rims 0.2.6 → 0.3.1

data/lib/rims/error.rb

@@ -2,6 +2,13 @@
 
 module RIMS
   class Error < StandardError
+    def initialize(*args, **kw_args)
+      super(*args)
+      @optional_data = kw_args.dup.freeze
+    end
+
+    attr_reader :optional_data
+
     def self.trace_error_chain(exception)
       return enum_for(:trace_error_chain, exception) unless block_given?
 
@@ -12,6 +19,14 @@ module RIMS
 
       nil
     end
+
+    def self.optional_data(error) # :yields: error, data
+      if (error.is_a? Error) then
+        unless (error.optional_data.empty?) then
+          yield(error, error.optional_data)
+        end
+      end
+    end
   end
 end
 

data/lib/rims/protocol.rb

@@ -4,6 +4,27 @@ module RIMS
   class ProtocolError < Error
   end
 
+  class ReadSizeError < ProtocolError
+  end
+
+  class LineTooLongError < ReadSizeError
+  end
+
+  class RecoverableReadSizeError < ReadSizeError
+    def initialize(message=nil, command_tag=nil, **kw_args)
+      super(message, **kw_args)
+      @command_tag = command_tag
+    end
+
+    attr_reader :command_tag
+  end
+
+  class LiteralSizeTooLargeError < RecoverableReadSizeError
+  end
+
+  class CommandSizeTooLargeError < RecoverableReadSizeError
+  end
+
   class SyntaxError < ProtocolError
   end
 
@@ -57,14 +78,14 @@ module RIMS
     end
     module_function :decode_base64
 
-    autoload :FetchBody, 'rims/protocol/parser'
-    autoload :RequestReader, 'rims/protocol/parser'
+    autoload :FetchBody,            'rims/protocol/parser'
+    autoload :RequestReader,        'rims/protocol/parser'
     autoload :AuthenticationReader, 'rims/protocol/parser'
-    autoload :SearchParser, 'rims/protocol/parser'
-    autoload :FetchParser, 'rims/protocol/parser'
-    autoload :ConnectionLimits, 'rims/protocol/connection'
-    autoload :ConnectionTimer, 'rims/protocol/connection'
-    autoload :Decoder, 'rims/protocol/decoder'
+    autoload :SearchParser,         'rims/protocol/parser'
+    autoload :FetchParser,          'rims/protocol/parser'
+    autoload :ConnectionLimits,     'rims/protocol/connection'
+    autoload :ConnectionTimer,      'rims/protocol/connection'
+    autoload :Decoder,              'rims/protocol/decoder'
 
     def body(symbol: nil, option: nil, section: nil, section_list: nil, partial_origin: nil, partial_size: nil)
       FetchBody.new(symbol, option, section, section_list, partial_origin, partial_size)

data/lib/rims/protocol/decoder.rb

@@ -2,6 +2,7 @@
 
 require 'logger'
 require 'net/imap'
+require 'pp'
 require 'time'
 
 module RIMS
@@ -21,125 +22,174 @@ module RIMS
         name.upcase
       end
 
+      def self.logging_error_chain(error, logger)
+        Error.trace_error_chain(error) do |exception|
+          if (logger.debug?) then
+            Error.optional_data(exception) do |error, data|
+              logger.debug("error message: #{error.message} (#{error.class})")
+              for name, value in data
+                logger.debug("error data [#{name}]: #{value.pretty_inspect}")
+              end
+            end
+          end
+          logger.error(exception)
+        end
+      end
+
       def self.repl(decoder, limits, input, output, logger)
-        input_gets = input.method(:gets)
-        output_write = lambda{|res|
-          last_line = nil
-          for data in res
+        output_write = lambda{|data|
+          begin
             if (data == :flush) then
               output.flush
             else
               logger.debug("response data: #{Protocol.io_data_log(data)}") if logger.debug?
               output << data
-              last_line = data
             end
+          rescue
+            logger.error('response write error.')
+            logging_error_chain($!, logger)
+            raise
+          end
+        }
+        server_output_write = lambda{|res|
+          for data in res
+            output_write.call(data)
           end
           output.flush
 
-          last_line
+          nil
         }
-        response_write = proc{|res|
-          begin
-            last_line = output_write.call(res)
-            logger.info("server response: #{last_line.strip}")
-          rescue
-            logger.error('response write error.')
-            logger.error($!)
-            raise
+        response_write = lambda{|response|
+          output_write.call(response)
+          output.flush
+          logger.info("server response: #{response.strip}")
+        }
+        apply_imap_command = lambda{|name, *args, uid: false|
+          last_line = nil
+          if (uid) then
+            decoder.__send__(name, *args, uid: true) {|response|
+              output_write.call(response)
+              last_line = response if (response.is_a? String)
+            }
+          else
+            decoder.__send__(name, *args) {|response|
+              output_write.call(response)
+              last_line = response if (response.is_a? String)
+            }
           end
+          output.flush
+          logger.info("server response: #{last_line.strip}") if last_line
         }
 
-        decoder.ok_greeting{|res| response_write.call(res) }
+        apply_imap_command.call(:ok_greeting)
 
         conn_timer = ConnectionTimer.new(limits, input.to_io)
-        request_reader = RequestReader.new(input, output, logger)
+        request_reader = decoder.make_requrest_reader(input, output)
+        input_gets = request_reader.method(:gets)
 
-        until (conn_timer.command_wait_timeout?)
-          conn_timer.command_wait or break
+        begin
+          until (conn_timer.command_wait_timeout?)
+            conn_timer.command_wait or break
 
-          begin
-            atom_list = request_reader.read_command
-          rescue
-            logger.error('invalid client command.')
-            Error.trace_error_chain($!) do |exception|
-              logger.error(exception)
+            begin
+              atom_list = request_reader.read_command
+            rescue LineTooLongError
+              raise
+            rescue LiteralSizeTooLargeError
+              logger.error('literal size too large error.')
+              logging_error_chain($!, logger)
+              response_write.call("#{request_reader.command_tag || '*'} BAD literal size too large\r\n")
+              next
+            rescue CommandSizeTooLargeError
+              logger.error('command size too large error.')
+              logging_error_chain($!, logger)
+              response_write.call("#{request_reader.command_tag || '*'} BAD command size too large\r\n")
+              next
+            rescue
+              logger.error('invalid client command.')
+              logging_error_chain($!, logger)
+              response_write.call("#{request_reader.command_tag || '*'} BAD client command syntax error\r\n")
+              next
             end
-            response_write.call([ "* BAD client command syntax error\r\n" ])
-            next
-          end
 
-          break unless atom_list
+            break unless atom_list
 
-          tag, command, *opt_args = atom_list
-          normalized_command = imap_command_normalize(command)
-          logger.info("client command: #{tag} #{command}")
-          if (logger.debug?) then
-            case (normalized_command)
-            when 'LOGIN'
-              log_opt_args = opt_args.dup
-              log_opt_args[-1] = '********'
-            when 'AUTHENTICATE'
-              if (opt_args[1]) then
+            tag, command, *opt_args = atom_list
+            normalized_command = imap_command_normalize(command)
+            logger.info("client command: #{tag} #{command}")
+            if (logger.debug?) then
+              case (normalized_command)
+              when 'LOGIN'
                 log_opt_args = opt_args.dup
-                log_opt_args[1] = '********'
+                log_opt_args[-1] = '********'
+              when 'AUTHENTICATE'
+                if (opt_args[1]) then
+                  log_opt_args = opt_args.dup
+                  log_opt_args[1] = '********'
+                else
+                  log_opt_args = opt_args
+                end
               else
                 log_opt_args = opt_args
               end
-            else
-              log_opt_args = opt_args
+              logger.debug("client command parameter: #{log_opt_args.inspect}")
             end
-            logger.debug("client command parameter: #{log_opt_args.inspect}")
-          end
 
-          begin
-            if (name = IMAP_CMDs[normalized_command]) then
-              case (name)
-              when :uid
-                unless (opt_args.empty?) then
-                  uid_command, *uid_args = opt_args
-                  logger.info("uid command: #{uid_command}")
-                  logger.debug("uid parameter: #{uid_args}") if logger.debug?
-                  if (uid_name = UID_CMDs[imap_command_normalize(uid_command)]) then
-                    decoder.__send__(uid_name, tag, *uid_args, uid: true) {|res| response_write.call(res) }
+            begin
+              if (name = IMAP_CMDs[normalized_command]) then
+                case (name)
+                when :uid
+                  unless (opt_args.empty?) then
+                    uid_command, *uid_args = opt_args
+                    logger.info("uid command: #{uid_command}")
+                    logger.debug("uid parameter: #{uid_args}") if logger.debug?
+                    if (uid_name = UID_CMDs[imap_command_normalize(uid_command)]) then
+                      apply_imap_command.call(uid_name, tag, *uid_args, uid: true)
+                    else
+                      logger.error("unknown uid command: #{uid_command}")
+                      response_write.call("#{tag} BAD unknown uid command\r\n")
+                    end
                   else
-                    logger.error("unknown uid command: #{uid_command}")
-                    response_write.call([ "#{tag} BAD unknown uid command\r\n" ])
+                    logger.error('empty uid parameter.')
+                    response_write.call("#{tag} BAD empty uid parameter\r\n")
                   end
+                when :authenticate
+                  apply_imap_command.call(:authenticate, tag, input_gets, server_output_write, *opt_args)
+                when :idle
+                  apply_imap_command.call(:idle, tag, input_gets, server_output_write, conn_timer, *opt_args)
                 else
-                  logger.error('empty uid parameter.')
-                  response_write.call([ "#{tag} BAD empty uid parameter\r\n" ])
+                  apply_imap_command.call(name, tag, *opt_args)
                 end
-              when :authenticate
-                decoder.authenticate(tag, input_gets, output_write, *opt_args) {|res| response_write.call(res) }
-              when :idle
-                decoder.idle(tag, input_gets, output_write, conn_timer, *opt_args) {|res| response_write.call(res) }
               else
-                decoder.__send__(name, tag, *opt_args) {|res| response_write.call(res) }
+                logger.error("unknown command: #{command}")
+                response_write.call("#{tag} BAD unknown command\r\n")
               end
-            else
-              logger.error("unknown command: #{command}")
-              response_write.call([ "#{tag} BAD unknown command\r\n" ])
+            rescue LineTooLongError
+              raise
+            rescue
+              logger.error('unexpected error.')
+              logging_error_chain($!, logger)
+              response_write.call("#{tag} BAD unexpected error\r\n")
             end
-          rescue
-            logger.error('unexpected error.')
-            Error.trace_error_chain($!) do |exception|
-              logger.error(exception)
+
+            if (normalized_command == 'LOGOUT') then
+              break
             end
-            response_write.call([ "#{tag} BAD unexpected error\r\n" ])
-          end
 
-          if (normalized_command == 'LOGOUT') then
-            break
+            decoder = decoder.next_decoder
           end
-
-          decoder = decoder.next_decoder
-        end
-
-        if (conn_timer.command_wait_timeout?) then
-          if (limits.command_wait_timeout_seconds > 0) then
-            response_write.call([ "* BYE server autologout: idle for too long\r\n" ])
-          else
-            response_write.call([ "* BYE server autologout: shutdown\r\n" ])
+        rescue LineTooLongError
+          logger.error('line too long error.')
+          logging_error_chain($!, logger)
+          response_write.call("* BAD line too long\r\n")
+          response_write.call("* BYE server autologout: connection terminated\r\n")
+        else
+          if (conn_timer.command_wait_timeout?) then
+            if (limits.command_wait_timeout_seconds > 0) then
+              response_write.call("* BYE server autologout: idle for too long\r\n")
+            else
+              response_write.call("* BYE server autologout: shutdown\r\n")
+            end
           end
         end
 
@@ -157,25 +207,10 @@ module RIMS
 
       attr_reader :next_decoder
 
-      def response_stream(tag)
-        Enumerator.new{|res|
-          begin
-            yield(res)
-          rescue SyntaxError
-            @logger.error('client command syntax error.')
-            @logger.error($!)
-            res << "#{tag} BAD client command syntax error\r\n"
-          rescue
-            raise if ($!.class.name =~ /AssertionFailedError/)
-            @logger.error('internal server error.')
-            Error.trace_error_chain($!) do |exception|
-              @logger.error(exception)
-            end
-            res << "#{tag} BAD internal server error\r\n"
-          end
-        }
+      def logging_error_chain(error)
+        self.class.logging_error_chain(error, @logger)
       end
-      private :response_stream
+      private :logging_error_chain
 
       def guard_error(imap_command, tag, *args, **kw_args, &block)
         begin
@@ -184,21 +219,21 @@ module RIMS
           else
             __send__(imap_command, tag, *args, **kw_args, &block)
           end
+        rescue LineTooLongError
+          raise
         rescue SyntaxError
           @logger.error('client command syntax error.')
-          @logger.error($!)
-          yield([ "#{tag} BAD client command syntax error\r\n" ])
+          logging_error_chain($!)
+          yield("#{tag} BAD client command syntax error\r\n")
         rescue ArgumentError
           @logger.error('invalid command parameter.')
-          @logger.error($!)
-          yield([ "#{tag} BAD invalid command parameter\r\n" ])
+          logging_error_chain($!)
+          yield("#{tag} BAD invalid command parameter\r\n")
         rescue
           raise if ($!.class.name =~ /AssertionFailedError/)
           @logger.error('internal server error.')
-          Error.trace_error_chain($!) do |exception|
-            @logger.error(exception)
-          end
-          yield([ "#{tag} BAD internal server error\r\n" ])
+          logging_error_chain($!)
+          yield("#{tag} BAD internal server error\r\n")
         end
       end
       private :guard_error
@@ -256,8 +291,7 @@ module RIMS
         end
         private :imap_command
 
-        def make_engine_and_recovery_if_needed(drb_services, username,
-                                               logger: Logger.new(STDOUT))
+        def make_engine_and_recovery_if_needed(drb_services, username, logger: Logger.new(STDOUT))
           unique_user_id = Authentication.unique_user_id(username)
           logger.debug("unique user ID: #{username} -> #{unique_user_id}") if logger.debug?
 
@@ -265,7 +299,7 @@ module RIMS
           engine = drb_services[:engine, unique_user_id]
 
           begin
-            engine.recovery_if_needed(username) {|msg| yield(msg) }
+            engine.recovery_if_needed(username) {|response| yield(response) }
           rescue
             engine.destroy
             raise
@@ -276,14 +310,13 @@ module RIMS
       end
 
       def make_logout_response(tag)
-        [ "* BYE server logout\r\n",
-          "#{tag} OK LOGOUT completed\r\n"
-        ]
+        yield("* BYE server logout\r\n")
+        yield("#{tag} OK LOGOUT completed\r\n")
       end
       private :make_logout_response
 
       def ok_greeting
-        yield([ "* OK RIMS v#{VERSION} IMAP4rev1 service ready.\r\n" ])
+        yield("* OK RIMS v#{VERSION} IMAP4rev1 service ready.\r\n")
       end
 
       # common IMAP command
@@ -292,10 +325,8 @@ module RIMS
       def capability(tag)
         capability_list = %w[ IMAP4rev1 UIDPLUS IDLE ]
         capability_list += @auth.capability.map{|auth_capability| "AUTH=#{auth_capability}" }
-        res = []
-        res << "* CAPABILITY #{capability_list.join(' ')}\r\n"
-        res << "#{tag} OK CAPABILITY completed\r\n"
-        yield(res)
+        yield("* CAPABILITY #{capability_list.join(' ')}\r\n")
+        yield("#{tag} OK CAPABILITY completed\r\n")
       end
       imap_command :capability
     end
@@ -326,10 +357,24 @@ module RIMS
       end
 
       def initialize(drb_services, auth, logger,
-                     mail_delivery_user: Service::DEFAULT_CONFIG.mail_delivery_user)
+                     mail_delivery_user: Service::DEFAULT_CONFIG.mail_delivery_user,
+                     line_length_limit: Service::DEFAULT_CONFIG.protocol_line_length_limit,
+                     literal_size_limit: Service::DEFAULT_CONFIG.protocol_literal_size_limit,
+                     command_size_limit: Service::DEFAULT_CONFIG.protocol_command_size_limit)
         super(auth, logger)
         @drb_services = drb_services
         @mail_delivery_user = mail_delivery_user
+        @line_length_limit = line_length_limit
+        @literal_size_limit = literal_size_limit
+        @command_size_limit = command_size_limit
+        @logger.debug("RIMS::Protocol::InitialDecoder#initialize at #{self}") if @logger.debug?
+      end
+
+      def make_requrest_reader(input, output)
+        RequestReader.new(input, output, @logger,
+                          line_length_limit: @line_length_limit,
+                          literal_size_limit: @literal_size_limit,
+                          command_size_limit: @command_size_limit)
       end
 
       def auth?
@@ -341,22 +386,23 @@ module RIMS
       end
 
       def cleanup
+        @logger.debug("RIMS::Protocol::InitialDecoder#cleanup at #{self}") if @logger.debug?
         nil
       end
 
       def make_not_authenticated_response(tag)
-        [ "#{tag} NO not authenticated\r\n" ]
+        yield("#{tag} NO not authenticated\r\n")
       end
       private :make_not_authenticated_response
 
       def noop(tag)
-        yield([ "#{tag} OK NOOP completed\r\n" ])
+        yield("#{tag} OK NOOP completed\r\n")
       end
       imap_command :noop
 
-      def logout(tag)
-        @next_decoder = LogoutDecoder.new(self)
-        yield(make_logout_response(tag))
+      def logout(tag, &block)
+        @next_decoder = LogoutDecoder.new(self, @logger)
+        make_logout_response(tag, &block)
       end
       imap_command :logout
 
@@ -366,7 +412,7 @@ module RIMS
           @logger.info("mail delivery user: #{username}")
           MailDeliveryDecoder.new(self, @drb_services, @auth, @logger)
         else
-          engine = self.class.make_engine_and_recovery_if_needed(@drb_services, username, logger: @logger) {|msg| yield(msg) }
+          engine = self.class.make_engine_and_recovery_if_needed(@drb_services, username, logger: @logger) {|untagged_response| yield(untagged_response) }
           UserMailboxDecoder.new(self, engine, @auth, @logger)
         end
       end
@@ -377,137 +423,137 @@ module RIMS
         auth_reader = AuthenticationReader.new(@auth, client_response_input_gets, server_challenge_output_write, @logger)
         if (username = auth_reader.authenticate_client(auth_type, inline_client_response_data_base64)) then
           if (username != :*) then
-            yield response_stream(tag) {|res|
-              @logger.info("authentication OK: #{username}")
-              @next_decoder = accept_authentication(username) {|msg| res << msg << :flush }
-              res << "#{tag} OK AUTHENTICATE #{auth_type} success\r\n"
+            @logger.info("authentication OK: #{username}")
+            @next_decoder = accept_authentication(username) {|untagged_response|
+              yield(untagged_response)
+              yield(:flush)
             }
+            yield("#{tag} OK AUTHENTICATE #{auth_type} success\r\n")
           else
             @logger.info('bad authentication.')
-            yield([ "#{tag} BAD AUTHENTICATE failed\r\n" ])
+            yield("#{tag} BAD AUTHENTICATE failed\r\n")
           end
         else
-          yield([ "#{tag} NO authentication failed\r\n" ])
+          yield("#{tag} NO authentication failed\r\n")
         end
       end
       imap_command :authenticate
 
       def login(tag, username, password)
         if (@auth.authenticate_login(username, password)) then
-          yield response_stream(tag) {|res|
-            @logger.info("login authentication OK: #{username}")
-            @next_decoder = accept_authentication(username) {|msg| res << msg << :flush }
-            res << "#{tag} OK LOGIN completed\r\n"
+          @logger.info("login authentication OK: #{username}")
+          @next_decoder = accept_authentication(username) {|untagged_response|
+            yield(untagged_response)
+            yield(:flush)
           }
+          yield("#{tag} OK LOGIN completed\r\n")
         else
-          yield([ "#{tag} NO failed to login\r\n" ])
+          yield("#{tag} NO failed to login\r\n")
         end
       end
       imap_command :login
 
-      def select(tag, mbox_name)
-        yield(make_not_authenticated_response(tag))
+      def select(tag, mbox_name, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :select
 
-      def examine(tag, mbox_name)
-        yield(make_not_authenticated_response(tag))
+      def examine(tag, mbox_name, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :examine
 
-      def create(tag, mbox_name)
-        yield(make_not_authenticated_response(tag))
+      def create(tag, mbox_name, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :create
 
-      def delete(tag, mbox_name)
-        yield(make_not_authenticated_response(tag))
+      def delete(tag, mbox_name, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :delete
 
-      def rename(tag, src_name, dst_name)
-        yield(make_not_authenticated_response(tag))
+      def rename(tag, src_name, dst_name, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :rename
 
-      def subscribe(tag, mbox_name)
-        yield(make_not_authenticated_response(tag))
+      def subscribe(tag, mbox_name, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :subscribe
 
-      def unsubscribe(tag, mbox_name)
-        yield(make_not_authenticated_response(tag))
+      def unsubscribe(tag, mbox_name, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :unsubscribe
 
-      def list(tag, ref_name, mbox_name)
-        yield(make_not_authenticated_response(tag))
+      def list(tag, ref_name, mbox_name, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :list
 
-      def lsub(tag, ref_name, mbox_name)
-        yield(make_not_authenticated_response(tag))
+      def lsub(tag, ref_name, mbox_name, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :lsub
 
-      def status(tag, mbox_name, data_item_group)
-        yield(make_not_authenticated_response(tag))
+      def status(tag, mbox_name, data_item_group, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :status
 
-      def append(tag, mbox_name, *opt_args, msg_text)
-        yield(make_not_authenticated_response(tag))
+      def append(tag, mbox_name, *opt_args, msg_text, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :append
 
-      def check(tag)
-        yield(make_not_authenticated_response(tag))
+      def check(tag, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :check
 
-      def close(tag)
-        yield(make_not_authenticated_response(tag))
+      def close(tag, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :close
 
-      def expunge(tag)
-        yield(make_not_authenticated_response(tag))
+      def expunge(tag, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :expunge
 
-      def search(tag, *cond_args, uid: false)
-        yield(make_not_authenticated_response(tag))
+      def search(tag, *cond_args, uid: false, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :search
 
-      def fetch(tag, msg_set, data_item_group, uid: false)
-        yield(make_not_authenticated_response(tag))
+      def fetch(tag, msg_set, data_item_group, uid: false, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :fetch
 
-      def store(tag, msg_set, data_item_name, data_item_value, uid: false)
-        yield(make_not_authenticated_response(tag))
+      def store(tag, msg_set, data_item_name, data_item_value, uid: false, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :store
 
-      def copy(tag, msg_set, mbox_name, uid: false)
-        yield(make_not_authenticated_response(tag))
+      def copy(tag, msg_set, mbox_name, uid: false, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :copy
 
-      def idle(tag, client_input_gets, server_output_write, connection_timer)
-        yield(make_not_authenticated_response(tag))
+      def idle(tag, client_input_gets, server_output_write, connection_timer, &block)
+        make_not_authenticated_response(tag, &block)
       end
       imap_command :idle
     end
 
     class LogoutDecoder < Decoder
-      def initialize(parent_decoder)
+      def initialize(parent_decoder, logger)
+        super(nil, logger)
         @parent_decoder = parent_decoder
-      end
-
-      def next_decoder
-        self
+        @logger.debug("RIMS::Protocol::LogoutDecoder#initialize at #{self}") if @logger.debug?
       end
 
       def auth?
@@ -519,6 +565,7 @@ module RIMS
       end
 
       def cleanup
+        @logger.debug("RIMS::Protocol::LogoutDecoder#cleanup at #{self}") if @logger.debug?
         unless (@parent_decoder.nil?) then
           @parent_decoder.cleanup
           @parent_decoder = nil
@@ -652,31 +699,81 @@ module RIMS
     class AuthenticatedDecoder < Decoder
       def authenticate(tag, client_response_input_gets, server_challenge_output_write,
                        auth_type, inline_client_response_data_base64=nil, &block)
-        yield([ "#{tag} NO duplicated authentication\r\n" ])
+        yield("#{tag} NO duplicated authentication\r\n")
       end
       imap_command :authenticate
 
       def login(tag, username, password, &block)
-        yield([ "#{tag} NO duplicated login\r\n" ])
+        yield("#{tag} NO duplicated login\r\n")
       end
       imap_command :login
     end
 
     class UserMailboxDecoder < AuthenticatedDecoder
+      class BulkResponse
+        def initialize(limit_count, limit_size)
+          @limit_count = limit_count
+          @limit_size = limit_size
+          @responses = []
+          @size = 0
+        end
+
+        def count
+          @responses.length
+        end
+
+        attr_reader :size
+
+        def add(response)
+          @responses << response
+          @size += response.bytesize
+          self
+        end
+
+        alias << add
+
+        def empty?
+          @responses.empty?
+        end
+
+        def full?
+          count >= @limit_count || size >= @limit_size
+        end
+
+        def flush
+          res = @responses
+          if (count >= @limit_count) then
+            res = [ res.join('') ]
+          end
+
+          @responses = []
+          @size = 0
+
+          res
+        end
+      end
+
       class Engine
         def initialize(unique_user_id, mail_store, logger,
                        bulk_response_count: 100,
+                       bulk_response_size: 1024**2 * 10,
                        read_lock_timeout_seconds: ReadWriteLock::DEFAULT_TIMEOUT_SECONDS,
                        write_lock_timeout_seconds: ReadWriteLock::DEFAULT_TIMEOUT_SECONDS,
-                       cleanup_write_lock_timeout_seconds: 1)
+                       cleanup_write_lock_timeout_seconds: 1,
+                       charset_aliases: RFC822::DEFAULT_CHARSET_ALIASES,
+                       charset_convert_options: nil)
           @unique_user_id = unique_user_id
           @mail_store = mail_store
           @logger = logger
           @bulk_response_count = bulk_response_count
+          @bulk_response_size = bulk_response_size
           @read_lock_timeout_seconds = read_lock_timeout_seconds
           @write_lock_timeout_seconds = write_lock_timeout_seconds
           @cleanup_write_lock_timeout_seconds = cleanup_write_lock_timeout_seconds
+          @charset_aliases = charset_aliases
+          @charset_convert_options = charset_convert_options
           @folders = {}
+          @logger.debug("RIMS::Protocol::UserMailboxDecoder::Engine#initialize at #{self}") if @logger.debug?
         end
 
         attr_reader :unique_user_id
@@ -704,11 +801,13 @@ module RIMS
           end
           @folders[token] = folder
 
+          @logger.debug("RIMS::Protocol::UserMailboxDecoder::Engine#open_folder: #{token}") if @logger.debug?
           token
         end
         private :open_folder
 
         def close_folder(token)
+          @logger.debug("RIMS::Protocol::UserMailboxDecoder::Engine#close_folder: #{token}") if @logger.debug?
           folder = @folders.delete(token) or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
           folder.reload if folder.updated?
           begin
@@ -752,9 +851,10 @@ module RIMS
         end
 
         def destroy
+          @logger.debug("RIMS::Protocol::UserMailboxDecoder::Engine#destroy at #{self}") if @logger.debug?
           tmp_mail_store = @mail_store
           ReadWriteLock.write_lock_timeout_detach(@cleanup_write_lock_timeout_seconds, @write_lock_timeout_seconds, logger: @logger) {|timeout_seconds|
-            @mail_store.write_synchronize(timeout_seconds) {
+            tmp_mail_store.write_synchronize(timeout_seconds) {
               @logger.info("close mail store: #{@unique_user_id}")
               tmp_mail_store.close
             }
@@ -804,40 +904,66 @@ module RIMS
 
         class << self
           def imap_command_authenticated(name, **guard_optional)
+            name = name.to_sym
             orig_name = "_#{name}".to_sym
             alias_method orig_name, name
-            define_method name, lambda{|token, tag, *args, **kw_args, &block|
-              guard_authenticated(orig_name, token, tag, *args, **kw_args, **guard_optional, &block)
-            }
-            name.to_sym
+
+            guard_options_name = "_#{name}_guard_options".to_sym
+            define_method guard_options_name, lambda{ guard_optional }
+            private guard_options_name
+
+            class_eval(<<-EOF, __FILE__, __LINE__ + 1)
+              def #{name}(token, tag, *args, **kw_args, &block)
+                guard_authenticated(:#{orig_name}, token, tag, *args, **kw_args, **#{guard_options_name}, &block)
+              end
+            EOF
+
+            name
           end
           private :imap_command_authenticated
 
           def imap_command_selected(name, **guard_optional)
+            name = name.to_sym
             orig_name = "_#{name}".to_sym
             alias_method orig_name, name
-            define_method name, lambda{|token, tag, *args, **kw_args, &block|
-              guard_selected(orig_name, token, tag, *args, **kw_args, **guard_optional, &block)
-            }
-            name.to_sym
+
+            guard_options_name = "_#{name}_guard_options".to_sym
+            define_method guard_options_name, lambda{ guard_optional }
+            private guard_options_name
+
+            class_eval(<<-EOF, __FILE__, __LINE__ + 1)
+              def #{name}(token, tag, *args, **kw_args, &block)
+                guard_selected(:#{orig_name}, token, tag, *args, **kw_args, **#{guard_options_name}, &block)
+              end
+            EOF
+
+            name
           end
           private :imap_command_selected
         end
 
+        def new_bulk_response
+          BulkResponse.new(@bulk_response_count, @bulk_response_size)
+        end
+        private :new_bulk_response
+
         def noop(token, tag)
-          res = []
+          res = new_bulk_response
           if (token) then
             folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
             begin
               @mail_store.read_synchronize(@read_lock_timeout_seconds) {
-                folder.server_response_fetch{|r| res << r } if folder.server_response?
+                folder.server_response_fetch{|untagged_response|
+                  res << untagged_response
+                  yield(res.flush) if res.full?
+                }
               }
             rescue ReadLockTimeoutError
               @logger.warn("give up to get folder status because of read-lock timeout over #{@read_lock_timeout_seconds} seconds")
             end
           end
           res << "#{tag} OK NOOP completed\r\n"
-          yield(res)
+          yield(res.flush)
         end
 
         def folder_open_msgs(token)
@@ -859,20 +985,21 @@ module RIMS
             close_no_response(token)
           end
 
-          res = []
+          res = new_bulk_response
           new_token = nil
           mbox_name_utf8 = Net::IMAP.decode_utf7(mbox_name)
 
           if (id = @mail_store.mbox_id(mbox_name_utf8)) then
             new_token = open_folder(id)
-            folder_open_msgs(new_token) do |msg|
-              res << msg
-            end
+            folder_open_msgs(new_token) {|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
             res << "#{tag} OK [READ-WRITE] SELECT completed\r\n"
           else
             res << "#{tag} NO not found a mailbox\r\n"
           end
-          yield(res)
+          yield(res.flush)
 
           new_token
         end
@@ -883,30 +1010,34 @@ module RIMS
             close_no_response(token)
           end
 
-          res = []
+          res = new_bulk_response
           new_token = nil
           mbox_name_utf8 = Net::IMAP.decode_utf7(mbox_name)
 
           if (id = @mail_store.mbox_id(mbox_name_utf8)) then
             new_token = open_folder(id, read_only: true)
-            folder_open_msgs(new_token) do |msg|
-              res << msg
-            end
+            folder_open_msgs(new_token) {|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
             res << "#{tag} OK [READ-ONLY] EXAMINE completed\r\n"
           else
             res << "#{tag} NO not found a mailbox\r\n"
           end
-          yield(res)
+          yield(res.flush)
 
           new_token
         end
         imap_command_authenticated :examine
 
         def create(token, tag, mbox_name)
-          res = []
+          res = new_bulk_response
           if (token) then
             folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
-            folder.server_response_fetch{|r| res << r }
+            folder.server_response_fetch{|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
           end
           mbox_name_utf8 = Net::IMAP.decode_utf7(mbox_name)
           if (@mail_store.mbox_id(mbox_name_utf8)) then
@@ -915,15 +1046,18 @@ module RIMS
             @mail_store.add_mbox(mbox_name_utf8)
             res << "#{tag} OK CREATE completed\r\n"
           end
-          yield(res)
+          yield(res.flush)
         end
         imap_command_authenticated :create, exclusive: true
 
         def delete(token, tag, mbox_name)
-          res = []
+          res = new_bulk_response
           if (token) then
             folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
-            folder.server_response_fetch{|r| res << r }
+            folder.server_response_fetch{|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
           end
           mbox_name_utf8 = Net::IMAP.decode_utf7(mbox_name)
           if (id = @mail_store.mbox_id(mbox_name_utf8)) then
@@ -936,38 +1070,47 @@ module RIMS
           else
             res << "#{tag} NO not found a mailbox\r\n"
           end
-          yield(res)
+          yield(res.flush)
         end
         imap_command_authenticated :delete, exclusive: true
 
         def rename(token, tag, src_name, dst_name)
-          res = []
+          res = new_bulk_response
           if (token) then
             folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
-            folder.server_response_fetch{|r| res << r }
+            folder.server_response_fetch{|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
           end
           src_name_utf8 = Net::IMAP.decode_utf7(src_name)
           dst_name_utf8 = Net::IMAP.decode_utf7(dst_name)
           unless (id = @mail_store.mbox_id(src_name_utf8)) then
-            return yield(res << "#{tag} NO not found a mailbox\r\n")
+            res << "#{tag} NO not found a mailbox\r\n"
+            return yield(res.flush)
           end
           if (id == @mail_store.mbox_id('INBOX')) then
-            return yield(res << "#{tag} NO not rename inbox\r\n")
+            res << "#{tag} NO not rename inbox\r\n"
+            return yield(res.flush)
           end
           if (@mail_store.mbox_id(dst_name_utf8)) then
-            return yield(res << "#{tag} NO duplicated mailbox\r\n")
+            res << "#{tag} NO duplicated mailbox\r\n"
+            return yield(res.flush)
           end
           @mail_store.rename_mbox(id, dst_name_utf8)
           res << "#{tag} OK RENAME completed\r\n"
-          yield(res)
+          yield(res.flush)
         end
         imap_command_authenticated :rename, exclusive: true
 
         def subscribe(token, tag, mbox_name)
-          res = []
+          res = new_bulk_response
           if (token) then
             folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
-            folder.server_response_fetch{|r| res << r }
+            folder.server_response_fetch{|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
           end
           mbox_name_utf8 = Net::IMAP.decode_utf7(mbox_name)
           if (@mail_store.mbox_id(mbox_name_utf8)) then
@@ -975,15 +1118,18 @@ module RIMS
           else
             res << "#{tag} NO not found a mailbox\r\n"
           end
-          yield(res)
+          yield(res.flush)
         end
         imap_command_authenticated :subscribe
 
         def unsubscribe(token, tag, mbox_name)
-          res = []
+          res = new_bulk_response
           if (token) then
             folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
-            folder.server_response_fetch{|r| res << r }
+            folder.server_response_fetch{|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
           end
           mbox_name_utf8 = Net::IMAP.decode_utf7(mbox_name)
           if (@mail_store.mbox_id(mbox_name_utf8)) then
@@ -991,7 +1137,7 @@ module RIMS
           else
             res << "#{tag} NO not found a mailbox\r\n"
           end
-          yield(res)
+          yield(res.flush)
         end
         imap_command_authenticated :unsubscribe
 
@@ -1020,46 +1166,57 @@ module RIMS
         private :list_mbox
 
         def list(token, tag, ref_name, mbox_name)
-          res = []
+          res = new_bulk_response
           if (token) then
             folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
-            folder.server_response_fetch{|r| res << r }
+            folder.server_response_fetch{|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
           end
           if (mbox_name.empty?) then
             res << "* LIST (\\Noselect) NIL \"\"\r\n"
           else
-            list_mbox(ref_name, mbox_name) do |mbox_entry|
+            list_mbox(ref_name, mbox_name) {|mbox_entry|
               res << "* LIST #{mbox_entry}\r\n"
-            end
+              yield(res.flush) if res.full?
+            }
           end
           res << "#{tag} OK LIST completed\r\n"
-          yield(res)
+          yield(res.flush)
         end
         imap_command_authenticated :list
 
         def lsub(token, tag, ref_name, mbox_name)
-          res = []
+          res = new_bulk_response
           if (token) then
             folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
-            folder.server_response_fetch{|r| res << r }
+            folder.server_response_fetch{|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
           end
           if (mbox_name.empty?) then
             res << "* LSUB (\\Noselect) NIL \"\"\r\n"
           else
-            list_mbox(ref_name, mbox_name) do |mbox_entry|
+            list_mbox(ref_name, mbox_name) {|mbox_entry|
               res << "* LSUB #{mbox_entry}\r\n"
-            end
+              yield(res.flush) if res.full?
+            }
           end
           res << "#{tag} OK LSUB completed\r\n"
-          yield(res)
+          yield(res.flush)
         end
         imap_command_authenticated :lsub
 
         def status(token, tag, mbox_name, data_item_group)
-          res = []
+          res = new_bulk_response
           if (token) then
             folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
-            folder.server_response_fetch{|r| res << r }
+            folder.server_response_fetch{|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
           end
           mbox_name_utf8 = Net::IMAP.decode_utf7(mbox_name)
           if (id = @mail_store.mbox_id(mbox_name_utf8)) then
@@ -1091,7 +1248,7 @@ module RIMS
           else
             res << "#{tag} NO not found a mailbox\r\n"
           end
-          yield(res)
+          yield(res.flush)
         end
         imap_command_authenticated :status
 
@@ -1112,7 +1269,7 @@ module RIMS
         private :mailbox_size_server_response_multicast_push
 
         def append(token, tag, mbox_name, *opt_args, msg_text)
-          res = []
+          res = new_bulk_response
           mbox_name_utf8 = Net::IMAP.decode_utf7(mbox_name)
           if (mbox_id = @mail_store.mbox_id(mbox_name_utf8)) then
             msg_flags = []
@@ -1161,30 +1318,39 @@ module RIMS
             mailbox_size_server_response_multicast_push(mbox_id)
             if (token) then
               folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
-              folder.server_response_fetch{|r| res << r }
+              folder.server_response_fetch{|untagged_response|
+                res << untagged_response
+                yield(res.flush) if res.full?
+              }
             end
 
             res << "#{tag} OK [APPENDUID #{mbox_id} #{uid}] APPEND completed\r\n"
           else
             if (token) then
               folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
-              folder.server_response_fetch{|r| res << r }
+              folder.server_response_fetch{|untagged_response|
+                res << untagged_response
+                yield(res.flush) if res.full?
+              }
             end
             res << "#{tag} NO [TRYCREATE] not found a mailbox\r\n"
           end
-          yield(res)
+          yield(res.flush)
         end
         imap_command_authenticated :append, exclusive: true
 
         def check(token, tag)
-          res = []
+          res = new_bulk_response
           if (token) then
             folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
-            folder.server_response_fetch{|r| res << r }
+            folder.server_response_fetch{|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
           end
           @mail_store.sync
           res << "#{tag} OK CHECK completed\r\n"
-          yield(res)
+          yield(res.flush)
         end
         imap_command_selected :check, exclusive: true
 
@@ -1212,27 +1378,21 @@ module RIMS
           return yield([ "#{tag} NO cannot expunge in read-only mode\r\n" ]) if folder.read_only?
           folder.reload if folder.updated?
 
-          res = []
-          folder.server_response_fetch{|r|
-            res << r
-            if (res.length >= @bulk_response_count) then
-              yield(res)
-              res = []
-            end
-          }
+          res = new_bulk_response
+          folder.server_response_fetch{|untagged_response|
+            res << untagged_response
+            yield(res.flush) if res.full?
+         }
 
           folder.expunge_mbox do |msg_num|
-            r = "* #{msg_num} EXPUNGE\r\n"
-            res << r
-            if (res.length >= @bulk_response_count) then
-              yield(res)
-              res = []
-            end
-            folder.server_response_multicast_push(r)
+            untagged_response = "* #{msg_num} EXPUNGE\r\n"
+            res << untagged_response
+            yield(res.flush) if res.full?
+            folder.server_response_multicast_push(untagged_response)
           end
 
           res << "#{tag} OK EXPUNGE completed\r\n"
-          yield(res)
+          yield(res.flush)
         end
         imap_command_selected :expunge, exclusive: true
 
@@ -1240,7 +1400,9 @@ module RIMS
           folder = @folders[token] or raise KeyError.new("undefined folder token: #{token}", key: token, receiver: self)
           folder.should_be_alive
           folder.reload if folder.updated?
-          parser = SearchParser.new(@mail_store, folder)
+          parser = SearchParser.new(@mail_store, folder,
+                                    charset_aliases: @charset_aliases,
+                                    charset_convert_options: @charset_convert_options)
 
           if (! cond_args.empty? && cond_args[0].upcase == 'CHARSET') then
             cond_args.shift
@@ -1250,7 +1412,7 @@ module RIMS
               parser.charset = charset_string
             rescue ArgumentError
               @logger.warn("unknown charset: #{charset_string}")
-              return yield([ "#{tag} NO [BADCHARSET (#{Encoding.list.map(&:to_s).join(' ')})] unknown charset\r\n" ])
+              return yield([ "#{tag} NO [BADCHARSET (#{Encoding.list.reject(&:dummy?).map(&:to_s).join(' ')})] unknown charset\r\n" ])
             end
           end
 
@@ -1261,35 +1423,32 @@ module RIMS
           if (cond_args[0].upcase == 'UID' && cond_args.length >= 2) then
             begin
               msg_set = folder.parse_msg_set(cond_args[1], uid: true)
-              msg_src = folder.msg_find_all(msg_set, uid: true)
+              msg_list = folder.msg_find_all(msg_set, uid: true)
               cond_args.shift(2)
             rescue MessageSetSyntaxError
-              msg_src = folder.each_msg
+              msg_list = folder.each_msg
             end
           else
             begin
               msg_set = folder.parse_msg_set(cond_args[0], uid: false)
-              msg_src = folder.msg_find_all(msg_set, uid: false)
+              msg_list = folder.msg_find_all(msg_set, uid: false)
               cond_args.shift
             rescue MessageSetSyntaxError
-              msg_src = folder.each_msg
+              msg_list = folder.each_msg
             end
           end
           cond = parser.parse(cond_args)
 
-          res = []
-          folder.server_response_fetch{|r|
-            res << r
-            if (res.length >= @bulk_response_count) then
-              yield(res)
-              res = []
-            end
+          res = new_bulk_response
+          folder.server_response_fetch{|untagged_response|
+            res << untagged_response
+            yield(res.flush) if res.full?
           }
 
           res << '* SEARCH'
           begin
             begin
-              for msg in msg_src
+              for msg in msg_list
                 begin
                   if (cond.call(msg)) then
                     if (uid) then
@@ -1297,10 +1456,7 @@ module RIMS
                     else
                       res << " #{msg.num}"
                     end
-                    if (res.length >= @bulk_response_count) then
-                      yield(res)
-                      res = []
-                    end
+                    yield(res.flush) if res.full?
                   end
                 rescue EncodingError
                   @logger.warn("encoding error at the message: uidvalidity(#{folder.mbox_id}) uid(#{msg.uid})")
@@ -1311,14 +1467,12 @@ module RIMS
               res << "\r\n"
             end
           rescue
-            # flush bulk response
-            yield(res)
-            res = []
+            yield(res.flush)
             raise
           end
 
           res << "#{tag} OK SEARCH completed\r\n"
-          yield(res)
+          yield(res.flush)
         end
         imap_command_selected :search
 
@@ -1339,28 +1493,22 @@ module RIMS
             end
           end
 
-          parser = FetchParser.new(@mail_store, folder)
+          parser = FetchParser.new(@mail_store, folder, charset_aliases: @charset_aliases)
           fetch = parser.parse(data_item_group)
 
-          res = []
-          folder.server_response_fetch{|r|
-            res << r
-            if (res.length >= @bulk_response_count) then
-              yield(res)
-              res = []
-            end
+          res = new_bulk_response
+          folder.server_response_fetch{|untagged_response|
+            res << untagged_response
+            yield(res.flush) if res.full?
           }
 
           for msg in msg_list
             res << ('* '.b << msg.num.to_s.b << ' FETCH '.b << fetch.call(msg) << "\r\n".b)
-            if (res.length >= @bulk_response_count) then
-              yield(res)
-              res = []
-            end
+            yield(res.flush) if res.full?
           end
 
           res << "#{tag} OK FETCH completed\r\n"
-          yield(res)
+          yield(res.flush)
         end
         imap_command_selected :fetch
 
@@ -1440,18 +1588,15 @@ module RIMS
             end
           end
 
-          res = []
-          folder.server_response_fetch{|r|
-            res << r
-            if (res.length >= @bulk_response_count) then
-              yield(res)
-              res = []
-            end
+          res = new_bulk_response
+          folder.server_response_fetch{|untagged_response|
+            res << untagged_response
+            yield(res.flush) if res.full?
           }
 
           if (is_silent) then
             res << "#{tag} OK STORE completed\r\n"
-            yield(res)
+            yield(res.flush)
           else
             for msg in msg_list
               flag_atom_list = nil
@@ -1471,17 +1616,14 @@ module RIMS
                 else
                   res << "* #{msg.num} FETCH (FLAGS (#{flag_atom_list.join(' ')}))\r\n"
                 end
-                if (res.length >= @bulk_response_count) then
-                  yield(res)
-                  res = []
-                end
+                yield(res.flush) if res.full?
               else
                 @logger.warn("not found a message and skipped: uidvalidity(#{folder.mbox_id}) uid(#{msg.uid})")
               end
             end
 
             res << "#{tag} OK STORE completed\r\n"
-            yield(res)
+            yield(res.flush)
           end
         end
         imap_command_selected :store, exclusive: true
@@ -1491,7 +1633,7 @@ module RIMS
           folder.should_be_alive
           folder.reload if folder.updated?
 
-          res = []
+          res = new_bulk_response
           mbox_name_utf8 = Net::IMAP.decode_utf7(mbox_name)
           msg_set = folder.parse_msg_set(msg_set, uid: uid)
 
@@ -1507,17 +1649,26 @@ module RIMS
 
             if (msg_list.size > 0) then
               mailbox_size_server_response_multicast_push(mbox_id)
-              folder.server_response_fetch{|r| res << r }
+              folder.server_response_fetch{|untagged_response|
+                res << untagged_response
+                yield(res.flush) if res.full?
+              }
               res << "#{tag} OK [COPYUID #{mbox_id} #{src_uids.join(',')} #{dst_uids.join(',')}] COPY completed\r\n"
             else
-              folder.server_response_fetch{|r| res << r }
+              folder.server_response_fetch{|untagged_response|
+                res << untagged_response
+                yield(res.flush) if res.full?
+              }
               res << "#{tag} OK COPY completed\r\n"
             end
           else
-            folder.server_response_fetch{|r| res << r }
+            folder.server_response_fetch{|untagged_response|
+              res << untagged_response
+              yield(res.flush) if res.full?
+            }
             res << "#{tag} NO [TRYCREATE] not found a mailbox\r\n"
           end
-          yield(res)
+          yield(res.flush)
         end
         imap_command_selected :copy, exclusive: true
 
@@ -1529,12 +1680,15 @@ module RIMS
           server_output_write.call([ "+ continue\r\n" ])
 
           server_response_thread = Thread.new{
-            @logger.info('idle server response thread start... ')
+            res = new_bulk_response
+            @logger.info('idle server response thread start...')
             folder.server_response_idle_wait{|server_response_list|
-              for server_response in server_response_list
-                @logger.debug("idle server response: #{server_response}") if @logger.debug?
+              for untagged_response in server_response_list
+                @logger.debug("idle server response: #{untagged_response}") if @logger.debug?
+                res << untagged_response
+                server_output_write.call(res.flush) if res.full?
               end
-              server_output_write.call(server_response_list)
+              server_output_write.call(res.flush) unless res.empty?
             }
             @logger.info('idle server response thread terminated.')
           }
@@ -1547,23 +1701,23 @@ module RIMS
             server_response_thread.join
           end
 
-          res = []
+          last_res = []
           if (line) then
             line.chomp!("\n")
             line.chomp!("\r")
             if (line.upcase == "DONE") then
               @logger.info('idle terminated.')
-              res << "#{tag} OK IDLE terminated\r\n"
+              last_res << "#{tag} OK IDLE terminated\r\n"
             else
               @logger.warn('unexpected client response and idle terminated.')
               @logger.debug("unexpected client response data: #{line}") if @logger.debug?
-              res << "#{tag} BAD unexpected client response\r\n"
+              last_res << "#{tag} BAD unexpected client response\r\n"
             end
           else
             @logger.warn('unexpected client connection close and idle terminated.')
-            res << "#{tag} BAD unexpected client connection close\r\n"
+            last_res << "#{tag} BAD unexpected client connection close\r\n"
           end
-          yield(res)
+          yield(last_res)
         end
         imap_command_selected :idle, exclusive: nil
       end
@@ -1573,6 +1727,7 @@ module RIMS
         @parent_decoder = parent_decoder
         @engine = engine
         @token = nil
+        @logger.debug("RIMS::Protocol::UserMailboxDecoder#initialize at #{self}") if @logger.debug?
       end
 
       def auth?
@@ -1583,43 +1738,57 @@ module RIMS
         ! @token.nil?
       end
 
-      def cleanup
+      # `not_cleanup_parent' keyword argument is defined for MailDeliveryDecoder
+      def cleanup(not_cleanup_parent: false)
+        @logger.debug("RIMS::Protocol::UserMailboxDecoder#cleanup at #{self}") if @logger.debug?
+
         unless (@engine.nil?) then
           begin
             @engine.cleanup(@token)
           ensure
             @token = nil
           end
+        end
 
-          begin
-            @engine.destroy
-          ensure
-            @engine = nil
+        unless (not_cleanup_parent) then
+          unless (@engine.nil?) then
+            begin
+              @engine.destroy
+            ensure
+              @engine = nil
+            end
           end
-        end
 
-        unless (@parent_decoder.nil?) then
-          @parent_decoder.cleanup
-          @parent_decoder = nil
+          unless (@parent_decoder.nil?) then
+            @parent_decoder.cleanup
+            @parent_decoder = nil
+          end
         end
 
         nil
       end
 
-      def noop(tag, &block)
-        @engine.noop(@token, tag, &block)
+      def noop(tag)
+        ret_val = nil
+        @engine.noop(@token, tag) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :noop
 
-      def logout(tag)
+      def logout(tag, &block)
         if (@token) then
           old_token = @token
           @token = nil
           @engine.cleanup(old_token)
         end
 
-        @next_decoder = LogoutDecoder.new(self)
-        yield(make_logout_response(tag))
+        @next_decoder = LogoutDecoder.new(self, @logger)
+        make_logout_response(tag, &block)
       end
       imap_command :logout
 
@@ -1627,7 +1796,9 @@ module RIMS
         ret_val = nil
         old_token = @token
         @token = @engine.select(old_token, tag, mbox_name) {|res|
-          ret_val = yield(res)
+          for response in res
+            ret_val = yield(response)
+          end
         }
 
         ret_val
@@ -1638,134 +1809,225 @@ module RIMS
         ret_val = nil
         old_token = @token
         @token = @engine.examine(old_token, tag, mbox_name) {|res|
-          ret_val = yield(res)
+          for response in res
+            ret_val = yield(response)
+          end
         }
 
         ret_val
       end
       imap_command :examine
 
-      def create(tag, mbox_name, &block)
-        @engine.create(@token, tag, mbox_name, &block)
+      def create(tag, mbox_name)
+        ret_val = nil
+        @engine.create(@token, tag, mbox_name) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :create
 
-      def delete(tag, mbox_name, &block)
-        @engine.delete(@token, tag, mbox_name, &block)
+      def delete(tag, mbox_name)
+        ret_val = nil
+        @engine.delete(@token, tag, mbox_name) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :delete
 
-      def rename(tag, src_name, dst_name, &block)
-        @engine.rename(@token, tag, src_name, dst_name, &block)
+      def rename(tag, src_name, dst_name)
+        ret_val = nil
+        @engine.rename(@token, tag, src_name, dst_name) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :rename
 
-      def subscribe(tag, mbox_name, &block)
-        @engine.subscribe(@token, tag, mbox_name, &block)
+      def subscribe(tag, mbox_name)
+        ret_val = nil
+        @engine.subscribe(@token, tag, mbox_name) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :subscribe
 
-      def unsubscribe(tag, mbox_name, &block)
-        @engine.unsubscribe(@token, tag, mbox_name, &block)
+      def unsubscribe(tag, mbox_name)
+        ret_val = nil
+        @engine.unsubscribe(@token, tag, mbox_name) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :unsubscribe
 
-      def list(tag, ref_name, mbox_name, &block)
-        @engine.list(@token, tag, ref_name, mbox_name, &block)
+      def list(tag, ref_name, mbox_name)
+        ret_val = nil
+        @engine.list(@token, tag, ref_name, mbox_name) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :list
 
-      def lsub(tag, ref_name, mbox_name, &block)
-        @engine.lsub(@token, tag, ref_name, mbox_name, &block)
+      def lsub(tag, ref_name, mbox_name)
+        ret_val = nil
+        @engine.lsub(@token, tag, ref_name, mbox_name) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :lsub
 
-      def status(tag, mbox_name, data_item_group, &block)
-        @engine.status(@token, tag, mbox_name, data_item_group, &block)
+      def status(tag, mbox_name, data_item_group)
+        ret_val = nil
+        @engine.status(@token, tag, mbox_name, data_item_group) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :status
 
-      def append(tag, mbox_name, *opt_args, msg_text, &block)
-        @engine.append(@token, tag, mbox_name, *opt_args, msg_text, &block)
+      def append(tag, mbox_name, *opt_args, msg_text)
+        ret_val = nil
+        @engine.append(@token, tag, mbox_name, *opt_args, msg_text) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :append
 
-      def check(tag, &block)
-        @engine.check(@token, tag, &block)
+      def check(tag)
+        ret_val = nil
+        @engine.check(@token, tag) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :check
 
-      def close(tag, &block)
+      def close(tag)
+        ret_val = nil
         old_token = @token
         @token = nil
-
-        yield response_stream(tag) {|res|
-          @engine.close(old_token, tag) {|bulk_res|
-            for r in bulk_res
-              res << r
-            end
-          }
+        @engine.close(old_token, tag) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
         }
+
+        ret_val
       end
       imap_command :close
 
       def expunge(tag)
-        yield response_stream(tag) {|res|
-          @engine.expunge(@token, tag) {|bulk_res|
-            for r in bulk_res
-              res << r
-            end
-          }
+        ret_val = nil
+        @engine.expunge(@token, tag) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
         }
+
+        ret_val
       end
       imap_command :expunge
 
       def search(tag, *cond_args, uid: false)
-        yield response_stream(tag) {|res|
-          @engine.search(@token, tag, *cond_args, uid: uid) {|bulk_res|
-            for r in bulk_res
-              res << r
-            end
-          }
+        ret_val = nil
+        @engine.search(@token, tag, *cond_args, uid: uid) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
         }
+
+        ret_val
       end
       imap_command :search
 
       def fetch(tag, msg_set, data_item_group, uid: false)
-        yield response_stream(tag) {|res|
-          @engine.fetch(@token, tag, msg_set, data_item_group, uid: uid) {|bulk_res|
-            for r in bulk_res
-              res << r
-            end
-          }
+        ret_val = nil
+        @engine.fetch(@token, tag, msg_set, data_item_group, uid: uid) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
         }
+
+        ret_val
       end
       imap_command :fetch
 
       def store(tag, msg_set, data_item_name, data_item_value, uid: false)
-        yield response_stream(tag) {|res|
-          @engine.store(@token, tag, msg_set, data_item_name, data_item_value, uid: uid) {|bulk_res|
-            for r in bulk_res
-              res << r
-            end
-          }
+        ret_val = nil
+        @engine.store(@token, tag, msg_set, data_item_name, data_item_value, uid: uid) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
         }
+
+        ret_val
       end
       imap_command :store
 
-      def copy(tag, msg_set, mbox_name, uid: false, &block)
-        @engine.copy(@token, tag, msg_set, mbox_name, uid: uid, &block)
+      def copy(tag, msg_set, mbox_name, uid: false)
+        ret_val = nil
+        @engine.copy(@token, tag, msg_set, mbox_name, uid: uid) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :copy
 
-      def idle(tag, client_input_gets, server_output_write, connection_timer, &block)
-        @engine.idle(@token, tag, client_input_gets, server_output_write, connection_timer, &block)
+      def idle(tag, client_input_gets, server_output_write, connection_timer)
+        ret_val = nil
+        @engine.idle(@token, tag, client_input_gets, server_output_write, connection_timer) {|res|
+          for response in res
+            ret_val = yield(response)
+          end
+        }
+
+        ret_val
       end
       imap_command :idle
     end
 
     # alias
-    Decoder::Engine = UserMailboxDecoder::Engine
+    Decoder::Engine       = UserMailboxDecoder::Engine
+    Decoder::BulkResponse = UserMailboxDecoder::BulkResponse
 
     def Decoder.encode_delivery_target_mailbox(username, mbox_name)
       "b64user-mbox #{Protocol.encode_base64(username)} #{mbox_name}"
@@ -1787,6 +2049,7 @@ module RIMS
         @auth = auth
         @last_user_cache_key_username = nil
         @last_user_cache_value_engine = nil
+        @logger.debug("RIMS::Protocol::MailDeliveryDecoder#initialize at #{self}") if @logger.debug?
       end
 
       def engine_cached?(username)
@@ -1834,6 +2097,8 @@ module RIMS
       end
 
       def cleanup
+        @logger.debug("RIMS::Protocol::MailDeliveryDecoder#cleanup at #{self}") if @logger.debug?
+
         release_engine_cache
         @drb_services = nil unless @drb_services.nil?
         @auth = nil unless @auth.nil?
@@ -1846,9 +2111,9 @@ module RIMS
         nil
       end
 
-      def logout(tag)
-        @next_decoder = LogoutDecoder.new(self)
-        yield(make_logout_response(tag))
+      def logout(tag, &block)
+        @next_decoder = LogoutDecoder.new(self, @logger)
+        make_logout_response(tag, &block)
       end
       imap_command :logout
 
@@ -1856,85 +2121,87 @@ module RIMS
       private :standard_capability
 
       def capability(tag)
-        standard_capability(tag) {|res|
-          yield res.map{|line|
-            if (line.start_with? '* CAPABILITY ') then
-              line.strip + " X-RIMS-MAIL-DELIVERY-USER\r\n"
-            else
-              line
-            end
-          }
+        standard_capability(tag) {|response|
+          if (response.start_with? '* CAPABILITY ') then
+            yield(response.strip + " X-RIMS-MAIL-DELIVERY-USER\r\n")
+          else
+            yield(response)
+          end
         }
       end
       imap_command :capability
 
       def make_not_allowed_command_response(tag)
-        [ "#{tag} NO not allowed command on mail delivery user\r\n" ]
+        yield("#{tag} NO not allowed command on mail delivery user\r\n")
       end
       private :make_not_allowed_command_response
 
-      def select(tag, mbox_name)
-        yield(make_not_allowed_command_response(tag))
+      def select(tag, mbox_name, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :select
 
-      def examine(tag, mbox_name)
-        yield(make_not_allowed_command_response(tag))
+      def examine(tag, mbox_name, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :examine
 
-      def create(tag, mbox_name)
-        yield(make_not_allowed_command_response(tag))
+      def create(tag, mbox_name, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :create
 
-      def delete(tag, mbox_name)
-        yield(make_not_allowed_command_response(tag))
+      def delete(tag, mbox_name, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :delete
 
-      def rename(tag, src_name, dst_name)
-        yield(make_not_allowed_command_response(tag))
+      def rename(tag, src_name, dst_name, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :rename
 
-      def subscribe(tag, mbox_name)
-        yield(make_not_allowed_command_response(tag))
+      def subscribe(tag, mbox_name, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :subscribe
 
-      def unsubscribe(tag, mbox_name)
-        yield(make_not_allowed_command_response(tag))
+      def unsubscribe(tag, mbox_name, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :unsubscribe
 
-      def list(tag, ref_name, mbox_name)
-        yield(make_not_allowed_command_response(tag))
+      def list(tag, ref_name, mbox_name, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :list
 
-      def lsub(tag, ref_name, mbox_name)
-        yield(make_not_allowed_command_response(tag))
+      def lsub(tag, ref_name, mbox_name, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :lsub
 
-      def status(tag, mbox_name, data_item_group)
-        yield(make_not_allowed_command_response(tag))
+      def status(tag, mbox_name, data_item_group, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :status
 
-      def deliver_to_user(tag, username, mbox_name, opt_args, msg_text, engine, res)
+      def deliver_to_user(tag, username, mbox_name, opt_args, msg_text, engine)
         user_decoder = UserMailboxDecoder.new(self, engine, @auth, @logger)
-        user_decoder.append(tag, mbox_name, *opt_args, msg_text) {|append_response|
-          if (append_response.last.split(' ', 3)[1] == 'OK') then
+        begin
+          last_response = nil
+          user_decoder.append(tag, mbox_name, *opt_args, msg_text) {|response|
+            last_response = response
+            yield(response)
+          }
+          if (last_response.split(' ', 3)[1] == 'OK') then
             @logger.info("message delivery: successed to deliver #{msg_text.bytesize} octets message.")
           else
             @logger.info("message delivery: failed to deliver message.")
           end
-          for response_data in append_response
-            res << response_data
-          end
-        }
+        ensure
+          user_decoder.cleanup(not_cleanup_parent: true)
+        end
       end
       private :deliver_to_user
 
@@ -1944,62 +2211,65 @@ module RIMS
 
         if (@auth.user? username) then
           if (engine_cached? username) then
-            res = []
             engine = engine_cache(username)
-            deliver_to_user(tag, username, mbox_name, opt_args, msg_text, engine, res)
+            deliver_to_user(tag, username, mbox_name, opt_args, msg_text, engine) {|response|
+              yield(response)
+            }
           else
-            res = response_stream(tag) {|stream_res|
-              engine = store_engine_cache(username) {
-                self.class.make_engine_and_recovery_if_needed(@drb_services, username, logger: @logger) {|msg| stream_res << msg << :flush }
+            engine = store_engine_cache(username) {
+              self.class.make_engine_and_recovery_if_needed(@drb_services, username, logger: @logger) {|untagged_response|
+                yield(untagged_response)
+                yield(:flush)
               }
-              deliver_to_user(tag, username, mbox_name, opt_args, msg_text, engine, stream_res)
+            }
+            deliver_to_user(tag, username, mbox_name, opt_args, msg_text, engine) {|response|
+              yield(response)
             }
           end
-          yield(res)
         else
           @logger.info('message delivery: not found a user.')
-          yield([ "#{tag} NO not found a user and couldn't deliver a message to the user's mailbox\r\n" ])
+          yield("#{tag} NO not found a user and couldn't deliver a message to the user's mailbox\r\n")
         end
       end
       imap_command :append
 
-      def check(tag)
-        yield(make_not_allowed_command_response(tag))
+      def check(tag, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :check
 
-      def close(tag)
-        yield(make_not_allowed_command_response(tag))
+      def close(tag, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :close
 
-      def expunge(tag)
-        yield(make_not_allowed_command_response(tag))
+      def expunge(tag, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :expunge
 
-      def search(tag, *cond_args, uid: false)
-        yield(make_not_allowed_command_response(tag))
+      def search(tag, *cond_args, uid: false, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :search
 
-      def fetch(tag, msg_set, data_item_group, uid: false)
-        yield(make_not_allowed_command_response(tag))
+      def fetch(tag, msg_set, data_item_group, uid: false, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :fetch
 
-      def store(tag, msg_set, data_item_name, data_item_value, uid: false)
-        yield(make_not_allowed_command_response(tag))
+      def store(tag, msg_set, data_item_name, data_item_value, uid: false, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :store
 
-      def copy(tag, msg_set, mbox_name, uid: false)
-        yield(make_not_allowed_command_response(tag))
+      def copy(tag, msg_set, mbox_name, uid: false, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :copy
 
-      def idle(tag, client_input_gets, server_output_write, connection_timer)
-        yield(make_not_allowed_command_response(tag))
+      def idle(tag, client_input_gets, server_output_write, connection_timer, &block)
+        make_not_allowed_command_response(tag, &block)
       end
       imap_command :idle
     end