rims 0.2.4 → 0.2.5

data/lib/rims/protocol/parser.rb

@@ -50,11 +50,16 @@ module RIMS
       def scan_line(line)
         atom_list = line.scan(/BODY(?:\.\S+)?\[.*?\](?:<\d+\.\d+>)?|[\[\]()]|".*?"|[^\[\]()\s]+/i).map{|s|
           case (s)
-          when '(', ')', '[', ']', /\ANIL\z/i
+          when '(', ')', '[', ']', /\A NIL \z/ix
             s.upcase.intern
-          when /\A"/
-            s.sub(/\A"/, '').sub(/"\z/, '')
-          when /\A(?<body_symbol>BODY)(?:\.(?<body_option>\S+))?\[(?<body_section>.*)\](?:<(?<partial_origin>\d+\.(?<partial_size>\d+)>))?\z/i
+          when /\A "/x
+            s.sub(/\A "/x, '').sub(/" \z/x, '')
+          when /
+                 \A
+                 (?<body_symbol>BODY) (?:\. (?<body_option>\S+))? \[ (?<body_section>.*) \]
+                 (?:< (?<partial_origin>\d+) \. (?<partial_size>\d+) >)?
+                 \z
+               /ix
             body_symbol = $~[:body_symbol]
             body_option = $~[:body_option]
             body_section = $~[:body_section]
@@ -71,7 +76,7 @@ module RIMS
             s
           end
         }
-        if ((atom_list[-1].is_a? String) && (atom_list[-1] =~ /\A{\d+}\z/)) then
+        if ((atom_list[-1].is_a? String) && (atom_list[-1] =~ /\A {\d+} \z/x)) then
           next_size = $&[1..-2].to_i
           @logger.debug("found literal: #{next_size} octets.") if @logger.debug?
           @output.write("+ continue\r\n")
@@ -107,7 +112,7 @@ module RIMS
         end
 
         if (atom == nil && last_atom != nil) then
-          raise 'syntax error.'
+          raise SyntaxError, "not found a terminator: `#{last_atom}'"
         end
 
         syntax_list
@@ -121,7 +126,7 @@ module RIMS
           if (atom_list.length < 2) then
             raise 'need for tag and command.'
           end
-          if (atom_list[0] =~ /\A[*+]/) then
+          if (atom_list[0] =~ /\A [*+]/x) then
             raise "invalid command tag: #{atom_list[0]}"
           end
           return parse(atom_list)
@@ -132,10 +137,10 @@ module RIMS
     end
 
     class AuthenticationReader
-      def initialize(auth, input, output, logger)
+      def initialize(auth, input_gets, output_write, logger)
         @auth = auth
-        @input = input
-        @output = output
+        @input_gets = input_gets
+        @output_write = output_write
         @logger = logger
       end
 
@@ -168,15 +173,13 @@ module RIMS
         if (server_challenge_data) then
           server_challenge_data_base64 = Protocol.encode_base64(server_challenge_data)
           @logger.debug("authenticate command: server challenge data: #{Protocol.io_data_log(server_challenge_data_base64)}") if @logger.debug?
-          @output.write("+ #{server_challenge_data_base64}\r\n")
-          @output.flush
+          @output_write.call([ "+ #{server_challenge_data_base64}\r\n" ])
         else
           @logger.debug("authenticate command: server challenge data is nil.") if @logger.debug?
-          @output.write("+ \r\n")
-          @output.flush
+          @output_write.call([ "+ \r\n" ])
         end
 
-        if (client_response_data_base64 = @input.gets) then
+        if (client_response_data_base64 = @input_gets.call) then
           client_response_data_base64.strip!
           @logger.debug("authenticate command: client response data: #{Protocol.io_data_log(client_response_data_base64)}") if @logger.debug?
           if (client_response_data_base64 == '*') then
@@ -326,7 +329,7 @@ module RIMS
         d = search_time.to_date
         proc{|next_cond|
           proc{|msg|
-            yield(@mail_store.msg_date(@folder.mbox_id, msg.uid).to_date, d) && next_cond.call(msg)
+            yield(@mail_store.msg_date(@folder.mbox_id, msg.uid).utc.to_date, d) && next_cond.call(msg)
           }
         }
       end
@@ -337,7 +340,7 @@ module RIMS
         proc{|next_cond|
           proc{|msg|
             if (mail_datetime = get_mail(msg).date) then
-              yield(mail_datetime.to_date, d) && next_cond.call(msg)
+              yield(mail_datetime.getutc.to_date, d) && next_cond.call(msg)
             else
               false
             end
@@ -418,12 +421,19 @@ module RIMS
       private :parse_or
 
       def parse_text(search_string)
-        search = proc{|text| string_include?(search_string, text) }
+        search_text = proc{|message_text| string_include?(search_string, message_text) }
+        search_mail = proc{|mail|
+          if (mail.multipart?) then
+            search_text.call(mail.header.raw_source) || mail.parts.any?{|m| search_mail.call(m) }
+          else
+            body_text = mail_body_text(mail)
+            search_text.call(mail.header.raw_source) || (body_text && search_text.call(body_text))
+          end
+        }
         proc{|next_cond|
           proc{|msg|
             mail = get_mail(msg)
-            body_txt = mail_body_text(mail)
-            (search.call(mail.header.raw_source) || (body_txt && search.call(body_txt))) && next_cond.call(msg)
+            search_mail.call(mail) && next_cond.call(msg)
           }
         }
       end
@@ -494,7 +504,7 @@ module RIMS
         unless (octet_size_string = search_key.shift) then
           raise SyntaxError, "need for a octet size of #{operation_name}."
         end
-        unless ((octet_size_string.is_a? String) && (octet_size_string =~ /\A\d+\z/)) then
+        unless ((octet_size_string.is_a? String) && (octet_size_string =~ /\A \d+ \z/x)) then
           raise SyntaxError, "#{operation_name} octet size is expected as numeric string but was <#{octet_size_string}>."
         end
 
@@ -650,26 +660,56 @@ module RIMS
 
     class FetchParser
       module Utils
+        def encode_value(object)
+          case (object)
+          when Symbol
+            object.to_s
+          when String
+            Protocol.quote(object)
+          when Integer
+            object.to_s
+          when NilClass
+            'NIL'.b
+          when Array
+            '('.b << object.map{|v| encode_value(v) }.join(' '.b) << ')'.b
+          else
+            raise "unknown value: #{object}"
+          end
+        end
+        module_function :encode_value
+
         def encode_list(array)
-          '('.b << array.map{|v|
-            case (v)
-            when Symbol
-              v.to_s
-            when String
-              Protocol.quote(v)
-            when Integer
-              v.to_s
-            when NilClass
-              'NIL'
-            when Array
-              encode_list(v)
-            else
-              raise "unknown value: #{v}"
-            end
-          }.join(' '.b) << ')'.b
+          unless (array.is_a? Array) then
+            raise TypeError, 'not a array type.'
+          end
+          encode_value(array)
         end
         module_function :encode_list
 
+        def encode_bodystructure(array)
+          if ((array.length > 0) && (array.first.is_a? Array)) then
+            s = '('.b
+            array = array.dup
+            while (object = array.shift)
+              case (object)
+              when Array
+                s << encode_bodystructure(object)
+              else
+                s << ' '.b << encode_value(object)
+              end
+            end
+            s << ')'.b
+          elsif ((array.length > 0) && (array.first.upcase == 'MESSAGE')) then
+            msg_body_list = array[0..7].map{|v| encode_value(v) }
+            msg_body_list << encode_bodystructure(array[8])
+            msg_body_list << array[9..-1].map{|v| encode_value(v) }
+            '('.b << msg_body_list.join(' '.b) << ')'.b
+          else
+            encode_list(array)
+          end
+        end
+        module_function :encode_bodystructure
+
         def encode_header(name_value_pair_list)
           name_value_pair_list.map{|n, v| ''.b << n << ': ' << v << "\r\n" }.join('') << "\r\n"
         end
@@ -886,7 +926,7 @@ module RIMS
           section_text = nil
           section_index_list = []
         else
-          if (body.section_list[0] =~ /\A(?<index>\d+(?:\.\d+)*)(?:\.(?<text>.+))?\z/) then
+          if (body.section_list[0] =~ /\A (?<index>\d+(?:\.\d+)*) (?:\.(?<text>.+))? \z/x) then
             section_text = $~[:text]
             section_index_list = $~[:index].split(/\./).map{|i| i.to_i }
           else
@@ -996,7 +1036,7 @@ module RIMS
 
       def parse_bodystructure(msg_att_name)
         proc{|msg|
-          ''.b << msg_att_name << ' '.b << encode_list(get_bodystructure_data(get_mail(msg)))
+          ''.b << msg_att_name << ' '.b << encode_bodystructure(get_bodystructure_data(get_mail(msg)))
         }
       end
       private :parse_bodystructure

data/lib/rims/rfc822.rb

@@ -25,9 +25,11 @@ module RIMS
 
       for _name, value in field_pair_list
         value.strip!
+        name.freeze
+        value.freeze
       end
 
-      field_pair_list
+      field_pair_list.freeze
     end
     module_function :parse_header
 
@@ -46,12 +48,12 @@ module RIMS
           else
             value = $~[:token]
           end
-          params[name.downcase] = [ name, value ]
+          params[name.downcase] = [ name.freeze, value.freeze ].freeze
         end
 
-        [ main_type, sub_type, params ]
+        [ main_type.freeze, sub_type.freeze, params.freeze ].freeze
       else
-        [ 'application', 'octet-stream', {} ]
+        [ 'application'.freeze, 'octet-stream'.freeze, {}.freeze ].freeze
       end
     end
     module_function :parse_content_type
@@ -68,12 +70,13 @@ module RIMS
             part_txt.lstrip!
             part_txt.chomp!("\n")
             part_txt.chomp!("\r")
+            part_txt.freeze
           end
-          return part_list
+          return part_list.freeze
         end
       end
 
-      []
+      [].freeze
     end
     module_function :parse_multipart_body
 
@@ -119,7 +122,7 @@ module RIMS
         end
       end
 
-      dst_txt
+      dst_txt.freeze
     end
     module_function :unquote_phrase
 
@@ -138,9 +141,9 @@ module RIMS
         then
           display_name = $~[:display_name]
           group_list = $~[:group_list]
-          addr_list << [ nil, nil, unquote_phrase(display_name), nil ]
+          addr_list << [ nil, nil, unquote_phrase(display_name), nil ].freeze
           addr_list.concat(parse_mail_address_list(group_list))
-          addr_list << [ nil, nil, nil, nil ]
+          addr_list << [ nil, nil, nil, nil ].freeze
         elsif (src_txt.sub!(%r{
                  \A
                  \s*
@@ -149,7 +152,7 @@ module RIMS
                  ,?
                }x, ''))
         then
-          addr_list << [ nil, nil, $~[:local_part], $~[:domain] ]
+          addr_list << [ nil, nil, $~[:local_part].freeze, $~[:domain].freeze ].freeze
         elsif (src_txt.sub!(%r{
                  \A
                  \s*
@@ -174,13 +177,13 @@ module RIMS
           route = $~[:route]
           local_part = $~[:local_part]
           domain = $~[:domain]
-          addr_list << [ unquote_phrase(display_name), route, local_part, domain ]
+          addr_list << [ unquote_phrase(display_name), route.freeze, local_part.freeze, domain.freeze ].freeze
         else
           break
         end
       end
 
-      addr_list
+      addr_list.freeze
     end
     module_function :parse_mail_address_list
 
@@ -188,7 +191,7 @@ module RIMS
       include Enumerable
 
       def initialize(header_txt)
-        @raw_source = header_txt
+        @raw_source = header_txt.freeze
         @field_list = nil
         @field_map = nil
       end
@@ -197,14 +200,17 @@ module RIMS
 
       def setup_header
         if (@field_list.nil? || @field_map.nil?) then
-          @field_list = []
+          @field_list = RFC822.parse_header(@raw_source)
           @field_map = {}
-          for name, value in RFC822.parse_header(@raw_source)
-            @field_list << [ name, value ]
+          for name, value in @field_list
             key = name.downcase
             @field_map[key] = [] unless (@field_map.key? key)
             @field_map[key] << value
           end
+          @field_map.each_value do |value_list|
+            value_list.freeze
+          end
+          @field_map.freeze
           self
         end
       end
@@ -248,7 +254,7 @@ module RIMS
 
     class Body
       def initialize(body_txt)
-        @raw_source = body_txt
+        @raw_source = body_txt.freeze
       end
 
       attr_reader :raw_source
@@ -256,7 +262,7 @@ module RIMS
 
     class Message
       def initialize(msg_txt)
-        @raw_source = msg_txt
+        @raw_source = msg_txt.freeze
         @header = nil
         @body = nil
         @content_type = nil
@@ -356,12 +362,12 @@ module RIMS
       end
 
       def text?
-        media_main_type.downcase == 'text'
+        media_main_type_upcase == 'TEXT'
       end
 
       def multipart?
         if (@is_multipart.nil?) then
-          @is_multipart = (media_main_type.downcase == 'multipart')
+          @is_multipart = (media_main_type_upcase == 'MULTIPART')
         end
         @is_multipart
       end
@@ -371,9 +377,9 @@ module RIMS
           if (@parts.nil?) then
             if (boundary = self.boundary) then
               part_list = RFC822.parse_multipart_body(boundary, body.raw_source)
-              @parts = part_list.map{|msg_txt| Message.new(msg_txt) }
+              @parts = part_list.map{|msg_txt| Message.new(msg_txt) }.freeze
             else
-              @parts = []
+              @parts = [].freeze
             end
           end
           @parts
@@ -382,7 +388,7 @@ module RIMS
 
       def message?
         if (@is_message.nil?) then
-          @is_message = (media_main_type.downcase == 'message')
+          @is_message = (media_main_type_upcase == 'MESSAGE')
         end
         @is_message
       end
@@ -406,7 +412,7 @@ module RIMS
             end
           end
 
-          @date
+          @date.freeze
         end
       end
 
@@ -416,6 +422,7 @@ module RIMS
           addr_list = instance_variable_get(ivar_name)
           if (addr_list.nil?) then
             addr_list = header.field_value_list(field_name).map{|addr_list_str| RFC822.parse_mail_address_list(addr_list_str) }.inject(:+)
+            addr_list.freeze
             instance_variable_set(ivar_name, addr_list)
           end
           addr_list

data/lib/rims/service.rb

@@ -130,6 +130,10 @@ module RIMS
       #       port: 143
       #       backlog: 64
       #     accept_polling_timeout_seconds: 0.1
+      #     process_num: 4
+      #     process_queue_size: 20
+      #     process_queue_polling_timeout_seconds: 0.1
+      #     process_send_io_polling_timeout_seconds: 0.1
       #     thread_num: 20
       #     thread_queue_size: 20
       #     thread_queue_polling_timeout_seconds: 0.1
@@ -153,10 +157,13 @@ module RIMS
       #     send_buffer_limit_size: 16384
       #     read_polling_interval_seconds: 1
       #     command_wait_timeout_seconds: 1800
-      #   lock:
-      #     read_lock_timeout_seconds: 30
-      #     write_lock_timeout_seconds: 30
-      #     cleanup_write_lock_timeout_seconds: 1
+      #   drb_services:
+      #     process_num: 4
+      #     engine:
+      #       bulk_response_count: 100
+      #       read_lock_timeout_seconds: 30
+      #       write_lock_timeout_seconds: 30
+      #       cleanup_write_lock_timeout_seconds: 1
       #   storage:
       #     meta_key_value_store:
       #       type: qdbm_depot
@@ -455,20 +462,19 @@ module RIMS
       end
 
       def process_num
-        # not yet supported multi-process server configuration.
-        0
+        @config.dig('server', 'process_num') || 0
       end
 
       def process_queue_size
-        20
+        @config.dig('server', 'process_queue_size') || 20
       end
 
       def process_queue_polling_timeout_seconds
-        0.1
+        @config.dig('server', 'process_queue_polling_timeout_seconds') || 0.1
       end
 
       def process_send_io_polling_timeout_seconds
-        0.1
+        @config.dig('server', 'process_send_io_polling_timeout_seconds') || 0.1
       end
 
       def thread_num
@@ -545,20 +551,28 @@ module RIMS
                                        @config.dig('connection', 'command_wait_timeout_seconds') || 60 * 30)
       end
 
+      def drb_process_num
+        @config.dig('drb_services', 'process_num') || 0
+      end
+
+      def bulk_response_count
+        @config.dig('drb_services', 'engine', 'bulk_response_count') || 100
+      end
+
       def read_lock_timeout_seconds
-        @config.dig('lock', 'read_lock_timeout_seconds') ||
+        @config.dig('drb_services', 'engine', 'read_lock_timeout_seconds') ||
           @config.dig('read_lock_timeout_seconds') || # for backward compatibility
           30
       end
 
       def write_lock_timeout_seconds
-        @config.dig('lock', 'write_lock_timeout_seconds') ||
+        @config.dig('drb_services', 'engine', 'write_lock_timeout_seconds') ||
           @config.dig('write_lock_timeout_seconds') || # for backward compatibility
           30
       end
 
       def cleanup_write_lock_timeout_seconds
-        @config.dig('lock', 'cleanup_write_lock_timeout_seconds') ||
+        @config.dig('drb_services', 'engine', 'cleanup_write_lock_timeout_seconds') ||
           @config.dig('cleanup_write_lock_timeout_seconds') || # for backward compatibility
           1
       end
@@ -756,7 +770,32 @@ module RIMS
       kvs_meta_open, kvs_meta_log = make_kvs_factory.call(@config.make_meta_key_value_store_params, 'meta')
       kvs_text_open, kvs_text_log = make_kvs_factory.call(@config.make_text_key_value_store_params, 'text')
       auth = @config.make_authentication
-      mail_store_pool = MailStore.build_pool(kvs_meta_open, kvs_text_open)
+
+      drb_process_num = @config.drb_process_num
+      if (@config.process_num > 0) then
+        unless (drb_process_num > 0) then
+          drb_process_num = 1
+          logger.warn('the number of dRuby processes needs to be 1 or more when the number of server processes is 1 or more.')
+          logger.warn("drb_services parameter: changed process_num: #{@config.drb_process_num} -> #{drb_process_num}")
+        end
+      end
+
+      services = Riser::DRbServices.new(drb_process_num)
+      services.add_sticky_process_service(:engine,
+                                          Riser::ResourceSet.build{|builder|
+                                            builder.at_create{|unique_user_id|
+                                              mail_store = MailStore.build(unique_user_id, kvs_meta_open, kvs_text_open)
+                                              Protocol::Decoder::Engine.new(unique_user_id, mail_store, logger,
+                                                                            bulk_response_count: @config.bulk_response_count,
+                                                                            read_lock_timeout_seconds: @config.read_lock_timeout_seconds,
+                                                                            write_lock_timeout_seconds: @config.write_lock_timeout_seconds,
+                                                                            cleanup_write_lock_timeout_seconds: @config.cleanup_write_lock_timeout_seconds)
+                                            }
+                                            builder.at_destroy{|engine|
+                                              engine.destroy
+                                            }
+                                            builder.alias_unref(:destroy)
+                                          })
 
       server.before_start{|server_socket|
         logger.info('start server.')
@@ -856,6 +895,8 @@ module RIMS
         logger.info("connection parameter: send_buffer_limit_size=#{@config.send_buffer_limit_size}")
         logger.info("connection parameter: read_polling_interval_seconds=#{conn_limits.read_polling_interval_seconds}")
         logger.info("connection parameter: command_wait_timeout_seconds=#{conn_limits.command_wait_timeout_seconds}")
+        logger.info("drb_services parameter: process_num=#{drb_process_num}")
+        logger.info("drb_services engine parameter: bulk_response_count=#{@config.bulk_response_count}")
         logger.info("lock parameter: read_lock_timeout_seconds=#{@config.read_lock_timeout_seconds}")
         logger.info("lock parameter: write_lock_timeout_seconds=#{@config.write_lock_timeout_seconds}")
         logger.info("lock parameter: cleanup_write_lock_timeout_seconds=#{@config.cleanup_write_lock_timeout_seconds}")
@@ -863,12 +904,18 @@ module RIMS
         kvs_text_log.call
         logger.info("authentication parameter: hostname=#{auth.hostname}")
         logger.info("authorization parameter: mail_delivery_user=#{@config.mail_delivery_user}")
+
+        logger.info('dRuby services: start server.')
+        services.start_server
+      }
+      server.at_fork{
+        logger.info('dRuby services: detach server.')
+        services.detach_server
       }
-      # server.at_fork{}
       server.at_stop{|stop_state|
         case (stop_state)
         when :graceful
-          logger.info('autologout mmediately.')
+          logger.info('autologout immediately.')
           conn_limits.command_wait_timeout_seconds = 0
         when :forced
           logger.info('forced shutdown.')
@@ -878,6 +925,8 @@ module RIMS
         logger.info("stat: #{info.to_json}")
       }
       server.preprocess{
+        logger.info('dRuby services: start client.')
+        services.start_client
         auth.start_plug_in(logger)
       }
       server.dispatch{|socket|
@@ -898,11 +947,7 @@ module RIMS
                   stream = Riser::WriteBufferStream.new(socket, @config.send_buffer_limit_size)
                 end
                 stream = Riser::LoggingStream.new(stream, protocol_logger)
-                decoder = Protocol::Decoder.new_decoder(mail_store_pool, auth, logger,
-                                                        mail_delivery_user: @config.mail_delivery_user,
-                                                        read_lock_timeout_seconds: @config.read_lock_timeout_seconds,
-                                                        write_lock_timeout_seconds: @config.write_lock_timeout_seconds,
-                                                        cleanup_write_lock_timeout_seconds: @config.cleanup_write_lock_timeout_seconds)
+                decoder = Protocol::Decoder.new_decoder(services, auth, logger, mail_delivery_user: @config.mail_delivery_user)
                 Protocol::Decoder.repl(decoder, conn_limits, stream, stream, logger)
               ensure
                 if (stream) then
@@ -934,6 +979,8 @@ module RIMS
         auth.stop_plug_in(logger)
       }
       server.after_stop{
+        logger.info('dRuby services: stop server.')
+        services.stop_server
         logger.info('stop server.')
       }