rims 0.2.5 → 0.3.0

data/lib/rims/service.rb

@@ -118,6 +118,7 @@ module RIMS
       #   daemon:
       #     daemonize: true
       #     debug: false
+      #     umask: 0037
       #     status_file: rims.pid
       #     server_polling_interval_seconds: 3
       #     server_privileged_user: nobody
@@ -157,10 +158,29 @@ module RIMS
       #     send_buffer_limit_size: 16384
       #     read_polling_interval_seconds: 1
       #     command_wait_timeout_seconds: 1800
+      #   protocol:
+      #     line_length_limit: 8192
+      #     literal_size_limit: 10485760
+      #     command_size_limit: 10485760
+      #   charset:
+      #     use_default_aliases: true
+      #     aliases:
+      #       - name: euc-jp
+      #         encoding: eucJP-ms
+      #       - name: ios-2022-jp
+      #         encoding: CP50221
+      #       - name: Shift_JIS
+      #         encoding: Windows-31J
+      #     convert_options:
+      #       replace_invalid_byte_sequence: false
+      #       replace_undefined_character: true
+      #       replaced_mark: "\uFFFD"
       #   drb_services:
       #     process_num: 4
+      #     load_limit: 134217728
       #     engine:
       #       bulk_response_count: 100
+      #       bulk_response_size: 33554432
       #       read_lock_timeout_seconds: 30
       #       write_lock_timeout_seconds: 30
       #       cleanup_write_lock_timeout_seconds: 1
@@ -385,9 +405,8 @@ module RIMS
       end
 
       def daemonize?
-        daemon_config = @config['daemon'] || {}
-        if (daemon_config.key? 'daemonize') then
-          daemon_config['daemonize']
+        if (@config.dig('daemon')&.key? 'daemonize') then
+          @config.dig('daemon', 'daemonize')
         else
           true
         end
@@ -398,14 +417,17 @@ module RIMS
       end
 
       def daemon_debug?
-        daemon_config = @config['daemon'] || {}
-        if (daemon_config.key? 'debug') then
-          daemon_config['debug']
+        if (@config.dig('daemon')&.key? 'debug') then
+          @config.dig('daemon', 'debug')
         else
           false
         end
       end
 
+      def daemon_umask
+        @config.dig('daemon', 'umask') || 0037
+      end
+
       def status_file
         file_path = @config.dig('daemon', 'status_file') || 'rims.pid'
         file_path = Pathname(file_path)
@@ -521,22 +543,20 @@ module RIMS
       end
 
       def ssl_context
-        if (openssl_config = @config['openssl']) then
-          if (openssl_config.key? 'use_ssl') then
-            use_ssl = openssl_config['use_ssl']
-          else
-            use_ssl = openssl_config.key? 'ssl_context'
-          end
-
-          if (use_ssl) then
-            ssl_context = OpenSSL::SSL::SSLContext.new
-            if (ssl_config_expr = openssl_config['ssl_context']) then
-              anon_mod = SSLContextConfigAttribute.new_module(ssl_context, base_dir)
-              SSLContextConfigAttribute.eval_config(anon_mod, ssl_config_expr, 'ssl_context')
-            end
+        if (@config.dig('openssl')&.key? 'use_ssl') then
+          use_ssl = @config.dig('openssl', 'use_ssl')
+        else
+          use_ssl = (@config.dig('openssl')&.key? 'ssl_context') || false
+        end
 
-            ssl_context
+        if (use_ssl) then
+          ssl_context = OpenSSL::SSL::SSLContext.new
+          if (ssl_config_expr = @config.dig('openssl', 'ssl_context')) then
+            anon_mod = SSLContextConfigAttribute.new_module(ssl_context, base_dir)
+            SSLContextConfigAttribute.eval_config(anon_mod, ssl_config_expr, 'ssl_context')
           end
+
+          ssl_context
         end
       end
 
@@ -551,14 +571,102 @@ module RIMS
                                        @config.dig('connection', 'command_wait_timeout_seconds') || 60 * 30)
       end
 
+      def protocol_line_length_limit
+        @config.dig('protocol', 'line_length_limit') || 1024 * 8
+      end
+
+      def protocol_literal_size_limit
+        @config.dig('protocol', 'literal_size_limit') || 1024**2 * 10
+      end
+
+      def protocol_command_size_limit
+        @config.dig('protocol', 'command_size_limit') || 1024**2 * 10
+      end
+
+      def charset_aliases
+        charset_aliases = RFC822::CharsetAliases.new
+
+        if (@config.dig('charset')&.key? 'use_default_aliases') then
+          use_default_aliases = @config.dig('charset', 'use_default_aliases')
+        else
+          use_default_aliases = true
+        end
+
+        if (use_default_aliases) then
+          for name, encoding in RFC822::DEFAULT_CHARSET_ALIASES
+            charset_aliases.add_alias(name, encoding)
+          end
+        end
+
+        if (alias_list = @config.dig('charset', 'aliases')) then
+          for an_alias in alias_list
+            charset_aliases.add_alias(an_alias['name'], Encoding.find(an_alias['encoding']))
+          end
+        end
+
+        charset_aliases
+      end
+
+      def charset_convert_options
+        options = {}
+
+        if (@config.dig('charset', 'convert_options')&.key? 'replace_invalid_byte_sequence') then
+          replace_invalid_byte_sequence = @config.dig('charset', 'convert_options', 'replace_invalid_byte_sequence')
+        else
+          replace_invalid_byte_sequence = false
+        end
+
+        if (replace_invalid_byte_sequence) then
+          options[:invalid] = :replace
+        end
+
+        if (@config.dig('charset', 'convert_options')&.key? 'replace_undefined_character') then
+          replace_undefined_character = @config.dig('charset', 'convert_options', 'replace_undefined_character')
+        else
+          replace_undefined_character = true
+        end
+
+        if (replace_undefined_character) then
+          options[:undef] = :replace
+        end
+
+        if (replaced_mark = @config.dig('charset', 'convert_options', 'replaced_mark')) then
+          options[:replace] = replaced_mark
+        end
+
+        options
+      end
+
       def drb_process_num
         @config.dig('drb_services', 'process_num') || 0
       end
 
+      def drb_server_config
+        config = {}
+        if (load_limit = @config.dig('drb_services', 'load_limit')) then
+          config[:load_limit] = load_limit
+        end
+
+        config
+      end
+
+      def drb_client_config
+        config = {}
+        if (load_limit = @config.dig('drb_services', 'load_limit')) then
+          config[:load_limit] = load_limit
+        end
+
+        config
+      end
+
       def bulk_response_count
         @config.dig('drb_services', 'engine', 'bulk_response_count') || 100
       end
 
+      def bulk_response_size
+        @config.dig('drb_services', 'engine', 'bulk_response_size') || 1024**2 * 10
+      end
+
       def read_lock_timeout_seconds
         @config.dig('drb_services', 'engine', 'read_lock_timeout_seconds') ||
           @config.dig('read_lock_timeout_seconds') || # for backward compatibility
@@ -639,8 +747,8 @@ module RIMS
       end
 
       def make_authentication
-        if ((@config.key? 'authentication') && (@config['authentication'].is_a? Hash)) then
-          auth_conf = @config['authentication']
+        if (@config.dig('authentication')&.is_a? Hash) then
+          auth_conf = @config.dig('authentication')
         else
           auth_conf = {}
         end
@@ -679,8 +787,8 @@ module RIMS
         end
 
         # for backward compatibility
-        if ((@config.key? 'authentication') && (@config['authentication'].is_a? Array)) then
-          plug_in_list = @config['authentication']
+        if (@config.dig('authentication')&.is_a? Array) then
+          plug_in_list = @config.dig('authentication')
           for plug_in_conf in plug_in_list
             plug_in_name = plug_in_conf['plug_in'] or raise KeyError, 'not found an authentication plug_in.'
             plug_in_config = get_configuration(plug_in_conf)
@@ -717,16 +825,16 @@ module RIMS
     using Logger::JointPlus
 
     def setup(server, daemon: false)
-      file_logger_params = @config.make_file_logger_params
-      logger = Logger.new(*file_logger_params)
+      *file_logger_params, file_logger_opts = @config.make_file_logger_params
+      logger = Logger.new(*file_logger_params, **file_logger_opts)
 
-      stdout_logger_params = @config.make_stdout_logger_params
+      *stdout_logger_params, stdout_logger_opts = @config.make_stdout_logger_params
       unless (daemon && @config.daemonize?) then
-        logger += Logger.new(*stdout_logger_params)
+        logger += Logger.new(*stdout_logger_params, **stdout_logger_opts)
       end
 
-      protocol_logger_params = @config.make_protocol_logger_params
-      protocol_logger = Logger.new(*protocol_logger_params)
+      *protocol_logger_params, protocol_logger_opts = @config.make_protocol_logger_params
+      protocol_logger = Logger.new(*protocol_logger_params, **protocol_logger_opts)
 
       logger.info('preload libraries.')
       Riser.preload
@@ -780,22 +888,27 @@ module RIMS
         end
       end
 
-      services = Riser::DRbServices.new(drb_process_num)
-      services.add_sticky_process_service(:engine,
-                                          Riser::ResourceSet.build{|builder|
-                                            builder.at_create{|unique_user_id|
-                                              mail_store = MailStore.build(unique_user_id, kvs_meta_open, kvs_text_open)
-                                              Protocol::Decoder::Engine.new(unique_user_id, mail_store, logger,
-                                                                            bulk_response_count: @config.bulk_response_count,
-                                                                            read_lock_timeout_seconds: @config.read_lock_timeout_seconds,
-                                                                            write_lock_timeout_seconds: @config.write_lock_timeout_seconds,
-                                                                            cleanup_write_lock_timeout_seconds: @config.cleanup_write_lock_timeout_seconds)
-                                            }
-                                            builder.at_destroy{|engine|
-                                              engine.destroy
-                                            }
-                                            builder.alias_unref(:destroy)
-                                          })
+      drb_services = Riser::DRbServices.new(drb_process_num,
+                                            server_config: @config.drb_server_config,
+                                            client_config: @config.drb_client_config)
+      drb_services.add_sticky_process_service(:engine,
+                                              Riser::ResourceSet.build{|builder|
+                                                builder.at_create{|unique_user_id|
+                                                  mail_store = MailStore.build(unique_user_id, kvs_meta_open, kvs_text_open)
+                                                  Protocol::Decoder::Engine.new(unique_user_id, mail_store, logger,
+                                                                                bulk_response_count: @config.bulk_response_count,
+                                                                                bulk_response_size: @config.bulk_response_size,
+                                                                                read_lock_timeout_seconds: @config.read_lock_timeout_seconds,
+                                                                                write_lock_timeout_seconds: @config.write_lock_timeout_seconds,
+                                                                                cleanup_write_lock_timeout_seconds: @config.cleanup_write_lock_timeout_seconds,
+                                                                                charset_aliases: @config.charset_aliases,
+                                                                                charset_convert_options: @config.charset_convert_options)
+                                                }
+                                                builder.at_destroy{|engine|
+                                                  engine.destroy
+                                                }
+                                                builder.alias_unref(:destroy)
+                                              })
 
       server.before_start{|server_socket|
         logger.info('start server.')
@@ -806,17 +919,17 @@ module RIMS
         file_logger_params[1..-2].each_with_index do |value, i|
           logger.info("file logging parameter: shift_args[#{i}]=#{value}")
         end
-        for name, value in file_logger_params[-1]
+        for name, value in file_logger_opts
           logger.info("file logging parameter: #{name}=#{value}")
         end
-        for name, value in stdout_logger_params[-1]
+        for name, value in stdout_logger_opts
           logger.info("stdout logging parameter: #{name}=#{value}")
         end
         logger.info("protocol logging parameter: path=#{protocol_logger_params[0]}")
         protocol_logger_params[1..-2].each_with_index do |value, i|
           logger.info("protocol logging parameter: shift_args[#{i}]=#{value}")
         end
-        for name, value in protocol_logger_params[-1]
+        for name, value in protocol_logger_opts
           logger.info("protocol logging parameter: #{name}=#{value}")
         end
         logger.info("listen address: #{server_socket.local_address.inspect_sockaddr}")
@@ -895,22 +1008,38 @@ module RIMS
         logger.info("connection parameter: send_buffer_limit_size=#{@config.send_buffer_limit_size}")
         logger.info("connection parameter: read_polling_interval_seconds=#{conn_limits.read_polling_interval_seconds}")
         logger.info("connection parameter: command_wait_timeout_seconds=#{conn_limits.command_wait_timeout_seconds}")
+        logger.info("protocol parameter: line_length_limit=#{@config.protocol_line_length_limit}")
+        logger.info("protocol parameter: literal_size_limit=#{@config.protocol_literal_size_limit}")
+        logger.info("protocol parameter: command_size_limit=#{@config.protocol_command_size_limit}")
+        @config.charset_aliases.each_with_index do |(name, enc), i|
+          logger.info("charset aliases parameter: alias[#{i}]: #{name} -> #{enc.name}")
+        end
+        for name, value in @config.charset_convert_options
+          logger.info("charset convert_options parameter: #{name}=#{value}")
+        end
         logger.info("drb_services parameter: process_num=#{drb_process_num}")
+        for name, value in @config.drb_server_config
+          logger.info("drb_services server config parameter: #{name}=#{value}")
+        end
+        for name, value in @config.drb_client_config
+          logger.info("drb_services client config parameter: #{name}=#{value}")
+        end
         logger.info("drb_services engine parameter: bulk_response_count=#{@config.bulk_response_count}")
-        logger.info("lock parameter: read_lock_timeout_seconds=#{@config.read_lock_timeout_seconds}")
-        logger.info("lock parameter: write_lock_timeout_seconds=#{@config.write_lock_timeout_seconds}")
-        logger.info("lock parameter: cleanup_write_lock_timeout_seconds=#{@config.cleanup_write_lock_timeout_seconds}")
+        logger.info("drb_services engine parameter: bulk_response_size=#{@config.bulk_response_size}")
+        logger.info("drb_services engine parameter: read_lock_timeout_seconds=#{@config.read_lock_timeout_seconds}")
+        logger.info("drb_services engine parameter: write_lock_timeout_seconds=#{@config.write_lock_timeout_seconds}")
+        logger.info("drb_services engine parameter: cleanup_write_lock_timeout_seconds=#{@config.cleanup_write_lock_timeout_seconds}")
         kvs_meta_log.call
         kvs_text_log.call
         logger.info("authentication parameter: hostname=#{auth.hostname}")
         logger.info("authorization parameter: mail_delivery_user=#{@config.mail_delivery_user}")
 
         logger.info('dRuby services: start server.')
-        services.start_server
+        drb_services.start_server
       }
       server.at_fork{
         logger.info('dRuby services: detach server.')
-        services.detach_server
+        drb_services.detach_server
       }
       server.at_stop{|stop_state|
         case (stop_state)
@@ -926,7 +1055,7 @@ module RIMS
       }
       server.preprocess{
         logger.info('dRuby services: start client.')
-        services.start_client
+        drb_services.start_client
         auth.start_plug_in(logger)
       }
       server.dispatch{|socket|
@@ -947,7 +1076,11 @@ module RIMS
                   stream = Riser::WriteBufferStream.new(socket, @config.send_buffer_limit_size)
                 end
                 stream = Riser::LoggingStream.new(stream, protocol_logger)
-                decoder = Protocol::Decoder.new_decoder(services, auth, logger, mail_delivery_user: @config.mail_delivery_user)
+                decoder = Protocol::Decoder.new_decoder(drb_services, auth, logger,
+                                                        mail_delivery_user: @config.mail_delivery_user,
+                                                        line_length_limit: @config.protocol_line_length_limit,
+                                                        literal_size_limit: @config.protocol_literal_size_limit,
+                                                        command_size_limit: @config.protocol_command_size_limit)
                 Protocol::Decoder.repl(decoder, conn_limits, stream, stream, logger)
               ensure
                 if (stream) then
@@ -980,7 +1113,7 @@ module RIMS
       }
       server.after_stop{
         logger.info('dRuby services: stop server.')
-        services.stop_server
+        drb_services.stop_server
         logger.info('stop server.')
       }
 

data/lib/rims/test.rb

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+require 'digest'
 require 'fileutils'
 require 'set'
 
@@ -37,8 +38,7 @@ module RIMS
       module_function :message_data_list
 
       def make_body(description)
-        reader = RIMS::Protocol::RequestReader.new(StringIO.new('', 'r'), StringIO.new('', 'w'), Logger.new(STDOUT))
-        reader.parse(reader.scan_line(description))[0]
+        RIMS::Protocol::RequestReader.parse(RIMS::Protocol::RequestReader.scan(description))[0]
       end
       private :make_body
 
@@ -67,23 +67,30 @@ module RIMS
     end
 
     module ProtocolFetchMailSample
-      def make_mail_simple
-        @simple_mail = RIMS::RFC822::Message.new(<<-'EOF')
+      simple_mail_body = <<-'EOF'.freeze
+Hello world.
+      EOF
+
+      MAIL_SIMPLE_TEXT = (<<-"EOF" + simple_mail_body).freeze
 To: foo@nonet.org
 From: bar@nonet.org
 Subject: test
 MIME-Version: 1.0
 Content-Type: text/plain; charset=us-ascii
 Content-Transfer-Encoding: 7bit
+Content-MD5: #{[ Digest::MD5.digest(simple_mail_body) ].pack('m').strip}
+Content-Language: en-US, en
 Date: Fri,  8 Nov 2013 06:47:50 +0900 (JST)
 
-Hello world.
-        EOF
+      EOF
+
+      def make_mail_simple
+        @simple_mail = RIMS::RFC822::Message.new(MAIL_SIMPLE_TEXT)
+        @simple_mail_body = @simple_mail.body.raw_source
       end
       private :make_mail_simple
 
-      def make_mail_multipart
-        @mpart_mail = RIMS::RFC822::Message.new(<<-'EOF')
+      MPART_MAIL_TEXT = <<-'EOF'.freeze
 To: bar@nonet.com
 From: foo@nonet.com
 Subject: multipart test
@@ -97,6 +104,9 @@ Content-Type: text/plain; charset=us-ascii
 Multipart test.
 --1383.905529.351297
 Content-Type: application/octet-stream
+Content-Disposition: attachment
+  ; filename=test.dat
+  ; modification-date="Wed, 12 Feb 1997 16:29:51 -0500"
 
 0123456789
 --1383.905529.351297
@@ -108,6 +118,9 @@ Subject: inner multipart
 MIME-Version: 1.0
 Date: Fri, 8 Nov 2013 19:31:03 +0900
 Content-Type: multipart/mixed; boundary="1383.905529.351298"
+Content-Disposition: attachment; filename=hello.txt
+Content-Language: en
+Content-Location: test
 
 --1383.905529.351298
 Content-Type: text/plain; charset=us-ascii
@@ -123,7 +136,9 @@ Content-Type: multipart/mixed; boundary="1383.905529.351299"
 
 --1383.905529.351299
 Content-Type: image/gif
+Content-Disposition: inline
 
+GIF image...
 --1383.905529.351299
 Content-Type: message/rfc822
 
@@ -136,6 +151,7 @@ Content-Type: multipart/mixed; boundary="1383.905529.351300"
 
 --1383.905529.351300
 Content-Type: text/plain; charset=us-ascii
+Content-Location: foo
 
 HALO
 --1383.905529.351300
@@ -143,10 +159,12 @@ Content-Type: multipart/alternative; boundary="1383.905529.351301"
 
 --1383.905529.351301
 Content-Type: text/plain; charset=us-ascii
+Content-Location: bar
 
 alternative message.
 --1383.905529.351301
 Content-Type: text/html; charset=us-ascii
+Content-Location: baz
 
 <html>
 <body><p>HTML message</p></body>
@@ -155,12 +173,14 @@ Content-Type: text/html; charset=us-ascii
 --1383.905529.351300--
 --1383.905529.351299--
 --1383.905529.351297--
-        EOF
+      EOF
+
+      def make_mail_multipart
+        @mpart_mail = RIMS::RFC822::Message.new(MPART_MAIL_TEXT)
       end
       private :make_mail_multipart
 
-      def make_mail_mime_subject
-        @mime_subject_mail = RIMS::RFC822::Message.new(<<-'EOF')
+      MAIL_MIME_SUBJECT_TEXT = <<-'EOF'.freeze
 Date: Fri, 8 Nov 2013 19:31:03 +0900
 Subject: =?ISO-2022-JP?B?GyRCJEYkOSRIGyhC?=
 From: foo@nonet.com, bar <bar@nonet.com>
@@ -176,22 +196,28 @@ In-Reply-To: <20131106081723.5KJU1774292@smtp.test.com>
 Message-Id: <20131107214750.445A1255B9F@smtp.nonet.com>
 
 Hello world.
-        EOF
+      EOF
+
+      def make_mail_mime_subject
+        @mime_subject_mail = RIMS::RFC822::Message.new(MAIL_MIME_SUBJECT_TEXT)
       end
       private :make_mail_mime_subject
 
+      MAIL_EMPTY_TEXT = ''.freeze
+
       def make_mail_empty
-        @empty_mail = RIMS::RFC822::Message.new('')
+        @empty_mail = RIMS::RFC822::Message.new(MAIL_EMPTY_TEXT)
       end
       private :make_mail_empty
 
+      MAIL_NO_BODY_TEXT = "Subject: foo\r\n\r\n".freeze
+
       def make_mail_no_body
-        @no_body_mail = RIMS::RFC822::Message.new('foo')
+        @no_body_mail = RIMS::RFC822::Message.new(MAIL_NO_BODY_TEXT)
       end
       private :make_mail_no_body
 
-      def make_mail_address_header_pattern
-        @address_header_pattern_mail = RIMS::RFC822::Message.new(<<-'EOF')
+      MAIL_ADDRESS_HEADER_PATTERN_TEXT = <<-'EOF'.freeze
 To: "foo@nonet.org" <foo@nonet.org>
 From: bar@nonet.org
 Subject: test
@@ -201,7 +227,10 @@ Content-Transfer-Encoding: 7bit
 Date: Fri,  8 Nov 2013 06:47:50 +0900 (JST)
 
 Hello world.
-EOF
+      EOF
+
+      def make_mail_address_header_pattern
+        @address_header_pattern_mail = RIMS::RFC822::Message.new(MAIL_ADDRESS_HEADER_PATTERN_TEXT)
       end
       private :make_mail_address_header_pattern
     end