rims 0.2.5 → 0.3.0

data/lib/rims/protocol/parser.rb

@@ -33,21 +33,7 @@ module RIMS
     end
 
     class RequestReader
-      def initialize(input, output, logger)
-        @input = input
-        @output = output
-        @logger = logger
-      end
-
-      def read_line
-        line = @input.gets or return
-        @logger.debug("read line: #{Protocol.io_data_log(line)}") if @logger.debug?
-        line.chomp!("\n")
-        line.chomp!("\r")
-        scan_line(line)
-      end
-
-      def scan_line(line)
+      def self.scan(line)
         atom_list = line.scan(/BODY(?:\.\S+)?\[.*?\](?:<\d+\.\d+>)?|[\[\]()]|".*?"|[^\[\]()\s]+/i).map{|s|
           case (s)
           when '(', ')', '[', ']', /\A NIL \z/ix
@@ -77,22 +63,14 @@ module RIMS
           end
         }
         if ((atom_list[-1].is_a? String) && (atom_list[-1] =~ /\A {\d+} \z/x)) then
-          next_size = $&[1..-2].to_i
-          @logger.debug("found literal: #{next_size} octets.") if @logger.debug?
-          @output.write("+ continue\r\n")
-          @output.flush
-          @logger.debug('continue literal.') if @logger.debug?
-          literal_string = @input.read(next_size) or raise 'unexpected client close.'
-          @logger.debug("read literal: #{Protocol.io_data_log(literal_string)}") if @logger.debug?
-          atom_list[-1] = literal_string
-          next_atom_list = read_line or raise 'unexpected client close.'
-          atom_list += next_atom_list
+          literal_size = $&[1..-2].to_i
+          atom_list[-1] = [ :literal, literal_size ]
         end
 
         atom_list
       end
 
-      def parse(atom_list, last_atom=nil)
+      def self.parse(atom_list, last_atom=nil)
         syntax_list = []
         while (atom = atom_list.shift)
           case (atom)
@@ -105,7 +83,7 @@ module RIMS
           else
             if ((atom.is_a? Array) && (atom[0] == :body)) then
               body = atom[1]
-              body.section_list = parse(scan_line(body.section))
+              body.section_list = parse(scan(body.section))
             end
             syntax_list.push(atom)
           end
@@ -118,18 +96,90 @@ module RIMS
         syntax_list
       end
 
+      def initialize(input, output, logger, line_length_limit: 1024*8, literal_size_limit: (1024**2)*10, command_size_limit: (1024**2)*10)
+        @input = input
+        @output = output
+        @logger = logger
+        @line_length_limit = line_length_limit
+        @literal_size_limit = literal_size_limit
+        @command_size_limit = command_size_limit
+        @command_tag = nil
+        @read_size = 0
+      end
+
+      attr_reader :command_tag
+
+      def gets
+        if (line = @input.gets($/, @line_length_limit)) then # arguments compatible with OpenSSL::Buffering#gets
+          if (line.bytesize < @line_length_limit) then
+            line
+          elsif (line.bytesize == @line_length_limit && (line.end_with? $/)) then
+            line
+          else
+            raise LineTooLongError.new('line too long.', line_fragment: line)
+          end
+        end
+      end
+
+      def read_literal(size)
+        @logger.debug("found literal: #{size} octets.") if @logger.debug?
+        if (size > @literal_size_limit) then
+          raise LiteralSizeTooLargeError.new('literal size too large', @command_tag)
+        end
+        if (@read_size + size > @command_size_limit) then
+          raise CommandSizeTooLargeError.new('command size too large', @command_tag)
+        end
+        @output.write("+ continue\r\n")
+        @output.flush
+        @logger.debug('continue literal.') if @logger.debug?
+        literal_string = @input.read(size) or raise 'unexpected client close.'
+        @read_size += size
+        @logger.debug("read literal: #{Protocol.io_data_log(literal_string)}") if @logger.debug?
+
+        literal_string
+      end
+      private :read_literal
+
+      def read_line
+        line = gets or return
+        @logger.debug("read line: #{Protocol.io_data_log(line)}") if @logger.debug?
+        line.chomp!("\n")
+        line.chomp!("\r")
+        @read_size += line.bytesize
+        if (@read_size > @command_size_limit) then
+          raise CommandSizeTooLargeError.new('command size too large', @command_tag)
+        end
+        atom_list = self.class.scan(line)
+
+        if (@command_tag.nil? && ! atom_list.empty?) then
+          unless ((atom_list[0].is_a? String) && ! (atom_list[0].start_with? '*', '+')) then
+            raise SyntaxError, "invalid command tag: #{atom_list[0]}"
+          end
+          @command_tag = atom_list[0]
+        end
+
+        if ((atom_list[-1].is_a? Array) && (atom_list[-1][0] == :literal)) then
+          atom_list[-1] = read_literal(atom_list[-1][1])
+          next_atom_list = read_line or raise 'unexpected client close.'
+          atom_list += next_atom_list
+        end
+
+        atom_list
+      end
+      private :read_line
+
       def read_command
+        @command_tag = nil
+        @read_size = 0
         while (atom_list = read_line)
           if (atom_list.empty?) then
+            @read_size = 0
             next
           end
           if (atom_list.length < 2) then
-            raise 'need for tag and command.'
-          end
-          if (atom_list[0] =~ /\A [*+]/x) then
-            raise "invalid command tag: #{atom_list[0]}"
+            raise SyntaxError, 'need for tag and command.'
           end
-          return parse(atom_list)
+          return self.class.parse(atom_list)
         end
 
         nil
@@ -228,13 +278,15 @@ module RIMS
     end
 
     class SearchParser
-      def initialize(mail_store, folder)
+      def initialize(mail_store, folder, charset_aliases: RFC822::DEFAULT_CHARSET_ALIASES, charset_convert_options: nil)
         @mail_store = mail_store
         @folder = folder
+        @charset_aliases = charset_aliases
+        @charset_convert_options = charset_convert_options || {}
         @charset = nil
         @mail_cache = Hash.new{|hash, uid|
           if (msg_txt = @mail_store.msg_text(@folder.mbox_id, uid)) then
-            hash[uid] = RFC822::Message.new(msg_txt)
+            hash[uid] = RFC822::Message.new(msg_txt, charset_aliases: @charset_aliases)
           end
         }
       end
@@ -244,38 +296,38 @@ module RIMS
       end
       private :get_mail
 
-      attr_accessor :charset
+      attr_reader :charset
 
-      def string_include?(search_string, text)
-        if (search_string.ascii_only?) then
-          unless (text.encoding.ascii_compatible?) then
-            text = text.encode('utf-8')
-          end
-        else
-          if (@charset) then
-            search_string = search_string.dup.force_encoding(@charset)
-            text = text.encode(@charset)
-          end
+      def charset=(new_charset)
+        charset_encoding = @charset_aliases[new_charset] || Encoding.find(new_charset)
+        if (charset_encoding.dummy?) then
+          # same error type as `Encoding.find'
+          raise ArgumentError, "not a searchable charset: #{new_charset}"
         end
+        @charset = charset_encoding
+      end
 
-        text.include? search_string
+      def force_charset(string)
+        string = string.dup
+        string.force_encoding(@charset)
+        string.valid_encoding? or raise SyntaxError, "invalid #{@charset} string: #{string.inspect}"
+        string
       end
-      private :string_include?
+      private :force_charset
 
-      def mail_body_text(mail)
-        if (mail.text? || mail.message?) then
-          body_txt = mail.body.raw_source
-          if (charset = mail.charset) then
-            if (body_txt.encoding != Encoding.find(charset)) then
-              body_txt = body_txt.dup.force_encoding(charset)
-            end
-          end
-          body_txt
+      def encode_charset(string)
+        if (string.encoding == @charset) then
+          string
         else
-          nil
+          string.encode(@charset, **@charset_convert_options)
         end
       end
-      private :mail_body_text
+      private :encode_charset
+
+      def compile_search_regexp(search_string)
+        Regexp.new(Regexp.quote(search_string), true)
+      end
+      private :compile_search_regexp
 
       def end_of_cond
         proc{|msg| true }
@@ -310,13 +362,29 @@ module RIMS
       private :parse_msg_flag_enabled
 
       def parse_search_header(field_name, search_string)
+        if (@charset) then
+          search_string = force_charset(search_string)
+          search_regexp = compile_search_regexp(search_string)
+          search_header = proc{|mail|
+            mail.mime_decoded_header_field_value_list(field_name, @charset, charset_convert_options: @charset_convert_options).any?{|field_value|
+              search_regexp.match? field_value
+            }
+          }
+        else
+          search_string = search_string.b
+          search_regexp = compile_search_regexp(search_string)
+          search_header = proc{|mail|
+            mail.header.field_value_list(field_name).any?{|field_value|
+              search_regexp.match? field_value
+            }
+          }
+        end
+
         proc{|next_cond|
           proc{|msg|
             mail = get_mail(msg)
             if (mail.header.key? field_name) then
-              mail.header.field_value_list(field_name).any?{|field_value|
-                string_include?(search_string, field_value)
-              } && next_cond.call(msg)
+              search_header.call(mail) && next_cond.call(msg)
             else
               false
             end
@@ -359,10 +427,40 @@ module RIMS
       private :parse_mail_bytesize
 
       def parse_body(search_string)
+        if (@charset)
+          search_string = force_charset(search_string)
+          search_regexp = compile_search_regexp(search_string)
+          search_body = proc{|mail|
+            if (mail.text? || mail.messge?) then
+              search_regexp.match? encode_charset(mail.mime_charset_body_text)
+            elsif (mail.multipart?) then
+              mail.parts.any?{|next_mail|
+                search_body.call(next_mail)
+              }
+            else
+              false
+            end
+          }
+        else
+          search_string = search_string.b
+          search_regexp = compile_search_regexp(search_string)
+          search_body = proc{|mail|
+            if (mail.text? || mail.message?)then
+              search_regexp.match? mail.mime_binary_body_string
+            elsif (mail.multipart?) then
+              mail.parts.any?{|next_mail|
+                search_body.call(next_mail)
+              }
+            else
+              false
+            end
+          }
+        end
+
         proc{|next_cond|
           proc{|msg|
-            if (text = mail_body_text(get_mail(msg))) then
-              string_include?(search_string, text) && next_cond.call(msg)
+            if (mail = get_mail(msg)) then
+              search_body.call(mail) && next_cond.call(msg)
             else
               false
             end
@@ -421,19 +519,44 @@ module RIMS
       private :parse_or
 
       def parse_text(search_string)
-        search_text = proc{|message_text| string_include?(search_string, message_text) }
-        search_mail = proc{|mail|
-          if (mail.multipart?) then
-            search_text.call(mail.header.raw_source) || mail.parts.any?{|m| search_mail.call(m) }
-          else
-            body_text = mail_body_text(mail)
-            search_text.call(mail.header.raw_source) || (body_text && search_text.call(body_text))
-          end
-        }
+        if (@charset) then
+          search_string = force_charset(search_string)
+          search_regexp = compile_search_regexp(search_string)
+          search_text = proc{|mail|
+            if (search_regexp.match? mail.mime_decoded_header_text(@charset, charset_convert_options: @charset_convert_options)) then
+              true
+            elsif (mail.text? || mail.message?) then
+              search_regexp.match? encode_charset(mail.mime_charset_body_text)
+            elsif (mail.multipart?) then
+              mail.parts.any?{|next_mail|
+                search_text.call(next_mail)
+              }
+            else
+              false
+            end
+          }
+        else
+          search_string = search_string.b
+          search_regexp = compile_search_regexp(search_string)
+          search_text = proc{|mail|
+            if (search_regexp.match? mail.header.raw_source) then
+              true
+            elsif (mail.text? || mail.message?) then
+              search_regexp.match? mail.mime_binary_body_string
+            elsif (mail.multipart?) then
+              mail.parts.any?{|next_mail|
+                search_text.call(next_mail)
+              }
+            else
+              false
+            end
+          }
+        end
+
         proc{|next_cond|
           proc{|msg|
             mail = get_mail(msg)
-            search_mail.call(mail) && next_cond.call(msg)
+            search_text.call(mail) && next_cond.call(msg)
           }
         }
       end
@@ -690,14 +813,10 @@ module RIMS
           if ((array.length > 0) && (array.first.is_a? Array)) then
             s = '('.b
             array = array.dup
-            while (object = array.shift)
-              case (object)
-              when Array
-                s << encode_bodystructure(object)
-              else
-                s << ' '.b << encode_value(object)
-              end
-            end
+            begin
+              s << encode_bodystructure(array.shift)
+            end while ((array.length > 0) && (array.first.is_a? Array))
+            s << ' '.b << array.map{|i| encode_value(i) }.join(' '.b)
             s << ')'.b
           elsif ((array.length > 0) && (array.first.upcase == 'MESSAGE')) then
             msg_body_list = array[0..7].map{|v| encode_value(v) }
@@ -757,13 +876,12 @@ module RIMS
       end
       include Utils
 
-      def initialize(mail_store, folder)
+      def initialize(mail_store, folder, charset_aliases: RFC822::DEFAULT_CHARSET_ALIASES)
         @mail_store = mail_store
         @folder = folder
-        @charset = nil
         @mail_cache = Hash.new{|hash, uid|
           if (msg_txt = @mail_store.msg_text(@folder.mbox_id, uid)) then
-            hash[uid] = RFC822::Message.new(msg_txt)
+            hash[uid] = RFC822::Message.new(msg_txt, charset_aliases: charset_aliases)
           end
         }
       end
@@ -773,29 +891,6 @@ module RIMS
       end
       private :get_mail
 
-      def make_array(value)
-        if (value) then
-          if (value.is_a? Array) then
-            list = value
-          else
-            list = [ value ]
-          end
-
-          if (block_given?) then
-            yield(list)
-          else
-            list
-          end
-        end
-      end
-      private :make_array
-
-      def make_address_list(email_address)
-        mailbox, host = email_address.split(/@/, 2)
-        [ nil, nil, mailbox, host ]
-      end
-      private :make_address_list
-
       def expand_macro(cmd_list)
         func_list = cmd_list.map{|name| parse_cached(name) }
         proc{|msg|
@@ -804,76 +899,116 @@ module RIMS
       end
       private :expand_macro
 
-      def get_header_field(mail, name, default=nil)
-        if (field = mail[name]) then
-          if (block_given?) then
-            yield(field)
-          else
-            field
-          end
+      def make_body_params(name_value_pair_list)
+        if (name_value_pair_list && ! name_value_pair_list.empty?) then
+          name_value_pair_list.flatten
         else
-          default
+          # not allowed empty body field parameters.
+          # RFC 3501 / 9. Formal Syntax:
+          #     body-fld-param  = "(" string SP string *(SP string SP string) ")" / nil
+          nil
         end
       end
-      private :get_header_field
+      private :make_body_params
 
-      def get_bodystructure_data(mail)
+      def get_body_disposition(mail)
+        if (disposition_type = mail.content_disposition_upcase) then
+          [ disposition_type,
+            make_body_params(mail.content_disposition_parameter_list)
+          ]
+        else
+          # not allowed empty body field disposition.
+          # RFC 3501 / 9. Formal Syntax:
+          #     body-fld-dsp    = "(" string SP body-fld-param ")" / nil
+          nil
+        end
+      end
+      private :get_body_disposition
+
+      def get_body_lang(mail)
+        if (tag_list = mail.content_language_upcase) then
+          unless (tag_list.empty?) then
+            if (tag_list.length == 1) then
+              tag_list[0]
+            else
+              tag_list
+            end
+          end
+        end
+      end
+      private :get_body_lang
+
+      def get_bodystructure_data(mail, extension: false)
+        body_data = []
         if (mail.multipart?) then       # body_type_mpart
-          mpart_data = []
-          mpart_data.concat(mail.parts.map{|part_msg| get_bodystructure_data(part_msg) })
-          mpart_data << mail.media_sub_type_upcase
-        elsif (mail.text?) then         # body_type_text
-          text_data = []
-
-          # media_text
-          text_data << mail.media_main_type_upcase
-          text_data << mail.media_sub_type_upcase
-
-          # body_fields
-          text_data << mail.content_type_parameters.flatten
-          text_data << mail.header['Content-Id']
-          text_data << mail.header['Content-Description']
-          text_data << mail.header.fetch_upcase('Content-Transfer-Encoding')
-          text_data << mail.raw_source.bytesize
-
-          # body_fld_lines
-          text_data << mail.raw_source.each_line.count
-        elsif (mail.message?) then      # body_type_msg
-          msg_data = []
-
-          # message_media
-          msg_data << mail.media_main_type_upcase
-          msg_data << mail.media_sub_type_upcase
-
-          # body_fields
-          msg_data << mail.content_type_parameters.flatten
-          msg_data << mail.header['Content-Id']
-          msg_data << mail.header['Content-Description']
-          msg_data << mail.header.fetch_upcase('Content-Transfer-Encoding')
-          msg_data << mail.raw_source.bytesize
-
-          # envelope
-          msg_data << get_envelope_data(mail.message)
-
-          # body
-          msg_data << get_bodystructure_data(mail.message)
-
-          # body_fld_lines
-          msg_data << mail.raw_source.each_line.count
-        else                            # body_type_basic
-          basic_data = []
-
-          # media_basic
-          basic_data << mail.media_main_type_upcase
-          basic_data << mail.media_sub_type_upcase
-
-          # body_fields
-          basic_data << mail.content_type_parameters.flatten
-          basic_data << mail.header['Content-Id']
-          basic_data << mail.header['Content-Description']
-          basic_data << mail.header.fetch_upcase('Content-Transfer-Encoding')
-          basic_data << mail.raw_source.bytesize
+          body_data.concat(mail.parts.map{|part_msg| get_bodystructure_data(part_msg, extension: extension) })
+          body_data << mail.media_sub_type_upcase
+
+          # body_ext_mpart
+          if (extension) then
+            body_data << make_body_params(mail.content_type_parameter_list)
+            body_data << get_body_disposition(mail)
+            body_data << get_body_lang(mail)
+            body_data << mail.header['Content-Location']
+          end
+        else
+          if (mail.text?) then          # body_type_text
+            # media_text
+            body_data << mail.media_main_type_upcase
+            body_data << mail.media_sub_type_upcase
+
+            # body_fields
+            body_data << make_body_params(mail.content_type_parameter_list)
+            body_data << mail.header['Content-Id']
+            body_data << mail.header['Content-Description']
+            body_data << mail.header.fetch_upcase('Content-Transfer-Encoding')
+            body_data << mail.raw_source.bytesize
+
+            # body_fld_lines
+            body_data << mail.raw_source.each_line.count
+          elsif (mail.message?) then    # body_type_msg
+            # message_media
+            body_data << mail.media_main_type_upcase
+            body_data << mail.media_sub_type_upcase
+
+            # body_fields
+            body_data << make_body_params(mail.content_type_parameter_list)
+            body_data << mail.header['Content-Id']
+            body_data << mail.header['Content-Description']
+            body_data << mail.header.fetch_upcase('Content-Transfer-Encoding')
+            body_data << mail.raw_source.bytesize
+
+            # envelope
+            body_data << get_envelope_data(mail.message)
+
+            # body
+            body_data << get_bodystructure_data(mail.message, extension: extension)
+
+            # body_fld_lines
+            body_data << mail.raw_source.each_line.count
+          else                          # body_type_basic
+            # media_basic
+            body_data << mail.media_main_type_upcase
+            body_data << mail.media_sub_type_upcase
+
+            # body_fields
+            body_data << make_body_params(mail.content_type_parameter_list)
+            body_data << mail.header['Content-Id']
+            body_data << mail.header['Content-Description']
+            body_data << mail.header.fetch_upcase('Content-Transfer-Encoding')
+            body_data << mail.raw_source.bytesize
+          end
+
+          # body_ext_1part
+          if (extension) then
+            body_data << mail.header['Content-MD5']
+            body_data << get_body_disposition(mail)
+            body_data << get_body_lang(mail)
+            body_data << mail.header['Content-Location']
+          end
         end
+
+        body_data
       end
       private :get_bodystructure_data
 
@@ -881,12 +1016,12 @@ module RIMS
         env_data = []
         env_data << mail.header['Date']
         env_data << mail.header['Subject']
-        env_data << mail.from
-        env_data << mail.sender
-        env_data << mail.reply_to
-        env_data << mail.to
-        env_data << mail.cc
-        env_data << mail.bcc
+        env_data << mail.from&.map(&:to_a)
+        env_data << mail.sender&.map(&:to_a)
+        env_data << mail.reply_to&.map(&:to_a)
+        env_data << mail.to&.map(&:to_a)
+        env_data << mail.cc&.map(&:to_a)
+        env_data << mail.bcc&.map(&:to_a)
         env_data << mail.header['In-Reply-To']
         env_data << mail.header['Message-Id']
       end
@@ -1034,9 +1169,9 @@ module RIMS
       end
       private :parse_body
 
-      def parse_bodystructure(msg_att_name)
+      def parse_bodystructure(msg_att_name, extension: false)
         proc{|msg|
-          ''.b << msg_att_name << ' '.b << encode_bodystructure(get_bodystructure_data(get_mail(msg)))
+          ''.b << msg_att_name << ' '.b << encode_bodystructure(get_bodystructure_data(get_mail(msg), extension: extension))
         }
       end
       private :parse_bodystructure
@@ -1095,9 +1230,9 @@ module RIMS
         when 'ALL'
           fetch = expand_macro(%w[ FLAGS INTERNALDATE RFC822.SIZE ENVELOPE ])
         when 'BODY'
-          fetch = parse_bodystructure(fetch_att)
+          fetch = parse_bodystructure(fetch_att, extension: false)
         when 'BODYSTRUCTURE'
-          fetch = parse_bodystructure(fetch_att)
+          fetch = parse_bodystructure(fetch_att, extension: true)
         when 'ENVELOPE'
           fetch = parse_envelope(fetch_att)
         when 'FAST'