mail 2.5.3 → 2.6.0

data/lib/mail/parsers/ragel/ruby/machines/received_machine.rb.rl

@@ -0,0 +1,37 @@
+%%{
+  machine received;
+
+  include rb_actions "rb_actions.rl";
+  include common "../../common.rl";
+
+  getkey data_unpacked[p];
+
+  main := received;
+}%%
+
+module Mail
+  module Parsers
+    module Ragel
+      module ReceivedMachine
+        %%write data;
+
+        def self.parse(data)
+          p = 0
+          eof = data.length
+          stack = []
+
+          actions = []
+          data_unpacked = data.bytes.to_a
+          %%write init;
+          %%write exec;
+
+          if p == eof && cs >= %%{ write first_final; }%%
+            return actions, nil
+          else
+            return [], "Only able to parse up to #{data[0..p]}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mail/parsers/ragel/ruby/parser.rb.rl.erb

@@ -0,0 +1,37 @@
+%%{
+  machine <%= parser_name %>;
+
+  include rb_actions "rb_actions.rl";
+  include common "../../common.rl";
+
+  getkey data_unpacked[p];
+
+  main := <%= parser_name %>;
+}%%
+
+module Mail
+  module Parsers
+    module Ragel
+      module <%= parser_name_cc %>
+        %%write data;
+
+        def self.parse(data)
+          p = 0
+          eof = data.length
+          stack = []
+
+          actions = []
+          data_unpacked = data.bytes.to_a
+          %%write init;
+          %%write exec;
+
+          if p == eof && cs >= %%{ write first_final; }%%
+            return actions, nil
+          else
+            return [], "Only able to parse up to #{data[0..p]}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mail/parsers/ragel/ruby.rb

@@ -0,0 +1,29 @@
+module Mail
+  module Parsers
+    module Ragel
+      module Ruby
+        Mail::Parsers::Ragel::FIELD_PARSERS.each do |field_parser|
+          require "mail/parsers/ragel/ruby/machines/#{field_parser}_machine"
+        end
+
+        MACHINE_LIST = {
+          :address_lists => AddressListsMachine,
+          :phrase_lists => PhraseListsMachine,
+          :date_time => DateTimeMachine,
+          :received => ReceivedMachine,
+          :message_ids => MessageIdsMachine,
+          :envelope_from => EnvelopeFromMachine,
+          :mime_version => MimeVersionMachine,
+          :content_type => ContentTypeMachine,
+          :content_disposition => ContentDispositionMachine,
+          :content_transfer_encoding => ContentTransferEncodingMachine,
+          :content_location => ContentLocationMachine
+        }
+
+        def self.parse(machine, string)
+          MACHINE_LIST[machine].parse(string)
+        end
+      end
+    end
+  end
+end

data/lib/mail/parsers/ragel.rb

@@ -0,0 +1,17 @@
+module Mail
+  module Parsers
+    module Ragel
+      require 'mail/parsers/ragel/parser_info'
+      require "mail/parsers/ragel/ruby"
+
+      def self.parse(machine, string)
+        @machine_module ||= Ruby
+        @machine_module.parse(machine, string)
+      end
+
+      def self.machine_module=(m)
+        @machine_module = m
+      end
+    end
+  end
+end

data/lib/mail/parsers/received_parser.rb

@@ -0,0 +1,47 @@
+module Mail::Parsers
+  class ReceivedParser
+
+    def parse(s)
+      actions, error = Ragel.parse(:received, s)
+      if error
+        raise Mail::Field::ParseError.new(Mail::ReceivedElement, s, error)
+      end
+
+      received = ReceivedStruct.new
+
+      received_tokens_s = date_s = time_s = nil
+      actions.each_slice(2) do |action_id, p|
+        action = Mail::Parsers::Ragel::ACTIONS[action_id]
+        case action
+
+        # Received Tokens:
+        when :received_tokens_s then received_tokens_s = p
+        when :received_tokens_e
+          received.info = s[received_tokens_s..(p-1)]
+
+        # Date
+        when :date_s then date_s = p
+        when :date_e
+          received.date = s[date_s..(p-1)].strip
+
+        # Time
+        when :time_s then time_s = p
+        when :time_e
+          received.time = s[time_s..(p-1)]
+
+        when :angle_addr_s, :comment_e, :comment_s,
+             :domain_e, :domain_s, :local_dot_atom_e,
+             :local_dot_atom_pre_comment_e,
+             :local_dot_atom_pre_comment_s,
+             :local_dot_atom_s, :qstr_e, :qstr_s,
+             :local_quoted_string_s, :local_quoted_string_e
+          # ignored actions
+
+        else
+          raise Mail::Field::ParseError.new(Mail::ReceivedElement, s, "Failed to process unknown action: #{action}")
+        end
+      end
+      received
+    end
+  end
+end

data/lib/mail/parsers.rb

@@ -0,0 +1,26 @@
+module Mail
+  module Parsers
+
+    # Low-level ragel based parsers
+    require 'mail/parsers/ragel'
+
+    AddressListStruct = Struct.new(:addresses, :group_names, :error)
+    AddressStruct = Struct.new(:raw, :domain, :comments, :local,
+                             :obs_domain_list, :display_name, :group, :error)
+    ContentDispositionStruct = Struct.new(:disposition_type, :parameters, :error)
+    ContentLocationStruct = Struct.new(:location, :error)
+    ContentTransferEncodingStruct = Struct.new(:encoding, :error)
+    ContentTypeStruct = Struct.new(:main_type, :sub_type, :parameters, :error)
+    DateTimeStruct = Struct.new(:date_string, :time_string, :error)
+    EnvelopeFromStruct = Struct.new(:address, :ctime_date, :error)
+    MessageIdsStruct = Struct.new(:message_ids, :error)
+    MimeVersionStruct = Struct.new(:major, :minor, :error)
+    PhraseListsStruct = Struct.new(:phrases, :error)
+    ReceivedStruct = Struct.new(:date, :time, :info, :error)
+
+    require 'mail/parsers/ragel/parser_info'
+    Ragel::FIELD_PARSERS.each do |field_parser|
+      require "mail/parsers/#{field_parser}_parser"
+    end
+  end
+end

data/lib/mail/part.rb

@@ -38,8 +38,12 @@ module Mail
     end
     
     def add_required_fields
-      add_content_id unless has_content_id?
       super
+      add_content_id if !has_content_id? && inline?
+    end
+
+    def add_required_message_fields
+      # Override so we don't add Date, MIME-Version, or Message-ID.
     end
     
     def delivery_status_report_part?
@@ -113,4 +117,4 @@ module Mail
 
   end
   
-end
+end

data/lib/mail/parts_list.rb

@@ -31,11 +31,13 @@ module Mail
     end
 
     def sort!(order)
-      sorted = self.sort do |a, b|
+      # stable sort should be used to maintain the relative order as the parts are added
+      i = 0;
+      sorted = self.sort_by do |a|
         # OK, 10000 is arbitrary... if anyone actually wants to explicitly sort 10000 parts of a
         # single email message... please show me a use case and I'll put more work into this method,
         # in the meantime, it works :)
-        get_order_value(a, order) <=> get_order_value(b, order)
+        [get_order_value(a, order), i += 1]
       end
       self.clear
       sorted.each { |p| self << p }
@@ -44,7 +46,7 @@ module Mail
   private
 
     def get_order_value(part, order)
-      if part.respond_to?(:content_type)
+      if part.respond_to?(:content_type) && !part[:content_type].nil?
         order.index(part[:content_type].string.downcase) || 10000
       else
         10000

data/lib/mail/patterns.rb

@@ -20,9 +20,12 @@ module Mail
     FWS           = /#{CRLF}#{WSP}*/
     TEXT          = /[#{text}]/ # + obs-text
     FIELD_NAME    = /[#{field_name}]+/
-    FIELD_BODY    = /.+/
+    FIELD_PREFIX  = /\A(#{FIELD_NAME})/
+    FIELD_BODY    = /.+/m
     FIELD_LINE    = /^[#{field_name}]+:\s*.+$/
+    FIELD_SPLIT   = /^(#{FIELD_NAME})\s*:\s*(#{FIELD_BODY})?$/
     HEADER_LINE   = /^([#{field_name}]+:\s*.+)$/
+    HEADER_SPLIT  = /#{CRLF}(?!#{WSP})/
 
     QP_UNSAFE     = /[^#{qp_safe}]/
     QP_SAFE       = /[#{qp_safe}]/

data/lib/mail/utilities.rb

@@ -21,7 +21,9 @@ module Mail
         original_encoding = str.encoding
         str.force_encoding('ASCII-8BIT')
         if (PHRASE_UNSAFE === str)
-          dquote(str).force_encoding(original_encoding)
+          quoted_str = dquote(str).force_encoding(original_encoding)
+          str.force_encoding(original_encoding)
+          quoted_str
         else
           str.force_encoding(original_encoding)
         end
@@ -41,28 +43,36 @@ module Mail
       token_safe?( str ) ? str : dquote(str)
     end
 
-    # Wraps supplied string in double quotes unless it is already wrapped.
-    # 
-    # Additionally will escape any double quotation marks in the string with a single
-    # backslash in front of the '"' character.
+    # Wraps supplied string in double quotes and applies \-escaping as necessary,
+    # unless it is already wrapped.
+    #
+    # Example:
+    #
+    #  string = 'This is a string'
+    #  dquote(string) #=> '"This is a string"'
+    #
+    #  string = 'This is "a string"'
+    #  dquote(string #=> '"This is \"a string\"'
     def dquote( str )
-      match = str.match(/^"(.*)?"$/)
-      str = match[1] if match
-      # First remove all escaped double quotes:
-      str = str.gsub(/\\"/, '"')
-      # Then wrap and re-escape all double quotes
-      '"' + str.gsub(/["]/n) {|s| '\\' + s } + '"'
+      '"' + unquote(str).gsub(/[\\"]/n) {|s| '\\' + s } + '"'
     end
-    
-    # Unwraps supplied string from inside double quotes.
-    # 
+
+    # Unwraps supplied string from inside double quotes and
+    # removes any \-escaping.
+    #
     # Example:
-    # 
+    #
     #  string = '"This is a string"'
     #  unquote(string) #=> 'This is a string'
+    #
+    #  string = '"This is \"a string\""'
+    #  unqoute(string) #=> 'This is "a string"'
     def unquote( str )
-      match = str.match(/^"(.*?)"$/)
-      match ? match[1] : str
+      if str =~ /^"(.*?)"$/
+        $1.gsub(/\\(.)/, '\1')
+      else
+        str
+      end
     end
     
     # Wraps a string in parenthesis and escapes any that are in the string itself.
@@ -135,7 +145,7 @@ module Mail
     #  obj1 = :this_IS_an_object
     #  match_to_s( obj1, obj2 ) #=> true
     def match_to_s( obj1, obj2 )
-      obj1.to_s.downcase == obj2.to_s.downcase
+      obj1.to_s.casecmp(obj2.to_s) == 0
     end
     
     # Capitalizes a string that is joined by hyphens correctly.

data/lib/mail/version.rb

@@ -3,7 +3,7 @@ module Mail
   module VERSION
     
     version = {}
-    File.read(File.join(File.dirname(__FILE__), '../', 'VERSION')).each_line do |line|
+    File.read(File.join(File.dirname(__FILE__), '../', '../', 'VERSION')).each_line do |line|
       type, value = line.chomp.split(":")
       next if type =~ /^\s+$/  || value =~ /^\s+$/
       version[type] = value

data/lib/mail/version_specific/ruby_1_8.rb

@@ -58,7 +58,7 @@ module Mail
       # Ruby 1.8 requires an encoding to work
       raise ArgumentError, "Must supply an encoding" if encoding.nil?
       encoding = encoding.to_s.upcase.gsub('_', '-')
-      [Encodings::Base64.encode(str), encoding]
+      [Encodings::Base64.encode(str), fix_encoding(encoding)]
     end
 
     def Ruby18.b_value_decode(str)
@@ -105,8 +105,12 @@ module Mail
 
     def Ruby18.fix_encoding(encoding)
       case encoding.upcase
-      when 'UTF8'
+      when 'UTF8', 'UTF_8'
         'UTF-8'
+      when 'UTF16', 'UTF-16'
+        'UTF-16BE'
+      when 'UTF32', 'UTF-32'
+        'UTF-32BE'
       else
         encoding
       end

data/lib/mail/version_specific/ruby_1_9.rb

@@ -51,9 +51,9 @@ module Mail
     def Ruby19.b_value_decode(str)
       match = str.match(/\=\?(.+)?\?[Bb]\?(.+)?\?\=/m)
       if match
-        encoding = match[1]
+        charset = match[1]
         str = Ruby19.decode_base64(match[2])
-        str.force_encoding(fix_encoding(encoding))
+        str.force_encoding(pick_encoding(charset))
       end
       decoded = str.encode("utf-8", :invalid => :replace, :replace => "")
       decoded.valid_encoding? ? decoded : decoded.encode("utf-16le", :invalid => :replace, :replace => "").encode("utf-8")
@@ -67,12 +67,15 @@ module Mail
     def Ruby19.q_value_decode(str)
       match = str.match(/\=\?(.+)?\?[Qq]\?(.+)?\?\=/m)
       if match
-        encoding = match[1]
+        charset = match[1]
         string = match[2].gsub(/_/, '=20')
         # Remove trailing = if it exists in a Q encoding
         string = string.sub(/\=$/, '')
         str = Encodings::QuotedPrintable.decode(string)
-        str.force_encoding(fix_encoding(encoding))
+        str.force_encoding(pick_encoding(charset))
+        # We assume that binary strings hold utf-8 directly to work around
+        # jruby/jruby#829 which subtly changes String#encode semantics.
+        str.force_encoding('utf-8') if str.encoding == Encoding::ASCII_8BIT
       end
       decoded = str.encode("utf-8", :invalid => :replace, :replace => "")
       decoded.valid_encoding? ? decoded : decoded.encode("utf-16le", :invalid => :replace, :replace => "").encode("utf-8")
@@ -96,23 +99,60 @@ module Mail
       @uri_parser ||= URI::Parser.new
     end
 
-     # mails somtimes includes invalid encodings like iso885915 or utf8 so we transform them to iso885915 or utf8
-     # TODO: add this as a test somewhere
-     # Encoding.list.map{|e| [e.to_s.upcase==fix_encoding(e.to_s.downcase.gsub("-", "")), e.to_s] }.select {|a,b| !b}
-     #  Encoding.list.map{|e| [e.to_s==fix_encoding(e.to_s), e.to_s] }.select {|a,b| !b}
-    def Ruby19.fix_encoding(encoding)
-      case encoding
-        # ISO-8859-15, ISO-2022-JP and alike
-        when /iso-?(\d{4})-?(\w{1,2})/i then return "ISO-#{$1}-#{$2}"
-        # "ISO-2022-JP-KDDI"  and alike
-        when /iso-?(\d{4})-?(\w{1,2})-?(\w*)/i then return "ISO-#{$1}-#{$2}-#{$3}"
-        # UTF-8, UTF-32BE and alike
-        when /utf-?(\d{1,2})?(\w{1,2})/i then return "UTF-#{$1}#{$2}".gsub(/\A(UTF-(?:16|32))\z/, '\\1BE')
-        # Windows-1252 and alike
-        when /Windows-?(.*)/i then return "Windows-#{$1}"
-        when /^8bit$/ then return "ASCII-8BIT"
-        #more aliases to be added if needed
-        else return encoding
+    # Pick a Ruby encoding corresponding to the message charset. Most
+    # charsets have a Ruby encoding, but some need manual aliasing here.
+    #
+    # TODO: add this as a test somewhere:
+    #   Encoding.list.map { |e| [e.to_s.upcase == pick_encoding(e.to_s.downcase.gsub("-", "")), e.to_s] }.select {|a,b| !b}
+    #   Encoding.list.map { |e| [e.to_s == pick_encoding(e.to_s), e.to_s] }.select {|a,b| !b}
+    def Ruby19.pick_encoding(charset)
+      case charset
+
+      # ISO-8859-8-I etc. http://en.wikipedia.org/wiki/ISO-8859-8-I
+      when /^iso-?8859-(\d+)(-i)?$/i
+        "ISO-8859-#{$1}"
+
+      # ISO-8859-15, ISO-2022-JP and alike
+      when /iso-?(\d{4})-?(\w{1,2})/i
+        "ISO-#{$1}-#{$2}"
+
+      # "ISO-2022-JP-KDDI"  and alike
+      when /iso-?(\d{4})-?(\w{1,2})-?(\w*)/i
+        "ISO-#{$1}-#{$2}-#{$3}"
+
+      # UTF-8, UTF-32BE and alike
+      when /utf[\-_]?(\d{1,2})?(\w{1,2})/i
+        "UTF-#{$1}#{$2}".gsub(/\A(UTF-(?:16|32))\z/, '\\1BE')
+
+      # Windows-1252 and alike
+      when /Windows-?(.*)/i
+        "Windows-#{$1}"
+
+      when /^8bit$/
+        Encoding::ASCII_8BIT
+
+      # alternatives/misspellings of us-ascii seen in the wild
+      when /^iso-?646(-us)?$/i, /us=ascii/i
+        Encoding::ASCII
+
+      # Microsoft-specific alias for MACROMAN
+      when /^macintosh$/i
+        Encoding::MACROMAN
+
+      # Microsoft-specific alias for CP949 (Korean)
+      when 'ks_c_5601-1987'
+        Encoding::CP949
+
+      # Wrongly written Shift_JIS (Japanese)
+      when 'shift-jis'
+        Encoding::Shift_JIS
+
+      # GB2312 (Chinese charset) is a subset of GB18030 (its replacement)
+      when /gb2312/i
+        Encoding::GB18030
+
+      else
+        charset
       end
     end
   end

data/lib/mail.rb

@@ -29,7 +29,6 @@ module Mail # :doc:
   require 'mail/core_extensions/nil'
   require 'mail/core_extensions/object'
   require 'mail/core_extensions/string'
-  require 'mail/core_extensions/shell_escape'
   require 'mail/core_extensions/smtp' if RUBY_VERSION < '1.9.3'
   require 'mail/indifferent_hash'
 
@@ -77,7 +76,7 @@ module Mail # :doc:
 
   require 'mail/envelope'
 
-  require 'load_parsers'
+  require 'mail/parsers'
 
   # Autoload header field elements and transfer encodings.
   require 'mail/elements'