mail 2.7.0.rc1 → 2.7.0

data/lib/mail/encodings/8bit.rb

@@ -6,32 +6,13 @@ module Mail
   module Encodings
     class EightBit < Binary
       NAME = '8bit'
-
       PRIORITY = 4
-
-      # 8bit is an identiy encoding, meaning nothing to do
-      
-      # Decode the string
-      def self.decode(str)
-        ::Mail::Utilities.to_lf str
-      end
-    
-      # Encode the string
-      def self.encode(str)
-        ::Mail::Utilities.to_crlf str
-      end
-     
-      # Idenity encodings have a fixed cost, 1 byte out per 1 byte in
-      def self.cost(str)
-        1.0
-      end
+      Encodings.register(NAME, self)
 
       # Per RFC 2821 4.5.3.1, SMTP lines may not be longer than 1000 octets including the <CRLF>.
       def self.compatible_input?(str)
-        !str.lines.find { |line| line.length > 998 }
+        !str.lines.find { |line| line.bytesize > 998 }
       end
-
-      Encodings.register(NAME, self) 
     end
   end
 end

data/lib/mail/encodings/base64.rb

@@ -4,36 +4,35 @@ require 'mail/encodings/7bit'
 
 module Mail
   module Encodings
+    # Base64 encoding handles binary content at the cost of 4 output bytes
+    # per input byte.
     class Base64 < SevenBit
       NAME = 'base64'
-     
       PRIORITY = 3
- 
+      Encodings.register(NAME, self)
+
       def self.can_encode?(enc)
         true
       end
 
-      # Decode the string from Base64
       def self.decode(str)
-        RubyVer.decode_base64( str )
+        RubyVer.decode_base64(str)
       end
-    
-      # Encode the string to Base64
+
       def self.encode(str)
-        ::Mail::Utilities.to_crlf(RubyVer.encode_base64( str ))
+        ::Mail::Utilities.binary_unsafe_to_crlf(RubyVer.encode_base64(str))
       end
 
-      # Base64 has a fixed cost, 4 bytes out per 3 bytes in
+      # 3 bytes in -> 4 bytes out
       def self.cost(str)
-        4.0/3
+        4.0 / 3
       end
 
-      # Base64 inserts newlines automatically and cannot violate the SMTP spec.
+      # Ruby Base64 inserts newlines automatically, so it doesn't exceed
+      # SMTP line length limits.
       def self.compatible_input?(str)
         true
       end
-
-      Encodings.register(NAME, self)      
     end
   end
 end

data/lib/mail/encodings/binary.rb

@@ -1,32 +1,13 @@
 # encoding: utf-8
 # frozen_string_literal: true
-require 'mail/encodings/transfer_encoding'
+require 'mail/encodings/identity'
 
 module Mail
   module Encodings
-    class Binary < TransferEncoding
+    class Binary < Identity
       NAME = 'binary'
-
       PRIORITY = 5
-
-      # Binary is an identiy encoding, meaning nothing to do
-      
-      # Decode the string
-      def self.decode(str)
-        str
-      end
-    
-      # Encode the string
-      def self.encode(str)
-        str
-      end
-     
-      # Idenity encodings have a fixed cost, 1 byte out per 1 byte in
-      def self.cost(str)
-        1.0
-      end
-
-      Encodings.register(NAME, self) 
+      Encodings.register(NAME, self)
     end
   end
 end

data/lib/mail/encodings/identity.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+# frozen_string_literal: true
+require 'mail/encodings/transfer_encoding'
+
+module Mail
+  module Encodings
+    # Identity encodings do no encoding/decoding and have a fixed cost:
+    # 1 byte in -> 1 byte out.
+    class Identity < TransferEncoding #:nodoc:
+      def self.decode(str)
+        str
+      end
+
+      def self.encode(str)
+        str
+      end
+
+      # 1 output byte per input byte.
+      def self.cost(str)
+        1.0
+      end
+    end
+  end
+end

data/lib/mail/encodings/quoted_printable.rb

@@ -6,27 +6,27 @@ module Mail
   module Encodings
     class QuotedPrintable < SevenBit
       NAME='quoted-printable'
-   
+
       PRIORITY = 2
 
-      def self.can_encode?(str)
-        EightBit.can_encode? str
+      def self.can_encode?(enc)
+        EightBit.can_encode? enc
       end
 
       # Decode the string from Quoted-Printable. Cope with hard line breaks
       # that were incorrectly encoded as hex instead of literal CRLF.
       def self.decode(str)
-        ::Mail::Utilities.to_lf str.gsub(/(?:=0D=0A|=0D|=0A)\r\n/, "\r\n").unpack("M*").first
+        str.gsub(/(?:=0D=0A|=0D|=0A)\r\n/, "\r\n").unpack("M*").first
       end
 
       def self.encode(str)
-        ::Mail::Utilities.to_crlf([::Mail::Utilities.to_lf(str)].pack("M"))
+        [str].pack("M")
       end
 
       def self.cost(str)
         # These bytes probably do not need encoding
         c = str.count("\x9\xA\xD\x20-\x3C\x3E-\x7E")
-        # Everything else turns into =XX where XX is a 
+        # Everything else turns into =XX where XX is a
         # two digit hex number (taking 3 bytes)
         total = (str.bytesize - c)*3 + c
         total.to_f/str.bytesize

data/lib/mail/encodings/transfer_encoding.rb

@@ -7,17 +7,17 @@ module Mail
 
       PRIORITY = -1
 
+      # And encoding's superclass can always transport it since the
+      # class hierarchy is arranged e.g. Base64 < 7bit < 8bit < Binary.
       def self.can_transport?(enc)
-        enc = Encodings.get_name(enc)
-        if Encodings.defined? enc
-          Encodings.get_encoding(enc).new.is_a? self
-        else
-          false
-        end
+        enc && enc <= self
       end
 
+      # Override in subclasses to indicate that they can encode text
+      # that couldn't be directly transported, e.g. Base64 has 7bit output,
+      # but it can encode binary.
       def self.can_encode?(enc)
-        can_transport? enc 
+        can_transport? enc
       end
 
       def self.cost(str)
@@ -32,36 +32,45 @@ module Mail
         self::NAME
       end
 
-      def self.get_best_compatible(source_encoding, str)
-        if self.can_transport?(source_encoding) && self.compatible_input?(str)
+      def self.negotiate(message_encoding, source_encoding, str, allowed_encodings = nil)
+        message_encoding = Encodings.get_encoding(message_encoding || '8bit')
+        source_encoding  = Encodings.get_encoding(source_encoding)
+
+        if message_encoding && source_encoding && message_encoding.can_transport?(source_encoding) && source_encoding.compatible_input?(str)
           source_encoding
         else
-          choices = Encodings.get_all.select do |enc|
-            self.can_transport?(enc) && enc.can_encode?(source_encoding)
-          end
+          renegotiate(message_encoding, source_encoding, str, allowed_encodings)
+        end
+      end
 
-          best = nil
-          best_cost = nil
+      def self.renegotiate(message_encoding, source_encoding, str, allowed_encodings = nil)
+        encodings = Encodings.get_all.select do |enc|
+          (allowed_encodings.nil? || allowed_encodings.include?(enc)) &&
+            message_encoding.can_transport?(enc) &&
+            enc.can_encode?(source_encoding)
+        end
 
-          choices.each do |enc|
-            # If the current choice cannot be transported safely,
-            # give priority to other choices but allow it to be used as a fallback.
-            this_cost = enc.cost(str) if enc.compatible_input?(str)
+        lowest_cost(str, encodings)
+      end
 
-            if !best_cost || (this_cost && this_cost < best_cost)
-              best_cost = this_cost
-              best = enc
-            elsif this_cost == best_cost
-              best = enc if enc::PRIORITY < best::PRIORITY
-            end
-          end
+      def self.lowest_cost(str, encodings)
+        best = nil
+        best_cost = nil
 
-          best
+        encodings.each do |enc|
+          # If the current choice cannot be transported safely, give priority
+          # to other choices but allow it to be used as a fallback.
+          this_cost = enc.cost(str) if enc.compatible_input?(str)
+
+          if !best_cost || (this_cost && this_cost < best_cost)
+            best_cost = this_cost
+            best = enc
+          elsif this_cost == best_cost
+            best = enc if enc::PRIORITY < best::PRIORITY
+          end
         end
-      end
 
-      def to_s
-        self.class.to_s
+        best
       end
     end
   end

data/lib/mail/encodings/unix_to_unix.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 module Mail
   module Encodings
-    module UnixToUnix
+    class UnixToUnix < TransferEncoding
       NAME = "x-uuencode"
 
       def self.decode(str)
@@ -13,6 +13,7 @@ module Mail
       end
 
       Encodings.register(NAME, self)
+      Encodings.register("uuencode", self)
     end
   end
 end

data/lib/mail/encodings.rb

@@ -7,7 +7,6 @@ module Mail
   end
 
   module Encodings
-
     include Mail::Constants
     extend  Mail::Utilities
 
@@ -19,7 +18,7 @@ module Mail
     #
     # Encodings.register "base64", Mail::Encodings::Base64
     def Encodings.register(name, cls)
-        @transfer_encodings[get_name(name)] = cls
+      @transfer_encodings[get_name(name)] = cls
     end
 
     # Is the encoding we want defined?
@@ -27,8 +26,8 @@ module Mail
     # Example:
     #
     #  Encodings.defined?(:base64) #=> true
-    def Encodings.defined?( str )
-      @transfer_encodings.include? get_name(str)
+    def Encodings.defined?(name)
+      @transfer_encodings.include? get_name(name)
     end
 
     # Gets a defined encoding type, QuotedPrintable or Base64 for now.
@@ -39,16 +38,16 @@ module Mail
     # Example:
     #
     #  Encodings.get_encoding(:base64) #=> Mail::Encodings::Base64
-    def Encodings.get_encoding( str )
-      @transfer_encodings[get_name(str)]
+    def Encodings.get_encoding(name)
+      @transfer_encodings[get_name(name)]
     end
 
     def Encodings.get_all
       @transfer_encodings.values
     end
 
-    def Encodings.get_name(enc)
-      underscoreize(enc).downcase
+    def Encodings.get_name(name)
+      underscoreize(name).downcase
     end
 
     def Encodings.transcode_charset(str, from_charset, to_charset = 'UTF-8')
@@ -168,31 +167,51 @@ module Mail
 
     def Encodings.address_encode(address, charset = 'utf-8')
       if address.is_a?(Array)
-        # loop back through for each element
         address.compact.map { |a| Encodings.address_encode(a, charset) }.join(", ")
-      else
-        # find any word boundary that is not ascii and encode it
-        encode_non_usascii(address, charset) if address
+      elsif address
+        encode_non_usascii(address, charset)
       end
     end
 
     def Encodings.encode_non_usascii(address, charset)
       return address if address.ascii_only? or charset.nil?
-      # Encode all strings embedded inside of quotes
-      address = address.gsub(/("[^"]*")/) { |s| Encodings.b_value_encode(unquote(s), charset) }
-      # Then loop through all remaining items and encode as needed
-      tokens = address.split(/\s/)
-      map_with_index(tokens) do |word, i|
-        if word.ascii_only?
-          word
-        else
-          previous_non_ascii = i>0 && tokens[i-1] && !tokens[i-1].ascii_only?
-          if previous_non_ascii #why are we adding an extra space here?
-            word = " #{word}"
+
+      # With KCODE=u we can't use regexps on other encodings. Go ASCII.
+      with_ascii_kcode do
+        # Encode all strings embedded inside of quotes
+        address = address.gsub(/("[^"]*[^\/]")/) { |s| Encodings.b_value_encode(unquote(s), charset) }
+
+        # Then loop through all remaining items and encode as needed
+        tokens = address.split(/\s/)
+
+        map_with_index(tokens) do |word, i|
+          if word.ascii_only?
+            word
+          else
+            previous_non_ascii = i>0 && tokens[i-1] && !tokens[i-1].ascii_only?
+            if previous_non_ascii #why are we adding an extra space here?
+              word = " #{word}"
+            end
+            Encodings.b_value_encode(word, charset)
           end
-          Encodings.b_value_encode(word, charset)
+        end.join(' ')
+      end
+    end
+
+    if RUBY_VERSION < '1.9'
+      # With KCODE=u we can't use regexps on other encodings. Go ASCII.
+      def Encodings.with_ascii_kcode #:nodoc:
+        if $KCODE
+          $KCODE, original_kcode = '', $KCODE
         end
-      end.join(' ')
+        yield
+      ensure
+        $KCODE = original_kcode if original_kcode
+      end
+    else
+      def Encodings.with_ascii_kcode #:nodoc:
+        yield
+      end
     end
 
     # Encode a string with Base64 Encoding and returns it ready to be inserted
@@ -202,12 +221,15 @@ module Mail
     #
     #  Encodings.b_value_encode('This is あ string', 'UTF-8')
     #  #=> "=?UTF-8?B?VGhpcyBpcyDjgYIgc3RyaW5n?="
-    def Encodings.b_value_encode(encoded_str, encoding = nil)
-      return encoded_str if encoded_str.to_s.ascii_only?
-      string, encoding = RubyVer.b_value_encode(encoded_str, encoding)
-      map_lines(string) do |str|
-        "=?#{encoding}?B?#{str.chomp}?="
-      end.join(" ")
+    def Encodings.b_value_encode(string, encoding = nil)
+      if string.to_s.ascii_only?
+        string
+      else
+        Encodings.each_base64_chunk_byterange(string, 60).map do |chunk|
+          str, encoding = RubyVer.b_value_encode(chunk, encoding)
+          "=?#{encoding}?B?#{str.chomp}?="
+        end.join(" ")
+      end
     end
 
     # Encode a string with Quoted-Printable Encoding and returns it ready to be inserted
@@ -281,5 +303,41 @@ module Mail
 
       results
     end
+
+    # Partition the string into bounded-size chunks without splitting
+    # multibyte characters.
+    def Encodings.each_base64_chunk_byterange(str, max_bytesize_per_base64_chunk, &block)
+      raise "size per chunk must be multiple of 4" if (max_bytesize_per_base64_chunk % 4).nonzero?
+
+      if block_given?
+        max_bytesize = ((3 * max_bytesize_per_base64_chunk) / 4.0).floor
+        each_chunk_byterange(str, max_bytesize, &block)
+      else
+        enum_for :each_base64_chunk_byterange, str, max_bytesize_per_base64_chunk
+      end
+    end
+
+    # Partition the string into bounded-size chunks without splitting
+    # multibyte characters.
+    def Encodings.each_chunk_byterange(str, max_bytesize_per_chunk)
+      return enum_for(:each_chunk_byterange, str, max_bytesize_per_chunk) unless block_given?
+
+      offset = 0
+      chunksize = 0
+
+      str.each_char do |chr|
+        charsize = chr.bytesize
+
+        if chunksize + charsize > max_bytesize_per_chunk
+          yield RubyVer.string_byteslice(str, offset, chunksize)
+          offset += chunksize
+          chunksize = charsize
+        else
+          chunksize += charsize
+        end
+      end
+
+      yield RubyVer.string_byteslice(str, offset, chunksize)
+    end
   end
 end

data/lib/mail/envelope.rb

@@ -12,7 +12,7 @@ module Mail
   class Envelope < StructuredField
     
     def initialize(*args)
-      super(FIELD_NAME, strip_field(FIELD_NAME, args.last))
+      super(FIELD_NAME, args.last.to_s)
     end
     
     def element

data/lib/mail/field.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require 'mail/fields'
+require 'mail/constants'
 
 # encoding: utf-8
 module Mail
@@ -83,9 +84,31 @@ module Mail
 
       def initialize(element, value, reason)
         @element = element
-        @value = value
-        @reason = reason
-        super("#{element} can not parse |#{value}|\nReason was: #{reason}")
+        @value = to_utf8(value)
+        @reason = to_utf8(reason)
+        super("#{@element} can not parse |#{@value}|: #{@reason}")
+      end
+
+      private
+        def to_utf8(text)
+          if text.respond_to?(:force_encoding)
+            text.dup.force_encoding(Encoding::UTF_8)
+          else
+            text
+          end
+        end
+    end
+
+    class NilParseError < ParseError #:nodoc:
+      def initialize(element)
+        super element, nil, 'nil is invalid'
+      end
+    end
+
+    class IncompleteParseError < ParseError #:nodoc:
+      def initialize(element, original_text, unparsed_index)
+        parsed_text = to_utf8(original_text[0...unparsed_index])
+        super element, original_text, "Only able to parse up to #{parsed_text.inspect}"
       end
     end
 
@@ -93,41 +116,62 @@ module Mail
     class SyntaxError < FieldError #:nodoc:
     end
 
-    # Accepts a string:
-    #
-    #  Field.new("field-name: field data")
-    #
-    # Or name, value pair:
-    #
-    #  Field.new("field-name", "value")
+    class << self
+      # Parse a field from a raw header line:
+      #
+      #  Mail::Field.parse("field-name: field data")
+      #  # => #<Mail::Field …>
+      def parse(field, charset = nil)
+        name, value = split(field)
+        if name && value
+          new name, value, charset
+        end
+      end
+
+      def split(raw_field) #:nodoc:
+        if raw_field.index(Constants::COLON)
+          name, value = raw_field.split(Constants::COLON, 2)
+          name.rstrip!
+          if name =~ /\A#{Constants::FIELD_NAME}\z/
+            [ name.rstrip, value.strip ]
+          else
+            Kernel.warn "WARNING: Ignoring unparsable header #{raw_field.inspect}: invalid header name syntax: #{name.inspect}"
+            nil
+          end
+        else
+          raw_field.strip
+        end
+      rescue => error
+        warn "WARNING: Ignoring unparsable header #{raw_field.inspect}: #{error.class}: #{error.message}"
+        nil
+      end
+    end
+
+    # Create a field by name and optional value:
     #
-    # Or a name by itself:
+    #  Mail::Field.new("field-name", "value")
+    #  # => #<Mail::Field …>
     #
-    #  Field.new("field-name")
+    # Values that aren't strings or arrays are coerced to Strings with `#to_s`.
     #
-    # Note, does not want a terminating carriage return.  Returns
-    # self appropriately parsed.  If value is not a string, then
-    # it will be passed through as is, for example, content-type
-    # field can accept an array with the type and a hash of
-    # parameters:
+    #  Mail::Field.new("field-name", 1234)
+    #  # => #<Mail::Field …>
     #
-    #  Field.new('content-type', ['text', 'plain', {:charset => 'UTF-8'}])
+    #  Mail::Field.new('content-type', ['text', 'plain', {:charset => 'UTF-8'}])
+    #  # => #<Mail::Field …>
     def initialize(name, value = nil, charset = 'utf-8')
       case
-      when name.index(COLON)            # Field.new("field-name: field data")
+      when name.index(COLON)
+        Kernel.warn 'Passing an unparsed header field to Mail::Field.new is deprecated and will be removed in Mail 2.8.0. Use Mail::Field.parse instead.'
+        @name, @unparsed_value = self.class.split(name)
         @charset = Utilities.blank?(value) ? charset : value
-        @name = name[FIELD_PREFIX]
-        @raw_value = name
-        @value = nil
-      when Utilities.blank?(value)   # Field.new("field-name")
+      when Utilities.blank?(value)
         @name = name
-        @value = nil
-        @raw_value = nil
+        @unparsed_value = nil
         @charset = charset
-      else                              # Field.new("field-name", "value")
+      else
         @name = name
-        @value = value
-        @raw_value = nil
+        @unparsed_value = value
         @charset = charset
       end
       @name = FIELD_NAME_MAP[@name.to_s.downcase] || @name
@@ -138,8 +182,7 @@ module Mail
     end
 
     def field
-      _, @value = split(@raw_value) if @raw_value && !@value
-      @field ||= create_field(@name, @value, @charset)
+      @field ||= create_field(@name, @unparsed_value, @charset)
     end
 
     def name
@@ -217,14 +260,26 @@ module Mail
 
     private
 
-    def split(raw_field)
-      match_data = Mail::Multibyte.mb_chars(raw_field).match(FIELD_SPLIT)
-      [
-        Mail::Multibyte.mb_chars(match_data[1].to_s).strip,
-        Mail::Multibyte.mb_chars(match_data[2].to_s).strip.to_s
-      ]
-    rescue
-      $stderr.puts "WARNING: Could not parse (and so ignoring) '#{raw_field}'"
+    def create_field(name, value, charset)
+      new_field(name, value, charset)
+    rescue Mail::Field::ParseError => e
+      field = Mail::UnstructuredField.new(name, value)
+      field.errors << [name, value, e]
+      field
+    end
+
+    def new_field(name, value, charset)
+      value = unfold(value) if value.is_a?(String)
+
+      if klass = field_class_for(name)
+        klass.new(value, charset)
+      else
+        OptionalField.new(name, value, charset)
+      end
+    end
+
+    def field_class_for(name)
+      FIELDS_MAP[name.to_s.downcase]
     end
 
     # 2.2.3. Long Header Fields
@@ -238,28 +293,5 @@ module Mail
     def unfold(string)
       string.gsub(/#{CRLF}(#{WSP})/m, '\1')
     end
-
-    def create_field(name, value, charset)
-      value = unfold(value) if value.is_a?(String)
-
-      begin
-        new_field(name, value, charset)
-      rescue Mail::Field::ParseError => e
-        field = Mail::UnstructuredField.new(name, value)
-        field.errors << [name, value, e]
-        field
-      end
-    end
-
-    def new_field(name, value, charset)
-      lower_case_name = name.to_s.downcase
-      if field_klass = FIELDS_MAP[lower_case_name]
-        field_klass.new(value, charset)
-      else
-        OptionalField.new(name, value, charset)
-      end
-    end
-
   end
-
 end