mail 2.1.5.3 → 2.2.0

data/CHANGELOG.rdoc

@@ -1,3 +1,19 @@
+== Sun Apr 11 07:49:15 UTC 2010 Mikel Lindsaar <raasdnil@gmail.com>
+
+* Lots of updates on encoding and decoding of headers and unstructured fields.  This
+  is now a lot cleaner and nicer.  Also more predictable.
+* Merged encoding branch back into head
+* Version bump to 2.2.0
+* Tagged 2.2.0
+
+== Sun Apr  4 06:41:46 UTC 2010 Mikel Lindsaar <raasdnil@gmail.com>
+
+* Created non-ascii header auto encoding for address fields and unstructured fields
+* Changed default behaviour of mail, if you specify a charset, it will use that charset
+  regardless of what is in the body.  Previously, if the body was all US-ASCII, it would
+  set the charset to US-ASCII in preference.
+* Many internal version jumps from 2.1.5.3 => 2.1.5.8 - unreleased development versions
+
 == Mon 29 Mar 2010 07:04:34 UTC Mikel Lindsaar <raasdnil@gmail.com>
 
 * Version bump to 2.1.5.3

data/Rakefile

@@ -8,7 +8,7 @@ require 'cucumber/rake/task'
 
 spec = Gem::Specification.new do |s|
   s.name        = "mail"
-  s.version     = "2.1.5.3"
+  s.version     = "2.2.0"
   s.author      = "Mike Lindsaar"
   s.email       = "raasdnil@gmail.com"
   s.homepage    = "http://github.com/mikel/mail"

data/lib/mail.rb

@@ -6,6 +6,7 @@ module Mail # :doc:
   require 'active_support'
   require 'active_support/core_ext/hash/indifferent_access'
   require 'active_support/core_ext/object/blank'
+  require 'active_support/core_ext/string'
 
   require 'uri'
   require 'net/smtp'
@@ -54,6 +55,7 @@ module Mail # :doc:
                 date_time received message_ids envelope_from rfc2045 
                 mime_version content_type content_disposition
                 content_transfer_encoding content_location ]
+
   parsers.each do |parser|
     begin
       # Try requiring the pre-compiled ruby version first
@@ -63,7 +65,7 @@ module Mail # :doc:
       # Otherwise, get treetop to compile and load it
       require 'treetop/runtime'
       require 'treetop/compiler'
-      Treetop.load("mail/parsers/#{parser}")
+      Treetop.load(File.join(File.dirname(__FILE__)) + "/mail/parsers/#{parser}")
     end
   end
 

data/lib/mail/body.rb

@@ -30,6 +30,7 @@ module Mail
       @boundary = nil
       @preamble = nil
       @epilogue = nil
+      @charset  = nil
       @part_sort_order = [ "text/plain", "text/enriched", "text/html" ]
       @parts = Mail::PartsList.new
       if string.blank?
@@ -125,8 +126,8 @@ module Mail
     end
    
     def get_best_encoding(target)
-        te = Mail::Encodings.get_encoding(target)
-        te.get_best_compatible(encoding, raw_source)
+      target_encoding = Mail::Encodings.get_encoding(target)
+      target_encoding.get_best_compatible(encoding, raw_source)
     end
  
     # Returns a body encoded using transfer_encoding.  Multipart always uses an

data/lib/mail/core_extensions/string.rb

@@ -16,6 +16,10 @@ class String #:nodoc:
       !(self =~ /[^#{US_ASCII_REGEXP}]/)
     end
   end
+  
+  def not_ascii_only?
+    !ascii_only?
+  end
 
   unless method_defined?(:bytesize)
     alias :bytesize :length

data/lib/mail/elements/address.rb

@@ -84,7 +84,7 @@ module Mail
     def display_name
       parse unless @parsed
       @display_name ||= get_display_name
-      Encodings.decode_encode(@display_name, @output_type) if @display_name
+      Encodings.decode_encode(@display_name.to_s, @output_type) if @display_name
     end
     
     # Provides a way to assign a display name to an already made Mail::Address object.
@@ -217,7 +217,7 @@ module Mail
     
     def get_comments
       if tree.respond_to?(:comments)
-        @comments ||= tree.comments.map { |c| unparen(c.text_value) } 
+        @comments = tree.comments.map { |c| unparen(c.text_value.to_str) } 
       else
         @comments = []
       end
@@ -226,7 +226,7 @@ module Mail
     def get_display_name
       if tree.respond_to?(:display_name)
         name = unquote(tree.display_name.text_value.strip)
-        str = strip_all_comments(name)
+        str = strip_all_comments(name.to_s)
       elsif comments
         if domain
           str = strip_domain_comments(format_comments)

data/lib/mail/elements/content_transfer_encoding_element.rb

@@ -9,7 +9,7 @@ module Mail
       case
       when string.blank?
         @encoding = ''
-      when tree = parser.parse(string.downcase)
+      when tree = parser.parse(string.to_s.downcase)
         @encoding = tree.encoding.text_value
       else
         raise Mail::Field::ParseError, "ContentTransferEncodingElement can not parse |#{string}|\nReason was: #{parser.failure_reason}\n"

data/lib/mail/encodings.rb

@@ -8,6 +8,7 @@ module Mail
   module Encodings
     
     include Mail::Patterns
+    extend  Mail::Utilities
 
     @transfer_encodings = {}
    
@@ -106,21 +107,25 @@ module Mail
         end
       end
     end
-    
+
     # Decodes a given string as Base64 or Quoted Printable, depending on what
     # type it is.
     # 
     # String has to be of the format =?<encoding>?[QB]?<string>?=
     def Encodings.value_decode(str)
-      str.gsub!(/\?=(\s*)=\?/, '?==?') # Remove whitespaces between 'encoded-word's
-      str.gsub(/(.*?)(=\?.*?\?.\?.*?\?=)|$/m) do
+      str = str.gsub(/\?=(\s*)=\?/, '?==?') # Remove whitespaces between 'encoded-word's
+
+      # Join QP encoded-words that are adjacent to avoid decoding partial chars
+      str.gsub!( /=\?==\?.+?\?[Qq]\?/m, '' ) if str =~ /\?==\?/
+      
+      str.gsub(/(.*?)(=\?.*?\?.\?.*?\?=)|$/m) do # Grab the insides of each encoded-word
         before = $1.to_s
         text = $2.to_s
-        
+
         case
-        when text =~ /=\?.+\?[Bb]\?/m
+        when text.to_str =~ /=\?.+\?[Bb]\?/m
           before + b_value_decode(text)
-        when text =~ /=\?.+\?[Qq]\?/m
+        when text.to_str =~ /=\?.+\?[Qq]\?/m
           before + q_value_decode(text)
         else
           before + text
@@ -154,6 +159,36 @@ module Mail
       end
     end
     
+    def Encodings.address_encode(address, charset = 'utf-8')
+      if address.is_a?(Array)
+        # loop back through for each element
+        address.map { |a| Encodings.address_encode(a, charset) }.join(", ")
+      else
+        # find any word boundary that is not ascii and encode it
+        encode_non_usascii(address, charset)
+      end
+    end
+
+    def Encodings.encode_non_usascii(address, charset)
+      return address if address.ascii_only?
+      us_ascii = %Q{\x00-\x7f}
+      # Encode any non usascii strings embedded inside of quotes
+      address.gsub!(/(".*?[^#{us_ascii}].+?")/) { |s| Encodings.b_value_encode(unquote(s), charset) }
+      # Then loop through all remaining items and encode as needed
+      tokens = address.split(/\s/)
+      tokens.each_with_index.map do |word, i|
+        if word.ascii_only?
+          word
+        else
+          previous_non_ascii = tokens[i-1] && !tokens[i-1].ascii_only?
+          if previous_non_ascii
+            word = " #{word}"
+          end
+          Encodings.b_value_encode(word, charset)
+        end
+      end.join(' ')
+    end
+    
     # Encode a string with Base64 Encoding and returns it ready to be inserted
     # as a value for a field, that is, in the =?<charset>?B?<string>?= format
     #
@@ -162,8 +197,9 @@ module Mail
     #  Encodings.b_value_encode('This is あ string', 'UTF-8') 
     #  #=> "=?UTF-8?B?VGhpcyBpcyDjgYIgc3RyaW5n?="
     def Encodings.b_value_encode(str, encoding = nil)
+      return str if str.to_s.ascii_only?
       string, encoding = RubyVer.b_value_encode(str, encoding)
-      string.split("\n").map do |str|
+      string.each_line.map do |str|
         "=?#{encoding}?B?#{str.chomp}?="
       end.join(" ")
     end
@@ -176,8 +212,12 @@ module Mail
     #  Encodings.q_value_encode('This is あ string', 'UTF-8') 
     #  #=> "=?UTF-8?Q?This_is_=E3=81=82_string?="
     def Encodings.q_value_encode(str, encoding = nil)
+      return str if str.to_s.ascii_only?
       string, encoding = RubyVer.q_value_encode(str, encoding)
-      "=?#{encoding}?Q?#{string.chomp.gsub(/ /, '_')}?="
+      string.gsub!("=\r\n=", '=') # We already have limited the string to the length we want
+      string.each_line.map do |str|
+        "=?#{encoding}?Q?#{str.chomp.gsub(/ /, '_')}?="
+      end.join(" ")
     end
     
     private

data/lib/mail/encodings/quoted_printable.rb

@@ -18,9 +18,7 @@ module Mail
       end
 
       def self.encode(str)
-        l = []
-        str.each_line{|line| l << qp_encode_line(line)}
-        l.join("\r\n")
+        [str].pack("M").gsub(/\n/, "\r\n")
       end
 
       def self.cost(str)
@@ -33,17 +31,6 @@ module Mail
       end
         
       private
-      def self.qp_encode_line(str)
-        str.chomp.gsub( /[^a-z ]/i ) { quoted_printable_encode($&) }
-      end
-
-      # Convert the given character to quoted printable format, taking into
-      # account multi-byte characters (if executing with $KCODE="u", for instance)
-      def self.quoted_printable_encode(character)
-        result = ""
-        character.each_byte { |b| result << "=%02X" % b }
-        result
-      end
 
       Encodings.register(NAME, self)
     end

data/lib/mail/field.rb

@@ -67,15 +67,15 @@ module Mail
     # parameters:
     # 
     #  Field.new('content-type', ['text', 'plain', {:charset => 'UTF-8'}])
-    def initialize(name, value = nil)
+    def initialize(name, value = nil, charset = 'utf-8')
       case
       when name =~ /:/ && value.blank?   # Field.new("field-name: field data")
         name, value = split(name)
-        create_field(name, value)
+        create_field(name, value, charset)
       when name !~ /:/ && value.blank?  # Field.new("field-name")
-        create_field(name, nil)
+        create_field(name, nil, charset)
       else                              # Field.new("field-name", "value")
-        create_field(name, value)
+        create_field(name, value, charset)
       end
       return self
     end
@@ -96,8 +96,8 @@ module Mail
       field.value
     end
     
-    def value=(str)
-      create_field(name, str)
+    def value=(val)
+      create_field(name, val, charset)
     end
     
     def to_s
@@ -105,7 +105,7 @@ module Mail
     end
     
     def update(name, value)
-      create_field(name, value)
+      create_field(name, value, charset)
     end
     
     def same( other )
@@ -134,86 +134,86 @@ module Mail
     private
     
     def split(raw_field)
-      match_data = raw_field.match(/^(#{FIELD_NAME})\s*:\s*(#{FIELD_BODY})?$/)
-      [match_data[1].to_s.strip, match_data[2].to_s.strip]
+      match_data = raw_field.mb_chars.match(/^(#{FIELD_NAME})\s*:\s*(#{FIELD_BODY})?$/)
+      [match_data[1].to_s.mb_chars.strip, match_data[2].to_s.mb_chars.strip]
     rescue
       STDERR.puts "WARNING: Could not parse (and so ignorning) '#{raw_field}'"
     end
-
-    def create_field(name, value)
+    
+    def create_field(name, value, charset)
       begin
-        self.field = new_field(name, value)
-      rescue => e
+        self.field = new_field(name, value, charset)
+      rescue Mail::Field::ParseError => e
         self.field = Mail::UnstructuredField.new(name, value)
         self.field.errors << [name, value, e]
         self.field
       end
     end
 
-    def new_field(name, value)
+    def new_field(name, value, charset)
       # Could do this with constantize and make it "as DRY as", but a simple case 
       # statement is, well, simpler... 
       case name.to_s.downcase
       when /^to$/i
-        ToField.new(name, value)
+        ToField.new(value, charset)
       when /^cc$/i
-        CcField.new(name, value)
+        CcField.new(value, charset)
       when /^bcc$/i
-        BccField.new(name, value)
+        BccField.new(value, charset)
       when /^message-id$/i
-        MessageIdField.new(name, value)
+        MessageIdField.new(value, charset)
       when /^in-reply-to$/i
-        InReplyToField.new(name, value)
+        InReplyToField.new(value, charset)
       when /^references$/i
-        ReferencesField.new(name, value)
+        ReferencesField.new(value, charset)
       when /^subject$/i
-        SubjectField.new(name, value)
+        SubjectField.new(value, charset)
       when /^comments$/i
-        CommentsField.new(name, value)
+        CommentsField.new(value, charset)
       when /^keywords$/i
-        KeywordsField.new(name, value)
+        KeywordsField.new(value, charset)
       when /^date$/i
-        DateField.new(name, value)
+        DateField.new(value, charset)
       when /^from$/i
-        FromField.new(name, value)
+        FromField.new(value, charset)
       when /^sender$/i
-        SenderField.new(name, value)
+        SenderField.new(value, charset)
       when /^reply-to$/i
-        ReplyToField.new(name, value)
+        ReplyToField.new(value, charset)
       when /^resent-date$/i
-        ResentDateField.new(name, value)
+        ResentDateField.new(value, charset)
       when /^resent-from$/i
-        ResentFromField.new(name, value)
+        ResentFromField.new(value, charset)
       when /^resent-sender$/i 
-        ResentSenderField.new(name, value)
+        ResentSenderField.new(value, charset)
       when /^resent-to$/i
-        ResentToField.new(name, value)
+        ResentToField.new(value, charset)
       when /^resent-cc$/i
-        ResentCcField.new(name, value)
+        ResentCcField.new(value, charset)
       when /^resent-bcc$/i
-        ResentBccField.new(name, value)
+        ResentBccField.new(value, charset)
       when /^resent-message-id$/i
-        ResentMessageIdField.new(name, value)
+        ResentMessageIdField.new(value, charset)
       when /^return-path$/i
-        ReturnPathField.new(name, value)
+        ReturnPathField.new(value, charset)
       when /^received$/i
-        ReceivedField.new(name, value)
+        ReceivedField.new(value, charset)
       when /^mime-version$/i
-        MimeVersionField.new(name, value)
+        MimeVersionField.new(value, charset)
       when /^content-transfer-encoding$/i
-        ContentTransferEncodingField.new(name, value)
+        ContentTransferEncodingField.new(value, charset)
       when /^content-description$/i
-        ContentDescriptionField.new(name, value)
+        ContentDescriptionField.new(value, charset)
       when /^content-disposition$/i
-        ContentDispositionField.new(name, value)
+        ContentDispositionField.new(value, charset)
       when /^content-type$/i
-        ContentTypeField.new(name, value)
+        ContentTypeField.new(value, charset)
       when /^content-id$/i
-        ContentIdField.new(name, value)
+        ContentIdField.new(value, charset)
       when /^content-location$/i
-        ContentLocationField.new(name, value)
+        ContentLocationField.new(value, charset)
       else 
-        OptionalField.new(name, value)
+        OptionalField.new(name, value, charset)
       end
       
     end

data/lib/mail/fields/bcc_field.rb

@@ -36,8 +36,9 @@ module Mail
     FIELD_NAME = 'bcc'
     CAPITALIZED_FIELD = 'Bcc'
     
-    def initialize(*args)
-      super(CAPITALIZED_FIELD, strip_field(FIELD_NAME, args.last))
+    def initialize(value = '', charset = 'utf-8')
+      @charset = charset
+      super(CAPITALIZED_FIELD, strip_field(FIELD_NAME, value), charset)
       self.parse
       self
     end

data/lib/mail/fields/cc_field.rb

@@ -36,8 +36,9 @@ module Mail
     FIELD_NAME = 'cc'
     CAPITALIZED_FIELD = 'Cc'
     
-    def initialize(*args)
-      super(CAPITALIZED_FIELD, strip_field(FIELD_NAME, args.last))
+    def initialize(value = nil, charset = 'utf-8')
+      self.charset = charset
+      super(CAPITALIZED_FIELD, strip_field(FIELD_NAME, value), charset)
       self.parse
       self
     end

data/lib/mail/fields/comments_field.rb

@@ -30,11 +30,11 @@ module Mail
     FIELD_NAME = 'comments'
     CAPITALIZED_FIELD = 'Comments'
     
-    def initialize(*args)
-      super(CAPITALIZED_FIELD, strip_field(FIELD_NAME, args.last))
+    def initialize(value = nil, charset = 'utf-8')
+      @charset = charset
+      super(CAPITALIZED_FIELD, strip_field(FIELD_NAME, value))
       self.parse
       self
-
     end
     
   end