mail 2.7.0.rc1 → 2.7.0.rc2

data/lib/mail/parsers/received_parser.rl

@@ -1,20 +1,22 @@
 # frozen_string_literal: true
 require 'mail/utilities'
+require 'mail/parser_tools'
 
 %%{
   machine date_time;
+  alphtype int;
 
   # Received Tokens
   action received_tokens_s { received_tokens_s = p }
-  action received_tokens_e { received.info = data[received_tokens_s..(p-1)] }
+  action received_tokens_e { received.info = chars(data, received_tokens_s, p-1) }
 
   # Date
   action date_s { date_s = p }
-  action date_e { received.date = data[date_s..(p-1)].strip }
+  action date_e { received.date = chars(data, date_s, p-1).strip }
 
   # Time
   action time_s { time_s = p }
-  action time_e { received.time = data[time_s..(p-1)] }
+  action time_e { received.time = chars(data, time_s, p-1) }
 
   # No-op actions
   action address_s {}
@@ -49,12 +51,16 @@ require 'mail/utilities'
 
 module Mail::Parsers
   module ReceivedParser
+    extend Mail::ParserTools
+
     ReceivedStruct = Struct.new(:date, :time, :info, :error)
 
     %%write data noprefix;
 
     def self.parse(data)
-      raise Mail::Field::ParseError.new(Mail::ReceivedElement, data, 'nil is invalid') if data.nil?
+      data = data.dup.force_encoding(Encoding::ASCII_8BIT) if data.respond_to?(:force_encoding)
+
+      raise Mail::Field::NilParseError.new(Mail::ReceivedElement) if data.nil?
 
       # Parser state
       received = ReceivedStruct.new
@@ -69,7 +75,7 @@ module Mail::Parsers
       %%write exec;
 
       if p != eof || cs < %%{ write first_final; }%%
-        raise Mail::Field::ParseError.new(Mail::ReceivedElement, data, "Only able to parse up to #{data[0..p]}")
+        raise Mail::Field::IncompleteParseError.new(Mail::ReceivedElement, data, p)
       end
 
       received

data/lib/mail/parsers/rfc2045_content_transfer_encoding.rl

@@ -2,6 +2,7 @@
   # RFC 2045 Section 6.1: Content-Transfer-Encoding Header Field
   # https://tools.ietf.org/html/rfc2045#section-6.1
   machine rfc2045_content_transfer_encoding;
+  alphtype int;
 
   include rfc2045_content_type "rfc2045_content_type.rl";
 

data/lib/mail/parsers/rfc2045_content_type.rl

@@ -3,6 +3,7 @@
   # https://tools.ietf.org/html/rfc2045#section-5.1
   # Previously: https://tools.ietf.org/html/rfc1049#section-3
   machine rfc2045_content_type;
+  alphtype int;
 
   include rfc5322_lexical_tokens "rfc5322_lexical_tokens.rl";
 

data/lib/mail/parsers/rfc2045_mime.rl

@@ -2,6 +2,7 @@
   # RFC 2045 MIME
   # https://tools.ietf.org/html/rfc2045
   machine rfc2045_mime;
+  alphtype int;
 
   include rfc5322_lexical_tokens "rfc5322_lexical_tokens.rl";
 

data/lib/mail/parsers/rfc2183_content_disposition.rl

@@ -4,6 +4,7 @@
   #
   # TODO: recognize filename, size, creation date, etc.
   machine rfc2183_content_disposition;
+  alphtype int;
 
   include rfc2045_content_type "rfc2045_content_type.rl";
 

data/lib/mail/parsers/rfc3629_utf8.rl

@@ -0,0 +1,19 @@
+%%{
+  # RFC 3629 4. Syntax of UTF-8 Byte Sequences
+  # https://tools.ietf.org/html/rfc3629#section-4
+  machine rfc3629_utf8;
+  alphtype int;
+
+  utf8_tail  = 0x80..0xBF;
+
+  utf8_2byte = 0xC2..0xDF utf8_tail;
+  utf8_3byte = 0xE0       0xA0..0xBF  utf8_tail |
+               0xE1..0xEC utf8_tail   utf8_tail |
+               0xED       0x80..0x9F  utf8_tail |
+               0xEE..0xEF utf8_tail   utf8_tail;
+  utf8_4byte = 0xF0       0x90..0xBF  utf8_tail utf8_tail |
+               0xF1..0xF3 utf8_tail   utf8_tail utf8_tail |
+               0xF4       0x80..0x8F  utf8_tail utf8_tail;
+
+  utf8_non_ascii = utf8_2byte | utf8_3byte | utf8_4byte;
+}%%

data/lib/mail/parsers/rfc5234_abnf_core_rules.rl

@@ -2,6 +2,9 @@
   # RFC 5234 B.1. Core Rules
   # https://tools.ietf.org/html/rfc5234#appendix-B.1
   machine rfc5234_abnf_core_rules;
+  alphtype int;
+
+  include rfc3629_utf8 "rfc3629_utf8.rl";
 
   LF = "\n";
   CR = "\r";
@@ -12,5 +15,8 @@
   DQUOTE = '"';
   DIGIT = [0-9];
   ALPHA = [a-zA-Z];
-  VCHAR = 0x21..0x7e;
+
+  # RFC6532 extension for UTF-8 content
+  rfc5234_VCHAR = 0x21..0x7e;
+  VCHAR = rfc5234_VCHAR | utf8_non_ascii;
 }%%

data/lib/mail/parsers/rfc5322.rl

@@ -5,6 +5,7 @@
   # RFC 6854 Update to Internet Message Format to Allow Group Syntax in the "From:" and "Sender:" Header Fields
   # https://tools.ietf.org/html/rfc6854
   machine rfc5322;
+  alphtype int;
 
   include rfc5234_abnf_core_rules "rfc5234_abnf_core_rules.rl";
 
@@ -18,7 +19,8 @@
   include rfc5322_address "rfc5322_address.rl";
 
   # 3.5. Overall Message Syntax
-  #text = 0x01..0x09 | "\v" | "\f" | 0x0e..0x1f;
+  #rfc5322_text = 0x01..0x09 | "\v" | "\f" | 0x0e..0x1f;
+  #text = rfc5322_text | utf8_non_ascii; # RFC6532 for UTF-8
   #obs_body = ((LF* CR* ((0x00 | text) LF* CR*)*) | CRLF)*
   #body = ((text{,998} CRLF)* text{,998}) | obs_body;
   #message = (fields | obs_fields) (CRLF body)?;

data/lib/mail/parsers/rfc5322_address.rl

@@ -3,6 +3,7 @@
   # Section 3.4. Address Specification
   # https://tools.ietf.org/html/rfc5322#section-3.4
   machine rfc5322_address;
+  alphtype int;
 
   include rfc5234_abnf_core_rules "rfc5234_abnf_core_rules.rl";
   include rfc5322_lexical_tokens "rfc5322_lexical_tokens.rl";
@@ -25,7 +26,8 @@
   # domain:
   domain_dot_atom_text = "."* domain_text ("."* domain_text)*;
   obs_dtext = obs_NO_WS_CTL | quoted_pair;
-  dtext = 0x21..0x5a | 0x5e..0x7e | obs_dtext;
+  rfc5322_dtext = 0x21..0x5a | 0x5e..0x7e | obs_dtext;
+  dtext = rfc5322_dtext | utf8_non_ascii; # RFC6532 for UTF-8
   domain_dot_atom = CFWS? domain_dot_atom_text (CFWS? >(comment_after_address,1));
   domain_literal = CFWS? "[" (FWS? dtext)* FWS? "]" CFWS?;
   obs_domain = atom ("." atom)*;

data/lib/mail/parsers/rfc5322_date_time.rl

@@ -3,6 +3,7 @@
   # Section 3.3. Date and Time Specification
   # https://tools.ietf.org/html/rfc5322#section-3.3
   machine rfc5322_date_time;
+  alphtype int;
 
   include rfc5322_lexical_tokens "rfc5322_lexical_tokens.rl";
 

data/lib/mail/parsers/rfc5322_lexical_tokens.rl

@@ -3,6 +3,7 @@
   # Section 3.2. Lexical Tokens
   # https://tools.ietf.org/html/rfc5322#section-3.2
   machine rfc5322_lexical_tokens;
+  alphtype int;
 
   include rfc5234_abnf_core_rules "rfc5234_abnf_core_rules.rl";
 
@@ -16,7 +17,8 @@
   FWS = (WSP* CRLF WSP+) | (CRLF WSP+) | obs_FWS;
 
   obs_ctext = obs_NO_WS_CTL;
-  ctext = 0x21..0x27 | 0x2a..0x5b | 0x5d..0x7e | obs_ctext;
+  rfc5322_ctext = 0x21..0x27 | 0x2a..0x5b | 0x5d..0x7e | obs_ctext;
+  ctext = rfc5322_ctext | utf8_non_ascii; # RFC6532 for UTF-8
 
   # Recursive comments
   action comment_begin { fcall comment_tail; }
@@ -27,16 +29,18 @@
   CFWS = ((FWS? comment)+ FWS?) | FWS;
 
   # 3.2.3. Atom
-  atext = ALPHA | DIGIT | "!" | "#" | "$" | "%" | "&" |
-          "'" | "*" | "+" | "-" | "/" | "=" | "?" | "^" |
-          "_" | "`" | "{" | "|" | "}" | "~";
+  rfc5322_atext = ALPHA | DIGIT | "!" | "#" | "$" | "%" | "&" |
+                  "'" | "*" | "+" | "-" | "/" | "=" | "?" | "^" |
+                  "_" | "`" | "{" | "|" | "}" | "~";
+  atext = rfc5322_atext | utf8_non_ascii; # RFC6532 for UTF-8
   atom = CFWS? atext+ CFWS?;
   dot_atom_text = atext ("." atext)*;
   dot_atom = CFWS? dot_atom_text CFWS?;
 
   # 3.2.4. Quoted Strings
   obs_qtext = obs_NO_WS_CTL;
-  qtext = 0x21 | 0x23..0x5b | 0x5d..0x7e | obs_qtext;
+  rfc5322_qtext = 0x21 | 0x23..0x5b | 0x5d..0x7e | obs_qtext;
+  qtext = rfc5322_qtext | utf8_non_ascii; # RFC6532 for UTF-8
 
   qcontent = qtext | quoted_pair;
   quoted_string = CFWS?

data/lib/mail/part.rb

@@ -21,7 +21,7 @@ module Mail
     
     def inline_content_id
       # TODO: Deprecated in 2.2.2 - Remove in 2.3
-      $stderr.puts("Part#inline_content_id is deprecated, please call Part#cid instead")
+      warn("Part#inline_content_id is deprecated, please call Part#cid instead")
       cid
     end
     

data/lib/mail/utilities.rb

@@ -248,33 +248,62 @@ module Mail
       (LF.encode(:crlf_newline => true) rescue nil) == CRLF
       # Using String#encode is better performing than Regexp
 
-      def self.to_lf(input)
-        input.kind_of?(String) ? input.to_str.encode(input.encoding, :universal_newline => true) : ''
+      def self.binary_unsafe_to_lf(string) #:nodoc:
+        string.encode(string.encoding, :universal_newline => true)
       end
 
-      def self.to_crlf(input)
-        input.kind_of?(String) ? input.to_str.encode(input.encoding, :universal_newline => true).encode!(input.encoding, :crlf_newline => true) : ''
+      def self.binary_unsafe_to_crlf(string) #:nodoc:
+        string.encode(string.encoding, :universal_newline => true).encode!(string.encoding, :crlf_newline => true)
       end
 
+      def self.safe_for_line_ending_conversion?(string) #:nodoc:
+        string.ascii_only?
+      end
     else
+      def self.binary_unsafe_to_lf(string) #:nodoc:
+        string.gsub(/\r\n|\r/, LF)
+      end
+
+      TO_CRLF_REGEX =
+        if RUBY_VERSION >= '1.9'
+          # This 1.9 only regex can save a reasonable amount of time (~20%)
+          # by not matching "\r\n" so the string is returned unchanged in
+          # the common case.
+          Regexp.new("(?<!\r)\n|\r(?!\n)")
+        else
+          /\n|\r\n|\r/
+        end
 
-      def self.to_lf(input)
-        input.kind_of?(String) ? input.to_str.gsub(/\r\n|\r/, LF) : ''
+      def self.binary_unsafe_to_crlf(string) #:nodoc:
+        string.gsub(TO_CRLF_REGEX, CRLF)
       end
 
-      if RUBY_VERSION >= '1.9'
-        # This 1.9 only regex can save a reasonable amount of time (~20%)
-        # by not matching "\r\n" so the string is returned unchanged in
-        # the common case.
-        CRLF_REGEX = Regexp.new("(?<!\r)\n|\r(?!\n)")
-      else
-        CRLF_REGEX = /\n|\r\n|\r/
+      def self.safe_for_line_ending_conversion?(string) #:nodoc:
+        string.ascii_only?
       end
+    end
 
-      def self.to_crlf(input)
-        input.kind_of?(String) ? input.to_str.gsub(CRLF_REGEX, CRLF) : ''
+    # Convert line endings to \n unless the string is binary. Used for
+    # sendmail delivery and for decoding 8bit Content-Transfer-Encoding.
+    def self.to_lf(string)
+      string = string.to_s
+      if safe_for_line_ending_conversion? string
+        binary_unsafe_to_lf string
+      else
+        string
       end
+    end
 
+    # Convert line endings to \r\n unless the string is binary. Used for
+    # encoding 8bit and base64 Content-Transfer-Encoding and for convenience
+    # when parsing emails with \n line endings instead of the required \r\n.
+    def self.to_crlf(string)
+      string = string.to_s
+      if safe_for_line_ending_conversion? string
+        binary_unsafe_to_crlf string
+      else
+        string
+      end
     end
 
     # Returns true if the object is considered blank.

data/lib/mail/version.rb

@@ -5,7 +5,7 @@ module Mail
     MAJOR = 2
     MINOR = 7
     PATCH = 0
-    BUILD = 'rc1'
+    BUILD = 'rc2'
 
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
 

data/lib/mail/version_specific/ruby_1_8.rb

@@ -132,6 +132,10 @@ module Mail
       "#{encoding}'#{language}'#{URI.escape(str)}"
     end
 
+    def Ruby18.string_byteslice(str, *args)
+      str.slice(*args)
+    end
+
     private
 
     def Ruby18.normalize_iconv_charset_encoding(encoding)
@@ -145,7 +149,14 @@ module Mail
       when 'KS_C_5601-1987'
         'CP949'
       else
-        encoding
+        # Fall back to ASCII for charsets that Iconv doesn't recognize
+        begin
+          Iconv.new('UTF-8', encoding)
+        rescue Iconv::InvalidEncoding => e
+          'ASCII'
+        else
+          encoding
+        end
       end
     end
   end

data/lib/mail/version_specific/ruby_1_9.rb

@@ -42,7 +42,7 @@ module Mail
     class << self
       attr_accessor :charset_encoder
     end
-    self.charset_encoder = StrictCharsetEncoder.new
+    self.charset_encoder = BestEffortCharsetEncoder.new
 
     # Escapes any parenthesis in a string that are unescaped this uses
     # a Ruby 1.9.1 regexp feature of negative look behind
@@ -88,6 +88,7 @@ module Mail
     end
 
     def Ruby19.transcode_charset(str, from_encoding, to_encoding = Encoding::UTF_8)
+      to_encoding = to_encoding.to_s if RUBY_VERSION < '1.9.3'
       to_encoding = Encoding.find(to_encoding)
       replacement_char = to_encoding == Encoding::UTF_8 ? '�' : '?'
       charset_encoder.encode(str.dup, from_encoding).encode(to_encoding, :undef => :replace, :invalid => :replace, :replace => replacement_char)
@@ -242,6 +243,16 @@ module Mail
       convert_to_encoding(encoding)
     end
 
+    if "string".respond_to?(:byteslice)
+      def Ruby19.string_byteslice(str, *args)
+        str.byteslice(*args)
+      end
+    else
+      def Ruby19.string_byteslice(str, *args)
+        str.unpack('C*').slice(*args).pack('C*').force_encoding(str.encoding)
+      end
+    end
+
     class << self
       private
 
@@ -249,6 +260,7 @@ module Mail
         if encoding.is_a?(Encoding)
           encoding
         else
+          # Fall back to ASCII for charsets that Ruby doesn't recognize
           begin
             Encoding.find(encoding)
           rescue ArgumentError

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mail
 version: !ruby/object:Gem::Version
-  version: 2.7.0.rc1
+  version: 2.7.0.rc2
 platform: ruby
 authors:
 - Mikel Lindsaar
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-09 00:00:00.000000000 Z
+date: 2017-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mini_mime
@@ -86,18 +86,9 @@ executables: []
 extensions: []
 extra_rdoc_files:
 - README.md
-- CONTRIBUTING.md
-- CHANGELOG.rdoc
-- TODO.rdoc
 files:
-- CHANGELOG.rdoc
-- CONTRIBUTING.md
-- Dependencies.txt
-- Gemfile
 - MIT-LICENSE
 - README.md
-- Rakefile
-- TODO.rdoc
 - lib/mail.rb
 - lib/mail/attachments_list.rb
 - lib/mail/body.rb
@@ -124,6 +115,7 @@ files:
 - lib/mail/encodings/8bit.rb
 - lib/mail/encodings/base64.rb
 - lib/mail/encodings/binary.rb
+- lib/mail/encodings/identity.rb
 - lib/mail/encodings/quoted_printable.rb
 - lib/mail/encodings/transfer_encoding.rb
 - lib/mail/encodings/unix_to_unix.rb
@@ -177,12 +169,12 @@ files:
 - lib/mail/message.rb
 - lib/mail/multibyte.rb
 - lib/mail/multibyte/chars.rb
-- lib/mail/multibyte/exceptions.rb
 - lib/mail/multibyte/unicode.rb
 - lib/mail/multibyte/utils.rb
 - lib/mail/network.rb
 - lib/mail/network/delivery_methods/exim.rb
 - lib/mail/network/delivery_methods/file_delivery.rb
+- lib/mail/network/delivery_methods/logger_delivery.rb
 - lib/mail/network/delivery_methods/sendmail.rb
 - lib/mail/network/delivery_methods/smtp.rb
 - lib/mail/network/delivery_methods/smtp_connection.rb
@@ -191,6 +183,7 @@ files:
 - lib/mail/network/retriever_methods/imap.rb
 - lib/mail/network/retriever_methods/pop3.rb
 - lib/mail/network/retriever_methods/test_retriever.rb
+- lib/mail/parser_tools.rb
 - lib/mail/parsers.rb
 - lib/mail/parsers/address_lists_parser.rb
 - lib/mail/parsers/address_lists_parser.rl
@@ -218,6 +211,7 @@ files:
 - lib/mail/parsers/rfc2045_content_type.rl
 - lib/mail/parsers/rfc2045_mime.rl
 - lib/mail/parsers/rfc2183_content_disposition.rl
+- lib/mail/parsers/rfc3629_utf8.rl
 - lib/mail/parsers/rfc5234_abnf_core_rules.rl
 - lib/mail/parsers/rfc5322.rl
 - lib/mail/parsers/rfc5322_address.rl
@@ -252,7 +246,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Mail provides a nice Ruby DSL for making, sending and reading emails.