mail 2.6.4 → 2.7.0.rc1

data/lib/mail/parsers/received_parser.rl

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+require 'mail/utilities'
+
+%%{
+  machine date_time;
+
+  # Received Tokens
+  action received_tokens_s { received_tokens_s = p }
+  action received_tokens_e { received.info = data[received_tokens_s..(p-1)] }
+
+  # Date
+  action date_s { date_s = p }
+  action date_e { received.date = data[date_s..(p-1)].strip }
+
+  # Time
+  action time_s { time_s = p }
+  action time_e { received.time = data[time_s..(p-1)] }
+
+  # No-op actions
+  action address_s {}
+  action address_e {}
+  action angle_addr_s {}
+  action ctime_date_s {}
+  action ctime_date_e {}
+  action comment_e {}
+  action comment_s {}
+  action phrase_s {}
+  action phrase_e {}
+  action domain_e {}
+  action domain_s {}
+  action local_dot_atom_e {}
+  action local_dot_atom_pre_comment_e {}
+  action local_dot_atom_pre_comment_s {}
+  action local_dot_atom_s {}
+  action qstr_e {}
+  action qstr_s {}
+  action local_quoted_string_s {}
+  action local_quoted_string_e {}
+  action obs_domain_list_s {}
+  action obs_domain_list_e {}
+  action group_name_s {}
+  action group_name_e {}
+  action msg_id_s {}
+  action msg_id_e {}
+
+  include rfc5322 "rfc5322.rl";
+  main := received;
+}%%
+
+module Mail::Parsers
+  module ReceivedParser
+    ReceivedStruct = Struct.new(:date, :time, :info, :error)
+
+    %%write data noprefix;
+
+    def self.parse(data)
+      raise Mail::Field::ParseError.new(Mail::ReceivedElement, data, 'nil is invalid') if data.nil?
+
+      # Parser state
+      received = ReceivedStruct.new
+      received_tokens_s = date_s = time_s = nil
+
+      # 5.1 Variables Used by Ragel
+      p = 0
+      eof = pe = data.length
+      stack = []
+
+      %%write init;
+      %%write exec;
+
+      if p != eof || cs < %%{ write first_final; }%%
+        raise Mail::Field::ParseError.new(Mail::ReceivedElement, data, "Only able to parse up to #{data[0..p]}")
+      end
+
+      received
+    end
+  end
+end

data/lib/mail/parsers/rfc2045_content_transfer_encoding.rl

@@ -0,0 +1,12 @@
+%%{
+  # RFC 2045 Section 6.1: Content-Transfer-Encoding Header Field
+  # https://tools.ietf.org/html/rfc2045#section-6.1
+  machine rfc2045_content_transfer_encoding;
+
+  include rfc2045_content_type "rfc2045_content_type.rl";
+
+  encoding = ('7bits' | '8bits' | '7bit' | '8bit' | 'binary' |
+              'quoted-printable' | 'base64' | ietf_token |
+              custom_x_token) >encoding_s %encoding_e;
+  content_transfer_encoding = CFWS? encoding CFWS? ";"? CFWS?;
+}%%

data/lib/mail/parsers/rfc2045_content_type.rl

@@ -0,0 +1,24 @@
+%%{
+  # RFC 2045 Section 5.1: Content-Type Header Field
+  # https://tools.ietf.org/html/rfc2045#section-5.1
+  # Previously: https://tools.ietf.org/html/rfc1049#section-3
+  machine rfc2045_content_type;
+
+  include rfc5322_lexical_tokens "rfc5322_lexical_tokens.rl";
+
+  token = 0x21..0x27 | 0x2a..0x2b | 0x2c..0x2e | 0x30..0x39 | 0x41..0x5a | 0x5e..0x7e;
+  value = (quoted_string | (token -- '"' | 0x3d)+) >param_val_s %param_val_e;
+  attribute = (token+) >param_attr_s %param_attr_e;
+  parameter = CFWS? attribute "=" value CFWS?;
+
+  ietf_token = token+;
+  custom_x_token = 'x'i "-" token+;
+  extension_token = ietf_token | custom_x_token;
+  discrete_type = 'text'i | 'image'i | 'audio'i | 'video'i |
+                  'application'i | extension_token;
+  composite_type = 'message'i | 'multipart'i | extension_token;
+  iana_token = token+;
+  main_type = (discrete_type | composite_type) >main_type_s %main_type_e;
+  sub_type = (extension_token | iana_token) >sub_type_s %sub_type_e;
+  content_type = main_type "/" sub_type (((CFWS? ";"+) | CFWS) parameter CFWS?)*;
+}%%

data/lib/mail/parsers/rfc2045_mime.rl

@@ -0,0 +1,15 @@
+%%{
+  # RFC 2045 MIME
+  # https://tools.ietf.org/html/rfc2045
+  machine rfc2045_mime;
+
+  include rfc5322_lexical_tokens "rfc5322_lexical_tokens.rl";
+
+  # 4. MIME-Version Header Field
+  # https://tools.ietf.org/html/rfc2045#section-4
+  mime_version = CFWS?
+            (DIGIT+ >major_digits_s %major_digits_e)
+            comment? "." comment?
+            (DIGIT+ >minor_digits_s %minor_digits_e)
+            CFWS?;
+}%%

data/lib/mail/parsers/rfc2183_content_disposition.rl

@@ -0,0 +1,14 @@
+%%{
+  # RFC 2183 The Content-Disposition Header Field
+  # https://tools.ietf.org/html/rfc2183#section-2
+  #
+  # TODO: recognize filename, size, creation date, etc.
+  machine rfc2183_content_disposition;
+
+  include rfc2045_content_type "rfc2045_content_type.rl";
+
+  disposition_type = 'inline'i | 'attachment'i | extension_token;
+  disposition_parm = parameter;
+  disposition = (disposition_type >disp_type_s %disp_type_e)
+                (";" disposition_parm)*;
+}%%

data/lib/mail/parsers/rfc5234_abnf_core_rules.rl

@@ -0,0 +1,16 @@
+%%{
+  # RFC 5234 B.1. Core Rules
+  # https://tools.ietf.org/html/rfc5234#appendix-B.1
+  machine rfc5234_abnf_core_rules;
+
+  LF = "\n";
+  CR = "\r";
+  CRLF = "\r\n";
+  SP = " ";
+  HTAB = "\t";
+  WSP = SP | HTAB;
+  DQUOTE = '"';
+  DIGIT = [0-9];
+  ALPHA = [a-zA-Z];
+  VCHAR = 0x21..0x7e;
+}%%

data/lib/mail/parsers/rfc5322.rl

@@ -0,0 +1,57 @@
+%%{
+  # RFC 5322 Internet Message Format
+  # https://tools.ietf.org/html/rfc5322
+  #
+  # RFC 6854 Update to Internet Message Format to Allow Group Syntax in the "From:" and "Sender:" Header Fields
+  # https://tools.ietf.org/html/rfc6854
+  machine rfc5322;
+
+  include rfc5234_abnf_core_rules "rfc5234_abnf_core_rules.rl";
+
+  # 3.2. Lexical Tokens
+  include rfc5322_lexical_tokens "rfc5322_lexical_tokens.rl";
+
+  # 3.3. Date and Time Specification
+  include rfc5322_date_time "rfc5322_date_time.rl";
+
+  # 3.4. Address Specification
+  include rfc5322_address "rfc5322_address.rl";
+
+  # 3.5. Overall Message Syntax
+  #text = 0x01..0x09 | "\v" | "\f" | 0x0e..0x1f;
+  #obs_body = ((LF* CR* ((0x00 | text) LF* CR*)*) | CRLF)*
+  #body = ((text{,998} CRLF)* text{,998}) | obs_body;
+  #message = (fields | obs_fields) (CRLF body)?;
+
+
+  # 3.6. Field Definitions
+
+  # 3.6.4. Identification Fields
+  obs_id_left = local_part;
+  id_left = dot_atom_text | obs_id_left;
+  # id_right modifications to support multiple '@' in msg_id.
+  msg_id_atext = ALPHA | DIGIT | "!" | "#" | "$" | "%" | "&" | "'" | "*" |
+                 "+" | "-" | "/" | "=" | "?" | "^" | "_" | "`" | "{" | "|" |
+                 "}" | "~" | "@";
+  msg_id_dot_atom_text = (msg_id_atext+ "."?)+;
+  obs_id_right = domain;
+  no_fold_literal = "[" (dtext)* "]";
+  id_right = msg_id_dot_atom_text | no_fold_literal | obs_id_right;
+  msg_id = (CFWS)?
+           (("<" id_left "@" id_right ">") >msg_id_s %msg_id_e)
+           (CFWS)?;
+  message_ids = msg_id (CFWS? msg_id)*;
+
+
+  # 3.6.7 Trace Fields
+  # Added CFWS? to increase robustness (qmail likes to include a comment)
+  received_token = word | angle_addr | addr_spec_no_angle_brackets | domain;
+  received = ((CFWS? received_token*) >received_tokens_s %received_tokens_e)
+              ";" date_time;
+
+  # Envelope From
+  ctime_date = day_name " "+ month " "+ day " " time_of_day " " year;
+  null_sender = ('<>' ' '{0,1});
+  envelope_from = (addr_spec_no_angle_brackets | null_sender) >address_s %address_e " "
+                  (ctime_date >ctime_date_s %ctime_date_e);
+}%%

data/lib/mail/parsers/rfc5322_address.rl

@@ -0,0 +1,70 @@
+%%{
+  # RFC 5322 Internet Message Format
+  # Section 3.4. Address Specification
+  # https://tools.ietf.org/html/rfc5322#section-3.4
+  machine rfc5322_address;
+
+  include rfc5234_abnf_core_rules "rfc5234_abnf_core_rules.rl";
+  include rfc5322_lexical_tokens "rfc5322_lexical_tokens.rl";
+
+  # local_part:
+  domain_text = (DQUOTE (FWS? qcontent)+ FWS? DQUOTE) | atext+;
+  local_dot_atom_text = ("."* domain_text "."*)+;
+  local_dot_atom = CFWS?
+                   (local_dot_atom_text >local_dot_atom_s %local_dot_atom_pre_comment_e)
+                   CFWS?;
+  obs_local_part = word ("." word)*;
+  local_part = (local_dot_atom >local_dot_atom_s %local_dot_atom_e |
+                (quoted_string %local_quoted_string_e) |
+                obs_local_part);
+
+  # Treetop parser behavior was to ignore addresses missing '@' inside of angle
+  # brackets. This construction preserves that behavior.
+  local_part_no_capture = (local_dot_atom | quoted_string | obs_local_part);
+
+  # domain:
+  domain_dot_atom_text = "."* domain_text ("."* domain_text)*;
+  obs_dtext = obs_NO_WS_CTL | quoted_pair;
+  dtext = 0x21..0x5a | 0x5e..0x7e | obs_dtext;
+  domain_dot_atom = CFWS? domain_dot_atom_text (CFWS? >(comment_after_address,1));
+  domain_literal = CFWS? "[" (FWS? dtext)* FWS? "]" CFWS?;
+  obs_domain = atom ("." atom)*;
+  domain = (domain_dot_atom | domain_literal | obs_domain) >domain_s %domain_e;
+
+  # 3.4.1. Addr-Spec Specification
+
+  # The %(end_addr,N) priority resolves uncertainty when whitespace
+  # after an addr_spec could cause it to be interpreted as a
+  # display name: "bar@example.com ,..."
+
+  addr_spec_in_angle_brackets =
+    (local_part "@" domain) %(end_addr,1) |
+    local_part_no_capture   %(end_addr,0);
+
+  addr_spec_no_angle_brackets =
+    (local_part "@" domain) %(end_addr,1) |
+    local_part              %(end_addr,0);
+
+  # angle_addr:
+  obs_domain_list = (CFWS | ",")* "@" domain ("," CFWS? ("@" domain)?)*;
+  obs_route = (obs_domain_list ":") >obs_domain_list_s %obs_domain_list_e;
+  obs_angle_addr = CFWS? "<" obs_route? addr_spec_in_angle_brackets ">" CFWS?;
+
+  angle_addr = CFWS? ("<" >angle_addr_s) addr_spec_in_angle_brackets ">" CFWS? |
+                obs_angle_addr;
+
+  # 3.4. Address Specification
+  display_name = phrase;
+  name_addr = display_name? %(end_addr,2) angle_addr;
+  mailbox = (name_addr | addr_spec_no_angle_brackets) >address_s %address_e;
+  obs_mbox_list = (CFWS? ",")* mailbox ("," (mailbox | CFWS)?)*;
+  mailbox_list = (mailbox (("," | ";") mailbox)*) | obs_mbox_list;
+  obs_group_list = (CFWS? ",")+ CFWS?;
+  group_list = mailbox_list | CFWS | obs_group_list;
+  group = (display_name >group_name_s %group_name_e) ":"
+            (group_list?) ";" CFWS?;
+  address = group | mailbox;
+  #obs_addr_list = (CFWS? ",")* address ("," (address | CFWS)?)*;
+  address_lists = address? %(comment_after_address,0)
+                  (FWS* ("," | ";") FWS* address?)*;
+}%%

data/lib/mail/parsers/{ragel/date_time.rl → rfc5322_date_time.rl}

@@ -1,6 +1,10 @@
 %%{
+  # RFC 5322 Internet Message Format
+  # Section 3.3. Date and Time Specification
+  # https://tools.ietf.org/html/rfc5322#section-3.3
+  machine rfc5322_date_time;
 
-  machine date_time;
+  include rfc5322_lexical_tokens "rfc5322_lexical_tokens.rl";
 
   # day_of_week
   day_name = "Mon" | "Tue" | "Wed" | "Thu" | "Fri" | "Sat" | "Sun";
@@ -27,4 +31,6 @@
   zone = FWS ((("+" | "-") DIGIT DIGIT DIGIT DIGIT) | obs_zone);
   time = time_of_day zone;
 
+  date_time = (day_of_week ",")?
+              (date >date_s %date_e) <: (time >time_s %time_e) CFWS?;
 }%%

data/lib/mail/parsers/rfc5322_lexical_tokens.rl

@@ -0,0 +1,56 @@
+%%{
+  # RFC 5322 Internet Message Format
+  # Section 3.2. Lexical Tokens
+  # https://tools.ietf.org/html/rfc5322#section-3.2
+  machine rfc5322_lexical_tokens;
+
+  include rfc5234_abnf_core_rules "rfc5234_abnf_core_rules.rl";
+
+  # 3.2.1.  Quoted characters
+  obs_NO_WS_CTL = 0x01..0x08 | "\v" | "\f" | 0x0e..0x1f | 0x7f;
+  obs_qp = "\\" (0x00 | obs_NO_WS_CTL | LF | CR);
+  quoted_pair = ("\\" (VCHAR | WSP)) | obs_qp;
+
+  # 3.2.2. Folding White Space and Comments
+  obs_FWS = (CRLF? WSP)+;
+  FWS = (WSP* CRLF WSP+) | (CRLF WSP+) | obs_FWS;
+
+  obs_ctext = obs_NO_WS_CTL;
+  ctext = 0x21..0x27 | 0x2a..0x5b | 0x5d..0x7e | obs_ctext;
+
+  # Recursive comments
+  action comment_begin { fcall comment_tail; }
+  action comment_exit { fret; }
+  ccontent = ctext | quoted_pair | "(" @comment_begin;
+  comment_tail := ((FWS? ccontent)* >comment_s) FWS? ")" @comment_exit;
+  comment = "(" @comment_begin %comment_e;
+  CFWS = ((FWS? comment)+ FWS?) | FWS;
+
+  # 3.2.3. Atom
+  atext = ALPHA | DIGIT | "!" | "#" | "$" | "%" | "&" |
+          "'" | "*" | "+" | "-" | "/" | "=" | "?" | "^" |
+          "_" | "`" | "{" | "|" | "}" | "~";
+  atom = CFWS? atext+ CFWS?;
+  dot_atom_text = atext ("." atext)*;
+  dot_atom = CFWS? dot_atom_text CFWS?;
+
+  # 3.2.4. Quoted Strings
+  obs_qtext = obs_NO_WS_CTL;
+  qtext = 0x21 | 0x23..0x5b | 0x5d..0x7e | obs_qtext;
+
+  qcontent = qtext | quoted_pair;
+  quoted_string = CFWS?
+                  (DQUOTE
+                    (((FWS? qcontent)* FWS?) >qstr_s %qstr_e)
+                  DQUOTE)
+                  CFWS?;
+
+  # 3.2.5. Miscellaneous Tokens
+  word = atom | quoted_string;
+
+  obs_phrase = (word | "." | "@")+;
+  phrase = (obs_phrase | word+) >phrase_s %phrase_e;
+
+  # Not part of RFC, used for keywords per 3.6.5 Information Fields
+  phrase_lists = phrase ("," FWS* phrase)*;
+}%%

data/lib/mail/parsers.rb

@@ -1,27 +1,19 @@
 # frozen_string_literal: true
-module Mail
-  module Parsers
+# Ragel-generated parsers are full of known warnings. Suppress them.
+begin
+  orig, $VERBOSE = $VERBOSE, nil
 
-    # Low-level ragel based parsers
-    require 'mail/parsers/ragel'
-
-    AddressListStruct = Struct.new(:addresses, :group_names, :error)
-    AddressStruct = Struct.new(:raw, :domain, :comments, :local,
-                             :obs_domain_list, :display_name, :group, :error)
-    ContentDispositionStruct = Struct.new(:disposition_type, :parameters, :error)
-    ContentLocationStruct = Struct.new(:location, :error)
-    ContentTransferEncodingStruct = Struct.new(:encoding, :error)
-    ContentTypeStruct = Struct.new(:main_type, :sub_type, :parameters, :error)
-    DateTimeStruct = Struct.new(:date_string, :time_string, :error)
-    EnvelopeFromStruct = Struct.new(:address, :ctime_date, :error)
-    MessageIdsStruct = Struct.new(:message_ids, :error)
-    MimeVersionStruct = Struct.new(:major, :minor, :error)
-    PhraseListsStruct = Struct.new(:phrases, :error)
-    ReceivedStruct = Struct.new(:date, :time, :info, :error)
-
-    require 'mail/parsers/ragel/parser_info'
-    Ragel::FIELD_PARSERS.each do |field_parser|
-      require "mail/parsers/#{field_parser}_parser"
-    end
-  end
+  require 'mail/parsers/address_lists_parser'
+  require 'mail/parsers/content_disposition_parser'
+  require 'mail/parsers/content_location_parser'
+  require 'mail/parsers/content_transfer_encoding_parser'
+  require 'mail/parsers/content_type_parser'
+  require 'mail/parsers/date_time_parser'
+  require 'mail/parsers/envelope_from_parser'
+  require 'mail/parsers/message_ids_parser'
+  require 'mail/parsers/mime_version_parser'
+  require 'mail/parsers/phrase_lists_parser'
+  require 'mail/parsers/received_parser'
+ensure
+  $VERBOSE = orig
 end

data/lib/mail/part.rb

@@ -21,7 +21,7 @@ module Mail
     
     def inline_content_id
       # TODO: Deprecated in 2.2.2 - Remove in 2.3
-      STDERR.puts("Part#inline_content_id is deprecated, please call Part#cid instead")
+      $stderr.puts("Part#inline_content_id is deprecated, please call Part#cid instead")
       cid
     end
     

data/lib/mail/parts_list.rb

@@ -55,7 +55,7 @@ module Mail
         # OK, 10000 is arbitrary... if anyone actually wants to explicitly sort 10000 parts of a
         # single email message... please show me a use case and I'll put more work into this method,
         # in the meantime, it works :)
-        [get_order_value(a, order), i += 1]
+        get_order_value(a, order) << (i += 1)
       end
       @parts.clear
       sorted.each { |p| @parts << p }
@@ -64,11 +64,10 @@ module Mail
   private
 
     def get_order_value(part, order)
-      if part.respond_to?(:content_type) && !part[:content_type].nil?
-        order.index(part[:content_type].string.downcase) || 10000
-      else
-        10000
-      end
+      is_attachment = part.respond_to?(:attachment?) && part.attachment?
+      has_content_type = part.respond_to?(:content_type) && !part[:content_type].nil?
+
+      [is_attachment ? 1 : 0, (has_content_type ? order.index(part[:content_type].string.downcase) : nil) || 10000]
     end
 
   end

data/lib/mail/utilities.rb

@@ -1,5 +1,7 @@
 # encoding: utf-8
 # frozen_string_literal: true
+require 'mail/constants'
+
 module Mail
   module Utilities
 
@@ -72,11 +74,26 @@ module Mail
     #  unqoute(string) #=> 'This is "a string"'
     def unquote( str )
       if str =~ /^"(.*?)"$/
-        $1.gsub(/\\(.)/, '\1')
+        unescape($1)
       else
         str
       end
     end
+    module_function :unquote
+
+    # Removes any \-escaping.
+    #
+    # Example:
+    #
+    #  string = 'This is \"a string\"'
+    #  unescape(string) #=> 'This is "a string"'
+    #
+    #  string = '"This is \"a string\""'
+    #  unescape(string) #=> '"This is "a string""'
+    def unescape( str )
+      str.gsub(/\\(.)/, '\1')
+    end
+    module_function :unescape
 
     # Wraps a string in parenthesis and escapes any that are in the string itself.
     #
@@ -178,7 +195,7 @@ module Mail
     # Example:
     #
     #  string = :resent_from_field
-    #  dasherize ( string ) #=> 'resent_from_field'
+    #  dasherize( string ) #=> 'resent-from-field'
     def dasherize( str )
       str.to_s.tr(UNDERSCORE, HYPHEN)
     end
@@ -274,6 +291,5 @@ module Mail
         value.respond_to?(:empty?) ? value.empty? : !value
       end
     end
-
   end
 end

data/lib/mail/version.rb

@@ -3,9 +3,9 @@ module Mail
   module VERSION
 
     MAJOR = 2
-    MINOR = 6
-    PATCH = 4
-    BUILD = nil
+    MINOR = 7
+    PATCH = 0
+    BUILD = 'rc1'
 
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
 

data/lib/mail/version_specific/ruby_1_8.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 # frozen_string_literal: true
+require 'net/imap' # for decode_utf7
 
 module Mail
   class Ruby18
@@ -56,7 +57,27 @@ module Mail
     end
 
     def Ruby18.transcode_charset(str, from_encoding, to_encoding = 'UTF-8')
-      Iconv.conv("#{normalize_iconv_charset_encoding(to_encoding)}//IGNORE", normalize_iconv_charset_encoding(from_encoding), str)
+      case from_encoding
+      when /utf-?7/i
+        decode_utf7(str)
+      else
+        retried = false
+        begin
+          Iconv.conv("#{normalize_iconv_charset_encoding(to_encoding)}//IGNORE", normalize_iconv_charset_encoding(from_encoding), str)
+        rescue Iconv::InvalidEncoding
+          if retried
+            raise
+          else
+            from_encoding = 'ASCII'
+            retried = true
+            retry
+          end
+        end
+      end
+    end
+
+    def Ruby18.decode_utf7(str)
+      Net::IMAP.decode_utf7(str)
     end
 
     def Ruby18.b_value_encode(str, encoding)
@@ -97,7 +118,12 @@ module Mail
     end
 
     def Ruby18.param_decode(str, encoding)
-      URI.unescape(str)
+      str = URI.unescape(str)
+      if encoding
+        transcode_charset(str, encoding)
+      else
+        str
+      end
     end
 
     def Ruby18.param_encode(str)

data/lib/mail/version_specific/ruby_1_9.rb

@@ -5,13 +5,23 @@ module Mail
   class Ruby19
     class StrictCharsetEncoder
       def encode(string, charset)
-        string.force_encoding(Mail::Ruby19.pick_encoding(charset))
+        case charset
+        when /utf-?7/i
+          Mail::Ruby19.decode_utf7(string)
+        else
+          string.force_encoding(Mail::Ruby19.pick_encoding(charset))
+        end
       end
     end
 
     class BestEffortCharsetEncoder
       def encode(string, charset)
-        string.force_encoding(pick_encoding(charset))
+        case charset
+        when /utf-?7/i
+          Mail::Ruby19.decode_utf7(string)
+        else
+          string.force_encoding(pick_encoding(charset))
+        end
       end
 
       private
@@ -59,6 +69,9 @@ module Mail
     end
 
     def Ruby19.decode_base64(str)
+      if !str.end_with?("=") && str.length % 4 != 0
+        str = str.ljust((str.length + 3) & ~3, "=")
+      end
       str.unpack( 'm' ).first
     end
 
@@ -75,7 +88,31 @@ module Mail
     end
 
     def Ruby19.transcode_charset(str, from_encoding, to_encoding = Encoding::UTF_8)
-      charset_encoder.encode(str.dup, from_encoding).encode(to_encoding, :undef => :replace, :invalid => :replace, :replace => '')
+      to_encoding = Encoding.find(to_encoding)
+      replacement_char = to_encoding == Encoding::UTF_8 ? '�' : '?'
+      charset_encoder.encode(str.dup, from_encoding).encode(to_encoding, :undef => :replace, :invalid => :replace, :replace => replacement_char)
+    end
+
+    # From Ruby stdlib Net::IMAP
+    def Ruby19.encode_utf7(string)
+      string.gsub(/(&)|[^\x20-\x7e]+/) do
+        if $1
+          "&-"
+        else
+          base64 = [$&.encode(Encoding::UTF_16BE)].pack("m0")
+          "&" + base64.delete("=").tr("/", ",") + "-"
+        end
+      end.force_encoding(Encoding::ASCII_8BIT)
+    end
+
+    def Ruby19.decode_utf7(utf7)
+      utf7.gsub(/&([^-]+)?-/n) do
+        if $1
+          ($1.tr(",", "/") + "===").unpack("m")[0].encode(Encoding::UTF_8, Encoding::UTF_16BE)
+        else
+          "&"
+        end
+      end
     end
 
     def Ruby19.b_value_encode(str, encoding = nil)
@@ -90,8 +127,7 @@ module Mail
         str = Ruby19.decode_base64(match[2])
         str = charset_encoder.encode(str, charset)
       end
-      decoded = str.encode(Encoding::UTF_8, :invalid => :replace, :replace => "")
-      decoded.valid_encoding? ? decoded : decoded.encode(Encoding::UTF_16LE, :invalid => :replace, :replace => "").encode(Encoding::UTF_8)
+      transcode_to_scrubbed_utf8(str)
     rescue Encoding::UndefinedConversionError, ArgumentError, Encoding::ConverterNotFoundError
       warn "Encoding conversion failed #{$!}"
       str.dup.force_encoding(Encoding::UTF_8)
@@ -115,8 +151,7 @@ module Mail
         # jruby/jruby#829 which subtly changes String#encode semantics.
         str.force_encoding(Encoding::UTF_8) if str.encoding == Encoding::ASCII_8BIT
       end
-      decoded = str.encode(Encoding::UTF_8, :invalid => :replace, :replace => "")
-      decoded.valid_encoding? ? decoded : decoded.encode(Encoding::UTF_16LE, :invalid => :replace, :replace => "").encode(Encoding::UTF_8)
+      transcode_to_scrubbed_utf8(str)
     rescue Encoding::UndefinedConversionError, ArgumentError, Encoding::ConverterNotFoundError
       warn "Encoding conversion failed #{$!}"
       str.dup.force_encoding(Encoding::UTF_8)
@@ -125,7 +160,10 @@ module Mail
     def Ruby19.param_decode(str, encoding)
       str = uri_parser.unescape(str)
       str = charset_encoder.encode(str, encoding) if encoding
-      str
+      transcode_to_scrubbed_utf8(str)
+    rescue Encoding::UndefinedConversionError, ArgumentError, Encoding::ConverterNotFoundError
+      warn "Encoding conversion failed #{$!}"
+      str.dup.force_encoding(Encoding::UTF_8)
     end
 
     def Ruby19.param_encode(str)
@@ -218,6 +256,11 @@ module Mail
           end
         end
       end
+
+      def transcode_to_scrubbed_utf8(str)
+        decoded = str.encode(Encoding::UTF_8, :undef => :replace, :invalid => :replace, :replace => "�")
+        decoded.valid_encoding? ? decoded : decoded.encode(Encoding::UTF_16LE, :invalid => :replace, :replace => "�").encode(Encoding::UTF_8)
+      end
     end
   end
 end