mailparser 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: acab11e0f5b50260e17f32e1fea879d8505fff45
-  data.tar.gz: f0ce38c8bb40d11a3332515b4aabb7d835cb692f
-!binary "U0hBNTEy":
-  metadata.gz: 4edf18f3211be9fb2d2114db16265fd728a4f5a69eb1d96ab9a7fc430e0cdcc740f1d3b00ae04290a6203b611cfcd0b22531ac8c5962d7a513798bbdda57c53d
-  data.tar.gz: ee05e15d52db8517094e5c9189787c7e3f557a5c17c9d2dd8a8d3868111e671e086121296b1d13493e9ee6d43fc0ab6576fcd35bd3dda4b39dbfb91451904a95
+SHA1:
+  metadata.gz: acf77838874db7d6c17b1e85420b5027eaea07c5
+  data.tar.gz: b67ae4ea3374d66de3cb11f40a5fe25074e682d6
+SHA512:
+  metadata.gz: 790f81bfd597d4b209efb81a29ffa6356484ab5b8570140b16dac2b0e33982269df2a0fa11f16362c06e198c759eef8b087d17a3a50e2967ef4daea6710d9a97
+  data.tar.gz: 2f2502e6975c84472da8675c4c42d65056e1ec4f3845a2ebe8a5477a3162f4ecaa0f43c37b729da9c189ca5dd12609704bac7c2dba002d095cabe2266ca931ff

data/lib/mailparser.rb

@@ -81,6 +81,8 @@ module MailParser
         r = @raw.chomp.gsub(/\r?\n/, '').gsub(/\t/, ' ')
         if @opt[:decode_mime_header] then
           @parsed = RFC2047.decode(r, @opt)
+        elsif @opt[:output_charset]
+          @parsed = @opt[:charset_converter].call(@opt[:output_charset], @opt[:output_charset], r)
         else
           @parsed = r
         end

data/lib/mailparser/loose.rb

@@ -67,7 +67,8 @@ module MailParser
       rescue
         t = Time.now
       end
-      return RFC2822::DateTime.new(t.year, t.month, t.day, t.hour, t.min, t.sec, t.zone)
+      year = t.year > 9999 ? 9999 : t.year
+      return RFC2822::DateTime.new(year, t.month, t.day, t.hour, t.min, t.sec, t.zone)
     end
 
     # parse From, To,Cc field
@@ -91,7 +92,7 @@ module MailParser
     # @param [Hash] opt options
     # @return [MailParser::RFC2822::MsgId]
     def parse_msg_id(str, opt={})
-      msg_id_list(str)[0]
+      msg_id_list(str, opt)[0]
     end
 
     # parse In-Reply-To, References field
@@ -99,7 +100,7 @@ module MailParser
     # @param [Hash] opt options
     # @return [MailParser::RFC2822::MsgIdList]
     def parse_msg_id_list(str, opt={})
-      msg_id_list(str)
+      msg_id_list(str, opt)
     end
 
     # parse Keywords field
@@ -112,7 +113,7 @@ module MailParser
       if opt[:decode_mime_header] then
         s.map!{|i| RFC2047.decode(i, opt)}
       end
-      s
+      s.map{|_| _conv(_, opt)}
     end
 
     # parse Return-Path field
@@ -145,6 +146,9 @@ module MailParser
           i += 1
         end
       end
+      name_val.keys.each do |k|
+        name_val[k] = _conv(name_val[k], opt)
+      end
       RFC2822::Received.new(name_val, date)
     end
 
@@ -158,13 +162,13 @@ module MailParser
       params = {}
       token.each do |param|
         pn, pv = param.join.split(/=/, 2)
-        params[pn.to_s] = pv.to_s.gsub(/\A"|"\z/,"")
+        params[pn.to_s] = _conv(pv.to_s.gsub(/\A"|"\z/,""), opt)
       end
       type = "text" if type.nil? or type.empty?
       if subtype.nil? or subtype.empty?
         subtype = type == "text" ? "plain" : ""
       end
-      RFC2045::ContentType.new(type, subtype, params)
+      RFC2045::ContentType.new(_conv(type, opt), _conv(subtype, opt), params)
     end
 
     # parse Content-Transfer-Encoding field
@@ -172,7 +176,7 @@ module MailParser
     # @param [Hash] opt options
     # @return [MailParser::RFC2045::ContentTransferEncoding]
     def parse_content_transfer_encoding(str, opt={})
-      RFC2045::ContentTransferEncoding.new(Tokenizer.token(str).first.to_s)
+      RFC2045::ContentTransferEncoding.new(_conv(Tokenizer.token(str).first.to_s, opt))
     end
 
     # parse Mime-Version field
@@ -180,7 +184,7 @@ module MailParser
     # @param [Hash] opt options
     # @return [String]
     def parse_mime_version(str, opt={})
-      Tokenizer.token(str).join
+      _conv(Tokenizer.token(str).join, opt)
     end
 
     # parse Content-Disposition field
@@ -193,9 +197,9 @@ module MailParser
       params = {}
       token.each do |param|
         pn, pv = param.join.split(/=/, 2)
-        params[pn.to_s] = pv.to_s.gsub(/\A"|"\z/,"")
+        params[pn.to_s] = _conv(pv.to_s.gsub(/\A"|"\z/,""), opt)
       end
-      RFC2183::ContentDisposition.new(type, params)
+      RFC2183::ContentDisposition.new(_conv(type, opt), params)
     end
 
     # split arry by delim
@@ -229,11 +233,11 @@ module MailParser
           if opt[:decode_mime_header] then
             display_name = RFC2047.decode(display_name, opt)
           end
-          mailaddr = m[a1+1..a2-1].join
+          mailaddr = _conv(m[a1+1..a2-1].join, opt)
           local_part, domain = mailaddr.split(/@/, 2)
-          ret << RFC2822::Mailbox.new(RFC2822::AddrSpec.new(local_part, domain), display_name)
+          ret << RFC2822::Mailbox.new(RFC2822::AddrSpec.new(local_part, domain), _conv(display_name, opt))
         else
-          local_part, domain = m.join.split(/@/, 2)
+          local_part, domain = _conv(m.join, opt).split(/@/, 2)
           ret << RFC2822::Mailbox.new(RFC2822::AddrSpec.new(local_part, domain))
         end
       end
@@ -243,7 +247,7 @@ module MailParser
     # parse MsgId type field
     # @param [String] str
     # @return [Array<MailParser::RFC2822::MsgId>]
-    def msg_id_list(str)
+    def msg_id_list(str, opt={})
       ret = []
       flag = false
       msgid = nil
@@ -257,14 +261,14 @@ module MailParser
         when ">"
           if flag
             flag = false
-            ret << RFC2822::MsgId.new(msgid)
+            ret << RFC2822::MsgId.new(_conv(msgid, opt))
           end
         else
           msgid << m if flag
         end
       end
       if ret.empty?
-        ret = str.split.map{|s| RFC2822::MsgId.new(s)}
+        ret = str.split.map{|s| RFC2822::MsgId.new(_conv(s, opt))}
       end
       return ret
     end
@@ -291,9 +295,9 @@ module MailParser
               @ss.pos = pos
               token << s
             end
-          elsif s = @ss.scan(/\"(\s*(\\[#{TEXT_RE}]|[#{QTEXT_RE}]))*\s*\"/o) ||
-              @ss.scan(/\[(\s*(\\[#{TEXT_RE}]|[#{DTEXT_RE}]))*\s*\]/o) ||
-              @ss.scan(/[#{ATEXT_RE}]+/o)
+          elsif s = @ss.scan(/\"(\s*(\\[#{TEXT_RE}]|[#{QTEXT_RE}\x80-\xff]))*\s*\"/o) ||
+              @ss.scan(/\[(\s*(\\[#{TEXT_RE}]|[#{DTEXT_RE}\x80-\xff]))*\s*\]/o) ||
+              @ss.scan(/[#{ATEXT_RE}\x80-\xff]+/o)
             token << s
           else
             token << @ss.scan(/./)
@@ -337,5 +341,11 @@ module MailParser
       end
     end
 
+    def _conv(str, opt)
+      cv = opt[:charset_converter]
+      cs = opt[:output_charset]
+      cv && cs ? cv.call(cs, cs, str) : str
+    end
+
   end
 end

data/test/test_loose.rb

@@ -31,6 +31,23 @@ class TC_Loose < Test::Unit::TestCase
     end
   end
 
+  def test_parse_date_y10000()
+    tzbak = ENV["TZ"]
+    begin
+      ENV["TZ"] = "GMT"
+      d = parse_date("Wed, 10 Jan 10000 12:53:55 +0900")
+      assert_equal(9999, d.year)
+      assert_equal(1, d.month)
+      assert_equal(10, d.day)
+      assert_equal(3, d.hour)
+      assert_equal(53, d.min)
+      assert_equal(55, d.sec)
+      assert_equal("+0000", d.zone)
+    ensure
+      ENV["TZ"] = tzbak
+    end
+  end
+
   def test_parse_phrase_list()
     p = parse_phrase_list("abc def, ghi jkl")
     assert_equal(2, p.size)
@@ -52,6 +69,15 @@ class TC_Loose < Test::Unit::TestCase
     assert_equal("GHI JKL", p[1])
   end
 
+  if String.method_defined? :force_encoding
+    def test_parse_phrase_list_output_charset_with_raw_utf8
+      p = parse_phrase_list("あいう, えお", :charset_converter=>MailParser::ConvCharset.method(:conv_charset), :output_charset=>"utf-8")
+      assert_equal(2, p.size)
+      assert_equal("あいう".force_encoding("utf-8"), p[0])
+      assert_equal("えお".force_encoding("utf-8"), p[1])
+    end
+  end
+
   def test_parse_received()
     tzbak = ENV["TZ"]
     begin
@@ -150,6 +176,28 @@ class TC_Loose < Test::Unit::TestCase
     end
   end
 
+  if String.method_defined? :force_encoding
+    def test_parse_received_output_charset_with_raw_utf8
+      tzbak = ENV["TZ"]
+      begin
+        ENV["TZ"] = "GMT"
+        r = parse_received("from ほげ by ふが for ぴよ; Wed, 10 Jan 2007 12:09:55 +0900", :charset_converter=>MailParser::ConvCharset.method(:conv_charset), :output_charset=>"utf-8")
+        assert_equal(2007, r.date_time.year)
+        assert_equal(1, r.date_time.month)
+        assert_equal(10, r.date_time.day)
+        assert_equal(3, r.date_time.hour)
+        assert_equal(9, r.date_time.min)
+        assert_equal(55, r.date_time.sec)
+        assert_equal("+0000", r.date_time.zone)
+        assert_equal("ほげ".force_encoding("utf-8"), r.name_val["from"])
+        assert_equal("ふが".force_encoding("utf-8"), r.name_val["by"])
+        assert_equal("ぴよ".force_encoding("utf-8"), r.name_val["for"])
+      ensure
+        ENV["TZ"] = tzbak
+      end
+    end
+  end
+
   def test_parse_content_type()
     ct = parse_content_type("text/plain; charset=iso-2022-jp")
     assert_equal("text", ct.type)
@@ -190,6 +238,15 @@ class TC_Loose < Test::Unit::TestCase
     assert_equal("", ct.subtype)
   end
 
+  if String.method_defined? :force_encoding
+    def test_parse_content_type_output_charset_with_raw_utf8
+      ct = parse_content_type("text/plain; name=ほげ", :charset_converter=>MailParser::ConvCharset.method(:conv_charset), :output_charset=>"utf-8")
+      assert_equal("text", ct.type)
+      assert_equal("plain", ct.subtype)
+      assert_equal({"name"=>"ほげ".force_encoding("utf-8")}, ct.params)
+    end
+  end
+
   def test_parse_content_transfer_encoding
     cte = parse_content_transfer_encoding("7BIT")
     assert_equal "7bit", cte.mechanism
@@ -200,6 +257,13 @@ class TC_Loose < Test::Unit::TestCase
     assert_equal "", cte.mechanism
   end
 
+  if String.method_defined? :force_encoding
+    def test_parse_content_transfer_encoding_output_charset_with_raw_utf8
+      cte = parse_content_transfer_encoding("あいう", :charset_converter=>MailParser::ConvCharset.method(:conv_charset), :output_charset=>"utf-8")
+      assert_equal "あいう".force_encoding("utf-8"), cte.mechanism
+    end
+  end
+
   def test_parse_mime_version
     assert_equal "1.0", parse_mime_version("1.0")
     assert_equal "1.0", parse_mime_version("1 . 0")
@@ -210,6 +274,12 @@ class TC_Loose < Test::Unit::TestCase
     assert_equal "", parse_mime_version("")
   end
 
+  if String.method_defined? :force_encoding
+    def test_parse_mime_version_output_charset_with_raw_utf8
+      assert_equal "ほげ".force_encoding("utf-8"), parse_mime_version("ほげ", :charset_converter=>MailParser::ConvCharset.method(:conv_charset), :output_charset=>"utf-8")
+    end
+  end
+
   def test_parse_content_disposition()
     c = parse_content_disposition("attachment; filename=hoge.txt")
     assert_equal("attachment", c.type)
@@ -227,6 +297,14 @@ class TC_Loose < Test::Unit::TestCase
     assert_equal "", c.type
   end
 
+  if String.method_defined? :force_encoding
+    def test_parse_content_disposition_output_charset_with_raw_utf8
+      c = parse_content_disposition("attachment; filename=ほげ", :charset_converter=>MailParser::ConvCharset.method(:conv_charset), :output_charset=>"utf-8")
+      assert_equal("attachment", c.type)
+      assert_equal({"filename"=>"ほげ".force_encoding("utf-8")}, c.params)
+    end
+  end
+
   def test_parse_other_header
     s = parse("subject", "=?euc-jp?q?=A4=A2=A4=A4?=")
     assert_equal "=?euc-jp?q?=A4=A2=A4=A4?=", s
@@ -282,8 +360,21 @@ class TC_Loose < Test::Unit::TestCase
     ml = mailbox_list("hoge =?us-ascii?q?hoge?= <hoge.hoge@example.com>", {:decode_mime_header=>true, :output_charset=>"us-ascii", :charset_converter=>proc{|_,_,s| s.upcase}})
     assert_equal(1, ml.size)
     assert_equal("HOGE HOGE", ml[0].phrase)
-    assert_equal("hoge.hoge", ml[0].addr_spec.local_part)
-    assert_equal("example.com", ml[0].addr_spec.domain)
+    assert_equal("HOGE.HOGE", ml[0].addr_spec.local_part)
+    assert_equal("EXAMPLE.COM", ml[0].addr_spec.domain)
+  end
+
+  if String.method_defined? :force_encoding
+    def test_mailbox_list_output_charset_with_raw_utf8
+      ml = mailbox_list("ほげ <ほげ@ぴよ>, ふが@ぴよ", :charset_converter=>MailParser::ConvCharset.method(:conv_charset), :output_charset=>"utf-8")
+      assert_equal(2, ml.size)
+      assert_equal("ほげ".force_encoding("utf-8"), ml[0].phrase)
+      assert_equal("ほげ".force_encoding("utf-8"), ml[0].addr_spec.local_part)
+      assert_equal("ぴよ".force_encoding("utf-8"), ml[0].addr_spec.domain)
+      assert_equal("", ml[1].phrase)
+      assert_equal("ふが".force_encoding("utf-8"), ml[1].addr_spec.local_part)
+      assert_equal("ぴよ".force_encoding("utf-8"), ml[1].addr_spec.domain)
+    end
   end
 
   def test_msg_id_list_old_in_reply_to()
@@ -317,6 +408,13 @@ class TC_Loose < Test::Unit::TestCase
     assert_equal m, []
   end
 
+  if String.method_defined? :force_encoding
+    def test_msg_id_output_charset_with_raw_utf8
+      m = msg_id_list "<ほげ>", :charset_converter=>MailParser::ConvCharset.method(:conv_charset), :output_charset=>"utf-8"
+      assert_equal "ほげ".force_encoding("utf-8"), m[0].msg_id
+    end
+  end
+
 end
 
 class TC_Loose_Tokenizer < Test::Unit::TestCase

data/test/test_mailparser.rb

@@ -344,6 +344,15 @@ EOS
     assert_equal("abcdefg", m.subject)
   end
 
+  def test_subject_raw_utf8_with_output_charset
+    msg = StringIO.new(<<EOS)
+Subject: あいうえお
+
+EOS
+    m = MailParser::Message.new(msg, :output_charset=>"utf-8")
+    assert_equal("あいうえお", m.subject)
+  end
+
   def test_content_type()
     msg = StringIO.new(<<EOS)
 Content-Type: text/plain; charset=us-ascii

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: mailparser
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - TOMITA Masahiro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-23 00:00:00.000000000 Z
+date: 2013-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mmapscanner
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: MailParser is a parser for mail message
@@ -64,17 +64,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0.preview3.1
+rubygems_version: 2.0.0
 signing_key: 
 specification_version: 4
 summary: Mail Parser