sisimai 4.18.0-java → 4.18.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1da191c307034c3f05ae22303f1b72d7ab9ee4b3
-  data.tar.gz: 14dd4e883a4d0ee9c827f9f86222f416fe185fe3
+  metadata.gz: faeb3a937063e9c2ac6f3eb598e3685af17a4df8
+  data.tar.gz: 8b71ea9646c6e9bbd38d95542bb45ed543966935
 SHA512:
-  metadata.gz: 566b746d973714fe370d3041db219bb1005599609ef17eb0f80bf0facd3aa5343bd524205dff013fcbced19effe5ac596bd6a8926f9f7334d1f77213b802651f
-  data.tar.gz: e270d541f5849a8beef3c12d09413b23d5b7fe24d6e699fa4b0ae0687b975747189ae836e38389687672c7f2a974d85637f19d1c2ed59a447d024cb5d0c2c9dc
+  metadata.gz: 000895ef83533fa8002d30944b14f8e7c57126a206594c2ea7b48902155474e35ea63000973b27ca1a3fa8f7ae0fb6b128c5fc229b5e3c00befe9d41df21f428
+  data.tar.gz: d52bff92353248023dd1c21d47860309643c96ba2ca96d7a423f7839913858f0fa9fb3859c0b33496e8218dc5be03c6157aedcb853658393adb482a38f7ee278

data/.travis.yml

@@ -5,6 +5,7 @@ rvm:
   - 2.2.4
   - 2.3.0
   - jruby-9.0.4.0
+  - jruby-9.0.5.0
 before_install:
 script:
   - rake spec

data/ANALYTICAL-PRECISION

@@ -4,10 +4,10 @@
 MTA::Activehunter .......    0/0014  0.0000  TransWARE Active!hunter
 MTA::ApacheJames ........    0/0006  0.0000  Java Apache Mail Enterprise Server
 MTA::Courier ............   14/0016  0.8750  Courier MTA
-MTA::Domino .............    7/0015  0.4667  IBM Domino Server
+MTA::Domino .............    6/0015  0.4000  IBM Domino Server
 MTA::Exchange2003 .......    1/0051  0.0196  Microsoft Exchange Server 2003
-MTA::Exchange2007 .......    3/0004  0.7500  Microsoft Exchange Server 2007
-MTA::Exim ...............  142/0172  0.8256  Exim
+MTA::Exchange2007 .......    4/0006  0.6667  Microsoft Exchange Server 2007
+MTA::Exim ...............  144/0174  0.8276  Exim
 MTA::IMailServer ........    0/0045  0.0000  IPSWITCH IMail Server
 MTA::InterScanMSS .......    0/0009  0.0000  Trend Micro InterScan Messaging Security Suite
 MTA::MXLogic ............    0/0014  0.0000  McAfee SaaS
@@ -18,7 +18,7 @@ MTA::MessagingServer ....   19/0022  0.8636  Oracle Communications Messaging Ser
 MTA::Notes ..............    0/0012  0.0000  Lotus Notes
 MTA::OpenSMTPD ..........    5/0023  0.2174  OpenSMTPD
 MTA::Postfix ............  165/0183  0.9016  Postfix
-MTA::Sendmail ...........  259/0247  1.0000  V8Sendmail: /usr/sbin/sendmail
+MTA::Sendmail ...........  330/0252  1.0000  V8Sendmail: /usr/sbin/sendmail
 MTA::SurfControl ........    6/0008  0.7500  WebSense SurfControl
 MTA::V5sendmail .........    0/0386  0.0000  Sendmail version 5
 MTA::X1 .................    0/0007  0.0000  Unknown MTA #1
@@ -42,7 +42,7 @@ MSP::US::Aol ............   18/0021  0.8571  Aol Mail: http://www.aol.com
 MSP::US::Bigfoot ........    2/0002  1.0000  Bigfoot: http://www.bigfoot.com
 MSP::US::Facebook .......    3/0003  1.0000  Facebook: https://www.facebook.com
 MSP::US::Google .........   58/0062  0.9355  Google Gmail: https://mail.google.com
-MSP::US::Office365 ......    7/0010  0.7000  Microsoft Office 365: http://office.microsoft.com/
+MSP::US::Office365 ......    8/0011  0.7273  Microsoft Office 365: http://office.microsoft.com/
 MSP::US::Outlook ........   19/0021  0.9048  Microsoft Outlook.com: https://www.outlook.com/
 MSP::US::ReceivingSES ...    0/0007  0.0000  Amazon SES(Receiving): http://aws.amazon.com/ses/
 MSP::US::SendGrid .......    7/0012  0.5833  SendGrid: http://sendgrid.com/
@@ -50,8 +50,8 @@ MSP::US::Verizon ........    3/0003  1.0000  Verizon Wireless: http://www.verizo
 MSP::US::Yahoo ..........    0/0009  0.0000  Yahoo! MAIL: https://www.yahoo.com
 MSP::US::Zoho ...........    0/0011  0.0000  Zoho Mail: https://www.zoho.com
 ARF .....................    9/0050  0.1800  Abuse Feedback Reporting Format
-RFC3464 .................  131/0272  0.4816  Fallback Module for MTAs
+RFC3464 .................  135/0276  0.4891  Fallback Module for MTAs
 RFC3834 .................    0/0004  0.0000  Detector for auto replied message
 -------------------------------------------------------------------------------
- bounceHammer 2.7.13p3    1110/2145  0.5175
- Sisimai 4.18.0           2145/2145  1.0000
+ bounceHammer 2.7.13p3    1188/2159  0.5503
+ Sisimai 4.18.1           2159/2159  1.0000

data/Changes

@@ -1,5 +1,14 @@
 Revision history for Ruby version of Sisimai
 
+4.18.1  Sun, 11 Sep 2016 20:05:20 +0900 (JST)
+    - Fix bug in Sisimai::Mail::STDIN.read() method reported at issue #61.
+      Thanks to @yaegassy.
+    - Fix bug in Sisimai::MIME.qprintd() reported at #60.
+    - Improved code related to MIME decoding.
+    - Implement Sisimai::String.to_plain() for converting from HTML message to
+      plain text before parsing. The method and related codes are imported from
+      pull-request #197 at p5-Sisimai.
+
 4.18.0  Mon, 22 Aug 2016 20:40:55 +0900 (JST)
     - Import pull-request https://github.com/sisimai/rb-Sisimai/pull/59 (soft
       bounce improvement) from Perl version of Sisimai.

data/Gemfile

@@ -3,10 +3,6 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in sisimai.gemspec
 gemspec name: 'sisimai'
 
-# To execute `rake spec` on Travis-CI
-gem 'oj', '>= 2.14.4', :platform => :ruby
-gem 'jrjackson', '>= 0.3.8', :platform => :jruby
-
 group :development, :test do
   gem 'rake',  :require => false
   gem 'rspec', :require => false

data/README-JA.md

@@ -38,7 +38,7 @@ Sisimaiの動作環境についての詳細は
 
 * [Ruby 2.1.0 or later](http://www.ruby-lang.org/)
   * [__Oj | The fastest JSON parser and object serializer__](https://rubygems.org/gems/oj)
-* Also works on [JRuby 9.0.0.0 or later](http://jruby.org)
+* Also works on [JRuby 9.0.4.0 or later](http://jruby.org)
   * [__JrJackson | A mostly native JRuby wrapper for the java jackson json processor jar__](https://rubygems.org/gems/jrjackson)
 
 インストール
@@ -148,8 +148,8 @@ Perl版Sisimaiとの違い
 | メール解析速度(1000通のメール)              | 3.30秒         | 2.33秒        |
 | インストール方法                            | gem install    | cpanm         |
 | 依存モジュール数(コアモジュールを除く)      | 1モジュール    | 2モジュール   |
-| LOC:ソースコードの行数                      | 11500行        | 8400行        |
-| テスト件数(t/,xt/ディレクトリ)              | 95000件        | 172000件      |
+| LOC:ソースコードの行数                      | 11600行        | 8500行        |
+| テスト件数(spec/,t/,xt/ディレクトリ)        | 97000件        | 177000件      |
 | ライセンス                                  | 二条項BSD      | 二条項BSD     |
 | 開発会社によるサポート契約                  | 準備中         | 提供中        |
 

data/README.md

@@ -37,7 +37,7 @@ More details about system requirements are available at
 
 * [Ruby 2.1.0 or later](http://www.ruby-lang.org/)
   * [__Oj | The fastest JSON parser and object serializer__](https://rubygems.org/gems/oj)
-* Also works on [JRuby 9.0.0.0 or later](http://jruby.org)
+* Also works on [JRuby 9.0.4.0 or later](http://jruby.org)
   * [__JrJackson | A mostly native JRuby wrapper for the java jackson json processor jar__](https://rubygems.org/gems/jrjackson)
 
 Install
@@ -148,8 +148,8 @@ and bounceHammer are available at
 | The speed of parsing email(1000 emails)     | 3.30s          | 2.33s         |
 | How to install                              | gem install    | cpanm         |
 | Dependencies (Except core modules)          | 1 modules      | 2 modules     |
-| LOC:Source lines of code                    | 11500 lines    | 8400 lines    |
-| The number of tests in t/, xt/ directory    | 95000 tests    | 172000 tests  |
+| LOC:Source lines of code                    | 11600 lines    | 8500 lines    |
+| The number of tests(spec/,t/,xt/) directory | 97000 tests    | 177000 tests  |
 | License                                     | BSD 2-Clause   | BSD 2-Clause  |
 | Support Contract provided by Developer      | Coming soon    | Available     |
 

data/lib/sisimai/mail.rb

@@ -27,7 +27,6 @@ module Sisimai
           classname = sprintf('%s::STDIN', self.class)
           parameter['type'] = 'stdin'
           parameter['path'] = $stdin
-
         else
           # The argumenet is a mailbox or a Maildir/.
           mediatype = File.ftype(argv1)
@@ -49,7 +48,6 @@ module Sisimai
         # The argument neither a mailbox nor a Maildir/.
         classname = sprintf('%s::STDIN', self.class)
         parameter['type'] = 'stdin'
-
       end
 
       return nil unless classname

data/lib/sisimai/mail/stdin.rb

@@ -35,6 +35,10 @@ module Sisimai
         readhandle = self.handle
         readbuffer = ''
 
+        if readhandle
+          return nil if readhandle.closed?
+        end
+
         begin
           readhandle = STDIN unless readhandle
           while r = readhandle.gets

data/lib/sisimai/message.rb

@@ -36,6 +36,7 @@ module Sisimai
     DefaultSet = Sisimai::Order.another
     PatternSet = Sisimai::Order.by('subject')
     ExtHeaders = Sisimai::Order.headers
+    ReEncoding = Sisimai::MIME.patterns
 
     # Constructor of Sisimai::Message
     # @param         [String] data      Email text data
@@ -412,7 +413,7 @@ module Sisimai
       mesgformat = (mailheader['content-type'] || '').downcase
       ctencoding = (mailheader['content-transfer-encoding'] || '').downcase
 
-      if mesgformat =~ %r|text/plain;|
+      if mesgformat =~ %r{text/(?:plain|html);?}
         # Content-Type: text/plain; charset=UTF-8
         if ctencoding == 'base64' || ctencoding == 'quoted-printable'
           # Content-Transfer-Encoding: base64
@@ -426,6 +427,27 @@ module Sisimai
             bodystring = Sisimai::MIME.qprintd(bodystring)
           end
         end
+
+        if mesgformat =~ %r|text/html;?|
+          # Content-Type: text/html;...
+          bodystring = Sisimai::String.to_plain(bodystring, true)
+        end
+
+      else
+        # NOT text/plain
+        if bodystring =~ ReEncoding[:'quoted-print']
+          # Content-Transfer-Encoding: quoted-printable
+          bodystring = Sisimai::MIME.qprintd(bodystring, mailheader)
+        end
+
+        if bodystring =~ ReEncoding[:'7bit-encoded'] &&
+           cv = bodystring.match(ReEncoding[:'some-iso2022'])
+          # Content-Transfer-Encoding: 7bit
+          # Content-Type: text/plain; charset=ISO-2022-JP
+          unless cv[1].downcase =~ /(?:us-ascii|utf[-]?8)/
+            bodystring = Sisimai::String.to_utf8(bodystring, cv[1])
+          end
+        end
       end
 
       # EXPAND_FORWARDED_MESSAGE:

data/lib/sisimai/mime.rb

@@ -4,6 +4,22 @@ module Sisimai
     # Imported from p5-Sisimail/lib/Sisimai/MIME.pm
     class << self
       require 'base64'
+      require 'sisimai/string'
+
+      ReE = {
+        :'7bit-encoded' => %r/^Content-Transfer-Encoding:[ ]*7bit$/im,
+        :'quoted-print' => %r/^Content-Transfer-Encoding:[ ]*quoted-printable$/im,
+        :'some-iso2022' => %r/^Content-Type:[ ]*.+;[ ]*charset=["']?(iso-2022-[-a-z0-9]+?)['"]?$/im,
+        :'with-charset' => %r/^Content[-]Type:[ ]*.+[;][ ]*charset=['"]?(.+?)['"]?$/i,
+        :'only-charset' => %r/^[\s\t]+charset=['"]?(.+?)['"]?$/i,
+        :'html-message' => %r|^Content-Type:[ ]*text/html;|mi,
+      }
+
+      # Make MIME-Encoding and Content-Type related headers regurlar expression
+      # @return   [Array] Regular expressions related to MIME encoding
+      def patterns
+        return ReE
+      end
 
       # Check that the argument is MIME-Encoded string or not
       # @param    [String] argvs  String to be checked
@@ -13,7 +29,22 @@ module Sisimai
         return false unless argv1
 
         argv1 = argv1.delete('"')
-        return true if argv1 =~ /[ \t]*=[?][-_0-9A-Za-z]+[?][BbQq][?].+[?]=[ \t]*\z/
+        parts = []
+        isnot = false
+
+        if argv1 =~ /[ ]/
+          # Multiple MIME-Encoded strings in a line
+          parts = argv1.split(' ')
+        else
+          parts << argv1
+        end
+
+        parts.each do |e|
+          # Check all the string in the array
+          next if e =~ /[ \t]*=[?][-_0-9A-Za-z]+[?][BbQq][?].+[?]=?[ \t]*\z/
+          isnot = true
+        end
+        return true unless isnot
         return false
       end
 
@@ -36,7 +67,7 @@ module Sisimai
 
           if self.is_mimeencoded(e)
             # MIME Encoded string
-            if cv = e.match(/\A=[?]([-_0-9A-Za-z]+)[?]([BbQq])[?](.+)[?]=\z/)
+            if cv = e.match(/\A=[?]([-_0-9A-Za-z]+)[?]([BbQq])[?](.+)[?]=?\z/)
               # =?utf-8?B?55m954yr44Gr44KD44KT44GT?=
               characterset ||= cv[1]
               encodingname ||= cv[2]
@@ -78,10 +109,95 @@ module Sisimai
 
       # Decode MIME Quoted-Printable Encoded string
       # @param  [String] argv1   MIME Encoded text
+      # @param  [Hash]   heads   Email header
       # @return [String]         MIME Decoded text
-      def qprintd(argv1)
+      def qprintd(argv1 = nil, heads = {})
         return nil unless argv1
-        return argv1.unpack('M').first
+        return argv1.unpack('M').first unless heads['content-type']
+        return argv1.unpack('M').first unless heads['content-type'].size > 0
+
+        # Quoted-printable encoded part is the part of the text
+        boundary00 = Sisimai::MIME.boundary(heads['content-type'], 0)
+
+        # Decoded using unpack('M') entire body string when the boundary string
+        # or "Content-Transfer-Encoding: quoted-printable" are not included in
+        # the message body.
+        return argv1.unpack('M').first if boundary00.size == 0
+        return argv1.unpack('M').first unless argv1 =~ ReE[:'quoted-print']
+
+        boundary01 = Sisimai::MIME.boundary(heads['content-type'], 1)
+        reboundary = {
+          :begin => Regexp.new('\A' + Regexp.escape(boundary00)),
+          :until => Regexp.new(Regexp.escape(boundary01) + '\z')
+        }
+        bodystring = ''
+        notdecoded = ''
+        getencoded = ''
+
+        encodename = nil
+        ctencoding = nil
+        mimeinside = false
+
+        argv1.split("\n").each do |e|
+          # This is a multi-part message in MIME format. Your mail reader does not
+          # understand MIME message format.
+          # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
+          # Content-Type: text/plain; charset=iso-8859-15
+          # Content-Transfer-Encoding: quoted-printable
+          if mimeinside 
+            # Quoted-Printable encoded text block
+            if e =~ reboundary[:begin]
+              # The next boundary string has appeared
+              # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
+              getencoded = Sisimai::String.to_utf8(notdecoded.unpack('M').first, encodename)
+
+              bodystring += getencoded
+              bodystring += e + "\n"
+
+              notdecoded = ''
+              mimeinside = false
+              ctencoding = false
+              encodename = nil
+            else
+              # Inside of Queoted printable encoded text
+              notdecoded += e + "\n"
+            end
+          else
+            # NOT Quoted-Printable encoded text block
+            if e =~ /\A[-]{2}[^\s]+[^-]\z/
+              # Start of the boundary block
+              # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
+              unless e == boundary00
+                # New boundary string has appeared
+                boundary00 = e
+                boundary01 = e + '--'
+                reboundary = {
+                  :begin => Regexp.new('\A' + Regexp.escape(boundary00)),
+                  :until => Regexp.new(Regexp.escape(boundary01) + '\z')
+                }
+              end
+            elsif cv = e.match(ReE[:'with-charset']) ||
+                  cv = e.match(ReE[:'only-charset'])
+              # Content-Type: text/plain; charset=ISO-2022-JP
+              encodename = cv[1]
+              mimeinside = true if ctencoding
+
+            elsif e =~ ReE[:'quoted-print']
+              # Content-Transfer-Encoding: quoted-printable
+              ctencoding = true
+              mimeinside = true if encodename
+
+            elsif e =~ reboundary[:until]
+              # The end of boundary block
+              # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ--
+              mimeinside = false
+            end
+
+            bodystring += e + "\n"
+          end
+        end
+
+        return bodystring
       end
 
       # Decode MIME BASE64 Encoded string
@@ -106,7 +222,7 @@ module Sisimai
       # @return   [String] Boundary string
       def boundary(argv1 = nil, start = -1)
         return nil unless argv1
-        value = nil
+        value = ''
 
         if cv = argv1.match(/\bboundary=([^ ]+)/)
           # Content-Type: multipart/mixed; boundary=Apple-Mail-5--931376066
@@ -114,10 +230,10 @@ module Sisimai
           #    boundary="n6H9lKZh014511.1247824040/mx.example.jp"
           value = cv[1]
           value = value.delete(%q|'"|)
+          value = sprintf('--%s', value) if start > -1
+          value = sprintf('%s--', value) if start >  0
         end
 
-        value = sprintf('--%s', value) if start > -1
-        value = sprintf('%s--', value) if start >  0
         return value
       end
     end

data/lib/sisimai/msp/us/amazonworkmail.rb

@@ -56,17 +56,6 @@ module Sisimai
           end
           return nil if match < 2
 
-          if mbody =~ /^Content-Transfer-Encoding: quoted-printable$/
-            # This is a multi-part message in MIME format. Your mail reader does not
-            # understand MIME message format.
-            # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
-            # Content-Type: text/plain; charset=iso-8859-15
-            # Content-Transfer-Encoding: quoted-printable
-            require 'sisimai/mime'
-            mbody = mbody.sub(/\A.+?quoted-printable/ms, '')
-            mbody = Sisimai::MIME.qprintd(mbody)
-          end
-
           dscontents = [Sisimai::MSP.DELIVERYSTATUS]
           hasdivided = mbody.split("\n")
           rfc822list = []     # (Array) Each line in message/rfc822 part string

data/lib/sisimai/msp/us/office365.rb

@@ -31,7 +31,7 @@ module Sisimai
           %r/\A4[.]7[.]26\z/       => 'securityerror',
           %r/\A4[.]7[.][56]\d\d\z/ => 'blocked',
           %r/\A4[.]7[.]8[5-9]\d\z/ => 'blocked',
-          %r/\A5[.]1[.]0\z/        => 'rejected',
+          %r/\A5[.]1[.]0\z/        => 'userunknown',
           %r/\A5[.]4[.]1\z/        => 'norelaying',
           %r/\A5[.]4[.]6\z/        => 'networkerror',
           %r/\A5[.]6[.]11\z/       => 'contenterror',
@@ -108,17 +108,6 @@ module Sisimai
           end
           return nil if match < 2
 
-          if mbody =~ /^Content-Transfer-Encoding: quoted-printable$/
-            # This is a multi-part message in MIME format. Your mail reader does not
-            # understand MIME message format.
-            # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
-            # Content-Type: text/plain; charset=iso-8859-15
-            # Content-Transfer-Encoding: quoted-printable
-            require 'sisimai/mime'
-            mbody = mbody.sub(/\A.+?quoted-printable/ms, '')
-            mbody = Sisimai::MIME.qprintd(mbody)
-          end
-
           dscontents = [Sisimai::MSP.DELIVERYSTATUS]
           hasdivided = mbody.split("\n")
           rfc822list = []     # (Array) Each line in message/rfc822 part string

data/lib/sisimai/mta/exchange2007.rb

@@ -21,9 +21,18 @@ module Sisimai
           :endof  => %r/\A__END_OF_EMAIL_MESSAGE__\z/,
         }
         NDRSubject = {
-          :'RESOLVER.ADR.RecipNotFound'     => 'userunknown',
-          :'RESOLVER.RST.RecipSizeLimit'    => 'mesgtoobig',
-          :'QUEUE.Expired'                  => 'expired',
+          :'SMTPSEND.DNS.NonExistentDomain'=> 'hostunknown',   # 554 5.4.4 SMTPSEND.DNS.NonExistentDomain
+          :'SMTPSEND.DNS.MxLoopback'       => 'networkerror',  # 554 5.4.4 SMTPSEND.DNS.MxLoopback
+          :'RESOLVER.ADR.BadPrimary'       => 'systemerror',   # 550 5.2.0 RESOLVER.ADR.BadPrimary
+          :'RESOLVER.ADR.RecipNotFound'    => 'userunknown',   # 550 5.1.1 RESOLVER.ADR.RecipNotFound
+          :'RESOLVER.ADR.ExRecipNotFound'  => 'userunknown',   # 550 5.1.1 RESOLVER.ADR.ExRecipNotFound
+          :'RESOLVER.ADR.RecipLimit'       => 'toomanyconn',   # 550 5.5.3 RESOLVER.ADR.RecipLimit
+          :'RESOLVER.ADR.InvalidInSmtp'    => 'systemerror',   # 550 5.1.0 RESOLVER.ADR.InvalidInSmtp
+          :'RESOLVER.ADR.Ambiguous'        => 'systemerror',   # 550 5.1.4 RESOLVER.ADR.Ambiguous, 420 4.2.0 RESOLVER.ADR.Ambiguous
+          :'RESOLVER.RST.AuthRequired'     => 'filtered',      # 550 5.7.1 RESOLVER.RST.AuthRequired
+          :'RESOLVER.RST.NotAuthorized'    => 'rejected',      # 550 5.7.1 RESOLVER.RST.NotAuthorized
+          :'RESOLVER.RST.RecipSizeLimit'   => 'mesgtoobig',    # 550 5.2.3 RESOLVER.RST.RecipSizeLimit
+          :'QUEUE.Expired'                 => 'expired',       # 550 4.4.7 QUEUE.Expired
         }
         Indicators = Sisimai::MTA.INDICATORS
 
@@ -112,6 +121,7 @@ module Sisimai
                     v = dscontents[-1]
                   end
                   v['recipient'] = cv[1]
+                  v['diagnosis'] = ''
                   recipients += 1
 
                 elsif cv = e.match(/\A[#]([45]\d{2})[ ]([45][.]\d[.]\d+)[ ].+\z/)

data/lib/sisimai/order.rb

@@ -187,6 +187,7 @@ module Sisimai
         rv.concat(AnotherList3)
         rv.concat(AnotherList4)
         rv.concat(AnotherList5)
+        rv.concat(AnotherList9)
         return rv
       end
 

data/lib/sisimai/reason/norelaying.rb

@@ -29,6 +29,7 @@ module Sisimai
             |mail[ ]server[ ]requires[ ]authentication[ ]when[ ]attempting[ ]to[ ]
               send[ ]to[ ]a[ ]non-local[ ]e-mail[ ]address    # MailEnable
             |not[ ]allowed[ ]to[ ]relay[ ]through[ ]this[ ]machine
+            |Not[ ]an[ ]open[ ]relay,[ ]so[ ]get[ ]lost
             |relay[ ](?:
                access[ ]denied
               |denied

data/lib/sisimai/reason/rejected.rb

@@ -44,6 +44,7 @@ module Sisimai
             |invalid[ ]domain,[ ]see[ ][<]url:.+[>]
             |mx[ ]records[ ]for[ ].+[ ]violate[ ]section[ ].+
             |name[ ]service[ ]error[ ]for[ ]    # Malformed MX RR or host not found
+            |Null[ ]Sender[ ]is[ ]not[ ]allowed
             |recipient[ ]not[ ]accepted[.][ ][(]batv:[ ]no[ ]tag
             |returned[ ]mail[ ]not[ ]accepted[ ]here
             |rfc[ ]1035[ ]violation:[ ]recursive[ ]cname[ ]records[ ]for

data/lib/sisimai/reason/securityerror.rb

@@ -44,10 +44,12 @@ module Sisimai
               # http://kb.mimecast.com/Mimecast_Knowledge_Base/Administration_Console/Monitoring/Mimecast_SMTP_Error_Codes#554
               |rejected[ ]due[ ]to[ ]security[ ]policies
               )
+            |Executable[ ]files[ ]are[ ]not[ ]allowed[ ]in[ ]compressed[ ]files
             |insecure[ ]mail[ ]relay
             |sorry,[ ]you[ ]don'?t[ ]authenticate[ ]or[ ]the[ ]domain[ ]isn'?t[ ]in[ ]my[ ]list[ ]of[ ]allowed[ ]rcpthosts
             |the[ ]message[ ]was[ ]rejected[ ]because[ ]it[ ]contains[ ]prohibited[ ]virus[ ]or[ ]spam[ ]content
             |TLS[ ]required[ ]but[ ]not[ ]supported # SendGrid
+            |you[ ]are[ ]not[ ]authorized[ ]to[ ]send[ ]mail,[ ]authentication[ ]is[ ]required
             |You[ ]have[ ]exceeded[ ]the[ ]the[ ]allowable[ ]number[ ]of[ ]posts[ ]without[ ]solving[ ]a[ ]captcha
             |verification[ ]failure
             )

data/lib/sisimai/string.rb

@@ -59,6 +59,72 @@ module Sisimai
         return argv1
       end
 
+      # Convert given HTML text to plain text
+      # @param    [String]  argv1 HTML text
+      # @param    [Boolean] loose Loose check flag
+      # @return   [String]  Plain text
+      def to_plain(argv1 = '', loose = false)
+        return '' if argv1.empty?
+
+        plain = argv1
+        match = {
+          :html => %r|<html[ >].+?</html>|sim,
+          :body => %r|<head>.+</head>.*<body[ >].+</body>|sim,
+        }
+
+        if loose || plain =~ match[:html] || plain =~ match[:body]
+          # <html> ... </html>
+          # Rewrite <a> elements
+          # 1. <a href = 'http://...'>...</a> to " http://... "
+          # 2. <a href = 'mailto:...'>...</a> to " Value <mailto:...> "
+          plain = plain.gsub(%r|<a\s+href\s*=\s*['"](https?://.+?)['"].*?>(.*?)</a>|i, '[\2](\1)')
+          plain = plain.gsub(%r|<a\s+href\s*=\s*["']mailto:([^\s]+?)["']>(.*?)</a>|i, '[\2](mailto:\1)')
+
+          plain = plain.gsub(/<[^<@>]+?>\s*/,' ') # Delete HTML tags except <neko@example.jp>
+          plain = plain.gsub(/&lt;/, '<')         # Convert to left angle brackets
+          plain = plain.gsub(/&gt;/, '>')         # Convert to right angle brackets
+          plain = plain.gsub(/&amp;/, '&')        # Convert to "&"
+          plain = plain.gsub(/&quot;/, '"')       # Convert to '"'
+          plain = plain.gsub(/&apos;/, "'")       # Convert to "'"
+          plain = plain.gsub(/&nbsp;/, ' ')       # Convert to ' '
+
+          if argv1.size > plain.size
+            plain  = plain.squeeze(' ')
+            plain += "\n"
+          end
+        end
+
+        return plain
+      end
+
+      # Convert given string to UTF-8
+      # @param    [String] argv1  String to be converted
+      # @param    [String] argv2  Encoding name before converting
+      # @return   [String]        UTF-8 Encoded string
+      def to_utf8(argv1 = '', argv2 = nil)
+        return '' unless argv1.size > 0
+
+        encodefrom = argv2 || false
+        getencoded = ''
+
+        begin
+          # Try to convert the string to UTF-8
+          if encodefrom
+            # String#encode('UTF-8', <FROM>)
+            getencoded = argv1.encode('UTF-8', encodefrom)
+          else
+            # Force encoding to UTF-8
+            getencoded = argv1.force_encoding('UTF-8')
+          end
+        rescue
+          # Unknown encoding name or failed to encode
+          getencoded = argv1.force_encoding('UTF-8')
+        end
+        getencoded = getencoded.scrub('?')
+
+        return getencoded
+      end
+
     end
   end
 end

data/lib/sisimai/version.rb

@@ -1,4 +1,4 @@
 # Define the version number of Sisimai
 module Sisimai
-  VERSION = '4.18.0'
+  VERSION = '4.18.1'
 end

data/set-of-emails/maildir/bsd/sendmail-40.eml

@@ -0,0 +1,71 @@
+Return-Path: <>
+Received: from mx1.example.jp (mx1.example.jp [192.0.2.111])
+	by 9jo.example.org (V8/cf) with ESMTP id 00000000000000
+	for <shironeko@example.org>; Thu, 29 Apr 2014 23:34:45 +0900
+X-SenderID: Sendmail Sender-ID Filter v1.0.0 9jo.example.org sAH7CLnA005966
+Authentication-Results: 9jo.example.org; sender-id=none header.from=MAILER-DAEMON@mx1.example.jp
+Received: from localhost (localhost)
+	by mx1.example.jp (V8/cf) id 00000000000000; Thu, 29 Apr 2014 23:34:45 +0900
+Date: Thu, 29 Apr 2014 23:34:45 +0900
+From: Mail Delivery Subsystem <MAILER-DAEMON@mx1.example.jp>
+Message-Id: <20140429233445.00000000000000@mx1.example.jp>
+To: postmaster@mx1.example.jp
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="f00000000000.000000000/mx1.example.jp"
+Subject: Postmaster notify: see transcript for details
+Auto-Submitted: auto-generated (postmaster-notification)
+
+This is a MIME-encapsulated message
+
+--f00000000000.000000000/mx1.example.jp
+
+The original message was received at Thu, 29 Apr 2014 23:34:45 +0900
+from localhost [127.0.0.1]
+with id 0000000000000
+
+   ----- The following addresses had permanent fatal errors -----
+<kijitora@ntt.example.ne.jp>
+    (reason: 550 Unknown user kijitora@ntt.example.ne.jp)
+
+   ----- Transcript of session follows -----
+... while talking to mfsmax.ntt.example.ne.jp.:
+>>> DATA
+<<< 550 Unknown user kijitora@ntt.example.ne.jp
+554 5.0.0 Service unavailable
+
+--f00000000000.000000000/mx1.example.jp
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; mx1.example.jp
+Received-From-MTA: DNS; localhost
+Arrival-Date: Thu, 29 Apr 2014 23:34:45 +0900
+
+Final-Recipient: RFC822; kijitora@ntt.example.ne.jp
+Action: failed
+Status: 5.2.0
+Remote-MTA: DNS; mfsmax.ntt.example.ne.jp
+Diagnostic-Code: SMTP; 550 Unknown user kijitora@ntt.example.ne.jp
+Last-Attempt-Date: Thu, 29 Apr 2014 23:34:45 +0900
+
+--f00000000000.000000000/mx1.example.jp
+Content-Type: text/rfc822-headers
+
+Return-Path: <httpd@mx1.example.jp>
+Received: from mx1.example.jp (localhost [127.0.0.1])
+	by mx1.example.jp (V8/cf) with ESMTP id 0000000000000;
+	Thu, 29 Apr 2014 23:34:45 +0900
+Received: (from httpd@example.com)
+	by mx1.example.jp (8.14.9/8.14.9/Submit) id 00000000000000;
+	Thu, 29 Apr 2014 23:34:45 +0900
+Message-Id: <20140429122334.00000000000000@mx1.example.jp>
+To: Nyanko <kijitora@ntt.example.ne.jp>
+Subject: =?ISO-2022-JP?B?GyRCIVowbEE3IVsbKEIgGyRCMnEwd0VQTz8kTkZiTUYkciQ0M04bKEI=?=  =?ISO-2022-JP?B?GyRCRyckLyRAJDUkJCEjGyhC?=
+MIME-Version: 1.0
+From: Shironeko <shironeko@example.org>
+Date: Thu, 29 Apr 2014 23:34:45 +0900
+Content-Transfer-Encoding: 7bit
+Content-Type: text/plain; charset="ISO-2022-JP"
+
+--f00000000000.000000000/mx1.example.jp--
+

data/sisimai-java.gemspec

@@ -25,5 +25,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 1.8'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 0'
-  spec.add_runtime_dependency 'jrjackson','~> 0.3','>= 0.3.8'
+
+  if RUBY_PLATFORM == 'java'
+    spec.add_runtime_dependency 'jrjackson','~> 0.3', '>= 0.3.8'
+  else
+    spec.add_runtime_dependency 'oj', '~> 2.14', '>= 2.14.4'
+  end
 end

data/sisimai.gemspec

@@ -24,5 +24,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler', '~> 1.8'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '~> 0'
-  spec.add_runtime_dependency 'oj', '~> 2.14', '>= 2.14.4'
+
+  if RUBY_PLATFORM == 'java'
+    spec.add_runtime_dependency 'jrjackson','~> 0.3', '>= 0.3.8'
+  else
+    spec.add_runtime_dependency 'oj', '~> 2.14', '>= 2.14.4'
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sisimai
 version: !ruby/object:Gem::Version
-  version: 4.18.0
+  version: 4.18.1
 platform: java
 authors:
 - azumakuniyuki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-22 00:00:00.000000000 Z
+date: 2016-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -53,25 +53,25 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: jrjackson
+  name: oj
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '2.14'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.8
+        version: 2.14.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '2.14'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.8
+        version: 2.14.4
 description: Sisimai is a Ruby library for analyzing RFC5322 bounce emails and generating
   structured data from parsed results.
 email:
@@ -431,6 +431,7 @@ files:
 - set-of-emails/maildir/bsd/sendmail-37.eml
 - set-of-emails/maildir/bsd/sendmail-38.eml
 - set-of-emails/maildir/bsd/sendmail-39.eml
+- set-of-emails/maildir/bsd/sendmail-40.eml
 - set-of-emails/maildir/bsd/surfcontrol-01.eml
 - set-of-emails/maildir/bsd/surfcontrol-02.eml
 - set-of-emails/maildir/bsd/surfcontrol-03.eml