sisimai 4.22.4 → 4.22.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d77b9e4c9f85d9c719c82e92a91d56a2dd504b10
-  data.tar.gz: 1b7309ee1aaedaa7a5119dc371d8ca1fdd5b1a10
+  metadata.gz: 383458bf9c7149bd37111bb9f8c3a447e8077a6c
+  data.tar.gz: a145574332b0193f166ac827a9f55ef1d4c21f6f
 SHA512:
-  metadata.gz: 26a656c7a48b5785f0515ad3c2df2ab595dc826f1c2a94cefa30f5eccd4f94b99b9449b853b0d8c1b760fb1f4299708cdd20ae57bd329267eb862d5ad4ba08a1
-  data.tar.gz: a4a6f09a0b3cf3e005a3acc8076bdbdae0ce598cc4efd20b7a374b6c3db56307f436ba1a723b9984afb42c9d00dda686d90179edc46d905c9974fd6ad6b74206
+  metadata.gz: e453822ac07cf13914b0f62ad02ef664e8f020d22a5c15f5947788f5e7588b4f17b3debfbbbd23d55b3207cd8ee18367c783f7c69976225959b56a81e7cc25e1
+  data.tar.gz: 93550044f8ee76e04dc4c8d9a53dfe0a1215d4c0f980ec60a9f7ffb4ab61dba857ce6ae511a9a51c7a56edd20de6f9aea0e0d7987009da3e989374b203053d4b

data/ChangeLog.md

@@ -3,6 +3,19 @@ RELEASE NOTES for Ruby version of Sisimai
 - releases: "https://github.com/sisimai/rb-Sisimai/releases"
 - download: "https://rubygems.org/gems/sisimai"
 
+v4.22.5
+--------------------------------------------------------------------------------
+- release: "Fri, 30 Mar 2018 12:29:36 +0900 (JST)"
+- version: "4.22.5"
+- changes:
+  - #112 The order to load MTA modules improvement.
+  - Sample emails in set-of-emails/ which are not owned by Sisimai project have
+    been removed.
+  - Update error message patterns in Sisimai::Reason::Expired.
+  - Less regular expression in each child class of Sisimai::Reason #113.
+  - Pre-Updates for au.com, the new domain of EZweb from Apr 2018 announced at
+    http://news.kddi.com/kddi/corporate/newsrelease/2017/08/22/2637.html #114
+
 v4.22.4
 --------------------------------------------------------------------------------
 - release: "Wed, 14 Feb 2018 10:44:00 +0900 (JST)"

data/README-JA.md

@@ -239,8 +239,8 @@ Differences between Ruby version and Perl version
 | メール解析速度(1000通のメール)              | 5.05秒         | 1.95秒        |
 | インストール方法                            | gem install    | cpanm, cpm    |
 | 依存モジュール数(コアモジュールを除く)      | 1モジュール    | 2モジュール   |
-| LOC:ソースコードの行数                      | 12000行        | 8700行        |
-| テスト件数(spec/,t/,xt/ディレクトリ)        | 213000件       | 230000件      |
+| LOC:ソースコードの行数                      | 12000行        | 8600行        |
+| テスト件数(spec/,t/,xt/ディレクトリ)        | 207000件       | 224000件      |
 | ライセンス                                  | 二条項BSD      | 二条項BSD     |
 | 開発会社によるサポート契約                  | 準備中         | 提供中        |
 

data/README.md

@@ -243,8 +243,8 @@ and bounceHammer are available at
 | The speed of parsing email(1000 emails)     | 5.05s          | 1.95s         |
 | How to install                              | gem install    | cpanm, cpm    |
 | Dependencies (Except core modules)          | 1 module       | 2 modules     |
-| LOC:Source lines of code                    | 12000 lines    | 8700 lines    |
-| The number of tests(spec/,t/,xt/) directory | 213000 tests   | 230000 tests  |
+| LOC:Source lines of code                    | 12000 lines    | 8600 lines    |
+| The number of tests(spec/,t/,xt/) directory | 207000 tests   | 224000 tests  |
 | License                                     | BSD 2-Clause   | BSD 2-Clause  |
 | Support Contract provided by Developer      | Coming soon    | Available     |
 

data/lib/sisimai.rb

@@ -31,7 +31,7 @@ module Sisimai
 
       input = argv1[:input] || nil
       field = argv1[:field] || []
-      raise ' ***error: "field" accepts an array reference only' unless field.is_a? Array
+      raise ' ***error: "field" accepts an array only' unless field.is_a? Array
 
       unless input
         # "input" did not specified, try to detect automatically.
@@ -114,7 +114,7 @@ module Sisimai
       return nil unless argv0
 
       nyaan = Sisimai.make(argv0, argv1) || []
-      if RUBY_PLATFORM.include?('java')
+      if RUBY_PLATFORM.start_with?('java')
         # java-based ruby environment like JRuby.
         require 'jrjackson'
         jsonstring = JrJackson::Json.dump(nyaan)

data/lib/sisimai/arf.rb

@@ -46,7 +46,7 @@ module Sisimai
           # Content-Type: multipart/report; report-type=feedback-report; ...
           match = true
 
-        elsif heads['content-type'].include?('multipart/mixed')
+        elsif heads['content-type'].start_with?('multipart/mixed')
           # Microsoft (Hotmail, MSN, Live, Outlook) uses its own report format.
           # Amazon SES Complaints bounces
           mfrom = %r{(?:

data/lib/sisimai/bite/email.rb

@@ -30,6 +30,17 @@ module Sisimai
             X1 X2 X3 X4 X5 V5sendmail FML]
         end
 
+        # @abstract MTA list which have one or more extra headers
+        # @return   [Array] MTA list (have extra headers)
+        def heads
+          return %w[
+            Exim Exchange2007 Exchange2003 Google GSuite Office365 Outlook SendGrid
+            AmazonSES ReceivingSES AmazonWorkMail Aol GMX MailRu MessageLabs Yahoo
+            Yandex Zoho EinsUndEins MXLogic McAfee MFILTER EZweb Activehunter IMailServer
+            SurfControl FML
+          ]
+        end
+
         # @abstract Parse bounce messages
         # @param         [Hash] mhead       Message header of a bounce email
         # @options mhead [String] from      From header

data/lib/sisimai/bite/email/amazonses.rb

@@ -41,7 +41,7 @@ module Sisimai::Bite::Email
 
         # :from    => %r/\AMAILER-DAEMON[@]email[-]bounces[.]amazonses[.]com\z/,
         # :subject => %r/\ADelivery Status Notification [(]Failure[)]\z/,
-        return nil if mhead['x-mailer'].to_s.include?('Amazon WorkMail')
+        return nil if mhead['x-mailer'].to_s.start_with?('Amazon WorkMail')
 
         match  = 0
         match += 1 if mhead['x-aws-outgoing']

data/lib/sisimai/bite/email/ezweb.rb

@@ -59,10 +59,12 @@ module Sisimai::Bite::Email
 
         match  = 0
         match += 1 if mhead['from'].include?('Postmaster@ezweb.ne.jp')
+        match += 1 if mhead['from'].include?('Postmaster@au.com')
         match += 1 if mhead['subject'] == 'Mail System Error - Returned Mail'
         match += 1 if mhead['received'].find { |a| a.include?('ezweb.ne.jp (EZweb Mail) with') }
+        match += 1 if mhead['received'].find { |a| a.include?('.au.com (') }
         if mhead['message-id']
-          match += 1 if mhead['message-id'].end_with?('.ezweb.ne.jp>')
+          match += 1 if mhead['message-id'].end_with?('.ezweb.ne.jp>', '.au.com>')
         end
         return nil if match < 2
 
@@ -227,7 +229,7 @@ module Sisimai::Bite::Email
             # The value of "reason" is not set yet.
             # Deal as "userunknown" when the domain part of the recipient
             # is "ezweb.ne.jp".
-            e['reason'] = 'userunknown' unless e['recipient'].end_with?('@ezweb.ne.jp')
+            e['reason'] = 'userunknown' unless e['recipient'].end_with?('@ezweb.ne.jp', '@au.com')
           end
           e['agent'] = self.smtpagent
         end

data/lib/sisimai/bite/email/google.rb

@@ -173,8 +173,8 @@ module Sisimai::Bite::Email
         #   The error that the other server returned was:
         #   550 5.1.1 <userunknown@example.jp>... User Unknown
         #
-        return nil unless mhead['from'].include?('<mailer-daemon@googlemail.com>')
-        return nil unless mhead['subject'].include?('Delivery Status Notification')
+        return nil unless mhead['from'].end_with?('<mailer-daemon@googlemail.com>')
+        return nil unless mhead['subject'].start_with?('Delivery Status Notification')
 
         require 'sisimai/address'
         dscontents = [Sisimai::Bite.DELIVERYSTATUS]
@@ -189,7 +189,7 @@ module Sisimai::Bite::Email
         while e = hasdivided.shift do
           if readcursor.zero?
             # Beginning of the bounce message or delivery status part
-            readcursor |= Indicators[:deliverystatus] if e.include?(StartingOf[:message][0])
+            readcursor |= Indicators[:deliverystatus] if e.start_with?(StartingOf[:message][0])
           end
 
           if (readcursor & Indicators[:'message-rfc822']).zero?

data/lib/sisimai/bite/email/gsuite.rb

@@ -38,8 +38,8 @@ module Sisimai::Bite::Email
         return nil unless mhead
         return nil unless mbody
 
-        return nil unless mhead['from'].include?('<mailer-daemon@googlemail.com>')
-        return nil unless mhead['subject'].include?('Delivery Status Notification')
+        return nil unless mhead['from'].end_with?('<mailer-daemon@googlemail.com>')
+        return nil unless mhead['subject'].start_with?('Delivery Status Notification')
         return nil unless mhead['x-gm-message-state']
 
         require 'sisimai/address'

data/lib/sisimai/bite/email/interscanmss.rb

@@ -41,7 +41,7 @@ module Sisimai::Bite::Email
           # メール配信に失敗しました
           '=?iso-2022-jp?B?GyRCJWEhPCVrR1s/LiRLPDpHVCQ3JF4kNyQ/GyhCDQo=?=',
         ]
-        match += 1 if mhead['from'].include?('InterScan MSS')
+        match += 1 if mhead['from'].start_with?('"InterScan MSS"')
         match += 1 if tryto.find { |a| mhead['subject'] == a }
         return nil if match.zero?
 

data/lib/sisimai/bite/email/kddi.rb

@@ -45,6 +45,7 @@ module Sisimai::Bite::Email
         match += 1 if mhead['from'] =~ /no-reply[@].+[.]dion[.]ne[.]jp/
         match += 1 if mhead['reply-to'].to_s == 'no-reply@app.auone-net.jp'
         match += 1 if mhead['received'].find { |a| a.include?('ezweb.ne.jp (') }
+        match += 1 if mhead['received'].find { |a| a.include?('.au.com (') }
         return nil if match.zero?
 
         require 'sisimai/string'

data/lib/sisimai/bite/email/mcafee.rb

@@ -34,7 +34,7 @@ module Sisimai::Bite::Email
         return nil unless mhead
         return nil unless mbody
         return nil unless mhead['x-nai-header']
-        return nil unless mhead['x-nai-header'].include?('Modified by McAfee ')
+        return nil unless mhead['x-nai-header'].start_with?('Modified by McAfee ')
         return nil unless mhead['subject'] == 'Delivery Status'
 
         require 'sisimai/address'

data/lib/sisimai/bite/email/opensmtpd.rb

@@ -33,8 +33,8 @@ module Sisimai::Bite::Email
         #   bounce.c/337:const char *notice_relay =
         #   bounce.c/338:    "    Your message was relayed to these recipients.\n\n";
         #   bounce.c/339:
-        message: [' This is the MAILER-DAEMON, please DO NOT REPLY to this '],
-        rfc822:  [' Below is a copy of the original message:'],
+        message: ['    This is the MAILER-DAEMON, please DO NOT REPLY to this '],
+        rfc822:  ['    Below is a copy of the original message:'],
       }.freeze
       ReFailures = {
         # smtpd/queue.c:221|  envelope_set_errormsg(&evp, "Envelope expired");
@@ -97,7 +97,7 @@ module Sisimai::Bite::Email
         while e = hasdivided.shift do
           if readcursor.zero?
             # Beginning of the bounce message or delivery status part
-            if e.include?(StartingOf[:message][0])
+            if e.start_with?(StartingOf[:message][0])
               readcursor |= Indicators[:deliverystatus]
               next
             end
@@ -105,7 +105,7 @@ module Sisimai::Bite::Email
 
           if (readcursor & Indicators[:'message-rfc822']).zero?
             # Beginning of the original message part
-            if e.include?(StartingOf[:rfc822][0])
+            if e.start_with?(StartingOf[:rfc822][0])
               readcursor |= Indicators[:'message-rfc822']
               next
             end

data/lib/sisimai/bite/email/outlook.rb

@@ -41,7 +41,7 @@ module Sisimai::Bite::Email
 
         # :from => %r/postmaster[@]/,
         match  = 0
-        match += 1 if mhead['subject'].include?('Delivery Status Notification')
+        match += 1 if mhead['subject'].start_with?('Delivery Status Notification')
         match += 1 if mhead['x-message-delivery']
         match += 1 if mhead['x-message-info']
         match += 1 if mhead['received'].find { |a| a.include?('.hotmail.com') }

data/lib/sisimai/bite/email/x1.rb

@@ -31,7 +31,7 @@ module Sisimai::Bite::Email
         return nil unless mhead
         return nil unless mbody
         return nil unless mhead['subject'].start_with?('Returned Mail: ')
-        return nil unless mhead['from'].include?('"Mail Deliver System" ')
+        return nil unless mhead['from'].start_with?('"Mail Deliver System" ')
 
         dscontents = [Sisimai::Bite.DELIVERYSTATUS]
         hasdivided = mbody.split("\n")

data/lib/sisimai/bite/email/x3.rb

@@ -8,7 +8,7 @@ module Sisimai::Bite::Email
 
       Indicators = Sisimai::Bite::Email.INDICATORS
       StartingOf = {
-        message: [' This is an automatically generated Delivery Status Notification'],
+        message: ['      This is an automatically generated Delivery Status Notification.'],
         rfc822:  ['Content-Type: message/rfc822'],
       }.freeze
 
@@ -44,7 +44,7 @@ module Sisimai::Bite::Email
         while e = hasdivided.shift do
           if readcursor.zero?
             # Beginning of the bounce message or delivery status part
-            if e.include?(StartingOf[:message][0])
+            if e.start_with?(StartingOf[:message][0])
               readcursor |= Indicators[:deliverystatus]
               next
             end

data/lib/sisimai/bite/json/amazonses.rb

@@ -68,7 +68,7 @@ module Sisimai::Bite::JSON
         end
 
         begin
-          if RUBY_PLATFORM.include?('java')
+          if RUBY_PLATFORM.start_with?('java')
             # java-based ruby environment like JRuby.
             require 'jrjackson'
             jsonobject = JrJackson::Json.load(jsonstring)

data/lib/sisimai/data/json.rb

@@ -13,7 +13,7 @@ module Sisimai
           return nil unless argvs
           return nil unless argvs.is_a? Sisimai::Data
 
-          if RUBY_PLATFORM.include?('java')
+          if RUBY_PLATFORM.start_with?('java')
             # java-based ruby environment like JRuby.
             begin
               require 'jrjackson'

data/lib/sisimai/message/email.rb

@@ -278,9 +278,10 @@ module Sisimai
 
         # Try to match the value of "Subject" with patterns generated by
         # Sisimai::Order->by('subject') method
+        title = heads['subject'].downcase
         SubjectTab.each_key do |e|
           # Get MTA list from the subject header
-          next unless heads['subject'] =~ e
+          next unless title.include?(e)
 
           # Matched and push MTA list
           order.concat(SubjectTab[e])
@@ -485,12 +486,7 @@ module Sisimai
             tobeloaded.each do |r|
               # Call user defined MTA modules
               next if haveloaded[r]
-              begin
-                require r.gsub('::', '/').downcase
-              rescue LoadError => ce
-                warn ' ***warning: Failed to load ' << ce.to_s
-                next
-              end
+              #require r.gsub('::', '/').downcase
               scannedset = Module.const_get(r).scan(mailheader, bodystring)
               haveloaded[r] = true
               throw :SCANNER if scannedset
@@ -500,12 +496,7 @@ module Sisimai
               # Try MTA module candidates which are detected from MTA specific
               # mail headers on first
               next if haveloaded.key?(r)
-              begin
-                require r.gsub('::', '/').downcase
-              rescue LoadError => ce
-                warn ' ***warning: ' << ce.to_s
-                next
-              end
+              require r.gsub('::', '/').downcase
               scannedset = Module.const_get(r).scan(mailheader, bodystring)
               haveloaded[r] = true
               throw :SCANNER if scannedset
@@ -515,15 +506,10 @@ module Sisimai
               # MTA modules which does not have MTA specific header and did
               # not match with any regular expressions of Subject header.
               next if haveloaded.key?(r)
-              begin
-                require r.gsub('::', '/').downcase
-                scannedset = Module.const_get(r).scan(mailheader, bodystring)
-                haveloaded[r] = true
-                throw :SCANNER if scannedset
-              rescue => ce
-                warn ' ***warning: ' << ce.to_s
-                next
-              end
+              require r.gsub('::', '/').downcase
+              scannedset = Module.const_get(r).scan(mailheader, bodystring)
+              haveloaded[r] = true
+              throw :SCANNER if scannedset
             end
 
             # When the all of Sisimai::Bite::Email::* modules did not return

data/lib/sisimai/order/email.rb

@@ -85,7 +85,7 @@ module Sisimai
         # header such as X-AWS-Outgoing, X-Yandex-Uniq.
         PatternTable = {
           'subject' => {
-            %r/delivery/i => [
+            'delivery' => [
               'Sisimai::Bite::Email::Exim',
               'Sisimai::Bite::Email::Courier',
               'Sisimai::Bite::Email::Google',
@@ -99,7 +99,7 @@ module Sisimai
               'Sisimai::Bite::Email::X3',
               'Sisimai::Bite::Email::X2',
             ],
-            %r/noti(?:ce|fi)/i => [
+            'noti' => [
               'Sisimai::Bite::Email::Qmail',
               'Sisimai::Bite::Email::Sendmail',
               'Sisimai::Bite::Email::Google',
@@ -111,7 +111,7 @@ module Sisimai
               'Sisimai::Bite::Email::X3',
               'Sisimai::Bite::Email::MFILTER',
             ],
-            %r/return/i => [
+            'return' => [
               'Sisimai::Bite::Email::Postfix',
               'Sisimai::Bite::Email::Sendmail',
               'Sisimai::Bite::Email::SendGrid',
@@ -121,7 +121,7 @@ module Sisimai
               'Sisimai::Bite::Email::Biglobe',
               'Sisimai::Bite::Email::V5sendmail',
             ],
-            %r/undeliver/i => [
+            'undeliver' => [
               'Sisimai::Bite::Email::Postfix',
               'Sisimai::Bite::Email::Exchange2007',
               'Sisimai::Bite::Email::Exchange2003',
@@ -132,7 +132,7 @@ module Sisimai
               'Sisimai::Bite::Email::IMailServer',
               'Sisimai::Bite::Email::MailMarshalSMTP',
             ],
-            %r/failure/i => [
+            'failure' => [
               'Sisimai::Bite::Email::Qmail',
               'Sisimai::Bite::Email::Domino',
               'Sisimai::Bite::Email::Google',
@@ -142,7 +142,7 @@ module Sisimai
               'Sisimai::Bite::Email::X2',
               'Sisimai::Bite::Email::MFILTER',
             ],
-            %r/warning/i => [
+            'warning' => [
               'Sisimai::Bite::Email::Postfix',
               'Sisimai::Bite::Email::Sendmail',
               'Sisimai::Bite::Email::Exim',
@@ -150,23 +150,10 @@ module Sisimai
           },
         }.freeze
 
-        make_default_order = lambda do
-          # Make default order of MTA modules to be loaded
-          rv = []
-          begin
-            rv.concat(Sisimai::Bite::Email.index.map { |e| 'Sisimai::Bite::Email::' << e })
-          rescue
-            # Failed to load an MTA module
-            next
-          end
-          return rv
-        end
-        DefaultOrder = make_default_order.call
-
         # @abstract Make default order of MTA modules to be loaded
         # @return   [Array] Default order list of MTA modules
         def default
-          return DefaultOrder
+          return Sisimai::Bite::Email.index.map { |e| 'Sisimai::Bite::Email::' << e }
         end
 
         # @abstract Get regular expression patterns for specified field
@@ -194,18 +181,13 @@ module Sisimai
         # @abstract Make email header list in each MTA module
         # @return   [Hash] Header list to be parsed
         def headers
-          order = self.default
+          order = Sisimai::Bite::Email.heads.map { |e| 'Sisimai::Bite::Email::' << e }
           table = {}
           skips = { 'return-path' => 1, 'x-mailer' => 1 }
 
           order.each do |e|
             # Load email headers from each MTA module
-            begin
-              require e.gsub('::', '/').downcase
-            rescue LoadError
-              warn ' ***warning: Failed to load ' << e
-              next
-            end
+            require e.gsub('::', '/').downcase
 
             Module.const_get(e).headerlist.each do |v|
               # Get header name which required each MTA module

data/lib/sisimai/reason/contenterror.rb

@@ -19,22 +19,16 @@ module Sisimai
         #                           true: Matched
         def match(argv1)
           return nil unless argv1
-          regex = %r{(?>
-             improper[ ]use[ ]of[ ]8-bit[ ]data[ ]in[ ]message[ ]header
-            |message[ ](?:
-               header[ ]size,[ ]or[ ]recipient[ ]list,[ ]exceeds[ ]policy[ ]limit
-              |mime[ ]complexity[ ]exceeds[ ]the[ ]policy[ ]maximum
-              )
-            |routing[ ]loop[ ]detected[ ][-][-][ ]too[ ]many[ ]received:[ ]headers
-            |this[ ]message[ ]contains?[ ](?:
-               invalid[ ]mime[ ]headers
-              |improperly[-]formatted[ ]binary[ ]content
-              |text[ ]that[ ]uses[ ]unnecessary[ ]base64[ ]encoding
-              )
-            )
-          }x
-
-          return true if argv1 =~ regex
+          index = [
+            'improper use of 8-bit data in message header',
+            'message header size, or recipient list, exceeds policy limit',
+            'message mime complexity exceeds the policy maximum',
+            'routing loop detected -- too many received: headers',
+            'this message contain invalid mime headers',
+            'this message contain improperly-formatted binary content',
+            'this message contain text that uses unnecessary base64 encoding',
+          ]
+          return true if index.find { |a| argv1.include?(a) }
           return false
         end