sisimai 4.24.1-java → 4.25.0-java

data/lib/sisimai/arf.rb

@@ -4,6 +4,7 @@ module Sisimai
     # Imported from p5-Sisimail/lib/Sisimai/ARF.pm
     class << self
       require 'sisimai/bite/email'
+      require 'sisimai/rfc5322'
 
       # http://tools.ietf.org/html/rfc5965
       # http://en.wikipedia.org/wiki/Feedback_loop_(email)
@@ -48,12 +49,13 @@ module Sisimai
         elsif heads['content-type'].start_with?('multipart/mixed')
           # Microsoft (Hotmail, MSN, Live, Outlook) uses its own report format.
           # Amazon SES Complaints bounces
+          hfrom = Sisimai::Address.s3s4(heads['from'])
           mfrom = %r{(?:
              staff[@]hotmail[.]com
             |complaints[@]email-abuse[.]amazonses[.]com
             )\z
           }x
-          if heads['from'] =~ mfrom && heads['subject'].include?('complaint about message from ')
+          if hfrom =~ mfrom && heads['subject'].include?('complaint about message from ')
             # From: staff@hotmail.com
             # From: complaints@email-abuse.amazonses.com
             # Subject: complaint about message from 192.0.2.1
@@ -85,16 +87,16 @@ module Sisimai
         recipients = 0    # (Integer) The number of 'Final-Recipient' header
         rcptintext = ''   # (String) Recipient address in the message body
         commondata = {
-          :diagnosis => '',  # Error message
-          :from      => '',  # Original-Mail-From:
-          :rhost     => '',  # Reporting-MTA:
+          'diagnosis' => '',  # Error message
+          'from'      => '',  # Original-Mail-From:
+          'rhost'     => '',  # Reporting-MTA:
         }
         arfheaders = {
-          :feedbacktype => nil,   # FeedBack-Type:
-          :rhost        => nil,   # Source-IP:
-          :agent        => nil,   # User-Agent:
-          :date         => nil,   # Arrival-Date:
-          :authres      => nil,   # Authentication-Results:
+          'feedbacktype' => nil,   # FeedBack-Type:
+          'rhost'        => nil,   # Source-IP:
+          'agent'        => nil,   # User-Agent:
+          'date'         => nil,   # Arrival-Date:
+          'authres'      => nil,   # Authentication-Results:
         }
         v = nil
 
@@ -120,7 +122,7 @@ module Sisimai
         #
         while e = hasdivided.shift do
           if readcursor == 0
-            # Beginning of the bounce message or delivery status part
+            # Beginning of the bounce message or message/delivery-status part
             if e =~ MarkingsOf[:message]
               readcursor |= Indicators[:deliverystatus]
               next
@@ -136,7 +138,7 @@ module Sisimai
           end
 
           if readcursor & Indicators[:'message-rfc822'] > 0
-            # After "message/rfc822"
+            # message/rfc822 OR text/rfc822-headers part
             if cv = e.match(/X-HmXmrOriginalRecipient:[ ]*(.+)\z/)
               # Microsoft ARF: original recipient.
               dscontents[-1]['recipient'] = Sisimai::Address.s3s4(cv[1])
@@ -145,17 +147,24 @@ module Sisimai
               # The "X-HmXmrOriginalRecipient" header appears only once so
               # we take this opportunity to hard-code ARF headers missing in
               # Microsoft's implementation.
-              arfheaders[:feedbacktype] = 'abuse'
-              arfheaders[:agent] = 'Microsoft Junk Mail Reporting Program'
+              arfheaders['feedbacktype'] = 'abuse'
+              arfheaders['agent'] = 'Microsoft Junk Mail Reporting Program'
 
             elsif cv = e.match(/\AFrom:[ ]*(.+)\z/)
               # Microsoft ARF: original sender.
-              commondata[:from] = Sisimai::Address.s3s4(cv[1]) if commondata[:from].empty?
+              commondata['from'] = Sisimai::Address.s3s4(cv[1]) if commondata['from'].empty?
 
-            elsif cv = e.match(/\A([-0-9A-Za-z]+?)[:][ ]*(.+)\z/)
+            elsif e.start_with?(' ', "\t")
+              # Continued line from the previous line
+              rfc822part << e + "\n" if LongFields.key?(previousfn)
+              next unless e.empty?
+              rcptintext << e if previousfn == 'to'
+
+            else
               # Get required headers only
-              lhs = cv[1].downcase
-              rhs = cv[2]
+              (lhs, rhs) = e.split(/:[ ]*/, 2)
+              next unless lhs
+              lhs.downcase!
 
               previousfn = ''
               next unless RFC822Head.key?(lhs)
@@ -163,15 +172,9 @@ module Sisimai
               previousfn  = lhs
               rfc822part << e + "\n"
               rcptintext  = rhs if lhs == 'to'
-
-            elsif e.start_with?(' ', "\t")
-              # Continued line from the previous line
-              rfc822part << e + "\n" if LongFields.key?(previousfn)
-              next unless e.empty?
-              rcptintext << e if previousfn == 'to'
             end
           else
-            # Before "message/rfc822"
+            # message/delivery-status part
             next unless readcursor & Indicators[:deliverystatus] > 0
             next if e.empty?
 
@@ -201,7 +204,7 @@ module Sisimai
             elsif cv = e.match(/\AFeedback-Type:[ ]*([^ ]+)\z/)
               # The header field MUST appear exactly once.
               # Feedback-Type: abuse
-              arfheaders[:feedbacktype] = cv[1]
+              arfheaders['feedbacktype'] = cv[1]
 
             elsif cv = e.match(/\AAuthentication-Results:[ ]*(.+)\z/)
               # "Authentication-Results" indicates the result of one or more
@@ -209,68 +212,68 @@ module Sisimai
               #
               # Authentication-Results: mail.example.com;
               #   spf=fail smtp.mail=somespammer@example.com
-              arfheaders[:authres] = cv[1]
+              arfheaders['authres'] = cv[1]
 
             elsif cv = e.match(/\AUser-Agent:[ ]*(.+)\z/)
               # The header field MUST appear exactly once.
               # User-Agent: SomeGenerator/1.0
-              arfheaders[:agent] = cv[1]
+              arfheaders['agent'] = cv[1]
 
             elsif cv = e.match(/\A(?:Received|Arrival)-Date:[ ]*(.+)\z/)
               # Arrival-Date header is optional and MUST NOT appear more than
               # once.
               # Received-Date: Thu, 29 Apr 2010 00:00:00 JST
               # Arrival-Date: Thu, 29 Apr 2010 00:00:00 +0000
-              arfheaders[:date] = cv[1]
+              arfheaders['date'] = cv[1]
 
             elsif cv = e.match(/\AReporting-MTA:[ ]*dns;[ ]*(.+)\z/)
               # The header is optional and MUST NOT appear more than once.
               # Reporting-MTA: dns; mx.example.jp
-              commondata[:rhost] = cv[1]
+              commondata['rhost'] = cv[1]
 
             elsif cv = e.match(/\ASource-IP:[ ]*(.+)\z/)
               # The header is optional and MUST NOT appear more than once.
               # Source-IP: 192.0.2.45
-              arfheaders[:rhost] = cv[1]
+              arfheaders['rhost'] = cv[1]
 
             elsif cv = e.match(/\AOriginal-Mail-From:[ ]*(.+)\z/)
               # the header is optional and MUST NOT appear more than once.
               # Original-Mail-From: <somespammer@example.net>
-              commondata[:from] = Sisimai::Address.s3s4(cv[1]) if commondata[:from].empty?
+              commondata['from'] = Sisimai::Address.s3s4(cv[1]) if commondata['from'].empty?
 
             elsif e =~ MarkingsOf[:message]
               # This is an email abuse report for an email message with the
               #   message-id of 0000-000000000000000000000000000000000@mx
               #   received from IP address 192.0.2.1 on
               #   Thu, 29 Apr 2010 00:00:00 +0900 (JST)
-              commondata[:diagnosis] = e
+              commondata['diagnosis'] = e
             end
           end
         end
 
-        if arfheaders[:feedbacktype] == 'auth-failure' && arfheaders[:authres]
+        if arfheaders['feedbacktype'] == 'auth-failure' && arfheaders['authres']
           # Append the value of Authentication-Results header
-          commondata[:diagnosis] << ' ' << arfheaders[:authres]
+          commondata['diagnosis'] << ' ' << arfheaders['authres']
         end
 
         unless recipients > 0
           # Insert pseudo recipient address when there is no valid recipient
           # address in the message.
-          dscontents[-1]['recipient'] = Sisimai::Address.undisclosed(:r)
+          dscontents[-1]['recipient'] = Sisimai::Address.undisclosed('r')
           recipients = 1
         end
 
         unless rfc822part =~ /\bFrom: [^ ]+[@][^ ]+\b/
           # There is no "From:" header in the original message
           # Append the value of "Original-Mail-From" value as a sender address.
-          rfc822part << 'From: ' << commondata[:from] + "\n" unless commondata[:from].empty?
+          rfc822part << 'From: ' << commondata['from'] + "\n" unless commondata['from'].empty?
         end
 
         if cv = mhead['subject'].match(/complaint about message from (\d{1,3}[.]\d{1,3}[.]\d{1,3}[.]\d{1,3})/)
           # Microsoft ARF: remote host address.
-          arfheaders[:rhost] = cv[1]
-          commondata[:diagnosis] = 
-            'This is a Microsoft email abuse report for an email message received from IP' << arfheaders[:rhost] + ' on ' << mhead['date']
+          arfheaders['rhost'] = cv[1]
+          commondata['diagnosis'] = 
+            'This is a Microsoft email abuse report for an email message received from IP' << arfheaders['rhost'] + ' on ' << mhead['date']
         end
 
         dscontents.each do |e|
@@ -278,32 +281,28 @@ module Sisimai
             # AOL = http://forums.cpanel.net/f43/aol-brutal-work-71473.html
             e['recipient'] = Sisimai::Address.s3s4(rcptintext)
           end
-          arfheaders.each_key { |a| e[a.to_s] ||= arfheaders[a] || '' }
+          arfheaders.each_key { |a| e[a] ||= arfheaders[a] || '' }
           e.delete('authres')
 
           e['softbounce'] = -1
-          e['diagnosis']  = commondata[:diagnosis] unless e['diagnosis']
+          e['diagnosis']  = commondata['diagnosis'] unless e['diagnosis']
           e['date']       = mhead['date'] if e['date'].empty?
-
-          if e['rhost'].empty?
-            # Get the remote IP address from the message body
-            if commondata[:rhost].size > 0
-              # The value of "Reporting-MTA" header
-              e['rhost'] = commondata[:rhost]
-
-            elsif cv = e['diagnosis'].match(/\breceived from IP address ([^ ]+)/)
-              # This is an email abuse report for an email message received
-              # from IP address 24.64.1.1 on Thu, 29 Apr 2010 00:00:00 +0000
-              e['rhost'] = cv[1]
-            end
+          e['reason']     = 'feedback'
+          e['agent']      = self.smtpagent if e['agent'].empty?
+          %w[command action status alias].each { |a| e[a] = '' }
+
+          # Get the remote IP address from the message body
+          next unless e['rhost'].empty?
+          if commondata['rhost'].size > 0
+            # The value of "Reporting-MTA" header
+            e['rhost'] = commondata['rhost']
+
+          elsif cv = e['diagnosis'].match(/\breceived from IP address ([^ ]+)/)
+            # This is an email abuse report for an email message received
+            # from IP address 24.64.1.1 on Thu, 29 Apr 2010 00:00:00 +0000
+            e['rhost'] = cv[1]
           end
 
-          e['reason']  = 'feedback'
-          e['command'] = ''
-          e['action']  = ''
-          e['status']  = ''
-          e['alias']   = ''
-          e['agent']   = self.smtpagent if e['agent'].empty?
         end
         return { 'ds' => dscontents, 'rfc822' => rfc822part }
       end

data/lib/sisimai/bite.rb

@@ -30,7 +30,6 @@ module Sisimai
 
       def smtpagent(v = ''); return v.to_s.sub(/\ASisimai::Bite::/, ''); end
       def description; return ''; end
-
     end
   end
 end

data/lib/sisimai/bite/email.rb

@@ -1,4 +1,4 @@
-module Sisimai
+  module Sisimai
   module Bite
     # Sisimai::Bite::Email- Base class for Sisimai::Bite::Email::*
     module Email
@@ -21,23 +21,23 @@ module Sisimai
         # @return   [Array] MTA list with order
         def index
           return %w[
-            Sendmail Postfix Qmail Exim Courier OpenSMTPD Exchange2007 Exchange2003
-            Google Yahoo GSuite Aol Outlook Office365 SendGrid AmazonSES MailRu
+            Sendmail Postfix Qmail Exim Courier OpenSMTPD Office365 Outlook
+            Exchange2007 Exchange2003 Yahoo GSuite Aol SendGrid AmazonSES MailRu
             Yandex MessagingServer Domino Notes ReceivingSES AmazonWorkMail Verizon
             GMX Bigfoot Facebook Zoho EinsUndEins MessageLabs EZweb KDDI Biglobe
-            ApacheJames McAfee MXLogic MailFoundry IMailServer
+            Amavis ApacheJames McAfee MXLogic MailFoundry IMailServer
             MFILTER Activehunter InterScanMSS SurfControl MailMarshalSMTP
-            X1 X2 X3 X4 X5 V5sendmail FML]
+            X1 X2 X3 X4 X5 V5sendmail FML Google]
         end
 
         # @abstract MTA list which have one or more extra headers
         # @return   [Array] MTA list (have extra headers)
         def heads
           return %w[
-            Exim Exchange2007 Exchange2003 Google GSuite Office365 Outlook SendGrid
+            Exim Office365 Outlook Exchange2007 Exchange2003 GSuite SendGrid
             AmazonSES ReceivingSES AmazonWorkMail Aol GMX MailRu MessageLabs Yahoo
             Yandex Zoho EinsUndEins MXLogic McAfee MFILTER EZweb Activehunter IMailServer
-            SurfControl FML
+            SurfControl FML Google
           ]
         end
 

data/lib/sisimai/bite/email/activehunter.rb

@@ -15,7 +15,7 @@ module Sisimai::Bite::Email
 
       def description; return 'TransWARE Active!hunter'; end
       def smtpagent;   return Sisimai::Bite.smtpagent(self); end
-      def headerlist;  return ['X-AHMAILID']; end
+      def headerlist;  return %w[x-ahmailid]; end
 
       # Parse bounce messages from TransWARE Active!hunter
       # @param         [Hash] mhead       Message headers of a bounce email
@@ -59,7 +59,7 @@ module Sisimai::Bite::Email
           end
 
           if readcursor & Indicators[:'message-rfc822'] > 0
-            # After "message/rfc822"
+            # Inside of the original message part
             if e.empty?
               blanklines += 1
               break if blanklines > 1
@@ -67,7 +67,7 @@ module Sisimai::Bite::Email
             end
             rfc822list << e
           else
-            # Before "message/rfc822"
+            # Error message part
             next if (readcursor & Indicators[:deliverystatus]) == 0
             next if e.empty?
 
@@ -100,6 +100,7 @@ module Sisimai::Bite::Email
         end
         return nil unless recipients > 0
 
+        require 'sisimai/string'
         dscontents.each do |e|
           e['diagnosis'] = Sisimai::String.sweep(e['diagnosis'])
           e['agent']     = self.smtpagent

data/lib/sisimai/bite/email/amavis.rb

@@ -0,0 +1,133 @@
+module Sisimai::Bite::Email
+  # Sisimai::Bite::Email::Amavis parses a bounce email which created by
+  # amavsid-new. Methods in the module are called from only Sisimai::Message.
+  module Amavis
+    class << self
+      # Imported from p5-Sisimail/lib/Sisimai/Bite/Email/Amavis.pm
+      require 'sisimai/bite/email'
+
+      Indicators = Sisimai::Bite::Email.INDICATORS
+      StartingOf = {
+        message: ['The message '],
+        rfc822:  ['Content-Type: text/rfc822-headers'],
+      }.freeze
+
+      def description; return 'amavisd-new: https://www.amavis.org/'; end
+      def smtpagent;   return Sisimai::Bite.smtpagent(self); end
+
+      # Parse bounce messages from amavisd-new
+      # @param         [Hash] mhead       Message headers of a bounce email
+      # @options mhead [String] from      From header
+      # @options mhead [String] date      Date header
+      # @options mhead [String] subject   Subject header
+      # @options mhead [Array]  received  Received headers
+      # @options mhead [String] others    Other required headers
+      # @param         [String] mbody     Message body of a bounce email
+      # @return        [Hash, Nil]        Bounce data list and message/rfc822
+      #                                   part or nil if it failed to parse or
+      #                                   the arguments are missing
+      def scan(mhead, mbody)
+        # From: "Content-filter at neko1.example.jp" <postmaster@neko1.example.jp>
+        # Subject: Undeliverable mail, MTA-BLOCKED
+        return nil unless mhead['from'].to_s.start_with?('"Content-filter at ')
+
+        require 'sisimai/rfc1894'
+        fieldtable = Sisimai::RFC1894.FIELDTABLE
+        permessage = {}     # (Hash) Store values of each Per-Message field
+
+        dscontents = [Sisimai::Bite.DELIVERYSTATUS]
+        hasdivided = mbody.split("\n")
+        havepassed = ['']
+        rfc822list = []     # (Array) Each line in message/rfc822 part string
+        blanklines = 0      # (Integer) The number of blank lines
+        readcursor = 0      # (Integer) Points the current cursor position
+        recipients = 0      # (Integer) The number of 'Final-Recipient' header
+        v = nil
+
+        while e = hasdivided.shift do
+          # Save the current line for the next loop
+          havepassed << e
+          p = havepassed[-2]
+
+          if readcursor == 0
+            # Beginning of the bounce message or message/delivery-status part
+            if e.start_with?(StartingOf[:message][0])
+              readcursor |= Indicators[:deliverystatus]
+              next
+            end
+          end
+
+          if (readcursor & Indicators[:'message-rfc822']) == 0
+            # Beginning of the original message part(message/rfc822)
+            if e == StartingOf[:rfc822][0]
+              readcursor |= Indicators[:'message-rfc822']
+              next
+            end
+          end
+
+          if readcursor & Indicators[:'message-rfc822'] > 0
+            # message/rfc822 OR text/rfc822-headers part
+            if e.empty?
+              blanklines += 1
+              break if blanklines > 1
+              next
+            end
+            rfc822list << e
+          else
+            # message/delivery-status part
+            next if (readcursor & Indicators[:deliverystatus]) == 0
+            next if e.empty?
+            next unless f = Sisimai::RFC1894.match(e)
+
+            # "e" matched with any field defined in RFC3464
+            next unless o = Sisimai::RFC1894.field(e)
+            v = dscontents[-1]
+
+            if o[-1] == 'addr'
+              # Final-Recipient: rfc822; kijitora@example.jp
+              # X-Actual-Recipient: rfc822; kijitora@example.co.jp
+              if o[0] == 'final-recipient'
+                # Final-Recipient: rfc822; kijitora@example.jp
+                if v['recipient']
+                  # There are multiple recipient addresses in the message body.
+                  dscontents << Sisimai::Bite.DELIVERYSTATUS
+                  v = dscontents[-1]
+                end
+                v['recipient'] = o[2]
+                recipients += 1
+              else
+                # X-Actual-Recipient: rfc822; kijitora@example.co.jp
+                v['alias'] = o[2]
+              end
+            elsif o[-1] == 'code'
+              # Diagnostic-Code: SMTP; 550 5.1.1 <userunknown@example.jp>... User Unknown
+              v['spec'] = o[1]
+              v['diagnosis'] = o[2]
+            else
+              # Other DSN fields defined in RFC3464
+              next unless fieldtable.key?(o[0])
+              v[fieldtable[o[0]]] = o[2]
+
+              next unless f == 1
+              permessage[fieldtable[o[0]]] = o[2]
+            end
+          end # End of message/delivery-status
+        end
+        return nil unless recipients > 0
+
+        dscontents.each do |e|
+          # Set default values if each value is empty.
+          permessage.each_key { |a| e[a] ||= permessage[a] || '' }
+
+          e['diagnosis'] = Sisimai::String.sweep(e['diagnosis'].to_s.tr("\n", ' '))
+          e['agent'] = self.smtpagent
+        end
+
+        rfc822part = Sisimai::RFC5322.weedout(rfc822list)
+        return { 'ds' => dscontents, 'rfc822' => rfc822part }
+      end
+
+    end
+  end
+end
+