sisimai 5.1.0 → 5.2.0

data/lib/sisimai/arf.rb

@@ -13,21 +13,21 @@ module Sisimai
       # OpenDMARC 1.3.0 uses:  This is an authentication failure report for an email message received from IP
       # Abusix ARF uses        this is an autogenerated email abuse complaint regarding your network.
       Indicators = Sisimai::Lhost.INDICATORS
-      StartingOf = {
-        rfc822:  ['Content-Type: message/rfc822', 'Content-Type: text/rfc822-headers'],
-        report:  ['Content-Type: message/feedback-report'],
-        message: [
-          ['this is a', 'abuse report'],
-          ['this is a', 'authentication', 'failure report'],
-          ['this is a', ' report for'],
-          ['this is an authentication', 'failure report'],
-          ['this is an autogenerated email abuse complaint'],
-          ['this is an email abuse report'],
-        ],
-      }.freeze
-      ReportFrom = ['staff@hotmail.com', 'complaints@email-abuse.amazonses.com'];
-      LongFields = Sisimai::RFC5322.LONGFIELDS
-      RFC822Head = Sisimai::RFC5322.HEADERFIELDS
+      AbuseAddrs = ['staff@hotmail.com', 'complaints@email-abuse.amazonses.com'].freeze
+      ReportFrom = "Content-Type: message/feedback-report".freeze
+      Boundaries = [
+        "Content-Type: message/rfc822",
+        "Content-Type: text/rfc822-headers",
+        "Content-Type: text/rfc822-header",  # ??
+      ].freeze
+      ARFPreface = [
+        ["this is a", "abuse report"],
+        ["this is a", "authentication", "failure report"],
+        ["this is a", " report for"],
+        ["this is an authentication", "failure report"],
+        ["this is an autogenerated email abuse complaint"],
+        ["this is an email abuse report"],
+      ].freeze
 
       def description; return 'Abuse Feedback Reporting Format'; end
 
@@ -39,23 +39,27 @@ module Sisimai
         return false unless heads
 
         # Content-Type: multipart/report; report-type=feedback-report; ...
-        return true if Sisimai::String.aligned(heads['content-type'], ['report-type=', 'feedback-report'])
+        return true if Sisimai::String.aligned(heads["content-type"], ["report-type=", "feedback-report"])
 
-        match = false
-        if heads['content-type'].include?('multipart/mixed')
+        if heads["content-type"].include?("multipart/mixed")
           # Microsoft (Hotmail, MSN, Live, Outlook) uses its own report format.
           # Amazon SES Complaints bounces
           cv = Sisimai::Address.s3s4(heads['from'])
-          if heads['subject'].include?('complaint about message from ')
+          if heads["subject"].include?("complaint about message from ")
             # From: staff@hotmail.com
             # From: complaints@email-abuse.amazonses.com
             # Subject: complaint about message from 192.0.2.1
-            match = true if ReportFrom.any? { |a| cv.include?(a) }
+            return true if AbuseAddrs.any? { |a| cv.include?(a) }
           end
         end
-        match = true if heads['x-apple-unsubscribe'] == 'true'
 
-        return match
+        while true
+          # X-Apple-Unsubscribe: true
+          break unless heads.has_key?("x-apple-unsubscribe")
+          return true if heads["x-apple-unsubscribe"] == "true"
+          break
+        end
+        return false
       end
 
       # @abstract Detect an error for Feedback Loop
@@ -67,25 +71,16 @@ module Sisimai
         return nil unless self.is_arf(mhead)
 
         dscontents = [Sisimai::Lhost.DELIVERYSTATUS]
-        bodyslices = mbody.split("\n")
-        rfc822part = ''   # (String) message/rfc822-headers part
-        previousfn = ''   # (String) Previous field name
-        readcursor = 0    # (Integer) Points the current cursor position
-        recipients = 0    # (Integer) The number of 'Final-Recipient' header
-        rcptintext = ''   # (String) Recipient address in the message body
-        commondata = {
-          'diagnosis' => '',  # Error message
-          'from'      => '',  # Original-Mail-From:
-          'rhost'     => '',  # Reporting-MTA:
-        }
-        arfheaders = {
-          'feedbacktype' => nil,   # FeedBack-Type:
-          'rhost'        => nil,   # Source-IP:
-          'agent'        => nil,   # User-Agent:
-          'date'         => nil,   # Arrival-Date:
-          'authres'      => nil,   # Authentication-Results:
-        }
-        v = nil
+        emailparts = Sisimai::RFC5322.part(mbody, Boundaries)
+        bodyslices = emailparts[0].split("\n")
+        reportpart = false
+        readcursor = 0    # Points the current cursor position
+        recipients = 0    # The number of 'Final-Recipient' header
+        timestamp0 = ""   # The value of "Arrival-Date" or "Received-Date"
+        remotehost = ""   # The value of "Source-IP" field
+        reportedby = ""   # The value of "Reporting-MTA" field
+        anotherone = ""   # Other fields(append to Diagnosis)
+        v = dscontents[-1]
 
         # 3.1.  Required Fields
         #
@@ -112,71 +107,24 @@ module Sisimai
           #   message-id of 0000-000000000000000000000000000000000@mx
           #   received from IP address 192.0.2.1 on
           #   Thu, 29 Apr 2010 00:00:00 +0900 (JST)
-          p = e.downcase
-          if commondata['diagnosis'].empty?
-            commondata['diagnosis'] = e if StartingOf[:message].any? { |a| Sisimai::String.aligned(p, a) }
-          end
-
           if readcursor == 0
             # Beginning of the bounce message or message/delivery-status part
-            readcursor |= Indicators[:deliverystatus] if e.start_with?(StartingOf[:report][0])
-          end
-
-          if (readcursor & Indicators[:'message-rfc822']) == 0
-            # Beginning of the original message part
-            if e.start_with?(StartingOf[:rfc822][0], StartingOf[:rfc822][1])
-              readcursor |= Indicators[:'message-rfc822']
-              next
+            r = e.downcase
+            ARFPreface.each do |f|
+              # Hello,
+              # this is an autogenerated email abuse complaint regarding your network.
+              next unless Sisimai::String.aligned(r, f)
+              readcursor |= Indicators[:deliverystatus]
+              v["diagnosis"] << " " + e
+              break
             end
+            next
           end
+          next unless readcursor & Indicators[:deliverystatus] > 0
+          next if e.empty?
+          if e == ReportFrom then reportpart = true; next; end
 
-          if readcursor & Indicators[:'message-rfc822'] > 0
-            # message/rfc822 OR text/rfc822-headers part
-            if e.start_with?('X-HmXmrOriginalRecipient:')
-              # Microsoft ARF: original recipient.
-              dscontents[-1]['recipient'] = Sisimai::Address.s3s4(e[e.index(':') + 1, e.size])
-              recipients += 1
-
-              # The "X-HmXmrOriginalRecipient" header appears only once so we take this opportunity
-              # to hard-code ARF headers missing in Microsoft's implementation.
-              arfheaders['feedbacktype'] = 'abuse'
-              arfheaders['agent'] = 'Microsoft Junk Mail Reporting Program'
-
-            elsif e.start_with?('From: ')
-              # Microsoft ARF: original sender.
-              commondata['from'] = Sisimai::Address.s3s4(e[6, e.size]) if commondata['from'].empty?
-              previousfn = 'from'
-
-            elsif e.start_with?(' ')
-              # Continued line from the previous line
-              if previousfn == 'from'
-                # Multiple lines at "From:" field
-                commondata['from'] << e
-                next
-              else
-                rfc822part << e + "\n" if LongFields[previousfn]
-                next unless e.empty?
-              end
-              rcptintext << e if previousfn == 'to'
-
-            else
-              # Get required headers only
-              (lhs, rhs) = e.split(/:[ ]/, 2)
-              next unless lhs
-              lhs.downcase!
-
-              previousfn = ''
-              next unless RFC822Head[lhs]
-
-              previousfn  = lhs
-              rfc822part << e + "\n"
-              rcptintext  = rhs if lhs == 'to'
-            end
-          else
-            # message/feedback-report part
-            next unless readcursor & Indicators[:deliverystatus] > 0
-            next if e.empty?
-
+          if reportpart
             # Feedback-Type: abuse
             # User-Agent: SomeGenerator/1.0
             # Version: 0.1
@@ -184,151 +132,114 @@ module Sisimai
             # Original-Rcpt-To: <kijitora@example.jp>
             # Received-Date: Thu, 29 Apr 2009 00:00:00 JST
             # Source-IP: 192.0.2.1
-            v = dscontents[-1]
-
-            if e.start_with?('Original-Rcpt-To: ', 'Redacted-Address: ')
+            if e.start_with?("Original-Rcpt-To: ") || e.start_with?("Removal-Recipient: ")
               # Original-Rcpt-To header field is optional and may appear any number of times as appropriate:
-              # Original-Rcpt-To: <user@example.com>
-              # Redacted-Address: localpart@
-              if v['recipient']
+              # Original-Rcpt-To: <kijitora@example.jp>
+              # Removal-Recipient: user@example.com
+              cv = Sisimai::Address.s3s4(e[e.index(" ") + 1, e.size]); next unless Sisimai::Address.is_emailaddress(cv)
+              cw = dscontents.size;                                    next if cw > 0 && cv == dscontents[cw - 1]["recipient"]
+
+              if v["recipient"] != ""
                 # There are multiple recipient addresses in the message body.
                 dscontents << Sisimai::Lhost.DELIVERYSTATUS
                 v = dscontents[-1]
               end
-              v['recipient'] = Sisimai::Address.s3s4(e[e.index(' ') + 1, e.size])
+              v["recipient"] = cv
               recipients += 1
 
-            elsif e.start_with?('Feedback-Type: ')
+            elsif e.start_with?("Feedback-Type: ")
               # The header field MUST appear exactly once.
               # Feedback-Type: abuse
-              arfheaders['feedbacktype'] = e[e.index(' ') + 1, e.size]
+              v["feedbacktype"] = e[e.index(" ") + 1, e.size]
 
-            elsif e.start_with?('Authentication-Results: ')
+            elsif e.start_with?("Authentication-Results: ")
               # "Authentication-Results" indicates the result of one or more authentication checks
               # run by the report generator.
-              #   Authentication-Results: mail.example.jp; spf=fail smtp.mail=spammer@example.com
-              arfheaders['authres'] = e[e.index(' ') + 1, e.size]
+              #
+              # Authentication-Results: mail.example.com;
+              #   spf=fail smtp.mail=somespammer@example.com
+              anotherone << e + ", "
 
-            elsif e.start_with?('User-Agent: ')
+            elsif e.start_with?("User-Agent: ")
               # The header field MUST appear exactly once.
               # User-Agent: SomeGenerator/1.0
-              arfheaders['agent'] = e[e.index(' ') + 1, e.size]
+              anotherone << e + ", "
 
-            elsif e.start_with?('Received-Date: ', 'Arrival-Date: ')
+            elsif e.start_with?("Received-Date: ") || e.start_with?("Arrival-Date: ")
               # Arrival-Date header is optional and MUST NOT appear more than once.
               # Received-Date: Thu, 29 Apr 2010 00:00:00 JST
               # Arrival-Date: Thu, 29 Apr 2010 00:00:00 +0000
-              arfheaders['date'] = e[e.index(' ') + 1, e.size]
+              timestamp0 = e[e.index(" ") + 1, e.size]
 
-            elsif e.start_with?('Reporting-MTA: dns; ')
+            elsif e.start_with?("Reporting-MTA: ")
               # The header is optional and MUST NOT appear more than once.
               # Reporting-MTA: dns; mx.example.jp
-              commondata['rhost'] = e[e.index(';') + 2, e.size]
+              cv = Sisimai::RFC1894.field(e); next if cv.size == 0
+              reportedby = cv[2]
 
-            elsif e.start_with?('Source-IP: ')
+            elsif e.start_with?("Source-IP: ")
               # The header is optional and MUST NOT appear more than once.
               # Source-IP: 192.0.2.45
-              arfheaders['rhost'] = e[e.index(' ') + 1, e.size]
+              remotehost = e[e.index(" ") + 1, e.size]
 
-            elsif e.start_with?('Original-Mail-From: ')
+            elsif e.start_with?("Original-Mail-From: ")
               # the header is optional and MUST NOT appear more than once.
               # Original-Mail-From: <somespammer@example.net>
-              commondata['from'] = Sisimai::Address.s3s4(e[e.index(' ') + 1, e.size]) if commondata['from'].empty?
-
+              anotherone << e + ", "
             end
-          end
-        end
-
-        if arfheaders['feedbacktype'] == 'auth-failure' && arfheaders['authres']
-          # Append the value of Authentication-Results header
-          commondata['diagnosis'] << ' ' << arfheaders['authres']
-        end
-
-        if recipients == 0
-          # The original recipient address was not found
-          if Sisimai::String.aligned(rfc822part, ["\nTo: ", '@'])
-            # pick the address from To: header in message/rfc822 part.
-            p1 = rfc822part.index("\nTo: ") + 5
-            p2 = rfc822part.index("\n", p1 + 1)
-            cm = p2 > 0 ? p2 - p1 : 255
-            dscontents[-1]['recipient'] = Sisimai::Address.s3s4(rfc822part[p1, cm])
-            recipients = 1
+          else
+            # Messages before "Content-Type: message/feedback-report" part
+            v["diagnosis"] << " " + e
           end
 
-          while true
-            # Insert pseudo recipient address when there is no valid recipient address in the message
-            # for example,
-            #   Date: Thu, 29 Apr 2015 23:34:45 +0000
-            #   To: "undisclosed"
-            #   Subject: Nyaan
-            #   Message-ID: <ffffffffffffffffffffffff00000000@example.net>
-            dscontents[-1]['recipient'] ||= ''
-            break if dscontents[-1]['recipient'].include?('@')
-            dscontents[-1]['recipient'] = Sisimai::Address.undisclosed(true)
-            recipients = 1
-            break
-          end
-        end
-
-        unless Sisimai::String.aligned(rfc822part, ['From: ', '@'])
-          # There is no "From:" header in the original message. Append the value of "Original-Mail-From"
-          # value as a sender address.
-          rfc822part << 'From: ' << commondata['from'] + "\n" unless commondata['from'].empty?
-        end
+          while recipients == 0
+            # There is no recipient address in the message
+            if mhead.has_key?("x-apple-unsubscribe")
+              # X-Apple-Unsubscribe: true
+              last unless mhead["x-apple-unsubscribe"] == "true"
+              last unless mhead["from"].include?('@')
+              dscontents[0]["recipient"]    = mhead["from"]
+              dscontents[0]["diagnosis"]    = Sisimai::String.sweep(emailparts[0])
+              dscontents[0]["feedbacktype"] = "opt-out"
 
-        if mhead['subject'].include?('complaint about message from ')
-          # Microsoft ARF: remote host address.
-          arfheaders['rhost'] = mhead['subject'][mhead['subject'].rindex(' ') + 1, mhead['subject'].size]
-          commondata['diagnosis'] =
-            'This is a Microsoft email abuse report for an email message received from IP' << arfheaders['rhost'] + ' on ' << mhead['date']
+              # Addpend To: field as a pseudo header
+              emailparts[1] = sprintf("To: <%s>\n", mhead["from"]) if emailparts[1].empty?
 
-        elsif mhead['subject'].include?('unsubscribe')
-          # Apple Mail sent this email to unsubscribe from the message
-          while true
-            # Subject: unsubscribe
-            # Content-Type: text/plain; charset=UTF-8
-            # Auto-Submitted: auto-replied
-            # X-Apple-Unsubscribe: true
-            #
-            # Apple Mail sent this email to unsubscribe from the message
-            break unless mhead['x-apple-unsubscribe']
-            break unless mhead['x-apple-unsubscribe'] == 'true'
-            break unless mbody.include?('Apple Mail sent this email to unsubscribe from the message');
-
-            dscontents[-1]['recipient'] = Sisimai::Address.s3s4(mhead['from'])
-            dscontents[-1]['feedbacktype'] = 'opt-out'
-            break
+            else
+              # Pick it from the original message part
+              p1 = emailparts[1].index("\nTo:");      break if p1.nil?
+              p2 = emailparts[1].index("\n", p1 + 4); break if p2.nil?
+              cv = Sisimai::Address.s3s4(emailparts[1][p1 + 4, p2 - p1])
+
+              # There is no valid email address in the To: header of the original message such as
+              # To: <Undisclosed Recipients>
+              cv = Sisimai::Address.undisclosed("r") unless Sisimai::Address.is_emailaddress(cv)
+              dscontents[0]["recipient"] = cv
+            end
+            recipients += 1
           end
-        end
-
-        dscontents.each do |e|
-          # AOL = http://forums.cpanel.net/f43/aol-brutal-work-71473.html
-          e['recipient'] = Sisimai::Address.s3s4(rcptintext) if e['recipient'][-1, 1] == '@'
-          arfheaders.each_key { |a| e[a] ||= arfheaders[a] || '' }
-          e.delete('authres')
-
-          e['diagnosis']  = commondata['diagnosis'] unless e['diagnosis']
-          e['diagnosis']  = Sisimai::String.sweep(e['diagnosis'])
-          e['date']       = mhead['date'] if e['date'].empty?
-          e['reason']     = 'feedback'
-          e['agent']      = 'Feedback-Loop'
-          %w[command action status alias].each { |a| e[a] = '' }
-
-          # Get the remote IP address from the message body
-          next unless e['rhost'].empty?
-          if commondata['rhost'].size > 0
-            # The value of "Reporting-MTA" header
-            e['rhost'] = commondata['rhost']
-
-          else
-            # Try to get an IP address from the error message
-            # This is an email abuse report for an email message received from IP address 24.64.1.1
-            # on Thu, 29 Apr 2010 00:00:00 +0000
-            ip = Sisimai::String.ipv4(e['diagnosis']) || []
-            e['rhost'] = ip[0] if ip.size > 0
+          return nil if recipients == 0
+
+          anotherone = ": " + Sisimai::String.sweep(anotherone) if anotherone != ""
+          anotherone = anotherone.chop if anotherone[-1, 1] == ","
+
+          j = -1
+          dscontents.each do |e|
+            # Tidy up the error message in e.Diagnosis, Try to detect the bounce reason.
+            j += 1
+            e["diagnosis"] = Sisimai::String.sweep(e["diagnosis"] + anotherone)
+            e["reason"]    = "feedback"
+            e["rhost"]     = remotehost
+            e["lhost"]     = reportedby
+            e["date"]      = timestamp0
+
+            # Copy some values from the previous element when the report have 2 or more email address
+            next if j == 0 || dscontents.size == 1
+            e["diagnosis"]    = dscontents[j - 1]["diagnosis"]
+            e["feedbacktype"] = dscontents[j - 1]["feedbacktype"]
           end
         end
-        return { 'ds' => dscontents, 'rfc822' => rfc822part }
+        return { "ds" => dscontents, "rfc822" => emailparts[1] }
       end
     end
   end