sisimai 5.0.0 → 5.0.2

data/lib/sisimai/reason/blocked.rb

@@ -46,6 +46,7 @@ module Sisimai
           'part of their network is on our block list',
           'please use the smtp server of your isp',
           'refused - see http',
+          'rejected - multi-blacklist', # junkemailfilter.com
           'rejected because the sending mta or the sender has not passed validation',
           'rejecting open proxy', # Sendmail(srvrsmtp.c)
           'sender ip address rejected',
@@ -78,6 +79,7 @@ module Sisimai
           ['message from ', ' rejected based on blacklist'],
           ['messages from ', ' temporarily deferred due to user complaints'], # Yahoo!
           ['server ip ', ' listed as abusive'],
+          ['sorry! your ip address', ' is blocked by rbl'], # junkemailfilter.com
           ['the domain ', ' is blacklisted'],
           ['the email ', ' is blacklisted'],
           ['the ip', ' is blacklisted'],

data/lib/sisimai/reason/expired.rb

@@ -18,6 +18,7 @@ module Sisimai
           'has been delayed',
           'it has not been collected after',
           'message expired after sitting in queue for',
+          'message expired, cannot connect to remote server',
           'message expired, connection refulsed',
           'message timed out',
           'retry time not reached for any host after a long failure period',

data/lib/sisimai/reason/mailboxfull.rb

@@ -15,6 +15,7 @@ module Sisimai
           'boite du destinataire pleine',
           'delivery failed: over quota',
           'disc quota exceeded',
+          'diskspace quota',
           'does not have enough space',
           'exceeded storage allocation',
           'exceeding its mailbox quota',

data/lib/sisimai/reason/securityerror.rb

@@ -29,6 +29,7 @@ module Sisimai
           'insecure mail relay',
           'recipient address rejected: access denied',
           "sorry, you don't authenticate or the domain isn't in my list of allowed rcpthosts",
+          'starttls is required to send mail',
           'tls required but not supported',   # SendGrid:the recipient mailserver does not support TLS or have a valid certificate
           'unauthenticated senders not allowed',
           'verification failure',

data/lib/sisimai/reason/spamdetected.rb

@@ -76,6 +76,7 @@ module Sisimai
           'spam score ',
           'spambouncer identified spam',  # SpamBouncer identified SPAM
           'spamming not allowed',
+          'too many spam complaints',
           'too much spam.',               # Earthlink
           'the email message was detected as spam',
           'the message has been rejected by spam filtering engine',

data/lib/sisimai/reason/suspend.rb

@@ -21,6 +21,7 @@ module Sisimai
           'recipient rejected: temporarily inactive',
           'recipient suspend the service',
           'this account has been disabled or discontinued',
+          'this address no longer accepts mail',
           'this mailbox is disabled',
           'user suspended',   # http://mail.163.com/help/help_spam_16.htm
           'vdelivermail: account is locked email bounced',

data/lib/sisimai/rfc5322.rb

@@ -3,6 +3,7 @@ module Sisimai
   module RFC5322
     class << self
       require 'sisimai/string'
+      require 'sisimai/address'
       HeaderTable = {
         :messageid => %w[message-id],
         :subject   => %w[subject],
@@ -52,79 +53,134 @@ module Sisimai
       # @return   [Array]         Received header as a structured data
       def received(argv1)
         return [] unless argv1.is_a?(::String)
+        return [] if argv1.include?(' invoked by uid')
+        return [] if argv1.include?(' invoked from network')
+
+        # - https://datatracker.ietf.org/doc/html/rfc5322
+        #   received        =   "Received:" *received-token ";" date-time CRLF
+        #   received-token  =   word / angle-addr / addr-spec / domain
+        #
+        # - Appendix A.4. Message with Trace Fields
+        #   Received:
+        #       from x.y.test
+        #       by example.net
+        #       via TCP
+        #       with ESMTP
+        #       id ABC12345
+        #       for <mary@example.net>;  21 Nov 1997 10:05:43 -0600
+        recvd = argv1.split(' ')
+        label = %w[from by via with id for]
+        token = {}
+        other = []
+        alter = []
+        right = false
+        range = recvd.size
+        index = -1
+
+        recvd.each do |e|
+          # Look up each label defined in "label" from Received header
+          index += 1
+          break unless index < range; f = e.downcase
+          next  unless label.any? { |a| f == a }
+          token[f] = recvd[index + 1] || next
+          token[f] = token[f].downcase.delete('();')
+
+          next  unless f == 'from'
+          break unless index + 2 < range
+          next  unless recvd[index + 2].start_with?('(')
+
+          # Get and keep a hostname in the comment as follows:
+          # from mx1.example.com (c213502.kyoto.example.ne.jp [192.0.2.135]) by mx.example.jp (V8/cf)
+          # [
+          #   "from",                         # index + 0
+          #   "mx1.example.com",              # index + 1
+          #   "(c213502.kyoto.example.ne.jp", # index + 2
+          #   "[192.0.2.135])",               # index + 3
+          #   "by",
+          #   "mx.example.jp",
+          #   "(V8/cf)",
+          #   ...
+          # ]
+          # The 2nd element after the current element is NOT a continuation of the current element
+          # such as "(c213502.kyoto.example.ne.jp)"
+          other << recvd[index + 2].delete('();')
+
+          # The 2nd element after the current element is a continuation of the current element.
+          # such as "(c213502.kyoto.example.ne.jp", "[192.0.2.135])"
+          break unless index + 3 < range
+          other << recvd[index + 3].delete('();')
+        end
 
-        hosts = []
-        value = { 'from' => '', 'by' => '' }
-
-        # Received: (qmail 10000 invoked by uid 999); 24 Apr 2013 00:00:00 +0900
-        return [] if argv1.include?('(qmail ') && argv1.include?(' invoked ')
-
-        p1 = argv1.index('from ')     || -1
-        p2 = argv1.index('by ')       || -1
-        p3 = argv1.index(' ', p2 + 3) || -1
-
-        if p1 == 0 && p2 > 1 && p2 < p3
-          # Received: from localhost (localhost) by nijo.example.jp (V8/cf) id s1QB5ma0018057;
-          #   Wed, 26 Feb 2014 06:05:48 -0500
-          value['from'] = Sisimai::String.sweep(argv1[p1 + 5, p2 - p1 - 5])
-          value['by']   = Sisimai::String.sweep(argv1[p2 + 3, p3 - p2 - 3])
+        other.each do |e|
+          # Check alternatives in "other", and then delete uninformative values.
+          next if e.nil?
+          next if e.size < 4
+          next if e == 'unknown'
+          next if e == 'localhost'
+          next if e == '[127.0.0.1]'
+          next if e == '[IPv6:::1]'
+          next unless e.include?('.')
+          next if e.include?('=')
+          alter << e
+        end
 
-        elsif p1 != 0 && p2 > -1
-          # Received: by 10.70.22.98 with SMTP id c2mr1838265pdf.3; Fri, 18 Jul 2014 00:31:02 -0700 (PDT)
-          value['from'] = Sisimai::String.sweep(argv1[p2 + 3, argv1.size])
-          value['by']   = Sisimai::String.sweep(argv1[p2 + 3, p3 - p2 - 3])
+        %w[from by].each do |e|
+          # Remove square brackets from the IP address such as "[192.0.2.25]"
+          next if token[e].nil?
+          next if token[e].empty?
+          next unless token[e].start_with?('[')
+          token[e] = Sisimai::String.ipv4(token[e]).shift || ''
         end
+        token['from'] ||= ''
 
-        if value['from'].include?(' ')
-          # Received: from [10.22.22.222] (smtp.kyoto.ocn.ne.jp [192.0.2.222]) (authenticated bits=0)
-          #   by nijo.example.jp (V8/cf) with ESMTP id s1QB5ka0018055; Wed, 26 Feb 2014 06:05:47 -0500
-          received = value['from'].split(' ')
-          namelist = []
-          addrlist = []
-          hostname = ''
-          hostaddr = ''
-
-          while e = received.shift do
-            # Received: from [10.22.22.222] (smtp-gateway.kyoto.ocn.ne.jp [192.0.2.222])
-            cv = Sisimai::String.ipv4(e) || []
-            if cv.size > 0
-              # [192.0.2.1] or (192.0.2.1)
-              addrlist.append(*cv)
-            else
-              # hostname
-              e = e.delete('()').strip
-              namelist << e
-            end
-          end
+        while true do
+          # Prefer hostnames over IP addresses, except for localhost.localdomain and similar.
+          break if token['from'] == 'localhost'
+          break if token['from'] == 'localhost.localdomain'
+          break unless token['from'].include?('.')  # A hostname without a domain name
+          break unless Sisimai::String.ipv4(token['from']).empty?
 
-          while e = namelist.shift do
-            # 1. Hostname takes priority over all other IP addresses
-            next unless e.include?('.')
-            hostname = e
-            break
-          end
+          # No need to rewrite token['from']
+          right = true
+          break
+        end
 
-          if hostname.empty?
-            # 2. Use IP address as a remote host name
-            addrlist.each do |e|
-              # Skip if the address is a private address
-              next if e.start_with?('10.', '127.', '192.168.')
-              next if e =~ /\A172[.](?:1[6-9]|2[0-9]|3[0-1])[.]/
-              hostaddr = e
-              break
-            end
+        while true do
+          # Try to rewrite uninformative hostnames and IP addresses in token['from']
+          break if right        # There is no need to rewrite
+          break if alter.empty? # There is no alternative to rewriting
+          break if alter[0].include?(token['from'])
+
+          if token['from'].start_with?('localhost')
+            # localhost or localhost.localdomain
+            token['from'] = alter[0]
+          elsif token['from'].index('.')
+            # A hostname without a domain name such as "mail", "mx", or "mbox"
+            token['from'] = alter[0] if alter[0].include?('.')
+          else
+            # An IPv4 address
+            token['from'] = alter[0]
           end
-
-          value['from'] = hostname || hostaddr || addrlist[-1]
+          break
         end
-
-        %w[from by].each do |e|
-          # Copy entries into hosts
-          next if value[e].empty?
-          value[e] = value[e].delete('[]();?')
-          hosts << value[e]
+        token.delete('from') if token['from'].nil?
+        token.delete('by')   if token['by'].nil?
+        token['for'] = Sisimai::Address.s3s4(token['for']) if token.has_key?('for')
+
+        token.keys.each do |e|
+          # Delete an invalid value
+          token[e] = '' if token[e].include?(' ')
+          token[e].delete!('[]')  # Remove "[]" from the IP address
         end
-        return hosts
+
+        return [
+          token['from'] || '',
+          token['by']   || '',
+          token['via']  || '',
+          token['with'] || '',
+          token['id']   || '',
+          token['for']  || '',
+        ]
       end
 
       # Split given entire message body into error message lines and the original message part only