sisimai 5.0.0-java → 5.0.3-java

data/lib/sisimai/lhost/sendmail.rb

@@ -168,7 +168,6 @@ module Sisimai::Lhost
         dscontents.each do |e|
           # Set default values if each value is empty.
           e['diagnosis'] ||= ''
-          e['lhost']     ||= permessage['rhost']
           permessage.each_key { |a| e[a] ||= permessage[a] || '' }
 
           if anotherset['diagnosis']

data/lib/sisimai/lhost/yandex.rb

@@ -103,7 +103,6 @@ module Sisimai::Lhost
 
         dscontents.each do |e|
           # Set default values if each value is empty.
-          e['lhost'] ||= permessage['rhost']
           permessage.each_key { |a| e[a] ||= permessage[a] || '' }
 
           e['diagnosis'] = Sisimai::String.sweep(e['diagnosis'].tr("\n", ' '))

data/lib/sisimai/message.rb

@@ -198,17 +198,26 @@ module Sisimai
         # Select and convert all the headers in $argv0. The following regular expression is based on
         # https://gist.github.com/xtetsuji/b080e1f5551d17242f6415aba8a00239
         headermaps = { 'subject' => '' }
-        recvheader = []
+        receivedby = []
         argv0.scan(/^([\w-]+):[ ]*(.*?)\n(?![\s\t])/m) { |e| headermaps[e[0].downcase] = e[1] }
         headermaps.delete('received')
         headermaps.each_key { |e| headermaps[e].gsub!(/\n[\s\t]+/, ' ') }
 
         if argv0.include?('Received:')
           # Capture values of each Received: header
-          recvheader = argv0.scan(/^Received:[ ]*(.*?)\n(?![\s\t])/m).flatten
-          recvheader.each { |e| e.gsub!(/\n[\s\t]+/, ' ') }
+          re = argv0.scan(/^Received:[ ]*(.*?)\n(?![\s\t])/m).flatten
+          re.each do |e|
+            # 1. Exclude the Received header including "(qmail ** invoked from network)".
+            # 2. Convert all consecutive spaces and line breaks into a single space character.
+            next if e.include?(' invoked by uid')
+            next if e.include?(' invoked from network')
+
+            e.gsub!(/\n[\s\t]+/, ' ')
+            e.squeeze!("\n\t ")
+            receivedby << e
+          end
         end
-        headermaps['received'] = recvheader
+        headermaps['received'] = receivedby
 
         return headermaps unless argv1
         return headermaps if headermaps['subject'].empty?
@@ -443,7 +452,7 @@ module Sisimai
             unless haveloaded['Sisimai::ARF']
               # Feedback Loop message
               require 'sisimai/arf'
-              havesifted = Sisimai::ARF.inquire(mailheader, bodystring) if Sisimai::ARF.is_arf(mailheader)
+              havesifted = Sisimai::ARF.inquire(mailheader, bodystring)
               throw :PARSER if havesifted
             end
 

data/lib/sisimai/reason/authfailure.rb

@@ -20,6 +20,7 @@ module Sisimai
           'dmarc policy',
           'please inspect your spf settings',
           'sender policy framework (spf) fail',
+          'sender policy framework violation',
           'spf (sender policy framework) domain authentication fail',
           'spf check: fail',
         ].freeze

data/lib/sisimai/reason/blocked.rb

@@ -46,6 +46,7 @@ module Sisimai
           'part of their network is on our block list',
           'please use the smtp server of your isp',
           'refused - see http',
+          'rejected - multi-blacklist', # junkemailfilter.com
           'rejected because the sending mta or the sender has not passed validation',
           'rejecting open proxy', # Sendmail(srvrsmtp.c)
           'sender ip address rejected',
@@ -61,6 +62,7 @@ module Sisimai
           'we do not accept mail from dynamic ips',   # @mail.ru
           'you are not allowed to connect',
           'you are sending spam',
+          'your ip address is listed in the rbl',
           'your network is temporary blacklisted',
           'your server requires confirmation',
         ].freeze
@@ -78,6 +80,7 @@ module Sisimai
           ['message from ', ' rejected based on blacklist'],
           ['messages from ', ' temporarily deferred due to user complaints'], # Yahoo!
           ['server ip ', ' listed as abusive'],
+          ['sorry! your ip address', ' is blocked by rbl'], # junkemailfilter.com
           ['the domain ', ' is blacklisted'],
           ['the email ', ' is blacklisted'],
           ['the ip', ' is blacklisted'],

data/lib/sisimai/reason/expired.rb

@@ -7,10 +7,13 @@ module Sisimai
     # the message you sent has been in the queue for long time.
     module Expired
       class << self
+        require 'sisimai/string'
+
         Index = [
           'connection timed out',
           'could not find a gateway for',
           'delivery attempts will continue to be',
+          'delivery expired',
           'delivery time expired',
           'failed to deliver to domain ',
           'giving up on',
@@ -18,6 +21,7 @@ module Sisimai
           'has been delayed',
           'it has not been collected after',
           'message expired after sitting in queue for',
+          'message expired, cannot connect to remote server',
           'message expired, connection refulsed',
           'message timed out',
           'retry time not reached for any host after a long failure period',
@@ -27,6 +31,9 @@ module Sisimai
           'was not reachable within the allowed queue period',
           'your message could not be delivered for more than',
         ].freeze
+        Pairs = [
+          ['could not be delivered for', ' days'],
+        ].freeze
 
         def text; return 'expired'; end
         def description; return 'Delivery time has expired due to a connection failure'; end
@@ -38,6 +45,7 @@ module Sisimai
         def match(argv1)
           return nil unless argv1
           return true if Index.any? { |a| argv1.include?(a) }
+          return true if Pairs.any? { |a| Sisimai::String.aligned(argv1, a) }
           return false
         end
 

data/lib/sisimai/reason/filtered.rb

@@ -21,6 +21,7 @@ module Sisimai
           'resolver.rst.notauthorized',   # Microsoft Exchange
           'this account is protected by',
           'user not found',   # Filter on MAIL.RU
+          'user refuses to receive this mail',
           'user reject',
           'we failed to deliver mail because the following address recipient id refuse to receive mail',  # Willcom
           'you have been blocked by the recipient',

data/lib/sisimai/reason/mailboxfull.rb

@@ -8,6 +8,7 @@ module Sisimai
     module MailboxFull
       class << self
         Index = [
+          '452 insufficient disk space',
           'account disabled temporarly for exceeding receiving limits',
           'account is exceeding their quota',
           'account is over quota',
@@ -15,6 +16,7 @@ module Sisimai
           'boite du destinataire pleine',
           'delivery failed: over quota',
           'disc quota exceeded',
+          'diskspace quota',
           'does not have enough space',
           'exceeded storage allocation',
           'exceeding its mailbox quota',
@@ -34,7 +36,9 @@ module Sisimai
           'maildir delivery failed: userdisk quota ',
           'maildir delivery failed: domaindisk quota ',
           'mailfolder is full',
+          'no space left on device',
           'not enough storage space in',
+          'not sufficient disk space',
           'over the allowed quota',
           'quota exceeded',
           'quota violation for',

data/lib/sisimai/reason/norelaying.rb

@@ -24,6 +24,7 @@ module Sisimai
           'relay not permitted',
           'relaying denied',  # Sendmail
           'relaying mail to ',
+          'specified domain is not allowed',
           "that domain isn't in my list of allowed rcpthost",
           'this system is not configured to relay mail',
           'unable to relay for',

data/lib/sisimai/reason/rejected.rb

@@ -38,7 +38,7 @@ module Sisimai
           'email address is on senderfilterconfig list',
           'emetteur invalide',
           'empty envelope senders not allowed',
-          'envelope blocked – ',
+          'envelope blocked - ',
           'error: no third-party dsns',   # SpamWall - block empty sender
           'from: domain is invalid. please provide a valid from:',
           'fully qualified email address required',   # McAfee

data/lib/sisimai/reason/securityerror.rb

@@ -29,6 +29,7 @@ module Sisimai
           'insecure mail relay',
           'recipient address rejected: access denied',
           "sorry, you don't authenticate or the domain isn't in my list of allowed rcpthosts",
+          'starttls is required to send mail',
           'tls required but not supported',   # SendGrid:the recipient mailserver does not support TLS or have a valid certificate
           'unauthenticated senders not allowed',
           'verification failure',

data/lib/sisimai/reason/spamdetected.rb

@@ -76,6 +76,7 @@ module Sisimai
           'spam score ',
           'spambouncer identified spam',  # SpamBouncer identified SPAM
           'spamming not allowed',
+          'too many spam complaints',
           'too much spam.',               # Earthlink
           'the email message was detected as spam',
           'the message has been rejected by spam filtering engine',

data/lib/sisimai/reason/suspend.rb

@@ -12,16 +12,21 @@ module Sisimai
           'boite du destinataire archivee',
           'email account that you tried to reach is disabled',
           'has been suspended',
+          'inactive account',
           'invalid/inactive user',
           'is a deactivated mailbox', # http://service.mail.qq.com/cgi-bin/help?subtype=1&&id=20022&&no=1000742
           'is unavailable: user is terminated',
           'mailbox currently suspended',
+          'mailbox disabled',
           'mailbox is frozen',
           'mailbox unavailable or access denied',
           'recipient rejected: temporarily inactive',
           'recipient suspend the service',
           'this account has been disabled or discontinued',
+          'this account has been temporarily suspended',
+          'this address no longer accepts mail',
           'this mailbox is disabled',
+          'user or domain is disabled',
           'user suspended',   # http://mail.163.com/help/help_spam_16.htm
           'vdelivermail: account is locked email bounced',
         ].freeze

data/lib/sisimai/reason/userunknown.rb

@@ -63,7 +63,6 @@ module Sisimai
           'no such mailbox',
           'no such person at this address',
           'no such recipient',
-
           'no such user',
           'no thank you rejected: account unavailable',
           'no valid recipients, bye',
@@ -76,7 +75,9 @@ module Sisimai
           'recipient address rejected: invalid user',
           'recipient address rejected: invalid-recipient',
           'recipient address rejected: unknown user',
+          'recipient address rejected: userunknown',
           'recipient does not exist',
+          'recipient is not accepted',
           'recipient is not local',
           'recipient not exist',
           'recipient not found',
@@ -125,6 +126,7 @@ module Sisimai
           ['no ', ' in name directory'],
           ['non', 'existent user'],
           ['rcpt <', ' does not exist'],
+          ['rcpt (', 't exist '],
           ['recipient ', ' was not found in'],
           ['recipient address rejected: user ', '  does not exist'],
           ['recipient address rejected: user unknown in ', '  table'],
@@ -135,6 +137,7 @@ module Sisimai
           ['unknown local', 'part'],
           ['user ', ' was not found'],
           ['user ', ' does not exist'],
+          ['user (', ') unknown'],
         ].freeze
 
         def text; return 'userunknown'; end

data/lib/sisimai/rfc5322.rb

@@ -3,6 +3,7 @@ module Sisimai
   module RFC5322
     class << self
       require 'sisimai/string'
+      require 'sisimai/address'
       HeaderTable = {
         :messageid => %w[message-id],
         :subject   => %w[subject],
@@ -52,79 +53,134 @@ module Sisimai
       # @return   [Array]         Received header as a structured data
       def received(argv1)
         return [] unless argv1.is_a?(::String)
+        return [] if argv1.include?(' invoked by uid')
+        return [] if argv1.include?(' invoked from network')
+
+        # - https://datatracker.ietf.org/doc/html/rfc5322
+        #   received        =   "Received:" *received-token ";" date-time CRLF
+        #   received-token  =   word / angle-addr / addr-spec / domain
+        #
+        # - Appendix A.4. Message with Trace Fields
+        #   Received:
+        #       from x.y.test
+        #       by example.net
+        #       via TCP
+        #       with ESMTP
+        #       id ABC12345
+        #       for <mary@example.net>;  21 Nov 1997 10:05:43 -0600
+        recvd = argv1.split(' ')
+        label = %w[from by via with id for]
+        token = {}
+        other = []
+        alter = []
+        right = false
+        range = recvd.size
+        index = -1
+
+        recvd.each do |e|
+          # Look up each label defined in "label" from Received header
+          index += 1
+          break unless index < range; f = e.downcase
+          next  unless label.any? { |a| f == a }
+          token[f] = recvd[index + 1] || next
+          token[f] = token[f].downcase.delete('();')
+
+          next  unless f == 'from'
+          break unless index + 2 < range
+          next  unless recvd[index + 2].start_with?('(')
+
+          # Get and keep a hostname in the comment as follows:
+          # from mx1.example.com (c213502.kyoto.example.ne.jp [192.0.2.135]) by mx.example.jp (V8/cf)
+          # [
+          #   "from",                         # index + 0
+          #   "mx1.example.com",              # index + 1
+          #   "(c213502.kyoto.example.ne.jp", # index + 2
+          #   "[192.0.2.135])",               # index + 3
+          #   "by",
+          #   "mx.example.jp",
+          #   "(V8/cf)",
+          #   ...
+          # ]
+          # The 2nd element after the current element is NOT a continuation of the current element
+          # such as "(c213502.kyoto.example.ne.jp)"
+          other << recvd[index + 2].delete('();')
+
+          # The 2nd element after the current element is a continuation of the current element.
+          # such as "(c213502.kyoto.example.ne.jp", "[192.0.2.135])"
+          break unless index + 3 < range
+          other << recvd[index + 3].delete('();')
+        end
 
-        hosts = []
-        value = { 'from' => '', 'by' => '' }
-
-        # Received: (qmail 10000 invoked by uid 999); 24 Apr 2013 00:00:00 +0900
-        return [] if argv1.include?('(qmail ') && argv1.include?(' invoked ')
-
-        p1 = argv1.index('from ')     || -1
-        p2 = argv1.index('by ')       || -1
-        p3 = argv1.index(' ', p2 + 3) || -1
-
-        if p1 == 0 && p2 > 1 && p2 < p3
-          # Received: from localhost (localhost) by nijo.example.jp (V8/cf) id s1QB5ma0018057;
-          #   Wed, 26 Feb 2014 06:05:48 -0500
-          value['from'] = Sisimai::String.sweep(argv1[p1 + 5, p2 - p1 - 5])
-          value['by']   = Sisimai::String.sweep(argv1[p2 + 3, p3 - p2 - 3])
+        other.each do |e|
+          # Check alternatives in "other", and then delete uninformative values.
+          next if e.nil?
+          next if e.size < 4
+          next if e == 'unknown'
+          next if e == 'localhost'
+          next if e == '[127.0.0.1]'
+          next if e == '[IPv6:::1]'
+          next unless e.include?('.')
+          next if e.include?('=')
+          alter << e
+        end
 
-        elsif p1 != 0 && p2 > -1
-          # Received: by 10.70.22.98 with SMTP id c2mr1838265pdf.3; Fri, 18 Jul 2014 00:31:02 -0700 (PDT)
-          value['from'] = Sisimai::String.sweep(argv1[p2 + 3, argv1.size])
-          value['by']   = Sisimai::String.sweep(argv1[p2 + 3, p3 - p2 - 3])
+        %w[from by].each do |e|
+          # Remove square brackets from the IP address such as "[192.0.2.25]"
+          next if token[e].nil?
+          next if token[e].empty?
+          next unless token[e].start_with?('[')
+          token[e] = Sisimai::String.ipv4(token[e]).shift || ''
         end
+        token['from'] ||= ''
 
-        if value['from'].include?(' ')
-          # Received: from [10.22.22.222] (smtp.kyoto.ocn.ne.jp [192.0.2.222]) (authenticated bits=0)
-          #   by nijo.example.jp (V8/cf) with ESMTP id s1QB5ka0018055; Wed, 26 Feb 2014 06:05:47 -0500
-          received = value['from'].split(' ')
-          namelist = []
-          addrlist = []
-          hostname = ''
-          hostaddr = ''
-
-          while e = received.shift do
-            # Received: from [10.22.22.222] (smtp-gateway.kyoto.ocn.ne.jp [192.0.2.222])
-            cv = Sisimai::String.ipv4(e) || []
-            if cv.size > 0
-              # [192.0.2.1] or (192.0.2.1)
-              addrlist.append(*cv)
-            else
-              # hostname
-              e = e.delete('()').strip
-              namelist << e
-            end
-          end
+        while true do
+          # Prefer hostnames over IP addresses, except for localhost.localdomain and similar.
+          break if token['from'] == 'localhost'
+          break if token['from'] == 'localhost.localdomain'
+          break unless token['from'].include?('.')  # A hostname without a domain name
+          break unless Sisimai::String.ipv4(token['from']).empty?
 
-          while e = namelist.shift do
-            # 1. Hostname takes priority over all other IP addresses
-            next unless e.include?('.')
-            hostname = e
-            break
-          end
+          # No need to rewrite token['from']
+          right = true
+          break
+        end
 
-          if hostname.empty?
-            # 2. Use IP address as a remote host name
-            addrlist.each do |e|
-              # Skip if the address is a private address
-              next if e.start_with?('10.', '127.', '192.168.')
-              next if e =~ /\A172[.](?:1[6-9]|2[0-9]|3[0-1])[.]/
-              hostaddr = e
-              break
-            end
+        while true do
+          # Try to rewrite uninformative hostnames and IP addresses in token['from']
+          break if right        # There is no need to rewrite
+          break if alter.empty? # There is no alternative to rewriting
+          break if alter[0].include?(token['from'])
+
+          if token['from'].start_with?('localhost')
+            # localhost or localhost.localdomain
+            token['from'] = alter[0]
+          elsif token['from'].index('.')
+            # A hostname without a domain name such as "mail", "mx", or "mbox"
+            token['from'] = alter[0] if alter[0].include?('.')
+          else
+            # An IPv4 address
+            token['from'] = alter[0]
           end
-
-          value['from'] = hostname || hostaddr || addrlist[-1]
+          break
         end
-
-        %w[from by].each do |e|
-          # Copy entries into hosts
-          next if value[e].empty?
-          value[e] = value[e].delete('[]();?')
-          hosts << value[e]
+        token.delete('from') if token['from'].nil?
+        token.delete('by')   if token['by'].nil?
+        token['for'] = Sisimai::Address.s3s4(token['for']) if token.has_key?('for')
+
+        token.keys.each do |e|
+          # Delete an invalid value
+          token[e] = '' if token[e].include?(' ')
+          token[e].delete!('[]')  # Remove "[]" from the IP address
         end
-        return hosts
+
+        return [
+          token['from'] || '',
+          token['by']   || '',
+          token['via']  || '',
+          token['with'] || '',
+          token['id']   || '',
+          token['for']  || '',
+        ]
       end
 
       # Split given entire message body into error message lines and the original message part only