sisimai 4.22.2 → 4.22.3

data/lib/sisimai/bite/email/imailserver.rb

@@ -106,7 +106,7 @@ module Sisimai::Bite::Email
             # Original message follows.
             v = dscontents[-1]
 
-            if cv = e.match(/\A(.+)[ ](.+)[:][ \t]*([^ ]+[@][^ ]+)/)
+            if cv = e.match(/\A([^ ]+)[ ](.+)[:][ \t]*([^ ]+[@][^ ]+)/)
               # Unknown user: kijitora@example.com
               if v['recipient']
                 # There are multiple recipient addresses in the message body.

data/lib/sisimai/bite/email/messagelabs.rb

@@ -18,7 +18,12 @@ module Sisimai::Bite::Email
         :endof   => %r/\A__END_OF_EMAIL_MESSAGE__\z/,
       }.freeze
       ReFailure = {
-        userunknown: %r/No[ ]such[ ]user/x,
+        userunknown: %r{(?:
+           542[ ].+[ ]Rejected
+          |No[ ]such[ ]user
+          )
+        }x,
+        securityerror: %r/Please turn on SMTP Authentication in your mail client/,
       }.freeze
       Indicators = Sisimai::Bite::Email.INDICATORS
 

data/lib/sisimai/bite/email/office365.rb

@@ -29,7 +29,6 @@ module Sisimai::Bite::Email
         %r/\A4[.]7[.]26\z/       => 'securityerror',
         %r/\A4[.]7[.][56]\d\d\z/ => 'blocked',
         %r/\A4[.]7[.]8[5-9]\d\z/ => 'blocked',
-        %r/\A5[.]1[.]0\z/        => 'userunknown',
         %r/\A5[.]4[.]1\z/        => 'norelaying',
         %r/\A5[.]4[.]6\z/        => 'networkerror',
         %r/\A5[.]6[.]11\z/       => 'contenterror',

data/lib/sisimai/data.rb

@@ -257,7 +257,7 @@ module Sisimai
 
         # Subject: header of the original message
         p['subject'] = rfc822data['subject'] || ''
-        p['subject'] = p['subject'].gsub(/\r\z/, '')
+        p['subject'] = p['subject'].scrub('?').gsub(/\r\z/, '')
 
         # The value of "List-Id" header
         p['listid'] = rfc822data['list-id'] || ''
@@ -285,20 +285,42 @@ module Sisimai
 
         # CHECK_DELIVERY_STATUS_VALUE:
         # Cleanup the value of "Diagnostic-Code:" header
-        p['diagnosticcode'] = p['diagnosticcode'].sub(/[ \t]+#{EndOfEmail}/, '')
-        d = Sisimai::SMTP::Status.find(p['diagnosticcode'])
-        if d =~ /\A[45][.][1-9][.][1-9]\z/
-          # Use the DSN value in Diagnostic-Code:
-          p['deliverystatus'] = d
-        end
+        p['diagnosticcode'] = p['diagnosticcode'].sub(/[ \t.]+#{EndOfEmail}/, '')
+
+        if p['diagnosticcode'].size > 0
+          # Count the number of D.S.N. and SMTP Reply Code
+          vs = Sisimai::SMTP::Status.find(p['diagnosticcode'])
+          vr = Sisimai::SMTP::Reply.find(p['diagnosticcode'])
+          vm = 0
+          re = nil
+
+          if vs.size > 0
+            # How many times does the D.S.N. appeared
+            vm += p['diagnosticcode'].scan(/\b#{vs}\b/).size
+            p['deliverystatus'] = vs if vs =~ /\A[45][.][1-9][.][1-9]\z/
+          end
 
-        if p['reason'] == 'mailererror'
-          p['diagnostictype'] ||= 'X-UNIX'
-        else
-          unless p['reason'] =~ /\A(?:feedback|vacation)\z/
-            p['diagnostictype'] ||= 'SMTP'
+          if vr.size > 0
+            # How many times does the SMTP reply code appeared
+            vm += p['diagnosticcode'].scan(/\b#{vr}\b/).size
+            p['replycode'] ||= vr
+          end
+
+          if vm > 2
+            # Build regular expression for removing string like '550-5.1.1'
+            # from the value of "diagnosticcode"
+            re = %r/[ ]#{vr}[- ](?:#{vs})?/
+
+            # 550-5.7.1 [192.0.2.222] Our system has detected that this message is
+            # 550-5.7.1 likely unsolicited mail. To reduce the amount of spam sent to Gmail,
+            # 550-5.7.1 this message has been blocked. Please visit
+            # 550 5.7.1 https://support.google.com/mail/answer/188131 for more information.
+            p['diagnosticcode'] = p['diagnosticcode'].gsub(re, ' ')
+            p['diagnosticdoee'] = Sisimai::String.sweep(p['diagnosticcode'])
           end
         end
+        p['diagnostictype'] ||= 'X-UNIX' if p['reason'] == 'mailererror'
+        p['diagnostictype'] ||= 'SMTP' unless p['reason'] =~ /\A(?:feedback|vacation)\z/
 
         # Check the value of SMTP command
         p['smtpcommand'] = '' unless p['smtpcommand'] =~ rxcommands

data/lib/sisimai/datetime.rb

@@ -346,12 +346,12 @@ module Sisimai
           elsif cr = p.match(/\A([0-2]\d):([0-5]\d)\z/)
             # Time; 12:34 => 12:34:00
             if cr[1].to_i < 24 && cr[2].to_i < 60
-              v[:T] = sprintf('%02d:%02d:00', cr[1], cr[2])
+              v[:T] = sprintf('%02d:%02d:00', cr[1].to_i, cr[2].to_i)
             end
 
           elsif cr = p.match(/\A(\d\d?):(\d\d?)\z/)
             # Time: 1:4 => 01:04:00
-            v[:T] = sprintf('%02d:%02d:00', cr[1], cr[2])
+            v[:T] = sprintf('%02d:%02d:00', cr[1].to_i, cr[2].to_i)
 
           elsif p =~ /\A[APap][Mm]\z/
             # AM or PM
@@ -382,7 +382,7 @@ module Sisimai
                 v[:d] = cr[3].to_i if cr[3].to_i < 32
 
                 if cr[4].to_i < 24 && cr[5].to_i < 60 && cr[6].to_i < 60
-                  v[:T] = sprintf('%02d:%02d:%02d', cr[4], cr[5], cr[6])
+                  v[:T] = sprintf('%02d:%02d:%02d', cr[4].to_i, cr[5].to_i, cr[6].to_i)
                 end
 
               elsif cr = p.match(%r|\A(\d{1,2})/(\d{1,2})/(\d{1,2})\z|)
@@ -391,6 +391,12 @@ module Sisimai
                 v[:d]  = cr[2].to_i
                 v[:Y]  = cr[3].to_i + 2000
                 v[:Y] -= 100 if v[:Y].to_i > ::DateTime.now.year + 1
+
+              elsif cr = p.match(%r|\A(\d{1,2})[-/](\d{1,2})[-/](\d{4})|)
+                # 29-04-2017 22:22
+                v[:d] = cr[1].to_i if cr[1].to_i < 32
+                v[:M] = MonthName[:abbr][cr[2].to_i - 1]
+                v[:Y] = cr[3].to_i
               end
             end
           end
@@ -400,7 +406,7 @@ module Sisimai
           # +12
           t0 = v[:T]
           t1 = v[:T].split(':')
-          v[:T] = sprintf('%02d:%02d:%02d', t1[0].to_i + 12, t1[1], t1[2])
+          v[:T] = sprintf('%02d:%02d:%02d', t1[0].to_i + 12, t1[1].to_i, t1[2].to_i)
           v[:T] = t0 if t1[0].to_i > 12
         end
         v[:a] ||= 'Thu' # There is no day of week

data/lib/sisimai/reason.rb

@@ -8,7 +8,10 @@ module Sisimai
       # Reason list better to retry detecting an error reason
       # @return   [Array] Reason list
       def retry
-        return %w|undefined onhold systemerror securityerror networkerror|
+        return %w|
+          undefined onhold systemerror securityerror networkerror
+          hostunknown userunknown
+        |
       end
       RetryReasons = Sisimai::Reason.retry
 
@@ -72,14 +75,16 @@ module Sisimai
 
           if reasontext == 'undefined' || reasontext.empty?
             # Action: delayed => "expired"
-            if reasontext.nil? || reasontext.empty?
-              reasontext = 'expired' if argvs.action == 'delayed'
-            end
-            if reasontext.nil? || reasontext.empty?
-              reasontext = 'onhold'  if argvs.diagnosticcode.size > 0
+            reasontext   = nil
+            reasontext ||= 'expired' if argvs.action == 'delayed'
+            unless reasontext
+              # Try to match with message patterns in Sisimai::Reason::Vacation
+              require 'sisimai/reason/vacation'
+              reasontext = 'vacation' if Sisimai::Reason::Vacation.match(argvs.diagnosticcode)
             end
+            reasontext ||= 'onhold'  if argvs.diagnosticcode.size > 0
           end
-          reasontext = 'undefined' if reasontext.nil? || reasontext.empty?
+          reasontext ||= 'undefined'
         end
         return reasontext
       end
@@ -97,6 +102,7 @@ module Sisimai
         statuscode = argvs.deliverystatus || ''
         diagnostic = argvs.diagnosticcode || ''
         commandtxt = argvs.smtpcommand    || ''
+        trytomatch = nil
         reasontext = ''
         classorder = %w|
           MailboxFull SpamDetected PolicyViolation VirusDetected SecurityError
@@ -107,60 +113,69 @@ module Sisimai
         require 'sisimai/smtp/status'
         reasontext = Sisimai::SMTP::Status.name(statuscode)
 
-        if reasontext.empty? || reasontext == 'userunknown' || RetryReasons.index(reasontext)
-          # Could not decide the reason by the value of Status:
-          classorder.each do |e|
-            # Trying to match with other patterns in Sisimai::Reason::* classes
-            p = 'Sisimai::Reason::' + e
-            r = nil
-            begin
-              require p.downcase.gsub('::', '/')
-              r = Module.const_get(p)
-            rescue
-              warn ' ***warning: Failed to load ' + p
-              next
-            end
+        catch :TRY_TO_MATCH do
+          loop do
+            trytomatch ||= true if reasontext.empty?
+            trytomatch ||= true if RetryReasons.index(reasontext)
+            trytomatch ||= true if argvs.diagnostictype != 'SMTP'
+            throw :TRY_TO_MATCH unless trytomatch
+
+            # Could not decide the reason by the value of Status:
+            classorder.each do |e|
+              # Trying to match with other patterns in Sisimai::Reason::* classes
+              p = 'Sisimai::Reason::' + e
+              r = nil
+              begin
+                require p.downcase.gsub('::', '/')
+                r = Module.const_get(p)
+              rescue
+                warn ' ***warning: Failed to load ' + p
+                next
+              end
 
-            next unless r.match(diagnostic)
-            reasontext = e.downcase
-            break
-          end
+              next unless r.match(diagnostic)
+              reasontext = e.downcase
+              break
+            end
 
-          if reasontext.empty?
-            # Check the value of Status:
-            v = statuscode[0, 3]
-            if v == '5.6' || v == '4.6'
-              #  X.6.0   Other or undefined media error
-              reasontext = 'contenterror'
-
-            elsif v == '5.7' || v == '4.7'
-              #  X.7.0   Other or undefined security status
-              reasontext = 'securityerror'
-
-            elsif argvs.diagnostictype =~ /\AX-(?:UNIX|POSTFIX)\z/
-              # Diagnostic-Code: X-UNIX; ...
-              reasontext = 'mailererror'
-
-            else
-              # 50X Syntax Error?
-              require 'sisimai/reason/syntaxerror'
-              reasontext = 'syntaxerror' if Sisimai::Reason::SyntaxError.true(argvs)
+            if reasontext.empty?
+              # Check the value of Status:
+              v = statuscode[0, 3]
+              if v == '5.6' || v == '4.6'
+                #  X.6.0   Other or undefined media error
+                reasontext = 'contenterror'
+
+              elsif v == '5.7' || v == '4.7'
+                #  X.7.0   Other or undefined security status
+                reasontext = 'securityerror'
+
+              elsif argvs.diagnostictype =~ /\AX-(?:UNIX|POSTFIX)\z/
+                # Diagnostic-Code: X-UNIX; ...
+                reasontext = 'mailererror'
+
+              else
+                # 50X Syntax Error?
+                require 'sisimai/reason/syntaxerror'
+                reasontext = 'syntaxerror' if Sisimai::Reason::SyntaxError.true(argvs)
+              end
             end
-          end
 
-          if reasontext.empty?
-            # Check the value of Action: field, first
-            if argvs.action =~ /\A(?:delayed|expired)/
-              # Action: delayed, expired
-              reasontext = 'expired'
-
-            else
-              # Check the value of SMTP command
-              if commandtxt =~ /\A(?:EHLO|HELO)\z/
-                # Rejected at connection or after EHLO|HELO
-                reasontext = 'blocked'
+            if reasontext.empty?
+              # Check the value of Action: field, first
+              if argvs.action =~ /\A(?:delayed|expired)/
+                # Action: delayed, expired
+                reasontext = 'expired'
+
+              else
+                # Check the value of SMTP command
+                if commandtxt =~ /\A(?:EHLO|HELO)\z/
+                  # Rejected at connection or after EHLO|HELO
+                  reasontext = 'blocked'
+                end
               end
             end
+
+            throw :TRY_TO_MATCH
           end
         end
         return reasontext

data/lib/sisimai/reason/blocked.rb

@@ -23,56 +23,126 @@ module Sisimai
           regex = %r{(?>
              access[ ]denied[.][ ]IP[ ]name[ ]lookup[ ]failed
             |access[ ]from[ ]ip[ ]address[ ].+[ ]blocked
+            |all[ ]mail[ ]servers[ ]must[ ]have[ ]a[ ]PTR[ ]record[ ]with[ ]a[ ]valid[ ]Reverse[ ]DNS[ ]entry
+            |Bad[ ]sender[ ]IP[ ]address
             |blacklisted[ ]by
-            |Blocked[ ]-[ ]see[ ]https://support[.]proofpoint[.]com/dnsbl-lookup[.]cgi[?]ip=.+
+            |(?:Blocked|Refused)[ ]-[ ]see[ ]https?://
             |can[']t[ ]determine[ ]Purported[ ]Responsible[ ]Address
             |cannot[ ]resolve[ ]your[ ]address
-            |client[ ]host[ ].+[ ]blocked[ ]using
-            |client[ ]host[ ]rejected:[ ](?:
-               may[ ]not[ ]be[ ]mail[ ]exchanger
-              |cannot[ ]find[ ]your[ ]hostname    # Yahoo!
-              |was[ ]not[ ]authenticated          # Microsoft
+            |client[ ]host[ ](?:
+               .+[ ]blocked[ ]using
+              |rejected:[ ](?:
+                 Abus[ ]detecte[ ]GU_EIB_0[24]      # SFR
+                |cannot[ ]find[ ]your[ ]hostname    # Yahoo!
+                |may[ ]not[ ]be[ ]mail[ ]exchanger
+                |was[ ]not[ ]authenticated          # Microsoft
+                )
               )
             |confirm[ ]this[ ]mail[ ]server
             |connection[ ](?:
-              dropped
-             |refused[ ]by
-             |reset[ ]by[ ]peer
-             |was[ ]dropped[ ]by[ ]remote[ ]host
-             )
-            |domain[ ]does[ ]not[ ]exist:
-            |domain[ ].+[ ]mismatches[ ]client[ ]ip
+               dropped
+              |refused[ ]by
+              |reset[ ]by[ ]peer
+              |was[ ]dropped[ ]by[ ]remote[ ]host
+              )
+            |Connections[ ](?:
+               not[ ]accepted[ ]from[ ]IP[ ]addresses[ ]on[ ]Spamhaus[ ]XBL
+              |will[ ]not[ ]be[ ]accepted[ ]from[ ].+because[ ]the[ ]ip[ ]is[ ]in[ ]Spamhaus's[ ]list
+              )
+            |Currently[ ]Sending[ ]Spam[ ]See:[ ]
+            |domain[ ](?:
+               .+[ ]mismatches[ ]client[ ]ip
+              |does[ ]not[ ]exist:
+              )
             |dns[ ]lookup[ ]failure:[ ].+[ ]try[ ]again[ ]later
             |DNSBL:ATTRBL
+            |Dynamic/zombied/spam[ ]IPs[ ]blocked
+            |Email[ ]blocked[ ]by[ ](?:.+[.]barracudacentral[.]org|SPAMHAUS)
+            |Fix[ ]reverse[ ]DNS[ ]for[ ].+
             |Go[ ]away
+            |host[ ].+[ ]refused[ ]to[ ]talk[ ]to[ ]me:[ ]\d+[ ]Blocked
             |hosts[ ]with[ ]dynamic[ ]ip
+            |http://(?:
+               spf[.]pobox[.]com/why[.]html
+              |www[.]spamcop[.]net/bl[.]
+              )
+            |INVALID[ ]IP[ ]FOR[ ]SENDING[ ]MAIL[ ]OF[ ]DOMAIN
             |IP[ ]\d{1,3}[.]\d{1,3}[.]\d{1,3}[.]\d{1,3}[ ]is[ ]blocked[ ]by[ ]EarthLink # Earthlink
             |IP[/]domain[ ]reputation[ ]problems
-            |is[ ]not[ ]allowed[ ]to[ ]send[ ]mail[ ]from
+            |is[ ](?:
+               in[ ]a[ ]black[ ]list[ ]at[ ].+[.]
+              |in[ ]an[ ].*RBL[ ]on[ ].+
+              |not[ ]allowed[ ]to[ ]send[ ](?:
+                 mail[ ]from
+                |from[ ].+[ ]per[ ]it's[ ]SPF[ ]Record
+                )
+              )
             |mail[ ]server[ ]at[ ].+[ ]is[ ]blocked
+            |Mail[ ]from[ ]\d+[.]\d+[.]\d+[.]\d[ ]refused:
+            |Message[ ]from[ ].+[ ]rejected[ ]based[ ]on[ ]blacklist
             |Messages[ ]from[ ].+[ ]temporarily[ ]deferred[ ]due[ ]to[ ]user[ ]complaints   # Yahoo!
-            |no[ ]access[ ]from[ ]mail[ ]server
+            |no[ ](?:
+               access[ ]from[ ]mail[ ]server
+              |PTR[ ]Record[ ]found[.]
+              )
             |Not[ ]currently[ ]accepting[ ]mail[ ]from[ ]your[ ]ip  # Microsoft
-            |Please[ ]get[ ]a[ ]custom[ ]reverse[ ]DNS[ ]name[ ]from[ ]your[ ]ISP[ ]for[ ]your[ ]host
-            |please[ ]use[ ]the[ ]smtp[ ]server[ ]of[ ]your[ ]ISP
+            |part[ ]of[ ]their[ ]network[ ]is[ ]on[ ]our[ ]block[ ]list
+            |Please[ ](?:
+               get[ ]a[ ]custom[ ]reverse[ ]DNS[ ]name[ ]from[ ]your[ ]ISP[ ]for[ ]your[ ]host
+              |inspect[ ]your[ ]SPF[ ]settings
+              |use[ ]the[ ]smtp[ ]server[ ]of[ ]your[ ]ISP
+              )
+            |PTR[ ]record[ ]setup
             |Rejecting[ ]open[ ]proxy   # Sendmail(srvrsmtp.c)
+            |Reverse[ ]DNS[ ](?:
+               failed
+              |required
+              |lookup[ ]for[ ]host[ ].+[ ]failed[ ]permanently
+              )
+            |Sender[ ]IP[ ](?:
+               address[ ]rejected
+              |reverse[ ]lookup[ ]rejected
+              )
+            |Server[ ]access[ ](?:
+               .+[ ]forbidden[ ]by[ ]invalid[ ]RDNS[ ]record[ ]of[ ]your[ ]mail[ ]server
+              |forbidden[ ]by[ ]your[ ]IP[ ]
+              )
+            |Server[ ]IP[ ].+[ ]listed[ ]as[ ]abusive
+            |service[ ]permits[ ]\d+[ ]unverifyable[ ]sending[ ]IPs
+            |SMTP[ ]error[ ]from[ ]remote[ ]mail[ ]server[ ]after[ ]initial[ ]connection:   # Exim
             |sorry,[ ](?:
                that[ ]domain[ ]isn'?t[ ]in[ ]my[ ]list[ ]of[ ]allowed[ ]rcpthosts
               |your[ ]remotehost[ ]looks[ ]suspiciously[ ]like[ ]spammer
               )
-            |SPF[ ]record
+            |SPF[ ](?:
+               .+[ ]domain[ ]authentication[ ]fail
+              |record
+              |check:[ ]fail
+              )
+            |SPF:[ ].+[ ]is[ ]not[ ]allowed[ ]to[ ]send[ ]mail.+[A-Z]{3}.+401
             |the[ ](?:email|domain|ip).+[ ]is[ ]blacklisted
+            |This[ ]system[ ]will[ ]not[ ]accept[ ]messages[ ]from[ ]servers[/]devices[ ]with[ ]no[ ]reverse[ ]DNS
+            |Too[ ]many[ ]spams[ ]from[ ]your[ ]IP  # free.fr
             |unresolvable[ ]relay[ ]host[ ]name
+            |Veuillez[ ]essayer[ ]plus[ ]tard.+[A-Z]{3}.+(?:103|510)
             |your[ ](?:
                network[ ]is[ ]temporary[ ]blacklisted
+              |sender's[ ]IP[ ]address[ ]is[ ]listed[ ]at[ ].+[.]abuseat[.]org
               |server[ ]requires[ ]confirmation
               )
             |was[ ]blocked[ ]by[ ].+
             |we[ ]do[ ]not[ ]accept[ ]mail[ ]from[ ](?: # @mail.ru
-               hosts[ ]with[ ]dynamic[ ]IP[ ]or[ ]generic[ ]dns[ ]PTR-records
-              |dynamic[ ]ips
+               dynamic[ ]ips
+              |hosts[ ]with[ ]dynamic[ ]IP[ ]or[ ]generic[ ]dns[ ]PTR-records
+              )
+            |You[ ]are[ ](?:
+               not[ ]allowed[ ]to[ ]connect
+              |sending[ ]spam
+              )
+            |Your[ ](?:
+               access[ ]to[ ]submit[ ]messages[ ]to[ ]this[ ]e-mail[ ]system[ ]has[ ]been[ ]rejected
+              |message[ ]was[ ]rejected[ ]for[ ]possible[ ]spam/virus[ ]content
               )
-            |http://www[.]spamcop[.]net/bl[.]
             )
           }xi