sisimai 4.25.3 → 4.25.8

data/lib/sisimai/mime.rb

@@ -24,12 +24,6 @@ module Sisimai
         )
       }x.freeze
 
-      # Make MIME-Encoding and Content-Type related headers regurlar expression
-      # @return   [Array] Regular expressions related to MIME encoding
-      def patterns
-        return ReE
-      end
-
       # Check that the argument is MIME-Encoded string or not
       # @param    [String] argvs  String to be checked
       # @return   [True,False]    false: Not MIME encoded string
@@ -106,7 +100,7 @@ module Sisimai
           end
         end
 
-        return decodedtext1.force_encoding('UTF-8')
+        return decodedtext1.force_encoding('UTF-8').scrub('?')
       end
 
       # Decode MIME Quoted-Printable Encoded string
@@ -115,8 +109,8 @@ module Sisimai
       # @return [String]         MIME Decoded text
       def qprintd(argv1 = nil, heads = {})
         return nil unless argv1
-        return argv1.unpack('M').first unless heads['content-type']
-        return argv1.unpack('M').first if heads['content-type'].empty?
+        return argv1.unpack('M').first.scrub('?') unless heads['content-type']
+        return argv1.unpack('M').first.scrub('?') if heads['content-type'].empty?
 
         # Quoted-printable encoded part is the part of the text
         boundary00 = Sisimai::MIME.boundary(heads['content-type'], 0)
@@ -124,8 +118,8 @@ module Sisimai
         # Decoded using unpack('M') entire body string when the boundary string
         # or "Content-Transfer-Encoding: quoted-printable" are not included in
         # the message body.
-        return argv1.unpack('M').first if boundary00.empty?
-        return argv1.unpack('M').first unless argv1.downcase =~ ReE[:'quoted-print']
+        return argv1.unpack('M').first.scrub('?') if boundary00.empty?
+        return argv1.unpack('M').first.scrub('?') unless argv1.downcase =~ ReE[:'quoted-print']
 
         boundary01 = Sisimai::MIME.boundary(heads['content-type'], 1)
         bodystring = ''
@@ -134,9 +128,9 @@ module Sisimai
         encodename = nil
         ctencoding = nil
         mimeinside = false
-        hasdivided = argv1.split("\n")
+        textslices = argv1.split("\n")
 
-        while e = hasdivided.shift do
+        while e = textslices.shift do
           # This is a multi-part message in MIME format. Your mail reader does not
           # understand MIME message format.
           # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
@@ -167,7 +161,7 @@ module Sisimai
                 #
                 # Content-Transfer-Encoding: quoted-printable
                 # X-Microsoft-Exchange-Diagnostics:
-                #     1;SLXP216MB0381;27:IdH7U/WHGgJu6J8lFrE7KvVxhnAwyKrNbSXMFYs3/Gzz6ZdXYYjzHj55K2O+cndpeVwkvBJqmo6y0IF4AhLfHtFzznw/BzhERU6wi/TCWRpyjYuW8v0/aTcflH3oAdgZ4Pwrp7PxLiiA8rYgU/E7SQ==
+                #     1;SLXP216MB0381;27:IdH7U/WHGgJu6J...LiiA8rYgU/E7SQ==
                 # ...
                 mustencode = true
                 while true do
@@ -177,10 +171,8 @@ module Sisimai
                     # Padding character of Base64 or not
                     break if e =~ /[\+\/0-9A-Za-z]{32,}[=]+\z/
                   else
-                    if e.include?('=') && ! e.upcase.include?('=3D')
-                      # Including "=" not as "=3D"
-                      break
-                    end
+                    # Including "=" not as "=3D"
+                    break if e.include?('=') && ! e.upcase.include?('=3D')
                   end
                   mustencode = false
                   break
@@ -222,7 +214,7 @@ module Sisimai
         end
 
         bodystring << notdecoded unless notdecoded.empty?
-        return bodystring
+        return bodystring.scrub('?')
       end
 
       # Decode MIME BASE64 Encoded string
@@ -251,6 +243,7 @@ module Sisimai
           # Content-Type: multipart/report; report-type=delivery-status;
           #    boundary="n6H9lKZh014511.1247824040/mx.example.jp"
           value = cv[1]
+          value = value.split(/\n/, 2).shift if value =~ /\n/
           value.delete!(%q|'";\\|)
           value = '--' + value if start > -1
           value = value + '--' if start >  0
@@ -278,8 +271,8 @@ module Sisimai
         return '' unless mimeformat =~ LeavesOnly
         return '' if alternates && mimeformat == 'text/html'
 
-        (upperchunk, lowerchunk) = argv0.split(/^$/m, 2)
-        upperchunk.tr!("\n", ' ').squeeze(' ')
+        (upperchunk, lowerchunk) = argv0.split(/^$/, 2)
+        upperchunk.tr("\n", ' ').squeeze!(' ')
 
         # Content-Description: Undelivered Message
         # Content-Type: message/rfc822
@@ -290,18 +283,16 @@ module Sisimai
           # Content-Type: multipart/*
           mpboundary = Regexp.new(Regexp.escape(Sisimai::MIME.boundary(upperchunk, 0)) << "\n")
           innerparts = lowerchunk.split(mpboundary)
+          innerparts.shift if innerparts[0].empty? || innerparts[0] == "\n"
 
-          innerparts.shift if innerparts[0].empty?
           while e = innerparts.shift do
             # Find internal multipart/* blocks and decode
             if cv = e.match(ThisFormat)
               # Found "Content-Type" field at the first or second line of this
               # splitted part
               nextformat = cv[1].downcase
-
               next unless nextformat =~ LeavesOnly
               next if nextformat == 'text/html'
-
               hasflatten << Sisimai::MIME.breaksup(e, mimeformat)
             else
               # The content of this part is almost '--': a part of boundary
@@ -405,8 +396,8 @@ module Sisimai
         # Some bounce messages include lower-cased "content-type:" field such as
         #   content-type: message/delivery-status
         #   content-transfer-encoding: quoted-printable
-        argv1.gsub!(/[Cc]ontent-[Tt]ype:/m, 'Content-Type:')
-        argv1.gsub!(/[Cc]ontent-[Tt]ransfer-[Ee]ncodeing:/m, 'Content-Transfer-Encoding:')
+        argv1.gsub!(/[Cc]ontent-[Tt]ype:/, 'Content-Type:')
+        argv1.gsub!(/[Cc]ontent-[Tt]ransfer-[Ee]ncodeing:/, 'Content-Transfer-Encoding:')
 
         # 1. Some bounce messages include upper-cased "Content-Transfer-Encoding",
         #    and "Content-Type" value such as
@@ -414,11 +405,10 @@ module Sisimai
         #      - Content-Transfer-Encoding: 7BIT
         # 2. Unused fields inside of mutipart/* block should be removed
         argv1.gsub!(/(Content-[A-Za-z-]+?):[ ]*([^\s]+)/) do "#{$1}: #{$2.downcase}" end
-        argv1.gsub!(/^Content-(?:Description|Disposition):.+?$/, '')
+        argv1.gsub!(/^Content-(?:Description|Disposition):.+?\n$/, '')
 
-        multiparts = argv1.split(Regexp.new(Regexp.escape(ehboundary) << "\n"))
+        multiparts = argv1.split(Regexp.new(Regexp.escape(ehboundary) << "\n?"))
         multiparts.shift if multiparts[0].empty?
-
         while e = multiparts.shift do
           # Find internal multipart blocks and decode
           catch :XCCT do
@@ -430,11 +420,16 @@ module Sisimai
               #   Message-ID: ...
               #   Content-Transfer-Encoding: quoted-printable
               #   Content-Type: text/plain; charset=us-ascii
-              throw :XCCT if e =~ /\AContent-T[ry]/
-              if cv = e.match(/\A(.+?)Content-Type:/)
-                throw :XCCT if cv[1] =~ /\n\n/
+              #
+              # Fields before "Content-Type:" in each part should have been removed
+              # and "Content-Type:" should be exist at the first line of each part.
+              # The field works as a delimiter to decode contents of each part.
+              #
+              throw :XCCT if e =~ /\AContent-T[ry]/       # The first field is "Content-Type:"
+              if cv = e.match(/\A(.+?)Content-Type:/m)    # Capture lines before "Content-Type:"
+                throw :XCCT if cv[1] =~ /\n\n/            # There is no field before "Content-Type:"
+                e.sub!(/\A.+?(Content-T[ry].+)\z/m, '\1') # Remove fields before "Content-Type:"
               end
-              e.sub!(/\A.+?(Content-T[ry].+)\z/, '\1')
               throw :XCCT
             end
           end

data/lib/sisimai/order.rb

@@ -4,11 +4,186 @@ module Sisimai
   module Order
     # Imported from p5-Sisimail/lib/Sisimai/Order.pm
     class << self
-      def by;      return {}; end
-      def default; return []; end
-      def another; return []; end
-      def headers; return {}; end
+      require 'sisimai/lhost'
 
+      # There are another patterns in the value of "Subject:" header of a bounce
+      # mail generated by the following MTA/ESP modules
+      OrderE0 = [
+        'Sisimai::Lhost::MailRu',
+        'Sisimai::Lhost::Yandex',
+        'Sisimai::Lhost::Exim',
+        'Sisimai::Lhost::Sendmail',
+        'Sisimai::Lhost::Aol',
+        'Sisimai::Lhost::Office365',
+        'Sisimai::Lhost::Exchange2007',
+        'Sisimai::Lhost::Exchange2003',
+        'Sisimai::Lhost::AmazonWorkMail',
+        'Sisimai::Lhost::AmazonSES',
+        'Sisimai::Lhost::Barracuda',
+        'Sisimai::Lhost::InterScanMSS',
+        'Sisimai::Lhost::KDDI',
+        'Sisimai::Lhost::SurfControl',
+        'Sisimai::Lhost::Verizon',
+        'Sisimai::Lhost::ApacheJames',
+        'Sisimai::Lhost::X2',
+        'Sisimai::Lhost::X5',
+        'Sisimai::Lhost::FML',
+      ].freeze
+
+      # Fallback list: The following MTA/ESP modules is not listed OrderE0
+      OrderE1 = [
+        'Sisimai::Lhost::Postfix',
+        'Sisimai::Lhost::GSuite',
+        'Sisimai::Lhost::Yahoo',
+        'Sisimai::Lhost::Outlook',
+        'Sisimai::Lhost::GMX',
+        'Sisimai::Lhost::MessagingServer',
+        'Sisimai::Lhost::EinsUndEins',
+        'Sisimai::Lhost::Domino',
+        'Sisimai::Lhost::Notes',
+        'Sisimai::Lhost::Qmail',
+        'Sisimai::Lhost::Courier',
+        'Sisimai::Lhost::OpenSMTPD',
+        'Sisimai::Lhost::Zoho',
+        'Sisimai::Lhost::MessageLabs',
+        'Sisimai::Lhost::MXLogic',
+        'Sisimai::Lhost::MailFoundry',
+        'Sisimai::Lhost::McAfee',
+        'Sisimai::Lhost::V5sendmail',
+        'Sisimai::Lhost::MFILTER',
+        'Sisimai::Lhost::SendGrid',
+        'Sisimai::Lhost::ReceivingSES',
+        'Sisimai::Lhost::Amavis',
+        'Sisimai::Lhost::PowerMTA',
+        'Sisimai::Lhost::GoogleGroups',
+        'Sisimai::Lhost::Gmail',
+        'Sisimai::Lhost::EZweb',
+        'Sisimai::Lhost::IMailServer',
+        'Sisimai::Lhost::MailMarshalSMTP',
+        'Sisimai::Lhost::Activehunter',
+        'Sisimai::Lhost::Bigfoot',
+        'Sisimai::Lhost::Biglobe',
+        'Sisimai::Lhost::Facebook',
+        'Sisimai::Lhost::X4',
+        'Sisimai::Lhost::X1',
+        'Sisimai::Lhost::X3',
+        'Sisimai::Lhost::X6',
+      ].freeze
+
+      # The following order is decided by the first 2 words of Subject: header
+      Subject = {
+        'abuse-report'     => ['Sisimai::ARF'],
+        'auto'             => ['Sisimai::RFC3834'],
+        'auto-reply'       => ['Sisimai::RFC3834'],
+        'automatic-reply'  => ['Sisimai::RFC3834'],
+        'aws-notification' => ['Sisimai::Lhost::AmazonSES'],
+        'complaint-about'  => ['Sisimai::ARF'],
+        'delivery-failure' => ['Sisimai::Lhost::Domino', 'Sisimai::Lhost::X2'],
+        'delivery-notification' => ['Sisimai::Lhost::MessagingServer'],
+        'delivery-report'  => ['Sisimai::Lhost::PowerMTA'],
+        'delivery-status'  => [
+          'Sisimai::Lhost::GSuite',
+          'Sisimai::Lhost::GoogleGroups',
+          'Sisimai::Lhost::Outlook',
+          'Sisimai::Lhost::McAfee',
+          'Sisimai::Lhost::OpenSMTPD',
+          'Sisimai::Lhost::AmazonSES',
+          'Sisimai::Lhost::AmazonWorkMail',
+          'Sisimai::Lhost::ReceivingSES',
+          'Sisimai::Lhost::Gmail',
+          'Sisimai::Lhost::X3',
+        ],
+        'dmarc-ietf-dmarc' => ['Sisimai::ARF'],
+        'email-feedback'   => ['Sisimai::ARF'],
+        'failed-delivery'  => ['Sisimai::Lhost::X2'],
+        'failure-delivery' => ['Sisimai::Lhost::X2'],
+        'failure-notice'   => [
+          'Sisimai::Lhost::Yahoo',
+          'Sisimai::Lhost::Qmail',
+          'Sisimai::Lhost::MFILTER',
+          'Sisimai::Lhost::Activehunter',
+          'Sisimai::Lhost::X4',
+        ],
+        'loop-alert'    => ['Sisimai::Lhost::FML'],
+        'mail-delivery' => [
+          'Sisimai::Lhost::Exim',
+          'Sisimai::Lhost::MailRu',
+          'Sisimai::Lhost::GMX',
+          'Sisimai::Lhost::EinsUndEins',
+          'Sisimai::Lhost::Zoho',
+          'Sisimai::Lhost::MessageLabs',
+          'Sisimai::Lhost::MXLogic',
+        ],
+        'mail-could'   => ['Sisimai::Lhost::InterScanMSS'],
+        'mail-failure' => ['Sisimai::Lhost::Exim'],
+        'mail-not'     => ['Sisimai::Lhost::X4'],
+        'mail-system'  => ['Sisimai::Lhost::EZweb'],
+        'message-delivery' => ['Sisimai::Lhost::MailFoundry'],
+        'message-frozen'   => ['Sisimai::Lhost::Exim'],
+        'message-you'      => ['Sisimai::Lhost::Barracuda'],
+        'non-recapitabile' => ['Sisimai::Lhost::Exchange2007'],
+        'non-remis' => ['Sisimai::Lhost::Exchange2007'],
+        'notice'    => ['Sisimai::Lhost::Courier'],
+        'permanent-delivery' => ['Sisimai::Lhost::X4'],
+        'postmaster-notify'  => ['Sisimai::Lhost::Sendmail'],
+        'returned-mail' => [
+          'Sisimai::Lhost::Sendmail',
+          'Sisimai::Lhost::Aol',
+          'Sisimai::Lhost::V5sendmail',
+          'Sisimai::Lhost::Bigfoot',
+          'Sisimai::Lhost::Biglobe',
+          'Sisimai::Lhost::X1',
+        ],
+        'sorry-your' => ['Sisimai::Lhost::Facebook'],
+        'there-was'  => ['Sisimai::Lhost::X6'],
+        'undeliverable' => [
+          'Sisimai::Lhost::Office365',
+          'Sisimai::Lhost::Exchange2007',
+          'Sisimai::Lhost::Aol',
+          'Sisimai::Lhost::Exchange2003',
+        ],
+        'undeliverable-mail' => [
+          'Sisimai::Lhost::Amavis',
+          'Sisimai::Lhost::MailMarshalSMTP',
+          'Sisimai::Lhost::IMailServer',
+        ],
+        'undeliverable-message' => ['Sisimai::Lhost::Notes', 'Sisimai::Lhost::Verizon'],
+        'undelivered-mail' => [
+          'Sisimai::Lhost::Postfix',
+          'Sisimai::Lhost::Aol',
+          'Sisimai::Lhost::SendGrid',
+          'Sisimai::Lhost::Zoho',
+        ],
+        'warning' => ['Sisimai::Lhost::Sendmail', 'Sisimai::Lhost::Exim'],
+      }.freeze
+
+      # @abstract Returns an MTA Order decided by the first word of the "Subject": header
+      # @param    [String] argv0 Subject header string
+      # @return   [Array]        Order of MTA modules
+      # @since    v4.25.4
+      def make(argv0 = '')
+        return [] if argv0.empty?
+        argv0 = argv0.downcase.tr('_[] ', ' ').squeeze(' ').sub(/\A[ ]+/, '')
+        words = argv0.split(/[ ]/, 3)
+
+        if words[0].include?(':')
+          # Undeliverable: ..., notify: ...
+          first = argv0.split(':').shift
+        else
+          # Postmaster notify, returned mail, ...
+          first = words.slice(0, 2).join('-')
+        end
+        first.delete!(':",*')
+        return Subject[first] || []
+      end
+
+      # @abstract Make default order of MTA modules to be loaded
+      # @return   [Array] Default order list of MTA modules
+      def default; return Sisimai::Lhost.index.map { |e| 'Sisimai::Lhost::' << e }; end
+
+      # @abstract Make MTA module list as a spare
+      # @return   [Array] Ordered module list
+      def another; return [OrderE0, OrderE1].flatten; end
     end
   end
 end

data/lib/sisimai/reason.rb

@@ -5,14 +5,36 @@ module Sisimai
   module Reason
     # Imported from p5-Sisimail/lib/Sisimai/Reason.pm
     class << self
+      # All the error reason list Sisimai support
+      # @return   [Array] Reason list
+      def index
+        return %w[
+          Blocked ContentError ExceedLimit Expired Filtered HasMoved HostUnknown
+          MailboxFull MailerError MesgTooBig NetworkError NotAccept OnHold
+          Rejected NoRelaying SpamDetected VirusDetected PolicyViolation SecurityError
+          Suspend SystemError SystemFull TooManyConn UserUnknown SyntaxError
+        ]
+      end
+
+      # @abstract Returns Sisimai::Reason::* module path table
+      # @return   [Hash] Module path table
+      # @since    v4.25.6
+      def path
+        index = Sisimai::Reason.index
+        table = {}
+        index.each { |e| table['Sisimai::Reason::' << e] = 'sisimai/reason/' << e.downcase }
+        return table
+      end
+
       # Reason list better to retry detecting an error reason
       # @return   [Array] Reason list
       def retry
-        return { 
+        return {
           'undefined' => 1, 'onhold' => 1, 'systemerror' => 1, 'securityerror' => 1,
           'networkerror' => 1, 'hostunknown' => 1, 'userunknown' => 1
         }.freeze
       end
+      ModulePath = Sisimai::Reason.path
       GetRetried = Sisimai::Reason.retry
       ClassOrder = [
         %w[
@@ -32,17 +54,6 @@ module Sisimai
         ]
       ]
 
-      # All the error reason list Sisimai support
-      # @return   [Array] Reason list
-      def index
-        return %w[
-          Blocked ContentError ExceedLimit Expired Filtered HasMoved HostUnknown
-          MailboxFull MailerError MesgTooBig NetworkError NotAccept OnHold
-          Rejected NoRelaying SpamDetected VirusDetected PolicyViolation SecurityError
-          Suspend SystemError SystemFull TooManyConn UserUnknown SyntaxError
-        ]
-      end
-
       # Detect the bounce reason
       # @param    [Sisimai::Data] argvs   Parsed email object
       # @return   [String, Nil]           Bounce reason or Nil if the argument
@@ -52,16 +63,14 @@ module Sisimai
         return nil unless argvs
         return nil unless argvs.is_a? Sisimai::Data
 
-        unless GetRetried.key?(argvs.reason)
+        unless GetRetried[argvs.reason]
           # Return reason text already decided except reason match with the
           # regular expression of retry() method.
           return argvs.reason unless argvs.reason.empty?
         end
+        return 'delivered' if argvs.deliverystatus.start_with?('2.')
 
-        statuscode = argvs.deliverystatus || ''
         reasontext = ''
-        return 'delivered' if statuscode.start_with?('2.')
-
         if argvs.diagnostictype == 'SMTP' || argvs.diagnostictype == ''
           # Diagnostic-Code: SMTP; ... or empty value
           ClassOrder[0].each do |e|
@@ -70,7 +79,7 @@ module Sisimai
             p = 'Sisimai::Reason::' << e
             r = nil
             begin
-              require p.downcase.gsub('::', '/')
+              require ModulePath[p]
               r = Module.const_get(p)
             rescue
               warn ' ***warning: Failed to load ' << p
@@ -119,7 +128,7 @@ module Sisimai
           while true
             diagnostic   = argvs.diagnosticcode.downcase || ''
             trytomatch   = reasontext.empty? ? true : false
-            trytomatch ||= true if GetRetried.key?(reasontext)
+            trytomatch ||= true if GetRetried[reasontext]
             trytomatch ||= true if argvs.diagnostictype != 'SMTP'
             throw :TRY_TO_MATCH unless trytomatch
 
@@ -129,7 +138,7 @@ module Sisimai
               p = 'Sisimai::Reason::' << e
               r = nil
               begin
-                require p.downcase.gsub('::', '/')
+                require ModulePath[p]
                 r = Module.const_get(p)
               rescue
                 warn ' ***warning: Failed to load ' << p
@@ -193,7 +202,7 @@ module Sisimai
           p = 'Sisimai::Reason::' << e
           r = nil
           begin
-            require p.downcase.gsub('::', '/')
+            require ModulePath[p]
             r = Module.const_get(p)
           rescue
             warn ' ***warning: Failed to load ' << p