sisimai 4.25.5-java → 4.25.11-java

data/lib/sisimai/mime.rb

@@ -24,12 +24,6 @@ module Sisimai
         )
       }x.freeze
 
-      # Make MIME-Encoding and Content-Type related headers regurlar expression
-      # @return   [Array] Regular expressions related to MIME encoding
-      def patterns
-        return ReE
-      end
-
       # Check that the argument is MIME-Encoded string or not
       # @param    [String] argvs  String to be checked
       # @return   [True,False]    false: Not MIME encoded string
@@ -106,7 +100,7 @@ module Sisimai
           end
         end
 
-        return decodedtext1.force_encoding('UTF-8')
+        return decodedtext1.force_encoding('UTF-8').scrub('?')
       end
 
       # Decode MIME Quoted-Printable Encoded string
@@ -115,8 +109,8 @@ module Sisimai
       # @return [String]         MIME Decoded text
       def qprintd(argv1 = nil, heads = {})
         return nil unless argv1
-        return argv1.unpack('M').first unless heads['content-type']
-        return argv1.unpack('M').first if heads['content-type'].empty?
+        return argv1.unpack('M').first.scrub('?') unless heads['content-type']
+        return argv1.unpack('M').first.scrub('?') if heads['content-type'].empty?
 
         # Quoted-printable encoded part is the part of the text
         boundary00 = Sisimai::MIME.boundary(heads['content-type'], 0)
@@ -124,8 +118,8 @@ module Sisimai
         # Decoded using unpack('M') entire body string when the boundary string
         # or "Content-Transfer-Encoding: quoted-printable" are not included in
         # the message body.
-        return argv1.unpack('M').first if boundary00.empty?
-        return argv1.unpack('M').first unless argv1.downcase =~ ReE[:'quoted-print']
+        return argv1.unpack('M').first.scrub('?') if boundary00.empty?
+        return argv1.unpack('M').first.scrub('?') unless argv1.downcase =~ ReE[:'quoted-print']
 
         boundary01 = Sisimai::MIME.boundary(heads['content-type'], 1)
         bodystring = ''
@@ -167,7 +161,7 @@ module Sisimai
                 #
                 # Content-Transfer-Encoding: quoted-printable
                 # X-Microsoft-Exchange-Diagnostics:
-                #     1;SLXP216MB0381;27:IdH7U/WHGgJu6J8lFrE7KvVxhnAwyKrNbSXMFYs3/Gzz6ZdXYYjzHj55K2O+cndpeVwkvBJqmo6y0IF4AhLfHtFzznw/BzhERU6wi/TCWRpyjYuW8v0/aTcflH3oAdgZ4Pwrp7PxLiiA8rYgU/E7SQ==
+                #     1;SLXP216MB0381;27:IdH7U/WHGgJu6J...LiiA8rYgU/E7SQ==
                 # ...
                 mustencode = true
                 while true do
@@ -177,10 +171,8 @@ module Sisimai
                     # Padding character of Base64 or not
                     break if e =~ /[\+\/0-9A-Za-z]{32,}[=]+\z/
                   else
-                    if e.include?('=') && ! e.upcase.include?('=3D')
-                      # Including "=" not as "=3D"
-                      break
-                    end
+                    # Including "=" not as "=3D"
+                    break if e.include?('=') && ! e.upcase.include?('=3D')
                   end
                   mustencode = false
                   break
@@ -222,7 +214,7 @@ module Sisimai
         end
 
         bodystring << notdecoded unless notdecoded.empty?
-        return bodystring
+        return bodystring.scrub('?')
       end
 
       # Decode MIME BASE64 Encoded string
@@ -251,6 +243,7 @@ module Sisimai
           # Content-Type: multipart/report; report-type=delivery-status;
           #    boundary="n6H9lKZh014511.1247824040/mx.example.jp"
           value = cv[1]
+          value = value.split(/\n/, 2).shift if value =~ /\n/
           value.delete!(%q|'";\\|)
           value = '--' + value if start > -1
           value = value + '--' if start >  0
@@ -278,8 +271,8 @@ module Sisimai
         return '' unless mimeformat =~ LeavesOnly
         return '' if alternates && mimeformat == 'text/html'
 
-        (upperchunk, lowerchunk) = argv0.split(/^$/m, 2)
-        upperchunk.tr!("\n", ' ').squeeze(' ')
+        (upperchunk, lowerchunk) = argv0.split(/^$/, 2)
+        upperchunk.tr("\n", ' ').squeeze!(' ')
 
         # Content-Description: Undelivered Message
         # Content-Type: message/rfc822
@@ -290,18 +283,16 @@ module Sisimai
           # Content-Type: multipart/*
           mpboundary = Regexp.new(Regexp.escape(Sisimai::MIME.boundary(upperchunk, 0)) << "\n")
           innerparts = lowerchunk.split(mpboundary)
+          innerparts.shift if innerparts[0].empty? || innerparts[0] == "\n"
 
-          innerparts.shift if innerparts[0].empty?
           while e = innerparts.shift do
             # Find internal multipart/* blocks and decode
             if cv = e.match(ThisFormat)
               # Found "Content-Type" field at the first or second line of this
               # splitted part
               nextformat = cv[1].downcase
-
               next unless nextformat =~ LeavesOnly
               next if nextformat == 'text/html'
-
               hasflatten << Sisimai::MIME.breaksup(e, mimeformat)
             else
               # The content of this part is almost '--': a part of boundary
@@ -405,8 +396,8 @@ module Sisimai
         # Some bounce messages include lower-cased "content-type:" field such as
         #   content-type: message/delivery-status
         #   content-transfer-encoding: quoted-printable
-        argv1.gsub!(/[Cc]ontent-[Tt]ype:/m, 'Content-Type:')
-        argv1.gsub!(/[Cc]ontent-[Tt]ransfer-[Ee]ncodeing:/m, 'Content-Transfer-Encoding:')
+        argv1.gsub!(/[Cc]ontent-[Tt]ype:/, 'Content-Type:')
+        argv1.gsub!(/[Cc]ontent-[Tt]ransfer-[Ee]ncodeing:/, 'Content-Transfer-Encoding:')
 
         # 1. Some bounce messages include upper-cased "Content-Transfer-Encoding",
         #    and "Content-Type" value such as
@@ -414,11 +405,10 @@ module Sisimai
         #      - Content-Transfer-Encoding: 7BIT
         # 2. Unused fields inside of mutipart/* block should be removed
         argv1.gsub!(/(Content-[A-Za-z-]+?):[ ]*([^\s]+)/) do "#{$1}: #{$2.downcase}" end
-        argv1.gsub!(/^Content-(?:Description|Disposition):.+?$/, '')
+        argv1.gsub!(/^Content-(?:Description|Disposition):.+?\n$/, '')
 
-        multiparts = argv1.split(Regexp.new(Regexp.escape(ehboundary) << "\n"))
+        multiparts = argv1.split(Regexp.new(Regexp.escape(ehboundary) << "\n?"))
         multiparts.shift if multiparts[0].empty?
-
         while e = multiparts.shift do
           # Find internal multipart blocks and decode
           catch :XCCT do
@@ -430,11 +420,16 @@ module Sisimai
               #   Message-ID: ...
               #   Content-Transfer-Encoding: quoted-printable
               #   Content-Type: text/plain; charset=us-ascii
-              throw :XCCT if e =~ /\AContent-T[ry]/
-              if cv = e.match(/\A(.+?)Content-Type:/)
-                throw :XCCT if cv[1] =~ /\n\n/
+              #
+              # Fields before "Content-Type:" in each part should have been removed
+              # and "Content-Type:" should be exist at the first line of each part.
+              # The field works as a delimiter to decode contents of each part.
+              #
+              throw :XCCT if e =~ /\AContent-T[ry]/       # The first field is "Content-Type:"
+              if cv = e.match(/\A(.+?)Content-Type:/m)    # Capture lines before "Content-Type:"
+                throw :XCCT if cv[1] =~ /\n\n/            # There is no field before "Content-Type:"
+                e.sub!(/\A.+?(Content-T[ry].+)\z/m, '\1') # Remove fields before "Content-Type:"
               end
-              e.sub!(/\A.+?(Content-T[ry].+)\z/, '\1')
               throw :XCCT
             end
           end

data/lib/sisimai/order.rb

@@ -6,218 +6,184 @@ module Sisimai
     class << self
       require 'sisimai/lhost'
 
-      OrderE1 = [
-        # These modules have many subject patterns or have MIME encoded subjects
-        # which is hard to code as regular expression
+      # There are another patterns in the value of "Subject:" header of a bounce
+      # mail generated by the following MTA/ESP modules
+      OrderE0 = [
+        'Sisimai::Lhost::MailRu',
+        'Sisimai::Lhost::Yandex',
         'Sisimai::Lhost::Exim',
-        'Sisimai::Lhost::Exchange2003',
-      ].freeze
-      OrderE2 = [
-        # These modules have no MTA specific header and did not listed in the
-        # following subject header based regular expressions.
+        'Sisimai::Lhost::Sendmail',
+        'Sisimai::Lhost::Aol',
+        'Sisimai::Lhost::Office365',
         'Sisimai::Lhost::Exchange2007',
-        'Sisimai::Lhost::Facebook',
+        'Sisimai::Lhost::Exchange2003',
+        'Sisimai::Lhost::AmazonWorkMail',
+        'Sisimai::Lhost::AmazonSES',
+        'Sisimai::Lhost::Barracuda',
+        'Sisimai::Lhost::InterScanMSS',
         'Sisimai::Lhost::KDDI',
+        'Sisimai::Lhost::SurfControl',
+        'Sisimai::Lhost::Verizon',
+        'Sisimai::Lhost::ApacheJames',
+        'Sisimai::Lhost::X2',
+        'Sisimai::Lhost::X5',
+        'Sisimai::Lhost::FML',
       ].freeze
-      OrderE3 = [
-        # These modules have no MTA specific header but listed in the following
-        # subject header based regular expressions.
-        'Sisimai::Lhost::Sendmail',
+
+      # Fallback list: The following MTA/ESP modules is not listed OrderE0
+      OrderE1 = [
         'Sisimai::Lhost::Postfix',
+        'Sisimai::Lhost::GSuite',
+        'Sisimai::Lhost::Yahoo',
+        'Sisimai::Lhost::Outlook',
+        'Sisimai::Lhost::GMX',
+        'Sisimai::Lhost::MessagingServer',
+        'Sisimai::Lhost::EinsUndEins',
+        'Sisimai::Lhost::Domino',
+        'Sisimai::Lhost::Notes',
         'Sisimai::Lhost::Qmail',
-        'Sisimai::Lhost::SendGrid',
         'Sisimai::Lhost::Courier',
         'Sisimai::Lhost::OpenSMTPD',
-        'Sisimai::Lhost::Notes',
-        'Sisimai::Lhost::MessagingServer',
-        'Sisimai::Lhost::Domino',
-        'Sisimai::Lhost::Bigfoot',
-        'Sisimai::Lhost::EinsUndEins',
+        'Sisimai::Lhost::Zoho',
+        'Sisimai::Lhost::MessageLabs',
         'Sisimai::Lhost::MXLogic',
-        'Sisimai::Lhost::Amavis',
-        'Sisimai::Lhost::IMailServer',
-        'Sisimai::Lhost::X4',
-      ].freeze
-      OrderE4 = [
-        # These modules have no MTA specific headers and there are few samples or
-        # too old MTA
-        'Sisimai::Lhost::Verizon',
-        'Sisimai::Lhost::InterScanMSS',
         'Sisimai::Lhost::MailFoundry',
-        'Sisimai::Lhost::ApacheJames',
-        'Sisimai::Lhost::Biglobe',
-        'Sisimai::Lhost::EZweb',
-        'Sisimai::Lhost::X5',
-        'Sisimai::Lhost::X3',
-        'Sisimai::Lhost::X2',
-        'Sisimai::Lhost::X1',
+        'Sisimai::Lhost::McAfee',
         'Sisimai::Lhost::V5sendmail',
-      ].freeze
-      OrderE5 = [
-        # These modules have one or more MTA specific headers but other headers
-        # also required for detecting MTA name
-        'Sisimai::Lhost::Outlook',
-        'Sisimai::Lhost::MailRu',
-        'Sisimai::Lhost::MessageLabs',
-        'Sisimai::Lhost::MailMarshalSMTP',
         'Sisimai::Lhost::MFILTER',
-        'Sisimai::Lhost::Google',
-      ].freeze
-      OrderE9 = [
-        # These modules have one or more MTA specific headers
-        'Sisimai::Lhost::Aol',
-        'Sisimai::Lhost::Yahoo',
-        'Sisimai::Lhost::AmazonSES',
-        'Sisimai::Lhost::GMX',
-        'Sisimai::Lhost::Yandex',
-        'Sisimai::Lhost::Office365',
+        'Sisimai::Lhost::SendGrid',
         'Sisimai::Lhost::ReceivingSES',
-        'Sisimai::Lhost::AmazonWorkMail',
-        'Sisimai::Lhost::Zoho',
-        'Sisimai::Lhost::McAfee',
+        'Sisimai::Lhost::Amavis',
+        'Sisimai::Lhost::PowerMTA',
+        'Sisimai::Lhost::GoogleGroups',
+        'Sisimai::Lhost::Gmail',
+        'Sisimai::Lhost::EZweb',
+        'Sisimai::Lhost::IMailServer',
+        'Sisimai::Lhost::MailMarshalSMTP',
         'Sisimai::Lhost::Activehunter',
-        'Sisimai::Lhost::SurfControl',
+        'Sisimai::Lhost::Bigfoot',
+        'Sisimai::Lhost::Biglobe',
+        'Sisimai::Lhost::Facebook',
+        'Sisimai::Lhost::X4',
+        'Sisimai::Lhost::X1',
+        'Sisimai::Lhost::X3',
+        'Sisimai::Lhost::X6',
       ].freeze
 
-      # This variable don't hold MTA name which have one or more MTA specific
-      # header such as X-AWS-Outgoing, X-Yandex-Uniq.
-      Pattern = {
-        'subject' => {
-          'delivery' => [
-            'Sisimai::Lhost::Exim',
-            'Sisimai::Lhost::Outlook',
-            'Sisimai::Lhost::Courier',
-            'Sisimai::Lhost::Domino',
-            'Sisimai::Lhost::OpenSMTPD',
-            'Sisimai::Lhost::EinsUndEins',
-            'Sisimai::Lhost::InterScanMSS',
-            'Sisimai::Lhost::MailFoundry',
-            'Sisimai::Lhost::X4',
-            'Sisimai::Lhost::X3',
-            'Sisimai::Lhost::X2',
-            'Sisimai::Lhost::Google',
-          ],
-          'noti' => [
-            'Sisimai::Lhost::Sendmail',
-            'Sisimai::Lhost::Qmail',
-            'Sisimai::Lhost::Outlook',
-            'Sisimai::Lhost::Courier',
-            'Sisimai::Lhost::MessagingServer',
-            'Sisimai::Lhost::OpenSMTPD',
-            'Sisimai::Lhost::X4',
-            'Sisimai::Lhost::X3',
-            'Sisimai::Lhost::MFILTER',
-            'Sisimai::Lhost::Google',
-          ],
-          'return' => [
-            'Sisimai::Lhost::Postfix',
-            'Sisimai::Lhost::Sendmail',
-            'Sisimai::Lhost::SendGrid',
-            'Sisimai::Lhost::Bigfoot',
-            'Sisimai::Lhost::EinsUndEins',
-            'Sisimai::Lhost::X1',
-            'Sisimai::Lhost::Biglobe',
-            'Sisimai::Lhost::V5sendmail',
-          ],
-          'undeliver' => [
-            'Sisimai::Lhost::Postfix',
-            'Sisimai::Lhost::Office365',
-            'Sisimai::Lhost::Exchange2007',
-            'Sisimai::Lhost::Exchange2003',
-            'Sisimai::Lhost::SendGrid',
-            'Sisimai::Lhost::Notes',
-            'Sisimai::Lhost::Verizon',
-            'Sisimai::Lhost::Amavis',
-            'Sisimai::Lhost::IMailServer',
-            'Sisimai::Lhost::MailMarshalSMTP',
-          ],
-          'failure' => [
-            'Sisimai::Lhost::Qmail',
-            'Sisimai::Lhost::Outlook',
-            'Sisimai::Lhost::MailRu',
-            'Sisimai::Lhost::Domino',
-            'Sisimai::Lhost::X4',
-            'Sisimai::Lhost::X2',
-            'Sisimai::Lhost::MFILTER',
-            'Sisimai::Lhost::Google',
-          ],
-          'warning' => [
-            'Sisimai::Lhost::Postfix',
-            'Sisimai::Lhost::Sendmail',
-            'Sisimai::Lhost::Exim',
-          ],
-        },
+      # The following order is decided by the first 2 words of Subject: header
+      Subject = {
+        'abuse-report'     => ['Sisimai::ARF'],
+        'auto'             => ['Sisimai::RFC3834'],
+        'auto-reply'       => ['Sisimai::RFC3834'],
+        'automatic-reply'  => ['Sisimai::RFC3834'],
+        'aws-notification' => ['Sisimai::Lhost::AmazonSES'],
+        'complaint-about'  => ['Sisimai::ARF'],
+        'delivery-failure' => ['Sisimai::Lhost::Domino', 'Sisimai::Lhost::X2'],
+        'delivery-notification' => ['Sisimai::Lhost::MessagingServer'],
+        'delivery-report'  => ['Sisimai::Lhost::PowerMTA'],
+        'delivery-status'  => [
+          'Sisimai::Lhost::GSuite',
+          'Sisimai::Lhost::GoogleGroups',
+          'Sisimai::Lhost::Outlook',
+          'Sisimai::Lhost::McAfee',
+          'Sisimai::Lhost::OpenSMTPD',
+          'Sisimai::Lhost::AmazonSES',
+          'Sisimai::Lhost::AmazonWorkMail',
+          'Sisimai::Lhost::ReceivingSES',
+          'Sisimai::Lhost::Gmail',
+          'Sisimai::Lhost::X3',
+        ],
+        'dmarc-ietf-dmarc' => ['Sisimai::ARF'],
+        'email-feedback'   => ['Sisimai::ARF'],
+        'failed-delivery'  => ['Sisimai::Lhost::X2'],
+        'failure-delivery' => ['Sisimai::Lhost::X2'],
+        'failure-notice'   => [
+          'Sisimai::Lhost::Yahoo',
+          'Sisimai::Lhost::Qmail',
+          'Sisimai::Lhost::MFILTER',
+          'Sisimai::Lhost::Activehunter',
+          'Sisimai::Lhost::X4',
+        ],
+        'loop-alert'    => ['Sisimai::Lhost::FML'],
+        'mail-delivery' => [
+          'Sisimai::Lhost::Exim',
+          'Sisimai::Lhost::MailRu',
+          'Sisimai::Lhost::GMX',
+          'Sisimai::Lhost::EinsUndEins',
+          'Sisimai::Lhost::Zoho',
+          'Sisimai::Lhost::MessageLabs',
+          'Sisimai::Lhost::MXLogic',
+        ],
+        'mail-could'   => ['Sisimai::Lhost::InterScanMSS'],
+        'mail-failure' => ['Sisimai::Lhost::Exim'],
+        'mail-not'     => ['Sisimai::Lhost::X4'],
+        'mail-system'  => ['Sisimai::Lhost::EZweb'],
+        'message-delivery' => ['Sisimai::Lhost::MailFoundry'],
+        'message-frozen'   => ['Sisimai::Lhost::Exim'],
+        'message-you'      => ['Sisimai::Lhost::Barracuda'],
+        'non-recapitabile' => ['Sisimai::Lhost::Exchange2007'],
+        'non-remis' => ['Sisimai::Lhost::Exchange2007'],
+        'notice'    => ['Sisimai::Lhost::Courier'],
+        'permanent-delivery' => ['Sisimai::Lhost::X4'],
+        'postmaster-notify'  => ['Sisimai::Lhost::Sendmail'],
+        'returned-mail' => [
+          'Sisimai::Lhost::Sendmail',
+          'Sisimai::Lhost::Aol',
+          'Sisimai::Lhost::V5sendmail',
+          'Sisimai::Lhost::Bigfoot',
+          'Sisimai::Lhost::Biglobe',
+          'Sisimai::Lhost::X1',
+        ],
+        'sorry-your' => ['Sisimai::Lhost::Facebook'],
+        'there-was'  => ['Sisimai::Lhost::X6'],
+        'undeliverable' => [
+          'Sisimai::Lhost::Office365',
+          'Sisimai::Lhost::Exchange2007',
+          'Sisimai::Lhost::Aol',
+          'Sisimai::Lhost::Exchange2003',
+        ],
+        'undeliverable-mail' => [
+          'Sisimai::Lhost::Amavis',
+          'Sisimai::Lhost::MailMarshalSMTP',
+          'Sisimai::Lhost::IMailServer',
+        ],
+        'undeliverable-message' => ['Sisimai::Lhost::Notes', 'Sisimai::Lhost::Verizon'],
+        'undelivered-mail' => [
+          'Sisimai::Lhost::Postfix',
+          'Sisimai::Lhost::Aol',
+          'Sisimai::Lhost::SendGrid',
+          'Sisimai::Lhost::Zoho',
+        ],
+        'warning' => ['Sisimai::Lhost::Sendmail', 'Sisimai::Lhost::Exim'],
       }.freeze
-      Skips   = { 'return-path' => 1, 'x-mailer' => 1 }.freeze
 
-      # @abstract Check headers for detecting MTA module and returns the order
-      #           of modules
-      # @param    [Hash] heads   Email header data
+      # @abstract Returns an MTA Order decided by the first word of the "Subject": header
+      # @param    [String] argv0 Subject header string
       # @return   [Array]        Order of MTA modules
       # @since    v4.25.4
-      def make(heads = {})
-        return [] unless heads['subject']
+      def make(argv0 = '')
+        return [] if argv0.empty?
+        argv0 = argv0.downcase.tr('_[] ', ' ').squeeze(' ').sub(/\A[ ]+/, '')
+        words = argv0.split(/[ ]/, 3)
 
-        # Try to match the value of "Subject" with patterns generated by
-        # Sisimai::Order->by('subject') method
-        order = []
-        title = heads['subject'].downcase
-        Subject.each_key do |e|
-          # Get MTA list from the subject header
-          next unless title.include?(e)
-          order = Subject[e]
-          break
+        if words[0].include?(':')
+          # Undeliverable: ..., notify: ...
+          first = argv0.split(':').shift
+        else
+          # Postmaster notify, returned mail, ...
+          first = words.slice(0, 2).join('-')
         end
-
-        return order
+        first.delete!(':",*')
+        return Subject[first] || []
       end
 
-      # @abstract Get regular expression patterns for specified field
-      # @param    [String] group  Group name for "ORDER BY"
-      # @return   [Hash]          Pattern table for the group
-      def by(group = '')
-        return {} if group.empty?
-        return Pattern[group] if Pattern[group]
-        return {}
-      end
-      Subject = Sisimai::Order.by('subject').freeze
-
       # @abstract Make default order of MTA modules to be loaded
       # @return   [Array] Default order list of MTA modules
-      def default
-        return Sisimai::Lhost.index.map { |e| 'Sisimai::Lhost::' << e }
-      end
+      def default; return Sisimai::Lhost.index.map { |e| 'Sisimai::Lhost::' << e }; end
 
       # @abstract Make MTA module list as a spare
       # @return   [Array] Ordered module list
-      def another
-        rv = []
-        [OrderE1, OrderE2, OrderE3, OrderE4, OrderE5, OrderE9].each do |e|
-          rv += e
-        end
-        return rv
-      end
-
-      # @abstract Make email header list in each MTA module
-      # @return   [Hash] Header list to be parsed
-      def headers
-        order = Sisimai::Lhost.heads.map { |e| 'Sisimai::Lhost::' << e }
-        table = {}
-
-        while e = order.shift do
-          # Load email headers from each MTA module
-          require e.gsub('::', '/').downcase
-
-          Module.const_get(e).headerlist.each do |v|
-            # Get header name which required each MTA module
-            next if Skips[v]
-            table[v] ||= []
-            table[v] << e
-          end
-        end
-        return table
-      end
-
+      def another; return [OrderE0, OrderE1].flatten; end
     end
   end
 end