sisimai 4.24.1-java → 4.25.0-java

data/lib/sisimai/message/email.rb

@@ -17,15 +17,10 @@ module Sisimai
       BorderLine = '__MIME_ENCODED_BOUNDARY__'
       EndOfEmail = Sisimai::String.EOM
       RFC822Head = Sisimai::RFC5322.HEADERFIELDS
-      RFC3834Set = Sisimai::RFC3834.headerlist.map(&:downcase)
+      RFC3834Set = Sisimai::RFC3834.headerlist
       HeaderList = %w[from to date subject content-type reply-to message-id
-                      received content-transfer-encoding return-path x-mailer].freeze
-      MultiHeads = { 'received' => true }.freeze
-      IgnoreList = { 'dkim-signature' => true }.freeze
-      Indicators = {
-        :begin => (1 << 1),
-        :endof => (1 << 2),
-      }.freeze
+                      received content-transfer-encoding return-path x-mailer]
+      IsMultiple = { 'received' => true }
       DefaultSet = Sisimai::Order::Email.another
       SubjectTab = Sisimai::Order::Email.by('subject')
       ExtHeaders = Sisimai::Order::Email.headers
@@ -56,8 +51,7 @@ module Sisimai
         tobeloaded = Sisimai::Message::Email.load(methodargv)
 
         # 1. Split email data to headers and a body part.
-        aftersplit = Sisimai::Message::Email.divideup(email)
-        return nil if aftersplit.empty?
+        return nil unless aftersplit = Sisimai::Message::Email.divideup(email)
 
         # 2. Convert email headers from text to hash reference
         headerargv = {
@@ -81,9 +75,7 @@ module Sisimai
           'tryonfirst' => headerargv['tryonfirst'],
           'tobeloaded' => tobeloaded,
         }
-        bouncedata = Sisimai::Message::Email.parse(methodargv)
-
-        return nil unless bouncedata
+        return nil unless bouncedata = Sisimai::Message::Email.parse(methodargv)
         return nil if bouncedata.empty?
         processing['ds']    = bouncedata['ds']
         processing['catch'] = bouncedata['catch']
@@ -153,48 +145,27 @@ module Sisimai
       # @param         [String] email  Email data
       # @return        [Hash]          Email data after split
       def self.divideup(email)
-        return {} if email.empty?
+        return nil if email.empty?
 
+        block = { 'from' => '', 'header' => '', 'body' => '' }
         email.scrub!('?')
         email.gsub!(/\r\n/, "\n")  if email.include?("\r\n")
         email.gsub!(/[ \t]+$/, '') if email =~ /[ \t]+$/
 
-        hasdivided = email.split("\n")
-        return {} if hasdivided.empty?
-
-        readcursor = 0
-        aftersplit = { 'from' => '', 'header' => '', 'body' => '' }
+        (block['header'], block['body']) = email.split(/\n\n/, 2)
+        return nil unless block['header']
+        return nil unless block['body']
 
-        if hasdivided[0][0, 5] == 'From '
+        if block['header'][0, 5] == 'From '
           # From MAILER-DAEMON Tue Feb 11 00:00:00 2014
-          aftersplit['from'] = hasdivided.shift
-          aftersplit['from'] = aftersplit['from'].delete("\n").delete("\r")
-        end
-
-        # Split email data to headers and a body part.
-        while e = hasdivided.shift do
-          # Split email data to headers and a body part.
-          if readcursor & Indicators[:endof] > 0
-            # The body part of the email
-            aftersplit['body'] << e + "\n"
-          else
-            # The boundary for splitting headers and a body part does not
-            # appeare yet.
-            if e.empty?
-              # Blank line, it is a boundary of headers and a body part
-              readcursor |= Indicators[:endof] if readcursor & Indicators[:begin] > 0
-            else
-              # The header part of the email
-              aftersplit['header'] << e + "\n"
-              readcursor |= Indicators[:begin]
-            end
-          end
+          block['from'] = block['header'].split(/\n/, 2)[0].delete("\r")
+        else
+          # Set pseudo UNIX From line
+          block['from'] = 'MAILER-DAEMON Tue Feb 11 00:00:00 2014'
         end
-        return {} if aftersplit['header'].empty?
-        return {} if aftersplit['body'].empty?
 
-        aftersplit['from'] = 'MAILER-DAEMON Tue Feb 11 00:00:00 2014' if aftersplit['from'].empty?
-        return aftersplit
+        block['body'] << "\n"
+        return block
       end
 
       # Convert email headers from text to hash reference
@@ -205,6 +176,7 @@ module Sisimai
       def self.headers(heads, argvs = {})
         return nil unless heads
 
+        currheader = ''
         allheaders = {}
         structured = {}
         extheaders = argvs['extheaders'] || []
@@ -214,50 +186,49 @@ module Sisimai
         HeaderList.each { |e| structured[e] = nil  }
         HeaderList.each { |e| allheaders[e] = true }
         RFC3834Set.each { |e| allheaders[e] = true }
-        MultiHeads.each_key { |e| structured[e.downcase] = [] }
+        IsMultiple.each_key { |e| structured[e] = [] }
         extheaders.each_key { |e| allheaders[e] = true }
         unless extrafield.empty?
-          extrafield.each { |e| allheaders[e.downcase] = true }
+          extrafield.each { |e| allheaders[e] = true }
         end
 
         while e = hasdivided.shift do
           # Convert email headers to hash
-          if cv = e.match(/\A([^ ]+?)[:][ ]*(.*?)\z/)
-            # split the line into a header name and a header content
-            lhs = cv[1]
-            rhs = cv[2]
+          if cv = e.match(/\A[ \t]+(.+)\z/)
+            # Continued (foled) header value from the previous line
+            next unless allheaders.key?(currheader)
 
-            currheader = lhs.downcase
+            # Header line continued from the previous line
+            if structured[currheader].is_a? Array
+              # Concatenate a header which have multi-lines such as 'Received'
+              structured[currheader][-1] << ' ' << cv[1]
+            else
+              structured[currheader] ||= ''
+              structured[currheader] << ' ' << cv[1]
+            end
+          else
+            # split the line into a header name and a header content
+            (lhs, rhs) = e.split(/:[ ]*/, 2)
+            currheader = lhs ? lhs.downcase : ''
             next unless allheaders.key?(currheader)
 
-            if MultiHeads.key?(currheader)
+            if IsMultiple.key?(currheader)
               # Such as 'Received' header, there are multiple headers in a single
               # email message.
-              rhs = rhs.tr("\t", ' ').squeeze(' ')
+              #rhs = rhs.tr("\t", ' ').squeeze(' ')
+              rhs = rhs.tr("\t", ' ')
               structured[currheader] << rhs
             else
               # Other headers except "Received" and so on
               if extheaders[currheader]
                 # MTA specific header
-                extheaders[currheader].keys.each do |r|
+                extheaders[currheader].each do |r|
                   next if argvs['tryonfirst'].index(r)
                   argvs['tryonfirst'] << r
                 end
               end
               structured[currheader] = rhs
             end
-          elsif cv = e.match(/\A[ \t]+(.+?)\z/)
-            # Ignore header?
-            next if IgnoreList[currheader]
-
-            # Header line continued from the previous line
-            if structured[currheader].is_a? Array
-              # Concatenate a header which have multi-lines such as 'Received'
-              structured[currheader][-1] << ' ' << cv[1]
-            else
-              structured[currheader] ||= ''
-              structured[currheader] << ' ' << cv[1]
-            end
           end
         end
         return structured
@@ -294,72 +265,68 @@ module Sisimai
         # 2. Convert from string to hash reference
         heads = heads.scrub('?').gsub(/^[>]+[ ]/m, '')
 
-        takenapart = {}
-        hasdivided = heads.split("\n")
         previousfn = '' # Previous field name
-        mimeborder = {}
+        asciiarmor = {} # Header names which has MIME encoded value
+        headerpart = {} # Required headers in the original message part
+        hasdivided = heads.split("\n")
 
         while e = hasdivided.shift do
           # Header name as a key, The value of header as a value
-          if cv = e.match(/\A([-0-9A-Za-z]+?)[:][ ]*(.*)\z/)
-            # Header
-            lhs = cv[1].downcase
-            rhs = cv[2]
-            previousfn = ''
-
-            next unless RFC822Head.key?(lhs)
-            previousfn = lhs
-            takenapart[previousfn] = rhs unless takenapart[previousfn]
-          else
-            # Continued line from the previous line
-            next unless e.start_with?(' ', "\t")
+          if e.start_with?(' ', "\t")
+            # Continued (foled) header value from the previous line
             next if previousfn.empty?
 
             # Concatenate the line if it is the value of required header
             if Sisimai::MIME.is_mimeencoded(e)
               # The line is MIME-Encoded test
-              takenapart[previousfn] << if previousfn == 'subject'
+              headerpart[previousfn] << if previousfn == 'subject'
                                           # Subject: header
                                           BorderLine + e
                                         else
                                           # Is not Subject header
                                           e
                                         end
-              mimeborder[previousfn] = true
+              asciiarmor[previousfn] = true
             else
               # ASCII Characters only: Not MIME-Encoded
-              takenapart[previousfn] << e.lstrip
-              mimeborder[previousfn] ||= false
+              headerpart[previousfn] << e.lstrip
+              asciiarmor[previousfn] ||= false
             end
+          else
+            # Header name as a key, The value of header as a value
+            (lhs, rhs) = e.split(/:[ ]*/, 2)
+            next unless lhs
+            lhs.downcase!
+            previousfn = ''
+
+            next unless RFC822Head.key?(lhs)
+            previousfn = lhs
+            headerpart[previousfn] = rhs unless headerpart[previousfn]
           end
         end
+        return headerpart unless headerpart['subject']
 
-        if takenapart['subject']
-          # Convert MIME-Encoded subject
-          v = takenapart['subject']
-
-          if Sisimai::String.is_8bit(v)
-            # The value of ``Subject'' header is including multibyte character,
-            # is not MIME-Encoded text.
-            v = 'MULTIBYTE CHARACTERS HAVE BEEN REMOVED'
-          else
-            # MIME-Encoded subject field or ASCII characters only
-            r = []
-            if mimeborder['subject']
-              # split the value of Subject by borderline
-              v.split(BorderLine).each do |m|
-                # Insert value to the array if the string is MIME encoded text
-                r << m if Sisimai::MIME.is_mimeencoded(m)
-              end
-            else
-              # Subject line is not MIME encoded
-              r << v
+        # Convert MIME-Encoded subject
+        if Sisimai::String.is_8bit(headerpart['subject'])
+          # The value of ``Subject'' header is including multibyte character,
+          # is not MIME-Encoded text.
+          headerpart['subject'] = 'MULTIBYTE CHARACTERS HAVE BEEN REMOVED'
+        else
+          # MIME-Encoded subject field or ASCII characters only
+          r = []
+          if asciiarmor['subject']
+            # split the value of Subject by borderline
+            headerpart['subject'].split(BorderLine).each do |v|
+              # Insert value to the array if the string is MIME encoded text
+              r << v if Sisimai::MIME.is_mimeencoded(v)
             end
-            v = Sisimai::MIME.mimedecode(r)
+          else
+            # Subject line is not MIME encoded
+            r << headerpart['subject']
           end
-          takenapart['subject'] = v
+          headerpart['subject'] = Sisimai::MIME.mimedecode(r)
         end
-        return takenapart
+        return headerpart
       end
 
       # @abstract Parse bounce mail with each MTA module
@@ -441,10 +408,13 @@ module Sisimai
         if mailheader['subject'].downcase =~ /\A[ \t]*fwd?:/
           # Delete quoted strings, quote symbols(>)
           bodystring = bodystring.gsub(/^[>]+[ ]/m, '').gsub(/^[>]$/m, '')
+        elsif Sisimai::MIME.is_mimeencoded(mailheader['subject'])
+          # Decode MIME-Encoded "Subject:" header
+          mailheader['subject'] = Sisimai::MIME.mimedecode(mailheader['subject'].split(/[ ]/))
+          mailheader['subject'].scrub!('?')
         end
         bodystring << EndOfEmail
         haveloaded = {}
-        defaultset = DefaultSet.dup
         scannedset = nil
 
         catch :SCANNER do
@@ -469,19 +439,9 @@ module Sisimai
               throw :SCANNER if scannedset
             end
 
+            tryonfirst.concat(DefaultSet)
             while r = tryonfirst.shift do
-              # Try MTA module candidates which are detected from MTA specific
-              # mail headers on first
-              next if haveloaded.key?(r)
-              require r.gsub('::', '/').downcase
-              scannedset = Module.const_get(r).scan(mailheader, bodystring)
-              haveloaded[r] = true
-              throw :SCANNER if scannedset
-            end
-
-            while r = defaultset.shift do
-              # MTA modules which does not have MTA specific header and did
-              # not match with any regular expressions of Subject header.
+              # Try MTA module candidates
               next if haveloaded.key?(r)
               require r.gsub('::', '/').downcase
               scannedset = Module.const_get(r).scan(mailheader, bodystring)

data/lib/sisimai/message/json.rb

@@ -37,14 +37,12 @@ module Sisimai
 
         # Rewrite message body for detecting the bounce reason
         methodargv = { 'hook' => hookmethod, 'json' => argvs['data'] }
-        bouncedata = Sisimai::Message::JSON.parse(methodargv)
-
-        return nil unless bouncedata
+        return nil unless bouncedata = Sisimai::Message::JSON.parse(methodargv)
         return nil if bouncedata.empty?
+
         processing['ds']     = bouncedata['ds']
         processing['catch']  = bouncedata['catch']
         processing['rfc822'] = bouncedata['rfc822']
-
         return processing
       end
 

data/lib/sisimai/mime.rb

@@ -15,6 +15,14 @@ module Sisimai
         :'only-charset' => %r/^[\s\t]+charset=['"]?(.+?)['"]?\b/,
         :'html-message' => %r|^content-type:[ ]*text/html;|m,
       }.freeze
+      AlsoAppend = %r{\A(?:text/rfc822-headers|message/)}.freeze
+      ThisFormat = %r/\A(?:Content-Transfer-Encoding:\s*.+\n)?Content-Type:\s*([^ ;]+)/.freeze
+      LeavesOnly = %r{\A(?>
+         text/(?:plain|html|rfc822-headers)
+        |message/(?:x?delivery-status|rfc822|partial|feedback-report)
+        |multipart/(?:report|alternative|mixed|related|partial)
+        )
+      }x.freeze
 
       # Make MIME-Encoding and Content-Type related headers regurlar expression
       # @return   [Array] Regular expressions related to MIME encoding
@@ -27,17 +35,17 @@ module Sisimai
       # @return   [True,False]    false: Not MIME encoded string
       #                           true:  MIME encoded string
       def is_mimeencoded(argv1)
-        return false unless argv1
+        return nil unless argv1
 
-        argv1.delete!('"')
-        piece = []
+        text1 = argv1.delete('"')
         mime1 = false
+        piece = []
 
-        if argv1.include?(' ')
+        if text1.include?(' ')
           # Multiple MIME-Encoded strings in a line
-          piece = argv1.split(' ')
+          piece = text1.split(' ')
         else
-          piece << argv1
+          piece << text1
         end
 
         while e = piece.shift do
@@ -67,19 +75,17 @@ module Sisimai
             characterset ||= cv[2]
             encodingname ||= cv[3]
             mimeencoded0   = cv[4]
-            decodedtext0  << cv[1]
 
-            if encodingname == 'Q'
-              # Quoted-Printable
-              decodedtext0 << mimeencoded0.unpack('M').first
-
-            elsif encodingname == 'B'
-              # Base64
-              decodedtext0 << Base64.decode64(mimeencoded0)
-            end
+            decodedtext0 << cv[1]
+            decodedtext0 << if encodingname == 'B'
+                              Base64.decode64(mimeencoded0)
+                            else
+                              mimeencoded0.unpack('M').first
+                            end
+            decodedtext0[-1].gsub!(/\r\n/, '')
             decodedtext0 << cv[5]
           else
-            decodedtext0 << e
+            decodedtext0 << if decodedtext0.empty? then e else ' ' << e end
           end
         end
 
@@ -261,28 +267,19 @@ module Sisimai
         return nil unless argv0
 
         hasflatten = '' # Message body including only text/plain and message/*
-        alsoappend = %r{\A(?:text/rfc822-headers|message/)}
-        thisformat = %r/\A(?:Content-Transfer-Encoding:\s*.+\n)?Content-Type:\s*([^ ;]+)/
-        leavesonly = %r{\A(?>
-           text/(?:plain|html|rfc822-headers)
-          |message/(?:x?delivery-status|rfc822|partial|feedback-report)
-          |multipart/(?:report|alternative|mixed|related|partial)
-          )
-        }x
-
         mimeformat = '' # MIME type string of this part
         alternates = argv1.start_with?('multipart/alternative') ? true : false
 
         # Get MIME type string from Content-Type: "..." field at the first line
         # or the second line of the part.
-        if cv = argv0.match(thisformat) then mimeformat = cv[1].downcase end
+        if cv = argv0.match(ThisFormat) then mimeformat = cv[1].downcase end
 
-        # Sisimai require only MIME types defined in $leavesonly variable
-        return '' unless mimeformat =~ leavesonly
+        # Sisimai require only MIME types defined in LeavesOnly variable
+        return '' unless mimeformat =~ LeavesOnly
         return '' if alternates && mimeformat == 'text/html'
 
         (upperchunk, lowerchunk) = argv0.split(/^$/m, 2)
-        upperchunk.gsub!("\n", ' ').squeeze(' ')
+        upperchunk.tr!("\n", ' ').squeeze(' ')
 
         # Content-Description: Undelivered Message
         # Content-Type: message/rfc822
@@ -297,12 +294,12 @@ module Sisimai
           innerparts.shift if innerparts[0].empty?
           while e = innerparts.shift do
             # Find internal multipart/* blocks and decode
-            if cv = e.match(thisformat)
+            if cv = e.match(ThisFormat)
               # Found "Content-Type" field at the first or second line of this
               # splitted part
               nextformat = cv[1].downcase
 
-              next unless nextformat =~ leavesonly
+              next unless nextformat =~ LeavesOnly
               next if nextformat == 'text/html'
 
               hasflatten << Sisimai::MIME.breaksup(e, mimeformat)
@@ -340,12 +337,12 @@ module Sisimai
               # Content-Transfer-Encoding: 8bit, binary, and so on
               getdecoded = lowerchunk
             end
-            getdecoded.gsub!(/\r\n/, "\n")  # Convert CRLF to LF
+            getdecoded.gsub!(/\r\n/, "\n") if getdecoded.include?("\r\n") # Convert CRLF to LF
 
-            if mimeformat =~ alsoappend
+            if mimeformat =~ AlsoAppend
               # Append field when the value of Content-Type: begins with
               # message/ or equals text/rfc822-headers.
-              upperchunk.sub!(/Content-Transfer-Encoding:.+\z/, '').gsub!(/[ ]\z/, '')
+              upperchunk.sub!(/Content-Transfer-Encoding:.+\z/, '').rstrip!
               hasflatten << upperchunk
 
             elsif mimeformat == 'text/html'