sisimai 4.23.0 → 4.24.0

data/lib/sisimai/bite/email/yahoo.rb

@@ -95,10 +95,8 @@ module Sisimai::Bite::Email
                 # Remote host said: 550 5.1.1 <kijitora@example.org>... User Unknown [RCPT_TO]
                 v['diagnosis'] = e
 
-                if cv = e.match(/\[([A-Z]{4}).*\]\z/)
-                  # Get SMTP command from the value of "Remote host said:"
-                  v['command'] = cv[1]
-                end
+                # Get SMTP command from the value of "Remote host said:"
+                if cv = e.match(/\[([A-Z]{4}).*\]\z/) then v['command'] = cv[1] end
               else
                 # <mailboxfull@example.jp>:
                 # Remote host said:

data/lib/sisimai/bite/json/amazonses.rb

@@ -149,10 +149,8 @@ module Sisimai::Bite::JSON
                 v['diagnosis'] = e['diagnosticCode']
               end
 
-              if cv = o['reportingMTA'].match(/\Adsn;[ ](.+)\z/)
-                # 'reportingMTA' => 'dsn; a27-23.smtp-out.us-west-2.amazonses.com',
-                v['lhost'] = cv[1]
-              end
+              # 'reportingMTA' => 'dsn; a27-23.smtp-out.us-west-2.amazonses.com',
+              if cv = o['reportingMTA'].match(/\Adsn;[ ](.+)\z/) then v['lhost'] = cv[1] end
 
               if BounceType.key?(o['bounceType'].to_sym) &&
                  BounceType[o['bounceType'].to_sym].key?(o['bounceSubType'].to_sym)

data/lib/sisimai/data.rb

@@ -94,7 +94,6 @@ module Sisimai
       return nil unless data.is_a? Sisimai::Message
 
       messageobj = data
-      mailheader = data.header
       rfc822data = messageobj.rfc822
       fieldorder = { :recipient => [], :addresser => [] }
       objectlist = []
@@ -197,14 +196,12 @@ module Sisimai
           break if datestring
         end
 
-        if datestring
+        if datestring && cv = datestring.match(/\A(.+)[ ]+([-+]\d{4})\z/)
           # Get the value of timezone offset from datestring
-          if cv = datestring.match(/\A(.+)[ ]+([-+]\d{4})\z/)
-            # Wed, 26 Feb 2014 06:05:48 -0500
-            datestring = cv[1]
-            zoneoffset = Sisimai::DateTime.tz2second(cv[2])
-            p['timezoneoffset'] = cv[2]
-          end
+          # Wed, 26 Feb 2014 06:05:48 -0500
+          datestring = cv[1]
+          zoneoffset = Sisimai::DateTime.tz2second(cv[2])
+          p['timezoneoffset'] = cv[2]
         end
 
         begin
@@ -218,7 +215,7 @@ module Sisimai
         next unless p['timestamp']
 
         # OTHER_TEXT_HEADERS:
-        recvheader = mailheader['received'] || []
+        recvheader = data.header['received'] || []
         unless recvheader.empty?
           # Get localhost and remote host name from Received header.
           %w[lhost rhost].each { |a| e[a] ||= '' }
@@ -244,11 +241,8 @@ module Sisimai
         # The value of "List-Id" header
         p['listid'] = rfc822data['list-id'] || ''
         unless p['listid'].empty?
-          # Get the value of List-Id header
-          if cv = p['listid'].match(/\A.*([<].+[>]).*\z/)
-            # List name <list-id@example.org>
-            p['listid'] = cv[1]
-          end
+          # Get the value of List-Id header like "List name <list-id@example.org>"
+          if cv = p['listid'].match(/\A.*([<].+[>]).*\z/) then p['listid'] = cv[1] end
           p['listid'] = p['listid'].delete('<>').chomp("\r")
           p['listid'] = '' if p['listid'].include?(' ')
         end
@@ -256,11 +250,9 @@ module Sisimai
         # The value of "Message-Id" header
         p['messageid'] = rfc822data['message-id'] || ''
         unless p['messageid'].empty?
-          # Remove angle brackets
-          if cv = p['messageid'].match(/\A([^ ]+)[ ].*/)
-            p['messageid'] = cv[1]
-          end
-          p['messageid'] = p['messageid'].delete('<>').chomp("\r")
+          # Leave only string inside of angle brackets(<>)
+          if cv = p['messageid'].match(/\A([^ ]+)[ ].*/) then p['messageid'] = cv[1] end
+          if cv = p['messageid'].match(/[<]([^ ]+?)[>]/) then p['messageid'] = cv[1] end
         end
 
         # CHECK_DELIVERY_STATUS_VALUE:
@@ -307,10 +299,8 @@ module Sisimai
 
         # Check the value of "action"
         if p['action'].size > 0
-          if cv = p['action'].match(/\A(.+?) .+/)
-            # Action: expanded (to multi-recipient alias)
-            p['action'] = cv[1]
-          end
+          # Action: expanded (to multi-recipient alias)
+          if cv = p['action'].match(/\A(.+?) .+/) then p['action'] = cv[1] end
 
           unless %w[failed delayed delivered relayed expanded].index(p['action'])
             # The value of "action" is not in the following values:

data/lib/sisimai/message/email.rb

@@ -29,7 +29,6 @@ module Sisimai
       DefaultSet = Sisimai::Order::Email.another
       SubjectTab = Sisimai::Order::Email.by('subject')
       ExtHeaders = Sisimai::Order::Email.headers
-      ReEncoding = Sisimai::MIME.patterns
 
       # Make data structure from the email message(a body part and headers)
       # @param         [Hash] argvs   Email data
@@ -392,22 +391,34 @@ module Sisimai
         mailheader['subject']      ||= ''
         mailheader['content-type'] ||= ''
 
+        if hookmethod.is_a? Proc
+          # Call the hook method
+          begin
+            p = {
+              'datasrc' => 'email',
+              'headers' => mailheader,
+              'message' => bodystring,
+              'bounces' => nil
+            }
+            havecaught = hookmethod.call(p)
+          rescue StandardError => ce
+            warn ' ***warning: Something is wrong in hook method :' << ce.to_s
+          end
+        end
+
         # Decode BASE64 Encoded message body, rewrite.
         mesgformat = (mailheader['content-type'] || '').downcase
         ctencoding = (mailheader['content-transfer-encoding'] || '').downcase
 
         if mesgformat.start_with?('text/plain', 'text/html')
           # Content-Type: text/plain; charset=UTF-8
-          if ctencoding == 'base64' || ctencoding == 'quoted-printable'
+          if ctencoding == 'base64'
             # Content-Transfer-Encoding: base64
+            bodystring = Sisimai::MIME.base64d(bodystring)
+
+          elsif ctencoding == 'quoted-printable'
             # Content-Transfer-Encoding: quoted-printable
-            bodystring = if ctencoding == 'base64'
-                           # Content-Transfer-Encoding: base64
-                           Sisimai::MIME.base64d(bodystring)
-                         else
-                           # Content-Transfer-Encoding: quoted-printable
-                           Sisimai::MIME.qprintd(bodystring)
-                         end
+            bodystring = Sisimai::MIME.qprintd(bodystring)
           end
 
           if mesgformat.start_with?('text/html;')
@@ -416,35 +427,10 @@ module Sisimai
           end
         else
           # NOT text/plain
-          lowercased = bodystring.downcase
-          if lowercased =~ ReEncoding[:'quoted-print']
-            # Content-Transfer-Encoding: quoted-printable
-            bodystring = Sisimai::MIME.qprintd(bodystring, mailheader)
-          end
-
-          if lowercased =~ ReEncoding[:'7bit-encoded'] &&
-             cv = lowercased.match(ReEncoding[:'some-iso2022'])
-            # Content-Transfer-Encoding: 7bit
-            # Content-Type: text/plain; charset=ISO-2022-JP
-            if ! cv[1].include?('us-ascii') && ! cv[1].include?('utf-8')
-              bodystring = Sisimai::String.to_utf8(bodystring, cv[1])
-            end
-          end
-        end
-
-        # Call the hook method
-        if hookmethod.is_a? Proc
-          # Execute hook method
-          begin
-            p = {
-              'datasrc' => 'email',
-              'headers' => mailheader,
-              'message' => bodystring,
-              'bounces' => nil
-            }
-            havecaught = hookmethod.call(p)
-          rescue StandardError => ce
-            warn ' ***warning: Something is wrong in hook method :' << ce.to_s
+          if mesgformat.start_with?('multipart/')
+            # In case of Content-Type: multipart/*
+            p = Sisimai::MIME.makeflat(mailheader['content-type'], bodystring)
+            bodystring = p unless p.empty?
           end
         end
 
@@ -469,7 +455,6 @@ module Sisimai
             # 4. Sisimai::Bite::Email::*
             # 5. Sisimai::RFC3464
             # 6. Sisimai::RFC3834
-            #
             if Sisimai::ARF.is_arf(mailheader)
               # Feedback Loop message
               scannedset = Sisimai::ARF.scan(mailheader, bodystring)

data/lib/sisimai/mime.rb

@@ -7,11 +7,12 @@ module Sisimai
       require 'sisimai/string'
 
       ReE = {
-        :'7bit-encoded' => %r/^content-transfer-encoding:[ ]*7bit$/m,
-        :'quoted-print' => %r/^content-transfer-encoding:[ ]*quoted-printable$/m,
-        :'some-iso2022' => %r/^content-type:[ ]*.+;[ ]*charset=["']?(iso-2022-[-a-z0-9]+?)['"]?$/m,
-        :'with-charset' => %r/^content[-]type:[ ]*.+[;][ ]*charset=['"]?(.+?)['"]?$/,
-        :'only-charset' => %r/^[\s\t]+charset=['"]?(.+?)['"]?$/,
+        :'7bit-encoded' => %r/^content-transfer-encoding:[ ]*7bit/m,
+        :'quoted-print' => %r/^content-transfer-encoding:[ ]*quoted-printable/m,
+        :'some-iso2022' => %r/^content-type:[ ]*.+;[ ]*charset=["']?(iso-2022-[-a-z0-9]+?)['"]?\b/m,
+        :'another-8bit' => %r/^content-type:[ ]*.+;[ ]*charset=["']?(.+?)['"]?\b/m,
+        :'with-charset' => %r/^content[-]type:[ ]*.+[;][ ]*charset=['"]?(.+?)['"]?\b/,
+        :'only-charset' => %r/^[\s\t]+charset=['"]?(.+?)['"]?\b/,
         :'html-message' => %r|^content-type:[ ]*text/html;|m,
       }.freeze
 
@@ -53,36 +54,30 @@ module Sisimai
       def mimedecode(argvs = [])
         characterset = nil
         encodingname = nil
-        mimeencoded0 = nil
         decodedtext0 = []
-        notmimetext0 = ''
-        notmimetext1 = ''
 
         while e = argvs.shift do
           # Check and decode each element
           e = e.strip.delete('"')
 
           if self.is_mimeencoded(e)
-            # MIME Encoded string
-            if cv = e.match(/\A(.*)=[?]([-_0-9A-Za-z]+)[?]([BbQq])[?](.+)[?]=?(.*)\z/)
-              # =?utf-8?B?55m954yr44Gr44KD44KT44GT?=
-              notmimetext0   = cv[1]
-              characterset ||= cv[2]
-              encodingname ||= cv[3]
-              mimeencoded0   = cv[4]
-              notmimetext1   = cv[5]
-
-              decodedtext0 << notmimetext0
-              if encodingname == 'Q'
-                # Quoted-Printable
-                decodedtext0 << mimeencoded0.unpack('M').first
-
-              elsif encodingname == 'B'
-                # Base64
-                decodedtext0 << Base64.decode64(mimeencoded0)
-              end
-              decodedtext0 << notmimetext1
+            # MIME Encoded string like "=?utf-8?B?55m954yr44Gr44KD44KT44GT?="
+            next unless cv = e.match(/\A(.*)=[?]([-_0-9A-Za-z]+)[?]([BbQq])[?](.+)[?]=?(.*)\z/)
+
+            characterset ||= cv[2]
+            encodingname ||= cv[3]
+            mimeencoded0   = cv[4]
+            decodedtext0  << cv[1]
+
+            if encodingname == 'Q'
+              # Quoted-Printable
+              decodedtext0 << mimeencoded0.unpack('M').first
+
+            elsif encodingname == 'B'
+              # Base64
+              decodedtext0 << Base64.decode64(mimeencoded0)
             end
+            decodedtext0 << cv[5]
           else
             decodedtext0 << e
           end
@@ -129,13 +124,10 @@ module Sisimai
         boundary01 = Sisimai::MIME.boundary(heads['content-type'], 1)
         bodystring = ''
         notdecoded = ''
-        getencoded = ''
-        lowercased = ''
 
         encodename = nil
         ctencoding = nil
         mimeinside = false
-        mustencode = false
         hasdivided = argv1.split("\n")
 
         while e = hasdivided.shift do
@@ -149,8 +141,8 @@ module Sisimai
             if e == boundary00
               # The next boundary string has appeared
               # --=_gy7C4Gpes0RP4V5Bs9cK4o2Us2ZT57b-3OLnRN+4klS8dTmQ
-              getencoded = Sisimai::String.to_utf8(notdecoded.unpack('M').first, encodename)
-              bodystring << getencoded << e + "\n"
+              hasdecoded = Sisimai::String.to_utf8(notdecoded.unpack('M').first, encodename)
+              bodystring << hasdecoded << e + "\n"
 
               notdecoded = ''
               mimeinside = false
@@ -234,10 +226,7 @@ module Sisimai
         return nil unless argv1
 
         plain = nil
-        if cv = argv1.match(%r|([+/\=0-9A-Za-z\r\n]+)|)
-          # Decode BASE64
-          plain = Base64.decode64(cv[1])
-        end
+        if cv = argv1.match(%r|([+/\=0-9A-Za-z\r\n]+)|) then plain = Base64.decode64(cv[1]) end
         return plain.force_encoding('UTF-8')
       end
 
@@ -256,15 +245,202 @@ module Sisimai
           # Content-Type: multipart/report; report-type=delivery-status;
           #    boundary="n6H9lKZh014511.1247824040/mx.example.jp"
           value = cv[1]
-          value.delete!(%q|'"|)
+          value.delete!(%q|'";\\|)
           value = '--' + value if start > -1
           value = value + '--' if start >  0
         end
 
         return value
       end
-    end
 
+      # Breaks up each multipart/* block
+      # @param    [String] argv0 Text block of multipart/*
+      # @param    [String] argv1 MIME type of the outside part
+      # @return   [String] Decoded part as a plain text(text part only)
+      def breaksup(argv0 = nil, argv1 = '')
+        return nil unless argv0
+
+        hasflatten = '' # Message body including only text/plain and message/*
+        alsoappend = %r{\A(?:text/rfc822-headers|message/)}
+        thisformat = %r/\A(?:Content-Transfer-Encoding:\s*.+\n)?Content-Type:\s*([^ ;]+)/
+        leavesonly = %r{\A(?>
+           text/(?:plain|html|rfc822-headers)
+          |message/(?:x?delivery-status|rfc822|partial|feedback-report)
+          |multipart/(?:report|alternative|mixed|related|partial)
+          )
+        }x
+
+        mimeformat = '' # MIME type string of this part
+        alternates = argv1.start_with?('multipart/alternative') ? true : false
+
+        # Get MIME type string from Content-Type: "..." field at the first line
+        # or the second line of the part.
+        if cv = argv0.match(thisformat) then mimeformat = cv[1].downcase end
+
+        # Sisimai require only MIME types defined in $leavesonly variable
+        return '' unless mimeformat =~ leavesonly
+        return '' if alternates && mimeformat == 'text/html'
+
+        (upperchunk, lowerchunk) = argv0.split(/^$/m, 2)
+        upperchunk.gsub!("\n", ' ').squeeze(' ')
+
+        # Content-Description: Undelivered Message
+        # Content-Type: message/rfc822
+        # <EOM>
+        lowerchunk ||= ''
+
+        if mimeformat.start_with?('multipart/')
+          # Content-Type: multipart/*
+          mpboundary = Regexp.new(Regexp.escape(Sisimai::MIME.boundary(upperchunk, 0)) << "\n")
+          innerparts = lowerchunk.split(mpboundary)
+
+          innerparts.shift if innerparts[0].empty?
+          while e = innerparts.shift do
+            # Find internal multipart/* blocks and decode
+            if cv = e.match(thisformat)
+              # Found "Content-Type" field at the first or second line of this
+              # splitted part
+              nextformat = cv[1].downcase
+
+              next unless nextformat =~ leavesonly
+              next if nextformat == 'text/html'
+
+              hasflatten << Sisimai::MIME.breaksup(e, mimeformat)
+            else
+              # The content of this part is almost '--': a part of boundary
+              # string which is used for splitting multipart/* blocks.
+              hasflatten << "\n"
+            end
+          end
+        else
+          # Is not "Content-Type: multipart/*"
+          if cv = upperchunk.match(/Content-Transfer-Encoding: ([^\s;]+)/)
+            # Content-Transfer-Encoding: quoted-printable|base64|7bit|...
+            ctencoding = cv[1].downcase
+            getdecoded = ''
+
+            if ctencoding == 'quoted-printable'
+              # Content-Transfer-Encoding: quoted-printable
+              getdecoded = Sisimai::MIME.qprintd(lowerchunk)
+
+            elsif ctencoding == 'base64'
+              # Content-Transfer-Encoding: base64
+              getdecoded = Sisimai::MIME.base64d(lowerchunk)
+
+            elsif ctencoding == '7bit'
+              # Content-Transfer-Encoding: 7bit
+              if cv = upperchunk.downcase.match(ReE[:'some-iso2022'])
+                # Content-Type: text/plain; charset=ISO-2022-JP
+                getdecoded = Sisimai::String.to_utf8(lowerchunk, cv[1])
+              else
+                # No "charset" parameter in Content-Type field
+                getdecoded = lowerchunk
+              end
+            else
+              # Content-Transfer-Encoding: 8bit, binary, and so on
+              getdecoded = lowerchunk
+            end
+            getdecoded.gsub!(/\r\n/, "\n")  # Convert CRLF to LF
+
+            if mimeformat =~ alsoappend
+              # Append field when the value of Content-Type: begins with
+              # message/ or equals text/rfc822-headers.
+              upperchunk.sub!(/Content-Transfer-Encoding:.+\z/, '').gsub!(/[ ]\z/, '')
+              hasflatten << upperchunk
+
+            elsif mimeformat == 'text/html'
+              # Delete HTML tags inside of text/html part whenever possible
+              getdecoded.gsub!(/[<][^@ ]+?[>]/, '')
+            end
+
+            unless getdecoded.empty?
+              # The string will be encoded to UTF-8 forcely and call String#scrub
+              # method to avoid the following errors:
+              #   - incompatible character encodings: ASCII-8BIT and UTF-8
+              #   - invalid byte sequence in UTF-8
+              unless getdecoded.encoding.to_s == 'UTF-8'
+                if cv = upperchunk.downcase.match(ReE[:'another-8bit'])
+                  # ISO-8859-1, GB2312, and so on
+                  getdecoded = Sisimai::String.to_utf8(getdecoded, cv[1])
+                end
+              end
+              # A part which has no "charset" parameter causes an ArgumentError:
+              # invalid byte sequence in UTF-8 so String#scrub should be called
+              hasflatten << getdecoded.scrub!('?') << "\n\n"
+            end
+          else
+            # Content-Type: text/plain OR text/rfc822-headers OR message/*
+            if mimeformat.start_with?('message/') || mimeformat == 'text/rfc822-headers'
+              # Append headers of multipart/* when the value of "Content-Type"
+              # is inlucded in the following MIME types:
+              #  - message/delivery-status
+              #  - message/rfc822
+              #  - text/rfc822-headers
+              hasflatten << upperchunk
+            end
+            lowerchunk.sub!(/^--\z/m, '')
+            lowerchunk << "\n" unless lowerchunk =~ /\n\z/
+            hasflatten << lowerchunk
+          end
+        end
+
+        return hasflatten
+      end
+
+      # MIME decode entire message body
+      # @param    [String] argv0 Content-Type header
+      # @param    [String] argv1 Entire message body
+      # @return   [String] Decoded message body
+      def makeflat(argv0 = nil, argv1 = nil)
+        return nil unless argv0
+        return nil unless argv1
+
+        ehboundary = Sisimai::MIME.boundary(argv0, 0)
+        mimeformat = ''
+        bodystring = ''
+
+        # Get MIME type string from an email header given as the 1st argument
+        if cv = argv0.match(%r|\A([0-9a-z]+/[^ ;]+)|) then mimeformat = cv[1] end
+
+        return '' unless mimeformat.include?('multipart/')
+        return '' if ehboundary.empty?
+
+        # Some bounce messages include lower-cased "content-type:" field such as
+        #   content-type: message/delivery-status
+        #   content-transfer-encoding: quoted-printable
+        argv1.gsub!(/[Cc]ontent-[Tt]ype:/m, 'Content-Type:')
+        argv1.gsub!(/[Cc]ontent-[Tt]ransfer-[Ee]ncodeing:/m, 'Content-Transfer-Encoding:')
+
+        # 1. Some bounce messages include upper-cased "Content-Transfer-Encoding",
+        #    and "Content-Type" value such as
+        #      - Content-Type: multipart/RELATED;
+        #      - Content-Transfer-Encoding: 7BIT
+        # 2. Unused fields inside of mutipart/* block should be removed
+        argv1.gsub!(/(Content-[A-Za-z-]+?):[ ]*([^\s]+)/) do "#{$1}: #{$2.downcase}" end
+        argv1.gsub!(/^Content-(?:Description|Disposition):.+$/, '')
+
+        multiparts = argv1.split(Regexp.new(Regexp.escape(ehboundary) << "\n"))
+        multiparts.shift if multiparts[0].empty?
+
+        while e = multiparts.shift do
+          # Find internal multipart blocks and decode
+          if e =~ /\A(?:Content-[A-Za-z-]+:.+?\r\n)?Content-Type:[ ]*[^\s]+/
+            # Content-Type: multipart/*
+            bodystring << Sisimai::MIME.breaksup(e, mimeformat)
+          else
+            # Is not multipart/* block
+            e.sub!(%r|^Content-Transfer-Encoding:.+?\n|mi, '')
+            e.sub!(%r|^Content-Type:\s*text/plain.+?\n|mi, '')
+            bodystring << e
+          end
+        end
+        bodystring.gsub!(%r{^(Content-Type:\s*message/(?:rfc822|delivery-status)).+$}, '\1')
+        bodystring.gsub!(/^\n{2,}/, "\n")
+
+        return bodystring
+      end
+
+    end
   end
 end