sisimai 4.14.2

data/lib/sisimai/rfc3834.rb

@@ -0,0 +1,150 @@
+module Sisimai
+  # Sisimai::RFC3834 - RFC3834 auto reply message detector
+  module RFC3834
+    # Imported from p5-Sisimail/lib/Sisimai/RFC3834.pm
+    class << self
+      # http://tools.ietf.org/html/rfc3834
+      Re0 = {
+        # http://www.iana.org/assignments/auto-submitted-keywords/auto-submitted-keywords.xhtml
+        :'auto-submitted' => %r/\Aauto-(?:generated|replied|notified)/i,
+        # https://msdn.microsoft.com/en-us/library/ee219609(v=exchg.80).aspx
+        :'x-auto-response-suppress' => %r/(?:OOF|AutoReply)/i,
+        :'precedence' => %r/\Aauto_reply\z/,
+        :'subject' => %r/\A(?:
+             Auto:
+            |Out[ ]of[ ]Office:
+          )
+        /xi,
+      }
+      Re1 = { :endof  => %r/\A__END_OF_EMAIL_MESSAGE__\z/ }
+      Re2 = {
+        :subject => %r/(?:
+             SECURITY[ ]information[ ]for  # sudo
+            |Mail[ ]failure[ ][-]          # Exim
+            )
+        /x,
+        :from    => %r/(?:root|postmaster|mailer-daemon)[@]/i,
+        :to      => %r/root[@]/,
+      }
+
+      def description; 'Detector for auto replied message'; end
+      def smtpagent;   'RFC3834'; end
+      def pattern;     return Re0; end
+      def headerlist
+        return [
+          'Auto-Submitted',
+          'Precedence',
+          'X-Auto-Response-Suppress',
+        ]
+      end
+
+      # Detect auto reply message as RFC3834
+      # @param         [Hash] mhead       Message header of a bounce email
+      # @options mhead [String] from      From header
+      # @options mhead [String] date      Date header
+      # @options mhead [String] subject   Subject header
+      # @options mhead [Array]  received  Received headers
+      # @options mhead [String] others    Other required headers
+      # @param         [String] mbody     Message body of a bounce email
+      # @return        [Hash, Undef]      Bounce data list and message/rfc822 part
+      #                                   or Undef if it failed to parse or the
+      #                                   arguments are missing
+      def scan(mhead, mbody)
+        return nil unless mhead
+        return nil unless mbody
+        return nil if mhead.keys.size == 0
+        return nil if mbody.size == 0
+
+        leave = 0
+        match = 0
+
+        # DETECT_EXCLUSION_MESSAGE
+        Re2.each_key do |e|
+          # Exclude message from root@
+          next unless mhead.key?(e.to_s)
+          next unless mhead[e.to_s]
+          next unless mhead[e.to_s] =~ Re2[e]
+          leave = 1
+          break
+        end
+        return nil if leave > 0
+
+        # DETECT_AUTO_REPLY_MESSAGE
+        Re0.each_key do |e|
+          # RFC3834 Auto-Submitted and other headers
+          next unless mhead.key?(e.to_s)
+          next unless mhead[e.to_s]
+          next unless mhead[e.to_s] =~ Re0[e]
+          match += 1
+          break
+        end
+        return nil if match < 1
+
+        require 'sisimai/mta'
+        require 'sisimai/address'
+        require 'sisimai/rfc5322'
+
+        dscontents = []; dscontents << Sisimai::MTA.DELIVERYSTATUS
+        hasdivided = mbody.split("\n")
+        recipients = 0  # (Integer) The number of 'Final-Recipient' header
+        maxmsgline = 5  # (Integer) Max message length(lines)
+        haveloaded = 0  # (Integer) The number of lines loaded from message body
+        blanklines = 0  # (Integer) Counter for countinuous blank lines
+        v = dscontents[-1]
+
+        # RECIPIENT_ADDRESS
+        ['from', 'return-path'].each do |e|
+          # Try to get the address of the recipient
+          next unless mhead.key?(e)
+          next unless mhead[e]
+          v['recipient'] = mhead[e]
+          break
+        end
+
+        if v['recipient']
+          # Clean-up the recipient address
+          v['recipient'] = Sisimai::Address.s3s4(v['recipient'])
+          recipients += 1
+        end
+        return nil unless recipients > 0
+
+        # BODY_PARSER: Get vacation message
+        hasdivided.each do |e|
+          # Read the first 5 lines except a blank line
+          unless e.size > 0
+            # Check a blank line
+            blanklines += 1
+            break if blanklines > 1
+            next
+          end
+          next unless e =~ / /
+
+          v['diagnosis'] ||= ''
+          v['diagnosis']  += e + ' '
+          haveloaded += 1
+          break if haveloaded >= maxmsgline
+        end
+        v['diagnosis'] ||= mhead['subject']
+
+        require 'sisimai/string'
+        v['diagnosis'] = Sisimai::String.sweep(v['diagnosis'])
+        v['reason']    = 'vacation'
+        v['agent']     = self.smtpagent
+        v['date']      = mhead['date']
+        v['status']    = ''
+
+        if mhead['received'].size > 0
+          # Get localhost and remote host name from Received header.
+          r = mhead['received']
+          ['lhost', 'rhost'].each { |a| v[a] ||= '' }
+          v['lhost'] = Sisimai::RFC5322.received(r[0]).shift if v['lhost'].empty?
+          v['rhost'] = Sisimai::RFC5322.received(r[-1]).pop  if v['rhost'].empty?
+        end
+
+        v.each_key { |a| v[a] ||= '' }
+        return { 'ds' => dscontents, 'rfc822' => '' }
+      end
+
+    end
+  end
+end

data/lib/sisimai/rfc5322.rb

@@ -0,0 +1,204 @@
+module Sisimai
+  # Sisimai::RFC5322 provide methods for checking email address.
+  module RFC5322
+    # Imported from p5-Sisimail/lib/Sisimai/RFC5322.pm
+    class << self
+      HeaderTable = {
+        'messageid' => ['Message-Id'],
+        'subject'   => ['Subject'],
+        'listid'    => ['List-Id'],
+        'date'      => ['Date', 'Posted-Date', 'Posted', 'Resent-Date'],
+        'addresser' => [
+          'From', 'Return-Path', 'Reply-To', 'Errors-To', 'Reverse-Path',
+          'X-Postfix-Sender', 'Envelope-From', 'X-Envelope-From',
+        ],
+        'recipient' => [
+          'To', 'Delivered-To', 'Forward-Path', 'Envelope-To',
+          'X-Envelope-To', 'Resent-To', 'Apparently-To'
+        ],
+      }
+
+      build_regular_expressions = lambda do
+        # See http://www.ietf.org/rfc/rfc5322.txt
+        #  or http://www.ex-parrot.com/pdw/Mail-RFC822-Address.html ...
+        #   addr-spec       = local-part "@" domain
+        #   local-part      = dot-atom / quoted-string / obs-local-part
+        #   domain          = dot-atom / domain-literal / obs-domain
+        #   domain-literal  = [CFWS] "[" *([FWS] dcontent) [FWS] "]" [CFWS]
+        #   dcontent        = dtext / quoted-pair
+        #   dtext           = NO-WS-CTL /     ; Non white space controls
+        #                     %d33-90 /       ; The rest of the US-ASCII
+        #                     %d94-126        ;  characters not including "[",
+        #                                     ;  "]", or "\"
+        re             = { 'rfc5322' => nil, 'ignored' => nil, 'domain' => nil }
+        atom           = %r([a-zA-Z0-9_!#\$\%&'*+/=?\^`{}~|\-]+)o
+        quoted_string  = %r/"(?:\\[^\r\n]|[^\\"])*"/o
+        domain_literal = %r/\[(?:\\[\x01-\x09\x0B-\x0c\x0e-\x7f]|[\x21-\x5a\x5e-\x7e])*\]/o
+        dot_atom       = %r/#{atom}(?:[.]#{atom})*/o
+        local_part     = %r/(?:#{dot_atom}|#{quoted_string})/o
+        domain         = %r/(?:#{dot_atom}|#{domain_literal})/o
+
+        re['rfc5322']  = %r/#{local_part}[@]#{domain}/o
+        re['ignored']  = %r/#{local_part}[.]*[@]#{domain}/o
+        re['domain']   = %r/#{domain}/o
+
+        return re
+      end
+
+      build_flatten_rfc822header_list = lambda do
+        # Convert HEADER: structured hash table to flatten hash table for being
+        # called from Sisimai::MTA::*
+        fv = {}
+        HeaderTable.each_value do |e|
+          e.each do |ee|
+            fv[ee.downcase] = 1
+          end
+        end
+        return fv
+      end
+
+      Re          = build_regular_expressions.call
+      HeaderIndex = build_flatten_rfc822header_list.call
+
+      # Grouped RFC822 headers
+      # @param    [String] group  RFC822 Header group name
+      # @return   [Array,Hash]    RFC822 Header list
+      def HEADERFIELDS(group = '')
+        return HeaderIndex unless group.size > 0
+        return HeaderTable[group] if HeaderTable.key?(group)
+        return HeaderTable
+      end
+
+      # Fields that might be long
+      # @return   [Hash] Long filed(email header) list
+      def LONGFIELDS
+        return { 'to' => 1, 'from' => 1, 'subject' => 1, 'message-id' => 1 }
+      end
+
+      # Check that the argument is an email address or not
+      # @param    [String] email  Email address string
+      # @return   [True,False]    true: is a email address
+      #                           false: is not an email address
+      def is_emailaddress(email)
+        return false unless email.is_a?(::String)
+        return false if email =~ %r/(?:[\x00-\x1f]|\x1f)/
+        return true  if email =~ Re['ignored']
+        return false
+      end
+
+      # Check that the argument is an domain part of email address or not
+      # @param    [String] dpart  Domain part of the email address
+      # @return   [True,False]    true: Valid domain part
+      #                           false: Not a valid domain part
+      def is_domainpart(dpart)
+        return false unless dpart.is_a?(::String)
+        return false if dpart =~ /(?:[\x00-\x1f]|\x1f)/
+        return false if dpart =~ /[@]/
+        return true  if dpart =~ Re['domain']
+        return false
+      end
+
+      # Check that the argument is mailer-daemon or not
+      # @param    [String] email  Email address
+      # @return   [True,False]    true: mailer-daemon
+      #                           false: Not mailer-daemon
+      def is_mailerdaemon(email)
+        return false unless email.is_a?(::String)
+
+        re = %r/(?:
+               mailer-daemon[@]
+              |[<(]mailer-daemon[)>]
+              |\Amailer-daemon\z
+              |[ ]?mailer-daemon[ ]
+              )
+        /xi
+        return true if email =~ re
+        return false
+      end
+
+      # Convert Received headers to a structured data
+      # @param    [String] argvs  Received header
+      # @return   [Array]         Received header as a structured data
+      def received(argvs)
+        return [] unless argvs.is_a?(::String)
+
+        hosts = []
+        value = { 'from' => '', 'by' => '' }
+
+        # Received: (qmail 10000 invoked by uid 999); 24 Apr 2013 00:00:00 +0900
+        return [] if argvs =~ /qmail[ ]+.+invoked[ ]+/
+
+        if cr = argvs.match(/\Afrom[ ]+(.+)[ ]+by[ ]+([^ ]+)/)
+          # Received: from localhost (localhost)
+          #   by nijo.example.jp (V8/cf) id s1QB5ma0018057;
+          #   Wed, 26 Feb 2014 06:05:48 -0500
+          value['from'] = cr[1]
+          value['by']   = cr[2]
+
+        elsif cr = argvs.match(/\bby[ ]+([^ ]+)(.+)/)
+          # Received: by 10.70.22.98 with SMTP id c2mr1838265pdf.3; Fri, 18 Jul 2014
+          #   00:31:02 -0700 (PDT)
+          value['from'] = cr[1] + cr[2]
+          value['by']   = cr[1]
+        end
+
+        if value['from'] =~ / /
+          # Received: from [10.22.22.222] (smtp-gateway.kyoto.ocn.ne.jp [192.0.2.222])
+          #   (authenticated bits=0)
+          #   by nijo.example.jp (V8/cf) with ESMTP id s1QB5ka0018055;
+          #   Wed, 26 Feb 2014 06:05:47 -0500
+          received = value['from'].split(' ')
+          namelist = []
+          addrlist = []
+          hostname = ''
+          hostaddr = ''
+
+          received.each do |e|
+            # Received: from [10.22.22.222] (smtp-gateway.kyoto.ocn.ne.jp [192.0.2.222])
+            if e =~ /\A[\[(]\d+[.]\d+[.]\d+[.]\d+[)\]]\z/
+              # [192.0.2.1] or (192.0.2.1)
+              e = e.delete('[]()')
+              addrlist << e
+
+            else
+              # hostname
+              e = e.delete('[]()')
+              namelist << e
+            end
+          end
+
+          namelist.each do |e|
+            # 1. Hostname takes priority over all other IP addresses
+            next unless e =~ /[.]/
+            hostname = e
+            break
+          end
+
+          if hostname.length == 0
+            # 2. Use IP address as a remote host name
+            addrlist.each do |e|
+              # Skip if the address is a private address
+              next if e =~ /\A(?:10|127)[.]/
+              next if e =~ /\A172[.](?:1[6-9]|2[0-9]|3[0-1])[.]/
+              next if e =~ /\A192[.]168[.]/
+              hostaddr = e
+              break
+            end
+          end
+
+          value['from'] = hostname || hostaddr || addrlist[-1]
+        end
+
+        %w[from by].each do |e|
+          # Copy entries into hosts
+          next unless value[e].length > 0
+          value[e] = value[e].delete('[]();?')
+          hosts << value[e]
+        end
+        return hosts
+      end
+
+    end
+  end
+end
+

data/lib/sisimai/rhost.rb

@@ -0,0 +1,47 @@
+module Sisimai
+  # Sisimai::Rhost detects the bounce reason from the content of Sisimai::Data
+  # object as an argument of get() method when the value of rhost of the object
+  # is listed in the results of Sisimai::Rhost->list method.
+  # This class is called only Sisimai::Data class.
+  module Rhost
+    class << self
+      # Imported from p5-Sisimail/lib/Sisimai/Rhost.pm
+      RhostClass = {
+        'aspmx.l.google.com' => 'GoogleApps',
+      }
+
+      # Retrun the list of remote hosts Sisimai support
+      # @return   [Array] Remote host list
+      def list
+        return RhostClass.keys
+      end
+
+      # The value of "rhost" is listed in $RhostClass or not
+      # @param    [String] argvs  Remote host name
+      # @return   [True,False]    True: matched
+      #                           False: did not match
+      def match(host)
+        return false unless host.is_a? ::String
+        return true  if RhostClass.key?(host.downcase)
+        return false
+      end
+
+      # Detect the bounce reason from certain remote hosts
+      # @param    [Sisimai::Data] argvs   Parsed email object
+      # @return   [String]                The value of bounce reason
+      def get(argvs)
+        return nil unless argvs
+        return nil unless argvs.is_a? Sisimai::Data
+        return argvs.reason if argvs.reason.size > 0
+
+        remotehost = argvs.rhost.downcase
+        modulename = 'Sisimai::Rhost::' + RhostClass[remotehost]
+        rhostclass = modulename.gsub('::', '/').downcase
+        require rhostclass
+
+        reasontext = Module.const_get(modulename).get(argvs)
+        return reasontext
+      end
+    end
+  end
+end

data/lib/sisimai/rhost/googleapps.rb

@@ -0,0 +1,237 @@
+module Sisimai
+  module Rhost
+    # Sisimai::Rhost detects the bounce reason from the content of Sisimai::Data
+    # object as an argument of get() method when the value of "rhost" of the object
+    # is "aspmx.l.google.com". This class is called only Sisimai::Data class.
+    module GoogleApps
+      class << self
+        # Imported from p5-Sisimail/lib/Sisimai/Rhost/GoogleApps.pm
+        SMTPErrorSet = {
+          'X.1.1' => [
+            {
+              'reason' => 'userunknown',
+              'regexp' => [%r/The email account that you tried to reach does not exist[.]/],
+            }
+          ],
+          'X.1.2' => [
+            {
+              'reason' => 'hostunknown',
+              'regexp' => [%r/We weren't able to find the recipient domain[.]/],
+            },
+          ],
+          'X.2.1' => [
+            {
+              'reason' => 'undefined',
+              'regexp' => [
+                %r/The user you are trying to contact is receiving mail too quickly[.]/,
+                %r/The user you are trying to contact is receiving mail at a rate/,
+              ],
+            },
+            {
+              'reason' => 'suspend',
+              'regexp' => [%r/The email account that you tried to reach is disabled[.]/],
+            },
+          ],
+          'X.2.2' => [
+            {
+              'reason' => 'mailboxfull',
+              'regexp' => [%r/The email account that you tried to reach is over quota[.]/],
+            },
+          ],
+          'X.2.3' => [
+            {
+              'reason' => 'exceedlimit',
+              'regexp' => [%r/Your message exceeded Google's message size limits[.]/],
+            },
+          ],
+          'X.3.0' => [
+            {
+              'reason' => 'undefined',
+              'regexp' => [
+                %r/Mail server temporarily rejected message[.]/,
+                %r/Multiple destination domains per transaction is unsupported[.]/,
+              ],
+            },
+          ],
+          'X.4.2' => [
+            {
+              'reason' => 'expired',
+              'regexp' => [%r/Timeout [-] closing connection[.]/],
+            },
+          ],
+          'X.4.5' => [
+            {
+              'reason' => 'undefined',
+              'regexp' => [%r/Server busy, try again later[.]/],
+            },
+            {
+              'reason' => 'exceedlimit',
+              'regexp' => [%r/Daily sending quota exceeded[.]/],
+            },
+          ],
+          'X.5.0' => [
+            {
+              'reason' => 'undefined',
+              'regexp' => [
+                %r/SMTP protocol violation, see RFC 2821[.]/,
+                %r/SMTP protocol violation, no commands allowed to pipeline after STARTTLS/,
+              ],
+            },
+          ],
+          'X.5.1' => [
+            {
+              'reason' => 'syntaxerror',
+              'regexp' => [
+                %r/STARTTLS may not be repeated[.]/,
+                %r/Too many unrecognized commands, goodbye[.]/,
+                %r/Unimplemented command[.]/,
+                %r/Unrecognized command[.]/,
+                %r|EHLO/HELO first[.]|,
+                %r/MAIL first[.]/,
+                %r/RCPT first[.]/,
+              ],
+            },
+            {
+              'reason' => 'securityerror',
+              'regexp' => [%r/Authentication Required[.]/],
+            },
+          ],
+          'X.5.2' => [
+            {
+              'reason' => 'undefined',
+              'regexp' => [%r/Cannot Decode response[.]/],
+            },
+            {
+              'reason' => 'syntaxerror',
+              'regexp' => [%r/Syntax error[.]/],
+            },
+          ],
+          'X.5.3' => [
+            {
+              'reason' => 'undefined',
+              'regexp' => [
+                %r/Domain policy size per transaction exceeded[,]/,
+                %r/Your message has too many recipients[.]/,
+              ],
+            },
+          ],
+          'X.5.4' => [
+            {
+              'reason' => 'syntaxerror',
+              'regexp' => [%r/Optional Argument not permitted for that AUTH mode[.]/],
+            },
+          ],
+          'X.6.0' => [
+            {
+              'reason' => 'contenterror',
+              'regexp' => [
+                %r/Mail message is malformed[.]/,
+                %r/Message exceeded 50 hops/,
+              ],
+            },
+          ],
+          'X.7.0' => [
+            {
+              'reason' => 'blocked',
+              'regexp' => [
+                %r/IP not in whitelist for RCPT domain, closing connection[.]/,
+                %r/Our system has detected an unusual rate of unsolicited mail originating from your IP address[.]/,
+              ],
+            },
+            {
+              'reason' => 'expired',
+              'regexp' => [
+                %r/Temporary System Problem. Try again later[.]/,
+                %r/Try again later, closing connection[.]/,
+              ],
+            },
+            {
+              'reason' => 'securityerror',
+              'regexp' => [
+                %r/TLS required for RCPT domain, closing connection[.]/,
+                %r/No identity changes permitted[.]/,
+                %r/Must issue a STARTTLS command first[.]/,
+                %r/Too Many Unauthenticated commands[.]/,
+              ],
+            },
+            {
+              'reason' => 'systemerror',
+              'regexp' => [%r/Cannot authenticate due to temporary system problem[.]/],
+            },
+            {
+              'reason' => 'norelaying',
+              'regexp' => [%r/Mail relay denied[.]/],
+            },
+            {
+              'reason' => 'rejected',
+              'regexp' => [%r/Mail Sending denied[.]/],
+            },
+          ],
+          'X.7.1' => [
+            {
+              'reason' => 'securityerror',
+              'regexp' => [
+                %r/Application-specific password required[.]/,
+                %r/Please log in with your web browser and then try again[.]/,
+                %r/Username and Password not accepted[.]/,
+              ],
+            },
+            {
+              'reason' => 'mailboxfull',
+              'regexp' => [%r/Email quota exceeded[.]/],
+            },
+            {
+              'reason' => 'blocked',
+              'regexp' => [
+                %r/Our system has detected an unusual rate of unsolicited mail originating from your IP address[.]/,
+                %r/The IP you[']re using to send mail is not authorized to send email directly to our servers[.]/,
+              ],
+            },
+            {
+              'reason' => 'contenterror',
+              'regexp' => [%r/Our system has detected that this message is likely unsolicited mail[.]/],
+            },
+            {
+              'reason' => 'filtered',
+              'regexp' => [%r/The user or domain that you are sending to [(]or from[)] has a policy/],
+            },
+            {
+              'reason' => 'rejected',
+              'regexp' => [%r/Unauthenticated email is not accepted from this domain[.]/],
+            },
+          ],
+          'X.7.4' => [
+            {
+              'reason' => 'securityerror',
+              'regexp' => [%r/Unrecognized Authentication Type[.]/],
+            },
+          ],
+        }
+
+        # Detect bounce reason from Google Apps
+        # @param    [Sisimai::Data] argvs   Parsed email object
+        # @return   [String]                The bounce reason for Google Apps
+        # @see      https://support.google.com/a/answer/3726730?hl=en
+        def get(argvs)
+          return nil unless argvs
+          return nil unless argvs.is_a? Sisimai::Data
+          return argvs.reason if argvs.reason.size > 0
+
+          statuscode = argvs.deliverystatus.sub(/\A\d[.](\d+[.]\d+)\z/, 'X.\1')
+          statusmesg = argvs.diagnosticcode
+          reasontext = ''
+          errortable = SMTPErrorSet[statuscode] || []
+
+          errortable.each do |e|
+            # Try to match
+            next unless e['regexp'].find { |a| statusmesg =~ a }
+            reasontext = e['reason']
+            break if reasontext.size > 0
+          end
+          return reasontext
+        end
+
+      end
+    end
+  end
+end