sisimai 4.25.14-java → 4.25.16-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44a59a8c29e1aed16238ad767faeb2f1f3ead742
-  data.tar.gz: 434a9f0e30a5c6257794aa628215faad448f4532
+  metadata.gz: 55c5f02d8a727fa2d8badf264fdd773c75eacade
+  data.tar.gz: 747708d2f576cabcf9ae6c60399514e1dd1bb167
 SHA512:
-  metadata.gz: e86e13a188372837ab2f7a4180e771c41ef68338e277f109e3edd007a8fc4ffe771586582163b359406359bc4b5e649de13081a38d54a9ac7b06bad6ea3bdbf5
-  data.tar.gz: 28cfe1bfaef9b1e984b03c5f86a71e425ce1b2327ba38c2d381bfdd2926a15054c9b8387d26a50977f20c9b100e1802d922fff3a66dc5ac93849dfa562107993
+  metadata.gz: e189c252c4b5646effd341cc6ef5c7736c5ae48c59d6b8009695186385512ade0367ab14a7eeed9d145b2f39231553e93ed5f3bdfbf898c7024e2ada797863ca
+  data.tar.gz: 6374615d1cf4969205033b7f3bb9e5a738e26d86a63b38da114b984d2054a54f9a669dcbc0020d53e55fcb2a7c85b6bc127869a78cab943ba62f5a41d604995d

data/ChangeLog.md

@@ -3,6 +3,57 @@ RELEASE NOTES for Ruby version of Sisimai
 - releases: "https://github.com/sisimai/rb-sisimai/releases"
 - download: "https://rubygems.org/gems/sisimai"
 
+v4.25.16
+--------------------------------------------------------------------------------
+- release: "Tue, 16 May 2023 15:11:22 +0900 (JST)"
+- version: "4.25.16"
+- changes:
+  - Fix `Sisimai::MIME.mimedecode()` method when the encoding name is lower case
+    "b" (not "B") like `=?utf-8?b?...` #251 Thanks to @revsystem
+  - #253 Fix code in `Sisimai::Data.make()` method to handle `nil` returned from
+    `Sisimai::Rhost.get()` method. Thanks to @axfcampos
+  - Tiny code improvement: import code from sisimai/p5-sisimai#493 (message.rb)
+  - Tiny code improvement: import code from sisimai/p5-sisimai#494 (rfc3834.rb)
+
+v4.25.15
+--------------------------------------------------------------------------------
+- release: "Thu, 22 Dec 2022 13:13:13 +0900 (JST)"
+- version: "4.25.15"
+- changes:
+  - Prevent ReDOS: Regex Denial of Service in `Sisimai::String.to_plain` #244.
+    Thanks to @gmcabrita
+  - Fix an error reason "5.2.1 exceedlimit" of an error message: "450-4.2.1 The
+    user you are trying to contact is receiving mail at a rate that prevents ad-
+    ditional messages from being delivered" to "toomanyconn" at Rhost::GoogleApp
+    class, imported from sisimai/p5-sisimai#465.
+  - Bug fix: getting a "name" part of an email address at Sisimai::Address.find
+  - Add many error message patterns at the following classes in `Sisimai/Reason`
+    - `Blocked`
+    - `ContentError`
+    - `Filtered`
+    - `NetworkError`
+    - `NotAccept`
+    - `PolicyViolation`
+    - `Rejected`
+    - `SpamDetected`
+    - `TooManyConn`
+    - `UserUnknown`
+  - When the value of `diagnosticcode` has enough error message for detecting a
+    bounce reason, `expired` will change to `networkerror`
+  - Update code in `Sisimai::Lhost::GoogleGroups` module to parse well a bounce
+    mail even if an email address of `X-Failed-Recipients:` header does not in-
+    clude a domain `@googlegroups.com` #234
+  - Import #235 from Sisimai v5, Implement `Sisimai::Rhost::NTTDOCOMO` to parse
+    more strictly a bounce mail returned from `mfsmax.docomo.ne.jp`
+  - Import #241 from Sisimai v5, Implement `Sisimai::Rhost::Mimecast`
+  - Add the following error messages at `Sisimai::Rhost::ExchangeOnline:
+    - 451 4.7.650 The mail server [...] has been temporarily rate limited due to
+      IP reputation ... (S775)
+    - 550 5.7.1 ... Please contact your Internet service provider since part of
+      their network is on our block list (S3150)
+    - Error messages of Exchange Server 2019 #242
+  - Add many error messages at `Sisimai::Rhost::GoogleApps`, Import #243
+
 v4.25.14
 --------------------------------------------------------------------------------
 - release: "Mon, 15 Aug 2022 14:22:22 +0900 (JST)"

data/README-JA.md

@@ -220,7 +220,7 @@ Differences between Ruby version and Perl version
 | メール解析速度(1000通のメール)              | 2.22秒[2]      | 1.35秒        |
 | インストール方法                            | gem install    | cpanm, cpm    |
 | 依存モジュール数(コアモジュールを除く)      | 1モジュール    | 2モジュール   |
-| LOC:ソースコードの行数                      | 10400行        | 10500行       |
+| LOC:ソースコードの行数                      | 10600行        | 10800行       |
 | テスト件数(spec/,t/,xt/ディレクトリ)        | 241000件       | 270000件      |
 | ライセンス                                  | 二条項BSD      | 二条項BSD     |
 | 開発会社によるサポート契約                  | 提供中         | 提供中        |
@@ -276,7 +276,7 @@ Author
 
 Copyright
 ===============================================================================
-Copyright (C) 2015-2020 azumakuniyuki, All Rights Reserved.
+Copyright (C) 2015-2022 azumakuniyuki, All Rights Reserved.
 
 License
 ===============================================================================

data/README.md

@@ -225,7 +225,7 @@ and bounceHammer are available at
 | The speed of parsing email(1000 emails)     | 2.22s[2]       | 1.35s         |
 | How to install                              | gem install    | cpanm, cpm    |
 | Dependencies (Except core modules)          | 1 module       | 2 modules     |
-| LOC:Source lines of code                    | 10400 lines    | 10500 lines   |
+| LOC:Source lines of code                    | 10600 lines    | 10800 lines   |
 | The number of tests(spec/,t/,xt/) directory | 241000 tests   | 270000 tests  |
 | License                                     | BSD 2-Clause   | BSD 2-Clause  |
 | Support Contract provided by Developer      | Available      | Available     |
@@ -283,7 +283,7 @@ Author
 
 Copyright
 ===============================================================================
-Copyright (C) 2015-2021 azumakuniyuki, All Rights Reserved.
+Copyright (C) 2015-2022 azumakuniyuki, All Rights Reserved.
 
 License
 ===============================================================================

data/lib/sisimai/address.rb

@@ -176,7 +176,7 @@ module Sisimai
             else
               # Deal as a display name
               readcursor &= ~Indicators[:'comment-block']
-              v[:name] = e
+              v[:name] << e
               p = ''
             end
             next

data/lib/sisimai/data.rb

@@ -256,13 +256,17 @@ module Sisimai
           if vm > 2
             # Build regular expression for removing string like '550-5.1.1'
             # from the value of "diagnosticcode"
-            re = %r/[ ]#{vr}[- ](?:#{vs})?/
+            re0 = %r/;?[ ]#{vr}[- ](?:#{vs})?/
+            re1 = %r/;?[ ][45]\d\d[- ](?:#{vs})?/
+            re2 = %r/;?[ ]#{vr}[- ](?:[45][.]\d[.]\d+)?/
 
             # 550-5.7.1 [192.0.2.222] Our system has detected that this message is
             # 550-5.7.1 likely unsolicited mail. To reduce the amount of spam sent to Gmail,
             # 550-5.7.1 this message has been blocked. Please visit
             # 550 5.7.1 https://support.google.com/mail/answer/188131 for more information.
-            p['diagnosticcode'] = p['diagnosticcode'].gsub(re, ' ')
+            p['diagnosticcode'] = p['diagnosticcode'].gsub(re0, ' ')
+            p['diagnosticcode'] = p['diagnosticcode'].gsub(re1, ' ')
+            p['diagnosticcode'] = p['diagnosticcode'].gsub(re2, ' ')
             p['diagnosticcode'] = Sisimai::String.sweep(p['diagnosticcode'].sub(%r|<html>.+</html>|i, ''))
           end
         end
@@ -299,7 +303,7 @@ module Sisimai
           if r.empty?
             # Failed to detect a bounce reason by the value of "rhost"
             r = Sisimai::Rhost.get(o, o.destination) if Sisimai::Rhost.match(o.destination)
-            r = Sisimai::Reason.get(o) if r.empty?
+            r = Sisimai::Reason.get(o) if r.nil? || r.empty?
             r = 'undefined' if r.empty?
           end
           o.reason = r

data/lib/sisimai/lhost/facebook.rb

@@ -22,20 +22,21 @@ module Sisimai::Lhost
           'RCP-P2', # The attempted recipient's preferences prevent messages from being delivered.
           'RCP-P3', # The attempted recipient's privacy settings blocked the delivery.
         ],
+        'blocked' => [
+          'POL-P1', # Your mail server's IP Address is listed on the Spamhaus PBL.
+          'POL-P2', # Facebook will no longer accept mail from your mail server's IP Address.
+        ],
         'mesgtoobig' => [
           'MSG-P1', # The message exceeds Facebook's maximum allowed size.
           'INT-P2', # The message exceeds Facebook's maximum allowed size.
         ],
         'contenterror' => [
           'MSG-P2', # The message contains an attachment type that Facebook does not accept.
-          'MSG-P3', # The message contains multiple instances of a header field that can only be present once. Please see RFC 5322, section 3.6 for more information
+          'MSG-P3', # The message contains multiple instances of a header field that can only be present once.
           'POL-P6', # The message contains a url that has been blocked by Facebook.
           'POL-P7', # The message does not comply with Facebook's abuse policies and will not be accepted.
         ],
         'securityerror' => [
-          'POL-P1', # Your mail server's IP Address is listed on the Spamhaus PBL.
-          'POL-P2', # Facebook will no longer accept mail from your mail server's IP Address.
-          'POL-P5', # The message contains a virus.
           'POL-P7', # The message does not comply with Facebook's Domain Authentication requirements.
         ],
         'notaccept' => [
@@ -54,17 +55,20 @@ module Sisimai::Lhost
         ],
         'systemerror' => [
           'CON-T1', # Facebook's mail server currently has too many connections open to allow another one.
+          'RCP-T1', # The attempted recipient address is not currently available due to an internal system issue. This is a temporary condition.
+        ],
+        'virusdetected' => [
+          'POL-P5', # The message contains a virus.
         ],
         'toomanyconn' => [
+          'CON-T2', # Your mail server currently has too many connections open to Facebook's mail servers.
           'CON-T3', # Your mail server has opened too many new connections to Facebook's mail servers in a short period of time.
         ],
         'suspend' => [
           'RCP-T4', # The attempted recipient address is currently deactivated. The user may or may not reactivate it.
         ],
         'undefined' => [
-          'RCP-T1', # The attempted recipient address is not currently available due to an internal system issue. This is a temporary condition.
           'MSG-T1', # The number of recipients on the message exceeds Facebook's allowed maximum.
-          'CON-T2', # Your mail server currently has too many connections open to Facebook's mail servers.
           'CON-T4', # Your mail server has exceeded the maximum number of recipients for its current connection.
         ],
       }.freeze

data/lib/sisimai/lhost/googlegroups.rb

@@ -19,7 +19,7 @@ module Sisimai::Lhost
         return nil unless mhead['from'].end_with?('<mailer-daemon@googlemail.com>')
         return nil unless mhead['subject'].start_with?('Delivery Status Notification')
         return nil unless mhead['x-failed-recipients']
-        return nil unless mhead['x-failed-recipients'].include?('@googlegroups.com')
+        return nil unless mhead['x-google-smtp-source']
 
         # Hello kijitora@libsisimai.org,
         #
@@ -55,7 +55,6 @@ module Sisimai::Lhost
 
         mhead['x-failed-recipients'].split(',').each do |e|
           # X-Failed-Recipients: neko@example.jp, nyaan@example.org, ...
-          next unless e.end_with?('@googlegroups.com')
           next unless Sisimai::RFC5322.is_emailaddress(e)
 
           if v['recipient']

data/lib/sisimai/lhost/imailserver.rb

@@ -21,7 +21,7 @@ module Sisimai::Lhost
         'hostunknown'   => %r/Unknown host/,
         'userunknown'   => %r/\A(?:Unknown user|Invalid final delivery userid)/,
         'mailboxfull'   => %r/\AUser mailbox exceeds allowed size/,
-        'securityerror' => %r/\ARequested action not taken: virus detected/,
+        'virusdetected' => %r/\ARequested action not taken: virus detected/,
         'undefined'     => %r/\Aundeliverable to /,
         'expired'       => %r/\ADelivery failed \d+ attempts/,
       }.freeze

data/lib/sisimai/message.rb

@@ -159,9 +159,22 @@ module Sisimai
         # Set pseudo UNIX From line
         block[0] = 'MAILER-DAEMON Tue Feb 11 00:00:00 2014'
       end
-
       block[1] << "\n" unless block[1].end_with?("\n")
+
+      %w[image/ application/ text/html].each do |e|
+        # https://github.com/sisimai/p5-sisimai/issues/492, Reduce email size
+        p0 = 0
+        p1 = 0
+        ep = e == 'text/html' ? '</html>' : "--\n"
+        while true
+          # Remove each part from "Content-Type: image/..." to "--\n" (the end of each boundary)
+          p0 = block[2].index('Content-Type: ' + e, p0); break unless p0
+          p1 = block[2].index(ep, p0 + 32);              break unless p1
+          block[2][p0, p1 - p0] = ''
+        end
+      end
       block[2] << "\n"
+
       return block
     end
 
@@ -245,7 +258,7 @@ module Sisimai
       # Decode BASE64 Encoded message body, rewrite.
       mesgformat = (mailheader['content-type'] || '').downcase
       ctencoding = (mailheader['content-transfer-encoding'] || '').downcase
-      if mesgformat.start_with?('text/plain', 'text/html')
+      if mesgformat.start_with?('text/')
         # Content-Type: text/plain; charset=UTF-8
         if ctencoding == 'base64'
           # Content-Transfer-Encoding: base64

data/lib/sisimai/mime.rb

@@ -71,7 +71,7 @@ module Sisimai
             mimeencoded0   = cv[4]
 
             decodedtext0 << cv[1]
-            decodedtext0 << if encodingname == 'B'
+            decodedtext0 << if encodingname.upcase == 'B'
                               Base64.decode64(mimeencoded0)
                             else
                               mimeencoded0.unpack('M').first

data/lib/sisimai/reason/blocked.rb

@@ -51,11 +51,12 @@ module Sisimai
             |does[ ]not[ ]exist:
             )
           |dns[ ]lookup[ ]failure:[ ][^ ]+[ ]try[ ]again[ ]later
-          |dnsbl:attrbl
+          |dnsbl:(?:attrbl|rbl[ ]\d+[<][ ].+[ ][>]_is_blocked)
           |dynamic/zombied/spam[ ]ips[ ]blocked
           |email[ ]blocked[ ]by[ ](?:[^ ]+[.]barracudacentral[.]org|spamhaus)
           |error:[ ]no[ ]valid[ ]recipients[ ]from[ ]
           |esmtp[ ]not[ ]accepting[ ]connections  # icloud.com
+          |extreme[ ]bad[ ]ip[ ]profile
           |fix[ ]reverse[ ]dns[ ]for[ ][^ ]+
           |go[ ]away
           |helo[ ]command[ ]rejected:

data/lib/sisimai/reason/contenterror.rb

@@ -11,6 +11,7 @@ module Sisimai
       # Imported from p5-Sisimail/lib/Sisimai/Reason/ContentError.pm
       class << self
         Index = [
+          'header size exceeds maximum permitted',
           'improper use of 8-bit data in message header',
           'message header size, or recipient list, exceeds policy limit',
           'message mime complexity exceeds the policy maximum',

data/lib/sisimai/reason/filtered.rb

@@ -19,6 +19,7 @@ module Sisimai
           'has restricted sms e-mail',    # AT&T
           'is not accepting any mail',
           'message rejected due to user rules',
+          'not found recipient account',
           'refused due to recipient preferences', # Facebook
           'resolver.rst.notauthorized',   # Microsoft Exchange
           'this account is protected by',

data/lib/sisimai/reason/networkerror.rb

@@ -18,6 +18,7 @@ module Sisimai
           'dns records for the destination computer could not be found',
           'hop count exceeded - possible mail loop',
           'host is unreachable',
+          'host name lookup failure',
           'host not found, try again',
           'mail forwarding loop for ',
           'malformed name server reply',

data/lib/sisimai/reason/notaccept.rb

@@ -13,6 +13,7 @@ module Sisimai
       class << self
         # Destination mail server does not accept any message
         Index = [
+          'does not accept mail (nullmx)',
           'host/domain does not accept mail', # iCloud
           'host does not accept mail',        # Sendmail
           'name server: .: host not found',   # Sendmail

data/lib/sisimai/reason/policyviolation.rb

@@ -32,10 +32,12 @@ module Sisimai
           'message bounced due to organizational settings',
           'message given low priority',
           'message not accepted for policy reasons',
+          'message rejected due to local policy',
           'messages with multiple addresses',
           'rejected for policy reasons',
           'protocol violation',
           'the email address used to send your message is not subscribed to this group',
+          'the message was rejected by organization policy',
           'this message was blocked because its content presents a potential',
           'we do not accept messages containing images or other attachments',
           'you have exceeded the allowable number of posts without solving a captcha',

data/lib/sisimai/reason/rejected.rb

@@ -28,6 +28,7 @@ module Sisimai
           'batv failed to verify',    # SoniWall
           'batv validation failure',  # SoniWall
           'backscatter protection detected an invalid or expired email address',  # MDaemon
+          "because the sender isn't on the recipient's list of senders to accept mail from",
           'bogus mail from',          # IMail - block empty sender
           'connections not accepted from servers without a valid sender domain',
           'denied [bouncedeny]',      # McAfee
@@ -35,6 +36,7 @@ module Sisimai
           'delivery not authorized, message refused',
           'does not exist e2110',
           'domain of sender address ',
+          'email address is on senderfilterconfig list',
           'emetteur invalide',
           'empty envelope senders not allowed',
           'envelope blocked – ',
@@ -55,6 +57,7 @@ module Sisimai
           'rule imposed mailbox access for',  # MailMarshal
           'sender address has been blacklisted',
           'sender email address rejected',
+          'sender is in my black list',
           'sender is spammer',
           'sender not pre-approved',
           'sender rejected',
@@ -65,7 +68,9 @@ module Sisimai
           'this server does not accept mail from',
           'transaction failed unsigned dsn for',
           'unroutable sender address',
+          'you are not allowed to post to this mailing list',
           'you are sending to/from an address that has been blacklisted',
+          'your access to submit messages to this e-mail system has been rejected'
         ]
 
         def text; return 'rejected'; end

data/lib/sisimai/reason/spamdetected.rb

@@ -46,12 +46,15 @@ module Sisimai
           |may[ ]consider[ ]spam
           |message[ ](?:
              considered[ ]as[ ]spam[ ]or[ ]virus
+            |contains[ ]spam[ ]or[ ]virus
             |content[ ]rejected
+            |detected[ ]as[ ]spam
             |filtered
             |filtered[.][ ](?:
                please[ ]see[ ]the[ ]faqs[ ]section[ ]on[ ]spam
               |refer[ ]to[ ]the[ ]troubleshooting[ ]page[ ]at[ ]
               )
+            |is[ ]being[ ]rejected[ ]as[ ]it[ ]seems[ ]to[ ]be[ ]a[ ]spam
             |looks[ ]like[ ]spam
             |refused[ ]by[ ](?:
               mailmarshal[ ]spamprofiler
@@ -99,6 +102,7 @@ module Sisimai
           |spambouncer[ ]identified[ ]spam    # SpamBouncer identified SPAM
           |spamming[ ]not[ ]allowed
           |too[ ]much[ ]spam[.]               # Earthlink
+          |the[ ]email[ ]message[ ]was[ ]detected[ ]as[ ]spam
           |the[ ]message[ ](?:
              has[ ]been[ ]rejected[ ]by[ ]spam[ ]filtering[ ]engine
             |was[ ]rejected[ ]due[ ]to[ ]classification[ ]as[ ]bulk[ ]mail

data/lib/sisimai/reason/userunknown.rb

@@ -16,13 +16,14 @@ module Sisimai
     module UserUnknown
       # Imported from p5-Sisimail/lib/Sisimai/Reason/UserUnknown.pm
       class << self
-        PreMatches = %w[NoRelaying Blocked MailboxFull HasMoved Rejected]
+        PreMatches = %w[NoRelaying Blocked MailboxFull HasMoved Rejected NotAccept]
         ModulePath = {
           'Sisimai::Reason::NoRelaying'  => 'sisimai/reason/norelaying',
           'Sisimai::Reason::Blocked'     => 'sisimai/reason/blocked',
           'Sisimai::Reason::MailboxFull' => 'sisimai/reason/mailboxfull',
           'Sisimai::Reason::HasMoved'    => 'sisimai/reason/hasmoved',
           'Sisimai::Reason::Rejected'    => 'sisimai/reason/rejected',
+          'Sisimai::Reason::NotAccept'   => 'sisimai/reason/notaccept',
         }
         Regex = %r{(?>
            [#]5[.]1[.]1[ ]bad[ ]address
@@ -98,6 +99,7 @@ module Sisimai
             |address[ ]rejected:[ ](?:
                access[ ]denied
               |invalid[ ]user
+              |invalid-recipient
               |user[ ][^ ]+[ ]does[ ]not[ ]exist
               |user[ ]unknown[ ]in[ ][^ ]+[ ]table
               |unknown[ ]user
@@ -105,6 +107,7 @@ module Sisimai
             |does[ ]not[ ]exist(?:[ ]on[ ]this[ ]system)?
             |is[ ]not[ ]local
             |not[ ](?:exist|found|ok)
+            |refuses[ ]to[ ]accept[ ]your[ ]mail
             |unknown
             )
           |requested[ ]action[ ]not[ ]taken:[ ]mailbox[ ]unavailable
@@ -114,7 +117,9 @@ module Sisimai
              user[ ]unknown
             |badrcptto
             |no[ ]mailbox[ ]here[ ]by[ ]that[ ]name
+            |your[ ]envelope[ ]recipient[ ]has[ ]been[ ]denied
             )
+          |that[ ]domain[ ]or[ ]user[ ]isn't[ ]in[ ]my[ ]list[ ]of[ ]allowed[ ]rcpthosts
           |the[ ](?:
              email[ ]account[ ]that[ ]you[ ]tried[ ]to[ ]reach[ ]does[ ]not[ ]exist
             |following[ ]recipients[ ]was[ ]undeliverable
@@ -124,6 +129,8 @@ module Sisimai
           |this[ ](?:
              address[ ]no[ ]longer[ ]accepts[ ]mail
             |email[ ]address[ ]is[ ]wrong[ ]or[ ]no[ ]longer[ ]valid
+            |recipient[ ]is[ ]in[ ]my[ ]badrecipientto[ ]list
+            |recipient[ ]is[ ]not[ ]in[ ]my[ ]validrcptto[ ]list
             |spectator[ ]does[ ]not[ ]exist
             |user[ ]doesn[']?t[ ]have[ ]a[ ][^ ]+[ ]account
             )

data/lib/sisimai/rfc3464.rb

@@ -83,7 +83,7 @@ module Sisimai
         |SMTP[ ]Server[ ][<].+[>][ ]rejected[ ]recipient[ ]
         |The[ ]following[ ]recipients[ ]returned[ ]permanent[ ]errors:[ ]
         |The[ ]following[ ]message[ ]to[ ]
-        |Unknown[ ]User:[ ]
+        |Unknown[ ]user:[ ]
         |undeliverable[ ]to[ ]
         |Undeliverable[ ]Address:[ ]*
         |You[ ]sent[ ]mail[ ]to[ ]

data/lib/sisimai/rfc3834.rb

@@ -8,8 +8,6 @@ module Sisimai
       AutoReply1 = {
         # http://www.iana.org/assignments/auto-submitted-keywords/auto-submitted-keywords.xhtml
         'auto-submitted' => %r/\Aauto-(?:generated|replied|notified)/,
-        # https://msdn.microsoft.com/en-us/library/ee219609(v=exchg.80).aspx
-        'x-auto-response-suppress' => %r/(?:oof|autoreply)/,
         'x-apple-action' => %r/\Avacation\z/,
         'precedence' => %r/\Aauto_reply\z/,
         'subject' => %r/\A(?>