sisimai 4.25.11 → 4.25.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b96e6f5eb8587091c2902d4bdf83d63fd5d36606d80120428b509afeae37ab4b
-  data.tar.gz: 074ee81152220a51456a3a4d2236075715b1aa7045ddcace81aa0fd406676f4f
+  metadata.gz: 74b7ce027bb94fdb0892c51dbdabafa7844a1bd99fae1828e318ca01842b42dd
+  data.tar.gz: 32774cf337f341ae6002fae61c9849bed9ff42d0d0c392ceaa5099350d3dcc0e
 SHA512:
-  metadata.gz: d3dbf3a7f80e30ef008b925c3f2105d005663a61e94bff2e361f9c9bf5bd80991cc9ca2e7ab522c657012ebe42a7f717de9d96a241c5b5b4e3d16921f8d6ac77
-  data.tar.gz: 8903044d48d468b905758a0b7c4abd8851d5609bbefb7f46c4f09379529f06751d6f0bec5214a50f782c7ea5902f2df33ea663a8806418584ff900cc13b5e019
+  metadata.gz: 1fd951bfff634447e4fbdf932d31bece6e0779950e9e6f5b4e65fa25f67390c8e53b28f3cac2a5552a0348b0bb1efd94773600268315064a5ef9820beda18219
+  data.tar.gz: 42e12f721657a1e2de88ceb3b247dda2c6c967d1650823809fcc7d3d96f9f4c5dc8404ebe43cf4ae28fcf697602438e6e303ee0c525fee7bd59f03ca6203e4e1

data/ChangeLog.md

@@ -3,6 +3,43 @@ RELEASE NOTES for Ruby version of Sisimai
 - releases: "https://github.com/sisimai/rb-sisimai/releases"
 - download: "https://rubygems.org/gems/sisimai"
 
+v4.25.14
+--------------------------------------------------------------------------------
+- release: "Mon, 15 Aug 2022 14:22:22 +0900 (JST)"
+- version: "4.25.14"
+- changes:
+  - Nothing changed, follow the Perl version of Sisimai 4.25.14.
+
+v4.25.13
+--------------------------------------------------------------------------------
+- release: "Mon,  1 Aug 2022 16:05:55 +0900 (JST)"
+- version: "4.25.13"
+- changes:
+  - Nothing changed, follow the Perl version of Sisimai 4.25.13.
+
+v4.25.12
+--------------------------------------------------------------------------------
+- release: "Mon, 22 Nov 2021 12:22:22 +0900 (JST)"
+- version: "4.25.12"
+- changes:
+  - Fix 2 encoding related errors:
+    - Fix an error with message `incompatible character encodings:ASCII-8BIT and
+      UTF-8` reported at issue #223. Thanks to @chahn
+    - Fix an error with message `invalid byte sequence in UTF-8 (ArgumentError)`
+      reported at issue #224. Thanks to @chahn
+    - Add `rfc3464-41.eml` and `rfc3464-42.eml` provided by @chahn
+    - Add a new error code of La Poste at `Sisimai::Rhost::FrancePTT` module and
+      a new sample email `rhost-franceptt-12.eml` into `set-of-emails` directory
+      imported from sisimai/p5-sisimai#441.
+    - Remove all the HTML elements from the value of `diagnosticcode`.
+  - Fix serious bugs:
+    - Both of the values of `deliverystatus` and `replycode` detected from the
+      message body did not use at `Sisimai::Lhost::Exim`.
+    - `true` method strictly checks the value of `smtpcommand` at some classes
+      in `Sisimai/Reason`. For example, when a detected reason is `spamdetected`
+      and `virusdetected` the value of `smtpcommand` should be `DATA` or an SMTP
+      command to be sent after `DATA`.
+
 v4.25.11
 --------------------------------------------------------------------------------
 - release: "Mon, 22 Feb 2021 21:15:22 +0900 (JST)"

data/README-JA.md

@@ -220,8 +220,8 @@ Differences between Ruby version and Perl version
 | メール解析速度(1000通のメール)              | 2.22秒[2]      | 1.35秒        |
 | インストール方法                            | gem install    | cpanm, cpm    |
 | 依存モジュール数(コアモジュールを除く)      | 1モジュール    | 2モジュール   |
-| LOC:ソースコードの行数                      | 10300行        | 10500行       |
-| テスト件数(spec/,t/,xt/ディレクトリ)        | 236000件       | 265000件      |
+| LOC:ソースコードの行数                      | 10400行        | 10500行       |
+| テスト件数(spec/,t/,xt/ディレクトリ)        | 241000件       | 270000件      |
 | ライセンス                                  | 二条項BSD      | 二条項BSD     |
 | 開発会社によるサポート契約                  | 提供中         | 提供中        |
 

data/README.md

@@ -225,8 +225,8 @@ and bounceHammer are available at
 | The speed of parsing email(1000 emails)     | 2.22s[2]       | 1.35s         |
 | How to install                              | gem install    | cpanm, cpm    |
 | Dependencies (Except core modules)          | 1 module       | 2 modules     |
-| LOC:Source lines of code                    | 10300 lines    | 10500 lines   |
-| The number of tests(spec/,t/,xt/) directory | 236000 tests   | 265000 tests  |
+| LOC:Source lines of code                    | 10400 lines    | 10500 lines   |
+| The number of tests(spec/,t/,xt/) directory | 241000 tests   | 270000 tests  |
 | License                                     | BSD 2-Clause   | BSD 2-Clause  |
 | Support Contract provided by Developer      | Available      | Available     |
 
@@ -283,7 +283,7 @@ Author
 
 Copyright
 ===============================================================================
-Copyright (C) 2015-2020 azumakuniyuki, All Rights Reserved.
+Copyright (C) 2015-2021 azumakuniyuki, All Rights Reserved.
 
 License
 ===============================================================================

data/lib/sisimai/data.rb

@@ -262,10 +262,16 @@ module Sisimai
             # 550-5.7.1 likely unsolicited mail. To reduce the amount of spam sent to Gmail,
             # 550-5.7.1 this message has been blocked. Please visit
             # 550 5.7.1 https://support.google.com/mail/answer/188131 for more information.
-            p['diagnosticcode'] = Sisimai::String.sweep(p['diagnosticcode'].gsub(re, ' '))
+            p['diagnosticcode'] = p['diagnosticcode'].gsub(re, ' ')
+            p['diagnosticcode'] = Sisimai::String.sweep(p['diagnosticcode'].sub(%r|<html>.+</html>|i, ''))
           end
         end
 
+        if Sisimai::String.is_8bit(p['diagnosticcode'])
+          # To avoid incompatible character encodings: ASCII-8BIT and UTF-8 (Encoding::CompatibilityError
+          p['diagnosticcode'] = p['diagnosticcode'].force_encoding('UTF-8').scrub('?')
+        end
+
         p['diagnostictype'] ||= 'X-UNIX' if p['reason'] == 'mailererror'
         p['diagnostictype'] ||= 'SMTP' unless %w[feedback vacation].include?(p['reason'])
 

data/lib/sisimai/lhost/exim.rb

@@ -440,8 +440,8 @@ module Sisimai::Lhost
           #   Diagnostic-Code: smtp; 450 TEMPERROR: retry timeout exceeded
           # The value of "Status:" indicates permanent error but the value
           # of SMTP reply code in Diagnostic-Code: field is "TEMPERROR"!!!!
-          sv = Sisimai::SMTP::Status.find(e['diagnosis'])
-          rv = Sisimai::SMTP::Reply.find(e['diagnosis'])
+          sv = e['status']    || Sisimai::SMTP::Status.find(e['diagnosis'])
+          rv = e['replycode'] || Sisimai::SMTP::Reply.find(e['diagnosis'])
           s1 = 0  # First character of Status as integer
           r1 = 0  # First character of SMTP reply code as integer
 

data/lib/sisimai/mime.rb

@@ -225,7 +225,7 @@ module Sisimai
 
         plain = nil
         if cv = argv1.match(%r|([+/\=0-9A-Za-z\r\n]+)|) then plain = Base64.decode64(cv[1]) end
-        return plain.force_encoding('UTF-8')
+        return plain.scrub('?')
       end
 
       # Get boundary string

data/lib/sisimai/reason/filtered.rb

@@ -41,7 +41,7 @@ module Sisimai
           return false
         end
 
-        # Rejected by domain or address filter ?
+        # Rejected by a sender domain or a sender address by a filter ?
         # @param    [Sisimai::Data] argvs   Object to be detected the reason
         # @return   [True,False]            true: is filtered
         #                                   false: is not filtered
@@ -53,18 +53,18 @@ module Sisimai
           tempreason = Sisimai::SMTP::Status.name(argvs.deliverystatus) || ''
           return false if tempreason == 'suspend'
 
-          commandtxt = argvs.smtpcommand || ''
           diagnostic = argvs.diagnosticcode.downcase || ''
-          alterclass = Sisimai::Reason::UserUnknown
-
           if tempreason == 'filtered'
             # Delivery status code points "filtered".
-            return true if alterclass.match(diagnostic) || match(diagnostic)
-
-          elsif commandtxt != 'RCPT' && commandtxt != 'MAIL'
-            # Check the value of Diagnostic-Code and the last SMTP command
+            return true if Sisimai::Reason::UserUnknown.match(diagnostic)
             return true if match(diagnostic)
-            return true if alterclass.match(diagnostic)
+          else
+            # The value of "reason" isn't "filtered" when the value of "smtpcommand" is an SMTP
+            # command to be sent before the SMTP DATA command because all the MTAs read the headers
+            # and the entire message body after the DATA command.
+            return false if %w[CONN EHLO HELO MAIL RCPT].include?(argvs.smtpcommand)
+            return true  if match(diagnostic)
+            return true  if Sisimai::Reason::UserUnknown.match(diagnostic)
           end
           return false
         end

data/lib/sisimai/reason/norelaying.rb

@@ -53,13 +53,9 @@ module Sisimai
         # @see http://www.ietf.org/rfc/rfc2822.txt
         def true(argvs)
           r = argvs.reason || ''
-          if r.size > 0
-            # Do not overwrite the reason
-            return false if r.start_with?('securityerror', 'systemerror', 'undefined')
-          else
-            # Check the value of Diagnosic-Code: header with patterns
-            return true if match(argvs.diagnosticcode.downcase)
-          end
+          return false if r.start_with?('securityerror', 'systemerror', 'undefined')
+          return false if %w[CONN EHLO HELO].include?(argvs.smtpcommand)
+          return true  if match(argvs.diagnosticcode.downcase)
           return false
         end
 

data/lib/sisimai/reason/notaccept.rb

@@ -43,9 +43,9 @@ module Sisimai
           return true if argvs.reason == 'notaccept'
 
           # SMTP Reply Code is 554 or 556
-          return true if [521, 554, 556].index(argvs.replycode.to_i)
-          return false if argvs.smtpcommand == 'MAIL'
-          return true if match(argvs.diagnosticcode.downcase)
+          return true  if [521, 554, 556].index(argvs.replycode.to_i)
+          return false if argvs.smtpcommand != 'MAIL'
+          return true  if match(argvs.diagnosticcode.downcase)
           return false
         end
 

data/lib/sisimai/reason/policyviolation.rb

@@ -56,7 +56,7 @@ module Sisimai
           return false
         end
 
-        # The bounce reason is security error or not
+        # The bounce reason is "policyviolation" or not
         # @param    [Sisimai::Data] argvs   Object to be detected the reason
         # @return   [True,False]            true: is policy violation
         #                                   false: is not policy violation

data/lib/sisimai/reason/spamdetected.rb

@@ -165,7 +165,12 @@ module Sisimai
           return nil  if argvs.deliverystatus.empty?
           return true if argvs.reason == 'spamdetected'
           return true if Sisimai::SMTP::Status.name(argvs.deliverystatus).to_s == 'spamdetected'
-          return true if match(argvs.diagnosticcode.downcase)
+
+          # The value of "reason" isn't "spamdetected" when the value of "smtpcommand" is an SMTP
+          # command to be sent before the SMTP DATA command because all the MTAs read the headers
+          # and the entire message body after the DATA command.
+          return false if %w[CONN EHLO HELO MAIL RCPT].include?(argvs.smtpcommand)
+          return true  if match(argvs.diagnosticcode.downcase)
           return false
         end
 

data/lib/sisimai/reason/systemerror.rb

@@ -26,6 +26,7 @@ module Sisimai
           'loop was found in the mail exchanger',
           'loops back to myself',
           'mail system configuration error',
+          'queue file write error',
           'remote server is misconfigured',
           'server configuration error',
           'service currently unavailable',

data/lib/sisimai/reason/virusdetected.rb

@@ -38,14 +38,20 @@ module Sisimai
           return false
         end
 
-        # The bounce reason is security error or not
+        # The bounce reason is "virusdetected" or not
         # @param    [Sisimai::Data] argvs   Object to be detected the reason
         # @return   [True,False]            true: virus detected
         #                                   false: virus was not detected
         # @since 4.22.0
         # @see http://www.ietf.org/rfc/rfc2822.txt
-        def true(_argvs)
-          return nil
+        def true(argvs)
+          # The value of "reason" isn't "visusdetected" when the value of "smtpcommand" is an SMTP
+          # command to be sent before the SMTP DATA command because all the MTAs read the headers
+          # and the entire message body after the DATA command.
+          return true  if argvs.reason == 'virusdetected'
+          return false if %w[CONN EHLO HELO MAIL RCPT].include?(argvs.smtpcommand)
+          return true  if match(argvs.diagnosticcode.downcase)
+          return false
         end
 
       end

data/lib/sisimai/rhost/franceptt.rb

@@ -123,6 +123,10 @@ module Sisimai
           # Message refuse, verification DMARC en echec LPN007_517
           '517' => 'policyviolation',
 
+          # 554 5.7.1 Client host rejected
+          # LPN000_630
+          '630' => 'policyviolation',
+
           # 421 mwinf5c77 ME Service refuse. Veuillez essayer plus tard.
           # Service refused, please try later. OFR_999 [999]
           '999' => 'blocked',

data/lib/sisimai/version.rb

@@ -1,4 +1,4 @@
 # Define the version number of Sisimai
 module Sisimai
-  VERSION = '4.25.11'.freeze
+  VERSION = '4.25.14'.freeze
 end

data/set-of-emails/maildir/bsd/rfc3464-41.eml

@@ -0,0 +1,19 @@
+From MAILER-DAEMON  Mon Sep 20 19:29:27 2021
+Return-Path: <>
+X-Original-To: postmaster@peter.otto.org
+Delivered-To: blackhole@localhost
+From: <foo@bar.net>
+To: postmaster@any.com
+Message-ID: <dummy@ott.pet>
+Date: Mon, 20 Sep 2021 21:29:21 +0200 (CEST)
+X-RZG-CLASS-ID: mi
+X-RZG-TRIGGERING-SMTP-ID: qcZaAFivUJ
+MIME-Version: 1.0
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: quoted-printable
+Subject: kitununulinitoquekegu
+In-Reply-To: <foobar@dummy.org>
+Auto-Submitted: auto-replied
+
+zuk=C3=BCnftig
+karl@hanz.de

data/set-of-emails/maildir/bsd/rfc3464-42.eml

@@ -0,0 +1,34 @@
+From MAILER-DAEMON  Mon Sep 20 19:33:02 2021
+Return-Path: <>
+X-Original-To: postmaster@blackhole.our-host.net
+Delivered-To: blackhole@localhost
+Received: from mx2.itnetwork.net (mx2.itnetwork.net [123.123.123.202])
+	by play.thmthlayer.com (Postfix) with ESMTPS id 94598E117B
+	for <postmaster@blackhole.our-host.net>; Mon, 20 Sep 2021 19:33:02 +0000 (UTC)
+Received: from 10.9.37.234 ([10.9.34.13]) by mx2.itnetwork.net with ESMTP id rTR3cr9xLoMiR5vX for <postmaster@blackhole.our-host.net>; Mon, 20 Sep 2021 21:33:01 +0200 (CEST)
+Date: Mon, 20 Sep 2021 21:32:59 +0200 (GMT+02:00)
+From: Postmaster@bit-onbreeeck.org
+To: MyName <message@gelaneeiuet.org>
+Subject: foobar
+Mime-Version: 1.0
+Message-ID: <OFB3228DED.DDDCAC8E-ONC1258756.006B648D-C1258756.006B6493@bit-onbreeeck.org>
+Content-Type: multipart/report; report-type=delivery-status; boundary="==IFJRGLKFGIR7891042UHRUHIHD"
+
+--==IFJRGLKFGIR7891042UHRUHIHD
+Content-Type: text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: base64
+
+ICBhdWZnZWb8aHJ0DQoNCg==
+
+--==IFJRGLKFGIR7891042UHRUHIHD
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns;10.9.37.234
+
+Final-Recipient: rfc822;jane.doe@some-domain.net
+Action: failed
+Status: 5.0.0
+Diagnostic-Code: The email account that you tried to reach does not exist.
+
+--==IFJRGLKFGIR7891042UHRUHIHD--
+

data/set-of-emails/maildir/bsd/rhost-franceptt-12.eml

@@ -0,0 +1,82 @@
+Return-Path: <>
+X-Original-To: xxxx@xxxx.net
+Delivered-To: xxxx@xxxx.xxxx.net
+Received: from xxxx.xxxx.net (xxxx.xxxx.net [#.#.#.#])
+	by xxxx.xxxx.net (Postfix) with ESMTP id D827520D624B
+	for <xxxx@xxxx.net>; Fri, 15 Oct 2021 14:42:03 +0200 (CEST)
+Received: by xxxx.xxxx.net (Postfix)
+	id C6602268DDF8; Fri, 15 Oct 2021 14:42:00 +0200 (CEST)
+Date: Fri, 15 Oct 2021 14:42:00 +0200 (CEST)
+From: MAILER-DAEMON@xxxx.xxxx.net (Mail Delivery System)
+Subject: Undelivered Mail Returned to Sender
+To: xxxx.xxxx.net
+Auto-Submitted: auto-replied
+MIME-Version: 1.0
+Content-Type: multipart/report; report-type=delivery-status;
+	boundary="64BD1268DDF4.xxxx.xxxx.net"
+Content-Transfer-Encoding: 8bit
+Message-Id: <20211015124200.xxxx.xxxx.net>
+
+This is a MIME-encapsulated message.
+
+--64BD1268DDF4.xxxx.xxxx.net
+Content-Description: Notification
+Content-Type: text/plain; charset=utf-8
+Content-Transfer-Encoding: 8bit
+
+This is the mail system at host xxxx.xxxx.net.
+
+I'm sorry to have to inform you that your message could not
+be delivered to one or more recipients. It's attached below.
+
+For further assistance, please send mail to postmaster.
+
+If you do so, please include this problem report. You can
+delete your own text from the attached returned message.
+
+                   The mail system
+
+<xxxx@laposte.net>: host smtpz4.laposte.net said: 554
+    5.7.1 <xxxx.xxxx.net>: Client host rejected:
+    #216 lpn LPN000_630 (in reply to RCPT TO command)
+
+--64BD1268DDF4.xxxx.xxxx.net
+Content-Description: Delivery report
+Content-Type: message/delivery-status
+
+Reporting-MTA: dns; xxxx.xxxx.net
+X-Postfix-Queue-ID: 64BD1268DDF4
+X-Postfix-Sender: rfc822; xxxx.xxxx.net
+Arrival-Date: Fri, 15 Oct 2021 14:41:59 +0200 (CEST)
+
+Final-Recipient: rfc822; xxxx@laposte.net
+Original-Recipient: rfc822;xxxx@laposte.net
+Action: failed
+Status: 5.7.1
+Remote-MTA: dns; smtpz4.laposte.net
+Diagnostic-Code: smtp; 554 5.7.1 <xxxx.xxxx.net[#.#.#.#]:49309>:
+    Client host rejected: #216 lpn LPN000_630
+
+--64BD1268DDF4.xxxx.xxxx.net
+Content-Description: Undelivered Message Headers
+Content-Type: text/rfc822-headers
+Content-Transfer-Encoding: 8bit
+
+Return-Path: <xxxx@xxxx.net>
+Received: from [#.#.#.#] (unknown [#.#.#.#])
+	by xxxx.xxxx.net (Postfix) with ESMTPS id 64BD1268DDF4
+	for <xxxx@laposte.net>; Fri, 15 Oct 2021 14:41:59 +0200 (CEST)
+Message-Id: <xxxx@xxxx.net>
+Feedback-Id: 657:1802
+List-Unsubscribe: <mailto:xxxx@lu.xxxx.net>
+Precedence: bulk
+From: "=?UTF-8?Q?Kipli_par_AM?=" <newsletter@xxxx.net>
+To: xxxx@r-v-b.com
+Reply-To: newsletter@xxxx.net
+Content-Type: multipart/alternative; boundary="_----------=_1634301717326132"
+Subject: =?UTF-8?Q?xxxx?=
+Content-Transfer-Encoding: binary
+MIME-Version: 1.0
+Date: Fri, 15 Oct 2021 14:41:57 +0200
+
+--64BD1268DDF4.xxxx.xxxx.net--

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sisimai
 version: !ruby/object:Gem::Version
-  version: 4.25.11
+  version: 4.25.14
 platform: ruby
 authors:
 - azumakuniyuki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-22 00:00:00.000000000 Z
+date: 2022-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -693,6 +693,8 @@ files:
 - set-of-emails/maildir/bsd/rfc3464-38.eml
 - set-of-emails/maildir/bsd/rfc3464-39.eml
 - set-of-emails/maildir/bsd/rfc3464-40.eml
+- set-of-emails/maildir/bsd/rfc3464-41.eml
+- set-of-emails/maildir/bsd/rfc3464-42.eml
 - set-of-emails/maildir/bsd/rfc3834-01.eml
 - set-of-emails/maildir/bsd/rfc3834-02.eml
 - set-of-emails/maildir/bsd/rfc3834-03.eml
@@ -711,6 +713,7 @@ files:
 - set-of-emails/maildir/bsd/rhost-franceptt-08.eml
 - set-of-emails/maildir/bsd/rhost-franceptt-10.eml
 - set-of-emails/maildir/bsd/rhost-franceptt-11.eml
+- set-of-emails/maildir/bsd/rhost-franceptt-12.eml
 - set-of-emails/maildir/bsd/rhost-godaddy-02.eml
 - set-of-emails/maildir/bsd/rhost-godaddy-03.eml
 - set-of-emails/maildir/bsd/rhost-googleapps-01.eml