sisimai 4.25.5 → 4.25.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf50fe472bdf358908619b5571800aa29c181ed59c11bcc52718c504f54ec991
-  data.tar.gz: d764cfda21b0b8c49e53f50b5ddd7527665e8c9721a5c4a1bebb8e9dc9eac2a6
+  metadata.gz: 9f6b7ab8483a8b45624cb35c088bea1e4ea2579781574bef1fd059036b5031e0
+  data.tar.gz: 37345455c029a5bd5809b03e07204825fecca0f62fe16a801129732fb160bbfe
 SHA512:
-  metadata.gz: ce283bb17ad9beb4a82186e653a67c92e6357652db7148c30ad69ae5ab9d6734a49a5c88b0ac2c6302cb13274350644ca4651fafa3aa5eaf96edc0a49c73a849
-  data.tar.gz: 9cbd0b351b7166af32183255e492cd4490017a037443e867bc63d6ca13939d7d5cc123b903464c711628bb4497a14d86de027e547abe55aed788e78286b85367
+  metadata.gz: b653224776d525a65748406246383613f85a1a3439bbd6d89376c6256d442f122d3fbbcec445ac9938f50cee93cfd3e14d66b894640df6ce3153de424d8da6d2
+  data.tar.gz: 6b8408d5b55b19baf00b82e2ec731a5445b5e324c90cbe2c71fb61079e6a407cc5184419016171ff3582cf2141bac7717669217af62928e928c6db32ae8e1d46

data/ANALYTICAL-PRECISION

@@ -1,60 +1,64 @@
  bounceHammer 2.7.13p3
  MTA MODULE NAME          CAN PARSE   RATIO   NOTES
 --------------------------------------------------------------------------------
-Email::Activehunter .....    0/0014  0.0000  TransWARE Active!hunter
-Email::Amavis ...........    4/0004  1.0000  amavisd-new: https://www.amavis.org/
-Email::AmazonSES ........   40/0049  0.8163  Amazon SES(Sending): http://aws.amazon.com/ses/
-Email::AmazonWorkMail ...    9/0013  0.6923  Amazon WorkMail: https://aws.amazon.com/workmail/
-Email::Aol ..............   20/0023  0.8696  Aol Mail: http://www.aol.com
-Email::ApacheJames ......    0/0006  0.0000  Java Apache Mail Enterprise Server
-Email::Bigfoot ..........    3/0003  1.0000  Bigfoot: http://www.bigfoot.com
-Email::Biglobe ..........    5/0007  0.7143  BIGLOBE: http://www.biglobe.ne.jp
-Email::Courier ..........   14/0016  0.8750  Courier MTA
-Email::Domino ...........   10/0019  0.5263  IBM Domino Server
-Email::EZweb ............  133/0128  1.0000  au EZweb: http://www.au.kddi.com/mobile/
-Email::EinsUndEins ......    0/0004  0.0000  1&1: http://www.1and1.de
-Email::Exchange2003 .....    1/0051  0.0196  Microsoft Exchange Server 2003
-Email::Exchange2007 .....   10/0012  0.8333  Microsoft Exchange Server 2007
-Email::Exim .............  234/0259  0.9035  Exim
-Email::FML ..............    0/0004  0.0000  fml mailing list server/manager
-Email::Facebook .........    5/0005  1.0000  Facebook: https://www.facebook.com
-Email::GMX ..............    0/0010  0.0000  GMX: http://www.gmx.net
-Email::GSuite ...........   19/0019  1.0000  G Suite: https://gsuite.google.com/
-Email::Google ...........   61/0063  0.9683  Google Gmail: https://mail.google.com
-Email::IMailServer ......    0/0045  0.0000  IPSWITCH IMail Server
-Email::InterScanMSS .....    0/0008  0.0000  Trend Micro InterScan Messaging Security Suite
-Email::KDDI .............    6/0006  1.0000  au by KDDI: http://www.au.kddi.com
-Email::MXLogic ..........    0/0014  0.0000  McAfee SaaS
-Email::MailFoundry ......    0/0007  0.0000  MailFoundry
-Email::MailMarshalSMTP ..    0/0004  0.0000  Trustwave Secure Email Gateway
-Email::MailRu ...........    4/0023  0.1739  @mail.ru: https://mail.ru
-Email::McAfee ...........    0/0014  0.0000  McAfee Email Appliance
-Email::MessageLabs ......    7/0007  1.0000  Symantec.cloud http://www.messagelabs.com
-Email::MessagingServer ..   30/0033  0.9091  Oracle Communications Messaging Server
-Email::Notes ............    0/0013  0.0000  Lotus Notes
-Email::Office365 ........   26/0029  0.8966  Microsoft Office 365: http://office.microsoft.com/
-Email::OpenSMTPD ........    6/0024  0.2500  OpenSMTPD
-Email::Outlook ..........   21/0023  0.9130  Microsoft Outlook.com: https://www.outlook.com/
-Email::Postfix ..........  260/0277  0.9386  Postfix
-Email::ReceivingSES .....    0/0007  0.0000  Amazon SES(Receiving): http://aws.amazon.com/ses/
-Email::SendGrid .........    7/0012  0.5833  SendGrid: http://sendgrid.com/
-Email::Sendmail .........  354/0278  1.0000  V8Sendmail: /usr/sbin/sendmail
-Email::SurfControl ......    6/0008  0.7500  WebSense SurfControl
-Email::V5sendmail .......    0/0386  0.0000  Sendmail version 5
-Email::Verizon ..........    4/0004  1.0000  Verizon Wireless: http://www.verizonwireless.com
-Email::X1 ...............    0/0008  0.0000  Unknown MTA #1
-Email::X2 ...............    0/0070  0.0000  Unknown MTA #2
-Email::X3 ...............    0/0012  0.0000  Unknown MTA #3
-Email::X4 ...............    0/0029  0.0000  Unknown MTA #4 qmail clones
-Email::X5 ...............    3/0003  1.0000  Unknown MTA #5
-Email::Yahoo ............    0/0025  0.0000  Yahoo! MAIL: https://www.yahoo.com
-Email::Yandex ...........    4/0007  0.5714  Yandex.Mail: http://www.yandex.ru
-Email::Zoho .............    0/0011  0.0000  Zoho Mail: https://www.zoho.com
-Email::mFILTER ..........    0/0012  0.0000  Digital Arts m-FILTER
-Email::qmail ............   58/0087  0.6667  qmail
-ARF .....................    8/0038  0.2105  Abuse Feedback Reporting Format
-RFC3464 .................  151/0322  0.4689  Fallback Module for MTAs
-RFC3834 .................    0/0014  0.0000  Detector for auto replied message
+Activehunter .....    0/0014  0.0000  TransWARE Active!hunter
+Amavis ...........    4/0004  1.0000  amavisd-new: https://www.amavis.org/
+AmazonSES ........   40/0049  0.8163  Amazon SES(Sending): http://aws.amazon.com/ses/
+AmazonWorkMail ...    9/0013  0.6923  Amazon WorkMail: https://aws.amazon.com/workmail/
+Aol ..............   20/0023  0.8696  Aol Mail: http://www.aol.com
+ApacheJames ......    0/0006  0.0000  Java Apache Mail Enterprise Server
+Barracuda ........    0/0004  0.0000  Barracuda: https://www.barracuda.com
+Bigfoot ..........    3/0003  1.0000  Bigfoot: http://www.bigfoot.com
+Biglobe ..........    5/0007  0.7143  BIGLOBE: http://www.biglobe.ne.jp
+Courier ..........   14/0016  0.8750  Courier MTA
+Domino ...........   10/0019  0.5263  IBM Domino Server
+EZweb ............  133/0128  1.0000  au EZweb: http://www.au.kddi.com/mobile/
+EinsUndEins ......    0/0004  0.0000  1&1: http://www.1and1.de
+Exchange2003 .....    1/0051  0.0196  Microsoft Exchange Server 2003
+Exchange2007 .....   10/0012  0.8333  Microsoft Exchange Server 2007
+Exim .............  234/0259  0.9035  Exim
+FML ..............    0/0004  0.0000  fml mailing list server/manager
+Facebook .........    5/0005  1.0000  Facebook: https://www.facebook.com
+GMX ..............    0/0010  0.0000  GMX: http://www.gmx.net
+GSuite ...........   19/0019  1.0000  G Suite: https://gsuite.google.com/
+Gmail ............   61/0063  0.9683  Gmail: https://mail.google.com
+GoogleGroups .....    0/0028  0.0000  Google Groups: https://groups.google.com
+IMailServer ......    0/0045  0.0000  IPSWITCH IMail Server
+InterScanMSS .....    0/0008  0.0000  Trend Micro InterScan Messaging Security Suite
+KDDI .............    6/0006  1.0000  au by KDDI: http://www.au.kddi.com
+MXLogic ..........    0/0014  0.0000  McAfee SaaS
+MailFoundry ......    0/0007  0.0000  MailFoundry
+MailMarshalSMTP ..    0/0004  0.0000  Trustwave Secure Email Gateway
+MailRu ...........    4/0023  0.1739  @mail.ru: https://mail.ru
+McAfee ...........    0/0014  0.0000  McAfee Email Appliance
+MessageLabs ......    7/0007  1.0000  Symantec.cloud http://www.messagelabs.com
+MessagingServer ..   30/0033  0.9091  Oracle Communications Messaging Server
+Notes ............    0/0013  0.0000  Lotus Notes
+Office365 ........   26/0029  0.8966  Microsoft Office 365: http://office.microsoft.com/
+OpenSMTPD ........    6/0024  0.2500  OpenSMTPD
+Outlook ..........   21/0023  0.9130  Microsoft Outlook.com: https://www.outlook.com/
+Postfix ..........  260/0277  0.9386  Postfix
+PowerMTA .........    0/0005  0.0000  PowerMTA: https://www.sparkpost.com/powermta/
+ReceivingSES .....    0/0007  0.0000  Amazon SES(Receiving): http://aws.amazon.com/ses/
+SendGrid .........    7/0012  0.5833  SendGrid: http://sendgrid.com/
+Sendmail .........  354/0278  1.0000  V8Sendmail: /usr/sbin/sendmail
+SurfControl ......    6/0008  0.7500  WebSense SurfControl
+V5sendmail .......    0/0386  0.0000  Sendmail version 5
+Verizon ..........    4/0004  1.0000  Verizon Wireless: http://www.verizonwireless.com
+X1 ...............    0/0008  0.0000  Unknown MTA #1
+X2 ...............    0/0070  0.0000  Unknown MTA #2
+X3 ...............    0/0012  0.0000  Unknown MTA #3
+X4 ...............    0/0029  0.0000  Unknown MTA #4 qmail clones
+X5 ...............    3/0003  1.0000  Unknown MTA #5
+X6 ...............    0/0029  0.0000  Unknown MTA #6
+Yahoo ............    0/0025  0.0000  Yahoo! MAIL: https://www.yahoo.com
+Yandex ...........    4/0007  0.5714  Yandex.Mail: http://www.yandex.ru
+Zoho .............    0/0011  0.0000  Zoho Mail: https://www.zoho.com
+mFILTER ..........    0/0012  0.0000  Digital Arts m-FILTER
+qmail ............   58/0087  0.6667  qmail
+ARF ..............    8/0038  0.2105  Abuse Feedback Reporting Format
+RFC3464 ..........  151/0322  0.4689  Fallback Module for MTAs
+RFC3834 ..........    0/0014  0.0000  Detector for auto replied message
 --------------------------------------------------------------------------------
- bounceHammer 2.7.13p3    1523/2569  0.5928
- Sisimai 4.25.3p2         2569/2569  1.0000
+ bounceHammer      1523/2635  0.5779
+ Sisimai 4.25.6    2635/2635  1.0000

data/Benchmarks.mk

@@ -22,6 +22,8 @@ SPEEDTESTDIR := tmp/emails-for-speed-test
 
 COMMANDARGVS := -I./lib -rsisimai
 TOBEEXECUTED := 'Sisimai.make($$*.shift, delivered: true)' $(PUBLICMAILS)
+PROFILEARGVS := -rrblineprof -rrblineprof-report
+PROFCOMMANDS := 'p = lineprof(%r|./lib/sisimai|) { Sisimai.make($$*.shift, delivered: true) }; LineProf.report(p, out: "pf")'
 HOWMANYMAILS := $(RUBY) $(COMMANDARGVS) -le 'puts Sisimai.make($$*.shift, delivered: true).size' 
 
 # -----------------------------------------------------------------------------
@@ -46,8 +48,8 @@ speed-test: emails-for-speed-test
 
 profile:
 	@ uptime
-	$(RUBY) -rprofile $(COMMANDARGVS) -e $(TOBEEXECUTED) $(SPEEDTESTDIR) 2> pf > /dev/null
-	cat ./pf | sed -e 's/^ *//g' | tr -s ' ' | sed -e 's/self self/self  self/' | tr ' ' '\t' > profiling-`date '+%Y-%m-%d-%T'`.txt
+	$(RUBY) $(COMMANDARGVS) $(PROFILEARGVS) -e $(PROFCOMMANDS) $(SPEEDTESTDIR)
+	cat ./pf > profile-`date '+%Y-%m-%d-%T'`.txt
 	$(RM) ./pf
 
 loc:

data/ChangeLog.md

@@ -3,6 +3,84 @@ RELEASE NOTES for Ruby version of Sisimai
 - releases: "https://github.com/sisimai/rb-Sisimai/releases"
 - download: "https://rubygems.org/gems/sisimai"
 
+v4.25.6
+--------------------------------------------------------------------------------
+- release: "Wed, 22 Apr 2020 16:22:22 +0900 (JST)"
+- version: "4.25.6"
+- changes:
+  - Performance improvement: 10% faster, reduced 9% of method calls
+  - #176 Make `Sisimai::Message` 27% faster
+    - Use the negative look-ahead regular expression code to convert all of the
+      email header strings into key-value pairs as a HASH at newly implemented
+      method `Sisiai::Message.makemap` #175. Thanks to @xtetsuji
+    - Remove `Sisimai::Message.takeapart` (replaced with `makemap`)
+    - Remove `Sisimai::Message.headers` (replaced with `makemap`)
+    - Code improvement for `require` statement before method calls #177
+  - Make `Sisimai::Order` 44% faster
+    - Rewrite `Sisimai::Order.make`
+    - Remove `Sisimai::Order.by`
+    - Remove `Sisimai::Order.headers`
+    - Remove `Sisimai::Lhost.headerlist`
+    - And all `headerlist` method have been removed from `Sisimai::Lhost::*`,
+      `Sisimai::RFC3834` and `Sisimai::ARF`
+    - The MTA module to be loaded at first is decided by the first 2 words of
+      each bounce mail subject, is defined at `Subject` in `Sisimai::Order`
+    - Some variables are replaced with `state`
+  - Each `field` parameter has been removed from the following methods because 
+    Sisimai detect all the email header fields by `Sisimai::Message.makemap()`
+    without having to specify field names at `field` parameter
+    - `Sisimai.make`
+    - `Sisimai::Message.new`
+    - `Sisimai::Message.make`
+  - Code improvement for `require` statement before calling `match()` method of
+    some modules defined in `$PreMatches` at `Sisimai::Reason::UserUnknown`
+  - Remove the following unused methods:
+    - `Sisimai::MIME.patterns`
+    - `Sisimai::SMTP.command`
+  - `Sisimai::Lhost::Google` has been renamed to `Sisimai::Lhost::Gmail`
+  - Implement 3 MTA modules: #178 #181
+    - `Sisimai::Lhost::Barracuda`
+    - `Sisimai::Lhost::PowerMTA`
+    - `Sisimai::Lhost::X6`
+    - `Sisimai::Lhost::GoogleGroups`
+  - "email-" prefix of each sample email in set-of-emails/maildir directory has
+    been replaced with "lhost-" sisimai/set-of-emails#14
+  - SMTP Agent improvement #158
+    - Remove `Email::` prefix from the value of `smtpagent` at parsed results
+    - Remove `Sisimai::Lhost->smtpagent` method
+  - Improved the following MTA modules:
+    - `Sisimai::Lhost::Amavis` #183
+    - `Sisimai::Lhost::InterScanMSS`
+    - `Sisimai::Lhost::Office365` improvement for reading MIME-encoded subject
+    - `Sisimai::Lhost::Exchange2007` supports error messages in `it-CH`
+  - Tiny bug fix for `Subject` header decoding
+  - Fix bug in code for getting an `"addresser"` address from `From:` field in
+    the original message part which are multiple lines at `Sisimai::ARF`. #185
+  - New accessor `origin` at `Sisimai::Data` and the parsed results for keeping
+    a path to the source email #186
+  - #187 `Sisimai::Mail` improvement for compatibilities with the Go language
+    version of Sisimai which will be released this summer
+    - Removed `Sisimai::Mail::STDIN.name` (not used)
+    - Removed `Sisimai::Mail::Maildir.inodes` (not needed to check the inode)
+    - Removed `Sisimai::Mail::Maildir.count` (use `offset` instead)
+    - Warning message is displayed when the following methods are called:
+      - `Sisimai::Mail.close` (automatically closes at the EOF)
+      - `Sisimai::Mail.type` (use `Sisimai::Mail.kind` instead)
+      - `Sisimai::Mail.mail.*` (use `Sisimai::Mail.data.*` instead)
+      - Methods above will be removed at v4.25.10
+    - `Sisimai::Mail::Memory.data` renamed to `Sisimai::Mail::Memory.payload`
+    - `Sisimai::Mail::Maildir.size` keeps the number of files in the Maildir/
+    - `Sisimai::Mail::Maildir.offset` keeps the number of email files in the
+      Maildir/ which have been read
+    - Call `Sisimai::Mail::*.read` directly instead of `Sisimai::Mail.read`
+    - Remove `Sisimai::Lhost::UserDefined` (not used)
+  - Add the following D.S.N. codes and error messages (not tested)
+    - `Mailbox does not exist!` at `Sisimai::Reason::UserUnknown` (Amazon SES)
+    - `Not a valid recipienet` at `Sisimai::Reason::UserUnknown` (Yahoo!)
+    - `Envelope blocked` at `Sisimai::Reason::Rejected` (Minecast.com)
+    - `5.2.122` is toomanyconn, `5.4.11` is contenterror, `5.7.51` is blocked
+      at `Sisimai::Rhost::ExchangeOnline`
+
 v4.25.5
 --------------------------------------------------------------------------------
 - release: "Wed, 22 Jan 2020 14:44:44 +0900 (JST)"

data/README-JA.md

@@ -53,8 +53,7 @@ Key features
   * git clone & make
 * __高い解析精度__
   * 解析精度はbounceHammerの2倍
-  * 29種類のMTAに対応
-  * 22種類の著名なMSPに対応
+  * 66種類のMTA/MDA/ESPに対応
   * Feedback Loopにも対応
   * 29種類のエラー理由を検出
 
@@ -107,7 +106,8 @@ Usage
 Basic usage
 -------------------------------------------------------------------------------
 下記のようにSisimaiの`make()`メソッドをmboxかMaildirのPATHを引数にして実行すると
-解析結果が配列で返ってきます。
+解析結果が配列で返ってきます。v4.25.6から元データとなった電子メールファイルへの
+PATHを保持する`origin`が利用できます。
 
 ```ruby
 #! /usr/bin/env ruby
@@ -135,6 +135,7 @@ if v.is_a? Array
     puts e.deliverystatus       # 5.1.1
     puts e.replycode            # 550
     puts e.reason               # userunknown
+    puts e.origin               # /var/spool/bounce/Maildir/new/1740074341.eml
 
     h = e.damn                  # Convert to HASH
     j = e.dump('json')          # Convert to JSON string
@@ -176,9 +177,8 @@ callbackto = lambda do |v|
   return r
 end
 
-list = ['X-Mailer']
-data = Sisimai.make('/path/to/mbox', hook: callbackto, field: list)
-json = Sisimai.dump('/path/to/mbox', hook: callbackto, field: list)
+data = Sisimai.make('/path/to/mbox', hook: callbackto)
+json = Sisimai.dump('/path/to/mbox', hook: callbackto)
 
 puts data[0].catch['x-mailer']      # Apple Mail (2.1283)
 ```
@@ -200,7 +200,7 @@ Output example
 ![](https://libsisimai.org/static/images/demo/sisimai-dump-02.gif)
 
 ```json
-[{"recipient": "kijitora@example.jp", "addresser": "shironeko@1jo.example.org", "feedbacktype": "", "action": "failed", "subject": "Nyaaaaan", "smtpcommand": "DATA", "diagnosticcode": "550 Unknown user kijitora@example.jp", "listid": "", "destination": "example.jp", "smtpagent": "Email::Courier", "lhost": "1jo.example.org", "deliverystatus": "5.0.0", "timestamp": 1291954879, "messageid": "201012100421.oBA4LJFU042012@1jo.example.org", "diagnostictype": "SMTP", "timezoneoffset": "+0900", "reason": "filtered", "token": "ce999a4c869e3f5e4d8a77b2e310b23960fb32ab", "alias": "", "senderdomain": "1jo.example.org", "rhost": "mfsmax.example.jp"}, {"diagnostictype": "SMTP", "timezoneoffset": "+0900", "reason": "userunknown", "timestamp": 1381900535, "messageid": "E1C50F1B-1C83-4820-BC36-AC6FBFBE8568@example.org", "token": "9fe754876e9133aae5d20f0fd8dd7f05b4e9d9f0", "alias": "", "senderdomain": "example.org", "rhost": "mx.bouncehammer.jp", "action": "failed", "addresser": "kijitora@example.org", "recipient": "userunknown@bouncehammer.jp", "feedbacktype": "", "smtpcommand": "DATA", "subject": "バウンスメールのテスト(日本語)", "destination": "bouncehammer.jp", "listid": "", "diagnosticcode": "550 5.1.1 <userunknown@bouncehammer.jp>... User Unknown", "deliverystatus": "5.1.1", "lhost": "p0000-ipbfpfx00kyoto.kyoto.example.co.jp", "smtpagent": "Email::Sendmail"}]
+[{"catch":{"x-mailer":"","return-path":"neko@example.com"},"token":"7e81d3b9306fc7a7f3fb4c7b705189d6806d3d6b","lhost":"omls-1.kuins.neko.example.jp","rhost":"nekonyaan0022.apcprd01.prod.exchangelabs.com","alias":"","listid":"","reason":"userunknown","action":"failed","origin":"set-of-emails/maildir/bsd/lhost-office365-13.eml","subject":"にゃーん","messageid":"","replycode":"550","smtpagent":"Office365","softbounce":0,"smtpcommand":"","destination":"neko.kyoto.example.jp","senderdomain":"example.com","feedbacktype":"","diagnosticcode":"Error Details Reported error: 550 5.1.10 RESOLVER.ADR.RecipientNotFound; Recipient not found by SMTP address lookup DSN generated by: NEKONYAAN0022.apcprd01.prod.exchangelabs.com","diagnostictype":"","deliverystatus":"5.1.10","timezoneoffset":"+0000","addresser":"neko@example.com","recipient":"kijitora-nyaan@neko.kyoto.example.jp","timestamp":1493508885}]
 ```
 
 Sisimai Specification
@@ -220,8 +220,8 @@ Differences between Ruby version and Perl version
 | メール解析速度(1000通のメール)              | 2.22秒[2]      | 1.35秒        |
 | インストール方法                            | gem install    | cpanm, cpm    |
 | 依存モジュール数(コアモジュールを除く)      | 1モジュール    | 2モジュール   |
-| LOC:ソースコードの行数                      | 10300行        | 7100行        |
-| テスト件数(spec/,t/,xt/ディレクトリ)        | 230000件       | 250000件      |
+| LOC:ソースコードの行数                      | 10200行        | 10400行       |
+| テスト件数(spec/,t/,xt/ディレクトリ)        | 236000件       | 266000件      |
 | ライセンス                                  | 二条項BSD      | 二条項BSD     |
 | 開発会社によるサポート契約                  | 提供中         | 提供中        |
 

data/README.md

@@ -52,8 +52,7 @@ Key Features
   * git clone & make
 * __High Precision of Analysis__
   * 2 times higher than bounceHammer
-  * Support 24 known MTAs and 5 unknown MTAs
-  * Support 22 major MSPs(Mail Service Providers)
+  * Support 66 MTAs/MDAs/ESPs
   * Support Feedback Loop Message(ARF)
   * Can detect 29 error reasons
 
@@ -112,7 +111,9 @@ Usage
 Basic usage
 -------------------------------------------------------------------------------
 `make()` method provides feature for getting parsed data from bounced email 
-messages like following.
+messages like following. Beginning with v4.25.6, new accessor `origin` which
+keeps the path to email file as a data source is available.
+
 
 ```ruby
 #! /usr/bin/env ruby
@@ -140,6 +141,7 @@ if v.is_a? Array
     puts e.deliverystatus       # 5.1.1
     puts e.replycode            # 550
     puts e.reason               # userunknown
+    puts e.origin               # /var/spool/bounce/Maildir/new/1740074341.eml
 
     h = e.damn                  # Convert to HASH
     j = e.dump('json')          # Convert to JSON string
@@ -180,9 +182,8 @@ callbackto = lambda do |v|
   return r
 end
 
-list = ['X-Mailer']
-data = Sisimai.make('/path/to/mbox', hook: callbackto, field: list)
-json = Sisimai.dump('/path/to/mbox', hook: callbackto, field: list)
+data = Sisimai.make('/path/to/mbox', hook: callbackto)
+json = Sisimai.dump('/path/to/mbox', hook: callbackto)
 
 puts data[0].catch['x-mailer']      # Apple Mail (2.1283)
 ```
@@ -203,7 +204,7 @@ Output example
 ![](https://libsisimai.org/static/images/demo/sisimai-dump-02.gif)
 
 ```json
-[{"action": "failed", "subject": "Nyaan", "catch": null, "token": "08acf78323edc7923a783c04749dd547ab45c433", "alias": "", "messageid": "201806090556.w595u8GZ093276@neko.example.jp", "listid": "", "smtpcommand": "MAIL", "smtpagent": "Email::Sendmail", "lhost": "localhost", "timezoneoffset": "+0900", "feedbacktype": "", "senderdomain": "neko.example.jp", "diagnostictype": "SMTP", "softbounce": 1, "deliverystatus": "5.0.0", "addresser": "kijitora@neko.example.jp", "diagnosticcode": "550 Unauthenticated senders not allowed", "timestamp": 1528523769, "destination": "example.com", "recipient": "sironeko@example.com", "reason": "securityerror", "replycode": "550", "rhost": "neko.example.jp"}]
+[{"catch":{"x-mailer":"","return-path":"<shironeko@mx.example.co.jp>"},"token":"cf17945938502bd876603a375f0e9517c921bbab","lhost":"localhost","rhost":"mx-s.neko.example.jp","alias":"","listid":"","reason":"hasmoved","action":"failed","origin":"set-of-emails/maildir/bsd/lhost-sendmail-22.eml","subject":"Nyaaaan","messageid":"0000000011111.fff0000000003@mx.example.co.jp","replycode":"","smtpagent":"Sendmail","softbounce":0,"smtpcommand":"DATA","destination":"example.net","senderdomain":"example.co.jp","feedbacktype":"","diagnosticcode":"450 busy - please try later 551 not our customer 503 need RCPT command [data]","diagnostictype":"SMTP","deliverystatus":"5.1.6","timezoneoffset":"+0900","addresser":"shironeko@example.co.jp","recipient":"kijitora@example.net","timestamp":1397054085}]
 ```
 
 Sisimai Specification
@@ -224,8 +225,8 @@ and bounceHammer are available at
 | The speed of parsing email(1000 emails)     | 2.22s[2]       | 1.35s         |
 | How to install                              | gem install    | cpanm, cpm    |
 | Dependencies (Except core modules)          | 1 module       | 2 modules     |
-| LOC:Source lines of code                    | 10300 lines    | 7100 lines    |
-| The number of tests(spec/,t/,xt/) directory | 230000 tests   | 250000 tests  |
+| LOC:Source lines of code                    | 10200 lines    | 10400 lines   |
+| The number of tests(spec/,t/,xt/) directory | 236000 tests   | 266000 tests  |
 | License                                     | BSD 2-Clause   | BSD 2-Clause  |
 | Support Contract provided by Developer      | Available      | Available     |
 

data/lib/sisimai.rb

@@ -1,10 +1,9 @@
-require 'sisimai/version'
-
 # Sisimai is the system formerly known as bounceHammer 4, is a Ruby module for
 # analyzing bounce mails and generate structured data in a JSON format (YAML is
 # also available if "YAML" module is installed on your system) from parsed bounce
 # messages. Sisimai is a coined word: Sisi (the number 4 is pronounced "Si" in
 # Japanese) and MAI (acronym of "Mail Analyzing Interface").
+require 'sisimai/version'
 module Sisimai
   # Imported from p5-Sisimail/lib/Sisimai.pm
   class << self
@@ -19,28 +18,23 @@ module Sisimai
     # @param         [Hash]   argv1      Parser options(delivered=false)
     # @options argv1 [Boolean] delivered true: Include "delivered" reason
     # @options argv1 [Lambda]  hook      Lambda object to be called back
-    # @options argv1 [Array]   field     Email header name to be captured
     # @return        [Array]             Parsed objects
     # @return        [nil]               nil if the argument was wrong or an empty array
     def make(argv0, **argv1)
       return nil unless argv0
-
-      field = argv1[:field] || []
-      raise ' ***error: "field" accepts an array only' unless field.is_a? Array
-
       require 'sisimai/data'
       require 'sisimai/message'
       require 'sisimai/mail'
 
       list = []
       return nil unless mail = Sisimai::Mail.new(argv0)
-      while r = mail.read do
-        # Read and parse each mail file
-        methodargv = { data: r, hook: argv1[:hook], field: field }
+      while r = mail.data.read do
+        # Read and parse each email file
+        methodargv = { data: r, hook: argv1[:hook] }
         mesg = Sisimai::Message.new(methodargv)
         next if mesg.void
 
-        methodargv = { data: mesg, delivered: argv1[:delivered] }
+        methodargv = { data: mesg, delivered: argv1[:delivered], origin: mail.data.path }
         next unless data = Sisimai::Data.make(methodargv)
         list += data unless data.empty?
       end

data/lib/sisimai/arf.rb

@@ -26,12 +26,11 @@ module Sisimai
           )
         }x,
       }.freeze
+      ReportFrom = /(?:staff[@]hotmail[.]com|complaints[@]email-abuse[.]amazonses[.]com)\z/.freeze
       LongFields = Sisimai::RFC5322.LONGFIELDS
       RFC822Head = Sisimai::RFC5322.HEADERFIELDS
 
       def description; return 'Abuse Feedback Reporting Format'; end
-      def smtpagent;   return 'FeedBack-Loop'; end
-      def headerlist;  return []; end
 
       # Email is a Feedback-Loop message or not
       # @param    [Hash] heads    Email header including "Content-Type", "From",
@@ -49,13 +48,8 @@ module Sisimai
         elsif heads['content-type'].start_with?('multipart/mixed')
           # Microsoft (Hotmail, MSN, Live, Outlook) uses its own report format.
           # Amazon SES Complaints bounces
-          hfrom = Sisimai::Address.s3s4(heads['from'])
-          mfrom = %r{(?:
-             staff[@]hotmail[.]com
-            |complaints[@]email-abuse[.]amazonses[.]com
-            )\z
-          }x
-          if hfrom =~ mfrom && heads['subject'].include?('complaint about message from ')
+          p = Sisimai::Address.s3s4(heads['from'])
+          if p =~ ReportFrom && heads['subject'].include?('complaint about message from ')
             # From: staff@hotmail.com
             # From: complaints@email-abuse.amazonses.com
             # Subject: complaint about message from 192.0.2.1
@@ -66,16 +60,10 @@ module Sisimai
       end
 
       # Detect an error for Feedback Loop
-      # @param         [Hash] mhead       Message headers of a bounce email
-      # @options mhead [String] from      From header
-      # @options mhead [String] date      Date header
-      # @options mhead [String] subject   Subject header
-      # @options mhead [Array]  received  Received headers
-      # @options mhead [String] others    Other required headers
-      # @param         [String] mbody     Message body of a bounce email
-      # @return        [Hash, Nil]        Bounce data list and message/rfc822 part
-      #                                   or nil if it failed to parse or the
-      #                                   arguments are missing
+      # @param  [Hash] mhead    Message headers of a bounce email
+      # @param  [String] mbody  Message body of a bounce email
+      # @return [Hash]          Bounce data list and message/rfc822 part
+      # @return [Nil]           it failed to parse or the arguments are missing
       def make(mhead, mbody)
         return nil unless self.is_arf(mhead)
 
@@ -153,11 +141,18 @@ module Sisimai
             elsif cv = e.match(/\AFrom:[ ]*(.+)\z/)
               # Microsoft ARF: original sender.
               commondata['from'] = Sisimai::Address.s3s4(cv[1]) if commondata['from'].empty?
+              previousfn = 'from'
 
             elsif e.start_with?(' ', "\t")
               # Continued line from the previous line
-              rfc822part << e + "\n" if LongFields[previousfn]
-              next unless e.empty?
+              if previousfn == 'from'
+                # Multiple lines at "From:" field
+                commondata['from'] << e
+                next
+              else
+                rfc822part << e + "\n" if LongFields[previousfn]
+                next unless e.empty?
+              end
               rcptintext << e if previousfn == 'to'
 
             else
@@ -294,7 +289,7 @@ module Sisimai
           e['diagnosis']  = commondata['diagnosis'] unless e['diagnosis']
           e['date']       = mhead['date'] if e['date'].empty?
           e['reason']     = 'feedback'
-          e['agent']      = self.smtpagent if e['agent'].empty?
+          e['agent']      = 'Feedback-Loop'
           %w[command action status alias].each { |a| e[a] = '' }
 
           # Get the remote IP address from the message body
@@ -308,7 +303,6 @@ module Sisimai
             # from IP address 24.64.1.1 on Thu, 29 Apr 2010 00:00:00 +0000
             e['rhost'] = cv[1]
           end
-
         end
         return { 'ds' => dscontents, 'rfc822' => rfc822part }
       end