sisimai 5.5.0 → 5.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/rake-test.yml +0 -4
  3. data/ChangeLog.md +53 -0
  4. data/LICENSE +1 -1
  5. data/README-JA.md +26 -20
  6. data/README.md +30 -25
  7. data/lib/sisimai/address.rb +8 -31
  8. data/lib/sisimai/arf.rb +3 -5
  9. data/lib/sisimai/fact.rb +26 -36
  10. data/lib/sisimai/lhost/activehunter.rb +0 -2
  11. data/lib/sisimai/lhost/amazonses.rb +7 -9
  12. data/lib/sisimai/lhost/apachejames.rb +0 -1
  13. data/lib/sisimai/lhost/biglobe.rb +0 -16
  14. data/lib/sisimai/lhost/courier.rb +5 -4
  15. data/lib/sisimai/lhost/deutschetelekom.rb +120 -0
  16. data/lib/sisimai/lhost/domino.rb +0 -3
  17. data/lib/sisimai/lhost/dragonfly.rb +0 -27
  18. data/lib/sisimai/lhost/einsundeins.rb +1 -10
  19. data/lib/sisimai/lhost/exchange2003.rb +4 -4
  20. data/lib/sisimai/lhost/exchange2007.rb +5 -6
  21. data/lib/sisimai/lhost/exim.rb +35 -85
  22. data/lib/sisimai/lhost/ezweb.rb +12 -49
  23. data/lib/sisimai/lhost/fml.rb +4 -29
  24. data/lib/sisimai/lhost/gmail.rb +0 -23
  25. data/lib/sisimai/lhost/gmx.rb +7 -24
  26. data/lib/sisimai/lhost/googlegroups.rb +3 -3
  27. data/lib/sisimai/lhost/googleworkspace.rb +0 -4
  28. data/lib/sisimai/lhost/imailserver.rb +3 -9
  29. data/lib/sisimai/lhost/kddi.rb +6 -20
  30. data/lib/sisimai/lhost/mailfoundry.rb +0 -2
  31. data/lib/sisimai/lhost/mailmarshal.rb +1 -3
  32. data/lib/sisimai/lhost/messagingserver.rb +4 -15
  33. data/lib/sisimai/lhost/mfilter.rb +0 -1
  34. data/lib/sisimai/lhost/mimecast.rb +0 -1
  35. data/lib/sisimai/lhost/notes.rb +1 -2
  36. data/lib/sisimai/lhost/opensmtpd.rb +0 -40
  37. data/lib/sisimai/lhost/postfix.rb +10 -11
  38. data/lib/sisimai/lhost/qmail.rb +14 -81
  39. data/lib/sisimai/lhost/sendmail.rb +4 -4
  40. data/lib/sisimai/lhost/trendmicro.rb +3 -3
  41. data/lib/sisimai/lhost/v5sendmail.rb +0 -1
  42. data/lib/sisimai/lhost/verizon.rb +1 -2
  43. data/lib/sisimai/lhost/x1.rb +1 -2
  44. data/lib/sisimai/lhost/x2.rb +0 -2
  45. data/lib/sisimai/lhost/x3.rb +4 -9
  46. data/lib/sisimai/lhost/x6.rb +0 -1
  47. data/lib/sisimai/lhost/zoho.rb +0 -12
  48. data/lib/sisimai/lhost.rb +38 -19
  49. data/lib/sisimai/message.rb +3 -1
  50. data/lib/sisimai/order.rb +4 -1
  51. data/lib/sisimai/reason/authfailure.rb +9 -13
  52. data/lib/sisimai/reason/badreputation.rb +7 -7
  53. data/lib/sisimai/reason/blocked.rb +57 -83
  54. data/lib/sisimai/reason/contenterror.rb +16 -8
  55. data/lib/sisimai/reason/{mesgtoobig.rb → emailtoolarge.rb} +22 -25
  56. data/lib/sisimai/reason/expired.rb +27 -23
  57. data/lib/sisimai/reason/filtered.rb +13 -17
  58. data/lib/sisimai/reason/hasmoved.rb +2 -1
  59. data/lib/sisimai/reason/hostunknown.rb +25 -19
  60. data/lib/sisimai/reason/mailboxfull.rb +28 -49
  61. data/lib/sisimai/reason/networkerror.rb +28 -16
  62. data/lib/sisimai/reason/norelaying.rb +21 -20
  63. data/lib/sisimai/reason/notaccept.rb +13 -8
  64. data/lib/sisimai/reason/notcompliantrfc.rb +6 -9
  65. data/lib/sisimai/reason/policyviolation.rb +17 -26
  66. data/lib/sisimai/reason/ratelimited.rb +62 -0
  67. data/lib/sisimai/reason/rejected.rb +51 -59
  68. data/lib/sisimai/reason/requireptr.rb +13 -25
  69. data/lib/sisimai/reason/securityerror.rb +14 -19
  70. data/lib/sisimai/reason/spamdetected.rb +51 -101
  71. data/lib/sisimai/reason/suspend.rb +30 -24
  72. data/lib/sisimai/reason/syntaxerror.rb +1 -8
  73. data/lib/sisimai/reason/systemerror.rb +37 -23
  74. data/lib/sisimai/reason/systemfull.rb +1 -1
  75. data/lib/sisimai/reason/userunknown.rb +81 -112
  76. data/lib/sisimai/reason/virusdetected.rb +6 -8
  77. data/lib/sisimai/reason.rb +15 -15
  78. data/lib/sisimai/rfc1123.rb +1 -1
  79. data/lib/sisimai/rfc1894.rb +7 -6
  80. data/lib/sisimai/rfc2045.rb +2 -2
  81. data/lib/sisimai/rfc3464/thirdparty.rb +1 -1
  82. data/lib/sisimai/rfc3464.rb +10 -14
  83. data/lib/sisimai/rfc3834.rb +3 -4
  84. data/lib/sisimai/rfc791.rb +3 -38
  85. data/lib/sisimai/rhost/apple.rb +5 -5
  86. data/lib/sisimai/rhost/cloudflare.rb +2 -0
  87. data/lib/sisimai/rhost/cox.rb +22 -20
  88. data/lib/sisimai/rhost/facebook.rb +16 -16
  89. data/lib/sisimai/rhost/franceptt.rb +8 -3
  90. data/lib/sisimai/rhost/godaddy.rb +33 -15
  91. data/lib/sisimai/rhost/google.rb +63 -64
  92. data/lib/sisimai/rhost/iua.rb +1 -1
  93. data/lib/sisimai/rhost/messagelabs.rb +12 -12
  94. data/lib/sisimai/rhost/microsoft.rb +86 -86
  95. data/lib/sisimai/rhost/mimecast.rb +34 -34
  96. data/lib/sisimai/rhost/nttdocomo.rb +2 -2
  97. data/lib/sisimai/rhost/spectrum.rb +7 -7
  98. data/lib/sisimai/rhost/tencent.rb +9 -11
  99. data/lib/sisimai/rhost/yahooinc.rb +7 -8
  100. data/lib/sisimai/rhost.rb +1 -1
  101. data/lib/sisimai/smtp/command.rb +2 -0
  102. data/lib/sisimai/smtp/status.rb +73 -109
  103. data/lib/sisimai/string.rb +0 -27
  104. data/lib/sisimai/version.rb +1 -1
  105. data/set-of-emails/maildir/bsd/lhost-deutschetelekom-01.eml +66 -0
  106. data/set-of-emails/maildir/bsd/lhost-deutschetelekom-02.eml +68 -0
  107. data/set-of-emails/maildir/bsd/lhost-deutschetelekom-03.eml +50 -0
  108. data/set-of-emails/should-not-crash/p5-664-iomart-mail-filter.eml +258 -0
  109. data/set-of-emails/to-be-debugged-because/sisimai-cannot-parse-yet/.gitkeep +0 -0
  110. metadata +10 -6
  111. data/lib/sisimai/reason/exceedlimit.rb +0 -47
  112. data/lib/sisimai/reason/speeding.rb +0 -47
  113. data/lib/sisimai/reason/toomanyconn.rb +0 -59
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 396f5b9d1247d49b1a13cd0b7c6f5f239b104fd6ec7f5f83f4d30cdd6f86d72a
4
- data.tar.gz: 5fdb40367c5649dfbfd3f8e8a50fc6932b9ecf9633d17527bfcd9fe5f1038928
3
+ metadata.gz: 7df8c7a329d0b47c042e69fb471062991ec6559ed7d231a25ae693bc8f00809a
4
+ data.tar.gz: b09714499bb39b23f62dc6b6c5e0a94767688df4305737d9343c6c603b0bfec6
5
5
  SHA512:
6
- metadata.gz: 43a56c25e1e1a5669a1f8e4c0f06f135e6f0d303f22f65f7bd7088229aea7e190173e6f02a7f5294352ff7adfc5f44305b5bdac13bf60d060ed28a56a92cff82
7
- data.tar.gz: c6cfafb04030c083c020626a268f5e142231a1944d2704ee1b8daf74b2a94f26a4f94897d6d54ea10fbedaee1665b30cd76c4bed69afb4a10eff90b10b6393f2
6
+ metadata.gz: cff9ac363a2a70bd91e376cd1150cff122e17d9797288f2c8042e3d3c74009f014abd98e9a46c4eeb7d51e1ffaceed62340889075e969b1c9341facde1027d25
7
+ data.tar.gz: d36372f77fdf9767b7b51b785580c3a3fa4f5b1ed582ad2d265390dacdd6babb248260dfccf4d879959cf9fec7dafce6fc71454bc5c96114b729ceab432436e7
@@ -15,8 +15,6 @@ jobs:
15
15
  steps:
16
16
  - name: Checkout the repository(CRuby)
17
17
  uses: actions/checkout@v4
18
- with:
19
- ref: ${{ github.event.pull_request.head.ref }}
20
18
  - name: Setup CRuby
21
19
  uses: ruby/setup-ruby@v1
22
20
  with:
@@ -41,8 +39,6 @@ jobs:
41
39
  steps:
42
40
  - name: Checkout the repository(JRuby)
43
41
  uses: actions/checkout@v4
44
- with:
45
- ref: ${{ github.event.pull_request.head.ref }}
46
42
  - name: Setup JRuby
47
43
  uses: ruby/setup-ruby@v1
48
44
  with:
data/ChangeLog.md CHANGED
@@ -2,6 +2,59 @@ RELEASE NOTES for Ruby version of Sisimai
2
2
  ===================================================================================================
3
3
  - releases: "https://github.com/sisimai/rb-sisimai/releases"
4
4
  - download: "https://rubygems.org/gems/sisimai"
5
+ - document: "https://libsisimai.org/"
6
+
7
+ v5.7.0
8
+ ---------------------------------------------------------------------------------------------------
9
+ - release: "Mon, 22 Jun 2026 16:22:22 +0900 (JST)"
10
+ - version: "5.7.0"
11
+ - changes:
12
+ - **Bug fixes**
13
+ - #414 Bug fix: The value of command should be `RCPT` when `RCPT first` in the error message.
14
+ - #433 Check whether `lowerchunk` is nil or not to avoid `NoMethodError`, `SystemStackError`
15
+ in `haircut` method of `Sisimai::RFC2045`. #434 #436 Thanks to @SAY-5.
16
+ - **MTA modules and error message patterns**
17
+ - #410 #412 Improvement in error message patterns.
18
+ - Update error message patterns in `AuthFailure`, `ContentError`, `NoRelaying`, `Rejected`,
19
+ `UserUnknown`, `SystemError`, `Suspend`, and `Sisimai::Rhost::Microsoft`.
20
+ - Remove error message patterns already defined in `Sisimai::Reason` from some MTA moudles of
21
+ `Sisimai::Lhost`.
22
+ - Move error message patterns from some MTA modules of `Sisimai::Lhost` to `Sisimai::Reason`.
23
+ - #413 List all the Zoho domains.
24
+ - Update the order of `ClassOrder` in `Sisimai::Reason`.
25
+ - #421 Code improvement for detecting bounce messages returned from Google Groups.
26
+ - #437 Implement `Sisimai::Lhost::DeutscheTelekom` to decode bounce messages generated by Smail
27
+ 3 or Deutsche Telekom.
28
+ - **Code improvements and Environment**
29
+ - **#430 Sisimai does not support Ruby 4.0.0 until the end of 2027.**
30
+ - #417 Use a switch statement instead of if-else for better readability.
31
+ - #423 Fix and update comments in `Sisimai::SMTP::Status`.
32
+ - #428 Remove useless code blocks.
33
+ - **EXPERIMENTAL Features**
34
+ - #424 #425 Change the data type of `Toxic` field implemented at v5.5.0 from Boolean to Integer
35
+ with `-1` as the default value to allow score-based evaluation of recipient address toxicity.
36
+ - #426 #427 Implement `Bogus` field as an Integer with `-1` as the default value to record the
37
+ unreliability score of a bounce message.
38
+
39
+ v5.6.0
40
+ ---------------------------------------------------------------------------------------------------
41
+ - release: "Mon, 2 Feb 2026 18:30:22 +0900 (JST)"
42
+ - version: "5.6.0"
43
+ - changes:
44
+ - **Changes in Bounce Reason Categorization**
45
+ - Update bounce status mappings in `Sisimai::SMTP::Status`.
46
+ - #399 Consolidate four bounce reasons into two:
47
+ - Merge `TooManyConn` and `Speeding` into `RateLimited`.
48
+ - Merge `ExceedLimit` and `MesgTooBig` into `EmailTooLarge`.
49
+ - #406 Update assigned reasons in `rhost/for-*.go`.
50
+ - #408 Error message patterns improvement.
51
+ - Merge similar error message patterns and remove ambiguous ones.
52
+ - Attachment-related errors have been moved from `PolicyViolation` to `ContentError`.
53
+ - **Reorganize the internal status code** #403
54
+ - Temporary error `4.0.9**` has been changed to `4.9.***`.
55
+ - Permanent error `5.0.9**` has been changed to `5.9.***`.
56
+ - #401 `Sisimai::String.token` has been moved to `Sisimai::Fact.token()`.
57
+ - Happy Birthday to Suzu (formerly known as "Neko-dono" Michitsuna).
5
58
 
6
59
  v5.5.0
7
60
  ---------------------------------------------------------------------------------------------------
data/LICENSE CHANGED
@@ -1,6 +1,6 @@
1
1
  THIS SOFTWARE IS DISTRIBUTED UNDER THE FOLLOWING BSD 2-CLAUSE LICENSE:
2
2
 
3
- Copyright (C) 2015-2017 azumakuniyuki, All rights reserved.
3
+ Copyright (C) 2015-2026 azumakuniyuki, All rights reserved.
4
4
 
5
5
  Redistribution and use in source and binary forms, with or without modification,
6
6
  are permitted provided that the following conditions are met:
data/README-JA.md CHANGED
@@ -1,5 +1,5 @@
1
1
  ![](https://libsisimai.org/static/images/logo/sisimai-x01.png)
2
- [![License](https://img.shields.io/badge/license-BSD%202--Clause-orange.svg)](https://github.com/sisimai/rb-sisimai/blob/master/LICENSE)
2
+ [![License](https://img.shields.io/badge/license-BSD%202--Clause-orange.svg)](https://github.com/sisimai/rb-sisimai/blob/5-stable/LICENSE)
3
3
  [![Ruby](https://img.shields.io/badge/ruby-v2.5.0--v3.4.0-red.svg)](https://www.ruby-lang.org/)
4
4
  [![Gem Version](https://badge.fury.io/rb/sisimai.svg)](https://badge.fury.io/rb/sisimai)
5
5
  [![codecov](https://codecov.io/github/sisimai/rb-sisimai/graph/badge.svg?token=YGkyluNWiZ)](https://codecov.io/github/sisimai/rb-sisimai)
@@ -65,14 +65,15 @@ Sisimai(シシマイ)は複雑で多種多様なバウンスメールを解析
65
65
  The key features of Sisimai
66
66
  ---------------------------------------------------------------------------------------------------
67
67
  * __バウンスメールを構造化したデータに変換__
68
- * 以下27項目の情報を含むデータ構造[^2]
68
+ * 以下28項目の情報を含むデータ構造[^2]
69
69
  * __基本的情報__: `timestamp`, `origin`
70
70
  * __発信者情報__: `addresser`, `senderdomain`,
71
71
  * __受信者情報__: `recipient`, `destination`, `alias`
72
72
  * __配信の情報__: `action`, `replycode`, `deliverystatus`, `command`
73
73
  * __エラー情報__: `reason`, `diagnosticcode`, `diagnostictype`, `feedbacktype`, `hardbounce`
74
74
  * __メール情報__: `subject`, `messageid`, `listid`,
75
- * __その他情報__: `decodedby`, `timezoneoffset`, `lhost`, `rhost`, `token`, `catch`
75
+ * __評価用項目__: `toxic`, `bogus`, `catch`
76
+ * __その他情報__: `decodedby`, `timezoneoffset`, `lhost`, `rhost`, `token`
76
77
  * __出力可能な形式__
77
78
  * Ruby (Hash, Array)
78
79
  * JSON
@@ -83,9 +84,9 @@ The key features of Sisimai
83
84
  * `gem install`
84
85
  * `git clone && make`
85
86
  * __高い解析精度__
86
- * [60種類のMTAs/MDAs/ESPs](https://libsisimai.org/en/engine/)に対応
87
+ * [61種類のMTAs/MDAs/ESPs](https://libsisimai.org/en/engine/)に対応
87
88
  * Feedback Loop(ARF)にも対応
88
- * [36種類のバウンス理由](https://libsisimai.org/en/reason/)を検出
89
+ * [34種類のバウンス理由](https://libsisimai.org/en/reason/)を検出
89
90
 
90
91
  [^2]: コールバック機能を使用すると`catch`アクセサの下に独自のデータを追加できます
91
92
 
@@ -115,10 +116,10 @@ Install
115
116
  ### From RubyGems.org
116
117
  ```shell
117
118
  $ sudo gem install sisimai
118
- Fetching: sisimai-5.5.0.gem (100%)
119
- Successfully installed sisimai-5.5.0
120
- Parsing documentation for sisimai-5.5.0
121
- Installing ri documentation for sisimai-5.5.0
119
+ Fetching: sisimai-5.7.0.gem (100%)
120
+ Successfully installed sisimai-5.7.0
121
+ Parsing documentation for sisimai-5.7.0
122
+ Installing ri documentation for sisimai-5.7.0
122
123
  Done installing documentation for sisimai after 6 seconds
123
124
  1 gem installed
124
125
  ```
@@ -146,13 +147,13 @@ if [ -d "/usr/local/jr" ]; then \
146
147
  ...
147
148
  3 gems installed
148
149
  /opt/local/bin/rake install
149
- sisimai 5.5.0 built to pkg/sisimai-5.5.0.gem.
150
- sisimai (5.5.0) installed.
150
+ sisimai 5.7.0 built to pkg/sisimai-5.7.0.gem.
151
+ sisimai (5.7.0) installed.
151
152
  if [ -d "/usr/local/jr" ]; then \
152
153
  PATH="/usr/local/jr/bin:$PATH" /usr/local/jr/bin/rake install; \
153
154
  fi
154
- sisimai 5.5.0 built to pkg/sisimai-5.5.0-java.gem.
155
- sisimai (5.5.0) installed.
155
+ sisimai 5.7.0 built to pkg/sisimai-5.7.0-java.gem.
156
+ sisimai (5.7.0) installed.
156
157
  ```
157
158
 
158
159
  Usage
@@ -333,7 +334,8 @@ Output example
333
334
  "timezoneoffset": "+0900",
334
335
  "replycode": 550,
335
336
  "token": "84656774898baa90660be3e12fe0526e108d4473",
336
- "toxic": false,
337
+ "bogus": -1,
338
+ "toxic": -1,
337
339
  "diagnostictype": "SMTP",
338
340
  "timestamp": 1650119685,
339
341
  "diagnosticcode": "host gmail-smtp-in.l.google.com[64.233.187.27] said: This mail has been blocked because the sender is unauthenticated. Gmail requires all senders to authenticate with either SPF or DKIM. Authentication results: DKIM = did not pass SPF [relay3.example.com] with ip: [192.0.2.22] = did not pass For instructions on setting up authentication, go to https://support.google.com/mail/answer/81126#authentication c2-202200202020202020222222cat.127 - gsmtp (in reply to end of DATA command)",
@@ -358,9 +360,9 @@ Sisimai 5.5.0から**Ruby 2.5以上**が必要になります。
358
360
  | 動作環境(JRuby) | 9.0.4.0 - 9.1.17.0 | **9.2** or later |
359
361
  | 元メールファイルを操作可能なコールバック機能 | なし | あり[^3] |
360
362
  | 解析エンジン(MTA/ESPモジュール)の数 | 68 | 60 |
361
- | 検出可能なバウンス理由の数 | 29 | 36 |
363
+ | 検出可能なバウンス理由の数 | 29 | 34 |
362
364
  | 依存Gem数(Ruby Standard Gemsを除く) | 1 Gem | 1 Gem |
363
- | ソースコードの行数 | 10,800 行 | 9,970 行 |
365
+ | ソースコードの行数 | 10,800 行 | 9,700 行 |
364
366
  | テストフレームワーク | rspec | minitest |
365
367
  | テスト件数(spec/またはtest/ディレクトリ) | 311,000 件 | 240,000 件 |
366
368
  | 1秒間に解析できるバウンスメール数[^4] | 620 通 | 620 通 |
@@ -405,6 +407,7 @@ Sisimai 5で3個のESPモジュール名(解析エンジン)が変更になり
405
407
  | Zoho (added at v5.5.0) | なし | `Rhost::Zoho` |
406
408
  | DragonFly Mail Agent (added at v5.1.0) | なし | `Lhost::DragonFly` |
407
409
  | Mimecast (added at v5.5.0) | なし | `Lhost::Mimecast` |
410
+ | DeutscheTelekom (added at v5.7.0) | なし | `Lhost::DeutscheTelekom` |
408
411
 
409
412
  Bounce Reasons
410
413
  ---------------------------------------------------------------------------------------------------
@@ -417,8 +420,11 @@ Sisimai 5では新たに5個のバウンス理由が増えました。検出可
417
420
  | 送信者のドメイン・IPアドレスの低いレピュテーション | `Blocked` | `BadReputation` |
418
421
  | PTRレコードが未設定または無効なPTRレコード | `Blocked` | `RequirePTR` |
419
422
  | RFCに準拠していないメール[^7] | `SecurityError` | `NotCompliantRFC` |
420
- | 単位時間の流量制限・送信速度が速すぎる | `SecurityError` | `Speeding` |
421
423
  | STARTTLS関連のエラー (added at v5.2.0) | `SecurityError` | `FailedSTARTTLS` |
424
+ | 単位時間の流量制限・送信速度が速すぎる | `SecurityError` | `RateLimited` |
425
+ | セッションあたりの受信者数制限や接続数を超過 | `TooManyConn` | `RateLimited` |
426
+ | メールが大きすぎる(ExceedLimit) | `ExceedLimit` | `EmailTooLarge` |
427
+ | メールが大きすぎる(MesgTooBig) | `MesgTooBig` | `EmailTooLarge` |
422
428
  | 宛先がサプレッションリストに一致 (added at v5.2.0) | `OnHold` | `Suppressed` |
423
429
 
424
430
  [^7]: RFC5322など
@@ -433,7 +439,7 @@ Bug report
433
439
  Emails could not be decoded
434
440
  ---------------------------------------------------------------------------------------------------
435
441
  Sisimaiで解析できないバウンスメールは
436
- [set-of-emails/to-be-debugged-because/sisimai-cannot-parse-yet](https://github.com/sisimai/set-of-emails/tree/master/to-be-debugged-because/sisimai-cannot-parse-yet)リポジトリに追加してPull-Requestを送ってください。
442
+ [set-of-emails/to-be-debugged-because/sisimai-cannot-parse-yet](https://github.com/sisimai/rb-sisimai/tree/5-stable/set-of-emails/to-be-debugged-because/sisimai-cannot-parse-yet)ディレクトリに追加してPull-Requestを送ってください。
437
443
 
438
444
  Other Information
439
445
  ===================================================================================================
@@ -450,7 +456,7 @@ Related sites
450
456
 
451
457
  See also
452
458
  ---------------------------------------------------------------------------------------------------
453
- * [README.md - README.md in English(🇬🇧)](https://github.com/sisimai/rb-sisimai/blob/master/README.md)
459
+ * [README.md - README.md in English(🇬🇧)](https://github.com/sisimai/rb-sisimai/blob/5-stable/README.md)
454
460
  * [RFC3463 - Enhanced Mail System Status Codes](https://tools.ietf.org/html/rfc3463)
455
461
  * [RFC3464 - An Extensible Message Format for Delivery Status Notifications](https://tools.ietf.org/html/rfc3464)
456
462
  * [RFC3834 - Recommendations for Automatic Responses to Electronic Mail](https://tools.ietf.org/html/rfc3834)
@@ -463,7 +469,7 @@ Author
463
469
 
464
470
  Copyright
465
471
  ===================================================================================================
466
- Copyright (C) 2015-2025 azumakuniyuki, All Rights Reserved.
472
+ Copyright (C) 2015-2026 azumakuniyuki, All Rights Reserved.
467
473
 
468
474
  License
469
475
  ===================================================================================================
data/README.md CHANGED
@@ -1,5 +1,5 @@
1
1
  ![](https://libsisimai.org/static/images/logo/sisimai-x01.png)
2
- [![License](https://img.shields.io/badge/license-BSD%202--Clause-orange.svg)](https://github.com/sisimai/rb-sisimai/blob/master/LICENSE)
2
+ [![License](https://img.shields.io/badge/license-BSD%202--Clause-orange.svg)](https://github.com/sisimai/rb-sisimai/blob/5-stable/LICENSE)
3
3
  [![Ruby](https://img.shields.io/badge/ruby-v2.5.0--v3.4.0-red.svg)](https://www.ruby-lang.org/)
4
4
  [![Gem Version](https://badge.fury.io/rb/sisimai.svg)](https://badge.fury.io/rb/sisimai)
5
5
  [![codecov](https://codecov.io/github/sisimai/rb-sisimai/graph/badge.svg?token=YGkyluNWiZ)](https://codecov.io/github/sisimai/rb-sisimai)
@@ -67,14 +67,15 @@ of Sisimai is ported from [the Perl version of Sisimai](https://github.com/sisim
67
67
  The key features of Sisimai
68
68
  ---------------------------------------------------------------------------------------------------
69
69
  * __Decode email bounces to structured data__
70
- * Sisimai provides detailed insights into bounce emails by extracting 27 key data points.[^2]
70
+ * Sisimai provides detailed insights into bounce emails by extracting 28 key data points.[^2]
71
71
  * __Essential information__: `timestamp`, `origin`
72
72
  * __Sender information__: `addresser`, `senderdomain`,
73
73
  * __Recipient information__: `recipient`, `destination`, `alias`
74
74
  * __Delivery information__: `action`, `replycode`, `deliverystatus`, `command`
75
75
  * __Bounce details__: `reason`, `diagnosticcode`, `diagnostictype`, `feedbacktype`, `feedbackid`, `hardbounce`
76
76
  * __Message details__: `subject`, `messageid`, `listid`,
77
- * __Additional information__: `decodedby`, `timezoneoffset`, `lhost`, `rhost`, `token`, `catch`, `toxic`
77
+ * __Evaluation metrics (User-calculated)__: `toxic`, `bogus`, `catch`
78
+ * __Additional information__: `decodedby`, `timezoneoffset`, `lhost`, `rhost`, `token`
78
79
  * Output formats
79
80
  * Ruby (Hash, Array)
80
81
  * JSON
@@ -85,9 +86,9 @@ The key features of Sisimai
85
86
  * `gem install`
86
87
  * `git clone && make`
87
88
  * __High Precision of Analysis__
88
- * Support [60 MTAs/MDAs/ESPs](https://libsisimai.org/en/engine/)
89
+ * Support [61 MTAs/MDAs/ESPs](https://libsisimai.org/en/engine/)
89
90
  * Support Feedback Loop Message(ARF)
90
- * Can detect [36 bounce reasons](https://libsisimai.org/en/reason/)
91
+ * Can detect [34 bounce reasons](https://libsisimai.org/en/reason/)
91
92
 
92
93
  [^2]: The callback function allows you to add your own data under the `catch` accessor.
93
94
 
@@ -114,10 +115,10 @@ Install
114
115
  ### From RubyGems
115
116
  ```shell
116
117
  $ sudo gem install sisimai
117
- Fetching: sisimai-5.5.0.gem (100%)
118
- Successfully installed sisimai-5.5.0
119
- Parsing documentation for sisimai-5.5.0
120
- Installing ri documentation for sisimai-5.5.0
118
+ Fetching: sisimai-5.7.0.gem (100%)
119
+ Successfully installed sisimai-5.7.0
120
+ Parsing documentation for sisimai-5.7.0
121
+ Installing ri documentation for sisimai-5.7.0
121
122
  Done installing documentation for sisimai after 6 seconds
122
123
  1 gem installed
123
124
  ```
@@ -145,13 +146,13 @@ if [ -d "/usr/local/jr" ]; then \
145
146
  ...
146
147
  3 gems installed
147
148
  /opt/local/bin/rake install
148
- sisimai 5.5.0 built to pkg/sisimai-5.5.0.gem.
149
- sisimai (5.5.0) installed.
149
+ sisimai 5.7.0 built to pkg/sisimai-5.7.0.gem.
150
+ sisimai (5.7.0) installed.
150
151
  if [ -d "/usr/local/jr" ]; then \
151
152
  PATH="/usr/local/jr/bin:$PATH" /usr/local/jr/bin/rake install; \
152
153
  fi
153
- sisimai 5.5.0 built to pkg/sisimai-5.5.0-java.gem.
154
- sisimai (5.5.0) installed.
154
+ sisimai 5.7.0 built to pkg/sisimai-5.7.0-java.gem.
155
+ sisimai (5.7.0) installed.
155
156
  ```
156
157
 
157
158
  Usage
@@ -332,7 +333,8 @@ Output example
332
333
  "timezoneoffset": "+0900",
333
334
  "replycode": 550,
334
335
  "token": "84656774898baa90660be3e12fe0526e108d4473",
335
- "toxic": false,
336
+ "bogus": -1,
337
+ "toxic": -1,
336
338
  "diagnostictype": "SMTP",
337
339
  "timestamp": 1650119685,
338
340
  "diagnosticcode": "host gmail-smtp-in.l.google.com[64.233.187.27] said: This mail has been blocked because the sender is unauthenticated. Gmail requires all senders to authenticate with either SPF or DKIM. Authentication results: DKIM = did not pass SPF [relay3.example.com] with ip: [192.0.2.22] = did not pass For instructions on setting up authentication, go to https://support.google.com/mail/answer/81126#authentication c2-202200202020202020222222cat.127 - gsmtp (in reply to end of DATA command)",
@@ -356,12 +358,12 @@ Beginning with v5.5.0, Sisimai requires **Ruby 2.5.0 or later.**
356
358
  | System requirements (CRuby) | 2.1 - 3.3.0 | **2.5** or later |
357
359
  | System requirements (JRuby) | 9.0.4.0 - 9.1.17.0 | **9.2** or later |
358
360
  | Callback feature for the original email file | N/A | Available[^3] |
359
- | The number of MTA/ESP modules | 68 | 60 |
360
- | The number of detectable bounce reasons | 29 | 36 |
361
+ | The number of MTA/ESP modules | 68 | 61 |
362
+ | The number of detectable bounce reasons | 29 | 34 |
361
363
  | Dependencies (Except Ruby Standard Gems) | 1 gem | 1 gem |
362
- | Source lines of code | 10,300 lines | 9,970 lines |
364
+ | Source lines of code | 10,300 lines | 9,500 lines |
363
365
  | Test frameworks | rspec | minitest |
364
- | The number of tests in spec/ or test/ directory | 311,000 tests | 240,000 tests |
366
+ | The number of tests in spec/ or test/ directory | 311,000 tests | 255,000 tests |
365
367
  | The number of bounce emails decoded/sec (CRuby)[^4] | 620 emails | 620 emails |
366
368
  | License | 2 Clause BSD | 2 Caluse BSD |
367
369
  | Commercial support | Available | Available |
@@ -406,6 +408,7 @@ available at [LIBSISIMAI.ORG/EN/ENGINE](https://libsisimai.org/en/engine/)
406
408
  | Zoho (added at v5.5.0) | None | `Rhost::Zoho` |
407
409
  | DragonFly Mail Agent (added at v5.1.0) | None | `Lhost::DragonFly` |
408
410
  | Mimecast (added at v5.5.0) | None | `Lhost::Mimecast` |
411
+ | DeutscheTelekom (added at v5.7.0) | None | `Lhost::DeutscheTelekom` |
409
412
 
410
413
  Bounce Reasons
411
414
  ---------------------------------------------------------------------------------------------------
@@ -418,8 +421,11 @@ detect is available at [LIBSISIMAI.ORG/EN/REASON](https://libsisimai.org/en/reas
418
421
  | low/bad reputation of the sender hostname/IP addr. | `Blocked` | `BadReputation` |
419
422
  | missing PTR/having invalid PTR | `Blocked` | `RequirePTR` |
420
423
  | non-compliance with RFC[^7] | `SecurityError` | `NotCompliantRFC` |
421
- | exceeding a rate limit or sending too fast | `SecurityError` | `Speeding` |
422
424
  | STARTTLS-related errors (added at v5.2.0) | `SecurityError` | `FailedSTARTTLS` |
425
+ | exceeding a rate limit or sending too fast | `SecurityError` | `RateLimited` |
426
+ | too many concurrency connections or recipients | `TooManyConn` | `RateLimited` |
427
+ | Email size is too large for the remote host | `ExceedLimit` | `EmailTooLarge` |
428
+ | Email size is too large for the remote host | `MesgTooBig` | `EmailTooLarge` |
423
429
  | Recipient in the suppression list (added at v5.2.0) | `OnHold` | `Suppressed` |
424
430
 
425
431
  [^7]: RFC5322 and related RFCs
@@ -433,10 +439,9 @@ Please use the [issue tracker](https://github.com/sisimai/rb-sisimai/issues) to
433
439
 
434
440
  Emails could not be decoded
435
441
  ---------------------------------------------------------------------------------------------------
436
- Bounce mails which could not be decoded by Sisimai are saved in the repository
437
- [set-of-emails/to-be-debugged-because/sisimai-cannot-parse-yet](https://github.com/sisimai/set-of-emails/tree/master/to-be-debugged-because/sisimai-cannot-parse-yet).
438
- If you have found any bounce email cannot be decoded using Sisimai, please add the email into the
439
- directory and send Pull-Request to this repository.
442
+ Bounce mails which could not be decoded by Sisimai are saved in
443
+ [set-of-emails/to-be-debugged-because/sisimai-cannot-parse-yet](https://github.com/sisimai/rb-sisimai/tree/5-stable/set-of-emails/to-be-debugged-because/sisimai-cannot-parse-yet) directory. If you have found any bounce email cannot be decoded
444
+ using Sisimai, please add the email into the directory and send Pull-Request to this repository.
440
445
 
441
446
  Other Information
442
447
  ===================================================================================================
@@ -453,7 +458,7 @@ Related Sites
453
458
 
454
459
  See also
455
460
  ---------------------------------------------------------------------------------------------------
456
- * [README-JA.md - README.md in Japanese(🇯🇵)](https://github.com/sisimai/rb-sisimai/blob/master/README-JA.md)
461
+ * [README-JA.md - README.md in Japanese(🇯🇵)](https://github.com/sisimai/rb-sisimai/blob/5-stable/README-JA.md)
457
462
  * [RFC3463 - Enhanced Mail System Status Codes](https://tools.ietf.org/html/rfc3463)
458
463
  * [RFC3464 - An Extensible Message Format for Delivery Status Notifications](https://tools.ietf.org/html/rfc3464)
459
464
  * [RFC3834 - Recommendations for Automatic Responses to Electronic Mail](https://tools.ietf.org/html/rfc3834)
@@ -466,7 +471,7 @@ Author
466
471
 
467
472
  Copyright
468
473
  ===================================================================================================
469
- Copyright (C) 2015-2025 azumakuniyuki, All Rights Reserved.
474
+ Copyright (C) 2015-2026 azumakuniyuki, All Rights Reserved.
470
475
 
471
476
  License
472
477
  ===================================================================================================
@@ -147,8 +147,8 @@ module Sisimai
147
147
  # Check each characters
148
148
  if Delimiters[e]
149
149
  # The character is a delimiter character
150
- if e == ','
151
- # Separator of email addresses or not
150
+ case e
151
+ when "," # Separator of email addresses or not
152
152
  if v[:address].start_with?('<') && v[:address].end_with?('>') && v[:address].include?('@')
153
153
  # An email address has already been picked
154
154
  if readcursor & Indicators[:'comment-block'] > 0
@@ -168,11 +168,7 @@ module Sisimai
168
168
  # "Neko, Nyaan" <neko@nyaan.example.org> OR <"neko,nyaan"@example.org>
169
169
  p.empty? ? (v[:name] += e) : (v[p] += e)
170
170
  end
171
- next
172
- end # End of if(',')
173
-
174
- if e == '<'
175
- # <: The beginning of an email address or not
171
+ when "<" # <: The beginning of an email address or not
176
172
  if v[:address].size > 0
177
173
  p.empty? ? (v[:name] += e) : (v[p] += e)
178
174
  else
@@ -181,12 +177,7 @@ module Sisimai
181
177
  v[:address] += e
182
178
  p = :address
183
179
  end
184
- next
185
- end
186
- # End of if('<')
187
-
188
- if e == '>'
189
- # >: The end of an email address or not
180
+ when ">" # >: The end of an email address or not
190
181
  if readcursor & Indicators[:'email-address'] > 0
191
182
  # <neko@example.org>
192
183
  readcursor &= ~Indicators[:'email-address']
@@ -196,11 +187,7 @@ module Sisimai
196
187
  # a comment block or a display name
197
188
  p.empty? ? (v[:name] == e) : (v[:comment] -= e)
198
189
  end
199
- next
200
- end # End of if('>')
201
-
202
- if e == '('
203
- # The beginning of a comment block or not
190
+ when "(" # The beginning of a comment block or not
204
191
  if readcursor & Indicators[:'email-address'] > 0
205
192
  # <"neko(nyaan)"@example.org> or <neko(nyaan)@example.org>
206
193
  if v[:address].include?('"')
@@ -228,11 +215,7 @@ module Sisimai
228
215
  v[:comment] += e
229
216
  p = :comment
230
217
  end
231
- next
232
- end # End of if('(')
233
-
234
- if e == ')'
235
- # The end of a comment block or not
218
+ when ")" # The end of a comment block or not
236
219
  if readcursor & Indicators[:'email-address'] > 0
237
220
  # <"neko(nyaan)"@example.org> OR <neko(nyaan)@example.org>
238
221
  if v[:address].include?('"')
@@ -255,11 +238,7 @@ module Sisimai
255
238
  v[:name] += e
256
239
  p = ''
257
240
  end
258
- next
259
- end # End of if(')')
260
-
261
- if e == '"'
262
- # The beginning or the end of a quoted-string
241
+ when '"' # The beginning or the end of a quoted-string
263
242
  if p.size > 0
264
243
  # email-address or comment-block
265
244
  v[p] += e
@@ -271,12 +250,10 @@ module Sisimai
271
250
  readcursor &= ~Indicators[:'quoted-string']
272
251
  p = ''
273
252
  end
274
- next
275
- end # End of if('"')
253
+ end # End of case-when
276
254
  else
277
255
  # The character is not a delimiter
278
256
  p.empty? ? (v[:name] += e) : (v[p] += e)
279
- next
280
257
  end
281
258
  end
282
259
 
data/lib/sisimai/arf.rb CHANGED
@@ -193,7 +193,7 @@ module Sisimai
193
193
  # X-Apple-Unsubscribe: true
194
194
  last if mhead["x-apple-unsubscribe"] != "true" || mhead["from"].include?('@') == false
195
195
  dscontents[0]["recipient"] = mhead["from"]
196
- dscontents[0]["diagnosis"] = Sisimai::String.sweep(emailparts[0])
196
+ dscontents[0]["diagnosis"] = emailparts[0]
197
197
  dscontents[0]["feedbacktype"] = "opt-out"
198
198
 
199
199
  # Addpend To: field as a pseudo header
@@ -214,14 +214,12 @@ module Sisimai
214
214
  end
215
215
  return nil if recipients == 0
216
216
 
217
- anotherone = ": #{Sisimai::String.sweep(anotherone)}" if anotherone != ""
218
- anotherone = anotherone.chop if anotherone[-1, 1] == ","
219
-
217
+ anotherone = ": #{anotherone.chop}" if anotherone != ""
220
218
  j = -1
221
219
  dscontents.each do |e|
222
220
  # Tidy up the error message in e.Diagnosis, Try to detect the bounce reason.
223
221
  j += 1
224
- e["diagnosis"] = Sisimai::String.sweep(e["diagnosis"] + anotherone)
222
+ e["diagnosis"] = e["diagnosis"] + anotherone
225
223
  e["reason"] = "feedback"
226
224
  e["rhost"] = remotehost
227
225
  e["lhost"] = reportedby
data/lib/sisimai/fact.rb CHANGED
@@ -20,6 +20,7 @@ module Sisimai
20
20
  :action, # [String] The value of Action: header
21
21
  :addresser, # [Sisimai::Address] From address
22
22
  :alias, # [String] Alias of the recipient address
23
+ :bogus, # [Integer] EXPERIMENTAL
23
24
  :catch, # [?] Results generated by hook method
24
25
  :command, # [String] The last SMTP command
25
26
  :decodedby, # [String] MTA module name since v5.2.0
@@ -43,7 +44,7 @@ module Sisimai
43
44
  :timestamp, # [Sisimai::Time] Date: header in the original message
44
45
  :timezoneoffset, # [Integer] Time zone offset(seconds)
45
46
  :token, # [String] Message token/MD5 Hex digest value
46
- :toxic, # [Boolean] EXPERIMENTAL
47
+ :toxic, # [Integer] EXPERIMENTAL
47
48
  ]
48
49
  attr_accessor(*@@rwaccessors)
49
50
 
@@ -71,6 +72,7 @@ module Sisimai
71
72
  @alias = argvs['alias'] || ''
72
73
  @addresser = argvs['addresser']
73
74
  @action = argvs['action']
75
+ @bogus = argvs['bogus']
74
76
  @catch = argvs['catch']
75
77
  @command = argvs['command']
76
78
  @decodedby = argvs['decodedby']
@@ -141,7 +143,6 @@ module Sisimai
141
143
  "replycode" => e["replycode"],
142
144
  "rhost" => e["rhost"],
143
145
  "decodedby" => e["agent"],
144
- "toxic" => e["toxic"],
145
146
  }
146
147
 
147
148
  # EMAILADDRESS: Detect an email address from message/rfc822 part
@@ -344,7 +345,7 @@ module Sisimai
344
345
  p1 = dc.index('<html>')
345
346
  p2 = dc.index('</html>')
346
347
  piece['diagnosticcode'][p1, p2 + 7 - p1] = '' if p1 && p2
347
- piece['diagnosticcode'] = Sisimai::String.sweep(piece['diagnosticcode'])
348
+ piece['diagnosticcode'] = piece['diagnosticcode'].split.join(" ")
348
349
  end
349
350
 
350
351
  if Sisimai::String.is_8bit(piece['diagnosticcode'])
@@ -357,8 +358,12 @@ module Sisimai
357
358
  piece["diagnostictype"] = "SMTP" if %w[feedback vacation].include?(piece["reason"]) == false
358
359
  end
359
360
 
360
- # Check the value of SMTP command
361
+ # When "RCPT first" in the error message, set "RCPT" as the last command.
362
+ # - <<< 503 RCPT first (#5.5.1)
363
+ # - <<< 503-5.5.1 RCPT first. A mail transaction protocol command was issued ...
364
+ # - RCPT first (in reply to DATA command)
361
365
  piece['command'] = '' if Sisimai::SMTP::Command.test(piece['command']) == false
366
+ piece['command'] = 'RCPT' if piece['diagnosticcode'].include?('RCPT first')
362
367
 
363
368
  # Create parameters for the constructor
364
369
  as = Sisimai::Address.new(piece['addresser']) || next; next if as.void
@@ -374,18 +379,19 @@ module Sisimai
374
379
  'senderdomain' => as.host,
375
380
  'destination' => ar.host,
376
381
  'alias' => piece['alias'] || ar.alias,
377
- 'token' => Sisimai::String.token(as.address, ar.address, piece['timestamp']),
382
+ 'token' => Sisimai::Fact.token(as.address, ar.address, piece['timestamp']),
378
383
  }
379
384
  ea.each { |q| thing[q] = piece[q] if thing[q].nil? || thing[q].empty? }
380
385
 
381
386
  # Other accessors
387
+ thing['bogus'] = 0
382
388
  thing['catch'] = piece['catch'] || nil
383
389
  thing["feedbackid"] = ""
384
390
  thing['hardbounce'] = piece['hardbounce']
385
- thing['toxic'] = piece['toxic']
386
391
  thing['replycode'] = Sisimai::SMTP::Reply.find(piece['diagnosticcode']) if thing['replycode'].empty?
387
392
  thing['timestamp'] = TimeModule.parse(::Time.at(piece['timestamp']).to_s)
388
393
  thing['timezoneoffset'] = piece['timezoneoffset'] || '+0000'
394
+ thing['toxic'] = 0
389
395
  ea.each { |q| thing[q] = piece[q] if thing[q].empty? }
390
396
 
391
397
  # ALIAS
@@ -483,42 +489,25 @@ module Sisimai
483
489
 
484
490
  # Feedback-ID: 1.us-west-2.QHuyeCQrGtIIMGKQfVdUhP9hCQR2LglVOrRamBc+Prk=:AmazonSES
485
491
  thing["feedbackid"] = rfc822data["feedback-id"] || ""
486
- thing["toxic"] ||= is_toxic(thing)
487
492
 
488
493
  listoffact << Sisimai::Fact.new(thing)
489
494
  end
490
495
  return listoffact
491
496
  end
492
497
 
493
- def self.is_toxic(thing = nil)
494
- return false unless thing
495
- cr = thing['reason'] || 'undefined'
496
- cv = thing['replycode'] || ''
497
- cw = thing['deliverystatus'] || ''
498
-
499
- # 1. Hard bounces or some soft bounces with a permanent error.
500
- # 1-1. Hard bounce: UserUnknown, HostUnknown, HasMoved, NotAccept
501
- # 1-2. Almost hard bounce: Suspend, Suppressed
502
- return false if cv.start_with?('4') || cw.start_with?('4')
503
- return true if %w[userunknown hostunknown hasmoved notaccept suspend suppressed].any? { |a| cr == a }
504
-
505
- if %w[mailboxfull filtered norelaying].any? { |a| cr == a }
506
- # 2. Several softbounces: MailboxFull, Filtered, NoRelaying
507
- # 2-1. The SMTP command is "RCPT" except "MailboxFull".
508
- # 2-2. The SMTP reply code begins with "5" such as "550".
509
- # 2-3. The SMTP status code is explicit code (not empty, not 5.0.9XX).
510
- # 2-4. The SMTP status code begins with "5." such as "5.1.1".
511
- return true if cr != 'mailboxfull' && thing['command'] == "RCPT"
512
- return true if cv.start_with?('5')
513
- return false if Sisimai::SMTP::Status.is_explicit(cw) == false
514
- return true if cw.start_with?('5.')
515
-
516
- elsif cr == 'feedback'
517
- # 3. Feedback Loop
518
- # 3-1. The Feedback Type is any of "abuse", "fraud", "opt-out"
519
- return true if %w[abuse fraud opt-out].any? { |a| thing['feedbacktype'] == a }
520
- end
521
- return false
498
+ # Create message token from addresser and recipient
499
+ # @param [String] addr1 Sender address
500
+ # @param [String] addr2 Recipient address
501
+ # @param [Integer] epoch Machine time of the email bounce
502
+ # @return [String] Message token(MD5 hex digest) or blank(failed to create token)
503
+ # @see http://en.wikipedia.org/wiki/ASCII
504
+ def self.token(addr1, addr2, epoch)
505
+ return "" if addr1.is_a?(::String) == false || addr2.is_a?(::String) == false
506
+ return "" if addr1.empty? || addr2.empty? || epoch.is_a?(Integer) == false
507
+
508
+ # Format: STX(0x02) Sender-Address RS(0x1e) Recipient-Address ETX(0x03)
509
+ require 'digest/sha1'
510
+ return Digest::SHA1.hexdigest(sprintf("\x02%s\x1e%s\x1e%d\x03", addr1.downcase, addr2.downcase, epoch))
522
511
  end
523
512
 
524
513
  # Convert from Sisimai::Fact object to a Hash
@@ -536,6 +525,7 @@ module Sisimai
536
525
  stringdata.each { |e| v[e] = self.send(e.to_sym) || '' }
537
526
  v['hardbounce'] = self.hardbounce
538
527
  v['toxic'] = self.toxic
528
+ v['bogus'] = self.bogus
539
529
  v['addresser'] = self.addresser.address
540
530
  v['recipient'] = self.recipient.address
541
531
  v['timestamp'] = self.timestamp.to_time.to_i