sisimai 5.1.0-java → 5.2.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/rake-test.yml +1 -1
  3. data/ChangeLog.md +102 -0
  4. data/Makefile +4 -2
  5. data/README-JA.md +23 -16
  6. data/README.md +22 -15
  7. data/lib/sisimai/arf.rb +121 -210
  8. data/lib/sisimai/fact.rb +208 -158
  9. data/lib/sisimai/lda.rb +98 -0
  10. data/lib/sisimai/lhost/activehunter.rb +1 -1
  11. data/lib/sisimai/lhost/amazonses.rb +185 -301
  12. data/lib/sisimai/lhost/apachejames.rb +48 -51
  13. data/lib/sisimai/lhost/biglobe.rb +1 -2
  14. data/lib/sisimai/lhost/courier.rb +10 -8
  15. data/lib/sisimai/lhost/domino.rb +25 -25
  16. data/lib/sisimai/lhost/dragonfly.rb +3 -4
  17. data/lib/sisimai/lhost/einsundeins.rb +3 -4
  18. data/lib/sisimai/lhost/exchange2003.rb +6 -8
  19. data/lib/sisimai/lhost/exchange2007.rb +111 -101
  20. data/lib/sisimai/lhost/exim.rb +232 -242
  21. data/lib/sisimai/lhost/ezweb.rb +43 -51
  22. data/lib/sisimai/lhost/fml.rb +2 -3
  23. data/lib/sisimai/lhost/gmail.rb +32 -28
  24. data/lib/sisimai/lhost/gmx.rb +4 -16
  25. data/lib/sisimai/lhost/googlegroups.rb +9 -8
  26. data/lib/sisimai/lhost/googleworkspace.rb +94 -0
  27. data/lib/sisimai/lhost/imailserver.rb +7 -16
  28. data/lib/sisimai/lhost/interscanmss.rb +1 -1
  29. data/lib/sisimai/lhost/kddi.rb +3 -4
  30. data/lib/sisimai/lhost/mailfoundry.rb +2 -5
  31. data/lib/sisimai/lhost/mailmarshalsmtp.rb +1 -2
  32. data/lib/sisimai/lhost/messagingserver.rb +14 -13
  33. data/lib/sisimai/lhost/mfilter.rb +4 -3
  34. data/lib/sisimai/lhost/notes.rb +2 -4
  35. data/lib/sisimai/lhost/opensmtpd.rb +2 -2
  36. data/lib/sisimai/lhost/postfix.rb +25 -27
  37. data/lib/sisimai/lhost/qmail.rb +130 -106
  38. data/lib/sisimai/lhost/sendmail.rb +19 -18
  39. data/lib/sisimai/lhost/v5sendmail.rb +88 -60
  40. data/lib/sisimai/lhost/verizon.rb +2 -2
  41. data/lib/sisimai/lhost/x1.rb +1 -1
  42. data/lib/sisimai/lhost/x2.rb +1 -2
  43. data/lib/sisimai/lhost/x3.rb +2 -2
  44. data/lib/sisimai/lhost/x6.rb +1 -1
  45. data/lib/sisimai/lhost/zoho.rb +2 -2
  46. data/lib/sisimai/lhost.rb +18 -21
  47. data/lib/sisimai/message.rb +93 -146
  48. data/lib/sisimai/order.rb +21 -77
  49. data/lib/sisimai/reason/authfailure.rb +1 -4
  50. data/lib/sisimai/reason/badreputation.rb +2 -2
  51. data/lib/sisimai/reason/blocked.rb +7 -10
  52. data/lib/sisimai/reason/contenterror.rb +7 -1
  53. data/lib/sisimai/reason/exceedlimit.rb +1 -4
  54. data/lib/sisimai/reason/failedstarttls.rb +42 -0
  55. data/lib/sisimai/reason/filtered.rb +5 -4
  56. data/lib/sisimai/reason/hasmoved.rb +1 -2
  57. data/lib/sisimai/reason/hostunknown.rb +3 -3
  58. data/lib/sisimai/reason/mailboxfull.rb +2 -4
  59. data/lib/sisimai/reason/mailererror.rb +1 -2
  60. data/lib/sisimai/reason/mesgtoobig.rb +2 -4
  61. data/lib/sisimai/reason/norelaying.rb +2 -3
  62. data/lib/sisimai/reason/notaccept.rb +2 -3
  63. data/lib/sisimai/reason/notcompliantrfc.rb +10 -4
  64. data/lib/sisimai/reason/rejected.rb +1 -1
  65. data/lib/sisimai/reason/requireptr.rb +2 -2
  66. data/lib/sisimai/reason/securityerror.rb +1 -3
  67. data/lib/sisimai/reason/spamdetected.rb +6 -8
  68. data/lib/sisimai/reason/speeding.rb +1 -2
  69. data/lib/sisimai/reason/suppressed.rb +36 -0
  70. data/lib/sisimai/reason/suspend.rb +1 -3
  71. data/lib/sisimai/reason/systemerror.rb +5 -0
  72. data/lib/sisimai/reason/toomanyconn.rb +1 -2
  73. data/lib/sisimai/reason/userunknown.rb +1 -1
  74. data/lib/sisimai/reason/virusdetected.rb +5 -6
  75. data/lib/sisimai/reason.rb +77 -73
  76. data/lib/sisimai/rfc1123.rb +152 -0
  77. data/lib/sisimai/rfc1894.rb +102 -62
  78. data/lib/sisimai/rfc2045.rb +2 -1
  79. data/lib/sisimai/rfc3464/thirdparty.rb +102 -0
  80. data/lib/sisimai/rfc3464.rb +222 -343
  81. data/lib/sisimai/rfc3834.rb +1 -1
  82. data/lib/sisimai/rfc5322.rb +7 -17
  83. data/lib/sisimai/rfc791.rb +69 -0
  84. data/lib/sisimai/rhost/aol.rb +36 -0
  85. data/lib/sisimai/rhost/apple.rb +5 -2
  86. data/lib/sisimai/rhost/cox.rb +3 -2
  87. data/lib/sisimai/rhost/facebook.rb +100 -0
  88. data/lib/sisimai/rhost/franceptt.rb +3 -2
  89. data/lib/sisimai/rhost/godaddy.rb +3 -2
  90. data/lib/sisimai/rhost/google.rb +19 -17
  91. data/lib/sisimai/rhost/gsuite.rb +42 -0
  92. data/lib/sisimai/rhost/iua.rb +3 -3
  93. data/lib/sisimai/rhost/kddi.rb +3 -2
  94. data/lib/sisimai/rhost/messagelabs.rb +37 -0
  95. data/lib/sisimai/rhost/microsoft.rb +56 -49
  96. data/lib/sisimai/rhost/mimecast.rb +29 -27
  97. data/lib/sisimai/rhost/nttdocomo.rb +4 -3
  98. data/lib/sisimai/rhost/outlook.rb +36 -0
  99. data/lib/sisimai/rhost/spectrum.rb +3 -2
  100. data/lib/sisimai/rhost/tencent.rb +3 -2
  101. data/lib/sisimai/rhost/yahooinc.rb +4 -3
  102. data/lib/sisimai/rhost.rb +69 -39
  103. data/lib/sisimai/smtp/command.rb +31 -21
  104. data/lib/sisimai/smtp/failure.rb +103 -0
  105. data/lib/sisimai/smtp/reply.rb +29 -25
  106. data/lib/sisimai/smtp/status.rb +36 -19
  107. data/lib/sisimai/smtp/transcript.rb +15 -15
  108. data/lib/sisimai/string.rb +0 -46
  109. data/lib/sisimai/version.rb +1 -1
  110. data/set-of-emails/maildir/bsd/lhost-postfix-30.eml +81 -81
  111. data/set-of-emails/maildir/bsd/{lhost-aol-03.eml → rhost-aol-03.eml} +1264 -1264
  112. data/set-of-emails/maildir/bsd/{lhost-aol-04.eml → rhost-aol-04.eml} +1260 -1260
  113. data/set-of-emails/maildir/bsd/{lhost-aol-05.eml → rhost-aol-05.eml} +105 -105
  114. data/set-of-emails/maildir/bsd/{lhost-aol-06.eml → rhost-aol-06.eml} +105 -105
  115. data/set-of-emails/maildir/bsd/rhost-gsuite-01.eml +189 -0
  116. data/set-of-emails/maildir/bsd/rhost-gsuite-02.eml +180 -0
  117. data/set-of-emails/maildir/bsd/rhost-gsuite-03.eml +251 -0
  118. data/set-of-emails/maildir/bsd/rhost-gsuite-04.eml +211 -0
  119. data/set-of-emails/maildir/bsd/rhost-gsuite-05.eml +226 -0
  120. data/set-of-emails/maildir/bsd/rhost-gsuite-06.eml +257 -0
  121. data/set-of-emails/maildir/bsd/rhost-gsuite-07.eml +289 -0
  122. data/set-of-emails/maildir/bsd/rhost-gsuite-08.eml +231 -0
  123. data/set-of-emails/maildir/bsd/rhost-gsuite-09.eml +231 -0
  124. data/set-of-emails/maildir/bsd/rhost-gsuite-10.eml +254 -0
  125. data/set-of-emails/maildir/bsd/rhost-gsuite-11.eml +228 -0
  126. data/set-of-emails/maildir/bsd/rhost-gsuite-12.eml +271 -0
  127. data/set-of-emails/maildir/bsd/rhost-gsuite-13.eml +261 -0
  128. data/set-of-emails/maildir/bsd/rhost-gsuite-14.eml +273 -0
  129. data/set-of-emails/maildir/bsd/rhost-gsuite-15.eml +229 -0
  130. data/set-of-emails/maildir/bsd/{lhost-messagelabs-01.eml → rhost-messagelabs-01.eml} +93 -93
  131. data/set-of-emails/maildir/bsd/rhost-outlook-01.eml +72 -0
  132. data/set-of-emails/maildir/bsd/rhost-outlook-02.eml +72 -0
  133. data/set-of-emails/maildir/bsd/rhost-outlook-03.eml +72 -0
  134. data/set-of-emails/maildir/bsd/rhost-outlook-04.eml +79 -0
  135. data/set-of-emails/maildir/bsd/rhost-outlook-06.eml +75 -0
  136. data/set-of-emails/maildir/bsd/rhost-outlook-07.eml +70 -0
  137. data/set-of-emails/maildir/bsd/rhost-outlook-08.eml +70 -0
  138. data/set-of-emails/maildir/bsd/rhost-outlook-09.eml +56 -0
  139. data/set-of-emails/maildir/tmp/arf-22.eml +49 -0
  140. data/set-of-emails/maildir/tmp/arf-23.eml +49 -0
  141. data/set-of-emails/maildir/tmp/arf-24.eml +50 -0
  142. data/set-of-emails/maildir/tmp/lhost-exim-07.eml +28 -0
  143. metadata +73 -56
  144. data/lib/sisimai/lhost/amavis.rb +0 -163
  145. data/lib/sisimai/lhost/amazonworkmail.rb +0 -127
  146. data/lib/sisimai/lhost/aol.rb +0 -125
  147. data/lib/sisimai/lhost/barracuda.rb +0 -92
  148. data/lib/sisimai/lhost/bigfoot.rb +0 -125
  149. data/lib/sisimai/lhost/facebook.rb +0 -188
  150. data/lib/sisimai/lhost/gsuite.rb +0 -194
  151. data/lib/sisimai/lhost/mailru.rb +0 -214
  152. data/lib/sisimai/lhost/mcafee.rb +0 -109
  153. data/lib/sisimai/lhost/messagelabs.rb +0 -120
  154. data/lib/sisimai/lhost/mxlogic.rb +0 -198
  155. data/lib/sisimai/lhost/office365.rb +0 -252
  156. data/lib/sisimai/lhost/outlook.rb +0 -129
  157. data/lib/sisimai/lhost/powermta.rb +0 -118
  158. data/lib/sisimai/lhost/receivingses.rb +0 -126
  159. data/lib/sisimai/lhost/sendgrid.rb +0 -150
  160. data/lib/sisimai/lhost/surfcontrol.rb +0 -105
  161. data/lib/sisimai/lhost/x4.rb +0 -269
  162. data/lib/sisimai/lhost/x5.rb +0 -112
  163. data/lib/sisimai/lhost/yahoo.rb +0 -102
  164. data/lib/sisimai/lhost/yandex.rb +0 -118
  165. data/lib/sisimai/mda.rb +0 -121
  166. data/lib/sisimai/smtp/error.rb +0 -119
  167. /data/set-of-emails/maildir/bsd/{lhost-googlegroups-15.eml → lhost-googleworkspace-01.eml} +0 -0
  168. /data/set-of-emails/maildir/bsd/{lhost-x4-08.eml → lhost-x2-06.eml} +0 -0
  169. /data/set-of-emails/maildir/bsd/{lhost-gsuite-01.eml → rfc3464-51.eml} +0 -0
  170. /data/set-of-emails/maildir/bsd/{lhost-gsuite-03.eml → rfc3464-52.eml} +0 -0
  171. /data/set-of-emails/maildir/bsd/{lhost-gsuite-04.eml → rfc3464-53.eml} +0 -0
  172. /data/set-of-emails/maildir/bsd/{lhost-gsuite-05.eml → rfc3464-54.eml} +0 -0
  173. /data/set-of-emails/maildir/bsd/{lhost-gsuite-06.eml → rfc3464-55.eml} +0 -0
  174. /data/set-of-emails/maildir/bsd/{lhost-gsuite-07.eml → rfc3464-56.eml} +0 -0
  175. /data/set-of-emails/maildir/bsd/{lhost-gsuite-08.eml → rfc3464-57.eml} +0 -0
  176. /data/set-of-emails/maildir/bsd/{lhost-gsuite-09.eml → rfc3464-58.eml} +0 -0
  177. /data/set-of-emails/maildir/bsd/{lhost-gsuite-10.eml → rfc3464-59.eml} +0 -0
  178. /data/set-of-emails/maildir/bsd/{lhost-gsuite-11.eml → rfc3464-60.eml} +0 -0
  179. /data/set-of-emails/maildir/bsd/{lhost-gsuite-12.eml → rfc3464-61.eml} +0 -0
  180. /data/set-of-emails/maildir/bsd/{lhost-gsuite-13.eml → rfc3464-62.eml} +0 -0
  181. /data/set-of-emails/maildir/bsd/{lhost-gsuite-14.eml → rfc3464-63.eml} +0 -0
  182. /data/set-of-emails/maildir/bsd/{lhost-gsuite-15.eml → rfc3464-64.eml} +0 -0
  183. /data/set-of-emails/maildir/bsd/{lhost-gsuite-02.eml → rfc3464-65.eml} +0 -0
  184. /data/set-of-emails/maildir/bsd/{lhost-aol-01.eml → rhost-aol-01.eml} +0 -0
  185. /data/set-of-emails/maildir/bsd/{lhost-aol-02.eml → rhost-aol-02.eml} +0 -0
  186. /data/set-of-emails/maildir/bsd/{lhost-facebook-03.eml → rhost-facebook-03.eml} +0 -0
  187. /data/set-of-emails/maildir/bsd/{lhost-facebook-04.eml → rhost-facebook-04.eml} +0 -0
  188. /data/set-of-emails/maildir/bsd/{lhost-messagelabs-02.eml → rhost-messagelabs-02.eml} +0 -0
  189. /data/set-of-emails/maildir/bsd/{lhost-messagelabs-03.eml → rhost-messagelabs-03.eml} +0 -0
  190. /data/set-of-emails/maildir/{bsd → tmp}/rfc3464-37.eml +0 -0
  191. /data/set-of-emails/maildir/{bsd → tmp}/rfc3464-38.eml +0 -0
  192. /data/set-of-emails/maildir/{bsd → tmp}/rfc3464-39.eml +0 -0
@@ -20,14 +20,17 @@ module Sisimai
20
20
  Fields5965 = Sisimai::RFC5965.FIELDINDEX.freeze
21
21
  FieldIndex = [Fields1894.flatten, Fields5322.flatten, Fields5965.flatten].flatten.freeze
22
22
  FieldTable = FieldIndex.map { |e| [e.downcase, e] }.to_h.freeze
23
- ReplacesAs = { 'Content-Type' => [%w[message/xdelivery-status message/delivery-status]] }.freeze
24
23
  Boundaries = ['Content-Type: message/rfc822', 'Content-Type: text/rfc822-headers'].freeze
24
+ ReplacesAs = {
25
+ "Content-Type" => [
26
+ %w[message/xdelivery-status message/delivery-status],
27
+ %w[message/disposition-notification message/delivery-status],
28
+ ]
29
+ }.freeze
25
30
 
26
31
  # Read an email message and convert to structured format
27
32
  # @param [Hash] argvs Module to be loaded
28
33
  # @options argvs [String] :data Entire email message
29
- # @options argvs [Array] :load User defined MTA module list
30
- # @options argvs [Array] :order The order of MTA modules
31
34
  # @options argvs [Code] :hook Reference to callback method
32
35
  # @return [Sisimai::Message] Structured email data or nil if each
33
36
  # value of the arguments are missing
@@ -37,16 +40,6 @@ module Sisimai
37
40
  thing = { 'from' => '','header' => {}, 'rfc822' => '', 'ds' => [], 'catch' => nil }
38
41
  param = {}
39
42
 
40
- # 0. Load specified MTA modules
41
- [:load, :order].each do |e|
42
- # Order of MTA modules
43
- next unless argvs[e]
44
- next unless argvs[e].is_a? Array
45
- next if argvs[e].empty?
46
- param[e.to_s] = argvs[e]
47
- end
48
- tobeloaded = Sisimai::Message.load(param)
49
-
50
43
  aftersplit = nil
51
44
  beforefact = nil
52
45
  parseagain = 0
@@ -71,7 +64,7 @@ module Sisimai
71
64
  if p1
72
65
  # Delete quoted strings, quote symbols(>)
73
66
  cq = Sisimai::String.sweep(cq[cq.index(':') + 1, cq.size])
74
- aftersplit[2] = aftersplit[2].gsub(/^[>]+[ ]/, '').gsub(/^[>]$/, '')
67
+ aftersplit[2] = aftersplit[2].gsub(/^[>][ ]/, '').gsub(/^[>]$/, '')
75
68
  end
76
69
  thing['header']['subject'] = cq
77
70
  end
@@ -81,7 +74,6 @@ module Sisimai
81
74
  'hook' => argvs[:hook] || nil,
82
75
  'mail' => thing,
83
76
  'body' => aftersplit[2],
84
- 'tobeloaded' => tobeloaded,
85
77
  'tryonfirst' => Sisimai::Order.make(thing['header']['subject'])
86
78
  }
87
79
  break if beforefact = Sisimai::Message.sift(param)
@@ -91,7 +83,7 @@ module Sisimai
91
83
  # There is a bounce message inside of mutipart/*, try to sift the first message/rfc822
92
84
  # part as a entire message body again.
93
85
  parseagain += 1
94
- email = Sisimai::RFC5322.part(aftersplit[2], Boundaries, true).pop.sub(/\A[\r\n\s]+/, '')
86
+ email = Sisimai::RFC5322.part(aftersplit[2], Boundaries, true).pop.sub(/\A\s+/, '')
95
87
  break unless email.size > 128
96
88
  end
97
89
  return nil unless beforefact
@@ -106,46 +98,7 @@ module Sisimai
106
98
  return thing
107
99
  end
108
100
 
109
- # Load MTA modules which specified at 'order' and 'load' in the argument
110
- # @param [Hash] argvs Module information to be loaded
111
- # @options argvs [Array] load User defined MTA module list
112
- # @options argvs [Array] order The order of MTA modules
113
- # @return [Array] Module list
114
- # @since v4.20.0
115
- def load(argvs)
116
- modulelist = []
117
- tobeloaded = []
118
-
119
- %w[load order].each do |e|
120
- # The order of MTA modules specified by user
121
- next unless argvs[e]
122
- next unless argvs[e].is_a? Array
123
- next if argvs[e].empty?
124
-
125
- modulelist += argvs['order'] if e == 'order'
126
- next unless e == 'load'
127
-
128
- # Load user defined MTA module
129
- argvs['load'].each do |v|
130
- # Load user defined MTA module
131
- begin
132
- require v.to_s.gsub('::', '/').downcase
133
- rescue LoadError
134
- warn ' ***warning: Failed to load ' << v
135
- next
136
- end
137
- tobeloaded << v
138
- end
139
- end
140
-
141
- while e = modulelist.shift do
142
- # Append the custom order of MTA modules
143
- next if tobeloaded.index(e)
144
- tobeloaded << e
145
- end
146
-
147
- return tobeloaded
148
- end
101
+ def load(argvs); ' ***warning: Sisimai::Message.load will be removed at v5.3.0'; return []; end
149
102
 
150
103
  # Divide email data up headers and a body part.
151
104
  # @param [String] email Email data
@@ -199,20 +152,20 @@ module Sisimai
199
152
  # https://gist.github.com/xtetsuji/b080e1f5551d17242f6415aba8a00239
200
153
  headermaps = { 'subject' => '' }
201
154
  receivedby = []
202
- argv0.scan(/^([\w-]+):[ ]*(.*?)\n(?![\s\t])/m) { |e| headermaps[e[0].downcase] = e[1] }
155
+ argv0.scan(/^([\w-]+):[ ]*(.*?)\n(?!\s)/m) { |e| headermaps[e[0].downcase] = e[1] }
203
156
  headermaps.delete('received')
204
- headermaps.each_key { |e| headermaps[e].gsub!(/\n[\s\t]+/, ' ') }
157
+ headermaps.each_key { |e| headermaps[e].gsub!(/\n\s+/, ' ') }
205
158
 
206
159
  if argv0.include?('Received:')
207
160
  # Capture values of each Received: header
208
- re = argv0.scan(/^Received:[ ]*(.*?)\n(?![\s\t])/m).flatten
161
+ re = argv0.scan(/^Received:[ ]*(.*?)\n(?!\s)/m).flatten
209
162
  re.each do |e|
210
163
  # 1. Exclude the Received header including "(qmail ** invoked from network)".
211
164
  # 2. Convert all consecutive spaces and line breaks into a single space character.
212
165
  next if e.include?(' invoked by uid')
213
166
  next if e.include?(' invoked from network')
214
167
 
215
- e.gsub!(/\n[\s\t]+/, ' ')
168
+ e.gsub!(/\n\s+/, ' ')
216
169
  e.squeeze!("\n\t ")
217
170
  receivedby << e
218
171
  end
@@ -252,93 +205,97 @@ module Sisimai
252
205
  return '' if argv0.empty?
253
206
 
254
207
  email = ''
255
- argv0.split("\n").each do |e|
208
+ lines = argv0.split("\n")
209
+ index = -1
210
+ lines.each do |e|
256
211
  # Find and tidy up fields defined in RFC5322, RFC1894, and RFC5965
257
212
  # 1. Find a field label defined in RFC5322, RFC1894, or RFC5965 from this line
258
- p0 = e.index(':') || 0
213
+ p0 = e.index(':') || -1
259
214
  cf = e.downcase[0, p0]
215
+ fn = FieldTable[cf] || ''
260
216
 
261
- unless FieldTable.has_key?(cf)
217
+ index += 1
218
+ if fn == ''
219
+ # There is neither ":" character nor the field listed in $FieldTable
262
220
  email << e + "\n"
263
221
  next
264
222
  end
265
223
 
266
- # 2. There is a field label defined in RFC5322, RFC1894, or RFC5965 from this line.
267
- # Code below replaces the field name with a valid name listed in @fieldindex when
268
- # the field name does not match with a valid name.
269
- # - Before: Message-id: <...>
270
- # - After: Message-Id: <...>
271
- fieldlabel = FieldTable[cf]
272
- substring0 = e[0, p0]
273
- e[0, p0] = fieldlabel unless substring0.empty?
274
-
275
- # 3. There is no " " (space character) immediately after ":"
276
- # - before: Content-Type:text/plain
277
- # - After: Content-Type: text/plain
278
- substring0 = e[p0 + 1, 1]
279
- e[p0, 1] = ': ' if substring0 != ' '
280
-
281
- # 4. Remove redundant space characters after ":"
282
- while true
283
- # - Before: Message-Id: <...>
284
- # - After: Message-Id: <...>
285
- break unless p0 + 2 < e.size
286
- break unless e[p0 + 2, 1] == ' '
287
- e[p0 + 2, 1] = ''
288
- end
289
-
290
- # 5. Tidy up a sub type of each field defined in RFC1894 such as Reporting-MTA: DNS;...
291
- p1 = e.index(';') || -1
224
+ # 2. Tidy up a sub type of each field defined in RFC1894 such as Reporting-MTA: DNS;...
225
+ ab = []
226
+ bf = e[p0 + 1, e.size - p0 - 1]
227
+ p1 = bf.index(';')
292
228
  while true
293
229
  # Such as Diagnostic-Code, Remote-MTA, and so on
294
230
  # - Before: Diagnostic-Code: SMTP;550 User unknown
295
231
  # - After: Diagnostic-Code: smtp; 550 User unknown
296
- break unless p1 > p0
297
- break unless ['Content-Type'].concat(Fields1894).any? { |a| a.start_with?(fieldlabel) }
232
+ break unless ['Content-Type'].concat(Fields1894).any? { |a| a == fn }
298
233
 
299
- substring0 = e[p0 + 2, p1 - p0 - 1]
300
- e[p0 + 2, substring0.size] = substring0.downcase + ' '
234
+ if p1
235
+ # The field including one or more ";"
236
+ bf.split(';').each do |f|
237
+ # 2-1. Trim leading and trailing space characters from the current buffer
238
+ f.strip!
239
+ ps = ''
240
+
241
+ # 2-2. Convert some parameters to the lower-cased string
242
+ while true
243
+ # For example,
244
+ # - Content-Type: Message/delivery-status => message/delivery-status
245
+ # - Content-Type: Charset=UTF8 => charset=utf8
246
+ # - Reporting-MTA: DNS; ... => dns
247
+ # - Final-Recipient: RFC822; ... => rfc822
248
+ break if f.include?(' ')
249
+
250
+ p2 = f.index('=')
251
+ if p2
252
+ # charset=, boundary=, and other pairs divided by "="
253
+ ps = f[0, p2].downcase
254
+ f[0, p2] = ps
255
+ end
256
+ f.downcase! unless ps == 'boundary'
257
+ break
258
+ end
259
+ ab << f
260
+ end
261
+
262
+ while true
263
+ # Diagnostic-Code: x-unix;
264
+ # /var/email/kijitora/Maildir/tmp/1000000000.A000000B00000.neko22:
265
+ # Disk quota exceeded
266
+ break unless fn == 'Diagnostic-Code'
267
+ break unless ab.size == 1
268
+ break unless lines[index + 1].start_with?(' ')
269
+
270
+ ab << ''
271
+ break
272
+ end
273
+ bf = ab.join('; ') # Insert " " (space characer) immediately after ";"
274
+ ab = []
275
+
276
+ else
277
+ # There is no ";" in the field
278
+ break if fn.end_with?('-Date') # Arrival-Date, Last-Attempt-Date
279
+ break if fn.end_with?('-Message-ID') # X-Original-Message-ID
280
+ bf.downcase!
281
+ end
301
282
  break
302
283
  end
303
284
 
304
- # 6. Remove redundant space characters after ";"
305
- while true
306
- # - Before: Diagnostic-Code: SMTP; 550 User unknown
307
- # - After: Diagnostic-Code: SMTP; 550 User unknown
308
- break unless p1 + 2 < e.size
309
- break unless e[p1 + 2, 1] == ' '
310
- e[p1 + 2, 1] = ''
311
- end
312
-
313
- # 7. Tidy up a value, and a parameter of Content-Type: field
314
- while true
285
+ # 3. Tidy up a value, and a parameter of Content-Type: field
286
+ if ReplacesAs.has_key?(fn)
315
287
  # Replace the value of "Content-Type" field
316
- break unless ReplacesAs.has_key?(fieldlabel)
317
- p2 = 0
318
-
319
- ReplacesAs[fieldlabel].each do |f|
320
- # Content-Type: message/xdelivery-status
321
- p2 = e.index(f[0]) || -1
322
- next unless p2 > 1
323
-
324
- e[p2, f[0].size] = f[1]
325
- p1 = e.index(';')
326
- break
288
+ ReplacesAs[fn].each do |f|
289
+ # - Before: Content-Type: message/xdelivery-status; ...
290
+ # - After: Content-Type: message/delivery-status; ...
291
+ p1 = bf.index(f[0]) || next
292
+ bf[p1, f[0].size] = f[1]
327
293
  end
328
-
329
- # A parameter name of Content-Type field should be a lower-cased string
330
- # - Before: Content-Type: text/plain; CharSet=ascii; Boundary=...
331
- # - After: Content-Type: text/plain; charset=ascii; boundary=...
332
- break unless fieldlabel == 'Content-Type'
333
- p2 = e.index('=') || -1
334
- break unless p2 > 0
335
- break unless p2 > p1
336
-
337
- substring0 = e[p1 + 2, p2 - p1 - 2]
338
- e[p1 + 2, p2 - p1 - 2] = substring0.downcase
339
- break
340
294
  end
341
- email << e + "\n"
295
+
296
+ # 4. Remove redundant space characters
297
+ bf = bf.squeeze(' ').strip
298
+ email << sprintf("%s: %s\n", fn, bf)
342
299
  end
343
300
 
344
301
  email << "\n" unless email.end_with?("\n\n")
@@ -354,7 +311,6 @@ module Sisimai
354
311
  # @param options mail [Array] ds Delivery status list(decoded data)
355
312
  # @param options argvs [String] body Email message body
356
313
  # @param options argvs [Array] tryonfirst MTA module list to load on first
357
- # @param options argvs [Array] tobeloaded User defined MTA module list
358
314
  # @return [Hash] Decoded and structured bounce mails
359
315
  def sift(argvs)
360
316
  return nil unless argvs['mail']
@@ -416,21 +372,11 @@ module Sisimai
416
372
 
417
373
  catch :DECODER do
418
374
  while true
419
- # 1. User-Defined Module
420
- # 2. MTA Module Candidates to be tried on first
421
- # 3. Sisimai::Lhost::*
422
- # 4. Sisimai::RFC3464
423
- # 5. Sisimai::ARF
424
- # 6. Sisimai::RFC3834
425
- while r = argvs['tobeloaded'].shift do
426
- # Call user defined MTA modules
427
- next if haveloaded[r]
428
- havesifted = Module.const_get(r).inquire(mailheader, bodystring)
429
- haveloaded[r] = true
430
- modulename = r
431
- throw :DECODER if havesifted
432
- end
433
-
375
+ # 1. MTA Module Candidates to be tried on first
376
+ # 2. Sisimai::Lhost::*
377
+ # 3. Sisimai::RFC3464
378
+ # 4. Sisimai::ARF
379
+ # 5. Sisimai::RFC3834
434
380
  [argvs['tryonfirst'], DefaultSet].flatten.each do |r|
435
381
  # Try MTA module candidates
436
382
  next if haveloaded[r]
@@ -453,6 +399,7 @@ module Sisimai
453
399
  # Feedback Loop message
454
400
  require 'sisimai/arf'
455
401
  havesifted = Sisimai::ARF.inquire(mailheader, bodystring)
402
+ modulename = "ARF"
456
403
  throw :DECODER if havesifted
457
404
  end
458
405
 
@@ -472,8 +419,8 @@ module Sisimai
472
419
  havesifted['catch'] = havecaught
473
420
  modulename = modulename.sub(/\A.+::/, '')
474
421
  havesifted['ds'].each do |e|
475
- e['agent'] = modulename unless e['agent']
476
- e.each_key { |a| e[a] ||= '' } # Replace nil with ""
422
+ e["agent"] = modulename if e["agent"].nil? || e["agent"].empty?
423
+ e.each_key { |a| e[a] = "" if e[a].nil? } # Replace nil with ""
477
424
  end
478
425
  return havesifted
479
426
  end
data/lib/sisimai/order.rb CHANGED
@@ -7,62 +7,42 @@ module Sisimai
7
7
  # There are another patterns in the value of "Subject:" header of a bounce mail generated by the
8
8
  # following MTA/ESP modules
9
9
  OrderE0 = [
10
- 'Sisimai::Lhost::MailRu',
11
- 'Sisimai::Lhost::Yandex',
12
10
  'Sisimai::Lhost::Exim',
13
11
  'Sisimai::Lhost::Sendmail',
14
- 'Sisimai::Lhost::Aol',
15
- 'Sisimai::Lhost::Office365',
16
12
  'Sisimai::Lhost::Exchange2007',
17
13
  'Sisimai::Lhost::Exchange2003',
18
- 'Sisimai::Lhost::AmazonWorkMail',
19
14
  'Sisimai::Lhost::AmazonSES',
20
- 'Sisimai::Lhost::Barracuda',
21
15
  'Sisimai::Lhost::InterScanMSS',
22
16
  'Sisimai::Lhost::KDDI',
23
- 'Sisimai::Lhost::SurfControl',
24
17
  'Sisimai::Lhost::Verizon',
25
18
  'Sisimai::Lhost::ApacheJames',
26
19
  'Sisimai::Lhost::X2',
27
- 'Sisimai::Lhost::X5',
28
20
  'Sisimai::Lhost::FML',
29
21
  ].freeze
30
22
 
31
23
  # Fallback list: The following MTA/ESP modules is not listed OrderE0
32
24
  OrderE1 = [
33
25
  'Sisimai::Lhost::Postfix',
34
- 'Sisimai::Lhost::GSuite',
35
- 'Sisimai::Lhost::Yahoo',
36
- 'Sisimai::Lhost::Outlook',
37
- 'Sisimai::Lhost::GMX',
26
+ 'Sisimai::Lhost::OpenSMTPD',
27
+ 'Sisimai::Lhost::Courier',
28
+ 'Sisimai::Lhost::Qmail',
38
29
  'Sisimai::Lhost::MessagingServer',
39
- 'Sisimai::Lhost::EinsUndEins',
30
+ 'Sisimai::Lhost::MailMarshalSMTP',
31
+ 'Sisimai::Lhost::GoogleWorkspace',
40
32
  'Sisimai::Lhost::Domino',
41
33
  'Sisimai::Lhost::Notes',
42
- 'Sisimai::Lhost::Qmail',
43
- 'Sisimai::Lhost::Courier',
44
- 'Sisimai::Lhost::OpenSMTPD',
34
+ 'Sisimai::Lhost::Gmail',
45
35
  'Sisimai::Lhost::Zoho',
46
- 'Sisimai::Lhost::MessageLabs',
47
- 'Sisimai::Lhost::MXLogic',
36
+ 'Sisimai::Lhost::GMX',
37
+ 'Sisimai::Lhost::GoogleGroups',
48
38
  'Sisimai::Lhost::MailFoundry',
49
- 'Sisimai::Lhost::McAfee',
50
39
  'Sisimai::Lhost::V5sendmail',
51
- 'Sisimai::Lhost::MFILTER',
52
- 'Sisimai::Lhost::SendGrid',
53
- 'Sisimai::Lhost::ReceivingSES',
54
- 'Sisimai::Lhost::Amavis',
55
- 'Sisimai::Lhost::PowerMTA',
56
- 'Sisimai::Lhost::GoogleGroups',
57
- 'Sisimai::Lhost::Gmail',
58
- 'Sisimai::Lhost::EZweb',
59
40
  'Sisimai::Lhost::IMailServer',
60
- 'Sisimai::Lhost::MailMarshalSMTP',
41
+ 'Sisimai::Lhost::MFILTER',
61
42
  'Sisimai::Lhost::Activehunter',
62
- 'Sisimai::Lhost::Bigfoot',
43
+ 'Sisimai::Lhost::EZweb',
63
44
  'Sisimai::Lhost::Biglobe',
64
- 'Sisimai::Lhost::Facebook',
65
- 'Sisimai::Lhost::X4',
45
+ 'Sisimai::Lhost::EinsUndEins',
66
46
  'Sisimai::Lhost::X1',
67
47
  'Sisimai::Lhost::X3',
68
48
  'Sisimai::Lhost::X6',
@@ -78,17 +58,12 @@ module Sisimai
78
58
  'complaint-about' => ['Sisimai::ARF'],
79
59
  'delivery-failure' => ['Sisimai::Lhost::Domino', 'Sisimai::Lhost::X2'],
80
60
  'delivery-notification' => ['Sisimai::Lhost::MessagingServer'],
81
- 'delivery-report' => ['Sisimai::Lhost::PowerMTA'],
82
61
  'delivery-status' => [
83
- 'Sisimai::Lhost::GSuite',
84
- 'Sisimai::Lhost::GoogleGroups',
85
- 'Sisimai::Lhost::Outlook',
86
- 'Sisimai::Lhost::McAfee',
87
62
  'Sisimai::Lhost::OpenSMTPD',
88
- 'Sisimai::Lhost::AmazonSES',
89
- 'Sisimai::Lhost::AmazonWorkMail',
90
- 'Sisimai::Lhost::ReceivingSES',
63
+ 'Sisimai::Lhost::GoogleWorkspace',
91
64
  'Sisimai::Lhost::Gmail',
65
+ 'Sisimai::Lhost::GoogleGroups',
66
+ 'Sisimai::Lhost::AmazonSES',
92
67
  'Sisimai::Lhost::X3',
93
68
  ],
94
69
  'dmarc-ietf-dmarc' => ['Sisimai::ARF'],
@@ -96,66 +71,39 @@ module Sisimai
96
71
  'failed-delivery' => ['Sisimai::Lhost::X2'],
97
72
  'failure-delivery' => ['Sisimai::Lhost::X2'],
98
73
  'failure-notice' => [
99
- 'Sisimai::Lhost::Yahoo',
100
74
  'Sisimai::Lhost::Qmail',
101
75
  'Sisimai::Lhost::MFILTER',
102
76
  'Sisimai::Lhost::Activehunter',
103
- 'Sisimai::Lhost::X4',
104
77
  ],
105
78
  'loop-alert' => ['Sisimai::Lhost::FML'],
106
79
  'mail-delivery' => [
107
80
  'Sisimai::Lhost::Exim',
108
81
  'Sisimai::Lhost::DragonFly',
109
- 'Sisimai::Lhost::MailRu',
110
82
  'Sisimai::Lhost::GMX',
111
- 'Sisimai::Lhost::EinsUndEins',
112
83
  'Sisimai::Lhost::Zoho',
113
- 'Sisimai::Lhost::MessageLabs',
114
- 'Sisimai::Lhost::MXLogic',
84
+ 'Sisimai::Lhost::EinsUndEins',
115
85
  ],
116
86
  'mail-could' => ['Sisimai::Lhost::InterScanMSS'],
117
87
  'mail-failure' => ['Sisimai::Lhost::Exim'],
118
- 'mail-not' => ['Sisimai::Lhost::X4'],
119
88
  'mail-system' => ['Sisimai::Lhost::EZweb'],
120
89
  'message-delivery' => ['Sisimai::Lhost::MailFoundry'],
121
90
  'message-frozen' => ['Sisimai::Lhost::Exim'],
122
- 'message-you' => ['Sisimai::Lhost::Barracuda'],
123
- 'não-entregue' => ['Sisimai::Lhost::Office365'],
124
91
  'non-recapitabile' => ['Sisimai::Lhost::Exchange2007'],
125
92
  'non-remis' => ['Sisimai::Lhost::Exchange2007'],
126
93
  'notice' => ['Sisimai::Lhost::Courier'],
127
- 'onbestelbaar' => ['Sisimai::Lhost::Office365'],
128
- 'permanent-delivery' => ['Sisimai::Lhost::X4'],
129
94
  'postmaster-notify' => ['Sisimai::Lhost::Sendmail'],
130
95
  'returned-mail' => [
131
96
  'Sisimai::Lhost::Sendmail',
132
- 'Sisimai::Lhost::Aol',
133
- 'Sisimai::Lhost::V5sendmail',
134
- 'Sisimai::Lhost::Bigfoot',
135
97
  'Sisimai::Lhost::Biglobe',
98
+ 'Sisimai::Lhost::V5sendmail',
136
99
  'Sisimai::Lhost::X1',
137
100
  ],
138
- 'sorry-your' => ['Sisimai::Lhost::Facebook'],
139
- 'there-was' => ['Sisimai::Lhost::X6'],
140
- 'undeliverable' => [
141
- 'Sisimai::Lhost::Office365',
142
- 'Sisimai::Lhost::Exchange2007',
143
- 'Sisimai::Lhost::Aol',
144
- 'Sisimai::Lhost::Exchange2003',
145
- ],
146
- 'undeliverable-mail' => [
147
- 'Sisimai::Lhost::Amavis',
148
- 'Sisimai::Lhost::MailMarshalSMTP',
149
- 'Sisimai::Lhost::IMailServer',
150
- ],
101
+ 'there-was' => ['Sisimai::Lhost::X6'],
102
+ 'undeliverable' => ['Sisimai::Lhost::Exchange2007', 'Sisimai::Lhost::Exchange2003'],
103
+ 'undeliverable-mail' => ['Sisimai::Lhost::MailMarshalSMTP', 'Sisimai::Lhost::IMailServer'],
151
104
  'undeliverable-message' => ['Sisimai::Lhost::Notes', 'Sisimai::Lhost::Verizon'],
152
- 'undelivered-mail' => [
153
- 'Sisimai::Lhost::Postfix',
154
- 'Sisimai::Lhost::Aol',
155
- 'Sisimai::Lhost::SendGrid',
156
- 'Sisimai::Lhost::Zoho',
157
- ],
158
- 'warning' => ['Sisimai::Lhost::Sendmail', 'Sisimai::Lhost::Exim'],
105
+ 'undelivered-mail' => ['Sisimai::Lhost::Postfix', 'Sisimai::Lhost::Zoho'],
106
+ 'warning' => ['Sisimai::Lhost::Sendmail', 'Sisimai::Lhost::Exim'],
159
107
  }.freeze
160
108
 
161
109
  # @abstract Returns an MTA Order decided by the first word of the "Subject": header
@@ -178,10 +126,6 @@ module Sisimai
178
126
  return Subject[first] || []
179
127
  end
180
128
 
181
- # @abstract Make default order of MTA modules to be loaded
182
- # @return [Array] Default order list of MTA modules
183
- def default; return Sisimai::Lhost.index.map { |e| 'Sisimai::Lhost::' << e }; end
184
-
185
129
  # @abstract Make MTA module list as a spare
186
130
  # @return [Array] Ordered module list
187
131
  def another; return [OrderE0, OrderE1].flatten; end
@@ -52,10 +52,7 @@ module Sisimai
52
52
  return nil if argvs['deliverystatus'].empty?
53
53
  return true if argvs['reason'] == 'authfailure'
54
54
  return true if Sisimai::SMTP::Status.name(argvs['deliverystatus']).to_s == 'authfailure'
55
-
56
- # Check the value of Diagnosic-Code: header with patterns
57
- return true if match(argvs['diagnosticcode'].downcase)
58
- return false
55
+ return match(argvs['diagnosticcode'].downcase)
59
56
  end
60
57
 
61
58
  end
@@ -19,6 +19,7 @@ module Sisimai
19
19
  'has been temporarily rate limited due to ip reputation',
20
20
  'ip/domain reputation problems',
21
21
  'likely suspicious due to the very low reputation',
22
+ 'none/bad reputation', # t-online.de
22
23
  'temporarily deferred due to unexpected volume or user complaints', # Yahoo Inc.
23
24
  "the sending mta's poor reputation",
24
25
  ].freeze
@@ -42,8 +43,7 @@ module Sisimai
42
43
  # @see http://www.ietf.org/rfc/rfc2822.txt
43
44
  def true(argvs)
44
45
  return true if argvs['reason'] == 'badreputation'
45
- return true if match(argvs['diagnosticcode'].downcase)
46
- return false
46
+ return match(argvs['diagnosticcode'].downcase)
47
47
  end
48
48
 
49
49
  end
@@ -39,6 +39,7 @@ module Sisimai
39
39
  'host network not allowed',
40
40
  'hosts with dynamic ip',
41
41
  'invalid ip for sending mail of domain',
42
+ 'is in a black list',
42
43
  'is not allowed to send mail from',
43
44
  'no access from mail server',
44
45
  'no matches to nameserver query',
@@ -59,6 +60,7 @@ module Sisimai
59
60
  'to submit messages to this e-mail system has been rejected',
60
61
  'too many spams from your ip', # free.fr
61
62
  'too many unwanted messages have been sent from the following ip address above',
63
+ 'was blocked by ',
62
64
  'we do not accept mail from dynamic ips', # @mail.ru
63
65
  'you are not allowed to connect',
64
66
  'you are sending spam',
@@ -67,16 +69,19 @@ module Sisimai
67
69
  'your server requires confirmation',
68
70
  ].freeze
69
71
  Pairs = [
72
+ ['(', '@', ':blocked)'],
70
73
  ['access from ip address ', ' blocked'],
71
74
  ['client host ', ' blocked using'],
72
75
  ['connections will not be accepted from ', " because the ip is in spamhaus's list"],
73
76
  ['dnsbl:rbl ', '>_is_blocked'],
74
77
  ['email blocked by ', '.barracudacentral.org'],
75
78
  ['email blocked by ', 'spamhaus'],
79
+ ['host ', ' refused to talk to me: ', ' blocked'],
76
80
  ['ip ', ' is blocked by earthlink'], # Earthlink
77
81
  ['is in an ', 'rbl on '],
78
82
  ['mail server at ', ' is blocked'],
79
83
  ['mail from ',' refused:'],
84
+ ["mail from "," refused, see ", "orbs"],
80
85
  ['message from ', ' rejected based on blacklist'],
81
86
  ['messages from ', ' temporarily deferred due to user complaints'], # Yahoo!
82
87
  ['server ip ', ' listed as abusive'],
@@ -86,15 +91,9 @@ module Sisimai
86
91
  ['the ip', ' is blacklisted'],
87
92
  ['veuillez essayer plus tard. service refused, please try later. ', '103'],
88
93
  ['veuillez essayer plus tard. service refused, please try later. ', '510'],
94
+ ["your access ip", " has been rejected"],
89
95
  ["your sender's ip address is listed at ", '.abuseat.org'],
90
96
  ].freeze
91
- Regex = %r{(?>
92
- [(][^ ]+[@][^ ]+:blocked[)]
93
- |host[ ][^ ]+[ ]refused[ ]to[ ]talk[ ]to[ ]me:[ ]\d+[ ]blocked
94
- |is[ ]in[ ]a[ ]black[ ]list(?:[ ]at[ ][^ ]+[.])?
95
- |was[ ]blocked[ ]by[ ][^ ]+
96
- )
97
- }x.freeze
98
97
 
99
98
  def text; return 'blocked'; end
100
99
  def description; return 'Email rejected due to client IP address or a hostname'; end
@@ -107,7 +106,6 @@ module Sisimai
107
106
  return nil unless argv1
108
107
  return true if Index.any? { |a| argv1.include?(a) }
109
108
  return true if Pairs.any? { |a| Sisimai::String.aligned(argv1, a) }
110
- return true if argv1 =~ Regex
111
109
  return false
112
110
  end
113
111
 
@@ -119,8 +117,7 @@ module Sisimai
119
117
  def true(argvs)
120
118
  return true if argvs['reason'] == 'blocked'
121
119
  return true if Sisimai::SMTP::Status.name(argvs['deliverystatus']).to_s == 'blocked'
122
- return true if match(argvs['diagnosticcode'].downcase)
123
- return false
120
+ return match(argvs['diagnosticcode'].downcase)
124
121
  end
125
122
 
126
123
  end
@@ -38,7 +38,13 @@ module Sisimai
38
38
  # @return [True,False] true: rejected due to content error
39
39
  # false: is not content error
40
40
  # @see http://www.ietf.org/rfc/rfc2822.txt
41
- def true(_argvs); return nil; end
41
+ def true(argvs)
42
+ require 'sisimai/reason/spamdetected'
43
+ return true if argvs["reason"] == "blocked"
44
+ return false if Sisimai::Reason::SpamDetected.true(argvs)
45
+ return true if Sisimai::SMTP::Status.name(argvs["deliverystatus"]).to_s == "contenterror"
46
+ return match(argvs["diagnosticcode"].downcase)
47
+ end
42
48
 
43
49
  end
44
50
  end
@@ -38,10 +38,7 @@ module Sisimai
38
38
  # Status: 5.2.3
39
39
  # Diagnostic-Code: SMTP; 552 5.2.3 Message size exceeds fixed maximum message size
40
40
  return true if Sisimai::SMTP::Status.name(argvs['deliverystatus']).to_s == 'exceedlimit'
41
-
42
- # Check the value of Diagnosic-Code: header with patterns
43
- return true if match(argvs['diagnosticcode'].downcase)
44
- return false
41
+ return match(argvs['diagnosticcode'].downcase)
45
42
  end
46
43
 
47
44
  end