fluent-plugin-sendmail 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/README.md +33 -197
- data/fluent-plugin-sendmail.gemspec +4 -4
- data/lib/fluent/plugin/in_sendmail.rb +102 -96
- data/lib/fluent/plugin/sendmailparser.rb +29 -63
- data/test/data/{data1 → data} +3 -2
- data/test/data/result +6 -0
- data/test/plugin/test_in_sendmail.rb +13 -29
- metadata +9 -11
- data/test/data/data1_bundle_result_expect +0 -1
- data/test/data/data1_unbundle_result_expect +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4706bdce3e130df90437459aedee773affd85751
|
4
|
+
data.tar.gz: 938c015312581cb20418af2a1a131990e0a678b3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d3a73dfd4dfcdd68a5927fb0a6c8af7161d9cb3ac2aa55f9f7109130c6ac2bb9c2e6ea044c38e7ad64d8335bf4b1bbc0442ca03bdd6bb17086419e44a2a851fc
|
7
|
+
data.tar.gz: bfb1709efd63f36f16292c8d1cdbdc5d81f238c185751f59e7c36730875f15dc3f5b20e0826abc5403294a0722b2dbc8b2d849279290ef5b2f9419101a887b3a
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -1,15 +1,16 @@
|
|
1
1
|
# Fluent::Plugin::Sendmail
|
2
2
|
|
3
|
-
Fluentd plugin to
|
3
|
+
Fluentd plugin to merge sender line and receiver line into one json data.
|
4
4
|
|
5
5
|
## Configuration
|
6
6
|
|
7
7
|
```
|
8
8
|
<source>
|
9
9
|
type sendmail
|
10
|
-
path
|
11
|
-
pos_file ./syslog.log.pos
|
10
|
+
path /var/log/maillog
|
12
11
|
tag sendmail
|
12
|
+
queuereturn 60m
|
13
|
+
path_cache_file /tmp/test.dat
|
13
14
|
</source>
|
14
15
|
```
|
15
16
|
|
@@ -29,205 +30,40 @@ Apr 2 00:15:25 mta001 sendmail[32302]: u31FFPtp032300: done; delay=00:00:00, nt
|
|
29
30
|
This plugin emit record like below:
|
30
31
|
|
31
32
|
```
|
32
|
-
2014-01-10 01:00:01 +0900 sendmail:
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
},
|
61
|
-
{
|
62
|
-
"to":[
|
63
|
-
"<sent3@example2.com>",
|
64
|
-
"<sent4@example2.com>"
|
65
|
-
],
|
66
|
-
"delay":"00:00:00",
|
67
|
-
"xdelay":"00:00:00",
|
68
|
-
"mailer":"esmtp",
|
69
|
-
"pri":"245938",
|
70
|
-
"relay":{
|
71
|
-
"ip":"93.184.216.34",
|
72
|
-
"host":null
|
73
|
-
},
|
74
|
-
"dsn":"2.0.0",
|
75
|
-
"stat":"Sent (ok: Message 40279895 accepted)"
|
76
|
-
},
|
77
|
-
{
|
78
|
-
"to":[
|
79
|
-
"<deferred1@example.com>"
|
80
|
-
],
|
81
|
-
"delay":"00:00:00",
|
82
|
-
"xdelay":"00:00:00",
|
83
|
-
"mailer":"esmtp",
|
84
|
-
"pri":"245938",
|
85
|
-
"relay":{
|
86
|
-
"ip":"93.184.216.34",
|
87
|
-
"host":null
|
88
|
-
},
|
89
|
-
"dsn":"2.0.0",
|
90
|
-
"stat":"Sent (ok: Message 40279894 accepted)"
|
91
|
-
}
|
92
|
-
]
|
93
|
-
}
|
94
|
-
```
|
95
|
-
|
96
|
-
### unbundle
|
97
|
-
|
98
|
-
unbundle mode
|
99
|
-
|
100
|
-
```
|
101
|
-
<source>
|
102
|
-
type sendmail
|
103
|
-
path ./syslog.log
|
104
|
-
pos_file ./syslog.log.pos
|
105
|
-
tag sendmail
|
106
|
-
unbundle yes
|
107
|
-
</source>
|
108
|
-
```
|
109
|
-
|
110
|
-
This plugin emit record like below:
|
111
|
-
|
112
|
-
```
|
113
|
-
2014-01-10 01:00:01 +0900 sendmail: {
|
114
|
-
"mta":"mta001",
|
115
|
-
"from":"<grandeur09@gmail.com>",
|
116
|
-
"relay":{
|
117
|
-
"ip":"93.184.216.34",
|
118
|
-
"host":null
|
119
|
-
},
|
120
|
-
"count":"5",
|
121
|
-
"size":"5938",
|
122
|
-
"msgid":"<201604011515.u31FFIAj012911@gmail.com>",
|
123
|
-
"popid":null,
|
124
|
-
"authid":null,
|
125
|
-
"to":"<sent1@example.com>",
|
126
|
-
"stat":"Sent (ok: Message 40279894 accepted)",
|
127
|
-
"dsn":"2.0.0",
|
128
|
-
"delay":null,
|
129
|
-
"xdelay":"00:00:00"
|
130
|
-
}
|
131
|
-
|
132
|
-
2014-01-10 01:00:01 +0900 sendmail: {
|
133
|
-
"mta":"mta001",
|
134
|
-
"from":"<grandeur09@gmail.com>",
|
135
|
-
"relay":{
|
136
|
-
"ip":"93.184.216.34",
|
137
|
-
"host":null
|
138
|
-
},
|
139
|
-
"count":"5",
|
140
|
-
"size":"5938",
|
141
|
-
"msgid":"<201604011515.u31FFIAj012911@gmail.com>",
|
142
|
-
"popid":null,
|
143
|
-
"authid":null,
|
144
|
-
"to":"<sent2@example.com>",
|
145
|
-
"stat":"Sent (ok: Message 40279894 accepted)",
|
146
|
-
"dsn":"2.0.0",
|
147
|
-
"delay":null,
|
148
|
-
"xdelay":"00:00:00"
|
149
|
-
}
|
150
|
-
|
151
|
-
2014-01-10 01:00:01 +0900 sendmail: {
|
152
|
-
"mta":"mta001",
|
153
|
-
"from":"<grandeur09@gmail.com>",
|
154
|
-
"relay":{
|
155
|
-
"ip":"93.184.216.34",
|
156
|
-
"host":null
|
157
|
-
},
|
158
|
-
"count":"5",
|
159
|
-
"size":"5938",
|
160
|
-
"msgid":"<201604011515.u31FFIAj012911@gmail.com>",
|
161
|
-
"popid":null,
|
162
|
-
"authid":null,
|
163
|
-
"to":"<deferred1@example.com>",
|
164
|
-
"stat":"Deferred: 451 4.3.5 Server configuration problem",
|
165
|
-
"dsn":"4.3.5",
|
166
|
-
"delay":"00:00:15",
|
167
|
-
"xdelay":"00:00:15"
|
168
|
-
}
|
169
|
-
|
170
|
-
2014-01-10 01:00:01 +0900 sendmail: {
|
171
|
-
"mta":"mta001",
|
172
|
-
"from":"<grandeur09@gmail.com>",
|
173
|
-
"relay":{
|
174
|
-
"ip":"93.184.216.34",
|
175
|
-
"host":null
|
176
|
-
},
|
177
|
-
"count":"5",
|
178
|
-
"size":"5938",
|
179
|
-
"msgid":"<201604011515.u31FFIAj012911@gmail.com>",
|
180
|
-
"popid":null,
|
181
|
-
"authid":null,
|
182
|
-
"to":"<sent3@example2.com>",
|
183
|
-
"stat":"Sent (ok: Message 40279895 accepted)",
|
184
|
-
"dsn":"2.0.0",
|
185
|
-
"delay":"00:00:00",
|
186
|
-
"xdelay":"00:00:00"
|
187
|
-
}
|
188
|
-
|
189
|
-
2014-01-10 01:00:01 +0900 sendmail: {
|
190
|
-
"mta":"mta001",
|
191
|
-
"from":"<grandeur09@gmail.com>",
|
192
|
-
"relay":{
|
193
|
-
"ip":"93.184.216.34",
|
194
|
-
"host":null
|
195
|
-
},
|
196
|
-
"count":"5",
|
197
|
-
"size":"5938",
|
198
|
-
"msgid":"<201604011515.u31FFIAj012911@gmail.com>",
|
199
|
-
"popid":null,
|
200
|
-
"authid":null,
|
201
|
-
"to":"<sent4@example2.com>",
|
202
|
-
"stat":"Sent (ok: Message 40279895 accepted)",
|
203
|
-
"dsn":"2.0.0",
|
204
|
-
"delay":"00:00:00",
|
205
|
-
"xdelay":"00:00:00"
|
206
|
-
}
|
207
|
-
|
208
|
-
2014-01-10 01:00:01 +0900 sendmail: {
|
209
|
-
"mta":"mta001",
|
210
|
-
"from":"<grandeur09@gmail.com>",
|
211
|
-
"relay":{
|
212
|
-
"ip":"93.184.216.34",
|
213
|
-
"host":null
|
214
|
-
},
|
215
|
-
"count":"5",
|
216
|
-
"size":"5938",
|
217
|
-
"msgid":"<201604011515.u31FFIAj012911@gmail.com>",
|
218
|
-
"popid":null,
|
219
|
-
"authid":null,
|
220
|
-
"to":"<deferred1@example.com>",
|
221
|
-
"stat":"Sent (ok: Message 40279894 accepted)",
|
222
|
-
"dsn":"2.0.0",
|
223
|
-
"delay":"00:00:00",
|
224
|
-
"xdelay":"00:00:00"
|
33
|
+
2014-01-10 01:00:01 +0900 sendmail:
|
34
|
+
{
|
35
|
+
"time":1459523725,
|
36
|
+
"mta":"mta001",
|
37
|
+
"qid":"u31FFPtp032300",
|
38
|
+
"from":"<grandeur09@gmail.com>",
|
39
|
+
"size":"5938",
|
40
|
+
"class":"0",
|
41
|
+
"nrcpts":"7",
|
42
|
+
"msgid":"<201604011515.u31FFIAj012911@gmail.com>",
|
43
|
+
"proto":"ESMTP",
|
44
|
+
"daemon":"MTA",
|
45
|
+
"relay":{
|
46
|
+
"ip":"93.184.216.34",
|
47
|
+
"host":null
|
48
|
+
},
|
49
|
+
"status_canonical":"sent",
|
50
|
+
"to":[
|
51
|
+
"<sent1@example.com>",
|
52
|
+
"<sent2@example.com>"
|
53
|
+
],
|
54
|
+
"delay":"00:00:00",
|
55
|
+
"xdelay":"00:00:00",
|
56
|
+
"mailer":"esmtp",
|
57
|
+
"pri":"245938",
|
58
|
+
"dsn":"2.0.0",
|
59
|
+
"stat":"Sent (ok: Message 40279894 accepted)",
|
60
|
+
"delay_in_sec":0
|
225
61
|
}
|
226
62
|
```
|
227
63
|
|
228
64
|
## TODO
|
229
65
|
|
230
|
-
|
66
|
+
write test for path_cache_file parmeter
|
231
67
|
|
232
68
|
## ChangeLog
|
233
69
|
|
@@ -4,12 +4,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
4
|
|
5
5
|
Gem::Specification.new do |spec|
|
6
6
|
spec.name = "fluent-plugin-sendmail"
|
7
|
-
spec.version = "0.1.
|
8
|
-
spec.authors = ["
|
9
|
-
spec.email = ["
|
7
|
+
spec.version = "0.1.1"
|
8
|
+
spec.authors = ["yudai09"]
|
9
|
+
spec.email = ["grandeur09@gmail.com"]
|
10
10
|
spec.summary = "Fluentd plugin to parse and merge sendmail syslog."
|
11
11
|
spec.description = spec.summary
|
12
|
-
spec.homepage = "https://github.com/
|
12
|
+
spec.homepage = "https://github.com/yudai09/fluent-plugin-sendmail"
|
13
13
|
spec.license = "MIT"
|
14
14
|
|
15
15
|
spec.files = `git ls-files`.split($/)
|
@@ -1,27 +1,56 @@
|
|
1
1
|
# -*- coding: utf-8 -*-
|
2
2
|
class Fluent::SendmailInput < Fluent::TailInput
|
3
|
-
|
4
|
-
|
5
|
-
Fluent::Plugin.register_input('sendmail', self)
|
3
|
+
Fluent::Plugin.register_input("sendmail", self)
|
6
4
|
|
7
|
-
|
8
|
-
|
9
|
-
|
5
|
+
config_param :lrucache_size, :integer, :default => (1024*1024)
|
6
|
+
# sendmail default value of queuereturn is 5d (432000sec)
|
7
|
+
config_param :queuereturn, :time, :default => 432000
|
8
|
+
config_param :path_cache_file, :string, :default => nil
|
10
9
|
|
11
|
-
|
12
|
-
|
10
|
+
require_relative "sendmailparser"
|
11
|
+
require "lru_redux"
|
13
12
|
|
14
|
-
def
|
13
|
+
def configure(conf)
|
15
14
|
super
|
16
15
|
@delivers = LruRedux::ThreadSafeCache.new(@lrucache_size)
|
16
|
+
if @path_cache_file != nil
|
17
|
+
if not File.exists?(@path_cache_file)
|
18
|
+
File.open(@path_cache_file, "w+"){|cache_file|
|
19
|
+
cache_file.puts('{}')
|
20
|
+
}
|
21
|
+
end
|
22
|
+
if not File.readable?(@path_cache_file)
|
23
|
+
raise ConfigError, "cache file exists but not readable."
|
24
|
+
end
|
25
|
+
if not File.writable?(@path_cache_file)
|
26
|
+
raise Fluent::ConfigError, "cache file not writable."
|
27
|
+
end
|
28
|
+
File.open(@path_cache_file, "r") {|cache_file|
|
29
|
+
line = cache_file.read()
|
30
|
+
data = JSON.parse(line)
|
31
|
+
data.each{|k, v|
|
32
|
+
@delivers[k] = SendmailLog.new(v['time'], v['from_line'], v['nrcpts'])
|
33
|
+
}
|
34
|
+
}
|
35
|
+
end
|
17
36
|
end
|
18
37
|
|
19
38
|
def configure_parser(conf)
|
20
39
|
@parser = SendmailParser.new(conf)
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
40
|
+
end
|
41
|
+
|
42
|
+
def shutdown
|
43
|
+
super
|
44
|
+
if @path_cache_file != nil
|
45
|
+
data = {}
|
46
|
+
if @path_cache_file != nil
|
47
|
+
@delivers.each{|k, v|
|
48
|
+
data[k] = v.to_json
|
49
|
+
}
|
50
|
+
end
|
51
|
+
File.open(@path_cache_file, "w+") {|cache_file|
|
52
|
+
cache_file.puts(data)
|
53
|
+
}
|
25
54
|
end
|
26
55
|
end
|
27
56
|
|
@@ -30,55 +59,38 @@ class Fluent::SendmailInput < Fluent::TailInput
|
|
30
59
|
lines.each {|line|
|
31
60
|
begin
|
32
61
|
line.chomp! # remove \n
|
33
|
-
|
34
|
-
|
35
|
-
if logline.nil?
|
62
|
+
record = parse_line(line)
|
63
|
+
if record.nil?
|
36
64
|
next
|
37
65
|
end
|
38
66
|
|
39
|
-
type =
|
40
|
-
mta =
|
41
|
-
qid =
|
42
|
-
time =
|
67
|
+
type = record["type"]
|
68
|
+
mta = record["mta"]
|
69
|
+
qid = record["qid"]
|
70
|
+
time = record["time"]
|
43
71
|
# a qid is not uniq worldwide.
|
44
|
-
# make delivery id uniq even if multiple MTA
|
72
|
+
# make delivery id uniq even if multiple MTA"s log are mixed.
|
45
73
|
deliveryid = mta + qid
|
46
|
-
type =
|
47
|
-
|
74
|
+
type = record["type"]
|
75
|
+
# remove unnecessary key `type"
|
76
|
+
record.delete("type")
|
48
77
|
|
49
78
|
case type
|
50
|
-
when
|
79
|
+
when "from"
|
51
80
|
# new log
|
52
|
-
from = noncommon
|
53
|
-
record = from.record
|
54
|
-
@delivers[deliveryid] = SendmailLog.new(mta, time, record)
|
55
|
-
when :to
|
56
|
-
to = noncommon
|
57
|
-
status = to.status
|
58
|
-
record = noncommon.record
|
59
81
|
if @delivers.has_key?(deliveryid)
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
log_bundled(es, deliveryid, time)
|
73
|
-
end
|
74
|
-
# log.destroy
|
75
|
-
@delivers.delete(qid)
|
76
|
-
end
|
77
|
-
when :deferred
|
78
|
-
if @do_unbundle
|
79
|
-
log_single(es, deliveryid, time, record)
|
80
|
-
end
|
81
|
-
when :other
|
82
|
+
$log.warn "duplicate sender line found. " + line.dump
|
83
|
+
else
|
84
|
+
@delivers[deliveryid] = SendmailLog.new(time, record)
|
85
|
+
end
|
86
|
+
when "to"
|
87
|
+
if @delivers.has_key?(deliveryid)
|
88
|
+
case record["status_canonical"]
|
89
|
+
when "sent", "sent_local", "bounced"
|
90
|
+
sent(es, deliveryid, time, record)
|
91
|
+
when "deferred"
|
92
|
+
queued(es, deliveryid, time, record)
|
93
|
+
when "other"
|
82
94
|
$log.warn "cannot find this kind of delivery status: " + line.dump
|
83
95
|
end
|
84
96
|
else
|
@@ -87,7 +99,7 @@ class Fluent::SendmailInput < Fluent::TailInput
|
|
87
99
|
end
|
88
100
|
rescue
|
89
101
|
$log.warn line.dump, :error=>$!.to_s
|
90
|
-
|
102
|
+
raise
|
91
103
|
end
|
92
104
|
}
|
93
105
|
|
@@ -96,64 +108,58 @@ class Fluent::SendmailInput < Fluent::TailInput
|
|
96
108
|
Fluent::Engine.emit_stream(@tag, es)
|
97
109
|
rescue
|
98
110
|
# ignore errors. Engine shows logs and backtraces.
|
111
|
+
raise
|
99
112
|
end
|
100
113
|
end
|
101
114
|
end
|
102
115
|
|
103
|
-
def
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
116
|
+
def sent(es, deliveryid, time, to_line)
|
117
|
+
from_line = @delivers[deliveryid].from_line
|
118
|
+
record = from_line.merge(to_line)
|
119
|
+
es.add(time, record)
|
120
|
+
nrcpts = @delivers[deliveryid].nrcpts
|
121
|
+
@delivers[deliveryid].nrcpts -= to_line["to"].length
|
122
|
+
# all done
|
123
|
+
if @delivers[deliveryid].nrcpts <= 0
|
124
|
+
@delivers.delete(deliveryid)
|
108
125
|
end
|
109
126
|
end
|
110
127
|
|
111
|
-
def
|
112
|
-
|
113
|
-
|
128
|
+
def queued(es, deliveryid, time, to_line)
|
129
|
+
from_line = @delivers[deliveryid].from_line
|
130
|
+
record = from_line.merge(to_line)
|
131
|
+
delay = to_line["delay_in_sec"]
|
132
|
+
# when a queue is expired, the mail will be bounced
|
133
|
+
if delay >= queuereturn
|
134
|
+
record["canonical_status"] = "bounced"
|
135
|
+
sent(es, deliveryid, time, record)
|
136
|
+
return
|
137
|
+
end
|
138
|
+
record = from_line.merge(to_line)
|
139
|
+
es.add(time, record)
|
114
140
|
end
|
115
|
-
|
116
141
|
end
|
117
142
|
|
118
143
|
class SendmailLog
|
119
|
-
attr_reader :status
|
120
144
|
attr_reader :time
|
121
|
-
|
122
|
-
|
123
|
-
|
145
|
+
attr_reader :from_line
|
146
|
+
attr_accessor :nrcpts
|
147
|
+
|
148
|
+
def initialize(time, from_line, nrcpts=nil)
|
124
149
|
@time = time
|
125
|
-
@
|
126
|
-
|
127
|
-
|
150
|
+
@from_line = from_line
|
151
|
+
if nrcpts == nil
|
152
|
+
@nrcpts = from_line["nrcpts"].to_i
|
153
|
+
else
|
154
|
+
@nrcpts = nrcpts
|
155
|
+
end
|
128
156
|
end
|
129
157
|
|
130
|
-
def
|
158
|
+
def to_json()
|
131
159
|
return {
|
132
|
-
"
|
133
|
-
"
|
134
|
-
"
|
160
|
+
"time" => @time,
|
161
|
+
"from_line" => @from_line,
|
162
|
+
"nrcpts" => @nrcpts,
|
135
163
|
}
|
136
164
|
end
|
137
|
-
|
138
|
-
def record_unbundle(time, record)
|
139
|
-
records_unbundled = []
|
140
|
-
for to in record["to"] do
|
141
|
-
record_single = record.dup
|
142
|
-
record_single["to"] = to
|
143
|
-
records_unbundled.push({
|
144
|
-
"mta" => @mta,
|
145
|
-
"from" => @from,
|
146
|
-
"to" => record_single
|
147
|
-
})
|
148
|
-
end
|
149
|
-
return records_unbundled
|
150
|
-
end
|
151
|
-
|
152
|
-
def dequeued(time, record)
|
153
|
-
@count = @count - record["to"].size
|
154
|
-
@tos.push(record)
|
155
|
-
if @count == 0
|
156
|
-
@status = :all_dequeued
|
157
|
-
end
|
158
|
-
end
|
159
165
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
class SendmailParser
|
2
2
|
def initialize(conf)
|
3
|
-
@base_regexp = /^(?<time>\w+\s+\w+\s+\d+:\d+:\d+) (?<
|
3
|
+
@base_regexp = /^(?<time>\w+\s+\w+\s+\d+:\d+:\d+) (?<mta>[^ ]+) (?<procowner>[^\[]+)\[(?<procid>\d+)\]: (?<qid>[^ ]+): (?<entry>(?<type>[^=]+).+)$/;
|
4
4
|
end
|
5
5
|
|
6
6
|
def parse(value)
|
@@ -9,60 +9,44 @@ class SendmailParser
|
|
9
9
|
# $log.warn "sendmail: pattern not match: #{value.inspect}"
|
10
10
|
return nil
|
11
11
|
end
|
12
|
-
|
13
|
-
logtype = m["type"]
|
14
|
-
entry = m["entry"]
|
15
|
-
mta = m["host"]
|
16
|
-
qid = m["qid"]
|
17
12
|
time = Time.parse(m["time"]).to_i || Fluent::Engine.now.to_i
|
18
|
-
|
19
|
-
logline = {
|
20
|
-
"type" => :from,
|
21
|
-
"mta" => mta,
|
22
|
-
"qid" => qid,
|
13
|
+
record = {
|
23
14
|
"time" => time,
|
24
|
-
"
|
25
|
-
"
|
15
|
+
"mta" => m["mta"],
|
16
|
+
"qid" => m["qid"],
|
17
|
+
"type" => m["type"],
|
26
18
|
}
|
27
|
-
|
28
|
-
case logtype
|
19
|
+
case m["type"]
|
29
20
|
when "from"
|
30
|
-
fromline = self.
|
31
|
-
|
32
|
-
logline["noncommon"] = fromline
|
21
|
+
fromline = self.from_line(m["entry"])
|
22
|
+
record.merge!(fromline)
|
33
23
|
when "to"
|
34
|
-
toline = self.
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
# not match
|
39
|
-
logline = nil
|
24
|
+
toline = self.to_line(m["entry"])
|
25
|
+
record.merge!(toline)
|
26
|
+
else # not match
|
27
|
+
m = nil
|
40
28
|
end
|
41
|
-
|
42
|
-
logline
|
29
|
+
record
|
43
30
|
end
|
44
31
|
|
45
32
|
def to_line(entry)
|
46
33
|
record = {}
|
47
34
|
status = nil
|
48
|
-
|
49
|
-
status = status_parser(entry)
|
50
|
-
|
35
|
+
record["status_canonical"] = status_parser(entry)
|
51
36
|
entry.split(", ").each {|param|
|
52
37
|
key, val = param.split("=")
|
53
38
|
record[key] = val
|
54
39
|
}
|
55
40
|
record["to"] = record["to"].split(",")
|
56
|
-
|
57
41
|
if record.has_key?("relay")
|
58
42
|
record["relay"] = relay_parser(record["relay"])
|
59
43
|
end
|
60
|
-
|
44
|
+
record["delay_in_sec"] = delay_parser(record["delay"])
|
45
|
+
return record
|
61
46
|
end
|
62
47
|
|
63
48
|
def from_line(entry)
|
64
49
|
record = {}
|
65
|
-
|
66
50
|
entry.split(", ").each {|param|
|
67
51
|
key, val = param.split("=")
|
68
52
|
record[key] = val
|
@@ -70,15 +54,7 @@ class SendmailParser
|
|
70
54
|
if record.has_key?("relay")
|
71
55
|
record["relay"] = relay_parser(record["relay"])
|
72
56
|
end
|
73
|
-
|
74
|
-
end
|
75
|
-
|
76
|
-
def to_parser(entry)
|
77
|
-
to_line(entry)
|
78
|
-
end
|
79
|
-
|
80
|
-
def from_parser(entry)
|
81
|
-
from_line(entry)
|
57
|
+
return record
|
82
58
|
end
|
83
59
|
|
84
60
|
def relay_parser(relays)
|
@@ -94,40 +70,30 @@ class SendmailParser
|
|
94
70
|
return {"ip" => relay_ip, "host" => relay_host}
|
95
71
|
end
|
96
72
|
|
97
|
-
def trim_bracket(val)
|
98
|
-
val[1..-2]
|
99
|
-
end
|
100
|
-
|
101
73
|
def status_parser(entry)
|
102
74
|
if entry.include?("stat=Sent")
|
103
75
|
if entry.include?("mailer=local,")
|
104
|
-
return
|
76
|
+
return "sent_local"
|
105
77
|
else
|
106
|
-
return
|
78
|
+
return "sent"
|
107
79
|
end
|
108
80
|
elsif entry.include?("dsn=5.")
|
109
|
-
return
|
81
|
+
return "bounced"
|
110
82
|
elsif entry.include?("stat=Deferred")
|
111
|
-
return
|
83
|
+
return "deferred"
|
112
84
|
else
|
113
|
-
return
|
85
|
+
return "other"
|
114
86
|
end
|
115
87
|
end
|
116
|
-
end
|
117
88
|
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
89
|
+
def delay_parser(delay)
|
90
|
+
/((?<day>[0-9]*)\+)?(?<hms>[0-9]{2,2}:[0-9]{2,2}:[0-9]{2,2})/ =~ delay
|
91
|
+
day = day.to_i
|
92
|
+
dtime = Time.parse(hms)
|
93
|
+
delay = (day * 24 * 60 * 60) + (dtime.hour * 60 * 60) + (dtime.min * 60) + (dtime.sec)
|
122
94
|
end
|
123
|
-
end
|
124
95
|
|
125
|
-
|
126
|
-
|
127
|
-
attr_reader :record
|
128
|
-
def initialize(status, record)
|
129
|
-
@status = status
|
130
|
-
@record = record
|
131
|
-
@record["canonical_status"] = status.to_s
|
96
|
+
def trim_bracket(val)
|
97
|
+
val[1..-2]
|
132
98
|
end
|
133
99
|
end
|
data/test/data/{data1 → data}
RENAMED
@@ -1,8 +1,9 @@
|
|
1
1
|
Apr 2 00:15:25 mta001 sendmail[32300]: u31FFPtp032300: Milter: no active filter
|
2
|
-
Apr 2 00:15:25 mta001 sendmail[32300]: u31FFPtp032300: from=<grandeur09@gmail.com>, size=5938, class=0, nrcpts=
|
2
|
+
Apr 2 00:15:25 mta001 sendmail[32300]: u31FFPtp032300: from=<grandeur09@gmail.com>, size=5938, class=0, nrcpts=7, msgid=<201604011515.u31FFIAj012911@gmail.com>, proto=ESMTP, daemon=MTA, relay=[64.233.187.27]
|
3
3
|
Apr 2 00:15:25 mta001 sendmail[32302]: u31FFPtp032300: SMTP outgoing connect on [192.168.198.81]
|
4
|
-
Apr 2 00:15:25 mta001 sendmail[32302]: u31FFPtp032300: to=<sent1@example.com>,<sent2@example.com>, 00:00:00, xdelay=00:00:00, mailer=esmtp, pri=245938, relay=[93.184.216.34] [93.184.216.34], dsn=2.0.0, stat=Sent (ok: Message 40279894 accepted)
|
4
|
+
Apr 2 00:15:25 mta001 sendmail[32302]: u31FFPtp032300: to=<sent1@example.com>,<sent2@example.com>, delay=00:00:00, xdelay=00:00:00, mailer=esmtp, pri=245938, relay=[93.184.216.34] [93.184.216.34], dsn=2.0.0, stat=Sent (ok: Message 40279894 accepted)
|
5
5
|
Apr 2 00:15:25 mta001 sendmail[12566]: u31FFPtp032300: to=<deferred1@example.com>, delay=00:00:15, xdelay=00:00:15, mailer=esmtp, pri=34527, relay=[93.184.216.34] [93.184.216.34], dsn=4.3.5, stat=Deferred: 451 4.3.5 Server configuration problem
|
6
|
+
Apr 3 00:15:40 mta001 sendmail[12566]: u31FFPtp032300: to=<deferred2@example.com>, delay=1+00:00:15, xdelay=00:00:15, mailer=esmtp, pri=34527, relay=[93.184.216.34] [93.184.216.34], dsn=4.3.5, stat=Deferred: 451 4.3.5 Server configuration problem
|
6
7
|
Apr 2 00:15:26 mta001 sendmail[32302]: u31FFPtp032300: to=<sent3@example2.com>,<sent4@example2.com>, delay=00:00:00, xdelay=00:00:00, mailer=esmtp, pri=245938, relay=[93.184.216.34] [93.184.216.34], dsn=2.0.0, stat=Sent (ok: Message 40279895 accepted)
|
7
8
|
Apr 2 00:18:50 mta001 sendmail[32302]: u31FFPtp032300: to=<deferred1@example.com>, delay=00:00:00, xdelay=00:00:00, mailer=esmtp, pri=245938, relay=[93.184.216.34] [93.184.216.34], dsn=2.0.0, stat=Sent (ok: Message 40279894 accepted)
|
8
9
|
Apr 2 00:15:25 mta001 sendmail[32302]: u31FFPtp032300: done; delay=00:00:00, ntries=2
|
data/test/data/result
ADDED
@@ -0,0 +1,6 @@
|
|
1
|
+
{"time":1459523725, "mta":"mta001", "qid":"u31FFPtp032300", "from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"7", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"93.184.216.34", "host":null}, "status_canonical":"sent", "to":["<sent1@example.com>", "<sent2@example.com>"], "delay":"00:00:00", "xdelay":"00:00:00", "mailer":"esmtp", "pri":"245938", "dsn":"2.0.0", "stat":"Sent (ok: Message 40279894 accepted)", "delay_in_sec":0}
|
2
|
+
{"time":1459523725, "mta":"mta001", "qid":"u31FFPtp032300", "from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"7", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"93.184.216.34", "host":null}, "status_canonical":"deferred", "to":["<deferred1@example.com>"], "delay":"00:00:15", "xdelay":"00:00:15", "mailer":"esmtp", "pri":"34527", "dsn":"4.3.5", "stat":"Deferred: 451 4.3.5 Server configuration problem", "delay_in_sec":15}
|
3
|
+
{"time":1459610140, "mta":"mta001", "qid":"u31FFPtp032300", "from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"7", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"93.184.216.34", "host":null}, "status_canonical":"deferred", "to":["<deferred2@example.com>"], "delay":"1+00:00:15", "xdelay":"00:00:15", "mailer":"esmtp", "pri":"34527", "dsn":"4.3.5", "stat":"Deferred: 451 4.3.5 Server configuration problem", "delay_in_sec":86415, "canonical_status":"bounced"}
|
4
|
+
{"time":1459523726, "mta":"mta001", "qid":"u31FFPtp032300", "from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"7", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"93.184.216.34", "host":null}, "status_canonical":"sent", "to":["<sent3@example2.com>", "<sent4@example2.com>"], "delay":"00:00:00", "xdelay":"00:00:00", "mailer":"esmtp", "pri":"245938", "dsn":"2.0.0", "stat":"Sent (ok: Message 40279895 accepted)", "delay_in_sec":0}
|
5
|
+
{"time":1459523930, "mta":"mta001", "qid":"u31FFPtp032300", "from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"7", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"93.184.216.34", "host":null}, "status_canonical":"sent", "to":["<deferred1@example.com>"], "delay":"00:00:00", "xdelay":"00:00:00", "mailer":"esmtp", "pri":"245938", "dsn":"2.0.0", "stat":"Sent (ok: Message 40279894 accepted)", "delay_in_sec":0}
|
6
|
+
{"time":1459523725, "mta":"mta001", "qid":"u31FFPtp032300", "from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"7", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"64.233.187.27", "host":null}, "status_canonical":"sent_local", "to":["root@localhost.localdomain"], "ctladdr":"root@localhost.localdomain (0/0)", "delay":"00:00:00", "xdelay":"00:00:00", "mailer":"local", "pri":"245938", "dsn":"2.0.0", "stat":"Sent", "delay_in_sec":0}
|
@@ -4,21 +4,17 @@ require 'json'
|
|
4
4
|
class SendmailInputTest < Test::Unit::TestCase
|
5
5
|
def setup
|
6
6
|
Fluent::Test.setup
|
7
|
+
log = Fluent::Engine.log
|
7
8
|
end
|
8
9
|
|
9
10
|
TMP_DIR = File.dirname(__FILE__) + "/../tmp"
|
10
11
|
DATA_DIR = File.dirname(__FILE__) + "/../data"
|
11
12
|
|
12
|
-
|
13
|
+
CONFIG = %[
|
13
14
|
path #{TMP_DIR}/sendmaillog
|
14
15
|
tag sendmail
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
CONFIG_BUNDLE = %[
|
19
|
-
path #{TMP_DIR}/sendmaillog
|
20
|
-
tag sendmail
|
21
|
-
unbundle no
|
16
|
+
queuereturn 1d
|
17
|
+
path_cache_file /tmp/fluent_plugin_sendmail_test_cache_file
|
22
18
|
]
|
23
19
|
|
24
20
|
def setup
|
@@ -27,34 +23,19 @@ class SendmailInputTest < Test::Unit::TestCase
|
|
27
23
|
FileUtils.mkdir_p(TMP_DIR)
|
28
24
|
end
|
29
25
|
|
30
|
-
def create_driver(conf =
|
26
|
+
def create_driver(conf = CONFIG, tag='test')
|
31
27
|
driver = Fluent::Test::InputTestDriver.new(Fluent::SendmailInput)
|
32
28
|
driver.configure(conf)
|
33
29
|
driver
|
34
30
|
end
|
35
31
|
|
36
32
|
def test_configure
|
37
|
-
#### set configurations
|
38
|
-
# d = create_driver %[
|
39
|
-
# path test_path
|
40
|
-
# compress gz
|
41
|
-
# ]
|
42
|
-
#### check configurations
|
43
|
-
# assert_equal 'test_path', d.instance.path
|
44
|
-
# assert_equal :gz, d.instance.compress
|
45
33
|
end
|
46
34
|
|
47
|
-
def
|
48
|
-
data_file = "#{DATA_DIR}/
|
49
|
-
expect_file = "#{DATA_DIR}/
|
50
|
-
driver = create_driver
|
51
|
-
do_test(driver, data_file, expect_file)
|
52
|
-
end
|
53
|
-
|
54
|
-
def test_bundled
|
55
|
-
data_file = "#{DATA_DIR}/data1"
|
56
|
-
expect_file = "#{DATA_DIR}/data1_bundle_result_expect"
|
57
|
-
driver = create_driver(conf=CONFIG_BUNDLE)
|
35
|
+
def test
|
36
|
+
data_file = "#{DATA_DIR}/data"
|
37
|
+
expect_file = "#{DATA_DIR}/result"
|
38
|
+
driver = create_driver(conf=CONFIG)
|
58
39
|
do_test(driver, data_file, expect_file)
|
59
40
|
end
|
60
41
|
|
@@ -64,9 +45,9 @@ class SendmailInputTest < Test::Unit::TestCase
|
|
64
45
|
|
65
46
|
# touch tail file
|
66
47
|
File.open(path, "a") {}
|
67
|
-
|
68
48
|
# result_expect
|
69
49
|
expects = []
|
50
|
+
|
70
51
|
File.open(expect_file, "r") {|expectfile|
|
71
52
|
expectfile.each_line {|line|
|
72
53
|
expects.push(JSON.parse(line))
|
@@ -86,8 +67,11 @@ class SendmailInputTest < Test::Unit::TestCase
|
|
86
67
|
end
|
87
68
|
|
88
69
|
emits = driver.emits
|
70
|
+
assert(emits.length > 0, "no emits")
|
71
|
+
|
89
72
|
emits.each_index {|i|
|
90
73
|
assert_equal(expects[i], emits[i][2])
|
91
74
|
}
|
75
|
+
|
92
76
|
end
|
93
77
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-sendmail
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
-
|
7
|
+
- yudai09
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: lru_redux
|
@@ -82,7 +82,7 @@ dependencies:
|
|
82
82
|
version: '0'
|
83
83
|
description: Fluentd plugin to parse and merge sendmail syslog.
|
84
84
|
email:
|
85
|
-
-
|
85
|
+
- grandeur09@gmail.com
|
86
86
|
executables: []
|
87
87
|
extensions: []
|
88
88
|
extra_rdoc_files: []
|
@@ -97,12 +97,11 @@ files:
|
|
97
97
|
- fluent-plugin-sendmail.gemspec
|
98
98
|
- lib/fluent/plugin/in_sendmail.rb
|
99
99
|
- lib/fluent/plugin/sendmailparser.rb
|
100
|
-
- test/data/
|
101
|
-
- test/data/
|
102
|
-
- test/data/data1_unbundle_result_expect
|
100
|
+
- test/data/data
|
101
|
+
- test/data/result
|
103
102
|
- test/helper.rb
|
104
103
|
- test/plugin/test_in_sendmail.rb
|
105
|
-
homepage: https://github.com/
|
104
|
+
homepage: https://github.com/yudai09/fluent-plugin-sendmail
|
106
105
|
licenses:
|
107
106
|
- MIT
|
108
107
|
metadata: {}
|
@@ -127,8 +126,7 @@ signing_key:
|
|
127
126
|
specification_version: 4
|
128
127
|
summary: Fluentd plugin to parse and merge sendmail syslog.
|
129
128
|
test_files:
|
130
|
-
- test/data/
|
131
|
-
- test/data/
|
132
|
-
- test/data/data1_unbundle_result_expect
|
129
|
+
- test/data/data
|
130
|
+
- test/data/result
|
133
131
|
- test/helper.rb
|
134
132
|
- test/plugin/test_in_sendmail.rb
|
@@ -1 +0,0 @@
|
|
1
|
-
{"mta":"mta001","from": {"from":"<grandeur09@gmail.com>","size":"5938", "class":"0", "nrcpts":"6","msgid":"<201604011515.u31FFIAj012911@gmail.com>","proto":"ESMTP","daemon":"MTA","relay":{"ip":"64.233.187.27", "host":null}},"to":[{"to":["<sent1@example.com>", "<sent2@example.com>"],"00:00:00":null,"xdelay":"00:00:00","mailer":"esmtp","pri":"245938","relay":{"ip":"93.184.216.34", "host":null},"dsn":"2.0.0","stat":"Sent (ok: Message 40279894 accepted)","canonical_status":"sent"},{"to":["<sent3@example2.com>", "<sent4@example2.com>"],"delay":"00:00:00","xdelay":"00:00:00","mailer":"esmtp","pri":"245938","relay":{"ip":"93.184.216.34", "host":null},"dsn":"2.0.0","stat":"Sent (ok: Message 40279895 accepted)","canonical_status":"sent"},{"to":["<deferred1@example.com>"],"delay":"00:00:00","xdelay":"00:00:00","mailer":"esmtp","pri":"245938","relay":{"ip":"93.184.216.34", "host":null},"dsn":"2.0.0","stat":"Sent (ok: Message 40279894 accepted)","canonical_status":"sent"},{"to":["root@localhost.localdomain"],"ctladdr":"root@localhost.localdomain (0/0)","delay":"00:00:00","xdelay":"00:00:00","mailer":"local","pri":"245938","dsn":"2.0.0","stat":"Sent","canonical_status":"sent_local"}]}
|
@@ -1,7 +0,0 @@
|
|
1
|
-
{"mta":"mta001", "from":{"from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"6", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"64.233.187.27", "host":null}}, "to":{"to":"<sent1@example.com>", "00:00:00":null, "xdelay":"00:00:00", "mailer":"esmtp", "pri":"245938", "relay":{"ip":"93.184.216.34", "host":null}, "dsn":"2.0.0", "stat":"Sent (ok: Message 40279894 accepted)", "canonical_status":"sent"}}
|
2
|
-
{"mta":"mta001", "from":{"from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"6", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"64.233.187.27", "host":null}}, "to":{"to":"<sent2@example.com>", "00:00:00":null, "xdelay":"00:00:00", "mailer":"esmtp", "pri":"245938", "relay":{"ip":"93.184.216.34", "host":null}, "dsn":"2.0.0", "stat":"Sent (ok: Message 40279894 accepted)", "canonical_status":"sent"}}
|
3
|
-
{"mta":"mta001", "from":{"from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"6", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"64.233.187.27", "host":null}}, "to":{"to":"<deferred1@example.com>", "delay":"00:00:15", "xdelay":"00:00:15", "mailer":"esmtp", "pri":"34527", "relay":{"ip":"93.184.216.34", "host":null}, "dsn":"4.3.5", "stat":"Deferred: 451 4.3.5 Server configuration problem", "canonical_status":"deferred"}}
|
4
|
-
{"mta":"mta001", "from":{"from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"6", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"64.233.187.27", "host":null}}, "to":{"to":"<sent3@example2.com>", "delay":"00:00:00", "xdelay":"00:00:00", "mailer":"esmtp", "pri":"245938", "relay":{"ip":"93.184.216.34", "host":null}, "dsn":"2.0.0", "stat":"Sent (ok: Message 40279895 accepted)", "canonical_status":"sent"}}
|
5
|
-
{"mta":"mta001", "from":{"from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"6", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"64.233.187.27", "host":null}}, "to":{"to":"<sent4@example2.com>", "delay":"00:00:00", "xdelay":"00:00:00", "mailer":"esmtp", "pri":"245938", "relay":{"ip":"93.184.216.34", "host":null}, "dsn":"2.0.0", "stat":"Sent (ok: Message 40279895 accepted)", "canonical_status":"sent"}}
|
6
|
-
{"mta":"mta001", "from":{"from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"6", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"64.233.187.27", "host":null}}, "to":{"to":"<deferred1@example.com>", "delay":"00:00:00", "xdelay":"00:00:00", "mailer":"esmtp", "pri":"245938", "relay":{"ip":"93.184.216.34", "host":null}, "dsn":"2.0.0", "stat":"Sent (ok: Message 40279894 accepted)", "canonical_status":"sent"}}
|
7
|
-
{"mta":"mta001", "from":{"from":"<grandeur09@gmail.com>", "size":"5938", "class":"0", "nrcpts":"6", "msgid":"<201604011515.u31FFIAj012911@gmail.com>", "proto":"ESMTP", "daemon":"MTA", "relay":{"ip":"64.233.187.27", "host":null}}, "to":{"to":"root@localhost.localdomain", "ctladdr":"root@localhost.localdomain (0/0)", "delay":"00:00:00", "xdelay":"00:00:00", "mailer":"local", "pri":"245938", "dsn":"2.0.0", "stat":"Sent", "canonical_status":"sent_local"}}
|