fluent-plugin-multiline-parser 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +36 -0
- data/Gemfile +3 -0
- data/LICENSE +201 -0
- data/README.md +1 -0
- data/Rakefile +11 -0
- data/fluent-plugin-multiline-parser.gemspec +21 -0
- data/lib/fluent/plugin/filter_deparser.rb +35 -0
- data/lib/fluent/plugin/filter_parser.rb +138 -0
- data/lib/fluent/plugin/out_deparser.rb +94 -0
- data/lib/fluent/plugin/out_parser.rb +172 -0
- data/test/custom_parser.rb +39 -0
- data/test/helper.rb +31 -0
- data/test/plugin/test_deparser.rb +149 -0
- data/test/plugin/test_filter_deparser.rb +82 -0
- data/test/plugin/test_filter_parser.rb +668 -0
- data/test/plugin/test_out_parser.rb +702 -0
- data/test/plugin/test_out_parser_for_parsers.rb +285 -0
- metadata +125 -0
@@ -0,0 +1,702 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class ParserOutputTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
end
|
7
|
+
|
8
|
+
CONFIG = %[
|
9
|
+
remove_prefix test
|
10
|
+
add_prefix parsed
|
11
|
+
key_name message
|
12
|
+
format /^(?<x>.)(?<y>.) (?<time>.+)$/
|
13
|
+
time_format %Y%m%d%H%M%S
|
14
|
+
reserve_data true
|
15
|
+
]
|
16
|
+
|
17
|
+
def create_driver(conf=CONFIG,tag='test')
|
18
|
+
Fluent::Test::OutputTestDriver.new(Fluent::ParserOutput, tag).configure(conf)
|
19
|
+
end
|
20
|
+
|
21
|
+
def test_configure
|
22
|
+
assert_raise(Fluent::ConfigError) {
|
23
|
+
d = create_driver('')
|
24
|
+
}
|
25
|
+
assert_raise(Fluent::ConfigError) {
|
26
|
+
d = create_driver %[
|
27
|
+
tag foo.bar
|
28
|
+
format unknown_format_that_will_never_be_implemented
|
29
|
+
key_name foo
|
30
|
+
]
|
31
|
+
}
|
32
|
+
assert_nothing_raised {
|
33
|
+
d = create_driver %[
|
34
|
+
tag foo.bar
|
35
|
+
format /(?<x>.)/
|
36
|
+
key_name foo
|
37
|
+
]
|
38
|
+
}
|
39
|
+
assert_nothing_raised {
|
40
|
+
d = create_driver %[
|
41
|
+
remove_prefix foo.bar
|
42
|
+
format /(?<x>.)/
|
43
|
+
key_name foo
|
44
|
+
]
|
45
|
+
}
|
46
|
+
assert_nothing_raised {
|
47
|
+
d = create_driver %[
|
48
|
+
add_prefix foo.bar
|
49
|
+
format /(?<x>.)/
|
50
|
+
key_name foo
|
51
|
+
]
|
52
|
+
}
|
53
|
+
assert_nothing_raised {
|
54
|
+
d = create_driver %[
|
55
|
+
remove_prefix foo.baz
|
56
|
+
add_prefix foo.bar
|
57
|
+
format /(?<x>.)/
|
58
|
+
key_name foo
|
59
|
+
]
|
60
|
+
}
|
61
|
+
assert_nothing_raised {
|
62
|
+
d = create_driver %[
|
63
|
+
remove_prefix foo.baz
|
64
|
+
add_prefix foo.bar
|
65
|
+
format json
|
66
|
+
key_name foo
|
67
|
+
]
|
68
|
+
}
|
69
|
+
assert_nothing_raised {
|
70
|
+
d = create_driver %[
|
71
|
+
remove_prefix foo.baz
|
72
|
+
add_prefix foo.bar
|
73
|
+
format ltsv
|
74
|
+
key_name foo
|
75
|
+
]
|
76
|
+
}
|
77
|
+
assert_nothing_raised {
|
78
|
+
d = create_driver %[
|
79
|
+
tag hogelog
|
80
|
+
format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
|
81
|
+
key_name message
|
82
|
+
suppress_parse_error_log true
|
83
|
+
]
|
84
|
+
}
|
85
|
+
assert_nothing_raised {
|
86
|
+
d = create_driver %[
|
87
|
+
tag hogelog
|
88
|
+
format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
|
89
|
+
key_name message
|
90
|
+
suppress_parse_error_log false
|
91
|
+
]
|
92
|
+
}
|
93
|
+
d = create_driver %[
|
94
|
+
tag foo.bar
|
95
|
+
key_name foo
|
96
|
+
format /(?<x>.)/
|
97
|
+
]
|
98
|
+
assert_equal false, d.instance.reserve_data
|
99
|
+
end
|
100
|
+
|
101
|
+
# CONFIG = %[
|
102
|
+
# remove_prefix test
|
103
|
+
# add_prefix parsed
|
104
|
+
# key_name message
|
105
|
+
# format /^(?<x>.)(?<y>.) (?<time>.+)$/
|
106
|
+
# time_format %Y%m%d%H%M%S
|
107
|
+
# reserve_data true
|
108
|
+
# ]
|
109
|
+
def test_emit
|
110
|
+
d1 = create_driver(CONFIG, 'test.in')
|
111
|
+
time = Time.parse("2012-01-02 13:14:15").to_i
|
112
|
+
d1.run do
|
113
|
+
d1.emit({'message' => '12 20120402182059'}, time)
|
114
|
+
d1.emit({'message' => '34 20120402182100'}, time)
|
115
|
+
d1.emit({'message' => '56 20120402182100'}, time)
|
116
|
+
d1.emit({'message' => '78 20120402182101'}, time)
|
117
|
+
d1.emit({'message' => '90 20120402182100'}, time)
|
118
|
+
end
|
119
|
+
emits = d1.emits
|
120
|
+
assert_equal 5, emits.length
|
121
|
+
|
122
|
+
first = emits[0]
|
123
|
+
assert_equal 'parsed.in', first[0]
|
124
|
+
assert_equal Time.parse("2012-04-02 18:20:59").to_i, first[1]
|
125
|
+
assert_equal '1', first[2]['x']
|
126
|
+
assert_equal '2', first[2]['y']
|
127
|
+
assert_equal '12 20120402182059', first[2]['message']
|
128
|
+
|
129
|
+
second = emits[1]
|
130
|
+
assert_equal 'parsed.in', second[0]
|
131
|
+
assert_equal Time.parse("2012-04-02 18:21:00").to_i, second[1]
|
132
|
+
assert_equal '3', second[2]['x']
|
133
|
+
assert_equal '4', second[2]['y']
|
134
|
+
|
135
|
+
third = emits[2]
|
136
|
+
assert_equal 'parsed.in', third[0]
|
137
|
+
assert_equal Time.parse("2012-04-02 18:21:00").to_i, third[1]
|
138
|
+
assert_equal '5', third[2]['x']
|
139
|
+
assert_equal '6', third[2]['y']
|
140
|
+
|
141
|
+
fourth = emits[3]
|
142
|
+
assert_equal 'parsed.in', fourth[0]
|
143
|
+
assert_equal Time.parse("2012-04-02 18:21:01").to_i, fourth[1]
|
144
|
+
assert_equal '7', fourth[2]['x']
|
145
|
+
assert_equal '8', fourth[2]['y']
|
146
|
+
|
147
|
+
fifth = emits[4]
|
148
|
+
assert_equal 'parsed.in', fifth[0]
|
149
|
+
assert_equal Time.parse("2012-04-02 18:21:00").to_i, fifth[1]
|
150
|
+
assert_equal '9', fifth[2]['x']
|
151
|
+
assert_equal '0', fifth[2]['y']
|
152
|
+
|
153
|
+
d2 = create_driver(%[
|
154
|
+
tag parsed
|
155
|
+
key_name data
|
156
|
+
format /^(?<x>.)(?<y>.) (?<t>.+)$/
|
157
|
+
], 'test.in')
|
158
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
159
|
+
d2.run do
|
160
|
+
d2.emit({'data' => '12 20120402182059'}, time)
|
161
|
+
d2.emit({'data' => '34 20120402182100'}, time)
|
162
|
+
end
|
163
|
+
emits = d2.emits
|
164
|
+
assert_equal 2, emits.length
|
165
|
+
|
166
|
+
first = emits[0]
|
167
|
+
assert_equal 'parsed', first[0]
|
168
|
+
assert_equal time, first[1]
|
169
|
+
assert_nil first[2]['data']
|
170
|
+
assert_equal '1', first[2]['x']
|
171
|
+
assert_equal '2', first[2]['y']
|
172
|
+
assert_equal '20120402182059', first[2]['t']
|
173
|
+
|
174
|
+
second = emits[1]
|
175
|
+
assert_equal 'parsed', second[0]
|
176
|
+
assert_equal time, second[1]
|
177
|
+
assert_nil second[2]['data']
|
178
|
+
assert_equal '3', second[2]['x']
|
179
|
+
assert_equal '4', second[2]['y']
|
180
|
+
assert_equal '20120402182100', second[2]['t']
|
181
|
+
|
182
|
+
d3 = create_driver(%[
|
183
|
+
tag parsed
|
184
|
+
key_name data
|
185
|
+
format /^(?<x>[0-9])(?<y>[0-9]) (?<t>.+)$/
|
186
|
+
], 'test.in')
|
187
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
188
|
+
d3.run do
|
189
|
+
d3.emit({'data' => '12 20120402182059'}, time)
|
190
|
+
d3.emit({'data' => '34 20120402182100'}, time)
|
191
|
+
d3.emit({'data' => 'xy 20120402182101'}, time)
|
192
|
+
end
|
193
|
+
emits = d3.emits
|
194
|
+
assert_equal 2, emits.length
|
195
|
+
|
196
|
+
d3x = create_driver(%[
|
197
|
+
tag parsed
|
198
|
+
key_name data
|
199
|
+
format /^(?<x>\\d)(?<y>\\d) (?<t>.+)$/
|
200
|
+
reserve_data yes
|
201
|
+
], 'test.in')
|
202
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
203
|
+
d3x.run do
|
204
|
+
d3x.emit({'data' => '12 20120402182059'}, time)
|
205
|
+
d3x.emit({'data' => '34 20120402182100'}, time)
|
206
|
+
d3x.emit({'data' => 'xy 20120402182101'}, time)
|
207
|
+
end
|
208
|
+
emits = d3x.emits
|
209
|
+
assert_equal 3, emits.length
|
210
|
+
|
211
|
+
d4 = create_driver(%[
|
212
|
+
tag parsed
|
213
|
+
key_name data
|
214
|
+
format json
|
215
|
+
], 'test.in')
|
216
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
217
|
+
d4.run do
|
218
|
+
d4.emit({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
|
219
|
+
d4.emit({'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'}, time)
|
220
|
+
end
|
221
|
+
emits = d4.emits
|
222
|
+
assert_equal 1, emits.length
|
223
|
+
|
224
|
+
d4x = create_driver(%[
|
225
|
+
tag parsed
|
226
|
+
key_name data
|
227
|
+
format json
|
228
|
+
reserve_data yes
|
229
|
+
], 'test.in')
|
230
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
231
|
+
d4x.run do
|
232
|
+
d4x.emit({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
|
233
|
+
d4x.emit({'data' => 'foobar', 'xxx' => 'x', 'yyy' => 'y'}, time)
|
234
|
+
end
|
235
|
+
emits = d4x.emits
|
236
|
+
assert_equal 2, emits.length
|
237
|
+
|
238
|
+
first = emits[0]
|
239
|
+
assert_equal 'parsed', first[0]
|
240
|
+
assert_equal time, first[1]
|
241
|
+
assert_equal '{"xxx":"first","yyy":"second"}', first[2]['data']
|
242
|
+
assert_equal 'first', first[2]['xxx']
|
243
|
+
assert_equal 'second', first[2]['yyy']
|
244
|
+
|
245
|
+
second = emits[1]
|
246
|
+
assert_equal 'parsed', second[0]
|
247
|
+
assert_equal time, second[1]
|
248
|
+
assert_equal 'foobar', second[2]['data']
|
249
|
+
assert_equal 'x', second[2]['xxx']
|
250
|
+
assert_equal 'y', second[2]['yyy']
|
251
|
+
end
|
252
|
+
|
253
|
+
CONFIG_LTSV = %[
|
254
|
+
remove_prefix foo.baz
|
255
|
+
add_prefix foo.bar
|
256
|
+
format ltsv
|
257
|
+
key_name data
|
258
|
+
]
|
259
|
+
def test_emit_ltsv
|
260
|
+
d = create_driver(CONFIG_LTSV, 'foo.baz.test')
|
261
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
262
|
+
d.run do
|
263
|
+
d.emit({'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'}, time)
|
264
|
+
d.emit({'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'}, time)
|
265
|
+
end
|
266
|
+
emits = d.emits
|
267
|
+
assert_equal 2, emits.length
|
268
|
+
|
269
|
+
first = emits[0]
|
270
|
+
assert_equal 'foo.bar.test', first[0]
|
271
|
+
assert_equal time, first[1]
|
272
|
+
assert_nil first[2]['data']
|
273
|
+
assert_equal 'first', first[2]['xxx']
|
274
|
+
assert_equal 'second', first[2]['yyy']
|
275
|
+
|
276
|
+
second = emits[1]
|
277
|
+
assert_equal 'foo.bar.test', second[0]
|
278
|
+
assert_equal time, second[1]
|
279
|
+
assert_nil first[2]['data']
|
280
|
+
assert_equal 'first', second[2]['xxx']
|
281
|
+
assert_equal 'second2', second[2]['yyy']
|
282
|
+
|
283
|
+
d = create_driver(CONFIG_LTSV + %[
|
284
|
+
reserve_data yes
|
285
|
+
], 'foo.baz.test')
|
286
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
287
|
+
d.run do
|
288
|
+
d.emit({'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'}, time)
|
289
|
+
d.emit({'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'}, time)
|
290
|
+
end
|
291
|
+
emits = d.emits
|
292
|
+
assert_equal 2, emits.length
|
293
|
+
|
294
|
+
first = emits[0]
|
295
|
+
assert_equal 'foo.bar.test', first[0]
|
296
|
+
assert_equal time, first[1]
|
297
|
+
assert_equal "xxx:first\tyyy:second", first[2]['data']
|
298
|
+
assert_equal 'first', first[2]['xxx']
|
299
|
+
assert_equal 'second', first[2]['yyy']
|
300
|
+
|
301
|
+
second = emits[1]
|
302
|
+
assert_equal 'foo.bar.test', second[0]
|
303
|
+
assert_equal time, second[1]
|
304
|
+
assert_equal "xxx:first\tyyy:second", first[2]['data']
|
305
|
+
assert_equal 'first', second[2]['xxx']
|
306
|
+
assert_equal 'second2', second[2]['yyy']
|
307
|
+
|
308
|
+
# convert types
|
309
|
+
d = create_driver(CONFIG_LTSV + %[
|
310
|
+
types i:integer,s:string,f:float,b:bool
|
311
|
+
], 'foo.baz.test')
|
312
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
313
|
+
d.run do
|
314
|
+
d.emit({'data' => "i:1\ts:2\tf:3\tb:true\tx:123"}, time)
|
315
|
+
end
|
316
|
+
emits = d.emits
|
317
|
+
assert_equal 1, emits.length
|
318
|
+
|
319
|
+
first = emits[0]
|
320
|
+
assert_equal 'foo.bar.test', first[0]
|
321
|
+
assert_equal time, first[1]
|
322
|
+
assert_equal 1, first[2]['i']
|
323
|
+
assert_equal '2', first[2]['s']
|
324
|
+
assert_equal 3.0, first[2]['f']
|
325
|
+
assert_equal true, first[2]['b']
|
326
|
+
assert_equal '123', first[2]['x']
|
327
|
+
end
|
328
|
+
|
329
|
+
CONFIG_TSV = %[
|
330
|
+
remove_prefix foo.baz
|
331
|
+
add_prefix foo.bar
|
332
|
+
format tsv
|
333
|
+
key_name data
|
334
|
+
keys key1,key2,key3
|
335
|
+
]
|
336
|
+
def test_emit_tsv
|
337
|
+
d = create_driver(CONFIG_TSV, 'foo.baz.test')
|
338
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
339
|
+
d.run do
|
340
|
+
d.emit({'data' => "value1\tvalue2\tvalueThree", 'xxx' => 'x', 'yyy' => 'y'}, time)
|
341
|
+
end
|
342
|
+
emits = d.emits
|
343
|
+
assert_equal 1, emits.length
|
344
|
+
|
345
|
+
first = emits[0]
|
346
|
+
assert_equal 'foo.bar.test', first[0]
|
347
|
+
assert_equal time, first[1]
|
348
|
+
assert_nil first[2]['data']
|
349
|
+
assert_equal 'value1', first[2]['key1']
|
350
|
+
assert_equal 'value2', first[2]['key2']
|
351
|
+
assert_equal 'valueThree', first[2]['key3']
|
352
|
+
end
|
353
|
+
|
354
|
+
CONFIG_CSV = %[
|
355
|
+
remove_prefix foo.baz
|
356
|
+
add_prefix foo.bar
|
357
|
+
format csv
|
358
|
+
key_name data
|
359
|
+
keys key1,key2,key3
|
360
|
+
]
|
361
|
+
def test_emit_csv
|
362
|
+
d = create_driver(CONFIG_CSV, 'foo.baz.test')
|
363
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
364
|
+
d.run do
|
365
|
+
d.emit({'data' => 'value1,"value2","value""ThreeYes!"', 'xxx' => 'x', 'yyy' => 'y'}, time)
|
366
|
+
end
|
367
|
+
emits = d.emits
|
368
|
+
assert_equal 1, emits.length
|
369
|
+
|
370
|
+
first = emits[0]
|
371
|
+
assert_equal 'foo.bar.test', first[0]
|
372
|
+
assert_equal time, first[1]
|
373
|
+
assert_nil first[2]['data']
|
374
|
+
assert_equal 'value1', first[2]['key1']
|
375
|
+
assert_equal 'value2', first[2]['key2']
|
376
|
+
assert_equal 'value"ThreeYes!', first[2]['key3']
|
377
|
+
end
|
378
|
+
|
379
|
+
CONFIG_KEY_PREFIX = %[
|
380
|
+
remove_prefix foo.baz
|
381
|
+
add_prefix foo.bar
|
382
|
+
format json
|
383
|
+
key_name data
|
384
|
+
reserve_data yes
|
385
|
+
inject_key_prefix data.
|
386
|
+
]
|
387
|
+
def test_inject_key_prefix
|
388
|
+
d = create_driver(CONFIG_KEY_PREFIX, 'foo.baz.test')
|
389
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
390
|
+
d.run do
|
391
|
+
d.emit({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
|
392
|
+
end
|
393
|
+
emits = d.emits
|
394
|
+
assert_equal 1, emits.length
|
395
|
+
|
396
|
+
first = emits[0]
|
397
|
+
assert_equal 'foo.bar.test', first[0]
|
398
|
+
assert_equal time, first[1]
|
399
|
+
|
400
|
+
assert_equal '{"xxx":"first","yyy":"second"}', first[2]['data']
|
401
|
+
assert_equal 'x', first[2]['xxx']
|
402
|
+
assert_equal 'y', first[2]['yyy']
|
403
|
+
assert_equal 'first', first[2]['data.xxx']
|
404
|
+
assert_equal 'second', first[2]['data.yyy']
|
405
|
+
assert_equal 5, first[2].keys.size
|
406
|
+
end
|
407
|
+
|
408
|
+
CONFIG_HASH_VALUE_FIELD = %[
|
409
|
+
remove_prefix foo.baz
|
410
|
+
add_prefix foo.bar
|
411
|
+
format json
|
412
|
+
key_name data
|
413
|
+
hash_value_field parsed
|
414
|
+
]
|
415
|
+
CONFIG_HASH_VALUE_FIELD_RESERVE_DATA = %[
|
416
|
+
remove_prefix foo.baz
|
417
|
+
add_prefix foo.bar
|
418
|
+
format json
|
419
|
+
key_name data
|
420
|
+
reserve_data yes
|
421
|
+
hash_value_field parsed
|
422
|
+
]
|
423
|
+
CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX = %[
|
424
|
+
remove_prefix foo.baz
|
425
|
+
add_prefix foo.bar
|
426
|
+
format json
|
427
|
+
key_name data
|
428
|
+
hash_value_field parsed
|
429
|
+
inject_key_prefix data.
|
430
|
+
]
|
431
|
+
def test_inject_hash_value_field
|
432
|
+
original = {'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}
|
433
|
+
|
434
|
+
d = create_driver(CONFIG_HASH_VALUE_FIELD, 'foo.baz.test')
|
435
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
436
|
+
d.run do
|
437
|
+
d.emit(original, time)
|
438
|
+
end
|
439
|
+
emits = d.emits
|
440
|
+
assert_equal 1, emits.length
|
441
|
+
|
442
|
+
first = emits[0]
|
443
|
+
assert_equal 'foo.bar.test', first[0]
|
444
|
+
assert_equal time, first[1]
|
445
|
+
|
446
|
+
record = first[2]
|
447
|
+
assert_equal 1, record.keys.size
|
448
|
+
assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
|
449
|
+
|
450
|
+
d = create_driver(CONFIG_HASH_VALUE_FIELD_RESERVE_DATA, 'foo.baz.test')
|
451
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
452
|
+
d.run do
|
453
|
+
d.emit(original, time)
|
454
|
+
end
|
455
|
+
emits = d.emits
|
456
|
+
assert_equal 1, emits.length
|
457
|
+
|
458
|
+
first = emits[0]
|
459
|
+
assert_equal 'foo.bar.test', first[0]
|
460
|
+
assert_equal time, first[1]
|
461
|
+
|
462
|
+
record = first[2]
|
463
|
+
assert_equal 4, record.keys.size
|
464
|
+
assert_equal original['data'], record['data']
|
465
|
+
assert_equal original['xxx'], record['xxx']
|
466
|
+
assert_equal original['yyy'], record['yyy']
|
467
|
+
assert_equal({"xxx"=>"first","yyy"=>"second"}, record['parsed'])
|
468
|
+
|
469
|
+
d = create_driver(CONFIG_HASH_VALUE_FIELD_WITH_INJECT_KEY_PREFIX, 'foo.baz.test')
|
470
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
471
|
+
d.run do
|
472
|
+
d.emit(original, time)
|
473
|
+
end
|
474
|
+
emits = d.emits
|
475
|
+
assert_equal 1, emits.length
|
476
|
+
|
477
|
+
first = emits[0]
|
478
|
+
assert_equal 'foo.bar.test', first[0]
|
479
|
+
assert_equal time, first[1]
|
480
|
+
|
481
|
+
record = first[2]
|
482
|
+
assert_equal 1, record.keys.size
|
483
|
+
assert_equal({"data.xxx"=>"first","data.yyy"=>"second"}, record['parsed'])
|
484
|
+
end
|
485
|
+
|
486
|
+
CONFIG_DONT_PARSE_TIME = %[
|
487
|
+
remove_prefix test
|
488
|
+
key_name data
|
489
|
+
format json
|
490
|
+
time_parse no
|
491
|
+
]
|
492
|
+
def test_time_should_be_reserved
|
493
|
+
t = Time.now.to_i
|
494
|
+
d = create_driver(CONFIG_DONT_PARSE_TIME, 'test.in')
|
495
|
+
|
496
|
+
d.run do
|
497
|
+
d.emit({'data' => '{"time":1383190430, "f1":"v1"}'}, t)
|
498
|
+
d.emit({'data' => '{"time":"1383190430", "f1":"v1"}'}, t)
|
499
|
+
d.emit({'data' => '{"time":"2013-10-31 12:34:03 +0900", "f1":"v1"}'}, t)
|
500
|
+
end
|
501
|
+
emits = d.emits
|
502
|
+
assert_equal 3, emits.length
|
503
|
+
|
504
|
+
assert_equal 'in', emits[0][0]
|
505
|
+
assert_equal 'v1', emits[0][2]['f1']
|
506
|
+
assert_equal 1383190430, emits[0][2]['time']
|
507
|
+
assert_equal t, emits[0][1]
|
508
|
+
|
509
|
+
assert_equal 'in', emits[1][0]
|
510
|
+
assert_equal 'v1', emits[1][2]['f1']
|
511
|
+
assert_equal "1383190430", emits[1][2]['time']
|
512
|
+
assert_equal t, emits[1][1]
|
513
|
+
|
514
|
+
assert_equal 'in', emits[2][0]
|
515
|
+
assert_equal 'v1', emits[2][2]['f1']
|
516
|
+
assert_equal '2013-10-31 12:34:03 +0900', emits[2][2]['time']
|
517
|
+
assert_equal t, emits[2][1]
|
518
|
+
end
|
519
|
+
|
520
|
+
CONFIG_INVALID_TIME_VALUE = %[
|
521
|
+
remove_prefix test
|
522
|
+
key_name data
|
523
|
+
format json
|
524
|
+
] # 'time' is implicit @time_key
|
525
|
+
def test_invalid_time_data
|
526
|
+
# should not raise errors
|
527
|
+
t = Time.now.to_i
|
528
|
+
d = create_driver(CONFIG_INVALID_TIME_VALUE, 'test.in')
|
529
|
+
assert_nothing_raised {
|
530
|
+
d.run do
|
531
|
+
d.emit({'data' => '{"time":[], "f1":"v1"}'}, t)
|
532
|
+
d.emit({'data' => '{"time":"thisisnottime", "f1":"v1"}'}, t)
|
533
|
+
end
|
534
|
+
}
|
535
|
+
emits = d.emits
|
536
|
+
assert_equal 1, emits.length
|
537
|
+
|
538
|
+
assert_equal 'in', emits[0][0]
|
539
|
+
assert_equal 0, emits[0][1]
|
540
|
+
assert_equal 'v1', emits[0][2]['f1']
|
541
|
+
assert_equal 0, emits[0][2]['time'].to_i
|
542
|
+
end
|
543
|
+
|
544
|
+
# REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
|
545
|
+
|
546
|
+
CONFIG_NOT_REPLACE = %[
|
547
|
+
remove_prefix test
|
548
|
+
key_name data
|
549
|
+
format /^(?<message>.*)$/
|
550
|
+
]
|
551
|
+
CONFIG_INVALID_BYTE = CONFIG_NOT_REPLACE + %[
|
552
|
+
replace_invalid_sequence true
|
553
|
+
]
|
554
|
+
def test_emit_invalid_byte
|
555
|
+
invalid_utf8 = "\xff".force_encoding('UTF-8')
|
556
|
+
|
557
|
+
d = create_driver(CONFIG_NOT_REPLACE, 'test.in')
|
558
|
+
assert_raise(ArgumentError) {
|
559
|
+
d.run do
|
560
|
+
d.emit({'data' => invalid_utf8}, Time.now.to_i)
|
561
|
+
end
|
562
|
+
}
|
563
|
+
|
564
|
+
d = create_driver(CONFIG_INVALID_BYTE, 'test.in')
|
565
|
+
assert_nothing_raised {
|
566
|
+
d.run do
|
567
|
+
d.emit({'data' => invalid_utf8}, Time.now.to_i)
|
568
|
+
end
|
569
|
+
}
|
570
|
+
emits = d.emits
|
571
|
+
assert_equal 1, emits.length
|
572
|
+
assert_nil emits[0][2]['data']
|
573
|
+
assert_equal '?'.force_encoding('UTF-8'), emits[0][2]['message']
|
574
|
+
|
575
|
+
d = create_driver(CONFIG_INVALID_BYTE + %[
|
576
|
+
reserve_data yes
|
577
|
+
], 'test.in')
|
578
|
+
assert_nothing_raised {
|
579
|
+
d.run do
|
580
|
+
d.emit({'data' => invalid_utf8}, Time.now.to_i)
|
581
|
+
end
|
582
|
+
}
|
583
|
+
emits = d.emits
|
584
|
+
assert_equal 1, emits.length
|
585
|
+
assert_equal invalid_utf8, emits[0][2]['data']
|
586
|
+
assert_equal '?'.force_encoding('UTF-8'), emits[0][2]['message']
|
587
|
+
|
588
|
+
invalid_ascii = "\xff".force_encoding('US-ASCII')
|
589
|
+
d = create_driver(CONFIG_INVALID_BYTE, 'test.in')
|
590
|
+
assert_nothing_raised {
|
591
|
+
d.run do
|
592
|
+
d.emit({'data' => invalid_ascii}, Time.now.to_i)
|
593
|
+
end
|
594
|
+
}
|
595
|
+
emits = d.emits
|
596
|
+
assert_equal 1, emits.length
|
597
|
+
assert_nil emits[0][2]['data']
|
598
|
+
assert_equal '?'.force_encoding('US-ASCII'), emits[0][2]['message']
|
599
|
+
end
|
600
|
+
|
601
|
+
# suppress_parse_error_log test
|
602
|
+
CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG = %[
|
603
|
+
tag hogelog
|
604
|
+
format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
|
605
|
+
key_name message
|
606
|
+
suppress_parse_error_log false
|
607
|
+
]
|
608
|
+
CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG = %[
|
609
|
+
tag hogelog
|
610
|
+
format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
|
611
|
+
key_name message
|
612
|
+
suppress_parse_error_log true
|
613
|
+
]
|
614
|
+
CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG = %[
|
615
|
+
tag hogelog
|
616
|
+
format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
|
617
|
+
key_name message
|
618
|
+
]
|
619
|
+
|
620
|
+
INVALID_MESSAGE = 'foo bar'
|
621
|
+
VALID_MESSAGE = 'col1=foo col2=bar'
|
622
|
+
|
623
|
+
# if call warn() raise exception
|
624
|
+
class DummyLoggerWarnedException < StandardError; end
|
625
|
+
class DummyLogger
|
626
|
+
def warn(message)
|
627
|
+
raise DummyLoggerWarnedException
|
628
|
+
end
|
629
|
+
end
|
630
|
+
|
631
|
+
def swap_logger(instance)
|
632
|
+
raise "use with block" unless block_given?
|
633
|
+
dummy = DummyLogger.new
|
634
|
+
saved_logger = instance.log
|
635
|
+
instance.log = dummy
|
636
|
+
restore = lambda{ instance.log = saved_logger }
|
637
|
+
|
638
|
+
yield
|
639
|
+
|
640
|
+
restore.call
|
641
|
+
end
|
642
|
+
|
643
|
+
def test_parser_error_warning
|
644
|
+
d = create_driver(CONFIG_INVALID_TIME_VALUE, 'test.in')
|
645
|
+
swap_logger(d.instance) do
|
646
|
+
assert_raise(DummyLoggerWarnedException) {
|
647
|
+
d.run do
|
648
|
+
d.emit({'data' => '{"time":[], "f1":"v1"}'}, Time.now.to_i)
|
649
|
+
end
|
650
|
+
}
|
651
|
+
end
|
652
|
+
end
|
653
|
+
|
654
|
+
def test_suppress_parse_error_log
|
655
|
+
# default(disabled) 'suppress_parse_error_log' is not specify
|
656
|
+
d = create_driver(CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
|
657
|
+
|
658
|
+
swap_logger(d.instance) do
|
659
|
+
assert_raise(DummyLoggerWarnedException) {
|
660
|
+
d.run do
|
661
|
+
d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
|
662
|
+
end
|
663
|
+
}
|
664
|
+
|
665
|
+
assert_nothing_raised {
|
666
|
+
d.run do
|
667
|
+
d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
|
668
|
+
end
|
669
|
+
}
|
670
|
+
end
|
671
|
+
|
672
|
+
# disabled 'suppress_parse_error_log'
|
673
|
+
d = create_driver(CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
|
674
|
+
|
675
|
+
swap_logger(d.instance) do
|
676
|
+
assert_raise(DummyLoggerWarnedException) {
|
677
|
+
d.run do
|
678
|
+
d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
|
679
|
+
end
|
680
|
+
}
|
681
|
+
|
682
|
+
assert_nothing_raised {
|
683
|
+
d.run do
|
684
|
+
d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
|
685
|
+
end
|
686
|
+
}
|
687
|
+
end
|
688
|
+
|
689
|
+
# enabled 'suppress_parse_error_log'
|
690
|
+
d = create_driver(CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
|
691
|
+
|
692
|
+
swap_logger(d.instance) do
|
693
|
+
assert_nothing_raised {
|
694
|
+
d.run do
|
695
|
+
d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
|
696
|
+
d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
|
697
|
+
end
|
698
|
+
}
|
699
|
+
end
|
700
|
+
end
|
701
|
+
|
702
|
+
end
|