fluentd 0.14.0 → 0.14.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. data/example/copy_roundrobin.conf +39 -0
  3. data/example/filter_stdout.conf +5 -5
  4. data/example/in_forward.conf +2 -2
  5. data/example/in_http.conf +2 -2
  6. data/example/in_syslog.conf +2 -2
  7. data/example/in_tail.conf +2 -2
  8. data/example/in_tcp.conf +2 -2
  9. data/example/in_udp.conf +2 -2
  10. data/example/out_buffered_null.conf +32 -0
  11. data/example/out_copy.conf +4 -4
  12. data/example/out_file.conf +2 -2
  13. data/example/out_forward.conf +2 -2
  14. data/example/v0_12_filter.conf +8 -8
  15. data/fluentd.gemspec +1 -1
  16. data/lib/fluent/command/fluentd.rb +6 -1
  17. data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
  18. data/lib/fluent/compat/input.rb +1 -0
  19. data/lib/fluent/compat/output.rb +1 -0
  20. data/lib/fluent/compat/record_filter_mixin.rb +34 -0
  21. data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
  22. data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
  23. data/lib/fluent/compat/type_converter.rb +90 -0
  24. data/lib/fluent/config/configure_proxy.rb +24 -4
  25. data/lib/fluent/config/dsl.rb +18 -1
  26. data/lib/fluent/config/v1_parser.rb +3 -2
  27. data/lib/fluent/configurable.rb +1 -1
  28. data/lib/fluent/event.rb +37 -9
  29. data/lib/fluent/mixin.rb +12 -286
  30. data/lib/fluent/plugin/buffer.rb +2 -2
  31. data/lib/fluent/plugin/in_dummy.rb +5 -1
  32. data/lib/fluent/plugin/in_gc_stat.rb +7 -37
  33. data/lib/fluent/plugin/in_http.rb +2 -0
  34. data/lib/fluent/plugin/{in_stream.rb → in_unix.rb} +0 -0
  35. data/lib/fluent/plugin/out_buffered_stdout.rb +60 -0
  36. data/lib/fluent/plugin/out_copy.rb +8 -51
  37. data/lib/fluent/plugin/out_null.rb +5 -5
  38. data/lib/fluent/plugin/out_relabel.rb +5 -5
  39. data/lib/fluent/plugin/out_roundrobin.rb +13 -40
  40. data/lib/fluent/plugin/output.rb +9 -0
  41. data/lib/fluent/plugin_helper.rb +2 -0
  42. data/lib/fluent/plugin_helper/formatter.rb +138 -0
  43. data/lib/fluent/plugin_helper/inject.rb +112 -0
  44. data/lib/fluent/plugin_helper/parser.rb +138 -0
  45. data/lib/fluent/plugin_helper/storage.rb +64 -50
  46. data/lib/fluent/process.rb +6 -1
  47. data/lib/fluent/registry.rb +1 -1
  48. data/lib/fluent/supervisor.rb +20 -2
  49. data/lib/fluent/test.rb +30 -5
  50. data/lib/fluent/test/base.rb +2 -66
  51. data/lib/fluent/test/driver/base.rb +3 -0
  52. data/lib/fluent/test/driver/base_owned.rb +106 -0
  53. data/lib/fluent/test/driver/formatter.rb +30 -0
  54. data/lib/fluent/test/driver/multi_output.rb +52 -0
  55. data/lib/fluent/test/driver/owner.rb +32 -0
  56. data/lib/fluent/test/driver/parser.rb +30 -0
  57. data/lib/fluent/test/helpers.rb +54 -0
  58. data/lib/fluent/test/log.rb +73 -0
  59. data/lib/fluent/time.rb +71 -0
  60. data/lib/fluent/version.rb +1 -1
  61. data/test/compat/test_parser.rb +82 -0
  62. data/test/config/test_configure_proxy.rb +15 -0
  63. data/test/config/test_dsl.rb +180 -2
  64. data/test/helper.rb +2 -24
  65. data/test/plugin/test_in_gc_stat.rb +6 -6
  66. data/test/plugin/test_in_http.rb +49 -32
  67. data/test/plugin/{test_in_stream.rb → test_in_unix.rb} +1 -1
  68. data/test/plugin/test_out_buffered_stdout.rb +108 -0
  69. data/test/plugin/test_out_copy.rb +88 -127
  70. data/test/plugin/test_out_null.rb +29 -0
  71. data/test/plugin/test_out_relabel.rb +28 -0
  72. data/test/plugin/test_out_roundrobin.rb +35 -29
  73. data/test/plugin/test_out_stdout.rb +4 -4
  74. data/test/plugin/test_output_as_buffered.rb +51 -0
  75. data/test/plugin/test_output_as_buffered_secondary.rb +13 -0
  76. data/test/plugin/test_parser_apache.rb +38 -0
  77. data/test/plugin/test_parser_apache2.rb +38 -0
  78. data/test/plugin/test_parser_apache_error.rb +40 -0
  79. data/test/plugin/test_parser_base.rb +32 -0
  80. data/test/plugin/test_parser_csv.rb +94 -0
  81. data/test/plugin/test_parser_json.rb +107 -0
  82. data/test/plugin/test_parser_labeled_tsv.rb +129 -0
  83. data/test/plugin/test_parser_multiline.rb +100 -0
  84. data/test/plugin/test_parser_nginx.rb +42 -0
  85. data/test/plugin/test_parser_none.rb +53 -0
  86. data/test/plugin/test_parser_regexp.rb +110 -0
  87. data/test/plugin/test_parser_syslog.rb +66 -0
  88. data/test/plugin/test_parser_time.rb +46 -0
  89. data/test/plugin/test_parser_tsv.rb +125 -0
  90. data/test/plugin_helper/test_child_process.rb +11 -2
  91. data/test/plugin_helper/test_formatter.rb +212 -0
  92. data/test/plugin_helper/test_inject.rb +388 -0
  93. data/test/plugin_helper/test_parser.rb +223 -0
  94. data/test/plugin_helper/test_retry_state.rb +40 -40
  95. data/test/plugin_helper/test_storage.rb +77 -10
  96. data/test/scripts/fluent/plugin/out_test.rb +22 -17
  97. data/test/scripts/fluent/plugin/out_test2.rb +80 -0
  98. data/test/test_event.rb +57 -0
  99. data/test/test_formatter.rb +0 -178
  100. data/test/test_output.rb +2 -152
  101. data/test/test_root_agent.rb +3 -2
  102. data/test/test_supervisor.rb +93 -26
  103. data/test/test_time_formatter.rb +186 -0
  104. metadata +69 -7
  105. data/test/test_parser.rb +0 -1087
@@ -0,0 +1,388 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin_helper/inject'
3
+ require 'fluent/event'
4
+ require 'time'
5
+
6
+ class InjectHelperTest < Test::Unit::TestCase
7
+ class Dummy < Fluent::Plugin::TestBase
8
+ helpers :inject
9
+ end
10
+
11
+ def config_inject_section(hash = {})
12
+ config_element('ROOT', '', {}, [config_element('inject', '', hash)])
13
+ end
14
+
15
+ setup do
16
+ Fluent::Test.setup
17
+ @d = Dummy.new
18
+ end
19
+
20
+ teardown do
21
+ if @d
22
+ @d.stop unless @d.stopped?
23
+ @d.shutdown unless @d.shutdown?
24
+ @d.close unless @d.closed?
25
+ @d.terminate unless @d.terminated?
26
+ end
27
+ end
28
+
29
+ test 'do nothing in default' do
30
+ @d.configure(config_inject_section())
31
+ @d.start
32
+ assert_nil @d.instance_eval{ @_inject_hostname_key }
33
+ assert_nil @d.instance_eval{ @_inject_hostname }
34
+ assert_nil @d.instance_eval{ @_inject_tag_key }
35
+ assert_nil @d.instance_eval{ @_inject_time_key }
36
+ assert_nil @d.instance_eval{ @_inject_time_formatter }
37
+
38
+ time = event_time()
39
+ record = {"key1" => "value1", "key2" => 2}
40
+ assert_equal record, @d.inject_values_to_record('tag', time, record)
41
+ assert_equal record.object_id, @d.inject_values_to_record('tag', time, record).object_id
42
+
43
+ es0 = Fluent::OneEventStream.new(time, {"key1" => "v", "key2" => 0})
44
+
45
+ es1 = Fluent::ArrayEventStream.new([ [time, {"key1" => "a", "key2" => 1}], [time, {"key1" => "b", "key2" => 2}] ])
46
+
47
+ es2 = Fluent::MultiEventStream.new
48
+ es2.add(event_time(), {"key1" => "a", "key2" => 1})
49
+ es2.add(event_time(), {"key1" => "b", "key2" => 2})
50
+
51
+ es3 = Fluent::MessagePackEventStream.new(es2.to_msgpack_stream)
52
+
53
+ [es0, es1, es2, es3].each do |es|
54
+ assert_equal es, @d.inject_values_to_event_stream('tag', es), "failed for #{es.class}"
55
+ assert_equal es.object_id, @d.inject_values_to_event_stream('tag', es).object_id, "failed for #{es.class}"
56
+ end
57
+ end
58
+
59
+ test 'can be configured as specified' do
60
+ @d.configure(config_inject_section(
61
+ "hostname_key" => "hostname",
62
+ "hostname" => "myhost.local",
63
+ "tag_key" => "tag",
64
+ "time_key" => "time",
65
+ "time_type" => "string",
66
+ "time_format" => "%Y-%m-%d %H:%M:%S.%N",
67
+ "timezone" => "-0700",
68
+ ))
69
+
70
+ assert_equal "hostname", @d.instance_eval{ @_inject_hostname_key }
71
+ assert_equal "myhost.local", @d.instance_eval{ @_inject_hostname }
72
+ assert_equal "tag", @d.instance_eval{ @_inject_tag_key }
73
+ assert_equal "time", @d.instance_eval{ @_inject_time_key }
74
+ assert_equal :string, @d.instance_eval{ @inject_config.time_type }
75
+ assert_not_nil @d.instance_eval{ @_inject_time_formatter }
76
+ end
77
+
78
+ sub_test_case 'using inject_record' do
79
+ test 'injects hostname automatically detected' do
80
+ detected_hostname = `hostname`.chomp
81
+ @d.configure(config_inject_section("hostname_key" => "host"))
82
+ logs = @d.log.out.logs
83
+ assert{ logs.any?{|l| l.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } }
84
+ @d.start
85
+
86
+ time = event_time()
87
+ record = {"key1" => "value1", "key2" => 2}
88
+ assert_equal record.merge({"host" => detected_hostname}), @d.inject_values_to_record('tag', time, record)
89
+ end
90
+
91
+ test 'injects hostname as specified value' do
92
+ @d.configure(config_inject_section("hostname_key" => "host", "hostname" => "myhost.yay.local"))
93
+ @d.start
94
+
95
+ time = event_time()
96
+ record = {"key1" => "value1", "key2" => 2}
97
+ assert_equal record.merge({"host" => "myhost.yay.local"}), @d.inject_values_to_record('tag', time, record)
98
+ end
99
+
100
+ test 'injects tag into specified key' do
101
+ @d.configure(config_inject_section("tag_key" => "mytag"))
102
+ @d.start
103
+
104
+ time = event_time()
105
+ record = {"key1" => "value1", "key2" => 2}
106
+ assert_equal record.merge({"mytag" => "tag.test"}), @d.inject_values_to_record('tag.test', time, record)
107
+ end
108
+
109
+ test 'injects time as floating point value into specified key as default' do
110
+ time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i # 1466464211 in unix time
111
+ time_subsecond = 320_101_224
112
+ time = Fluent::EventTime.new(time_in_unix, time_subsecond)
113
+ float_time = 1466464211.320101 # microsecond precision in float
114
+
115
+ @d.configure(config_inject_section("time_key" => "timedata"))
116
+ @d.start
117
+
118
+ record = {"key1" => "value1", "key2" => 2}
119
+ assert_equal record.merge({"timedata" => float_time}), @d.inject_values_to_record('tag', time, record)
120
+ end
121
+
122
+ test 'injects time as unix time into specified key' do
123
+ time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i
124
+ time_subsecond = 320_101_224
125
+ time = Fluent::EventTime.new(time_in_unix, time_subsecond)
126
+ int_time = 1466464211
127
+
128
+ @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "unixtime"))
129
+ @d.start
130
+
131
+ record = {"key1" => "value1", "key2" => 2}
132
+ assert_equal record.merge({"timedata" => int_time}), @d.inject_values_to_record('tag', time, record)
133
+ end
134
+
135
+ test 'injects time as formatted string in localtime if timezone not specified' do
136
+ local_timezone = Time.now.strftime('%z')
137
+ time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i
138
+ time_subsecond = 320_101_224
139
+ time = Fluent::EventTime.new(time_in_unix, time_subsecond)
140
+
141
+ @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z"))
142
+ @d.start
143
+
144
+ record = {"key1" => "value1", "key2" => 2}
145
+ assert_equal record.merge({"timedata" => "2016_06_21 08:10:11 #{local_timezone}"}), @d.inject_values_to_record('tag', time, record)
146
+ end
147
+
148
+ test 'injects time as formatted string with nanosecond in localtime if timezone not specified' do
149
+ local_timezone = Time.now.strftime('%z')
150
+ time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i
151
+ time_subsecond = 320_101_224
152
+ time = Fluent::EventTime.new(time_in_unix, time_subsecond)
153
+
154
+ @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%N %z"))
155
+ @d.start
156
+
157
+ record = {"key1" => "value1", "key2" => 2}
158
+ assert_equal record.merge({"timedata" => "2016_06_21 08:10:11.320101224 #{local_timezone}"}), @d.inject_values_to_record('tag', time, record)
159
+ end
160
+
161
+ test 'injects time as formatted string with millisecond in localtime if timezone not specified' do
162
+ local_timezone = Time.now.strftime('%z')
163
+ time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i
164
+ time_subsecond = 320_101_224
165
+ time = Fluent::EventTime.new(time_in_unix, time_subsecond)
166
+
167
+ @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%3N %z"))
168
+ @d.start
169
+
170
+ record = {"key1" => "value1", "key2" => 2}
171
+ assert_equal record.merge({"timedata" => "2016_06_21 08:10:11.320 #{local_timezone}"}), @d.inject_values_to_record('tag', time, record)
172
+ end
173
+
174
+ test 'injects time as formatted string in specified timezone' do
175
+ time_in_unix = Time.parse("2016-06-21 08:10:11 +0000").to_i
176
+ time_subsecond = 320_101_224
177
+ time = Fluent::EventTime.new(time_in_unix, time_subsecond)
178
+
179
+ @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z", "timezone" => "-0800"))
180
+ @d.start
181
+
182
+ record = {"key1" => "value1", "key2" => 2}
183
+ assert_equal record.merge({"timedata" => "2016_06_21 00:10:11 -0800"}), @d.inject_values_to_record('tag', time, record)
184
+ end
185
+
186
+ test 'injects hostname, tag and time' do
187
+ time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i
188
+ time_subsecond = 320_101_224
189
+ time = Fluent::EventTime.new(time_in_unix, time_subsecond)
190
+
191
+ @d.configure(config_inject_section(
192
+ "hostname_key" => "hostnamedata",
193
+ "hostname" => "myname.local",
194
+ "tag_key" => "tagdata",
195
+ "time_key" => "timedata",
196
+ "time_type" => "string",
197
+ "time_format" => "%Y_%m_%d %H:%M:%S.%N %z",
198
+ "timezone" => "+0000",
199
+ ))
200
+ @d.start
201
+
202
+ record = {"key1" => "value1", "key2" => 2}
203
+ injected = {"hostnamedata" => "myname.local", "tagdata" => "tag", "timedata" => "2016_06_20 23:10:11.320101224 +0000"}
204
+ assert_equal record.merge(injected), @d.inject_values_to_record('tag', time, record)
205
+ end
206
+ end
207
+ sub_test_case 'using inject_event_stream' do
208
+ local_timezone = Time.now.strftime('%z')
209
+ time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i
210
+ time_subsecond = 320_101_224
211
+ time_in_rational = Rational(time_in_unix * 1_000_000_000 + time_subsecond, 1_000_000_000)
212
+ time_in_localtime = Time.at(time_in_rational).localtime
213
+ time_in_utc = Time.at(time_in_rational).utc
214
+ time = Fluent::EventTime.new(time_in_unix, time_subsecond)
215
+ time_float = time.to_r.truncate(+6).to_f
216
+
217
+ data(
218
+ "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
219
+ "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
220
+ )
221
+ test 'injects hostname automatically detected' do |data|
222
+ detected_hostname = `hostname`.chomp
223
+ @d.configure(config_inject_section("hostname_key" => "host"))
224
+ logs = @d.log.out.logs
225
+ assert{ logs.any?{|l| l.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } }
226
+ @d.start
227
+
228
+ injected = {"host" => detected_hostname}
229
+ expected_es = Fluent::MultiEventStream.new
230
+ data.each do |t, r|
231
+ expected_es.add(t, r.merge(injected))
232
+ end
233
+ assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
234
+ end
235
+
236
+ data(
237
+ "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
238
+ "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
239
+ )
240
+ test 'injects hostname as specified value' do |data|
241
+ @d.configure(config_inject_section("hostname_key" => "host", "hostname" => "myhost.yay.local"))
242
+ @d.start
243
+
244
+ injected = {"host" => "myhost.yay.local"}
245
+ expected_es = Fluent::MultiEventStream.new
246
+ data.each do |t, r|
247
+ expected_es.add(t, r.merge(injected))
248
+ end
249
+ assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
250
+ end
251
+
252
+ data(
253
+ "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
254
+ "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
255
+ )
256
+ test 'injects tag into specified key' do |data|
257
+ @d.configure(config_inject_section("tag_key" => "mytag"))
258
+ @d.start
259
+
260
+ injected = {"mytag" => "tag"}
261
+ expected_es = Fluent::MultiEventStream.new
262
+ data.each do |t, r|
263
+ expected_es.add(t, r.merge(injected))
264
+ end
265
+ assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
266
+ end
267
+
268
+ data(
269
+ "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
270
+ "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
271
+ )
272
+ test 'injects time as floating point value into specified key as default' do |data|
273
+ @d.configure(config_inject_section("time_key" => "timedata"))
274
+ @d.start
275
+
276
+ injected = {"timedata" => time_float }
277
+ expected_es = Fluent::MultiEventStream.new
278
+ data.each do |t, r|
279
+ expected_es.add(t, r.merge(injected))
280
+ end
281
+ assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
282
+ end
283
+
284
+ data(
285
+ "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
286
+ "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
287
+ )
288
+ test 'injects time as unix time into specified key' do |data|
289
+ @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "unixtime"))
290
+ @d.start
291
+
292
+ injected = {"timedata" => time_in_localtime.to_i}
293
+ expected_es = Fluent::MultiEventStream.new
294
+ data.each do |t, r|
295
+ expected_es.add(t, r.merge(injected))
296
+ end
297
+ assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
298
+ end
299
+
300
+ data(
301
+ "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
302
+ "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
303
+ )
304
+ test 'injects time as formatted string in localtime if timezone not specified' do |data|
305
+ @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z"))
306
+ @d.start
307
+
308
+ injected = {"timedata" => time_in_localtime.strftime("%Y_%m_%d %H:%M:%S %z")}
309
+ expected_es = Fluent::MultiEventStream.new
310
+ data.each do |t, r|
311
+ expected_es.add(t, r.merge(injected))
312
+ end
313
+ assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
314
+ end
315
+
316
+ data(
317
+ "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
318
+ "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
319
+ )
320
+ test 'injects time as formatted string with nanosecond in localtime if timezone not specified' do |data|
321
+ @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%N %z"))
322
+ @d.start
323
+
324
+ injected = {"timedata" => time_in_localtime.strftime("%Y_%m_%d %H:%M:%S.%N %z")}
325
+ expected_es = Fluent::MultiEventStream.new
326
+ data.each do |t, r|
327
+ expected_es.add(t, r.merge(injected))
328
+ end
329
+ assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
330
+ end
331
+
332
+ data(
333
+ "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
334
+ "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
335
+ )
336
+ test 'injects time as formatted string with millisecond in localtime if timezone not specified' do |data|
337
+ @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%3N %z"))
338
+ @d.start
339
+
340
+ injected = {"timedata" => time_in_localtime.strftime("%Y_%m_%d %H:%M:%S.%3N %z")}
341
+ expected_es = Fluent::MultiEventStream.new
342
+ data.each do |t, r|
343
+ expected_es.add(t, r.merge(injected))
344
+ end
345
+ assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
346
+ end
347
+
348
+ data(
349
+ "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
350
+ "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
351
+ )
352
+ test 'injects time as formatted string in specified timezone' do |data|
353
+ @d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z", "timezone" => "-0800"))
354
+ @d.start
355
+
356
+ injected = {"timedata" => Time.at(time_in_unix).localtime("-08:00").strftime("%Y_%m_%d %H:%M:%S -0800")}
357
+ expected_es = Fluent::MultiEventStream.new
358
+ data.each do |t, r|
359
+ expected_es.add(t, r.merge(injected))
360
+ end
361
+ assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
362
+ end
363
+
364
+ data(
365
+ "OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
366
+ "ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
367
+ )
368
+ test 'injects hostname, tag and time' do |data|
369
+ @d.configure(config_inject_section(
370
+ "hostname_key" => "hostnamedata",
371
+ "hostname" => "myname.local",
372
+ "tag_key" => "tagdata",
373
+ "time_key" => "timedata",
374
+ "time_type" => "string",
375
+ "time_format" => "%Y_%m_%d %H:%M:%S.%N %z",
376
+ "timezone" => "+0000",
377
+ ))
378
+ @d.start
379
+
380
+ injected = {"hostnamedata" => "myname.local", "tagdata" => "tag", "timedata" => time_in_utc.strftime("%Y_%m_%d %H:%M:%S.%N %z")}
381
+ expected_es = Fluent::MultiEventStream.new
382
+ data.each do |t, r|
383
+ expected_es.add(t, r.merge(injected))
384
+ end
385
+ assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
386
+ end
387
+ end
388
+ end
@@ -0,0 +1,223 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin_helper/parser'
3
+ require 'fluent/plugin/base'
4
+ require 'fluent/time'
5
+
6
+ class ParserHelperTest < Test::Unit::TestCase
7
+ class ExampleParser < Fluent::Plugin::Parser
8
+ Fluent::Plugin.register_parser('example', self)
9
+ def parse(text)
10
+ ary = text.split(/\s*,\s*/)
11
+ r = {}
12
+ ary.each_with_index do |v, i|
13
+ r[i.to_s] = v
14
+ end
15
+ yield Fluent::EventTime.now, r
16
+ end
17
+ end
18
+ class Example2Parser < Fluent::Plugin::Parser
19
+ Fluent::Plugin.register_parser('example2', self)
20
+ def parse(text)
21
+ ary = text.split(/\s*,\s*/)
22
+ r = {}
23
+ ary.each_with_index do |v, i|
24
+ r[i.to_s] = v
25
+ end
26
+ yield Fluent::EventTime.now, r
27
+ end
28
+ end
29
+ class Dummy < Fluent::Plugin::TestBase
30
+ helpers :parser
31
+ end
32
+
33
+ setup do
34
+ @d = nil
35
+ end
36
+
37
+ teardown do
38
+ if @d
39
+ @d.stop unless @d.stopped?
40
+ @d.shutdown unless @d.shutdown?
41
+ @d.close unless @d.closed?
42
+ @d.terminate unless @d.terminated?
43
+ end
44
+ end
45
+
46
+ test 'can be initialized without any parsers at first' do
47
+ d = Dummy.new
48
+ assert_equal 0, d._parsers.size
49
+ end
50
+
51
+ test 'can be configured without parse sections' do
52
+ d = Dummy.new
53
+ assert_nothing_raised do
54
+ d.configure(config_element())
55
+ end
56
+ assert_equal 0, d._parsers.size
57
+ end
58
+
59
+ test 'can be configured with a parse section' do
60
+ d = Dummy.new
61
+ conf = config_element('ROOT', '', {}, [
62
+ config_element('parse', '', {'@type' => 'example'})
63
+ ])
64
+ assert_nothing_raised do
65
+ d.configure(conf)
66
+ end
67
+ assert_equal 1, d._parsers.size
68
+ assert{ d._parsers.values.all?{ |parser| !parser.started? } }
69
+ end
70
+
71
+ test 'can be configured with 2 or more parse sections with different usages with each other' do
72
+ d = Dummy.new
73
+ conf = config_element('ROOT', '', {}, [
74
+ config_element('parse', 'default', {'@type' => 'example'}),
75
+ config_element('parse', 'extra', {'@type' => 'example2'}),
76
+ ])
77
+ assert_nothing_raised do
78
+ d.configure(conf)
79
+ end
80
+ assert_equal 2, d._parsers.size
81
+ assert{ d._parsers.values.all?{ |parser| !parser.started? } }
82
+ end
83
+
84
+ test 'cannot be configured with 2 parse sections with same usage' do
85
+ d = Dummy.new
86
+ conf = config_element('ROOT', '', {}, [
87
+ config_element('parse', 'default', {'@type' => 'example'}),
88
+ config_element('parse', 'extra', {'@type' => 'example2'}),
89
+ config_element('parse', 'extra', {'@type' => 'example2'}),
90
+ ])
91
+ assert_raises Fluent::ConfigError do
92
+ d.configure(conf)
93
+ end
94
+ end
95
+
96
+ test 'creates a parse plugin instance which is already configured without usage' do
97
+ @d = d = Dummy.new
98
+ conf = config_element('ROOT', '', {}, [
99
+ config_element('parse', '', {'@type' => 'example'})
100
+ ])
101
+ d.configure(conf)
102
+ d.start
103
+
104
+ parser = d.parser_create
105
+ assert{ parser.is_a? ExampleParser }
106
+ assert parser.started?
107
+ end
108
+
109
+ test 'creates a parser plugin instance which is already configured with usage' do
110
+ @d = d = Dummy.new
111
+ conf = config_element('ROOT', '', {}, [
112
+ config_element('parse', 'mydata', {'@type' => 'example'})
113
+ ])
114
+ d.configure(conf)
115
+ d.start
116
+
117
+ parser = d.parser_create(usage: 'mydata')
118
+ assert{ parser.is_a? ExampleParser }
119
+ assert parser.started?
120
+ end
121
+
122
+ test 'creates a parser plugin without configurations' do
123
+ @d = d = Dummy.new
124
+ d.configure(config_element())
125
+ d.start
126
+
127
+ parser = d.parser_create(usage: 'mydata', type: 'example', conf: config_element('parse', 'mydata'))
128
+ assert{ parser.is_a? ExampleParser }
129
+ assert parser.started?
130
+ end
131
+
132
+ test 'creates 2 or more parser plugin instances' do
133
+ @d = d = Dummy.new
134
+ conf = config_element('ROOT', '', {}, [
135
+ config_element('parse', 'mydata', {'@type' => 'example'}),
136
+ config_element('parse', 'secret', {'@type' => 'example2'})
137
+ ])
138
+ d.configure(conf)
139
+ d.start
140
+
141
+ p1 = d.parser_create(usage: 'mydata')
142
+ p2 = d.parser_create(usage: 'secret')
143
+ assert{ p1.is_a? ExampleParser }
144
+ assert p1.started?
145
+ assert{ p2.is_a? Example2Parser }
146
+ assert p2.started?
147
+ end
148
+
149
+ test 'calls lifecycle methods for all plugin instances via owner plugin' do
150
+ @d = d = Dummy.new
151
+ conf = config_element('ROOT', '', {}, [ config_element('parse', '', {'@type' => 'example'}), config_element('parse', 'e2', {'@type' => 'example'}) ])
152
+ d.configure(conf)
153
+ d.start
154
+
155
+ i1 = d.parser_create(usage: '')
156
+ i2 = d.parser_create(usage: 'e2')
157
+ i3 = d.parser_create(usage: 'e3', type: 'example2')
158
+
159
+ assert i1.started?
160
+ assert i2.started?
161
+ assert i3.started?
162
+
163
+ assert !i1.stopped?
164
+ assert !i2.stopped?
165
+ assert !i3.stopped?
166
+
167
+ d.stop
168
+
169
+ assert i1.stopped?
170
+ assert i2.stopped?
171
+ assert i3.stopped?
172
+
173
+ assert !i1.before_shutdown?
174
+ assert !i2.before_shutdown?
175
+ assert !i3.before_shutdown?
176
+
177
+ d.before_shutdown
178
+
179
+ assert i1.before_shutdown?
180
+ assert i2.before_shutdown?
181
+ assert i3.before_shutdown?
182
+
183
+ assert !i1.shutdown?
184
+ assert !i2.shutdown?
185
+ assert !i3.shutdown?
186
+
187
+ d.shutdown
188
+
189
+ assert i1.shutdown?
190
+ assert i2.shutdown?
191
+ assert i3.shutdown?
192
+
193
+ assert !i1.after_shutdown?
194
+ assert !i2.after_shutdown?
195
+ assert !i3.after_shutdown?
196
+
197
+ d.after_shutdown
198
+
199
+ assert i1.after_shutdown?
200
+ assert i2.after_shutdown?
201
+ assert i3.after_shutdown?
202
+
203
+ assert !i1.closed?
204
+ assert !i2.closed?
205
+ assert !i3.closed?
206
+
207
+ d.close
208
+
209
+ assert i1.closed?
210
+ assert i2.closed?
211
+ assert i3.closed?
212
+
213
+ assert !i1.terminated?
214
+ assert !i2.terminated?
215
+ assert !i3.terminated?
216
+
217
+ d.terminate
218
+
219
+ assert i1.terminated?
220
+ assert i2.terminated?
221
+ assert i3.terminated?
222
+ end
223
+ end