fluentd 0.14.0 → 0.14.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/example/copy_roundrobin.conf +39 -0
- data/example/filter_stdout.conf +5 -5
- data/example/in_forward.conf +2 -2
- data/example/in_http.conf +2 -2
- data/example/in_syslog.conf +2 -2
- data/example/in_tail.conf +2 -2
- data/example/in_tcp.conf +2 -2
- data/example/in_udp.conf +2 -2
- data/example/out_buffered_null.conf +32 -0
- data/example/out_copy.conf +4 -4
- data/example/out_file.conf +2 -2
- data/example/out_forward.conf +2 -2
- data/example/v0_12_filter.conf +8 -8
- data/fluentd.gemspec +1 -1
- data/lib/fluent/command/fluentd.rb +6 -1
- data/lib/fluent/compat/handle_tag_name_mixin.rb +53 -0
- data/lib/fluent/compat/input.rb +1 -0
- data/lib/fluent/compat/output.rb +1 -0
- data/lib/fluent/compat/record_filter_mixin.rb +34 -0
- data/lib/fluent/compat/set_tag_key_mixin.rb +50 -0
- data/lib/fluent/compat/set_time_key_mixin.rb +69 -0
- data/lib/fluent/compat/type_converter.rb +90 -0
- data/lib/fluent/config/configure_proxy.rb +24 -4
- data/lib/fluent/config/dsl.rb +18 -1
- data/lib/fluent/config/v1_parser.rb +3 -2
- data/lib/fluent/configurable.rb +1 -1
- data/lib/fluent/event.rb +37 -9
- data/lib/fluent/mixin.rb +12 -286
- data/lib/fluent/plugin/buffer.rb +2 -2
- data/lib/fluent/plugin/in_dummy.rb +5 -1
- data/lib/fluent/plugin/in_gc_stat.rb +7 -37
- data/lib/fluent/plugin/in_http.rb +2 -0
- data/lib/fluent/plugin/{in_stream.rb → in_unix.rb} +0 -0
- data/lib/fluent/plugin/out_buffered_stdout.rb +60 -0
- data/lib/fluent/plugin/out_copy.rb +8 -51
- data/lib/fluent/plugin/out_null.rb +5 -5
- data/lib/fluent/plugin/out_relabel.rb +5 -5
- data/lib/fluent/plugin/out_roundrobin.rb +13 -40
- data/lib/fluent/plugin/output.rb +9 -0
- data/lib/fluent/plugin_helper.rb +2 -0
- data/lib/fluent/plugin_helper/formatter.rb +138 -0
- data/lib/fluent/plugin_helper/inject.rb +112 -0
- data/lib/fluent/plugin_helper/parser.rb +138 -0
- data/lib/fluent/plugin_helper/storage.rb +64 -50
- data/lib/fluent/process.rb +6 -1
- data/lib/fluent/registry.rb +1 -1
- data/lib/fluent/supervisor.rb +20 -2
- data/lib/fluent/test.rb +30 -5
- data/lib/fluent/test/base.rb +2 -66
- data/lib/fluent/test/driver/base.rb +3 -0
- data/lib/fluent/test/driver/base_owned.rb +106 -0
- data/lib/fluent/test/driver/formatter.rb +30 -0
- data/lib/fluent/test/driver/multi_output.rb +52 -0
- data/lib/fluent/test/driver/owner.rb +32 -0
- data/lib/fluent/test/driver/parser.rb +30 -0
- data/lib/fluent/test/helpers.rb +54 -0
- data/lib/fluent/test/log.rb +73 -0
- data/lib/fluent/time.rb +71 -0
- data/lib/fluent/version.rb +1 -1
- data/test/compat/test_parser.rb +82 -0
- data/test/config/test_configure_proxy.rb +15 -0
- data/test/config/test_dsl.rb +180 -2
- data/test/helper.rb +2 -24
- data/test/plugin/test_in_gc_stat.rb +6 -6
- data/test/plugin/test_in_http.rb +49 -32
- data/test/plugin/{test_in_stream.rb → test_in_unix.rb} +1 -1
- data/test/plugin/test_out_buffered_stdout.rb +108 -0
- data/test/plugin/test_out_copy.rb +88 -127
- data/test/plugin/test_out_null.rb +29 -0
- data/test/plugin/test_out_relabel.rb +28 -0
- data/test/plugin/test_out_roundrobin.rb +35 -29
- data/test/plugin/test_out_stdout.rb +4 -4
- data/test/plugin/test_output_as_buffered.rb +51 -0
- data/test/plugin/test_output_as_buffered_secondary.rb +13 -0
- data/test/plugin/test_parser_apache.rb +38 -0
- data/test/plugin/test_parser_apache2.rb +38 -0
- data/test/plugin/test_parser_apache_error.rb +40 -0
- data/test/plugin/test_parser_base.rb +32 -0
- data/test/plugin/test_parser_csv.rb +94 -0
- data/test/plugin/test_parser_json.rb +107 -0
- data/test/plugin/test_parser_labeled_tsv.rb +129 -0
- data/test/plugin/test_parser_multiline.rb +100 -0
- data/test/plugin/test_parser_nginx.rb +42 -0
- data/test/plugin/test_parser_none.rb +53 -0
- data/test/plugin/test_parser_regexp.rb +110 -0
- data/test/plugin/test_parser_syslog.rb +66 -0
- data/test/plugin/test_parser_time.rb +46 -0
- data/test/plugin/test_parser_tsv.rb +125 -0
- data/test/plugin_helper/test_child_process.rb +11 -2
- data/test/plugin_helper/test_formatter.rb +212 -0
- data/test/plugin_helper/test_inject.rb +388 -0
- data/test/plugin_helper/test_parser.rb +223 -0
- data/test/plugin_helper/test_retry_state.rb +40 -40
- data/test/plugin_helper/test_storage.rb +77 -10
- data/test/scripts/fluent/plugin/out_test.rb +22 -17
- data/test/scripts/fluent/plugin/out_test2.rb +80 -0
- data/test/test_event.rb +57 -0
- data/test/test_formatter.rb +0 -178
- data/test/test_output.rb +2 -152
- data/test/test_root_agent.rb +3 -2
- data/test/test_supervisor.rb +93 -26
- data/test/test_time_formatter.rb +186 -0
- metadata +69 -7
- data/test/test_parser.rb +0 -1087
@@ -0,0 +1,388 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/plugin_helper/inject'
|
3
|
+
require 'fluent/event'
|
4
|
+
require 'time'
|
5
|
+
|
6
|
+
class InjectHelperTest < Test::Unit::TestCase
|
7
|
+
class Dummy < Fluent::Plugin::TestBase
|
8
|
+
helpers :inject
|
9
|
+
end
|
10
|
+
|
11
|
+
def config_inject_section(hash = {})
|
12
|
+
config_element('ROOT', '', {}, [config_element('inject', '', hash)])
|
13
|
+
end
|
14
|
+
|
15
|
+
setup do
|
16
|
+
Fluent::Test.setup
|
17
|
+
@d = Dummy.new
|
18
|
+
end
|
19
|
+
|
20
|
+
teardown do
|
21
|
+
if @d
|
22
|
+
@d.stop unless @d.stopped?
|
23
|
+
@d.shutdown unless @d.shutdown?
|
24
|
+
@d.close unless @d.closed?
|
25
|
+
@d.terminate unless @d.terminated?
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
test 'do nothing in default' do
|
30
|
+
@d.configure(config_inject_section())
|
31
|
+
@d.start
|
32
|
+
assert_nil @d.instance_eval{ @_inject_hostname_key }
|
33
|
+
assert_nil @d.instance_eval{ @_inject_hostname }
|
34
|
+
assert_nil @d.instance_eval{ @_inject_tag_key }
|
35
|
+
assert_nil @d.instance_eval{ @_inject_time_key }
|
36
|
+
assert_nil @d.instance_eval{ @_inject_time_formatter }
|
37
|
+
|
38
|
+
time = event_time()
|
39
|
+
record = {"key1" => "value1", "key2" => 2}
|
40
|
+
assert_equal record, @d.inject_values_to_record('tag', time, record)
|
41
|
+
assert_equal record.object_id, @d.inject_values_to_record('tag', time, record).object_id
|
42
|
+
|
43
|
+
es0 = Fluent::OneEventStream.new(time, {"key1" => "v", "key2" => 0})
|
44
|
+
|
45
|
+
es1 = Fluent::ArrayEventStream.new([ [time, {"key1" => "a", "key2" => 1}], [time, {"key1" => "b", "key2" => 2}] ])
|
46
|
+
|
47
|
+
es2 = Fluent::MultiEventStream.new
|
48
|
+
es2.add(event_time(), {"key1" => "a", "key2" => 1})
|
49
|
+
es2.add(event_time(), {"key1" => "b", "key2" => 2})
|
50
|
+
|
51
|
+
es3 = Fluent::MessagePackEventStream.new(es2.to_msgpack_stream)
|
52
|
+
|
53
|
+
[es0, es1, es2, es3].each do |es|
|
54
|
+
assert_equal es, @d.inject_values_to_event_stream('tag', es), "failed for #{es.class}"
|
55
|
+
assert_equal es.object_id, @d.inject_values_to_event_stream('tag', es).object_id, "failed for #{es.class}"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
test 'can be configured as specified' do
|
60
|
+
@d.configure(config_inject_section(
|
61
|
+
"hostname_key" => "hostname",
|
62
|
+
"hostname" => "myhost.local",
|
63
|
+
"tag_key" => "tag",
|
64
|
+
"time_key" => "time",
|
65
|
+
"time_type" => "string",
|
66
|
+
"time_format" => "%Y-%m-%d %H:%M:%S.%N",
|
67
|
+
"timezone" => "-0700",
|
68
|
+
))
|
69
|
+
|
70
|
+
assert_equal "hostname", @d.instance_eval{ @_inject_hostname_key }
|
71
|
+
assert_equal "myhost.local", @d.instance_eval{ @_inject_hostname }
|
72
|
+
assert_equal "tag", @d.instance_eval{ @_inject_tag_key }
|
73
|
+
assert_equal "time", @d.instance_eval{ @_inject_time_key }
|
74
|
+
assert_equal :string, @d.instance_eval{ @inject_config.time_type }
|
75
|
+
assert_not_nil @d.instance_eval{ @_inject_time_formatter }
|
76
|
+
end
|
77
|
+
|
78
|
+
sub_test_case 'using inject_record' do
|
79
|
+
test 'injects hostname automatically detected' do
|
80
|
+
detected_hostname = `hostname`.chomp
|
81
|
+
@d.configure(config_inject_section("hostname_key" => "host"))
|
82
|
+
logs = @d.log.out.logs
|
83
|
+
assert{ logs.any?{|l| l.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } }
|
84
|
+
@d.start
|
85
|
+
|
86
|
+
time = event_time()
|
87
|
+
record = {"key1" => "value1", "key2" => 2}
|
88
|
+
assert_equal record.merge({"host" => detected_hostname}), @d.inject_values_to_record('tag', time, record)
|
89
|
+
end
|
90
|
+
|
91
|
+
test 'injects hostname as specified value' do
|
92
|
+
@d.configure(config_inject_section("hostname_key" => "host", "hostname" => "myhost.yay.local"))
|
93
|
+
@d.start
|
94
|
+
|
95
|
+
time = event_time()
|
96
|
+
record = {"key1" => "value1", "key2" => 2}
|
97
|
+
assert_equal record.merge({"host" => "myhost.yay.local"}), @d.inject_values_to_record('tag', time, record)
|
98
|
+
end
|
99
|
+
|
100
|
+
test 'injects tag into specified key' do
|
101
|
+
@d.configure(config_inject_section("tag_key" => "mytag"))
|
102
|
+
@d.start
|
103
|
+
|
104
|
+
time = event_time()
|
105
|
+
record = {"key1" => "value1", "key2" => 2}
|
106
|
+
assert_equal record.merge({"mytag" => "tag.test"}), @d.inject_values_to_record('tag.test', time, record)
|
107
|
+
end
|
108
|
+
|
109
|
+
test 'injects time as floating point value into specified key as default' do
|
110
|
+
time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i # 1466464211 in unix time
|
111
|
+
time_subsecond = 320_101_224
|
112
|
+
time = Fluent::EventTime.new(time_in_unix, time_subsecond)
|
113
|
+
float_time = 1466464211.320101 # microsecond precision in float
|
114
|
+
|
115
|
+
@d.configure(config_inject_section("time_key" => "timedata"))
|
116
|
+
@d.start
|
117
|
+
|
118
|
+
record = {"key1" => "value1", "key2" => 2}
|
119
|
+
assert_equal record.merge({"timedata" => float_time}), @d.inject_values_to_record('tag', time, record)
|
120
|
+
end
|
121
|
+
|
122
|
+
test 'injects time as unix time into specified key' do
|
123
|
+
time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i
|
124
|
+
time_subsecond = 320_101_224
|
125
|
+
time = Fluent::EventTime.new(time_in_unix, time_subsecond)
|
126
|
+
int_time = 1466464211
|
127
|
+
|
128
|
+
@d.configure(config_inject_section("time_key" => "timedata", "time_type" => "unixtime"))
|
129
|
+
@d.start
|
130
|
+
|
131
|
+
record = {"key1" => "value1", "key2" => 2}
|
132
|
+
assert_equal record.merge({"timedata" => int_time}), @d.inject_values_to_record('tag', time, record)
|
133
|
+
end
|
134
|
+
|
135
|
+
test 'injects time as formatted string in localtime if timezone not specified' do
|
136
|
+
local_timezone = Time.now.strftime('%z')
|
137
|
+
time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i
|
138
|
+
time_subsecond = 320_101_224
|
139
|
+
time = Fluent::EventTime.new(time_in_unix, time_subsecond)
|
140
|
+
|
141
|
+
@d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z"))
|
142
|
+
@d.start
|
143
|
+
|
144
|
+
record = {"key1" => "value1", "key2" => 2}
|
145
|
+
assert_equal record.merge({"timedata" => "2016_06_21 08:10:11 #{local_timezone}"}), @d.inject_values_to_record('tag', time, record)
|
146
|
+
end
|
147
|
+
|
148
|
+
test 'injects time as formatted string with nanosecond in localtime if timezone not specified' do
|
149
|
+
local_timezone = Time.now.strftime('%z')
|
150
|
+
time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i
|
151
|
+
time_subsecond = 320_101_224
|
152
|
+
time = Fluent::EventTime.new(time_in_unix, time_subsecond)
|
153
|
+
|
154
|
+
@d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%N %z"))
|
155
|
+
@d.start
|
156
|
+
|
157
|
+
record = {"key1" => "value1", "key2" => 2}
|
158
|
+
assert_equal record.merge({"timedata" => "2016_06_21 08:10:11.320101224 #{local_timezone}"}), @d.inject_values_to_record('tag', time, record)
|
159
|
+
end
|
160
|
+
|
161
|
+
test 'injects time as formatted string with millisecond in localtime if timezone not specified' do
|
162
|
+
local_timezone = Time.now.strftime('%z')
|
163
|
+
time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i
|
164
|
+
time_subsecond = 320_101_224
|
165
|
+
time = Fluent::EventTime.new(time_in_unix, time_subsecond)
|
166
|
+
|
167
|
+
@d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%3N %z"))
|
168
|
+
@d.start
|
169
|
+
|
170
|
+
record = {"key1" => "value1", "key2" => 2}
|
171
|
+
assert_equal record.merge({"timedata" => "2016_06_21 08:10:11.320 #{local_timezone}"}), @d.inject_values_to_record('tag', time, record)
|
172
|
+
end
|
173
|
+
|
174
|
+
test 'injects time as formatted string in specified timezone' do
|
175
|
+
time_in_unix = Time.parse("2016-06-21 08:10:11 +0000").to_i
|
176
|
+
time_subsecond = 320_101_224
|
177
|
+
time = Fluent::EventTime.new(time_in_unix, time_subsecond)
|
178
|
+
|
179
|
+
@d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z", "timezone" => "-0800"))
|
180
|
+
@d.start
|
181
|
+
|
182
|
+
record = {"key1" => "value1", "key2" => 2}
|
183
|
+
assert_equal record.merge({"timedata" => "2016_06_21 00:10:11 -0800"}), @d.inject_values_to_record('tag', time, record)
|
184
|
+
end
|
185
|
+
|
186
|
+
test 'injects hostname, tag and time' do
|
187
|
+
time_in_unix = Time.parse("2016-06-21 08:10:11 +0900").to_i
|
188
|
+
time_subsecond = 320_101_224
|
189
|
+
time = Fluent::EventTime.new(time_in_unix, time_subsecond)
|
190
|
+
|
191
|
+
@d.configure(config_inject_section(
|
192
|
+
"hostname_key" => "hostnamedata",
|
193
|
+
"hostname" => "myname.local",
|
194
|
+
"tag_key" => "tagdata",
|
195
|
+
"time_key" => "timedata",
|
196
|
+
"time_type" => "string",
|
197
|
+
"time_format" => "%Y_%m_%d %H:%M:%S.%N %z",
|
198
|
+
"timezone" => "+0000",
|
199
|
+
))
|
200
|
+
@d.start
|
201
|
+
|
202
|
+
record = {"key1" => "value1", "key2" => 2}
|
203
|
+
injected = {"hostnamedata" => "myname.local", "tagdata" => "tag", "timedata" => "2016_06_20 23:10:11.320101224 +0000"}
|
204
|
+
assert_equal record.merge(injected), @d.inject_values_to_record('tag', time, record)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
sub_test_case 'using inject_event_stream' do
|
208
|
+
local_timezone = Time.now.strftime('%z')
|
209
|
+
time_in_unix = Time.parse("2016-06-21 08:10:11 #{local_timezone}").to_i
|
210
|
+
time_subsecond = 320_101_224
|
211
|
+
time_in_rational = Rational(time_in_unix * 1_000_000_000 + time_subsecond, 1_000_000_000)
|
212
|
+
time_in_localtime = Time.at(time_in_rational).localtime
|
213
|
+
time_in_utc = Time.at(time_in_rational).utc
|
214
|
+
time = Fluent::EventTime.new(time_in_unix, time_subsecond)
|
215
|
+
time_float = time.to_r.truncate(+6).to_f
|
216
|
+
|
217
|
+
data(
|
218
|
+
"OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
|
219
|
+
"ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
|
220
|
+
)
|
221
|
+
test 'injects hostname automatically detected' do |data|
|
222
|
+
detected_hostname = `hostname`.chomp
|
223
|
+
@d.configure(config_inject_section("hostname_key" => "host"))
|
224
|
+
logs = @d.log.out.logs
|
225
|
+
assert{ logs.any?{|l| l.include?("[info]: using hostname for specified field host_key=\"host\" host_name=\"#{detected_hostname}\"") } }
|
226
|
+
@d.start
|
227
|
+
|
228
|
+
injected = {"host" => detected_hostname}
|
229
|
+
expected_es = Fluent::MultiEventStream.new
|
230
|
+
data.each do |t, r|
|
231
|
+
expected_es.add(t, r.merge(injected))
|
232
|
+
end
|
233
|
+
assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
|
234
|
+
end
|
235
|
+
|
236
|
+
data(
|
237
|
+
"OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
|
238
|
+
"ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
|
239
|
+
)
|
240
|
+
test 'injects hostname as specified value' do |data|
|
241
|
+
@d.configure(config_inject_section("hostname_key" => "host", "hostname" => "myhost.yay.local"))
|
242
|
+
@d.start
|
243
|
+
|
244
|
+
injected = {"host" => "myhost.yay.local"}
|
245
|
+
expected_es = Fluent::MultiEventStream.new
|
246
|
+
data.each do |t, r|
|
247
|
+
expected_es.add(t, r.merge(injected))
|
248
|
+
end
|
249
|
+
assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
|
250
|
+
end
|
251
|
+
|
252
|
+
data(
|
253
|
+
"OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
|
254
|
+
"ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
|
255
|
+
)
|
256
|
+
test 'injects tag into specified key' do |data|
|
257
|
+
@d.configure(config_inject_section("tag_key" => "mytag"))
|
258
|
+
@d.start
|
259
|
+
|
260
|
+
injected = {"mytag" => "tag"}
|
261
|
+
expected_es = Fluent::MultiEventStream.new
|
262
|
+
data.each do |t, r|
|
263
|
+
expected_es.add(t, r.merge(injected))
|
264
|
+
end
|
265
|
+
assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
|
266
|
+
end
|
267
|
+
|
268
|
+
data(
|
269
|
+
"OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
|
270
|
+
"ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
|
271
|
+
)
|
272
|
+
test 'injects time as floating point value into specified key as default' do |data|
|
273
|
+
@d.configure(config_inject_section("time_key" => "timedata"))
|
274
|
+
@d.start
|
275
|
+
|
276
|
+
injected = {"timedata" => time_float }
|
277
|
+
expected_es = Fluent::MultiEventStream.new
|
278
|
+
data.each do |t, r|
|
279
|
+
expected_es.add(t, r.merge(injected))
|
280
|
+
end
|
281
|
+
assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
|
282
|
+
end
|
283
|
+
|
284
|
+
data(
|
285
|
+
"OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
|
286
|
+
"ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
|
287
|
+
)
|
288
|
+
test 'injects time as unix time into specified key' do |data|
|
289
|
+
@d.configure(config_inject_section("time_key" => "timedata", "time_type" => "unixtime"))
|
290
|
+
@d.start
|
291
|
+
|
292
|
+
injected = {"timedata" => time_in_localtime.to_i}
|
293
|
+
expected_es = Fluent::MultiEventStream.new
|
294
|
+
data.each do |t, r|
|
295
|
+
expected_es.add(t, r.merge(injected))
|
296
|
+
end
|
297
|
+
assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
|
298
|
+
end
|
299
|
+
|
300
|
+
data(
|
301
|
+
"OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
|
302
|
+
"ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
|
303
|
+
)
|
304
|
+
test 'injects time as formatted string in localtime if timezone not specified' do |data|
|
305
|
+
@d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z"))
|
306
|
+
@d.start
|
307
|
+
|
308
|
+
injected = {"timedata" => time_in_localtime.strftime("%Y_%m_%d %H:%M:%S %z")}
|
309
|
+
expected_es = Fluent::MultiEventStream.new
|
310
|
+
data.each do |t, r|
|
311
|
+
expected_es.add(t, r.merge(injected))
|
312
|
+
end
|
313
|
+
assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
|
314
|
+
end
|
315
|
+
|
316
|
+
data(
|
317
|
+
"OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
|
318
|
+
"ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
|
319
|
+
)
|
320
|
+
test 'injects time as formatted string with nanosecond in localtime if timezone not specified' do |data|
|
321
|
+
@d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%N %z"))
|
322
|
+
@d.start
|
323
|
+
|
324
|
+
injected = {"timedata" => time_in_localtime.strftime("%Y_%m_%d %H:%M:%S.%N %z")}
|
325
|
+
expected_es = Fluent::MultiEventStream.new
|
326
|
+
data.each do |t, r|
|
327
|
+
expected_es.add(t, r.merge(injected))
|
328
|
+
end
|
329
|
+
assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
|
330
|
+
end
|
331
|
+
|
332
|
+
data(
|
333
|
+
"OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
|
334
|
+
"ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
|
335
|
+
)
|
336
|
+
test 'injects time as formatted string with millisecond in localtime if timezone not specified' do |data|
|
337
|
+
@d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S.%3N %z"))
|
338
|
+
@d.start
|
339
|
+
|
340
|
+
injected = {"timedata" => time_in_localtime.strftime("%Y_%m_%d %H:%M:%S.%3N %z")}
|
341
|
+
expected_es = Fluent::MultiEventStream.new
|
342
|
+
data.each do |t, r|
|
343
|
+
expected_es.add(t, r.merge(injected))
|
344
|
+
end
|
345
|
+
assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
|
346
|
+
end
|
347
|
+
|
348
|
+
data(
|
349
|
+
"OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
|
350
|
+
"ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
|
351
|
+
)
|
352
|
+
test 'injects time as formatted string in specified timezone' do |data|
|
353
|
+
@d.configure(config_inject_section("time_key" => "timedata", "time_type" => "string", "time_format" => "%Y_%m_%d %H:%M:%S %z", "timezone" => "-0800"))
|
354
|
+
@d.start
|
355
|
+
|
356
|
+
injected = {"timedata" => Time.at(time_in_unix).localtime("-08:00").strftime("%Y_%m_%d %H:%M:%S -0800")}
|
357
|
+
expected_es = Fluent::MultiEventStream.new
|
358
|
+
data.each do |t, r|
|
359
|
+
expected_es.add(t, r.merge(injected))
|
360
|
+
end
|
361
|
+
assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
|
362
|
+
end
|
363
|
+
|
364
|
+
data(
|
365
|
+
"OneEventStream" => Fluent::OneEventStream.new(time, {"key1" => "value1", "key2" => 0}),
|
366
|
+
"ArrayEventStream" => Fluent::ArrayEventStream.new([ [time, {"key1" => "value1", "key2" => 1}], [time, {"key1" => "value2", "key2" => 2}] ]),
|
367
|
+
)
|
368
|
+
test 'injects hostname, tag and time' do |data|
|
369
|
+
@d.configure(config_inject_section(
|
370
|
+
"hostname_key" => "hostnamedata",
|
371
|
+
"hostname" => "myname.local",
|
372
|
+
"tag_key" => "tagdata",
|
373
|
+
"time_key" => "timedata",
|
374
|
+
"time_type" => "string",
|
375
|
+
"time_format" => "%Y_%m_%d %H:%M:%S.%N %z",
|
376
|
+
"timezone" => "+0000",
|
377
|
+
))
|
378
|
+
@d.start
|
379
|
+
|
380
|
+
injected = {"hostnamedata" => "myname.local", "tagdata" => "tag", "timedata" => time_in_utc.strftime("%Y_%m_%d %H:%M:%S.%N %z")}
|
381
|
+
expected_es = Fluent::MultiEventStream.new
|
382
|
+
data.each do |t, r|
|
383
|
+
expected_es.add(t, r.merge(injected))
|
384
|
+
end
|
385
|
+
assert_equal expected_es, @d.inject_values_to_event_stream('tag', data)
|
386
|
+
end
|
387
|
+
end
|
388
|
+
end
|
@@ -0,0 +1,223 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/plugin_helper/parser'
|
3
|
+
require 'fluent/plugin/base'
|
4
|
+
require 'fluent/time'
|
5
|
+
|
6
|
+
class ParserHelperTest < Test::Unit::TestCase
|
7
|
+
class ExampleParser < Fluent::Plugin::Parser
|
8
|
+
Fluent::Plugin.register_parser('example', self)
|
9
|
+
def parse(text)
|
10
|
+
ary = text.split(/\s*,\s*/)
|
11
|
+
r = {}
|
12
|
+
ary.each_with_index do |v, i|
|
13
|
+
r[i.to_s] = v
|
14
|
+
end
|
15
|
+
yield Fluent::EventTime.now, r
|
16
|
+
end
|
17
|
+
end
|
18
|
+
class Example2Parser < Fluent::Plugin::Parser
|
19
|
+
Fluent::Plugin.register_parser('example2', self)
|
20
|
+
def parse(text)
|
21
|
+
ary = text.split(/\s*,\s*/)
|
22
|
+
r = {}
|
23
|
+
ary.each_with_index do |v, i|
|
24
|
+
r[i.to_s] = v
|
25
|
+
end
|
26
|
+
yield Fluent::EventTime.now, r
|
27
|
+
end
|
28
|
+
end
|
29
|
+
class Dummy < Fluent::Plugin::TestBase
|
30
|
+
helpers :parser
|
31
|
+
end
|
32
|
+
|
33
|
+
setup do
|
34
|
+
@d = nil
|
35
|
+
end
|
36
|
+
|
37
|
+
teardown do
|
38
|
+
if @d
|
39
|
+
@d.stop unless @d.stopped?
|
40
|
+
@d.shutdown unless @d.shutdown?
|
41
|
+
@d.close unless @d.closed?
|
42
|
+
@d.terminate unless @d.terminated?
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
test 'can be initialized without any parsers at first' do
|
47
|
+
d = Dummy.new
|
48
|
+
assert_equal 0, d._parsers.size
|
49
|
+
end
|
50
|
+
|
51
|
+
test 'can be configured without parse sections' do
|
52
|
+
d = Dummy.new
|
53
|
+
assert_nothing_raised do
|
54
|
+
d.configure(config_element())
|
55
|
+
end
|
56
|
+
assert_equal 0, d._parsers.size
|
57
|
+
end
|
58
|
+
|
59
|
+
test 'can be configured with a parse section' do
|
60
|
+
d = Dummy.new
|
61
|
+
conf = config_element('ROOT', '', {}, [
|
62
|
+
config_element('parse', '', {'@type' => 'example'})
|
63
|
+
])
|
64
|
+
assert_nothing_raised do
|
65
|
+
d.configure(conf)
|
66
|
+
end
|
67
|
+
assert_equal 1, d._parsers.size
|
68
|
+
assert{ d._parsers.values.all?{ |parser| !parser.started? } }
|
69
|
+
end
|
70
|
+
|
71
|
+
test 'can be configured with 2 or more parse sections with different usages with each other' do
|
72
|
+
d = Dummy.new
|
73
|
+
conf = config_element('ROOT', '', {}, [
|
74
|
+
config_element('parse', 'default', {'@type' => 'example'}),
|
75
|
+
config_element('parse', 'extra', {'@type' => 'example2'}),
|
76
|
+
])
|
77
|
+
assert_nothing_raised do
|
78
|
+
d.configure(conf)
|
79
|
+
end
|
80
|
+
assert_equal 2, d._parsers.size
|
81
|
+
assert{ d._parsers.values.all?{ |parser| !parser.started? } }
|
82
|
+
end
|
83
|
+
|
84
|
+
test 'cannot be configured with 2 parse sections with same usage' do
|
85
|
+
d = Dummy.new
|
86
|
+
conf = config_element('ROOT', '', {}, [
|
87
|
+
config_element('parse', 'default', {'@type' => 'example'}),
|
88
|
+
config_element('parse', 'extra', {'@type' => 'example2'}),
|
89
|
+
config_element('parse', 'extra', {'@type' => 'example2'}),
|
90
|
+
])
|
91
|
+
assert_raises Fluent::ConfigError do
|
92
|
+
d.configure(conf)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
test 'creates a parse plugin instance which is already configured without usage' do
|
97
|
+
@d = d = Dummy.new
|
98
|
+
conf = config_element('ROOT', '', {}, [
|
99
|
+
config_element('parse', '', {'@type' => 'example'})
|
100
|
+
])
|
101
|
+
d.configure(conf)
|
102
|
+
d.start
|
103
|
+
|
104
|
+
parser = d.parser_create
|
105
|
+
assert{ parser.is_a? ExampleParser }
|
106
|
+
assert parser.started?
|
107
|
+
end
|
108
|
+
|
109
|
+
test 'creates a parser plugin instance which is already configured with usage' do
|
110
|
+
@d = d = Dummy.new
|
111
|
+
conf = config_element('ROOT', '', {}, [
|
112
|
+
config_element('parse', 'mydata', {'@type' => 'example'})
|
113
|
+
])
|
114
|
+
d.configure(conf)
|
115
|
+
d.start
|
116
|
+
|
117
|
+
parser = d.parser_create(usage: 'mydata')
|
118
|
+
assert{ parser.is_a? ExampleParser }
|
119
|
+
assert parser.started?
|
120
|
+
end
|
121
|
+
|
122
|
+
test 'creates a parser plugin without configurations' do
|
123
|
+
@d = d = Dummy.new
|
124
|
+
d.configure(config_element())
|
125
|
+
d.start
|
126
|
+
|
127
|
+
parser = d.parser_create(usage: 'mydata', type: 'example', conf: config_element('parse', 'mydata'))
|
128
|
+
assert{ parser.is_a? ExampleParser }
|
129
|
+
assert parser.started?
|
130
|
+
end
|
131
|
+
|
132
|
+
test 'creates 2 or more parser plugin instances' do
|
133
|
+
@d = d = Dummy.new
|
134
|
+
conf = config_element('ROOT', '', {}, [
|
135
|
+
config_element('parse', 'mydata', {'@type' => 'example'}),
|
136
|
+
config_element('parse', 'secret', {'@type' => 'example2'})
|
137
|
+
])
|
138
|
+
d.configure(conf)
|
139
|
+
d.start
|
140
|
+
|
141
|
+
p1 = d.parser_create(usage: 'mydata')
|
142
|
+
p2 = d.parser_create(usage: 'secret')
|
143
|
+
assert{ p1.is_a? ExampleParser }
|
144
|
+
assert p1.started?
|
145
|
+
assert{ p2.is_a? Example2Parser }
|
146
|
+
assert p2.started?
|
147
|
+
end
|
148
|
+
|
149
|
+
test 'calls lifecycle methods for all plugin instances via owner plugin' do
|
150
|
+
@d = d = Dummy.new
|
151
|
+
conf = config_element('ROOT', '', {}, [ config_element('parse', '', {'@type' => 'example'}), config_element('parse', 'e2', {'@type' => 'example'}) ])
|
152
|
+
d.configure(conf)
|
153
|
+
d.start
|
154
|
+
|
155
|
+
i1 = d.parser_create(usage: '')
|
156
|
+
i2 = d.parser_create(usage: 'e2')
|
157
|
+
i3 = d.parser_create(usage: 'e3', type: 'example2')
|
158
|
+
|
159
|
+
assert i1.started?
|
160
|
+
assert i2.started?
|
161
|
+
assert i3.started?
|
162
|
+
|
163
|
+
assert !i1.stopped?
|
164
|
+
assert !i2.stopped?
|
165
|
+
assert !i3.stopped?
|
166
|
+
|
167
|
+
d.stop
|
168
|
+
|
169
|
+
assert i1.stopped?
|
170
|
+
assert i2.stopped?
|
171
|
+
assert i3.stopped?
|
172
|
+
|
173
|
+
assert !i1.before_shutdown?
|
174
|
+
assert !i2.before_shutdown?
|
175
|
+
assert !i3.before_shutdown?
|
176
|
+
|
177
|
+
d.before_shutdown
|
178
|
+
|
179
|
+
assert i1.before_shutdown?
|
180
|
+
assert i2.before_shutdown?
|
181
|
+
assert i3.before_shutdown?
|
182
|
+
|
183
|
+
assert !i1.shutdown?
|
184
|
+
assert !i2.shutdown?
|
185
|
+
assert !i3.shutdown?
|
186
|
+
|
187
|
+
d.shutdown
|
188
|
+
|
189
|
+
assert i1.shutdown?
|
190
|
+
assert i2.shutdown?
|
191
|
+
assert i3.shutdown?
|
192
|
+
|
193
|
+
assert !i1.after_shutdown?
|
194
|
+
assert !i2.after_shutdown?
|
195
|
+
assert !i3.after_shutdown?
|
196
|
+
|
197
|
+
d.after_shutdown
|
198
|
+
|
199
|
+
assert i1.after_shutdown?
|
200
|
+
assert i2.after_shutdown?
|
201
|
+
assert i3.after_shutdown?
|
202
|
+
|
203
|
+
assert !i1.closed?
|
204
|
+
assert !i2.closed?
|
205
|
+
assert !i3.closed?
|
206
|
+
|
207
|
+
d.close
|
208
|
+
|
209
|
+
assert i1.closed?
|
210
|
+
assert i2.closed?
|
211
|
+
assert i3.closed?
|
212
|
+
|
213
|
+
assert !i1.terminated?
|
214
|
+
assert !i2.terminated?
|
215
|
+
assert !i3.terminated?
|
216
|
+
|
217
|
+
d.terminate
|
218
|
+
|
219
|
+
assert i1.terminated?
|
220
|
+
assert i2.terminated?
|
221
|
+
assert i3.terminated?
|
222
|
+
end
|
223
|
+
end
|