fluentd 0.14.7-x86-mingw32 → 0.14.10-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/.travis.yml +2 -0
- data/CONTRIBUTING.md +6 -1
- data/ChangeLog +95 -0
- data/Rakefile +21 -0
- data/appveyor.yml +1 -0
- data/code-of-conduct.md +3 -0
- data/example/out_exec_filter.conf +42 -0
- data/fluentd.gemspec +1 -1
- data/lib/fluent/agent.rb +2 -2
- data/lib/fluent/command/binlog_reader.rb +1 -1
- data/lib/fluent/command/cat.rb +15 -4
- data/lib/fluent/compat/output.rb +14 -9
- data/lib/fluent/compat/parser.rb +141 -11
- data/lib/fluent/config/configure_proxy.rb +2 -11
- data/lib/fluent/config/section.rb +8 -1
- data/lib/fluent/configurable.rb +1 -3
- data/lib/fluent/env.rb +1 -1
- data/lib/fluent/log.rb +1 -1
- data/lib/fluent/plugin/base.rb +17 -0
- data/lib/fluent/plugin/filter_parser.rb +108 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
- data/lib/fluent/plugin/filter_stdout.rb +1 -1
- data/lib/fluent/plugin/formatter.rb +5 -0
- data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
- data/lib/fluent/plugin/formatter_stdout.rb +3 -2
- data/lib/fluent/plugin/formatter_tsv.rb +34 -0
- data/lib/fluent/plugin/in_exec.rb +48 -93
- data/lib/fluent/plugin/in_forward.rb +66 -265
- data/lib/fluent/plugin/in_http.rb +68 -65
- data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
- data/lib/fluent/plugin/in_syslog.rb +42 -58
- data/lib/fluent/plugin/in_tail.rb +29 -14
- data/lib/fluent/plugin/in_tcp.rb +54 -14
- data/lib/fluent/plugin/in_udp.rb +49 -13
- data/lib/fluent/plugin/multi_output.rb +1 -3
- data/lib/fluent/plugin/out_exec.rb +58 -71
- data/lib/fluent/plugin/out_exec_filter.rb +199 -279
- data/lib/fluent/plugin/out_file.rb +172 -81
- data/lib/fluent/plugin/out_forward.rb +229 -206
- data/lib/fluent/plugin/out_stdout.rb +6 -21
- data/lib/fluent/plugin/output.rb +90 -59
- data/lib/fluent/plugin/parser.rb +121 -61
- data/lib/fluent/plugin/parser_csv.rb +9 -3
- data/lib/fluent/plugin/parser_json.rb +37 -35
- data/lib/fluent/plugin/parser_ltsv.rb +11 -19
- data/lib/fluent/plugin/parser_msgpack.rb +50 -0
- data/lib/fluent/plugin/parser_regexp.rb +15 -42
- data/lib/fluent/plugin/parser_tsv.rb +8 -3
- data/lib/fluent/plugin_helper.rb +10 -1
- data/lib/fluent/plugin_helper/child_process.rb +139 -73
- data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
- data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
- data/lib/fluent/plugin_helper/event_loop.rb +24 -6
- data/lib/fluent/plugin_helper/extract.rb +16 -4
- data/lib/fluent/plugin_helper/formatter.rb +9 -11
- data/lib/fluent/plugin_helper/inject.rb +16 -1
- data/lib/fluent/plugin_helper/parser.rb +3 -3
- data/lib/fluent/plugin_helper/server.rb +494 -0
- data/lib/fluent/plugin_helper/socket.rb +101 -0
- data/lib/fluent/plugin_helper/socket_option.rb +84 -0
- data/lib/fluent/plugin_helper/timer.rb +1 -0
- data/lib/fluent/root_agent.rb +1 -1
- data/lib/fluent/test/driver/base.rb +95 -49
- data/lib/fluent/test/driver/base_owner.rb +18 -8
- data/lib/fluent/test/driver/multi_output.rb +2 -1
- data/lib/fluent/test/driver/output.rb +29 -6
- data/lib/fluent/test/helpers.rb +3 -1
- data/lib/fluent/test/log.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +13 -0
- data/lib/fluent/time.rb +14 -8
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +1 -1
- data/test/command/test_binlog_reader.rb +5 -1
- data/test/compat/test_parser.rb +10 -0
- data/test/config/test_configurable.rb +193 -0
- data/test/config/test_configure_proxy.rb +0 -43
- data/test/helper.rb +36 -1
- data/test/plugin/test_base.rb +16 -0
- data/test/plugin/test_filter_parser.rb +665 -0
- data/test/plugin/test_filter_record_transformer.rb +36 -100
- data/test/plugin/test_filter_stdout.rb +18 -27
- data/test/plugin/test_in_dummy.rb +1 -1
- data/test/plugin/test_in_exec.rb +206 -94
- data/test/plugin/test_in_forward.rb +268 -347
- data/test/plugin/test_in_http.rb +310 -186
- data/test/plugin/test_in_monitor_agent.rb +65 -35
- data/test/plugin/test_in_syslog.rb +39 -3
- data/test/plugin/test_in_tcp.rb +78 -62
- data/test/plugin/test_in_udp.rb +101 -80
- data/test/plugin/test_out_exec.rb +223 -68
- data/test/plugin/test_out_exec_filter.rb +520 -169
- data/test/plugin/test_out_file.rb +637 -177
- data/test/plugin/test_out_forward.rb +242 -234
- data/test/plugin/test_out_null.rb +1 -1
- data/test/plugin/test_out_secondary_file.rb +4 -2
- data/test/plugin/test_out_stdout.rb +14 -35
- data/test/plugin/test_output_as_buffered.rb +60 -2
- data/test/plugin/test_parser.rb +359 -0
- data/test/plugin/test_parser_csv.rb +1 -2
- data/test/plugin/test_parser_json.rb +3 -4
- data/test/plugin/test_parser_labeled_tsv.rb +1 -2
- data/test/plugin/test_parser_none.rb +1 -2
- data/test/plugin/test_parser_regexp.rb +8 -4
- data/test/plugin/test_parser_tsv.rb +4 -3
- data/test/plugin_helper/test_child_process.rb +184 -0
- data/test/plugin_helper/test_compat_parameters.rb +88 -1
- data/test/plugin_helper/test_extract.rb +0 -1
- data/test/plugin_helper/test_formatter.rb +5 -2
- data/test/plugin_helper/test_inject.rb +21 -0
- data/test/plugin_helper/test_parser.rb +6 -5
- data/test/plugin_helper/test_server.rb +905 -0
- data/test/test_event_time.rb +3 -1
- data/test/test_output.rb +53 -2
- data/test/test_plugin_classes.rb +20 -0
- data/test/test_root_agent.rb +139 -0
- data/test/test_test_drivers.rb +135 -0
- metadata +28 -8
- data/test/plugin/test_parser_base.rb +0 -32
@@ -80,7 +80,7 @@ class NullOutputTest < Test::Unit::TestCase
|
|
80
80
|
d.instance.delayed = true
|
81
81
|
|
82
82
|
t = event_time("2016-05-23 00:22:13 -0800")
|
83
|
-
d.run(default_tag: 'test', flush: true, shutdown: false) do
|
83
|
+
d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
|
84
84
|
d.feed(t, {"message" => "null null null"})
|
85
85
|
d.feed(t, {"message" => "null null"})
|
86
86
|
d.feed(t, {"message" => "null"})
|
@@ -97,7 +97,9 @@ class FileOutputSecondaryTest < Test::Unit::TestCase
|
|
97
97
|
|
98
98
|
test 'should be passed directory' do
|
99
99
|
assert_raise Fluent::ConfigError do
|
100
|
-
|
100
|
+
i = Fluent::Plugin::SecondaryFileOutput.new
|
101
|
+
i.acts_as_secondary(create_primary)
|
102
|
+
i.configure(config_element())
|
101
103
|
end
|
102
104
|
|
103
105
|
assert_nothing_raised do
|
@@ -166,7 +168,7 @@ class FileOutputSecondaryTest < Test::Unit::TestCase
|
|
166
168
|
sleep 0.1 until File.stat(path).size == msgpack_binary.size
|
167
169
|
end
|
168
170
|
|
169
|
-
assert_equal File.
|
171
|
+
assert_equal msgpack_binary, File.binread(path)
|
170
172
|
end
|
171
173
|
|
172
174
|
test 'path should be incremental when append option is false' do
|
@@ -18,15 +18,19 @@ class StdoutOutputTest < Test::Unit::TestCase
|
|
18
18
|
sub_test_case 'non-buffered' do
|
19
19
|
test 'configure' do
|
20
20
|
d = create_driver
|
21
|
-
assert_equal
|
21
|
+
assert_equal 1, d.instance.formatter_configs.size # init: true
|
22
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
23
|
+
assert_equal 'json', d.instance.formatter.output_type
|
22
24
|
end
|
23
25
|
|
24
26
|
test 'configure output_type' do
|
25
27
|
d = create_driver(CONFIG + "\noutput_type json")
|
26
|
-
|
28
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
29
|
+
assert_equal 'json', d.instance.formatter.output_type
|
27
30
|
|
28
31
|
d = create_driver(CONFIG + "\noutput_type hash")
|
29
|
-
|
32
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
33
|
+
assert_equal 'hash', d.instance.formatter.output_type
|
30
34
|
|
31
35
|
assert_raise(Fluent::ConfigError) do
|
32
36
|
d = create_driver(CONFIG + "\noutput_type foo")
|
@@ -83,7 +87,9 @@ class StdoutOutputTest < Test::Unit::TestCase
|
|
83
87
|
sub_test_case 'buffered' do
|
84
88
|
test 'configure' do
|
85
89
|
d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
|
86
|
-
assert_equal
|
90
|
+
assert_equal 1, d.instance.formatter_configs.size
|
91
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
92
|
+
assert_equal 'json', d.instance.formatter.output_type
|
87
93
|
assert_equal 10 * 1024, d.instance.buffer_config.chunk_limit_size
|
88
94
|
assert d.instance.buffer_config.flush_at_shutdown
|
89
95
|
assert_equal ['tag'], d.instance.buffer_config.chunk_keys
|
@@ -94,10 +100,12 @@ class StdoutOutputTest < Test::Unit::TestCase
|
|
94
100
|
|
95
101
|
test 'configure with output_type' do
|
96
102
|
d = create_driver(config_element("ROOT", "", {"output_type" => "json"}, [config_element("buffer")]))
|
97
|
-
|
103
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
104
|
+
assert_equal 'json', d.instance.formatter.output_type
|
98
105
|
|
99
106
|
d = create_driver(config_element("ROOT", "", {"output_type" => "hash"}, [config_element("buffer")]))
|
100
|
-
|
107
|
+
assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
|
108
|
+
assert_equal 'hash', d.instance.formatter.output_type
|
101
109
|
|
102
110
|
assert_raise(Fluent::ConfigError) do
|
103
111
|
create_driver(config_element("ROOT", "", {"output_type" => "foo"}, [config_element("buffer")]))
|
@@ -131,21 +139,6 @@ class StdoutOutputTest < Test::Unit::TestCase
|
|
131
139
|
end
|
132
140
|
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
|
133
141
|
end
|
134
|
-
|
135
|
-
data('oj' => 'oj', 'yajl' => 'yajl')
|
136
|
-
test '#try_write(asynchronous)' do |data|
|
137
|
-
d = create_driver(config_element("ROOT", "", {"output_type" => "json", "json_parser" => data}, [config_element("buffer")]))
|
138
|
-
time = event_time()
|
139
|
-
d.instance.delayed = true
|
140
|
-
|
141
|
-
out = capture_log do
|
142
|
-
d.run(default_tag: 'test', flush: true, shutdown: false) do
|
143
|
-
d.feed(time, {'test' => 'test'})
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
|
-
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
|
148
|
-
end
|
149
142
|
end
|
150
143
|
|
151
144
|
sub_test_case 'emit hash' do
|
@@ -161,20 +154,6 @@ class StdoutOutputTest < Test::Unit::TestCase
|
|
161
154
|
|
162
155
|
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out
|
163
156
|
end
|
164
|
-
|
165
|
-
test '#try_write(asynchronous)' do
|
166
|
-
d = create_driver(config_element("ROOT", "", {"output_type" => "hash"}, [config_element("buffer")]))
|
167
|
-
time = event_time()
|
168
|
-
d.instance.delayed = true
|
169
|
-
|
170
|
-
out = capture_log do
|
171
|
-
d.run(default_tag: 'test', flush: true, shutdown: false) do
|
172
|
-
d.feed(time, {'test' => 'test'})
|
173
|
-
end
|
174
|
-
end
|
175
|
-
|
176
|
-
assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out
|
177
|
-
end
|
178
157
|
end
|
179
158
|
end
|
180
159
|
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require_relative '../helper'
|
2
2
|
require 'fluent/plugin/output'
|
3
3
|
require 'fluent/plugin/buffer'
|
4
|
+
require 'fluent/output'
|
4
5
|
require 'fluent/event'
|
5
6
|
|
6
7
|
require 'json'
|
@@ -126,6 +127,28 @@ module FluentPluginOutputAsBufferedTest
|
|
126
127
|
@try_write ? @try_write.call(chunk) : nil
|
127
128
|
end
|
128
129
|
end
|
130
|
+
module OldPluginMethodMixin
|
131
|
+
def initialize
|
132
|
+
super
|
133
|
+
@format = nil
|
134
|
+
@write = nil
|
135
|
+
end
|
136
|
+
def register(name, &block)
|
137
|
+
instance_variable_set("@#{name}", block)
|
138
|
+
end
|
139
|
+
def format(tag, time, record)
|
140
|
+
@format ? @format.call(tag, time, record) : [tag, time, record].to_json
|
141
|
+
end
|
142
|
+
def write(chunk)
|
143
|
+
@write ? @write.call(chunk) : nil
|
144
|
+
end
|
145
|
+
end
|
146
|
+
class DummyOldBufferedOutput < Fluent::BufferedOutput
|
147
|
+
include OldPluginMethodMixin
|
148
|
+
end
|
149
|
+
class DummyOldObjectBufferedOutput < Fluent::ObjectBufferedOutput
|
150
|
+
include OldPluginMethodMixin
|
151
|
+
end
|
129
152
|
end
|
130
153
|
|
131
154
|
class BufferedOutputTest < Test::Unit::TestCase
|
@@ -138,6 +161,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
138
161
|
when :standard then FluentPluginOutputAsBufferedTest::DummyStandardBufferedOutput.new
|
139
162
|
when :custom then FluentPluginOutputAsBufferedTest::DummyCustomFormatBufferedOutput.new
|
140
163
|
when :full then FluentPluginOutputAsBufferedTest::DummyFullFeatureOutput.new
|
164
|
+
when :old_buf then FluentPluginOutputAsBufferedTest::DummyOldBufferedOutput.new
|
165
|
+
when :old_obj then FluentPluginOutputAsBufferedTest::DummyOldObjectBufferedOutput.new
|
141
166
|
else
|
142
167
|
raise ArgumentError, "unknown type: #{type}"
|
143
168
|
end
|
@@ -346,6 +371,37 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
346
371
|
assert_equal events, each_pushed.map{|tag,time,record| [time,record]}
|
347
372
|
end
|
348
373
|
end
|
374
|
+
|
375
|
+
data(:BufferedOutput => :old_buf,
|
376
|
+
:ObjectBufferedOutput => :old_obj)
|
377
|
+
test 'old plugin types can iterate chunk by msgpack_each in #write' do |plugin_type|
|
378
|
+
events_from_chunk = []
|
379
|
+
# event_emitter helper requires Engine.root_agent for routing
|
380
|
+
ra = Fluent::RootAgent.new(log: $log)
|
381
|
+
stub(Fluent::Engine).root_agent { ra }
|
382
|
+
@i = create_output(plugin_type)
|
383
|
+
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', @hash)]))
|
384
|
+
@i.register(:format) { |tag, time, record| [time, record].to_msgpack }
|
385
|
+
@i.register(:write) { |chunk| e = []; chunk.msgpack_each { |t, r| e << [t, r] }; events_from_chunk << [:write, e]; }
|
386
|
+
@i.start
|
387
|
+
@i.after_start
|
388
|
+
|
389
|
+
events = [
|
390
|
+
[event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
|
391
|
+
[event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
|
392
|
+
]
|
393
|
+
|
394
|
+
@i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
|
395
|
+
@i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
|
396
|
+
|
397
|
+
waiting(5) { sleep 0.1 until events_from_chunk.size == 2 }
|
398
|
+
|
399
|
+
assert_equal 2, events_from_chunk.size
|
400
|
+
2.times.each do |i|
|
401
|
+
assert_equal :write, events_from_chunk[i][0]
|
402
|
+
assert_equal events, events_from_chunk[i][1]
|
403
|
+
end
|
404
|
+
end
|
349
405
|
end
|
350
406
|
|
351
407
|
sub_test_case 'buffered output configured with many chunk keys' do
|
@@ -1404,6 +1460,7 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1404
1460
|
@i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
|
1405
1461
|
@i.start
|
1406
1462
|
@i.after_start
|
1463
|
+
@i.log = Fluent::Test::TestLogger.new
|
1407
1464
|
end
|
1408
1465
|
|
1409
1466
|
test '#format is called for each event streams' do
|
@@ -1684,8 +1741,9 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1684
1741
|
|
1685
1742
|
assert{ chunks[0...3].all?{|c| !c.empty? } }
|
1686
1743
|
|
1687
|
-
# rollback is in progress, but some may be flushed again after rollback
|
1688
|
-
|
1744
|
+
# rollback is in progress, but some may be flushed again in retry state, after rollback
|
1745
|
+
# retry.next_time is 14:04:49
|
1746
|
+
Timecop.freeze( Time.parse('2016-04-13 14:04:51 +0900') )
|
1689
1747
|
@i.enqueue_thread_wait
|
1690
1748
|
@i.flush_thread_wakeup
|
1691
1749
|
|
@@ -0,0 +1,359 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test/driver/parser'
|
3
|
+
require 'fluent/plugin/parser'
|
4
|
+
require 'json'
|
5
|
+
require 'timecop'
|
6
|
+
|
7
|
+
class ParserTest < ::Test::Unit::TestCase
|
8
|
+
class ExampleParser < Fluent::Plugin::Parser
|
9
|
+
def parse(data)
|
10
|
+
r = JSON.parse(data)
|
11
|
+
yield convert_values(parse_time(r), r)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def create_driver(conf={})
|
16
|
+
Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser).configure(conf)
|
17
|
+
end
|
18
|
+
|
19
|
+
def setup
|
20
|
+
Fluent::Test.setup
|
21
|
+
end
|
22
|
+
|
23
|
+
sub_test_case 'base class works as plugin' do
|
24
|
+
def test_init
|
25
|
+
i = Fluent::Plugin::Parser.new
|
26
|
+
assert_nil i.types
|
27
|
+
assert_nil i.null_value_pattern
|
28
|
+
assert !i.null_empty_string
|
29
|
+
assert i.estimate_current_event
|
30
|
+
assert !i.keep_time_key
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_configure_against_string_literal
|
34
|
+
d = create_driver('keep_time_key true')
|
35
|
+
assert_true d.instance.keep_time_key
|
36
|
+
end
|
37
|
+
|
38
|
+
def test_parse
|
39
|
+
d = create_driver
|
40
|
+
assert_raise NotImplementedError do
|
41
|
+
d.instance.parse('')
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
sub_test_case '#string_like_null' do
|
47
|
+
setup do
|
48
|
+
@i = ExampleParser.new
|
49
|
+
end
|
50
|
+
|
51
|
+
test 'returns false if null_empty_string is false and null_value_regexp is nil' do
|
52
|
+
assert ! @i.string_like_null('a', false, nil)
|
53
|
+
assert ! @i.string_like_null('', false, nil)
|
54
|
+
end
|
55
|
+
|
56
|
+
test 'returns true if null_empty_string is true and string value is empty' do
|
57
|
+
assert ! @i.string_like_null('a', true, nil)
|
58
|
+
assert @i.string_like_null('', true, nil)
|
59
|
+
end
|
60
|
+
|
61
|
+
test 'returns true if null_value_regexp has regexp and it matches string value' do
|
62
|
+
assert ! @i.string_like_null('a', false, /null/i)
|
63
|
+
assert @i.string_like_null('NULL', false, /null/i)
|
64
|
+
assert @i.string_like_null('empty', false, /null|empty/i)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
sub_test_case '#build_type_converters converters' do
|
69
|
+
setup do
|
70
|
+
@i = ExampleParser.new
|
71
|
+
types_config = {
|
72
|
+
"s" => "string",
|
73
|
+
"i" => "integer",
|
74
|
+
"f" => "float",
|
75
|
+
"b" => "bool",
|
76
|
+
"t1" => "time",
|
77
|
+
"t2" => "time:%Y-%m-%d %H:%M:%S.%N",
|
78
|
+
"t3" => "time:+0100:%Y-%m-%d %H:%M:%S.%N",
|
79
|
+
"t4" => "time:unixtime",
|
80
|
+
"t5" => "time:float",
|
81
|
+
"a1" => "array",
|
82
|
+
"a2" => "array:|",
|
83
|
+
}
|
84
|
+
@hash = {
|
85
|
+
'types' => JSON.dump(types_config),
|
86
|
+
}
|
87
|
+
end
|
88
|
+
|
89
|
+
test 'to do #to_s by "string" type' do
|
90
|
+
@i.configure(config_element('parse', '', @hash))
|
91
|
+
c = @i.type_converters["s"]
|
92
|
+
assert_equal "", c.call("")
|
93
|
+
assert_equal "a", c.call("a")
|
94
|
+
assert_equal "1", c.call(1)
|
95
|
+
assert_equal "1.01", c.call(1.01)
|
96
|
+
assert_equal "true", c.call(true)
|
97
|
+
assert_equal "false", c.call(false)
|
98
|
+
end
|
99
|
+
|
100
|
+
test 'to do #to_i by "integer" type' do
|
101
|
+
@i.configure(config_element('parse', '', @hash))
|
102
|
+
c = @i.type_converters["i"]
|
103
|
+
assert_equal 0, c.call("")
|
104
|
+
assert_equal 0, c.call("0")
|
105
|
+
assert_equal 0, c.call("a")
|
106
|
+
assert_equal(-1000, c.call("-1000"))
|
107
|
+
assert_equal 1, c.call(1)
|
108
|
+
assert_equal 1, c.call(1.01)
|
109
|
+
assert_equal 0, c.call(true)
|
110
|
+
assert_equal 0, c.call(false)
|
111
|
+
end
|
112
|
+
|
113
|
+
test 'to do #to_f by "float" type' do
|
114
|
+
@i.configure(config_element('parse', '', @hash))
|
115
|
+
c = @i.type_converters["f"]
|
116
|
+
assert_equal 0.0, c.call("")
|
117
|
+
assert_equal 0.0, c.call("0")
|
118
|
+
assert_equal 0.0, c.call("a")
|
119
|
+
assert_equal(-1000.0, c.call("-1000"))
|
120
|
+
assert_equal 1.0, c.call(1)
|
121
|
+
assert_equal 1.01, c.call(1.01)
|
122
|
+
assert_equal 0.0, c.call(true)
|
123
|
+
assert_equal 0.0, c.call(false)
|
124
|
+
end
|
125
|
+
|
126
|
+
test 'to return true/false, which returns true only for true/yes/1 (C & perl style), by "bool"' do
|
127
|
+
@i.configure(config_element('parse', '', @hash))
|
128
|
+
c = @i.type_converters["b"]
|
129
|
+
assert_false c.call("")
|
130
|
+
assert_false c.call("0")
|
131
|
+
assert_false c.call("a")
|
132
|
+
assert_true c.call("1")
|
133
|
+
assert_true c.call("true")
|
134
|
+
assert_true c.call("True")
|
135
|
+
assert_true c.call("YES")
|
136
|
+
assert_true c.call(true)
|
137
|
+
assert_false c.call(false)
|
138
|
+
assert_false c.call("1.0")
|
139
|
+
end
|
140
|
+
|
141
|
+
test 'to parse time string by ruby default time parser without any options' do
|
142
|
+
# "t1" => "time",
|
143
|
+
with_timezone("UTC+02") do # -0200
|
144
|
+
@i.configure(config_element('parse', '', @hash))
|
145
|
+
c = @i.type_converters["t1"]
|
146
|
+
assert_nil c.call("")
|
147
|
+
assert_equal_event_time event_time("2016-10-21 01:54:30 -0200"), c.call("2016-10-21 01:54:30")
|
148
|
+
assert_equal_event_time event_time("2016-10-21 03:54:30 -0200"), c.call("2016-10-21 01:54:30 -0400")
|
149
|
+
assert_equal_event_time event_time("2016-10-21 01:55:24 -0200"), c.call("2016-10-21T01:55:24-02:00")
|
150
|
+
assert_equal_event_time event_time("2016-10-21 01:55:24 -0200"), c.call("2016-10-21T03:55:24Z")
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
test 'to parse time string with specified time format' do
|
155
|
+
# "t2" => "time:%Y-%m-%d %H:%M:%S.%N",
|
156
|
+
with_timezone("UTC+02") do # -0200
|
157
|
+
@i.configure(config_element('parse', '', @hash))
|
158
|
+
c = @i.type_converters["t2"]
|
159
|
+
assert_nil c.call("")
|
160
|
+
assert_equal_event_time event_time("2016-10-21 01:54:30.123000000 -0200"), c.call("2016-10-21 01:54:30.123")
|
161
|
+
assert_equal_event_time event_time("2016-10-21 01:54:30.012345678 -0200"), c.call("2016-10-21 01:54:30.012345678")
|
162
|
+
assert_nil c.call("2016/10/21 015430")
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
test 'to parse time string with specified time format and timezone' do
|
167
|
+
# "t3" => "time:+0100:%Y-%m-%d %H:%M:%S.%N",
|
168
|
+
with_timezone("UTC+02") do # -0200
|
169
|
+
@i.configure(config_element('parse', '', @hash))
|
170
|
+
c = @i.type_converters["t3"]
|
171
|
+
assert_nil c.call("")
|
172
|
+
assert_equal_event_time event_time("2016-10-21 01:54:30.123000000 +0100"), c.call("2016-10-21 01:54:30.123")
|
173
|
+
assert_equal_event_time event_time("2016-10-21 01:54:30.012345678 +0100"), c.call("2016-10-21 01:54:30.012345678")
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
test 'to parse time string in unix timestamp' do
|
178
|
+
# "t4" => "time:unixtime",
|
179
|
+
with_timezone("UTC+02") do # -0200
|
180
|
+
@i.configure(config_element('parse', '', @hash))
|
181
|
+
c = @i.type_converters["t4"]
|
182
|
+
assert_equal_event_time event_time("1970-01-01 00:00:00.0 +0000"), c.call("")
|
183
|
+
assert_equal_event_time event_time("2016-10-21 01:54:30.0 -0200"), c.call("1477022070")
|
184
|
+
assert_equal_event_time event_time("2016-10-21 01:54:30.0 -0200"), c.call("1477022070.01")
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
test 'to parse time string in floating poing value' do
|
189
|
+
# "t5" => "time:float",
|
190
|
+
with_timezone("UTC+02") do # -0200
|
191
|
+
@i.configure(config_element('parse', '', @hash))
|
192
|
+
c = @i.type_converters["t5"]
|
193
|
+
assert_equal_event_time event_time("1970-01-01 00:00:00.0 +0000"), c.call("")
|
194
|
+
assert_equal_event_time event_time("2016-10-21 01:54:30.012 -0200"), c.call("1477022070.012")
|
195
|
+
assert_equal_event_time event_time("2016-10-21 01:54:30.123456789 -0200"), c.call("1477022070.123456789")
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
test 'to return array of string' do
|
200
|
+
@i.configure(config_element('parse', '', @hash))
|
201
|
+
c = @i.type_converters["a1"]
|
202
|
+
assert_equal [], c.call("")
|
203
|
+
assert_equal ["0"], c.call("0")
|
204
|
+
assert_equal ["0"], c.call(0)
|
205
|
+
assert_equal ["0", "1"], c.call("0,1")
|
206
|
+
assert_equal ["0|1", "2"], c.call("0|1,2")
|
207
|
+
assert_equal ["true"], c.call(true)
|
208
|
+
end
|
209
|
+
|
210
|
+
test 'to return array of string using specified delimiter' do
|
211
|
+
@i.configure(config_element('parse', '', @hash))
|
212
|
+
c = @i.type_converters["a2"]
|
213
|
+
assert_equal [], c.call("")
|
214
|
+
assert_equal ["0"], c.call("0")
|
215
|
+
assert_equal ["0"], c.call(0)
|
216
|
+
assert_equal ["0,1"], c.call("0,1")
|
217
|
+
assert_equal ["0", "1,2"], c.call("0|1,2")
|
218
|
+
assert_equal ["true"], c.call(true)
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
sub_test_case 'example parser without any configurations' do
|
223
|
+
setup do
|
224
|
+
@current_time = Time.parse("2016-10-21 14:22:01.0 +1000")
|
225
|
+
@current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)
|
226
|
+
# @current_time.to_i #=> 1477023721
|
227
|
+
Timecop.freeze(@current_time)
|
228
|
+
@i = ExampleParser.new
|
229
|
+
@i.configure(config_element('parse', '', {}))
|
230
|
+
end
|
231
|
+
|
232
|
+
teardown do
|
233
|
+
Timecop.return
|
234
|
+
end
|
235
|
+
|
236
|
+
test 'parser returns parsed JSON object, leaving empty/NULL strings, with current time' do
|
237
|
+
json = '{"t1":"1477023720.101","s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
|
238
|
+
@i.parse(json) do |time, record|
|
239
|
+
assert_equal_event_time @current_event_time, time
|
240
|
+
assert_equal "1477023720.101", record["t1"]
|
241
|
+
assert_equal "", record["s1"]
|
242
|
+
assert_equal "NULL", record["s2"]
|
243
|
+
assert_equal "null", record["s3"]
|
244
|
+
assert_equal 1, record["k1"]
|
245
|
+
assert_equal "13.1", record["k2"]
|
246
|
+
assert_equal "1", record["k3"]
|
247
|
+
assert_equal "yes", record["k4"]
|
248
|
+
end
|
249
|
+
end
|
250
|
+
end
|
251
|
+
|
252
|
+
sub_test_case 'example parser fully configured' do
|
253
|
+
setup do
|
254
|
+
@current_time = Time.parse("2016-10-21 14:22:01.0 +1000")
|
255
|
+
@current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)
|
256
|
+
# @current_time.to_i #=> 1477023721
|
257
|
+
Timecop.freeze(@current_time)
|
258
|
+
@i = ExampleParser.new
|
259
|
+
hash = {
|
260
|
+
'keep_time_key' => "no",
|
261
|
+
'estimate_current_event' => "yes",
|
262
|
+
'time_key' => "t1",
|
263
|
+
'time_type' => "float",
|
264
|
+
'null_empty_string' => 'yes',
|
265
|
+
'null_value_pattern' => 'NULL|null',
|
266
|
+
'types' => "k1:string, k2:integer, k3:float, k4:bool",
|
267
|
+
}
|
268
|
+
@i.configure(config_element('parse', '', hash))
|
269
|
+
end
|
270
|
+
|
271
|
+
teardown do
|
272
|
+
Timecop.return
|
273
|
+
end
|
274
|
+
|
275
|
+
test 'parser returns parsed JSON object, leaving empty/NULL strings, with current time' do
|
276
|
+
json = '{"t1":"1477023720.101","s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
|
277
|
+
@i.parse(json) do |time, record|
|
278
|
+
assert_equal_event_time Fluent::EventTime.new(1477023720, 101_000_000), time
|
279
|
+
assert !record.has_key?("t1")
|
280
|
+
assert{ record.has_key?("s1") && record["s1"].nil? }
|
281
|
+
assert{ record.has_key?("s2") && record["s2"].nil? }
|
282
|
+
assert{ record.has_key?("s3") && record["s3"].nil? }
|
283
|
+
assert_equal "1", record["k1"]
|
284
|
+
assert_equal 13, record["k2"]
|
285
|
+
assert_equal 1.0, record["k3"]
|
286
|
+
assert_equal true, record["k4"]
|
287
|
+
end
|
288
|
+
end
|
289
|
+
|
290
|
+
test 'parser returns current time if a field is missing specified by time_key' do
|
291
|
+
json = '{"s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
|
292
|
+
@i.parse(json) do |time, record|
|
293
|
+
assert_equal_event_time @current_event_time, time
|
294
|
+
assert !record.has_key?("t1")
|
295
|
+
assert{ record.has_key?("s1") && record["s1"].nil? }
|
296
|
+
assert{ record.has_key?("s2") && record["s2"].nil? }
|
297
|
+
assert{ record.has_key?("s3") && record["s3"].nil? }
|
298
|
+
assert_equal "1", record["k1"]
|
299
|
+
assert_equal 13, record["k2"]
|
300
|
+
assert_equal 1.0, record["k3"]
|
301
|
+
assert_equal true, record["k4"]
|
302
|
+
end
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
sub_test_case 'example parser configured not to estimate current time, and to keep time key' do
|
307
|
+
setup do
|
308
|
+
@current_time = Time.parse("2016-10-21 14:22:01.0 +1000")
|
309
|
+
@current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)
|
310
|
+
# @current_time.to_i #=> 1477023721
|
311
|
+
Timecop.freeze(@current_time)
|
312
|
+
@i = ExampleParser.new
|
313
|
+
hash = {
|
314
|
+
'keep_time_key' => "yes",
|
315
|
+
'estimate_current_event' => "no",
|
316
|
+
'time_key' => "t1",
|
317
|
+
'time_type' => "float",
|
318
|
+
'null_empty_string' => 'yes',
|
319
|
+
'null_value_pattern' => 'NULL|null',
|
320
|
+
'types' => "k1:string, k2:integer, k3:float, k4:bool",
|
321
|
+
}
|
322
|
+
@i.configure(config_element('parse', '', hash))
|
323
|
+
end
|
324
|
+
|
325
|
+
teardown do
|
326
|
+
Timecop.return
|
327
|
+
end
|
328
|
+
|
329
|
+
test 'parser returns parsed time with original field and value if the field of time exists' do
|
330
|
+
json = '{"t1":"1477023720.101","s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
|
331
|
+
@i.parse(json) do |time, record|
|
332
|
+
assert_equal_event_time Fluent::EventTime.new(1477023720, 101_000_000), time
|
333
|
+
assert_equal "1477023720.101", record["t1"]
|
334
|
+
assert{ record.has_key?("s1") && record["s1"].nil? }
|
335
|
+
assert{ record.has_key?("s2") && record["s2"].nil? }
|
336
|
+
assert{ record.has_key?("s3") && record["s3"].nil? }
|
337
|
+
assert_equal "1", record["k1"]
|
338
|
+
assert_equal 13, record["k2"]
|
339
|
+
assert_equal 1.0, record["k3"]
|
340
|
+
assert_equal true, record["k4"]
|
341
|
+
end
|
342
|
+
end
|
343
|
+
|
344
|
+
test 'parser returns nil as time if the field of time is missing' do
|
345
|
+
json = '{"s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
|
346
|
+
@i.parse(json) do |time, record|
|
347
|
+
assert_nil time
|
348
|
+
assert !record.has_key?("t1")
|
349
|
+
assert{ record.has_key?("s1") && record["s1"].nil? }
|
350
|
+
assert{ record.has_key?("s2") && record["s2"].nil? }
|
351
|
+
assert{ record.has_key?("s3") && record["s3"].nil? }
|
352
|
+
assert_equal "1", record["k1"]
|
353
|
+
assert_equal 13, record["k2"]
|
354
|
+
assert_equal 1.0, record["k3"]
|
355
|
+
assert_equal true, record["k4"]
|
356
|
+
end
|
357
|
+
end
|
358
|
+
end
|
359
|
+
end
|