fluentd 0.14.7-x86-mingw32 → 0.14.10-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +2 -0
  4. data/CONTRIBUTING.md +6 -1
  5. data/ChangeLog +95 -0
  6. data/Rakefile +21 -0
  7. data/appveyor.yml +1 -0
  8. data/code-of-conduct.md +3 -0
  9. data/example/out_exec_filter.conf +42 -0
  10. data/fluentd.gemspec +1 -1
  11. data/lib/fluent/agent.rb +2 -2
  12. data/lib/fluent/command/binlog_reader.rb +1 -1
  13. data/lib/fluent/command/cat.rb +15 -4
  14. data/lib/fluent/compat/output.rb +14 -9
  15. data/lib/fluent/compat/parser.rb +141 -11
  16. data/lib/fluent/config/configure_proxy.rb +2 -11
  17. data/lib/fluent/config/section.rb +8 -1
  18. data/lib/fluent/configurable.rb +1 -3
  19. data/lib/fluent/env.rb +1 -1
  20. data/lib/fluent/log.rb +1 -1
  21. data/lib/fluent/plugin/base.rb +17 -0
  22. data/lib/fluent/plugin/filter_parser.rb +108 -0
  23. data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
  24. data/lib/fluent/plugin/filter_stdout.rb +1 -1
  25. data/lib/fluent/plugin/formatter.rb +5 -0
  26. data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
  27. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  28. data/lib/fluent/plugin/formatter_tsv.rb +34 -0
  29. data/lib/fluent/plugin/in_exec.rb +48 -93
  30. data/lib/fluent/plugin/in_forward.rb +66 -265
  31. data/lib/fluent/plugin/in_http.rb +68 -65
  32. data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
  33. data/lib/fluent/plugin/in_syslog.rb +42 -58
  34. data/lib/fluent/plugin/in_tail.rb +29 -14
  35. data/lib/fluent/plugin/in_tcp.rb +54 -14
  36. data/lib/fluent/plugin/in_udp.rb +49 -13
  37. data/lib/fluent/plugin/multi_output.rb +1 -3
  38. data/lib/fluent/plugin/out_exec.rb +58 -71
  39. data/lib/fluent/plugin/out_exec_filter.rb +199 -279
  40. data/lib/fluent/plugin/out_file.rb +172 -81
  41. data/lib/fluent/plugin/out_forward.rb +229 -206
  42. data/lib/fluent/plugin/out_stdout.rb +6 -21
  43. data/lib/fluent/plugin/output.rb +90 -59
  44. data/lib/fluent/plugin/parser.rb +121 -61
  45. data/lib/fluent/plugin/parser_csv.rb +9 -3
  46. data/lib/fluent/plugin/parser_json.rb +37 -35
  47. data/lib/fluent/plugin/parser_ltsv.rb +11 -19
  48. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  49. data/lib/fluent/plugin/parser_regexp.rb +15 -42
  50. data/lib/fluent/plugin/parser_tsv.rb +8 -3
  51. data/lib/fluent/plugin_helper.rb +10 -1
  52. data/lib/fluent/plugin_helper/child_process.rb +139 -73
  53. data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
  54. data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
  55. data/lib/fluent/plugin_helper/event_loop.rb +24 -6
  56. data/lib/fluent/plugin_helper/extract.rb +16 -4
  57. data/lib/fluent/plugin_helper/formatter.rb +9 -11
  58. data/lib/fluent/plugin_helper/inject.rb +16 -1
  59. data/lib/fluent/plugin_helper/parser.rb +3 -3
  60. data/lib/fluent/plugin_helper/server.rb +494 -0
  61. data/lib/fluent/plugin_helper/socket.rb +101 -0
  62. data/lib/fluent/plugin_helper/socket_option.rb +84 -0
  63. data/lib/fluent/plugin_helper/timer.rb +1 -0
  64. data/lib/fluent/root_agent.rb +1 -1
  65. data/lib/fluent/test/driver/base.rb +95 -49
  66. data/lib/fluent/test/driver/base_owner.rb +18 -8
  67. data/lib/fluent/test/driver/multi_output.rb +2 -1
  68. data/lib/fluent/test/driver/output.rb +29 -6
  69. data/lib/fluent/test/helpers.rb +3 -1
  70. data/lib/fluent/test/log.rb +4 -0
  71. data/lib/fluent/test/startup_shutdown.rb +13 -0
  72. data/lib/fluent/time.rb +14 -8
  73. data/lib/fluent/version.rb +1 -1
  74. data/lib/fluent/winsvc.rb +1 -1
  75. data/test/command/test_binlog_reader.rb +5 -1
  76. data/test/compat/test_parser.rb +10 -0
  77. data/test/config/test_configurable.rb +193 -0
  78. data/test/config/test_configure_proxy.rb +0 -43
  79. data/test/helper.rb +36 -1
  80. data/test/plugin/test_base.rb +16 -0
  81. data/test/plugin/test_filter_parser.rb +665 -0
  82. data/test/plugin/test_filter_record_transformer.rb +36 -100
  83. data/test/plugin/test_filter_stdout.rb +18 -27
  84. data/test/plugin/test_in_dummy.rb +1 -1
  85. data/test/plugin/test_in_exec.rb +206 -94
  86. data/test/plugin/test_in_forward.rb +268 -347
  87. data/test/plugin/test_in_http.rb +310 -186
  88. data/test/plugin/test_in_monitor_agent.rb +65 -35
  89. data/test/plugin/test_in_syslog.rb +39 -3
  90. data/test/plugin/test_in_tcp.rb +78 -62
  91. data/test/plugin/test_in_udp.rb +101 -80
  92. data/test/plugin/test_out_exec.rb +223 -68
  93. data/test/plugin/test_out_exec_filter.rb +520 -169
  94. data/test/plugin/test_out_file.rb +637 -177
  95. data/test/plugin/test_out_forward.rb +242 -234
  96. data/test/plugin/test_out_null.rb +1 -1
  97. data/test/plugin/test_out_secondary_file.rb +4 -2
  98. data/test/plugin/test_out_stdout.rb +14 -35
  99. data/test/plugin/test_output_as_buffered.rb +60 -2
  100. data/test/plugin/test_parser.rb +359 -0
  101. data/test/plugin/test_parser_csv.rb +1 -2
  102. data/test/plugin/test_parser_json.rb +3 -4
  103. data/test/plugin/test_parser_labeled_tsv.rb +1 -2
  104. data/test/plugin/test_parser_none.rb +1 -2
  105. data/test/plugin/test_parser_regexp.rb +8 -4
  106. data/test/plugin/test_parser_tsv.rb +4 -3
  107. data/test/plugin_helper/test_child_process.rb +184 -0
  108. data/test/plugin_helper/test_compat_parameters.rb +88 -1
  109. data/test/plugin_helper/test_extract.rb +0 -1
  110. data/test/plugin_helper/test_formatter.rb +5 -2
  111. data/test/plugin_helper/test_inject.rb +21 -0
  112. data/test/plugin_helper/test_parser.rb +6 -5
  113. data/test/plugin_helper/test_server.rb +905 -0
  114. data/test/test_event_time.rb +3 -1
  115. data/test/test_output.rb +53 -2
  116. data/test/test_plugin_classes.rb +20 -0
  117. data/test/test_root_agent.rb +139 -0
  118. data/test/test_test_drivers.rb +135 -0
  119. metadata +28 -8
  120. data/test/plugin/test_parser_base.rb +0 -32
@@ -80,7 +80,7 @@ class NullOutputTest < Test::Unit::TestCase
80
80
  d.instance.delayed = true
81
81
 
82
82
  t = event_time("2016-05-23 00:22:13 -0800")
83
- d.run(default_tag: 'test', flush: true, shutdown: false) do
83
+ d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
84
84
  d.feed(t, {"message" => "null null null"})
85
85
  d.feed(t, {"message" => "null null"})
86
86
  d.feed(t, {"message" => "null"})
@@ -97,7 +97,9 @@ class FileOutputSecondaryTest < Test::Unit::TestCase
97
97
 
98
98
  test 'should be passed directory' do
99
99
  assert_raise Fluent::ConfigError do
100
- create_driver %[]
100
+ i = Fluent::Plugin::SecondaryFileOutput.new
101
+ i.acts_as_secondary(create_primary)
102
+ i.configure(config_element())
101
103
  end
102
104
 
103
105
  assert_nothing_raised do
@@ -166,7 +168,7 @@ class FileOutputSecondaryTest < Test::Unit::TestCase
166
168
  sleep 0.1 until File.stat(path).size == msgpack_binary.size
167
169
  end
168
170
 
169
- assert_equal File.read(path), msgpack_binary
171
+ assert_equal msgpack_binary, File.binread(path)
170
172
  end
171
173
 
172
174
  test 'path should be incremental when append option is false' do
@@ -18,15 +18,19 @@ class StdoutOutputTest < Test::Unit::TestCase
18
18
  sub_test_case 'non-buffered' do
19
19
  test 'configure' do
20
20
  d = create_driver
21
- assert_equal [], d.instance.formatter_configs
21
+ assert_equal 1, d.instance.formatter_configs.size # init: true
22
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
23
+ assert_equal 'json', d.instance.formatter.output_type
22
24
  end
23
25
 
24
26
  test 'configure output_type' do
25
27
  d = create_driver(CONFIG + "\noutput_type json")
26
- assert_equal 'json', d.instance.formatter_configs.first[:@type]
28
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
29
+ assert_equal 'json', d.instance.formatter.output_type
27
30
 
28
31
  d = create_driver(CONFIG + "\noutput_type hash")
29
- assert_equal 'hash', d.instance.formatter_configs.first[:@type]
32
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
33
+ assert_equal 'hash', d.instance.formatter.output_type
30
34
 
31
35
  assert_raise(Fluent::ConfigError) do
32
36
  d = create_driver(CONFIG + "\noutput_type foo")
@@ -83,7 +87,9 @@ class StdoutOutputTest < Test::Unit::TestCase
83
87
  sub_test_case 'buffered' do
84
88
  test 'configure' do
85
89
  d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
86
- assert_equal [], d.instance.formatter_configs
90
+ assert_equal 1, d.instance.formatter_configs.size
91
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
92
+ assert_equal 'json', d.instance.formatter.output_type
87
93
  assert_equal 10 * 1024, d.instance.buffer_config.chunk_limit_size
88
94
  assert d.instance.buffer_config.flush_at_shutdown
89
95
  assert_equal ['tag'], d.instance.buffer_config.chunk_keys
@@ -94,10 +100,12 @@ class StdoutOutputTest < Test::Unit::TestCase
94
100
 
95
101
  test 'configure with output_type' do
96
102
  d = create_driver(config_element("ROOT", "", {"output_type" => "json"}, [config_element("buffer")]))
97
- assert_equal 'json', d.instance.formatter_configs.first[:@type]
103
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
104
+ assert_equal 'json', d.instance.formatter.output_type
98
105
 
99
106
  d = create_driver(config_element("ROOT", "", {"output_type" => "hash"}, [config_element("buffer")]))
100
- assert_equal 'hash', d.instance.formatter_configs.first[:@type]
107
+ assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
108
+ assert_equal 'hash', d.instance.formatter.output_type
101
109
 
102
110
  assert_raise(Fluent::ConfigError) do
103
111
  create_driver(config_element("ROOT", "", {"output_type" => "foo"}, [config_element("buffer")]))
@@ -131,21 +139,6 @@ class StdoutOutputTest < Test::Unit::TestCase
131
139
  end
132
140
  assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
133
141
  end
134
-
135
- data('oj' => 'oj', 'yajl' => 'yajl')
136
- test '#try_write(asynchronous)' do |data|
137
- d = create_driver(config_element("ROOT", "", {"output_type" => "json", "json_parser" => data}, [config_element("buffer")]))
138
- time = event_time()
139
- d.instance.delayed = true
140
-
141
- out = capture_log do
142
- d.run(default_tag: 'test', flush: true, shutdown: false) do
143
- d.feed(time, {'test' => 'test'})
144
- end
145
- end
146
-
147
- assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
148
- end
149
142
  end
150
143
 
151
144
  sub_test_case 'emit hash' do
@@ -161,20 +154,6 @@ class StdoutOutputTest < Test::Unit::TestCase
161
154
 
162
155
  assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out
163
156
  end
164
-
165
- test '#try_write(asynchronous)' do
166
- d = create_driver(config_element("ROOT", "", {"output_type" => "hash"}, [config_element("buffer")]))
167
- time = event_time()
168
- d.instance.delayed = true
169
-
170
- out = capture_log do
171
- d.run(default_tag: 'test', flush: true, shutdown: false) do
172
- d.feed(time, {'test' => 'test'})
173
- end
174
- end
175
-
176
- assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out
177
- end
178
157
  end
179
158
  end
180
159
 
@@ -1,6 +1,7 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/plugin/output'
3
3
  require 'fluent/plugin/buffer'
4
+ require 'fluent/output'
4
5
  require 'fluent/event'
5
6
 
6
7
  require 'json'
@@ -126,6 +127,28 @@ module FluentPluginOutputAsBufferedTest
126
127
  @try_write ? @try_write.call(chunk) : nil
127
128
  end
128
129
  end
130
+ module OldPluginMethodMixin
131
+ def initialize
132
+ super
133
+ @format = nil
134
+ @write = nil
135
+ end
136
+ def register(name, &block)
137
+ instance_variable_set("@#{name}", block)
138
+ end
139
+ def format(tag, time, record)
140
+ @format ? @format.call(tag, time, record) : [tag, time, record].to_json
141
+ end
142
+ def write(chunk)
143
+ @write ? @write.call(chunk) : nil
144
+ end
145
+ end
146
+ class DummyOldBufferedOutput < Fluent::BufferedOutput
147
+ include OldPluginMethodMixin
148
+ end
149
+ class DummyOldObjectBufferedOutput < Fluent::ObjectBufferedOutput
150
+ include OldPluginMethodMixin
151
+ end
129
152
  end
130
153
 
131
154
  class BufferedOutputTest < Test::Unit::TestCase
@@ -138,6 +161,8 @@ class BufferedOutputTest < Test::Unit::TestCase
138
161
  when :standard then FluentPluginOutputAsBufferedTest::DummyStandardBufferedOutput.new
139
162
  when :custom then FluentPluginOutputAsBufferedTest::DummyCustomFormatBufferedOutput.new
140
163
  when :full then FluentPluginOutputAsBufferedTest::DummyFullFeatureOutput.new
164
+ when :old_buf then FluentPluginOutputAsBufferedTest::DummyOldBufferedOutput.new
165
+ when :old_obj then FluentPluginOutputAsBufferedTest::DummyOldObjectBufferedOutput.new
141
166
  else
142
167
  raise ArgumentError, "unknown type: #{type}"
143
168
  end
@@ -346,6 +371,37 @@ class BufferedOutputTest < Test::Unit::TestCase
346
371
  assert_equal events, each_pushed.map{|tag,time,record| [time,record]}
347
372
  end
348
373
  end
374
+
375
+ data(:BufferedOutput => :old_buf,
376
+ :ObjectBufferedOutput => :old_obj)
377
+ test 'old plugin types can iterate chunk by msgpack_each in #write' do |plugin_type|
378
+ events_from_chunk = []
379
+ # event_emitter helper requires Engine.root_agent for routing
380
+ ra = Fluent::RootAgent.new(log: $log)
381
+ stub(Fluent::Engine).root_agent { ra }
382
+ @i = create_output(plugin_type)
383
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', @hash)]))
384
+ @i.register(:format) { |tag, time, record| [time, record].to_msgpack }
385
+ @i.register(:write) { |chunk| e = []; chunk.msgpack_each { |t, r| e << [t, r] }; events_from_chunk << [:write, e]; }
386
+ @i.start
387
+ @i.after_start
388
+
389
+ events = [
390
+ [event_time('2016-10-05 16:16:16 -0700'), {"message" => "yaaaaaaaaay!"}],
391
+ [event_time('2016-10-05 16:16:17 -0700'), {"message" => "yoooooooooy!"}],
392
+ ]
393
+
394
+ @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
395
+ @i.emit_events("test.tag", Fluent::ArrayEventStream.new(events))
396
+
397
+ waiting(5) { sleep 0.1 until events_from_chunk.size == 2 }
398
+
399
+ assert_equal 2, events_from_chunk.size
400
+ 2.times.each do |i|
401
+ assert_equal :write, events_from_chunk[i][0]
402
+ assert_equal events, events_from_chunk[i][1]
403
+ end
404
+ end
349
405
  end
350
406
 
351
407
  sub_test_case 'buffered output configured with many chunk keys' do
@@ -1404,6 +1460,7 @@ class BufferedOutputTest < Test::Unit::TestCase
1404
1460
  @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
1405
1461
  @i.start
1406
1462
  @i.after_start
1463
+ @i.log = Fluent::Test::TestLogger.new
1407
1464
  end
1408
1465
 
1409
1466
  test '#format is called for each event streams' do
@@ -1684,8 +1741,9 @@ class BufferedOutputTest < Test::Unit::TestCase
1684
1741
 
1685
1742
  assert{ chunks[0...3].all?{|c| !c.empty? } }
1686
1743
 
1687
- # rollback is in progress, but some may be flushed again after rollback
1688
- Timecop.freeze( Time.parse('2016-04-13 14:04:46 +0900') )
1744
+ # rollback is in progress, but some may be flushed again in retry state, after rollback
1745
+ # retry.next_time is 14:04:49
1746
+ Timecop.freeze( Time.parse('2016-04-13 14:04:51 +0900') )
1689
1747
  @i.enqueue_thread_wait
1690
1748
  @i.flush_thread_wakeup
1691
1749
 
@@ -0,0 +1,359 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/parser'
3
+ require 'fluent/plugin/parser'
4
+ require 'json'
5
+ require 'timecop'
6
+
7
+ class ParserTest < ::Test::Unit::TestCase
8
+ class ExampleParser < Fluent::Plugin::Parser
9
+ def parse(data)
10
+ r = JSON.parse(data)
11
+ yield convert_values(parse_time(r), r)
12
+ end
13
+ end
14
+
15
+ def create_driver(conf={})
16
+ Fluent::Test::Driver::Parser.new(Fluent::Plugin::Parser).configure(conf)
17
+ end
18
+
19
+ def setup
20
+ Fluent::Test.setup
21
+ end
22
+
23
+ sub_test_case 'base class works as plugin' do
24
+ def test_init
25
+ i = Fluent::Plugin::Parser.new
26
+ assert_nil i.types
27
+ assert_nil i.null_value_pattern
28
+ assert !i.null_empty_string
29
+ assert i.estimate_current_event
30
+ assert !i.keep_time_key
31
+ end
32
+
33
+ def test_configure_against_string_literal
34
+ d = create_driver('keep_time_key true')
35
+ assert_true d.instance.keep_time_key
36
+ end
37
+
38
+ def test_parse
39
+ d = create_driver
40
+ assert_raise NotImplementedError do
41
+ d.instance.parse('')
42
+ end
43
+ end
44
+ end
45
+
46
+ sub_test_case '#string_like_null' do
47
+ setup do
48
+ @i = ExampleParser.new
49
+ end
50
+
51
+ test 'returns false if null_empty_string is false and null_value_regexp is nil' do
52
+ assert ! @i.string_like_null('a', false, nil)
53
+ assert ! @i.string_like_null('', false, nil)
54
+ end
55
+
56
+ test 'returns true if null_empty_string is true and string value is empty' do
57
+ assert ! @i.string_like_null('a', true, nil)
58
+ assert @i.string_like_null('', true, nil)
59
+ end
60
+
61
+ test 'returns true if null_value_regexp has regexp and it matches string value' do
62
+ assert ! @i.string_like_null('a', false, /null/i)
63
+ assert @i.string_like_null('NULL', false, /null/i)
64
+ assert @i.string_like_null('empty', false, /null|empty/i)
65
+ end
66
+ end
67
+
68
+ sub_test_case '#build_type_converters converters' do
69
+ setup do
70
+ @i = ExampleParser.new
71
+ types_config = {
72
+ "s" => "string",
73
+ "i" => "integer",
74
+ "f" => "float",
75
+ "b" => "bool",
76
+ "t1" => "time",
77
+ "t2" => "time:%Y-%m-%d %H:%M:%S.%N",
78
+ "t3" => "time:+0100:%Y-%m-%d %H:%M:%S.%N",
79
+ "t4" => "time:unixtime",
80
+ "t5" => "time:float",
81
+ "a1" => "array",
82
+ "a2" => "array:|",
83
+ }
84
+ @hash = {
85
+ 'types' => JSON.dump(types_config),
86
+ }
87
+ end
88
+
89
+ test 'to do #to_s by "string" type' do
90
+ @i.configure(config_element('parse', '', @hash))
91
+ c = @i.type_converters["s"]
92
+ assert_equal "", c.call("")
93
+ assert_equal "a", c.call("a")
94
+ assert_equal "1", c.call(1)
95
+ assert_equal "1.01", c.call(1.01)
96
+ assert_equal "true", c.call(true)
97
+ assert_equal "false", c.call(false)
98
+ end
99
+
100
+ test 'to do #to_i by "integer" type' do
101
+ @i.configure(config_element('parse', '', @hash))
102
+ c = @i.type_converters["i"]
103
+ assert_equal 0, c.call("")
104
+ assert_equal 0, c.call("0")
105
+ assert_equal 0, c.call("a")
106
+ assert_equal(-1000, c.call("-1000"))
107
+ assert_equal 1, c.call(1)
108
+ assert_equal 1, c.call(1.01)
109
+ assert_equal 0, c.call(true)
110
+ assert_equal 0, c.call(false)
111
+ end
112
+
113
+ test 'to do #to_f by "float" type' do
114
+ @i.configure(config_element('parse', '', @hash))
115
+ c = @i.type_converters["f"]
116
+ assert_equal 0.0, c.call("")
117
+ assert_equal 0.0, c.call("0")
118
+ assert_equal 0.0, c.call("a")
119
+ assert_equal(-1000.0, c.call("-1000"))
120
+ assert_equal 1.0, c.call(1)
121
+ assert_equal 1.01, c.call(1.01)
122
+ assert_equal 0.0, c.call(true)
123
+ assert_equal 0.0, c.call(false)
124
+ end
125
+
126
+ test 'to return true/false, which returns true only for true/yes/1 (C & perl style), by "bool"' do
127
+ @i.configure(config_element('parse', '', @hash))
128
+ c = @i.type_converters["b"]
129
+ assert_false c.call("")
130
+ assert_false c.call("0")
131
+ assert_false c.call("a")
132
+ assert_true c.call("1")
133
+ assert_true c.call("true")
134
+ assert_true c.call("True")
135
+ assert_true c.call("YES")
136
+ assert_true c.call(true)
137
+ assert_false c.call(false)
138
+ assert_false c.call("1.0")
139
+ end
140
+
141
+ test 'to parse time string by ruby default time parser without any options' do
142
+ # "t1" => "time",
143
+ with_timezone("UTC+02") do # -0200
144
+ @i.configure(config_element('parse', '', @hash))
145
+ c = @i.type_converters["t1"]
146
+ assert_nil c.call("")
147
+ assert_equal_event_time event_time("2016-10-21 01:54:30 -0200"), c.call("2016-10-21 01:54:30")
148
+ assert_equal_event_time event_time("2016-10-21 03:54:30 -0200"), c.call("2016-10-21 01:54:30 -0400")
149
+ assert_equal_event_time event_time("2016-10-21 01:55:24 -0200"), c.call("2016-10-21T01:55:24-02:00")
150
+ assert_equal_event_time event_time("2016-10-21 01:55:24 -0200"), c.call("2016-10-21T03:55:24Z")
151
+ end
152
+ end
153
+
154
+ test 'to parse time string with specified time format' do
155
+ # "t2" => "time:%Y-%m-%d %H:%M:%S.%N",
156
+ with_timezone("UTC+02") do # -0200
157
+ @i.configure(config_element('parse', '', @hash))
158
+ c = @i.type_converters["t2"]
159
+ assert_nil c.call("")
160
+ assert_equal_event_time event_time("2016-10-21 01:54:30.123000000 -0200"), c.call("2016-10-21 01:54:30.123")
161
+ assert_equal_event_time event_time("2016-10-21 01:54:30.012345678 -0200"), c.call("2016-10-21 01:54:30.012345678")
162
+ assert_nil c.call("2016/10/21 015430")
163
+ end
164
+ end
165
+
166
+ test 'to parse time string with specified time format and timezone' do
167
+ # "t3" => "time:+0100:%Y-%m-%d %H:%M:%S.%N",
168
+ with_timezone("UTC+02") do # -0200
169
+ @i.configure(config_element('parse', '', @hash))
170
+ c = @i.type_converters["t3"]
171
+ assert_nil c.call("")
172
+ assert_equal_event_time event_time("2016-10-21 01:54:30.123000000 +0100"), c.call("2016-10-21 01:54:30.123")
173
+ assert_equal_event_time event_time("2016-10-21 01:54:30.012345678 +0100"), c.call("2016-10-21 01:54:30.012345678")
174
+ end
175
+ end
176
+
177
+ test 'to parse time string in unix timestamp' do
178
+ # "t4" => "time:unixtime",
179
+ with_timezone("UTC+02") do # -0200
180
+ @i.configure(config_element('parse', '', @hash))
181
+ c = @i.type_converters["t4"]
182
+ assert_equal_event_time event_time("1970-01-01 00:00:00.0 +0000"), c.call("")
183
+ assert_equal_event_time event_time("2016-10-21 01:54:30.0 -0200"), c.call("1477022070")
184
+ assert_equal_event_time event_time("2016-10-21 01:54:30.0 -0200"), c.call("1477022070.01")
185
+ end
186
+ end
187
+
188
+ test 'to parse time string in floating poing value' do
189
+ # "t5" => "time:float",
190
+ with_timezone("UTC+02") do # -0200
191
+ @i.configure(config_element('parse', '', @hash))
192
+ c = @i.type_converters["t5"]
193
+ assert_equal_event_time event_time("1970-01-01 00:00:00.0 +0000"), c.call("")
194
+ assert_equal_event_time event_time("2016-10-21 01:54:30.012 -0200"), c.call("1477022070.012")
195
+ assert_equal_event_time event_time("2016-10-21 01:54:30.123456789 -0200"), c.call("1477022070.123456789")
196
+ end
197
+ end
198
+
199
+ test 'to return array of string' do
200
+ @i.configure(config_element('parse', '', @hash))
201
+ c = @i.type_converters["a1"]
202
+ assert_equal [], c.call("")
203
+ assert_equal ["0"], c.call("0")
204
+ assert_equal ["0"], c.call(0)
205
+ assert_equal ["0", "1"], c.call("0,1")
206
+ assert_equal ["0|1", "2"], c.call("0|1,2")
207
+ assert_equal ["true"], c.call(true)
208
+ end
209
+
210
+ test 'to return array of string using specified delimiter' do
211
+ @i.configure(config_element('parse', '', @hash))
212
+ c = @i.type_converters["a2"]
213
+ assert_equal [], c.call("")
214
+ assert_equal ["0"], c.call("0")
215
+ assert_equal ["0"], c.call(0)
216
+ assert_equal ["0,1"], c.call("0,1")
217
+ assert_equal ["0", "1,2"], c.call("0|1,2")
218
+ assert_equal ["true"], c.call(true)
219
+ end
220
+ end
221
+
222
+ sub_test_case 'example parser without any configurations' do
223
+ setup do
224
+ @current_time = Time.parse("2016-10-21 14:22:01.0 +1000")
225
+ @current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)
226
+ # @current_time.to_i #=> 1477023721
227
+ Timecop.freeze(@current_time)
228
+ @i = ExampleParser.new
229
+ @i.configure(config_element('parse', '', {}))
230
+ end
231
+
232
+ teardown do
233
+ Timecop.return
234
+ end
235
+
236
+ test 'parser returns parsed JSON object, leaving empty/NULL strings, with current time' do
237
+ json = '{"t1":"1477023720.101","s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
238
+ @i.parse(json) do |time, record|
239
+ assert_equal_event_time @current_event_time, time
240
+ assert_equal "1477023720.101", record["t1"]
241
+ assert_equal "", record["s1"]
242
+ assert_equal "NULL", record["s2"]
243
+ assert_equal "null", record["s3"]
244
+ assert_equal 1, record["k1"]
245
+ assert_equal "13.1", record["k2"]
246
+ assert_equal "1", record["k3"]
247
+ assert_equal "yes", record["k4"]
248
+ end
249
+ end
250
+ end
251
+
252
+ sub_test_case 'example parser fully configured' do
253
+ setup do
254
+ @current_time = Time.parse("2016-10-21 14:22:01.0 +1000")
255
+ @current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)
256
+ # @current_time.to_i #=> 1477023721
257
+ Timecop.freeze(@current_time)
258
+ @i = ExampleParser.new
259
+ hash = {
260
+ 'keep_time_key' => "no",
261
+ 'estimate_current_event' => "yes",
262
+ 'time_key' => "t1",
263
+ 'time_type' => "float",
264
+ 'null_empty_string' => 'yes',
265
+ 'null_value_pattern' => 'NULL|null',
266
+ 'types' => "k1:string, k2:integer, k3:float, k4:bool",
267
+ }
268
+ @i.configure(config_element('parse', '', hash))
269
+ end
270
+
271
+ teardown do
272
+ Timecop.return
273
+ end
274
+
275
+ test 'parser returns parsed JSON object, leaving empty/NULL strings, with current time' do
276
+ json = '{"t1":"1477023720.101","s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
277
+ @i.parse(json) do |time, record|
278
+ assert_equal_event_time Fluent::EventTime.new(1477023720, 101_000_000), time
279
+ assert !record.has_key?("t1")
280
+ assert{ record.has_key?("s1") && record["s1"].nil? }
281
+ assert{ record.has_key?("s2") && record["s2"].nil? }
282
+ assert{ record.has_key?("s3") && record["s3"].nil? }
283
+ assert_equal "1", record["k1"]
284
+ assert_equal 13, record["k2"]
285
+ assert_equal 1.0, record["k3"]
286
+ assert_equal true, record["k4"]
287
+ end
288
+ end
289
+
290
+ test 'parser returns current time if a field is missing specified by time_key' do
291
+ json = '{"s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
292
+ @i.parse(json) do |time, record|
293
+ assert_equal_event_time @current_event_time, time
294
+ assert !record.has_key?("t1")
295
+ assert{ record.has_key?("s1") && record["s1"].nil? }
296
+ assert{ record.has_key?("s2") && record["s2"].nil? }
297
+ assert{ record.has_key?("s3") && record["s3"].nil? }
298
+ assert_equal "1", record["k1"]
299
+ assert_equal 13, record["k2"]
300
+ assert_equal 1.0, record["k3"]
301
+ assert_equal true, record["k4"]
302
+ end
303
+ end
304
+ end
305
+
306
+ sub_test_case 'example parser configured not to estimate current time, and to keep time key' do
307
+ setup do
308
+ @current_time = Time.parse("2016-10-21 14:22:01.0 +1000")
309
+ @current_event_time = Fluent::EventTime.new(@current_time.to_i, 0)
310
+ # @current_time.to_i #=> 1477023721
311
+ Timecop.freeze(@current_time)
312
+ @i = ExampleParser.new
313
+ hash = {
314
+ 'keep_time_key' => "yes",
315
+ 'estimate_current_event' => "no",
316
+ 'time_key' => "t1",
317
+ 'time_type' => "float",
318
+ 'null_empty_string' => 'yes',
319
+ 'null_value_pattern' => 'NULL|null',
320
+ 'types' => "k1:string, k2:integer, k3:float, k4:bool",
321
+ }
322
+ @i.configure(config_element('parse', '', hash))
323
+ end
324
+
325
+ teardown do
326
+ Timecop.return
327
+ end
328
+
329
+ test 'parser returns parsed time with original field and value if the field of time exists' do
330
+ json = '{"t1":"1477023720.101","s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
331
+ @i.parse(json) do |time, record|
332
+ assert_equal_event_time Fluent::EventTime.new(1477023720, 101_000_000), time
333
+ assert_equal "1477023720.101", record["t1"]
334
+ assert{ record.has_key?("s1") && record["s1"].nil? }
335
+ assert{ record.has_key?("s2") && record["s2"].nil? }
336
+ assert{ record.has_key?("s3") && record["s3"].nil? }
337
+ assert_equal "1", record["k1"]
338
+ assert_equal 13, record["k2"]
339
+ assert_equal 1.0, record["k3"]
340
+ assert_equal true, record["k4"]
341
+ end
342
+ end
343
+
344
+ test 'parser returns nil as time if the field of time is missing' do
345
+ json = '{"s1":"","s2":"NULL","s3":"null","k1":1,"k2":"13.1","k3":"1","k4":"yes"}'
346
+ @i.parse(json) do |time, record|
347
+ assert_nil time
348
+ assert !record.has_key?("t1")
349
+ assert{ record.has_key?("s1") && record["s1"].nil? }
350
+ assert{ record.has_key?("s2") && record["s2"].nil? }
351
+ assert{ record.has_key?("s3") && record["s3"].nil? }
352
+ assert_equal "1", record["k1"]
353
+ assert_equal 13, record["k2"]
354
+ assert_equal 1.0, record["k3"]
355
+ assert_equal true, record["k4"]
356
+ end
357
+ end
358
+ end
359
+ end