fluentd 0.14.5-x64-mingw32 → 0.14.7-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (103) hide show
  1. checksums.yaml +4 -4
  2. data/ChangeLog +55 -0
  3. data/bin/fluent-binlog-reader +7 -0
  4. data/example/in_dummy_with_compression.conf +23 -0
  5. data/lib/fluent/agent.rb +8 -12
  6. data/lib/fluent/command/binlog_reader.rb +234 -0
  7. data/lib/fluent/command/fluentd.rb +17 -1
  8. data/lib/fluent/compat/file_util.rb +1 -1
  9. data/lib/fluent/compat/output.rb +5 -1
  10. data/lib/fluent/config/configure_proxy.rb +18 -3
  11. data/lib/fluent/config/element.rb +1 -1
  12. data/lib/fluent/config/section.rb +1 -1
  13. data/lib/fluent/config/v1_parser.rb +1 -1
  14. data/lib/fluent/env.rb +1 -0
  15. data/lib/fluent/event.rb +49 -2
  16. data/lib/fluent/event_router.rb +6 -2
  17. data/lib/fluent/label.rb +8 -0
  18. data/lib/fluent/log.rb +30 -1
  19. data/lib/fluent/plugin.rb +1 -1
  20. data/lib/fluent/plugin/base.rb +3 -0
  21. data/lib/fluent/plugin/buf_file.rb +2 -2
  22. data/lib/fluent/plugin/buf_memory.rb +1 -1
  23. data/lib/fluent/plugin/buffer.rb +12 -2
  24. data/lib/fluent/plugin/buffer/chunk.rb +68 -7
  25. data/lib/fluent/plugin/buffer/file_chunk.rb +4 -4
  26. data/lib/fluent/plugin/buffer/memory_chunk.rb +4 -4
  27. data/lib/fluent/plugin/compressable.rb +91 -0
  28. data/lib/fluent/plugin/filter_grep.rb +4 -4
  29. data/lib/fluent/plugin/formatter.rb +2 -2
  30. data/lib/fluent/plugin/formatter_json.rb +2 -1
  31. data/lib/fluent/plugin/formatter_out_file.rb +3 -30
  32. data/lib/fluent/plugin/in_forward.rb +6 -5
  33. data/lib/fluent/plugin/in_monitor_agent.rb +7 -21
  34. data/lib/fluent/plugin/in_syslog.rb +1 -1
  35. data/lib/fluent/plugin/in_tail.rb +11 -2
  36. data/lib/fluent/plugin/multi_output.rb +63 -3
  37. data/lib/fluent/plugin/out_exec.rb +1 -1
  38. data/lib/fluent/plugin/out_file.rb +5 -1
  39. data/lib/fluent/plugin/out_forward.rb +17 -5
  40. data/lib/fluent/plugin/out_stdout.rb +2 -1
  41. data/lib/fluent/plugin/output.rb +205 -19
  42. data/lib/fluent/plugin/parser.rb +5 -49
  43. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  44. data/lib/fluent/plugin/parser_json.rb +4 -4
  45. data/lib/fluent/plugin/parser_multiline.rb +5 -5
  46. data/lib/fluent/plugin/parser_regexp.rb +1 -2
  47. data/lib/fluent/plugin/parser_syslog.rb +2 -2
  48. data/lib/fluent/plugin/storage_local.rb +2 -1
  49. data/lib/fluent/plugin_helper.rb +1 -0
  50. data/lib/fluent/plugin_helper/compat_parameters.rb +39 -21
  51. data/lib/fluent/plugin_helper/extract.rb +92 -0
  52. data/lib/fluent/plugin_helper/inject.rb +10 -12
  53. data/lib/fluent/plugin_helper/thread.rb +23 -3
  54. data/lib/fluent/registry.rb +1 -1
  55. data/lib/fluent/root_agent.rb +2 -1
  56. data/lib/fluent/supervisor.rb +28 -8
  57. data/lib/fluent/test/base.rb +0 -7
  58. data/lib/fluent/test/driver/base.rb +1 -0
  59. data/lib/fluent/test/driver/output.rb +3 -0
  60. data/lib/fluent/test/helpers.rb +18 -0
  61. data/lib/fluent/test/input_test.rb +4 -2
  62. data/lib/fluent/test/log.rb +3 -1
  63. data/lib/fluent/time.rb +232 -1
  64. data/lib/fluent/timezone.rb +1 -1
  65. data/lib/fluent/version.rb +1 -1
  66. data/test/command/test_binlog_reader.rb +351 -0
  67. data/test/config/test_config_parser.rb +6 -0
  68. data/test/config/test_configurable.rb +47 -1
  69. data/test/helper.rb +0 -1
  70. data/test/plugin/test_buffer.rb +22 -2
  71. data/test/plugin/test_buffer_chunk.rb +34 -4
  72. data/test/plugin/test_buffer_file_chunk.rb +73 -0
  73. data/test/plugin/test_buffer_memory_chunk.rb +73 -0
  74. data/test/plugin/test_compressable.rb +81 -0
  75. data/test/plugin/test_formatter_json.rb +14 -1
  76. data/test/plugin/test_in_forward.rb +67 -3
  77. data/test/plugin/test_in_monitor_agent.rb +17 -1
  78. data/test/plugin/test_in_tail.rb +8 -8
  79. data/test/plugin/test_out_file.rb +0 -8
  80. data/test/plugin/test_out_forward.rb +85 -0
  81. data/test/plugin/test_out_secondary_file.rb +20 -12
  82. data/test/plugin/test_out_stdout.rb +11 -10
  83. data/test/plugin/test_output.rb +234 -0
  84. data/test/plugin/test_output_as_buffered.rb +223 -0
  85. data/test/plugin/test_output_as_buffered_compress.rb +165 -0
  86. data/test/plugin/test_parser_json.rb +8 -0
  87. data/test/plugin/test_parser_regexp.rb +1 -1
  88. data/test/plugin_helper/test_child_process.rb +2 -2
  89. data/test/plugin_helper/test_extract.rb +195 -0
  90. data/test/plugin_helper/test_inject.rb +0 -7
  91. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +7 -0
  92. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +7 -0
  93. data/test/test_event.rb +186 -0
  94. data/test/test_event_router.rb +1 -1
  95. data/test/test_formatter.rb +0 -7
  96. data/test/test_log.rb +121 -0
  97. data/test/test_plugin_classes.rb +62 -0
  98. data/test/test_root_agent.rb +125 -0
  99. data/test/test_supervisor.rb +25 -2
  100. data/test/test_time_formatter.rb +103 -7
  101. data/test/test_time_parser.rb +211 -0
  102. metadata +22 -4
  103. data/test/plugin/test_parser_time.rb +0 -46
@@ -55,6 +55,19 @@ class MonitorAgentInputTest < Test::Unit::TestCase
55
55
  @id test_out
56
56
  </match>
57
57
  </label>
58
+ <label @copy>
59
+ <match **>
60
+ @type copy
61
+ <store>
62
+ @type test_out
63
+ @id copy_out_1
64
+ </store>
65
+ <store>
66
+ @type test_out
67
+ @id copy_out_2
68
+ </store>
69
+ </match>
70
+ </label>
58
71
  <label @ERROR>
59
72
  <match>
60
73
  @type null
@@ -153,7 +166,10 @@ EOC
153
166
  assert_equal([FluentTest::FluentTestInput,
154
167
  Fluent::Plugin::RelabelOutput,
155
168
  FluentTest::FluentTestFilter,
156
- FluentTest::FluentTestOutput,
169
+ FluentTest::FluentTestOutput, # in label @test
170
+ Fluent::Plugin::CopyOutput,
171
+ FluentTest::FluentTestOutput, # in label @copy 1
172
+ FluentTest::FluentTestOutput, # in label @copy 2
157
173
  Fluent::Plugin::NullOutput], plugins)
158
174
  end
159
175
 
@@ -5,6 +5,7 @@ require 'fluent/plugin/buffer'
5
5
  require 'fluent/system_config'
6
6
  require 'net/http'
7
7
  require 'flexmock/test_unit'
8
+ require 'timecop'
8
9
 
9
10
  class TailInputTest < Test::Unit::TestCase
10
11
  include FlexMock::TestCase
@@ -401,7 +402,7 @@ class TailInputTest < Test::Unit::TestCase
401
402
 
402
403
  d = create_driver
403
404
 
404
- d.run do
405
+ d.run(expect_emits: 1) do
405
406
  File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
406
407
  f.print "test3"
407
408
  }
@@ -410,7 +411,6 @@ class TailInputTest < Test::Unit::TestCase
410
411
  File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
411
412
  f.puts "test4"
412
413
  }
413
- sleep 1
414
414
  end
415
415
 
416
416
  events = d.events
@@ -766,9 +766,7 @@ class TailInputTest < Test::Unit::TestCase
766
766
  @loop = Coolio::Loop.new
767
767
  end
768
768
 
769
- flexstub(Time) do |timeclass|
770
- timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5), Time.new(2010, 1, 2, 3, 4, 6), Time.new(2010, 1, 2, 3, 4, 7))
771
-
769
+ Timecop.freeze(2010, 1, 2, 3, 4, 5) do
772
770
  flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
773
771
  EX_PATHS.each do |path|
774
772
  watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
@@ -781,11 +779,13 @@ class TailInputTest < Test::Unit::TestCase
781
779
  end
782
780
  plugin.refresh_watchers
783
781
  end
782
+ end
784
783
 
785
- plugin.instance_eval do
786
- @tails['test/plugin/data/2010/01/20100102-030405.log'].should_receive(:close).zero_or_more_times
787
- end
784
+ plugin.instance_eval do
785
+ @tails['test/plugin/data/2010/01/20100102-030405.log'].should_receive(:close).zero_or_more_times
786
+ end
788
787
 
788
+ Timecop.freeze(2010, 1, 2, 3, 4, 6) do
789
789
  flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
790
790
  watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
791
791
  flexmock('TailWatcher') do |watcher|
@@ -24,14 +24,6 @@ class FileOutputTest < Test::Unit::TestCase
24
24
  Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
25
25
  end
26
26
 
27
- def with_timezone(timezone = 'UTC', &block)
28
- old = ENV['TZ']
29
- ENV['TZ'] = timezone
30
- output = yield
31
- ENV['TZ'] = old
32
- output
33
- end
34
-
35
27
  def test_configure
36
28
  d = create_driver %[
37
29
  path test_path
@@ -2,6 +2,7 @@ require_relative '../helper'
2
2
  require 'fluent/test'
3
3
  require 'fluent/test/startup_shutdown'
4
4
  require 'fluent/plugin/out_forward'
5
+ require 'flexmock/test_unit'
5
6
 
6
7
  class ForwardOutputTest < Test::Unit::TestCase
7
8
  extend Fluent::Test::StartupShutdown
@@ -107,6 +108,40 @@ class ForwardOutputTest < Test::Unit::TestCase
107
108
  end
108
109
  end
109
110
 
111
+ def test_compress_default_value
112
+ d = create_driver
113
+ assert_equal :text, d.instance.compress
114
+
115
+ node = d.instance.nodes.first
116
+ assert_equal :text, node.instance_variable_get(:@compress)
117
+ end
118
+
119
+ def test_set_compress_is_gzip
120
+ d = create_driver(CONFIG + %[compress gzip])
121
+ assert_equal :gzip, d.instance.compress
122
+ assert_equal :gzip, d.instance.buffer.compress
123
+
124
+ node = d.instance.nodes.first
125
+ assert_equal :gzip, node.instance_variable_get(:@compress)
126
+ end
127
+
128
+ def test_set_compress_is_gzip_in_buffer_section
129
+ mock = flexmock($log)
130
+ mock.should_receive(:log).with("buffer is compressed. If you also want to save the bandwidth of a network, Add `compress` configuration in <match>")
131
+
132
+ d = create_driver(CONFIG + %[
133
+ <buffer>
134
+ type memory
135
+ compress gzip
136
+ </buffer>
137
+ ])
138
+ assert_equal :text, d.instance.compress
139
+ assert_equal :gzip, d.instance.buffer.compress
140
+
141
+ node = d.instance.nodes.first
142
+ assert_equal :text, node.instance_variable_get(:@compress)
143
+ end
144
+
110
145
  def test_phi_failure_detector
111
146
  d = create_driver(CONFIG + %[phi_failure_detector false \n phi_threshold 0])
112
147
  node = d.instance.nodes.first
@@ -202,6 +237,40 @@ class ForwardOutputTest < Test::Unit::TestCase
202
237
  assert_empty d.instance.exceptions
203
238
  end
204
239
 
240
+ def test_send_comprssed_message_pack_stream_if_compress_is_gzip
241
+ target_input_driver = create_target_input_driver
242
+
243
+ d = create_driver(CONFIG + %[
244
+ flush_interval 1s
245
+ compress gzip
246
+ ])
247
+
248
+ time = event_time('2011-01-02 13:14:15 UTC')
249
+
250
+ records = [
251
+ {"a" => 1},
252
+ {"a" => 2}
253
+ ]
254
+ d.register_run_post_condition do
255
+ d.instance.responses.length == 1
256
+ end
257
+
258
+ target_input_driver.run do
259
+ d.run do
260
+ records.each do |record|
261
+ d.emit record, time
262
+ end
263
+ end
264
+ end
265
+
266
+ event_streams = target_input_driver.event_streams
267
+ assert_true event_streams[0].is_a?(Fluent::CompressedMessagePackEventStream)
268
+
269
+ emits = target_input_driver.emits
270
+ assert_equal ['test', time, records[0]], emits[0]
271
+ assert_equal ['test', time, records[1]], emits[1]
272
+ end
273
+
205
274
  def test_send_to_a_node_supporting_responses
206
275
  target_input_driver = create_target_input_driver(response_stub: true)
207
276
 
@@ -529,4 +598,20 @@ class ForwardOutputTest < Test::Unit::TestCase
529
598
  node.tick
530
599
  assert_equal node.available, true
531
600
  end
601
+
602
+ def test_heartbeat_type_udp
603
+ d = create_driver(CONFIG + "\nheartbeat_type udp")
604
+
605
+ d.instance.start
606
+ usock = d.instance.instance_variable_get(:@usock)
607
+ timer = d.instance.instance_variable_get(:@timer)
608
+ hb = d.instance.instance_variable_get(:@hb)
609
+ assert_equal UDPSocket, usock.class
610
+ assert_equal Fluent::ForwardOutput::HeartbeatRequestTimer, timer.class
611
+ assert_equal Fluent::ForwardOutput::HeartbeatHandler, hb.class
612
+
613
+ mock(usock).send("\0", 0, Socket.pack_sockaddr_in(TARGET_PORT, '127.0.0.1')).once
614
+ timer.disable # call send_heartbeat at just once
615
+ timer.on_timer
616
+ end
532
617
  end
@@ -110,16 +110,19 @@ class FileOutputSecondaryTest < Test::Unit::TestCase
110
110
  # Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790
111
111
  # Following code from https://www.ruby-forum.com/topic/971591#979520
112
112
  result = ""
113
- File.open(path, "rb") { |io|
114
- loop do
115
- gzr = Zlib::GzipReader.new(io)
116
- result << gzr.read
117
- unused = gzr.unused
118
- gzr.finish
119
- break if unused.nil?
120
- io.pos -= unused.length
121
- end
122
- }
113
+ waiting(10) do
114
+ # we can expect that GzipReader#read can wait unflushed raw data of `io` on disk
115
+ File.open(path, "rb") { |io|
116
+ loop do
117
+ gzr = Zlib::GzipReader.new(io)
118
+ result << gzr.read
119
+ unused = gzr.unused
120
+ gzr.finish
121
+ break if unused.nil?
122
+ io.pos -= unused.length
123
+ end
124
+ }
125
+ end
123
126
 
124
127
  assert_equal expect, result
125
128
  end
@@ -155,10 +158,15 @@ class FileOutputSecondaryTest < Test::Unit::TestCase
155
158
  basename out_file_test
156
159
  ], @primary)
157
160
 
158
- path = d.instance.write(@chunk)
161
+ msgpack_binary = @es.to_msgpack_stream.force_encoding('ASCII-8BIT')
159
162
 
163
+ path = d.instance.write(@chunk)
160
164
  assert_equal "#{TMP_DIR}/out_file_test.0", path
161
- assert_equal File.read(path), @es.to_msgpack_stream.force_encoding('ASCII-8BIT')
165
+ waiting(5) do
166
+ sleep 0.1 until File.stat(path).size == msgpack_binary.size
167
+ end
168
+
169
+ assert_equal File.read(path), msgpack_binary
162
170
  end
163
171
 
164
172
  test 'path should be incremental when append option is false' do
@@ -9,6 +9,7 @@ class StdoutOutputTest < Test::Unit::TestCase
9
9
 
10
10
  CONFIG = %[
11
11
  ]
12
+ TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
12
13
 
13
14
  def create_driver(conf = CONFIG)
14
15
  Fluent::Test::Driver::Output.new(Fluent::Plugin::StdoutOutput).configure(conf)
@@ -40,7 +41,7 @@ class StdoutOutputTest < Test::Unit::TestCase
40
41
  d.feed(time, {'test' => 'test1'})
41
42
  end
42
43
  end
43
- assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test1\"}\n", out
44
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test1\"}\n", out
44
45
  end
45
46
 
46
47
  data('oj' => 'oj', 'yajl' => 'yajl')
@@ -52,14 +53,14 @@ class StdoutOutputTest < Test::Unit::TestCase
52
53
  d.feed(time, {'test' => 'test1'})
53
54
  end
54
55
  end
55
- assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test1\"}\n", out
56
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test1\"}\n", out
56
57
 
57
58
  if data == 'yajl'
58
59
  # NOTE: Float::NAN is not jsonable
59
60
  assert_raise(Yajl::EncodeError) { d.feed('test', time, {'test' => Float::NAN}) }
60
61
  else
61
62
  out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
62
- assert_equal "#{Time.at(time).localtime} test: {\"test\":NaN}\n", out
63
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":NaN}\n", out
63
64
  end
64
65
  end
65
66
 
@@ -71,11 +72,11 @@ class StdoutOutputTest < Test::Unit::TestCase
71
72
  d.feed(time, {'test' => 'test2'})
72
73
  end
73
74
  end
74
- assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test2\"}\n", out
75
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test2\"}\n", out
75
76
 
76
77
  # NOTE: Float::NAN is not jsonable, but hash string can output it.
77
78
  out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
78
- assert_equal "#{Time.at(time).localtime} test: {\"test\"=>NaN}\n", out
79
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>NaN}\n", out
79
80
  end
80
81
  end
81
82
 
@@ -113,7 +114,7 @@ class StdoutOutputTest < Test::Unit::TestCase
113
114
  d.feed(time, {'test' => 'test'})
114
115
  end
115
116
  end
116
- assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test\"}\n", out
117
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
117
118
  end
118
119
  end
119
120
 
@@ -128,7 +129,7 @@ class StdoutOutputTest < Test::Unit::TestCase
128
129
  d.feed(time, {'test' => 'test'})
129
130
  end
130
131
  end
131
- assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test\"}\n", out
132
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
132
133
  end
133
134
 
134
135
  data('oj' => 'oj', 'yajl' => 'yajl')
@@ -143,7 +144,7 @@ class StdoutOutputTest < Test::Unit::TestCase
143
144
  end
144
145
  end
145
146
 
146
- assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test\"}\n", out
147
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
147
148
  end
148
149
  end
149
150
 
@@ -158,7 +159,7 @@ class StdoutOutputTest < Test::Unit::TestCase
158
159
  end
159
160
  end
160
161
 
161
- assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test\"}\n", out
162
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out
162
163
  end
163
164
 
164
165
  test '#try_write(asynchronous)' do
@@ -172,7 +173,7 @@ class StdoutOutputTest < Test::Unit::TestCase
172
173
  end
173
174
  end
174
175
 
175
- assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test\"}\n", out
176
+ assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out
176
177
  end
177
178
  end
178
179
  end
@@ -287,6 +287,208 @@ class OutputTest < Test::Unit::TestCase
287
287
  assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/////tail", @i.extract_placeholders(tmpl, m)
288
288
  end
289
289
 
290
+ sub_test_case '#placeholder_validators' do
291
+ test 'returns validators for time, tag and keys when a template has placeholders even if plugin is not configured with these keys' do
292
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
293
+ validators = @i.placeholder_validators(:path, "/my/path/${tag}/${username}/file.%Y%m%d_%H%M.log")
294
+ assert_equal 3, validators.size
295
+ assert_equal 1, validators.select(&:time?).size
296
+ assert_equal 1, validators.select(&:tag?).size
297
+ assert_equal 1, validators.select(&:keys?).size
298
+ end
299
+
300
+ test 'returns validators for time, tag and keys when a plugin is configured with these keys even if a template does not have placeholders' do
301
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,username', {'timekey' => 60})]))
302
+ validators = @i.placeholder_validators(:path, "/my/path/file.log")
303
+ assert_equal 3, validators.size
304
+ assert_equal 1, validators.select(&:time?).size
305
+ assert_equal 1, validators.select(&:tag?).size
306
+ assert_equal 1, validators.select(&:keys?).size
307
+ end
308
+
309
+ test 'returns a validator for time if a template has timestamp placeholders' do
310
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
311
+ validators = @i.placeholder_validators(:path, "/my/path/file.%Y-%m-%d.log")
312
+ assert_equal 1, validators.size
313
+ assert_equal 1, validators.select(&:time?).size
314
+ assert_raise Fluent::ConfigError.new("Parameter 'path' has timestamp placeholders, but chunk key 'time' is not configured") do
315
+ validators.first.validate!
316
+ end
317
+ end
318
+
319
+ test 'returns a validator for time if a plugin is configured with time key' do
320
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => '30'})]))
321
+ validators = @i.placeholder_validators(:path, "/my/path/to/file.log")
322
+ assert_equal 1, validators.size
323
+ assert_equal 1, validators.select(&:time?).size
324
+ assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have timestamp placeholders for timekey 30") do
325
+ validators.first.validate!
326
+ end
327
+ end
328
+
329
+ test 'returns a validator for tag if a template has tag placeholders' do
330
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
331
+ validators = @i.placeholder_validators(:path, "/my/path/${tag}/file.log")
332
+ assert_equal 1, validators.size
333
+ assert_equal 1, validators.select(&:tag?).size
334
+ assert_raise Fluent::ConfigError.new("Parameter 'path' has tag placeholders, but chunk key 'tag' is not configured") do
335
+ validators.first.validate!
336
+ end
337
+ end
338
+
339
+ test 'returns a validator for tag if a plugin is configured with tag key' do
340
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
341
+ validators = @i.placeholder_validators(:path, "/my/path/file.log")
342
+ assert_equal 1, validators.size
343
+ assert_equal 1, validators.select(&:tag?).size
344
+ assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have tag placeholder") do
345
+ validators.first.validate!
346
+ end
347
+ end
348
+
349
+ test 'returns a validator for variable keys if a template has variable placeholders' do
350
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
351
+ validators = @i.placeholder_validators(:path, "/my/path/${username}/file.${group}.log")
352
+ assert_equal 1, validators.size
353
+ assert_equal 1, validators.select(&:keys?).size
354
+ assert_raise Fluent::ConfigError.new("Parameter 'path' has placeholders, but chunk keys doesn't have keys group,username") do
355
+ validators.first.validate!
356
+ end
357
+ end
358
+
359
+ test 'returns a validator for variable keys if a plugin is configured with variable keys' do
360
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,group')]))
361
+ validators = @i.placeholder_validators(:path, "/my/path/file.log")
362
+ assert_equal 1, validators.size
363
+ assert_equal 1, validators.select(&:keys?).size
364
+ assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys group,username") do
365
+ validators.first.validate!
366
+ end
367
+ end
368
+ end
369
+
370
+ sub_test_case '#placeholder_validate!' do
371
+ test 'raises configuration error for a templace when timestamp placeholders exist but time key is missing' do
372
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
373
+ assert_raise Fluent::ConfigError.new("Parameter 'path' has timestamp placeholders, but chunk key 'time' is not configured") do
374
+ @i.placeholder_validate!(:path, "/path/without/timestamp/file.%Y%m%d-%H%M.log")
375
+ end
376
+ end
377
+
378
+ test 'raises configuration error for a template without timestamp placeholders when timekey is configured' do
379
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
380
+ assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have timestamp placeholders for timekey 180") do
381
+ @i.placeholder_validate!(:path, "/my/path/file.log")
382
+ end
383
+ assert_nothing_raised do
384
+ @i.placeholder_validate!(:path, "/my/path/%Y%m%d/file.%H%M.log")
385
+ end
386
+ end
387
+
388
+ test 'raises configuration error for a template with timestamp placeholders when plugin is configured more fine timekey' do
389
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
390
+ assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have timestamp placeholder for hour('%H') for timekey 180") do
391
+ @i.placeholder_validate!(:path, "/my/path/file.%Y%m%d_%H.log")
392
+ end
393
+ assert_nothing_raised do
394
+ @i.placeholder_validate!(:path, "/my/path/file.%Y%m%d_%H%M.log")
395
+ end
396
+ end
397
+
398
+ test 'raises configuration error for a template when tag placeholders exist but tag key is missing' do
399
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
400
+ assert_raise Fluent::ConfigError.new("Parameter 'path' has tag placeholders, but chunk key 'tag' is not configured") do
401
+ @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
402
+ end
403
+ end
404
+
405
+ test 'raises configuration error for a template without tag placeholders when tagkey is configured' do
406
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
407
+ assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have tag placeholder") do
408
+ @i.placeholder_validate!(:path, "/my/path/file.log")
409
+ end
410
+ assert_nothing_raised do
411
+ @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
412
+ end
413
+ end
414
+
415
+ test 'raises configuration error for a template when variable key placeholders exist but chunk keys are missing' do
416
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
417
+ assert_raise Fluent::ConfigError.new("Parameter 'path' has placeholders, but chunk keys doesn't have keys service,username") do
418
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
419
+ end
420
+ end
421
+
422
+ test 'raises configuration error for a template without variable key placeholders when chunk keys are configured' do
423
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
424
+ assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys service,username") do
425
+ @i.placeholder_validate!(:path, "/my/path/file.log")
426
+ end
427
+ assert_nothing_raised do
428
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
429
+ end
430
+ end
431
+
432
+ test 'raise configuration error for a template and configuration with keys mismatch' do
433
+ @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
434
+ assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys service") do
435
+ @i.placeholder_validate!(:path, "/my/path/file.${username}.log")
436
+ end
437
+ assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys username") do
438
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.log")
439
+ end
440
+ assert_nothing_raised do
441
+ @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
442
+ end
443
+ end
444
+ end
445
+
446
+ test '#get_placeholders_time returns seconds,title and example placeholder for a template' do
447
+ s, t, e = @i.get_placeholders_time("/path/to/dir/yay")
448
+ assert_nil s
449
+ assert_nil t
450
+ assert_nil e
451
+
452
+ s, t, e = @i.get_placeholders_time("/path/to/%Y%m%d/yay")
453
+ assert_equal 86400, s
454
+ assert_equal :day, t
455
+ assert_equal '%d', e
456
+ s, t, e = @i.get_placeholders_time("my birthiday! at %F")
457
+ assert_equal 86400, s
458
+ assert_equal :day, t
459
+ assert_equal '%d', e
460
+
461
+ s, t, e = @i.get_placeholders_time("myfile.%Y-%m-%d_%H.log")
462
+ assert_equal 3600, s
463
+ assert_equal :hour, t
464
+ assert_equal '%H', e
465
+
466
+ s, t, e = @i.get_placeholders_time("part-%Y%m%d-%H%M.ts")
467
+ assert_equal 60, s
468
+ assert_equal :minute, t
469
+ assert_equal '%M', e
470
+
471
+ s, t, e = @i.get_placeholders_time("my first data at %F %T %z")
472
+ assert_equal 1, s
473
+ assert_equal :second, t
474
+ assert_equal '%S', e
475
+ end
476
+
477
+ test '#get_placeholders_tag returns a list of tag part position for a template' do
478
+ assert_equal [], @i.get_placeholders_tag("db.table")
479
+ assert_equal [], @i.get_placeholders_tag("db.table_${non_tag}")
480
+ assert_equal [-1], @i.get_placeholders_tag("table_${tag}")
481
+ assert_equal [0, 1], @i.get_placeholders_tag("db_${tag[0]}.table_${tag[1]}")
482
+ assert_equal [-1, 0], @i.get_placeholders_tag("/treedir/${tag[0]}/${tag}")
483
+ end
484
+
485
+ test '#get_placeholders_keys returns a list of keys for a template' do
486
+ assert_equal [], @i.get_placeholders_keys("/path/to/my/data/file.log")
487
+ assert_equal [], @i.get_placeholders_keys("/path/to/my/${tag}/file.log")
488
+ assert_equal ['key1', 'key2'], @i.get_placeholders_keys("/path/to/${key2}/${tag}/file.${key1}.log")
489
+ assert_equal ['.hidden', '0001', '@timestamp', 'a_key', 'my-domain'], @i.get_placeholders_keys("http://${my-domain}/${.hidden}/${0001}/${a_key}?timestamp=${@timestamp}")
490
+ end
491
+
290
492
  test '#metadata returns object which contains tag/timekey/variables from records as specified in configuration' do
291
493
  tag = 'test.output'
292
494
  time = event_time('2016-04-12 15:31:23 -0700')
@@ -565,4 +767,36 @@ class OutputTest < Test::Unit::TestCase
565
767
  @i.stop; @i.before_shutdown; @i.shutdown; @i.after_shutdown; @i.close; @i.terminate
566
768
  end
567
769
  end
770
+
771
+ sub_test_case '#generate_format_proc' do
772
+ test "when output doesn't have <buffer>" do
773
+ i = create_output(:sync)
774
+ i.configure(config_element('ROOT', '', {}, []))
775
+ assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
776
+ end
777
+
778
+ test "when output doesn't have <buffer> and time_as_integer is true" do
779
+ i = create_output(:sync)
780
+ i.configure(config_element('ROOT', '', {'time_as_integer' => true}))
781
+ assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, i.generate_format_proc
782
+ end
783
+
784
+ test 'when output has <buffer> and compress is gzip' do
785
+ i = create_output(:buffered)
786
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'gzip'})]))
787
+ assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM, i.generate_format_proc
788
+ end
789
+
790
+ test 'when output has <buffer> and compress is gzip and time_as_integer is true' do
791
+ i = create_output(:buffered)
792
+ i.configure(config_element('ROOT', '', {'time_as_integer' => true}, [config_element('buffer', '', {'compress' => 'gzip'})]))
793
+ assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT, i.generate_format_proc
794
+ end
795
+
796
+ test 'when output has <buffer> and compress is text' do
797
+ i = create_output(:buffered)
798
+ i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'text'})]))
799
+ assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
800
+ end
801
+ end
568
802
  end