fluentd 0.14.7-x64-mingw32 → 0.14.10-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (120) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/.travis.yml +2 -0
  4. data/CONTRIBUTING.md +6 -1
  5. data/ChangeLog +95 -0
  6. data/Rakefile +21 -0
  7. data/appveyor.yml +1 -0
  8. data/code-of-conduct.md +3 -0
  9. data/example/out_exec_filter.conf +42 -0
  10. data/fluentd.gemspec +1 -1
  11. data/lib/fluent/agent.rb +2 -2
  12. data/lib/fluent/command/binlog_reader.rb +1 -1
  13. data/lib/fluent/command/cat.rb +15 -4
  14. data/lib/fluent/compat/output.rb +14 -9
  15. data/lib/fluent/compat/parser.rb +141 -11
  16. data/lib/fluent/config/configure_proxy.rb +2 -11
  17. data/lib/fluent/config/section.rb +8 -1
  18. data/lib/fluent/configurable.rb +1 -3
  19. data/lib/fluent/env.rb +1 -1
  20. data/lib/fluent/log.rb +1 -1
  21. data/lib/fluent/plugin/base.rb +17 -0
  22. data/lib/fluent/plugin/filter_parser.rb +108 -0
  23. data/lib/fluent/plugin/filter_record_transformer.rb +14 -35
  24. data/lib/fluent/plugin/filter_stdout.rb +1 -1
  25. data/lib/fluent/plugin/formatter.rb +5 -0
  26. data/lib/fluent/plugin/formatter_msgpack.rb +4 -0
  27. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  28. data/lib/fluent/plugin/formatter_tsv.rb +34 -0
  29. data/lib/fluent/plugin/in_exec.rb +48 -93
  30. data/lib/fluent/plugin/in_forward.rb +66 -265
  31. data/lib/fluent/plugin/in_http.rb +68 -65
  32. data/lib/fluent/plugin/in_monitor_agent.rb +8 -4
  33. data/lib/fluent/plugin/in_syslog.rb +42 -58
  34. data/lib/fluent/plugin/in_tail.rb +29 -14
  35. data/lib/fluent/plugin/in_tcp.rb +54 -14
  36. data/lib/fluent/plugin/in_udp.rb +49 -13
  37. data/lib/fluent/plugin/multi_output.rb +1 -3
  38. data/lib/fluent/plugin/out_exec.rb +58 -71
  39. data/lib/fluent/plugin/out_exec_filter.rb +199 -279
  40. data/lib/fluent/plugin/out_file.rb +172 -81
  41. data/lib/fluent/plugin/out_forward.rb +229 -206
  42. data/lib/fluent/plugin/out_stdout.rb +6 -21
  43. data/lib/fluent/plugin/output.rb +90 -59
  44. data/lib/fluent/plugin/parser.rb +121 -61
  45. data/lib/fluent/plugin/parser_csv.rb +9 -3
  46. data/lib/fluent/plugin/parser_json.rb +37 -35
  47. data/lib/fluent/plugin/parser_ltsv.rb +11 -19
  48. data/lib/fluent/plugin/parser_msgpack.rb +50 -0
  49. data/lib/fluent/plugin/parser_regexp.rb +15 -42
  50. data/lib/fluent/plugin/parser_tsv.rb +8 -3
  51. data/lib/fluent/plugin_helper.rb +10 -1
  52. data/lib/fluent/plugin_helper/child_process.rb +139 -73
  53. data/lib/fluent/plugin_helper/compat_parameters.rb +93 -4
  54. data/lib/fluent/plugin_helper/event_emitter.rb +14 -1
  55. data/lib/fluent/plugin_helper/event_loop.rb +24 -6
  56. data/lib/fluent/plugin_helper/extract.rb +16 -4
  57. data/lib/fluent/plugin_helper/formatter.rb +9 -11
  58. data/lib/fluent/plugin_helper/inject.rb +16 -1
  59. data/lib/fluent/plugin_helper/parser.rb +3 -3
  60. data/lib/fluent/plugin_helper/server.rb +494 -0
  61. data/lib/fluent/plugin_helper/socket.rb +101 -0
  62. data/lib/fluent/plugin_helper/socket_option.rb +84 -0
  63. data/lib/fluent/plugin_helper/timer.rb +1 -0
  64. data/lib/fluent/root_agent.rb +1 -1
  65. data/lib/fluent/test/driver/base.rb +95 -49
  66. data/lib/fluent/test/driver/base_owner.rb +18 -8
  67. data/lib/fluent/test/driver/multi_output.rb +2 -1
  68. data/lib/fluent/test/driver/output.rb +29 -6
  69. data/lib/fluent/test/helpers.rb +3 -1
  70. data/lib/fluent/test/log.rb +4 -0
  71. data/lib/fluent/test/startup_shutdown.rb +13 -0
  72. data/lib/fluent/time.rb +14 -8
  73. data/lib/fluent/version.rb +1 -1
  74. data/lib/fluent/winsvc.rb +1 -1
  75. data/test/command/test_binlog_reader.rb +5 -1
  76. data/test/compat/test_parser.rb +10 -0
  77. data/test/config/test_configurable.rb +193 -0
  78. data/test/config/test_configure_proxy.rb +0 -43
  79. data/test/helper.rb +36 -1
  80. data/test/plugin/test_base.rb +16 -0
  81. data/test/plugin/test_filter_parser.rb +665 -0
  82. data/test/plugin/test_filter_record_transformer.rb +36 -100
  83. data/test/plugin/test_filter_stdout.rb +18 -27
  84. data/test/plugin/test_in_dummy.rb +1 -1
  85. data/test/plugin/test_in_exec.rb +206 -94
  86. data/test/plugin/test_in_forward.rb +268 -347
  87. data/test/plugin/test_in_http.rb +310 -186
  88. data/test/plugin/test_in_monitor_agent.rb +65 -35
  89. data/test/plugin/test_in_syslog.rb +39 -3
  90. data/test/plugin/test_in_tcp.rb +78 -62
  91. data/test/plugin/test_in_udp.rb +101 -80
  92. data/test/plugin/test_out_exec.rb +223 -68
  93. data/test/plugin/test_out_exec_filter.rb +520 -169
  94. data/test/plugin/test_out_file.rb +637 -177
  95. data/test/plugin/test_out_forward.rb +242 -234
  96. data/test/plugin/test_out_null.rb +1 -1
  97. data/test/plugin/test_out_secondary_file.rb +4 -2
  98. data/test/plugin/test_out_stdout.rb +14 -35
  99. data/test/plugin/test_output_as_buffered.rb +60 -2
  100. data/test/plugin/test_parser.rb +359 -0
  101. data/test/plugin/test_parser_csv.rb +1 -2
  102. data/test/plugin/test_parser_json.rb +3 -4
  103. data/test/plugin/test_parser_labeled_tsv.rb +1 -2
  104. data/test/plugin/test_parser_none.rb +1 -2
  105. data/test/plugin/test_parser_regexp.rb +8 -4
  106. data/test/plugin/test_parser_tsv.rb +4 -3
  107. data/test/plugin_helper/test_child_process.rb +184 -0
  108. data/test/plugin_helper/test_compat_parameters.rb +88 -1
  109. data/test/plugin_helper/test_extract.rb +0 -1
  110. data/test/plugin_helper/test_formatter.rb +5 -2
  111. data/test/plugin_helper/test_inject.rb +21 -0
  112. data/test/plugin_helper/test_parser.rb +6 -5
  113. data/test/plugin_helper/test_server.rb +905 -0
  114. data/test/test_event_time.rb +3 -1
  115. data/test/test_output.rb +53 -2
  116. data/test/test_plugin_classes.rb +20 -0
  117. data/test/test_root_agent.rb +139 -0
  118. data/test/test_test_drivers.rb +135 -0
  119. metadata +28 -8
  120. data/test/plugin/test_parser_base.rb +0 -32
@@ -2,9 +2,11 @@ require_relative '../helper'
2
2
  require 'timecop'
3
3
  require 'fluent/test/driver/filter'
4
4
  require 'fluent/plugin/filter_record_transformer'
5
+ require 'flexmock/test_unit'
5
6
 
6
7
  class RecordTransformerFilterTest < Test::Unit::TestCase
7
8
  include Fluent
9
+ include FlexMock::TestCase
8
10
 
9
11
  setup do
10
12
  Test.setup
@@ -53,7 +55,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
53
55
  hostname ${hostname}
54
56
  tag ${tag}
55
57
  time ${time}
56
- message ${hostname} ${tag_parts[-1]} ${message}
58
+ message ${hostname} ${tag_parts[-1]} ${record["message"]}
57
59
  </record>
58
60
  ]
59
61
 
@@ -119,7 +121,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
119
121
  config = %[
120
122
  enable_ruby yes
121
123
  <record>
122
- message ${hostname} ${tag_parts.last} ${"'" + message + "'"}
124
+ message ${hostname} ${tag_parts.last} ${"'" + record["message"] + "'"}
123
125
  </record>
124
126
  ]
125
127
  msgs = ['1', '2']
@@ -250,7 +252,11 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
250
252
  ]
251
253
  filtered = filter(config)
252
254
  filtered.each do |t, r|
253
- assert_equal(Time.at(@time).localtime.to_s, r['message'])
255
+ if enable_ruby == "yes"
256
+ assert_equal(Time.at(@time).localtime, r['message'])
257
+ else
258
+ assert_equal(Time.at(@time).localtime.to_s, r['message'])
259
+ end
254
260
  end
255
261
  end
256
262
 
@@ -259,8 +265,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
259
265
  enable_ruby #{enable_ruby}
260
266
  remove_keys eventType0
261
267
  <record>
262
- message bar ${message}
263
- eventtype ${eventType0}
268
+ message bar ${record["message"]}
269
+ eventtype ${record["eventType0"]}
264
270
  </record>
265
271
  ]
266
272
  msgs = ['1', '2']
@@ -356,11 +362,11 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
356
362
  auto_typecast false
357
363
  enable_ruby #{enable_ruby}
358
364
  <record>
359
- single ${source}
360
- multiple ${source}${source}
361
- with_prefix prefix-${source}
362
- with_suffix ${source}-suffix
363
- with_quote source[""]
365
+ single ${record["source"]}
366
+ multiple ${record["source"]}${record["source"]}
367
+ with_prefix prefix-${record["source"]}
368
+ with_suffix ${record["source"]}-suffix
369
+ with_quote record["source"][""]
364
370
  </record>
365
371
  ]
366
372
  msgs = [
@@ -375,27 +381,27 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
375
381
  multiple: "stringstring",
376
382
  with_prefix: "prefix-string",
377
383
  with_suffix: "string-suffix",
378
- with_quote: %Q{source[""]} },
384
+ with_quote: %Q{record["source"][""]} },
379
385
  { single: 123.to_s,
380
386
  multiple: "#{123.to_s}#{123.to_s}",
381
387
  with_prefix: "prefix-#{123.to_s}",
382
388
  with_suffix: "#{123.to_s}-suffix",
383
- with_quote: %Q{source[""]} },
389
+ with_quote: %Q{record["source"][""]} },
384
390
  { single: [1, 2].to_s,
385
391
  multiple: "#{[1, 2].to_s}#{[1, 2].to_s}",
386
392
  with_prefix: "prefix-#{[1, 2].to_s}",
387
393
  with_suffix: "#{[1, 2].to_s}-suffix",
388
- with_quote: %Q{source[""]} },
394
+ with_quote: %Q{record["source"][""]} },
389
395
  { single: {a:1, b:2}.to_s,
390
396
  multiple: "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
391
397
  with_prefix: "prefix-#{{a:1, b:2}.to_s}",
392
398
  with_suffix: "#{{a:1, b:2}.to_s}-suffix",
393
- with_quote: %Q{source[""]} },
399
+ with_quote: %Q{record["source"][""]} },
394
400
  { single: nil.to_s,
395
401
  multiple: "#{nil.to_s}#{nil.to_s}",
396
402
  with_prefix: "prefix-#{nil.to_s}",
397
403
  with_suffix: "#{nil.to_s}-suffix",
398
- with_quote: %Q{source[""]} },
404
+ with_quote: %Q{record["source"][""]} },
399
405
  ]
400
406
  actual_results = []
401
407
  filtered = filter(config, msgs)
@@ -416,10 +422,10 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
416
422
  auto_typecast yes
417
423
  enable_ruby #{enable_ruby}
418
424
  <record>
419
- single ${source}
420
- multiple ${source}${source}
421
- with_prefix prefix-${source}
422
- with_suffix ${source}-suffix
425
+ single ${record["source"]}
426
+ multiple ${record["source"]}${record["source"]}
427
+ with_prefix prefix-${record["source"]}
428
+ with_suffix ${record["source"]}-suffix
423
429
  </record>
424
430
  ]
425
431
  msgs = [
@@ -463,28 +469,6 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
463
469
  end
464
470
  assert_equal(expected_results, actual_results)
465
471
  end
466
-
467
- test %Q[record["key"] with enable_ruby #{enable_ruby}] do
468
- config = %[
469
- enable_ruby #{enable_ruby}
470
- auto_typecast yes
471
- <record>
472
- _timestamp ${record["@timestamp"]}
473
- _foo_bar ${record["foo.bar"]}
474
- </record>
475
- ]
476
- d = create_driver(config)
477
- record = {
478
- "foo.bar" => "foo.bar",
479
- "@timestamp" => 10,
480
- }
481
- d.run { d.feed(@tag, @time, record) }
482
- filtered = d.filtered
483
- filtered.each do |t, r|
484
- assert { r['_timestamp'] == record['@timestamp'] }
485
- assert { r['_foo_bar'] == record['foo.bar'] }
486
- end
487
- end
488
472
  end
489
473
 
490
474
  test 'unknown placeholder (enable_ruby no)' do
@@ -499,26 +483,11 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
499
483
  }
500
484
  end
501
485
 
502
- test 'failed to expand (enable_ruby yes)' do
503
- config = %[
504
- enable_ruby yes
505
- <record>
506
- message ${unknown['bar']}
507
- </record>
508
- ]
509
- filtered = filter(config) { |d|
510
- mock(d.instance.log).warn("failed to expand `%Q[\#{unknown['bar']}]`", anything)
511
- }
512
- filtered.each do |t, r|
513
- assert_nil(r['message'])
514
- end
515
- end
516
-
517
486
  test 'expand fields starting with @ (enable_ruby no)' do
518
487
  config = %[
519
488
  enable_ruby no
520
489
  <record>
521
- foo ${@timestamp}
490
+ foo ${record["@timestamp"]}
522
491
  </record>
523
492
  ]
524
493
  d = create_driver(config)
@@ -547,54 +516,21 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
547
516
  assert_equal([message["@timestamp"]], r['foo'])
548
517
  end
549
518
  end
519
+ end # test placeholders
550
520
 
551
- test 'expand fields starting with @ (enable_ruby yes)' do
552
- config = %[
521
+ sub_test_case 'test error record' do
522
+ test 'invalid record for placeholders' do
523
+ d = create_driver(%[
553
524
  enable_ruby yes
554
525
  <record>
555
- foo ${__send__("@timestamp")}
526
+ foo ${record["unknown"]["key"]}
556
527
  </record>
557
- ]
558
- d = create_driver(config)
559
- message = {"@timestamp" => "foo"}
560
- d.run { d.feed(@tag, @time, message) }
561
- filtered = d.filtered
562
- filtered.each do |t, r|
563
- assert_equal(message["@timestamp"], r['foo'])
528
+ ])
529
+ flexmock(d.instance.router).should_receive(:emit_error_event).
530
+ with(String, Fluent::EventTime, Hash, RuntimeError).once
531
+ d.run do
532
+ d.feed(@tag, Fluent::EventTime.now, {'key' => 'value'})
564
533
  end
565
534
  end
566
- end # test placeholders
567
-
568
- test "compatibility test (enable_ruby yes)" do
569
- config = %[
570
- enable_ruby yes
571
- auto_typecast yes
572
- <record>
573
- _message prefix-${message}-suffix
574
- _time ${Time.at(time)}
575
- _number ${number == '-' ? 0 : number}
576
- _match ${/0x[0-9a-f]+/.match(hex)[0]}
577
- _timestamp ${__send__("@timestamp")}
578
- _foo_bar ${__send__('foo.bar')}
579
- </record>
580
- ]
581
- d = create_driver(config)
582
- record = {
583
- "number" => "-",
584
- "hex" => "0x10",
585
- "foo.bar" => "foo.bar",
586
- "@timestamp" => 10,
587
- "message" => "10",
588
- }
589
- d.run { d.feed(@tag, @time, record) }
590
- filtered = d.filtered
591
- filtered.each do |t, r|
592
- assert { r['_message'] == "prefix-#{record['message']}-suffix" }
593
- assert { r['_time'] == Time.at(@time) }
594
- assert { r['_number'] == 0 }
595
- assert { r['_match'] == record['hex'] }
596
- assert { r['_timestamp'] == record['@timestamp'] }
597
- assert { r['_foo_bar'] == record['foo.bar'] }
598
- end
599
535
  end
600
536
  end
@@ -67,10 +67,9 @@ class StdoutFilterTest < Test::Unit::TestCase
67
67
 
68
68
  def test_output_type_json
69
69
  d = create_driver(CONFIG + config_element("", "", { "output_type" => "json" }))
70
- etime = event_time
71
- time = Time.at(etime.sec)
70
+ etime = event_time("2016-10-07 21:09:31.012345678 UTC")
72
71
  out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
73
- assert_equal "#{time.localtime} filter.test: {\"test\":\"test\"}\n", out
72
+ assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\"}\n", out
74
73
 
75
74
  # NOTE: Float::NAN is not jsonable
76
75
  d = create_driver(CONFIG + config_element("", "", { "output_type" => "json" }))
@@ -80,15 +79,14 @@ class StdoutFilterTest < Test::Unit::TestCase
80
79
 
81
80
  def test_output_type_hash
82
81
  d = create_driver(CONFIG + config_element("", "", { "output_type" => "hash" }))
83
- etime = event_time
84
- time = Time.at(etime.sec)
82
+ etime = event_time("2016-10-07 21:09:31.012345678 UTC")
85
83
  out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
86
- assert_equal "#{time.localtime} filter.test: {\"test\"=>\"test\"}\n", out
84
+ assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>\"test\"}\n", out
87
85
 
88
86
  # NOTE: Float::NAN is not jsonable, but hash string can output it.
89
87
  d = create_driver(CONFIG + config_element("", "", { "output_type" => "hash" }))
90
88
  out = capture_log(d) { filter(d, etime, {'test' => Float::NAN}) }
91
- assert_equal "#{time.localtime} filter.test: {\"test\"=>NaN}\n", out
89
+ assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>NaN}\n", out
92
90
  end
93
91
 
94
92
  # Use include_time_key to output the message's time
@@ -99,11 +97,9 @@ class StdoutFilterTest < Test::Unit::TestCase
99
97
  "localtime" => false
100
98
  })
101
99
  d = create_driver(config)
102
- etime = event_time
103
- time = Time.at(etime.sec)
104
- message_time = event_time("2011-01-02 13:14:15 UTC")
105
- out = capture_log(d) { filter(d, message_time, {'test' => 'test'}) }
106
- assert_equal "#{time.localtime} filter.test: {\"test\":\"test\",\"time\":\"2011-01-02T13:14:15Z\"}\n", out
100
+ etime = event_time("2016-10-07 21:09:31.012345678 UTC")
101
+ out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
102
+ assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\",\"time\":\"2016-10-07T21:09:31Z\"}\n", out
107
103
  end
108
104
 
109
105
  # out_stdout formatter itself can also be replaced
@@ -150,10 +146,9 @@ class StdoutFilterTest < Test::Unit::TestCase
150
146
  conf = config_element
151
147
  conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "json" })
152
148
  d = create_driver(conf)
153
- etime = event_time
154
- time = Time.at(etime.sec)
149
+ etime = event_time("2016-10-07 21:09:31.012345678 UTC")
155
150
  out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
156
- assert_equal "#{time.localtime} filter.test: {\"test\":\"test\"}\n", out
151
+ assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\"}\n", out
157
152
  end
158
153
 
159
154
  def test_json_nan
@@ -161,7 +156,7 @@ class StdoutFilterTest < Test::Unit::TestCase
161
156
  conf = config_element
162
157
  conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "json" })
163
158
  d = create_driver(conf)
164
- etime = event_time
159
+ etime = event_time("2016-10-07 21:09:31.012345678 UTC")
165
160
  flexmock(d.instance.router).should_receive(:emit_error_event)
166
161
  filter(d, etime, {'test' => Float::NAN})
167
162
  end
@@ -170,10 +165,9 @@ class StdoutFilterTest < Test::Unit::TestCase
170
165
  conf = config_element
171
166
  conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "hash" })
172
167
  d = create_driver(conf)
173
- etime = event_time
174
- time = Time.at(etime.sec)
168
+ etime = event_time("2016-10-07 21:09:31.012345678 UTC")
175
169
  out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
176
- assert_equal "#{time.localtime} filter.test: {\"test\"=>\"test\"}\n", out
170
+ assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>\"test\"}\n", out
177
171
  end
178
172
 
179
173
  def test_hash_nan
@@ -181,10 +175,9 @@ class StdoutFilterTest < Test::Unit::TestCase
181
175
  conf = config_element
182
176
  conf.elements << config_element("format", "", { "@type" => "stdout", "output_type" => "hash" })
183
177
  d = create_driver(conf)
184
- etime = event_time
185
- time = Time.at(etime.sec)
178
+ etime = event_time("2016-10-07 21:09:31.012345678 UTC")
186
179
  out = capture_log(d) { filter(d, etime, {'test' => Float::NAN}) }
187
- assert_equal "#{time.localtime} filter.test: {\"test\"=>NaN}\n", out
180
+ assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\"=>NaN}\n", out
188
181
  end
189
182
 
190
183
  # Use include_time_key to output the message's time
@@ -200,11 +193,9 @@ class StdoutFilterTest < Test::Unit::TestCase
200
193
  "localtime" => false
201
194
  })
202
195
  d = create_driver(conf)
203
- etime = event_time
204
- time = Time.at(etime.sec)
205
- message_time = event_time("2011-01-02 13:14:15 UTC")
206
- out = capture_log(d) { filter(d, message_time, {'test' => 'test'}) }
207
- assert_equal "#{time.localtime} filter.test: {\"test\":\"test\",\"time\":\"2011-01-02T13:14:15Z\"}\n", out
196
+ etime = event_time("2016-10-07 21:09:31.012345678 UTC")
197
+ out = capture_log(d) { filter(d, etime, {'test' => 'test'}) }
198
+ assert_equal "2016-10-07 21:09:31.012345678 +0000 filter.test: {\"test\":\"test\",\"time\":\"2016-10-07T21:09:31Z\"}\n", out
208
199
  end
209
200
  end
210
201
  end
@@ -15,7 +15,7 @@ class DummyTest < Test::Unit::TestCase
15
15
  sub_test_case 'configure' do
16
16
  test 'required parameters' do
17
17
  assert_raise_message("'tag' parameter is required") do
18
- create_driver('')
18
+ Fluent::Plugin::DummyInput.new.configure(config_element('ROOT',''))
19
19
  end
20
20
  end
21
21
 
@@ -1,133 +1,245 @@
1
1
  require_relative '../helper'
2
2
  require 'fluent/test/driver/input'
3
3
  require 'fluent/plugin/in_exec'
4
- require 'net/http'
4
+ require 'timecop'
5
5
 
6
6
  class ExecInputTest < Test::Unit::TestCase
7
+ SCRIPT_PATH = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'exec_script.rb'))
8
+ TEST_TIME = "2011-01-02 13:14:15"
9
+ TEST_UNIX_TIME = Time.parse(TEST_TIME)
10
+
7
11
  def setup
8
12
  Fluent::Test.setup
9
- @test_time = event_time("2011-01-02 13:14:15")
10
- @script = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'exec_script.rb'))
13
+ @test_time = event_time()
11
14
  end
12
15
 
13
- def create_driver(conf = tsv_config)
16
+ def create_driver(conf)
14
17
  Fluent::Test::Driver::Input.new(Fluent::Plugin::ExecInput).configure(conf)
15
18
  end
16
19
 
17
- def tsv_config
18
- %[
19
- command ruby #{@script} "2011-01-02 13:14:15" 0
20
+ DEFAULT_CONFIG_ONLY_WITH_KEYS = %[
21
+ command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 0
22
+ run_interval 1s
23
+ tag "my.test.data"
24
+ <parse>
25
+ keys ["k1", "k2", "k3"]
26
+ </parse>
27
+ ]
28
+
29
+ TSV_CONFIG = %[
30
+ command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 0
31
+ run_interval 1s
32
+ <parse>
33
+ @type tsv
34
+ keys time, tag, k1
35
+ </parse>
36
+ <extract>
37
+ tag_key tag
38
+ time_key time
39
+ time_type string
40
+ time_format %Y-%m-%d %H:%M:%S
41
+ </extract>
42
+ ]
43
+
44
+ JSON_CONFIG = %[
45
+ command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 1
46
+ run_interval 1s
47
+ <parse>
48
+ @type json
49
+ </parse>
50
+ <extract>
51
+ tag_key tag
52
+ time_key time
53
+ time_type unixtime
54
+ </extract>
55
+ ]
56
+
57
+ MSGPACK_CONFIG = %[
58
+ command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 2
59
+ run_interval 1s
60
+ <parse>
61
+ @type msgpack
62
+ </parse>
63
+ <extract>
64
+ tag_key tagger
65
+ time_key datetime
66
+ time_type unixtime
67
+ </extract>
68
+ ]
69
+
70
+ # here document for not de-quoting backslashes
71
+ REGEXP_CONFIG = %[
72
+ command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 3
73
+ run_interval 1s
74
+ tag regex_tag
75
+ ] + <<'EOC'
76
+ <parse>
77
+ @type regexp
78
+ expression "(?<time>[^\\]]*) (?<message>[^ ]*)"
79
+ time_key time
80
+ time_type string
81
+ time_format %Y-%m-%d %H:%M:%S
82
+ </parse>
83
+ EOC
84
+
85
+ sub_test_case 'with configuration with sections' do
86
+ test 'configure with default tsv format without extract' do
87
+ d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS
88
+ assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }
89
+ assert_equal "my.test.data", d.instance.tag
90
+ assert_equal ["k1", "k2", "k3"], d.instance.parser.keys
91
+ end
92
+
93
+ test 'configure raises error if both of tag and extract.tag_key are missing' do
94
+ assert_raise Fluent::ConfigError.new("'tag' or 'tag_key' option is required on exec input") do
95
+ create_driver %[
96
+ command ruby -e 'puts "yay"'
97
+ <parse>
98
+ keys y1
99
+ </parse>
100
+ ]
101
+ end
102
+ end
103
+
104
+ test 'configure for tsv' do
105
+ d = create_driver TSV_CONFIG
106
+ assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }
107
+ assert_equal ["time", "tag", "k1"], d.instance.parser.keys
108
+ assert_equal "tag", d.instance.extract_config.tag_key
109
+ assert_equal "time", d.instance.extract_config.time_key
110
+ assert_equal :string, d.instance.extract_config.time_type
111
+ assert_equal "%Y-%m-%d %H:%M:%S", d.instance.extract_config.time_format
112
+ end
113
+
114
+ test 'configure for json' do
115
+ d = create_driver JSON_CONFIG
116
+ assert{ d.instance.parser.is_a? Fluent::Plugin::JSONParser }
117
+ assert_equal "tag", d.instance.extract_config.tag_key
118
+ assert_equal "time", d.instance.extract_config.time_key
119
+ assert_equal :unixtime, d.instance.extract_config.time_type
120
+ end
121
+
122
+ test 'configure for msgpack' do
123
+ d = create_driver MSGPACK_CONFIG
124
+ assert{ d.instance.parser.is_a? Fluent::Plugin::MessagePackParser }
125
+ assert_equal "tagger", d.instance.extract_config.tag_key
126
+ assert_equal "datetime", d.instance.extract_config.time_key
127
+ assert_equal :unixtime, d.instance.extract_config.time_type
128
+ end
129
+
130
+ test 'configure for regexp' do
131
+ d = create_driver REGEXP_CONFIG
132
+ assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }
133
+ assert_equal "regex_tag", d.instance.tag
134
+ expression = <<'EXP'.chomp
135
+ (?<time>[^\]]*) (?<message>[^ ]*)
136
+ EXP
137
+ assert_equal expression, d.instance.parser.expression
138
+ assert_nil d.instance.extract_config
139
+ end
140
+ end
141
+
142
+ TSV_CONFIG_COMPAT = %[
143
+ command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 0
20
144
  keys time,tag,k1
21
145
  time_key time
22
146
  tag_key tag
23
147
  time_format %Y-%m-%d %H:%M:%S
24
148
  run_interval 1s
25
- ]
26
- end
149
+ ]
27
150
 
28
- def json_config
29
- %[
30
- command ruby #{@script} #{@test_time} 1
151
+ JSON_CONFIG_COMPAT = %[
152
+ command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 1
31
153
  format json
32
154
  tag_key tag
33
155
  time_key time
34
156
  run_interval 1s
35
- ]
36
- end
157
+ ]
37
158
 
38
- def msgpack_config
39
- %[
40
- command ruby #{@script} #{@test_time} 2
159
+ MSGPACK_CONFIG_COMPAT = %[
160
+ command ruby #{SCRIPT_PATH} #{TEST_UNIX_TIME.to_i} 2
41
161
  format msgpack
42
162
  tag_key tagger
43
163
  time_key datetime
44
164
  run_interval 1s
45
- ]
46
- end
165
+ ]
47
166
 
48
- def regexp_config
49
- %[
50
- command ruby #{@script} "2011-01-02 13:14:15" 3
167
+ REGEXP_CONFIG_COMPAT = %[
168
+ command ruby #{SCRIPT_PATH} "#{TEST_TIME}" 3
51
169
  format /(?<time>[^\\\]]*) (?<message>[^ ]*)/
52
170
  tag regex_tag
53
171
  run_interval 1s
54
- ]
55
- end
56
-
57
- def test_configure
58
- d = create_driver
59
- assert_equal 'tsv', d.instance.format
60
- assert_equal ["time","tag","k1"], d.instance.keys
61
- assert_equal "tag", d.instance.tag_key
62
- assert_equal "time", d.instance.time_key
63
- assert_equal "%Y-%m-%d %H:%M:%S", d.instance.time_format
64
- end
65
-
66
- def test_configure_with_json
67
- d = create_driver json_config
68
- assert_equal 'json', d.instance.format
69
- assert_equal [], d.instance.keys
70
- end
71
-
72
- def test_configure_with_msgpack
73
- d = create_driver msgpack_config
74
- assert_equal 'msgpack', d.instance.format
75
- assert_equal [], d.instance.keys
172
+ ]
173
+
174
+ sub_test_case 'with traditional configuration' do
175
+ test 'configure' do
176
+ d = create_driver TSV_CONFIG_COMPAT
177
+ assert{ d.instance.parser.is_a? Fluent::Plugin::TSVParser }
178
+ assert_equal ["time","tag","k1"], d.instance.parser.keys
179
+ assert_equal "tag", d.instance.extract_config.tag_key
180
+ assert_equal "time", d.instance.extract_config.time_key
181
+ assert_equal "%Y-%m-%d %H:%M:%S", d.instance.extract_config.time_format
182
+ end
183
+
184
+ test 'configure_with_json' do
185
+ d = create_driver JSON_CONFIG_COMPAT
186
+ assert{ d.instance.parser.is_a? Fluent::Plugin::JSONParser }
187
+ end
188
+
189
+ test 'configure_with_msgpack' do
190
+ d = create_driver MSGPACK_CONFIG_COMPAT
191
+ assert{ d.instance.parser.is_a? Fluent::Plugin::MessagePackParser }
192
+ end
193
+
194
+ test 'configure_with_regexp' do
195
+ d = create_driver REGEXP_CONFIG_COMPAT
196
+ assert{ d.instance.parser.is_a? Fluent::Plugin::RegexpParser }
197
+ assert_equal '(?<time>[^\]]*) (?<message>[^ ]*)', d.instance.parser.expression
198
+ assert_equal 'regex_tag', d.instance.tag
199
+ end
76
200
  end
77
201
 
78
- def test_configure_with_regexp
79
- d = create_driver regexp_config
80
- assert_equal '/(?<time>[^\]]*) (?<message>[^ ]*)/', d.instance.format
81
- assert_equal 'regex_tag', d.instance.tag
202
+ sub_test_case 'with default configuration' do
203
+ setup do
204
+ @current_event_time = event_time('2016-10-31 20:01:30.123 -0700')
205
+ Timecop.freeze(Time.at(@current_event_time))
206
+ end
207
+
208
+ teardown do
209
+ Timecop.return
210
+ end
211
+
212
+ test 'emits events with current timestamp if time key is not specified' do
213
+ d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS
214
+ d.run(expect_records: 2, timeout: 10)
215
+
216
+ assert{ d.events.length > 0 }
217
+ d.events.each do |event|
218
+ assert_equal ["my.test.data", @current_event_time, {"k1"=>"2011-01-02 13:14:15", "k2"=>"tag1", "k3"=>"ok"}], event
219
+ end
220
+ end
82
221
  end
83
222
 
84
- # TODO: Merge following tests into one case with parameters
85
-
86
- def test_emit
87
- d = create_driver
88
-
89
- d.run(expect_emits: 2)
90
-
91
- assert_equal true, d.events.length > 0
92
- d.events.each_with_index {|event, i|
93
- assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
94
- assert_equal_event_time(@test_time, event[1])
95
- }
96
- end
97
-
98
- def test_emit_json
99
- d = create_driver json_config
100
-
101
- d.run(expect_emits: 2)
102
-
103
- assert_equal true, d.events.length > 0
104
- d.events.each_with_index {|event, i|
105
- assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
106
- assert_equal_event_time(@test_time, event[1])
107
- }
108
- end
109
-
110
- def test_emit_msgpack
111
- d = create_driver msgpack_config
112
-
113
- d.run(expect_emits: 2)
114
-
115
- assert_equal true, d.events.length > 0
116
- d.events.each_with_index {|event, i|
117
- assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
118
- assert_equal_event_time(@test_time, event[1])
119
- }
120
- end
121
-
122
- def test_emit_regexp
123
- d = create_driver regexp_config
124
-
125
- d.run(expect_emits: 2)
126
-
127
- assert_equal true, d.events.length > 0
223
+ data(
224
+ 'default' => [TSV_CONFIG, "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
225
+ 'json' => [JSON_CONFIG, "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
226
+ 'msgpack' => [MSGPACK_CONFIG, "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
227
+ 'regexp' => [REGEXP_CONFIG, "regex_tag", event_time("2011-01-02 13:14:15"), {"message"=>"hello"}],
228
+ 'default_c' => [TSV_CONFIG_COMPAT, "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
229
+ 'json_c' => [JSON_CONFIG_COMPAT, "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
230
+ 'msgpack_c' => [MSGPACK_CONFIG_COMPAT, "tag1", event_time("2011-01-02 13:14:15"), {"k1"=>"ok"}],
231
+ 'regexp_c' => [REGEXP_CONFIG_COMPAT, "regex_tag", event_time("2011-01-02 13:14:15"), {"message"=>"hello"}],
232
+ )
233
+ test 'emit with formats' do |data|
234
+ config, tag, time, record = data
235
+ d = create_driver(config)
236
+
237
+ d.run(expect_emits: 2, timeout: 10)
238
+
239
+ assert{ d.events.length > 0 }
128
240
  d.events.each_with_index {|event, i|
129
- assert_equal ["regex_tag", @test_time, {"message"=>"hello"}], event
130
- assert_equal_event_time(@test_time, event[1])
241
+ assert_equal_event_time(time, event[1])
242
+ assert_equal [tag, time, record], event
131
243
  }
132
244
  end
133
245
  end