fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (159) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +16 -5
  3. data/ADOPTERS.md +5 -0
  4. data/{ChangeLog → CHANGELOG.md} +495 -6
  5. data/CONTRIBUTING.md +5 -2
  6. data/GOVERNANCE.md +55 -0
  7. data/LICENSE +202 -0
  8. data/MAINTAINERS.md +7 -5
  9. data/README.md +17 -10
  10. data/bin/fluent-ca-generate +6 -0
  11. data/example/counter.conf +18 -0
  12. data/example/secondary_file.conf +3 -2
  13. data/fluentd.gemspec +3 -3
  14. data/lib/fluent/agent.rb +1 -1
  15. data/lib/fluent/command/binlog_reader.rb +11 -2
  16. data/lib/fluent/command/ca_generate.rb +181 -0
  17. data/lib/fluent/command/cat.rb +28 -15
  18. data/lib/fluent/command/debug.rb +4 -4
  19. data/lib/fluent/command/fluentd.rb +2 -2
  20. data/lib/fluent/command/plugin_config_formatter.rb +24 -2
  21. data/lib/fluent/command/plugin_generator.rb +26 -8
  22. data/lib/fluent/config/configure_proxy.rb +7 -1
  23. data/lib/fluent/config/dsl.rb +8 -5
  24. data/lib/fluent/config/element.rb +5 -0
  25. data/lib/fluent/config/literal_parser.rb +7 -1
  26. data/lib/fluent/config/types.rb +28 -2
  27. data/lib/fluent/config/v1_parser.rb +1 -2
  28. data/lib/fluent/configurable.rb +1 -0
  29. data/lib/fluent/counter.rb +23 -0
  30. data/lib/fluent/counter/base_socket.rb +46 -0
  31. data/lib/fluent/counter/client.rb +297 -0
  32. data/lib/fluent/counter/error.rb +86 -0
  33. data/lib/fluent/counter/mutex_hash.rb +163 -0
  34. data/lib/fluent/counter/server.rb +273 -0
  35. data/lib/fluent/counter/store.rb +205 -0
  36. data/lib/fluent/counter/validator.rb +145 -0
  37. data/lib/fluent/env.rb +1 -0
  38. data/lib/fluent/event_router.rb +1 -1
  39. data/lib/fluent/log.rb +119 -29
  40. data/lib/fluent/plugin/base.rb +12 -0
  41. data/lib/fluent/plugin/buf_file.rb +20 -16
  42. data/lib/fluent/plugin/buffer.rb +130 -32
  43. data/lib/fluent/plugin/buffer/file_chunk.rb +23 -4
  44. data/lib/fluent/plugin/compressable.rb +1 -1
  45. data/lib/fluent/plugin/filter_grep.rb +135 -21
  46. data/lib/fluent/plugin/filter_parser.rb +13 -2
  47. data/lib/fluent/plugin/filter_record_transformer.rb +16 -14
  48. data/lib/fluent/plugin/formatter_stdout.rb +3 -2
  49. data/lib/fluent/plugin/formatter_tsv.rb +5 -1
  50. data/lib/fluent/plugin/in_debug_agent.rb +8 -1
  51. data/lib/fluent/plugin/in_forward.rb +1 -1
  52. data/lib/fluent/plugin/in_http.rb +84 -3
  53. data/lib/fluent/plugin/in_monitor_agent.rb +7 -1
  54. data/lib/fluent/plugin/in_syslog.rb +31 -10
  55. data/lib/fluent/plugin/in_tail.rb +142 -53
  56. data/lib/fluent/plugin/in_tcp.rb +5 -6
  57. data/lib/fluent/plugin/in_udp.rb +6 -2
  58. data/lib/fluent/plugin/in_unix.rb +1 -1
  59. data/lib/fluent/plugin/multi_output.rb +1 -0
  60. data/lib/fluent/plugin/out_copy.rb +25 -2
  61. data/lib/fluent/plugin/out_file.rb +26 -7
  62. data/lib/fluent/plugin/out_forward.rb +81 -42
  63. data/lib/fluent/plugin/out_secondary_file.rb +2 -2
  64. data/lib/fluent/plugin/out_stdout.rb +0 -1
  65. data/lib/fluent/plugin/out_stream.rb +1 -1
  66. data/lib/fluent/plugin/output.rb +221 -57
  67. data/lib/fluent/plugin/parser_apache.rb +1 -1
  68. data/lib/fluent/plugin/parser_apache2.rb +5 -1
  69. data/lib/fluent/plugin/parser_apache_error.rb +1 -1
  70. data/lib/fluent/plugin/parser_json.rb +10 -3
  71. data/lib/fluent/plugin/parser_ltsv.rb +7 -0
  72. data/lib/fluent/plugin/parser_multiline.rb +2 -1
  73. data/lib/fluent/plugin/parser_nginx.rb +1 -1
  74. data/lib/fluent/plugin/parser_none.rb +1 -0
  75. data/lib/fluent/plugin/parser_regexp.rb +15 -14
  76. data/lib/fluent/plugin/parser_syslog.rb +9 -5
  77. data/lib/fluent/plugin_helper.rb +2 -0
  78. data/lib/fluent/plugin_helper/cert_option.rb +28 -9
  79. data/lib/fluent/plugin_helper/compat_parameters.rb +3 -1
  80. data/lib/fluent/plugin_helper/counter.rb +51 -0
  81. data/lib/fluent/plugin_helper/event_loop.rb +9 -0
  82. data/lib/fluent/plugin_helper/record_accessor.rb +210 -0
  83. data/lib/fluent/plugin_helper/retry_state.rb +15 -7
  84. data/lib/fluent/plugin_helper/server.rb +87 -25
  85. data/lib/fluent/plugin_helper/socket_option.rb +5 -2
  86. data/lib/fluent/plugin_helper/timer.rb +8 -7
  87. data/lib/fluent/root_agent.rb +18 -9
  88. data/lib/fluent/supervisor.rb +63 -23
  89. data/lib/fluent/system_config.rb +30 -2
  90. data/lib/fluent/test/helpers.rb +1 -1
  91. data/lib/fluent/time.rb +15 -7
  92. data/lib/fluent/timezone.rb +26 -2
  93. data/lib/fluent/version.rb +1 -1
  94. data/templates/new_gem/README.md.erb +2 -2
  95. data/templates/new_gem/lib/fluent/plugin/filter.rb.erb +1 -1
  96. data/templates/new_gem/lib/fluent/plugin/input.rb.erb +1 -1
  97. data/templates/new_gem/lib/fluent/plugin/output.rb.erb +1 -1
  98. data/templates/new_gem/lib/fluent/plugin/parser.rb.erb +4 -4
  99. data/test/command/test_ca_generate.rb +70 -0
  100. data/test/command/test_fluentd.rb +2 -2
  101. data/test/command/test_plugin_config_formatter.rb +8 -7
  102. data/test/command/test_plugin_generator.rb +65 -39
  103. data/test/config/test_config_parser.rb +7 -2
  104. data/test/config/test_configurable.rb +7 -2
  105. data/test/config/test_configure_proxy.rb +41 -3
  106. data/test/config/test_dsl.rb +10 -10
  107. data/test/config/test_element.rb +10 -0
  108. data/test/config/test_literal_parser.rb +8 -0
  109. data/test/config/test_plugin_configuration.rb +56 -0
  110. data/test/config/test_system_config.rb +19 -1
  111. data/test/config/test_types.rb +37 -0
  112. data/test/counter/test_client.rb +559 -0
  113. data/test/counter/test_error.rb +44 -0
  114. data/test/counter/test_mutex_hash.rb +179 -0
  115. data/test/counter/test_server.rb +589 -0
  116. data/test/counter/test_store.rb +258 -0
  117. data/test/counter/test_validator.rb +137 -0
  118. data/test/plugin/test_buf_file.rb +124 -0
  119. data/test/plugin/test_buffer.rb +3 -2
  120. data/test/plugin/test_filter_grep.rb +580 -2
  121. data/test/plugin/test_filter_parser.rb +33 -2
  122. data/test/plugin/test_filter_record_transformer.rb +22 -1
  123. data/test/plugin/test_formatter_ltsv.rb +3 -0
  124. data/test/plugin/test_formatter_tsv.rb +68 -0
  125. data/test/plugin/test_in_debug_agent.rb +21 -0
  126. data/test/plugin/test_in_exec.rb +3 -5
  127. data/test/plugin/test_in_http.rb +178 -0
  128. data/test/plugin/test_in_monitor_agent.rb +1 -1
  129. data/test/plugin/test_in_syslog.rb +64 -0
  130. data/test/plugin/test_in_tail.rb +116 -6
  131. data/test/plugin/test_in_tcp.rb +21 -0
  132. data/test/plugin/test_in_udp.rb +78 -0
  133. data/test/plugin/test_metadata.rb +89 -0
  134. data/test/plugin/test_out_copy.rb +31 -0
  135. data/test/plugin/test_out_file.rb +108 -2
  136. data/test/plugin/test_out_forward.rb +195 -2
  137. data/test/plugin/test_out_secondary_file.rb +14 -0
  138. data/test/plugin/test_output.rb +159 -45
  139. data/test/plugin/test_output_as_buffered.rb +19 -0
  140. data/test/plugin/test_output_as_buffered_backup.rb +307 -0
  141. data/test/plugin/test_output_as_buffered_retries.rb +70 -0
  142. data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
  143. data/test/plugin/test_parser_apache2.rb +1 -0
  144. data/test/plugin/test_parser_labeled_tsv.rb +17 -0
  145. data/test/plugin/test_parser_nginx.rb +40 -0
  146. data/test/plugin/test_parser_regexp.rb +6 -7
  147. data/test/plugin/test_parser_syslog.rb +155 -5
  148. data/test/plugin_helper/test_child_process.rb +4 -4
  149. data/test/plugin_helper/test_compat_parameters.rb +22 -0
  150. data/test/plugin_helper/test_record_accessor.rb +197 -0
  151. data/test/plugin_helper/test_retry_state.rb +20 -0
  152. data/test/plugin_helper/test_server.rb +30 -2
  153. data/test/test_config.rb +3 -3
  154. data/test/test_configdsl.rb +2 -2
  155. data/test/test_log.rb +51 -1
  156. data/test/test_root_agent.rb +33 -0
  157. data/test/test_supervisor.rb +105 -0
  158. metadata +68 -8
  159. data/COPYING +0 -14
@@ -455,7 +455,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
455
455
  d.instance.start
456
456
  expected_test_out_fail_write_response = {
457
457
  "buffer_queue_length" => 1,
458
- "buffer_total_queued_size" => 0,
458
+ "buffer_total_queued_size" => 40,
459
459
  "output_plugin" => true,
460
460
  "plugin_category" => "output",
461
461
  "plugin_id" => "test_out_fail_write",
@@ -37,6 +37,24 @@ class SyslogInputTest < Test::Unit::TestCase
37
37
  assert_equal bind_addr, d.instance.bind
38
38
  end
39
39
 
40
+ sub_test_case 'source_hostname_key and source_address_key features' do
41
+ test 'resolve_hostname must be true with source_hostname_key' do
42
+ assert_raise(Fluent::ConfigError) {
43
+ create_driver(CONFIG + <<EOS)
44
+ resolve_hostname false
45
+ source_hostname_key hostname
46
+ EOS
47
+ }
48
+ end
49
+
50
+ data('resolve_hostname' => 'resolve_hostname true',
51
+ 'source_hostname_key' => 'source_hostname_key source_host')
52
+ def test_configure_resolve_hostname(param)
53
+ d = create_driver([CONFIG, param].join("\n"))
54
+ assert_true d.instance.resolve_hostname
55
+ end
56
+ end
57
+
40
58
  data(
41
59
  ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
42
60
  ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
@@ -268,4 +286,50 @@ class SyslogInputTest < Test::Unit::TestCase
268
286
  assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
269
287
  }
270
288
  end
289
+
290
+ sub_test_case 'octet counting frame' do
291
+ def test_msg_size_with_tcp
292
+ d = create_driver([CONFIG, 'protocol_type tcp', 'frame_type octet_count'].join("\n"))
293
+ tests = create_test_case
294
+
295
+ d.run(expect_emits: 2) do
296
+ tests.each {|test|
297
+ TCPSocket.open('127.0.0.1', PORT) do |s|
298
+ s.send(test['msg'], 0)
299
+ end
300
+ }
301
+ end
302
+
303
+ assert(d.events.size > 0)
304
+ compare_test_result(d.events, tests)
305
+ end
306
+
307
+ def test_msg_size_with_same_tcp_connection
308
+ d = create_driver([CONFIG, 'protocol_type tcp', 'frame_type octet_count'].join("\n"))
309
+ tests = create_test_case
310
+
311
+ d.run(expect_emits: 2) do
312
+ TCPSocket.open('127.0.0.1', PORT) do |s|
313
+ tests.each {|test|
314
+ s.send(test['msg'], 0)
315
+ }
316
+ end
317
+ end
318
+
319
+ assert(d.events.size > 0)
320
+ compare_test_result(d.events, tests)
321
+ end
322
+
323
+ def create_test_case(large_message: false)
324
+ msgs = [
325
+ {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100, 'expected' => 'x' * 100},
326
+ {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024, 'expected' => 'x' * 1024},
327
+ ]
328
+ msgs.each { |msg|
329
+ m = msg['msg']
330
+ msg['msg'] = "#{m.size + 1} #{m}"
331
+ }
332
+ msgs
333
+ end
334
+ end
271
335
  end
@@ -41,6 +41,7 @@ class TailInputTest < Test::Unit::TestCase
41
41
  COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
42
42
  CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
43
43
  CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
44
+ CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
44
45
  CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
45
46
  SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
46
47
  PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
@@ -83,6 +84,12 @@ class TailInputTest < Test::Unit::TestCase
83
84
  end
84
85
  end
85
86
 
87
+ test "both enable_watch_timer and enable_stat_watcher are false" do
88
+ assert_raise(Fluent::ConfigError) do
89
+ create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
90
+ end
91
+ end
92
+
86
93
  sub_test_case "encoding" do
87
94
  test "valid" do
88
95
  conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" })
@@ -256,6 +263,30 @@ class TailInputTest < Test::Unit::TestCase
256
263
  assert_equal({"message" => "test3"}, events[0][2])
257
264
  assert_equal({"message" => "test4"}, events[1][2])
258
265
  end
266
+
267
+ data(flat: CONFIG_DISABLE_STAT_WATCHER + SINGLE_LINE_CONFIG,
268
+ parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
269
+ def test_emit_with_disable_stat_watcher(data)
270
+ config = data
271
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
272
+ f.puts "test1"
273
+ f.puts "test2"
274
+ }
275
+
276
+ d = create_driver(config)
277
+
278
+ d.run(expect_emits: 1) do
279
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
280
+ f.puts "test3"
281
+ f.puts "test4"
282
+ }
283
+ end
284
+
285
+ events = d.events
286
+ assert(events.length > 0)
287
+ assert_equal({"message" => "test3"}, events[0][2])
288
+ assert_equal({"message" => "test4"}, events[1][2])
289
+ end
259
290
  end
260
291
 
261
292
  class TestWithSystem < self
@@ -440,6 +471,71 @@ class TailInputTest < Test::Unit::TestCase
440
471
  end
441
472
  end
442
473
 
474
+ def test_truncate_file
475
+ omit "Permission denied error happen on Windows. Need fix" if Fluent.windows?
476
+
477
+ config = SINGLE_LINE_CONFIG
478
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
479
+ f.puts "test1"
480
+ f.puts "test2"
481
+ }
482
+
483
+ d = create_driver(config)
484
+
485
+ d.run(expect_emits: 2) do
486
+ File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
487
+ f.puts "test3\ntest4"
488
+ f.flush
489
+ }
490
+ sleep 1
491
+ File.truncate("#{TMP_DIR}/tail.txt", 6)
492
+ end
493
+
494
+ events = d.events
495
+ assert_equal(3, events.length)
496
+ assert_equal({"message" => "test3"}, events[0][2])
497
+ assert_equal({"message" => "test4"}, events[1][2])
498
+ assert_equal({"message" => "test1"}, events[2][2])
499
+ assert(events[0][1].is_a?(Fluent::EventTime))
500
+ assert(events[1][1].is_a?(Fluent::EventTime))
501
+ assert(events[2][1].is_a?(Fluent::EventTime))
502
+ assert_equal(2, d.emit_count)
503
+ end
504
+
505
+ def test_move_truncate_move_back
506
+ omit "Permission denied error happen on Windows. Need fix" if Fluent.windows?
507
+
508
+ config = SINGLE_LINE_CONFIG
509
+ File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
510
+ f.puts "test1"
511
+ f.puts "test2"
512
+ }
513
+
514
+ d = create_driver(config)
515
+
516
+ d.run(expect_emits: 1) do
517
+ if Fluent.windows?
518
+ FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt", force: true)
519
+ else
520
+ FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
521
+ end
522
+ sleep(1)
523
+ File.truncate("#{TMP_DIR}/tail2.txt", 6)
524
+ sleep(1)
525
+ if Fluent.windows?
526
+ FileUtils.mv("#{TMP_DIR}/tail2.txt", "#{TMP_DIR}/tail.txt", force: true)
527
+ else
528
+ FileUtils.mv("#{TMP_DIR}/tail2.txt", "#{TMP_DIR}/tail.txt")
529
+ end
530
+ end
531
+
532
+ events = d.events
533
+ assert_equal(1, events.length)
534
+ assert_equal({"message" => "test1"}, events[0][2])
535
+ assert(events[0][1].is_a?(Fluent::EventTime))
536
+ assert_equal(1, d.emit_count)
537
+ end
538
+
443
539
  def test_lf
444
540
  File.open("#{TMP_DIR}/tail.txt", "wb") {|f| }
445
541
 
@@ -823,7 +919,7 @@ class TailInputTest < Test::Unit::TestCase
823
919
  sub_test_case "path" do
824
920
  # * path test
825
921
  # TODO: Clean up tests
826
- EX_RORATE_WAIT = 0
922
+ EX_ROTATE_WAIT = 0
827
923
 
828
924
  EX_CONFIG = config_element("", "", {
829
925
  "tag" => "tail",
@@ -832,7 +928,7 @@ class TailInputTest < Test::Unit::TestCase
832
928
  "pos_file" => "#{TMP_DIR}/tail.pos",
833
929
  "read_from_head" => true,
834
930
  "refresh_interval" => 30,
835
- "rotate_wait" => "#{EX_RORATE_WAIT}s",
931
+ "rotate_wait" => "#{EX_ROTATE_WAIT}s",
836
932
  })
837
933
  EX_PATHS = [
838
934
  'test/plugin/data/2010/01/20100102-030405.log',
@@ -873,7 +969,7 @@ class TailInputTest < Test::Unit::TestCase
873
969
  end
874
970
 
875
971
  # For https://github.com/fluent/fluentd/issues/1455
876
- # This test is fragile because test content depends on internal implementaion.
972
+ # This test is fragile because test content depends on internal implementation.
877
973
  # So if you modify in_tail internal, this test may break.
878
974
  def test_unwatched_files_should_be_removed
879
975
  config = config_element("", "", {
@@ -894,7 +990,7 @@ class TailInputTest < Test::Unit::TestCase
894
990
  waiting(20) { sleep 0.1 until Dir.glob("#{TMP_DIR}/*.txt").size == 0 } # Ensure file is deleted on Windows
895
991
  waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 0 }
896
992
 
897
- # Previous implementaion has an infinite watcher creation bug.
993
+ # Previous implementation has an infinite watcher creation bug.
898
994
  # Following code checks such unexpected bug by couting actual object allocation.
899
995
  base_num = count_timer_object
900
996
  2.times {
@@ -926,7 +1022,7 @@ class TailInputTest < Test::Unit::TestCase
926
1022
  Timecop.freeze(2010, 1, 2, 3, 4, 5) do
927
1023
  flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
928
1024
  EX_PATHS.each do |path|
929
- watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
1025
+ watcherclass.should_receive(:new).with(path, EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, true, 1000, any, any, any, any, any, any).once.and_return do
930
1026
  flexmock('TailWatcher') { |watcher|
931
1027
  watcher.should_receive(:attach).once
932
1028
  watcher.should_receive(:unwatched=).zero_or_more_times
@@ -944,7 +1040,7 @@ class TailInputTest < Test::Unit::TestCase
944
1040
 
945
1041
  Timecop.freeze(2010, 1, 2, 3, 4, 6) do
946
1042
  flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
947
- watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
1043
+ watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, true, 1000, any, any, any, any, any, any).once.and_return do
948
1044
  flexmock('TailWatcher') do |watcher|
949
1045
  watcher.should_receive(:attach).once
950
1046
  watcher.should_receive(:unwatched=).zero_or_more_times
@@ -972,6 +1068,20 @@ class TailInputTest < Test::Unit::TestCase
972
1068
  plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
973
1069
  end
974
1070
 
1071
+ def test_tag_with_only_star
1072
+ config = config_element("", "", {
1073
+ "tag" => "*",
1074
+ "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
1075
+ "format" => "none",
1076
+ "read_from_head" => true
1077
+ })
1078
+ d = create_driver(config, false)
1079
+ d.run {}
1080
+ plugin = d.instance
1081
+ mock(plugin.router).emit_stream('foo.bar.log', anything).once
1082
+ plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
1083
+ end
1084
+
975
1085
  def test_tag_prefix
976
1086
  config = config_element("", "", {
977
1087
  "tag" => "pre.*",
@@ -115,4 +115,25 @@ class TcpInputTest < Test::Unit::TestCase
115
115
  assert_equal expected_record, d.events[i][2]
116
116
  end
117
117
  end
118
+
119
+ test 'source_hostname_key' do
120
+ d = create_driver(BASE_CONFIG + %!
121
+ format none
122
+ source_hostname_key host
123
+ !)
124
+ hostname = nil
125
+ d.run(expect_records: 1) do
126
+ create_tcp_socket('127.0.0.1', PORT) do |sock|
127
+ sock.do_not_reverse_lookup = false
128
+ hostname = sock.peeraddr[2]
129
+ sock.send("test\n", 0)
130
+ end
131
+ end
132
+
133
+ assert_equal 1, d.events.size
134
+ event = d.events[0]
135
+ assert_equal "tcp", event[0]
136
+ assert event[1].is_a?(Fluent::EventTime)
137
+ assert_equal hostname, event[2]['host']
138
+ end
118
139
  end
@@ -31,6 +31,7 @@ class UdpInputTest < Test::Unit::TestCase
31
31
  else
32
32
  UDPSocket.new(Socket::AF_INET6)
33
33
  end
34
+ u.do_not_reverse_lookup = false
34
35
  u.connect(host, port)
35
36
  if block_given?
36
37
  begin
@@ -55,6 +56,7 @@ class UdpInputTest < Test::Unit::TestCase
55
56
  assert_equal PORT, d.instance.port
56
57
  assert_equal bind, d.instance.bind
57
58
  assert_equal 4096, d.instance.message_length_limit
59
+ assert_equal nil, d.instance.receive_buffer_size
58
60
  end
59
61
 
60
62
  data(
@@ -149,4 +151,80 @@ class UdpInputTest < Test::Unit::TestCase
149
151
  assert_equal expected_record, d.events[i][2]
150
152
  end
151
153
  end
154
+
155
+ test 'remove_newline' do
156
+ d = create_driver(BASE_CONFIG + %!
157
+ format none
158
+ remove_newline false
159
+ !)
160
+ payloads = ["test1\n", "test2\n"]
161
+ d.run(expect_records: 2) do
162
+ create_udp_socket('127.0.0.1', PORT) do |u|
163
+ payloads.each do |payload|
164
+ u.send(payload, 0)
165
+ end
166
+ end
167
+ end
168
+
169
+ expecteds = payloads.map { |payload| {'message' => payload} }
170
+ assert_equal 2, d.events.size
171
+ expecteds.each_with_index do |expected_record, i|
172
+ assert_equal "udp", d.events[i][0]
173
+ assert d.events[i][1].is_a?(Fluent::EventTime)
174
+ assert_equal expected_record, d.events[i][2]
175
+ end
176
+ end
177
+
178
+ test 'source_hostname_key' do
179
+ d = create_driver(BASE_CONFIG + %!
180
+ format none
181
+ source_hostname_key host
182
+ !)
183
+ hostname = nil
184
+ d.run(expect_records: 1) do
185
+ create_udp_socket('127.0.0.1', PORT) do |u|
186
+ u.send("test", 0)
187
+ hostname = u.peeraddr[2]
188
+ end
189
+ end
190
+
191
+ expected = {'message' => 'test'}
192
+ assert_equal 1, d.events.size
193
+ assert_equal "udp", d.events[0][0]
194
+ assert d.events[0][1].is_a?(Fluent::EventTime)
195
+ assert_equal hostname, d.events[0][2]['host']
196
+ end
197
+
198
+ test 'receive_buffer_size' do
199
+ # doesn't check exact value because it depends on platform and condition
200
+
201
+ # check if default socket and in_udp's one without receive_buffer_size have same size buffer
202
+ d0 = create_driver(BASE_CONFIG + %!
203
+ format none
204
+ !)
205
+ d0.run do
206
+ sock = d0.instance.instance_variable_get(:@_servers)[0].server.instance_variable_get(:@sock)
207
+ begin
208
+ default_sock = UDPSocket.new
209
+ assert_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
210
+ ensure
211
+ default_sock.close
212
+ end
213
+ end
214
+
215
+ # check if default socket and in_udp's one with receive_buffer_size have different size buffer
216
+ d1 = create_driver(BASE_CONFIG + %!
217
+ format none
218
+ receive_buffer_size 1001
219
+ !)
220
+ d1.run do
221
+ sock = d1.instance.instance_variable_get(:@_servers)[0].server.instance_variable_get(:@sock)
222
+ begin
223
+ default_sock = UDPSocket.new
224
+ assert_not_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
225
+ ensure
226
+ default_sock.close
227
+ end
228
+ end
229
+ end
152
230
  end
@@ -0,0 +1,89 @@
1
+ require_relative '../helper'
2
+ require 'fluent/plugin/buffer'
3
+
4
+ class BufferMetadataTest < Test::Unit::TestCase
5
+
6
+ def meta(timekey=nil, tag=nil, variables=nil)
7
+ Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
8
+ end
9
+
10
+ setup do
11
+ Fluent::Test.setup
12
+ end
13
+
14
+ sub_test_case 'about metadata' do
15
+ test 'comparison of variables should be stable' do
16
+ m = meta(nil, nil, nil)
17
+ # different sets of keys
18
+ assert_equal(-1, m.cmp_variables({}, {a: 1}))
19
+ assert_equal(1, m.cmp_variables({a: 1}, {}))
20
+ assert_equal(1, m.cmp_variables({c: 1}, {a: 1}))
21
+ assert_equal(-1, m.cmp_variables({a: 1}, {a: 1, b: 2}))
22
+ assert_equal(1, m.cmp_variables({a: 1, c: 1}, {a: 1, b: 2}))
23
+ assert_equal(1, m.cmp_variables({a: 1, b: 0, c: 1}, {a: 1, b: 2}))
24
+ # same set of keys
25
+ assert_equal(-1, m.cmp_variables({a: 1}, {a: 2}))
26
+ assert_equal(-1, m.cmp_variables({a: 1, b: 0}, {a: 1, b: 1}))
27
+ assert_equal(-1, m.cmp_variables({a: 1, b: 1, c: 100}, {a: 1, b: 1, c: 200}))
28
+ assert_equal(-1, m.cmp_variables({b: 1, c: 100, a: 1}, {a: 1, b: 1, c: 200})) # comparison sorts keys
29
+ assert_equal(-1, m.cmp_variables({a: nil}, {a: 1}))
30
+ assert_equal(-1, m.cmp_variables({a: 1, b: nil}, {a: 1, b: 1}))
31
+ end
32
+
33
+ test 'comparison of metadata should be stable' do
34
+ n = Time.now.to_i
35
+
36
+ assert_equal(0, meta(nil, nil, nil) <=> meta(nil, nil, nil))
37
+ assert_equal(0, meta(n, nil, nil) <=> meta(n, nil, nil))
38
+ assert_equal(0, meta(nil, "t1", nil) <=> meta(nil, "t1", nil))
39
+ assert_equal(0, meta(nil, nil, {}) <=> meta(nil, nil, {}))
40
+ assert_equal(0, meta(nil, nil, {a: "1"}) <=> meta(nil, nil, {a: "1"}))
41
+ assert_equal(0, meta(n, nil, {}) <=> meta(n, nil, {}))
42
+ assert_equal(0, meta(n, "t1", {}) <=> meta(n, "t1", {}))
43
+ assert_equal(0, meta(n, "t1", {a: "x", b: 10}) <=> meta(n, "t1", {a: "x", b: 10}))
44
+
45
+ # timekey is 1st comparison key
46
+ assert_equal(-1, meta(n - 300, nil, nil) <=> meta(n - 270, nil, nil))
47
+ assert_equal(1, meta(n + 1, "a", nil) <=> meta(n - 1, "b", nil))
48
+ assert_equal(-1, meta(n - 1, nil, {a: 100}) <=> meta(n + 1, nil, {}))
49
+
50
+ # tag is 2nd
51
+ assert_equal(-1, meta(nil, "a", {}) <=> meta(nil, "b", {}))
52
+ assert_equal(-1, meta(n, "a", {}) <=> meta(n, "b", {}))
53
+ assert_equal(1, meta(nil, "x", {a: 1}) <=> meta(nil, "t", {}))
54
+ assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {}))
55
+ assert_equal(1, meta(nil, "x", {a: 1}) <=> meta(nil, "t", {a: 1}))
56
+ assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {a: 2}))
57
+ assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {a: 10, b: 1}))
58
+
59
+ # variables is the last
60
+ assert_equal(-1, meta(nil, nil, {}) <=> meta(nil, nil, {a: 1}))
61
+ assert_equal(-1, meta(n, "t", {}) <=> meta(n, "t", {a: 1}))
62
+ assert_equal(1, meta(n, "t", {a: 1}) <=> meta(n, "t", {}))
63
+ assert_equal(-1, meta(n, "t", {a: 1}) <=> meta(n, "t", {a: 2}))
64
+ assert_equal(-1, meta(n, "t", {a: 1}) <=> meta(n, "t", {a: 1, b: 1}))
65
+ assert_equal(1, meta(nil, nil, {b: 1}) <=> meta(nil, nil, {a: 1}))
66
+ assert_equal(1, meta(n, "t", {b: 1}) <=> meta(n, "t", {a: 1}))
67
+ end
68
+
69
+ test 'metadata can be sorted' do
70
+ n = Time.now.to_i
71
+ m0 = meta(nil, nil, nil)
72
+ m1 = meta(n - 1, nil, nil)
73
+ m2 = meta(n - 1, "a", nil)
74
+ m3 = meta(n - 1, "a", {a: 1})
75
+ m4 = meta(n - 1, "a", {a: 100})
76
+ m5 = meta(n - 1, "a", {a: 100, b: 1})
77
+ m6 = meta(n - 1, "aa", nil)
78
+ m7 = meta(n - 1, "aa", {a: 1})
79
+ m8 = meta(n - 1, "b", nil)
80
+ m9 = meta(n, nil, nil)
81
+ m10 = meta(n + 1, nil, {a: 1})
82
+ expected = [m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10].freeze
83
+ ary = expected.dup
84
+ 100.times do
85
+ assert_equal expected, ary.shuffle.sort
86
+ end
87
+ end
88
+ end
89
+ end