fluentd 1.6.3 → 1.7.1

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (84) hide show
  1. checksums.yaml +4 -4
  2. data/.drone.yml +35 -0
  3. data/.github/ISSUE_TEMPLATE/bug_report.md +2 -0
  4. data/CHANGELOG.md +83 -0
  5. data/README.md +5 -1
  6. data/fluentd.gemspec +3 -2
  7. data/lib/fluent/clock.rb +4 -0
  8. data/lib/fluent/compat/output.rb +3 -3
  9. data/lib/fluent/compat/socket_util.rb +1 -1
  10. data/lib/fluent/config/element.rb +3 -3
  11. data/lib/fluent/config/literal_parser.rb +1 -1
  12. data/lib/fluent/config/section.rb +4 -1
  13. data/lib/fluent/error.rb +4 -0
  14. data/lib/fluent/event.rb +28 -24
  15. data/lib/fluent/event_router.rb +2 -1
  16. data/lib/fluent/log.rb +1 -1
  17. data/lib/fluent/msgpack_factory.rb +8 -0
  18. data/lib/fluent/plugin/bare_output.rb +4 -4
  19. data/lib/fluent/plugin/buf_file.rb +10 -1
  20. data/lib/fluent/plugin/buf_file_single.rb +219 -0
  21. data/lib/fluent/plugin/buffer.rb +62 -63
  22. data/lib/fluent/plugin/buffer/chunk.rb +21 -3
  23. data/lib/fluent/plugin/buffer/file_chunk.rb +44 -12
  24. data/lib/fluent/plugin/buffer/file_single_chunk.rb +314 -0
  25. data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -1
  26. data/lib/fluent/plugin/compressable.rb +10 -6
  27. data/lib/fluent/plugin/filter_grep.rb +2 -2
  28. data/lib/fluent/plugin/formatter_csv.rb +10 -6
  29. data/lib/fluent/plugin/in_syslog.rb +10 -3
  30. data/lib/fluent/plugin/in_tail.rb +7 -2
  31. data/lib/fluent/plugin/in_tcp.rb +34 -7
  32. data/lib/fluent/plugin/multi_output.rb +4 -4
  33. data/lib/fluent/plugin/out_exec_filter.rb +1 -0
  34. data/lib/fluent/plugin/out_file.rb +13 -3
  35. data/lib/fluent/plugin/out_forward.rb +144 -588
  36. data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
  37. data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
  38. data/lib/fluent/plugin/out_forward/error.rb +28 -0
  39. data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
  40. data/lib/fluent/plugin/out_forward/handshake_protocol.rb +121 -0
  41. data/lib/fluent/plugin/out_forward/load_balancer.rb +111 -0
  42. data/lib/fluent/plugin/out_forward/socket_cache.rb +138 -0
  43. data/lib/fluent/plugin/out_http.rb +231 -0
  44. data/lib/fluent/plugin/output.rb +29 -35
  45. data/lib/fluent/plugin/parser.rb +77 -0
  46. data/lib/fluent/plugin/parser_csv.rb +75 -0
  47. data/lib/fluent/plugin/parser_syslog.rb +106 -3
  48. data/lib/fluent/plugin_helper/server.rb +2 -2
  49. data/lib/fluent/plugin_helper/socket.rb +14 -1
  50. data/lib/fluent/plugin_helper/thread.rb +1 -0
  51. data/lib/fluent/root_agent.rb +1 -1
  52. data/lib/fluent/time.rb +4 -2
  53. data/lib/fluent/timezone.rb +21 -7
  54. data/lib/fluent/version.rb +1 -1
  55. data/test/command/test_fluentd.rb +1 -1
  56. data/test/command/test_plugin_generator.rb +18 -2
  57. data/test/config/test_configurable.rb +78 -40
  58. data/test/counter/test_store.rb +1 -1
  59. data/test/helper.rb +1 -0
  60. data/test/helpers/process_extenstion.rb +33 -0
  61. data/test/plugin/out_forward/test_ack_handler.rb +101 -0
  62. data/test/plugin/out_forward/test_connection_manager.rb +145 -0
  63. data/test/plugin/out_forward/test_handshake_protocol.rb +103 -0
  64. data/test/plugin/out_forward/test_load_balancer.rb +60 -0
  65. data/test/plugin/out_forward/test_socket_cache.rb +139 -0
  66. data/test/plugin/test_buf_file.rb +172 -2
  67. data/test/plugin/test_buf_file_single.rb +801 -0
  68. data/test/plugin/test_buffer.rb +4 -48
  69. data/test/plugin/test_buffer_file_chunk.rb +38 -1
  70. data/test/plugin/test_buffer_file_single_chunk.rb +621 -0
  71. data/test/plugin/test_buffer_memory_chunk.rb +1 -0
  72. data/test/plugin/test_formatter_csv.rb +16 -0
  73. data/test/plugin/test_in_syslog.rb +56 -6
  74. data/test/plugin/test_in_tail.rb +1 -1
  75. data/test/plugin/test_in_tcp.rb +25 -0
  76. data/test/plugin/test_out_forward.rb +150 -201
  77. data/test/plugin/test_out_http.rb +352 -0
  78. data/test/plugin/test_output_as_buffered.rb +27 -24
  79. data/test/plugin/test_parser.rb +40 -0
  80. data/test/plugin/test_parser_csv.rb +83 -0
  81. data/test/plugin/test_parser_syslog.rb +118 -19
  82. data/test/plugin_helper/test_record_accessor.rb +1 -1
  83. data/test/test_time_formatter.rb +140 -121
  84. metadata +35 -6
@@ -331,6 +331,7 @@ class BufferMemoryChunkTest < Test::Unit::TestCase
331
331
  assert_equal @gzipped_src, c.read(compressed: :gzip)
332
332
 
333
333
  io = StringIO.new
334
+ io.set_encoding(Encoding::ASCII_8BIT)
334
335
  c.write_to(io, compressed: :gzip)
335
336
  assert_equal @gzipped_src, io.string
336
337
  end
@@ -108,4 +108,20 @@ class CsvFormatterTest < ::Test::Unit::TestCase
108
108
  d = create_driver('fields' => data)
109
109
  assert_equal %w(one two three), d.instance.fields
110
110
  end
111
+
112
+ def test_format_with_multiple_records
113
+ d = create_driver("fields" => "message,message2")
114
+ r = {'message' => 'hello', 'message2' => 'fluentd'}
115
+
116
+ formatted = d.instance.format(tag, @time, r)
117
+ assert_equal("\"hello\",\"fluentd\"\n", formatted)
118
+
119
+ r = {'message' => 'hey', 'message2' => 'ho'}
120
+ formatted = d.instance.format(tag, @time, r)
121
+ assert_equal("\"hey\",\"ho\"\n", formatted)
122
+
123
+ r = {'message' => 'longer message', 'message2' => 'longer longer message'}
124
+ formatted = d.instance.format(tag, @time, r)
125
+ assert_equal("\"longer message\",\"longer longer message\"\n", formatted)
126
+ end
111
127
  end
@@ -362,9 +362,8 @@ EOS
362
362
  end
363
363
  end
364
364
 
365
- def test_emit_unmatched_lines
366
- d = create_driver([CONFIG, 'emit_unmatched_lines true'].join("\n"))
367
- tests = [
365
+ def create_unmatched_lines_test_case
366
+ [
368
367
  # valid message
369
368
  {'msg' => '<6>Sep 10 00:00:00 localhost logger: xxx', 'expected' => {'host'=>'localhost', 'ident'=>'logger', 'message'=>'xxx'}},
370
369
  # missing priority
@@ -372,6 +371,22 @@ EOS
372
371
  # timestamp parsing failure
373
372
  {'msg' => '<6>ZZZ 99 99:99:99 localhost logger: xxx', 'expected' => {'unmatched_line' => '<6>ZZZ 99 99:99:99 localhost logger: xxx'}},
374
373
  ]
374
+ end
375
+
376
+ def compare_unmatched_lines_test_result(events, tests, options = {})
377
+ events.each_index { |i|
378
+ tests[i]['expected'].each { |k,v|
379
+ assert_equal v, events[i][2][k], "No key <#{k}> in response or value mismatch"
380
+ }
381
+ assert_equal('syslog.unmatched', events[i][0], 'tag does not match syslog.unmatched') unless i==0
382
+ assert_equal(options[:address], events[i][2]['source_address'], 'response has no source_address or mismatch') if options[:address]
383
+ assert_equal(options[:hostname], events[i][2]['source_hostname'], 'response has no source_hostname or mismatch') if options[:hostname]
384
+ }
385
+ end
386
+
387
+ def test_emit_unmatched_lines
388
+ d = create_driver([CONFIG, 'emit_unmatched_lines true'].join("\n"))
389
+ tests = create_unmatched_lines_test_case
375
390
 
376
391
  d.run(expect_emits: 3) do
377
392
  u = UDPSocket.new
@@ -383,9 +398,44 @@ EOS
383
398
  end
384
399
 
385
400
  assert_equal tests.size, d.events.size
386
- tests.size.times do |i|
387
- assert_equal tests[i]['expected'], d.events[i][2]
388
- assert_equal 'syslog.unmatched', d.events[i][0] unless i==0
401
+ compare_unmatched_lines_test_result(d.events, tests)
402
+ end
403
+
404
+ def test_emit_unmatched_lines_with_hostname
405
+ d = create_driver([CONFIG, 'emit_unmatched_lines true', 'source_hostname_key source_hostname'].join("\n"))
406
+ tests = create_unmatched_lines_test_case
407
+
408
+ hostname = nil
409
+ d.run(expect_emits: 3) do
410
+ u = UDPSocket.new
411
+ u.do_not_reverse_lookup = false
412
+ u.connect('127.0.0.1', PORT)
413
+ hostname = u.peeraddr[2]
414
+ tests.each {|test|
415
+ u.send(test['msg'], 0)
416
+ }
417
+ end
418
+
419
+ assert_equal tests.size, d.events.size
420
+ compare_unmatched_lines_test_result(d.events, tests, {hostname: hostname})
421
+ end
422
+
423
+ def test_emit_unmatched_lines_with_address
424
+ d = create_driver([CONFIG, 'emit_unmatched_lines true', 'source_address_key source_address'].join("\n"))
425
+ tests = create_unmatched_lines_test_case
426
+
427
+ address = nil
428
+ d.run(expect_emits: 3) do
429
+ u = UDPSocket.new
430
+ u.do_not_reverse_lookup = false
431
+ u.connect('127.0.0.1', PORT)
432
+ address = u.peeraddr[3]
433
+ tests.each {|test|
434
+ u.send(test['msg'], 0)
435
+ }
389
436
  end
437
+
438
+ assert_equal tests.size, d.events.size
439
+ compare_unmatched_lines_test_result(d.events, tests, {address: address})
390
440
  end
391
441
  end
@@ -1011,7 +1011,7 @@ class TailInputTest < Test::Unit::TestCase
1011
1011
  waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 0 }
1012
1012
 
1013
1013
  # Previous implementation has an infinite watcher creation bug.
1014
- # Following code checks such unexpected bug by couting actual object allocation.
1014
+ # Following code checks such unexpected bug by counting actual object allocation.
1015
1015
  base_num = count_timer_object
1016
1016
  2.times {
1017
1017
  sleep 1
@@ -162,4 +162,29 @@ class TcpInputTest < Test::Unit::TestCase
162
162
  assert event[1].is_a?(Fluent::EventTime)
163
163
  assert_equal address, event[2]['addr']
164
164
  end
165
+
166
+ sub_test_case '<extract>' do
167
+ test 'extract tag from record field' do
168
+ d = create_driver(BASE_CONFIG + %!
169
+ <parse>
170
+ @type json
171
+ </parse>
172
+ <extract>
173
+ tag_key tag
174
+ </extract>
175
+ !)
176
+ d.run(expect_records: 1) do
177
+ create_tcp_socket('127.0.0.1', PORT) do |sock|
178
+ data = {'msg' => 'hello', 'tag' => 'helper_test'}
179
+ sock.send("#{data.to_json}\n", 0)
180
+ end
181
+ end
182
+
183
+ assert_equal 1, d.events.size
184
+ event = d.events[0]
185
+ assert_equal 'helper_test', event[0]
186
+ assert event[1].is_a?(Fluent::EventTime)
187
+ assert_equal 'hello', event[2]['msg']
188
+ end
189
+ end
165
190
  end
@@ -39,13 +39,11 @@ class ForwardOutputTest < Test::Unit::TestCase
39
39
 
40
40
  def create_driver(conf=CONFIG)
41
41
  Fluent::Test::Driver::Output.new(Fluent::Plugin::ForwardOutput) {
42
- attr_reader :response_chunk_ids, :exceptions, :sent_chunk_ids
42
+ attr_reader :sent_chunk_ids, :ack_handler
43
43
 
44
44
  def initialize
45
45
  super
46
46
  @sent_chunk_ids = []
47
- @response_chunk_ids = []
48
- @exceptions = []
49
47
  end
50
48
 
51
49
  def try_write(chunk)
@@ -53,15 +51,6 @@ class ForwardOutputTest < Test::Unit::TestCase
53
51
  @sent_chunk_ids << chunk.unique_id
54
52
  retval
55
53
  end
56
-
57
- def read_ack_from_sock(sock, unpacker)
58
- retval = super
59
- @response_chunk_ids << retval
60
- retval
61
- rescue => e
62
- @exceptions << e
63
- raise e
64
- end
65
54
  }.configure(conf)
66
55
  end
67
56
 
@@ -196,6 +185,81 @@ EOL
196
185
  assert_equal([dummy_cert_path], d.instance.tls_ca_cert_path)
197
186
  end
198
187
 
188
+ sub_test_case "certstore loading parameters for Windows" do
189
+ test 'certstore related config parameters' do
190
+ omit "certstore related values raise error on not Windows" if Fluent.windows?
191
+ conf = %[
192
+ send_timeout 5
193
+ transport tls
194
+ tls_cert_logical_store_name Root
195
+ tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b
196
+ <server>
197
+ host #{TARGET_HOST}
198
+ port #{TARGET_PORT}
199
+ </server>
200
+ ]
201
+
202
+ assert_raise(Fluent::ConfigError) do
203
+ create_driver(conf)
204
+ end
205
+ end
206
+
207
+ test 'cert_logical_store_name and tls_cert_thumbprint default values' do
208
+ conf = %[
209
+ send_timeout 5
210
+ transport tls
211
+ <server>
212
+ host #{TARGET_HOST}
213
+ port #{TARGET_PORT}
214
+ </server>
215
+ ]
216
+
217
+ @d = d = create_driver(conf)
218
+ assert_nil d.instance.tls_cert_logical_store_name
219
+ assert_nil d.instance.tls_cert_thumbprint
220
+ end
221
+
222
+ data('CA cert' => 'tls_ca_cert_path',
223
+ 'non CA cert' => 'tls_cert_path')
224
+ test 'specify tls_cert_logical_store_name and tls_cert_path should raise error' do |param|
225
+ omit "Loading CertStore feature works only Windows" unless Fluent.windows?
226
+ dummy_cert_path = File.join(TMP_DIR, "dummy_cert.pem")
227
+ FileUtils.touch(dummy_cert_path)
228
+ conf = %[
229
+ send_timeout 5
230
+ transport tls
231
+ #{param} #{dummy_cert_path}
232
+ tls_cert_logical_store_name Root
233
+ <server>
234
+ host #{TARGET_HOST}
235
+ port #{TARGET_PORT}
236
+ </server>
237
+ ]
238
+
239
+ assert_raise(Fluent::ConfigError) do
240
+ create_driver(conf)
241
+ end
242
+ end
243
+
244
+ test 'configure cert_logical_store_name and tls_cert_thumbprint' do
245
+ omit "Loading CertStore feature works only Windows" unless Fluent.windows?
246
+ conf = %[
247
+ send_timeout 5
248
+ transport tls
249
+ tls_cert_logical_store_name Root
250
+ tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b
251
+ <server>
252
+ host #{TARGET_HOST}
253
+ port #{TARGET_PORT}
254
+ </server>
255
+ ]
256
+
257
+ @d = d = create_driver(conf)
258
+ assert_equal "Root", d.instance.tls_cert_logical_store_name
259
+ assert_equal "a909502dd82ae41433e6f83886b00d4277a32a7b", d.instance.tls_cert_thumbprint
260
+ end
261
+ end
262
+
199
263
  test 'compress_default_value' do
200
264
  @d = d = create_driver
201
265
  assert_equal :text, d.instance.compress
@@ -235,14 +299,14 @@ EOL
235
299
  node = d.instance.nodes.first
236
300
  stub(node.failure).phi { raise 'Should not be called' }
237
301
  node.tick
238
- assert_equal node.available, true
302
+ assert_true node.available?
239
303
  end
240
304
 
241
305
  test 'phi_failure_detector enabled' do
242
306
  @d = d = create_driver(CONFIG + %[phi_failure_detector true \n phi_threshold 0])
243
307
  node = d.instance.nodes.first
244
308
  node.tick
245
- assert_equal node.available, false
309
+ assert_false node.available?
246
310
  end
247
311
 
248
312
  test 'require_ack_response is disabled in default' do
@@ -287,6 +351,7 @@ EOL
287
351
  [tag_in_ascii, time, {"a" => 2}],
288
352
  ]
289
353
 
354
+ stub(d.instance.ack_handler).read_ack_from_sock(anything).never
290
355
  target_input_driver.run(expect_records: 2) do
291
356
  d.run do
292
357
  emit_events.each do |tag, t, record|
@@ -302,8 +367,6 @@ EOL
302
367
  assert_equal_event_time(time, events[1][1])
303
368
  assert_equal ['test.ascii', time, emit_events[1][2]], events[1]
304
369
  assert_equal Encoding::UTF_8, events[1][0].encoding
305
-
306
- assert_empty d.instance.exceptions
307
370
  end
308
371
 
309
372
  test 'send_with_time_as_integer' do
@@ -317,6 +380,8 @@ EOL
317
380
  {"a" => 1},
318
381
  {"a" => 2}
319
382
  ]
383
+
384
+ stub(d.instance.ack_handler).read_ack_from_sock(anything).never
320
385
  target_input_driver.run(expect_records: 2) do
321
386
  d.run(default_tag: 'test') do
322
387
  records.each do |record|
@@ -330,8 +395,6 @@ EOL
330
395
  assert_equal ['test', time, records[0]], events[0]
331
396
  assert_equal_event_time(time, events[1][1])
332
397
  assert_equal ['test', time, records[1]], events[1]
333
-
334
- assert_empty d.instance.exceptions
335
398
  end
336
399
 
337
400
  test 'send_without_time_as_integer' do
@@ -348,6 +411,7 @@ EOL
348
411
  {"a" => 1},
349
412
  {"a" => 2}
350
413
  ]
414
+ stub(d.instance.ack_handler).read_ack_from_sock(anything).never
351
415
  target_input_driver.run(expect_records: 2) do
352
416
  d.run(default_tag: 'test') do
353
417
  records.each do |record|
@@ -361,8 +425,6 @@ EOL
361
425
  assert_equal ['test', time, records[0]], events[0]
362
426
  assert_equal_event_time(time, events[1][1])
363
427
  assert_equal ['test', time, records[1]], events[1]
364
-
365
- assert_empty d.instance.exceptions
366
428
  end
367
429
 
368
430
  test 'send_comprssed_message_pack_stream_if_compress_is_gzip' do
@@ -406,6 +468,8 @@ EOL
406
468
  {"a" => 1},
407
469
  {"a" => 2}
408
470
  ]
471
+ # not attempt to receive responses
472
+ stub(d.instance.ack_handler).read_ack_from_sock(anything).never
409
473
  target_input_driver.run(expect_records: 2) do
410
474
  d.run(default_tag: 'test') do
411
475
  records.each do |record|
@@ -417,9 +481,6 @@ EOL
417
481
  events = target_input_driver.events
418
482
  assert_equal ['test', time, records[0]], events[0]
419
483
  assert_equal ['test', time, records[1]], events[1]
420
-
421
- assert_empty d.instance.response_chunk_ids # not attempt to receive responses, so it's empty
422
- assert_empty d.instance.exceptions
423
484
  end
424
485
 
425
486
  test 'send_to_a_node_not_supporting_responses' do
@@ -433,6 +494,8 @@ EOL
433
494
  {"a" => 1},
434
495
  {"a" => 2}
435
496
  ]
497
+ # not attempt to receive responses
498
+ stub(d.instance.ack_handler).read_ack_from_sock(anything).never
436
499
  target_input_driver.run(expect_records: 2) do
437
500
  d.run(default_tag: 'test') do
438
501
  records.each do |record|
@@ -444,9 +507,6 @@ EOL
444
507
  events = target_input_driver.events
445
508
  assert_equal ['test', time, records[0]], events[0]
446
509
  assert_equal ['test', time, records[1]], events[1]
447
-
448
- assert_empty d.instance.response_chunk_ids # not attempt to receive responses, so it's empty
449
- assert_empty d.instance.exceptions
450
510
  end
451
511
 
452
512
  test 'a node supporting responses' do
@@ -465,12 +525,20 @@ EOL
465
525
 
466
526
  time = event_time("2011-01-02 13:14:15 UTC")
467
527
 
528
+ acked_chunk_ids = []
529
+ mock.proxy(d.instance.ack_handler).read_ack_from_sock(anything) do |info, success|
530
+ if success
531
+ acked_chunk_ids << info.chunk_id
532
+ end
533
+ [chunk_id, success]
534
+ end
535
+
468
536
  records = [
469
537
  {"a" => 1},
470
538
  {"a" => 2}
471
539
  ]
472
540
  target_input_driver.run(expect_records: 2) do
473
- d.end_if{ d.instance.response_chunk_ids.length > 0 }
541
+ d.end_if { acked_chunk_ids.size > 0 }
474
542
  d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
475
543
  d.feed([[time, records[0]], [time,records[1]]])
476
544
  end
@@ -480,9 +548,8 @@ EOL
480
548
  assert_equal ['test', time, records[0]], events[0]
481
549
  assert_equal ['test', time, records[1]], events[1]
482
550
 
483
- assert_equal 1, d.instance.response_chunk_ids.size
484
- assert_equal d.instance.sent_chunk_ids.first, d.instance.response_chunk_ids.first
485
- assert_empty d.instance.exceptions
551
+ assert_equal 1, acked_chunk_ids.size
552
+ assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first
486
553
  end
487
554
 
488
555
  data('ack true' => true,
@@ -555,7 +622,7 @@ EOL
555
622
  {"a" => 2}
556
623
  ]
557
624
  target_input_driver.end_if{ d.instance.rollback_count > 0 }
558
- target_input_driver.end_if{ !node.available }
625
+ target_input_driver.end_if{ !node.available? }
559
626
  target_input_driver.run(expect_records: 2, timeout: 25) do
560
627
  d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do
561
628
  delayed_commit_timeout_value = d.instance.delayed_commit_timeout
@@ -600,7 +667,7 @@ EOL
600
667
  {"a" => 2}
601
668
  ]
602
669
  target_input_driver.end_if{ d.instance.rollback_count > 0 }
603
- target_input_driver.end_if{ !node.available }
670
+ target_input_driver.end_if{ !node.available? }
604
671
  target_input_driver.run(expect_records: 2, timeout: 25) do
605
672
  d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do
606
673
  delayed_commit_timeout_value = d.instance.delayed_commit_timeout
@@ -840,7 +907,7 @@ EOL
840
907
 
841
908
  stub(node.failure).phi { raise 'Should not be called' }
842
909
  node.tick
843
- assert_equal node.available, true
910
+ assert_true node.available?
844
911
  end
845
912
 
846
913
  test 'heartbeat_type_udp' do
@@ -856,7 +923,7 @@ EOL
856
923
  assert timers.include?(:out_forward_heartbeat_request)
857
924
 
858
925
  mock(usock).send("\0", 0, Socket.pack_sockaddr_in(TARGET_PORT, '127.0.0.1')).once
859
- d.instance.send(:on_timer)
926
+ d.instance.send(:on_heartbeat_timer)
860
927
  end
861
928
 
862
929
  test 'acts_as_secondary' do
@@ -882,16 +949,12 @@ EOL
882
949
  test 'nodes are not available' do
883
950
  @d = d = create_driver(CONFIG + %[
884
951
  verify_connection_at_startup true
885
- <buffer tag>
886
- flush_mode immediate
887
- retry_type periodic
888
- retry_wait 30s
889
- flush_at_shutdown false # suppress errors in d.instance_shutdown
890
- </buffer>
891
952
  ])
892
- assert_raise Fluent::UnrecoverableError do
953
+ e = assert_raise Fluent::UnrecoverableError do
893
954
  d.instance_start
894
955
  end
956
+ assert_match(/Connection refused/, e.message)
957
+
895
958
  d.instance_shutdown
896
959
  end
897
960
 
@@ -904,24 +967,47 @@ EOL
904
967
  ]
905
968
  target_input_driver = create_target_input_driver(conf: input_conf)
906
969
  output_conf = %[
907
- send_timeout 30
908
- heartbeat_type transport
970
+ transport tcp
971
+ verify_connection_at_startup true
972
+ <security>
973
+ self_hostname localhost
974
+ shared_key key_miss_match
975
+ </security>
976
+
977
+ <server>
978
+ host #{TARGET_HOST}
979
+ port #{TARGET_PORT}
980
+ </server>
981
+ ]
982
+ @d = d = create_driver(output_conf)
983
+
984
+ target_input_driver.run(expect_records: 1, timeout: 1) do
985
+ e = assert_raise Fluent::UnrecoverableError do
986
+ d.instance_start
987
+ end
988
+ assert_match(/Failed to establish connection/, e.message)
989
+ end
990
+ end
991
+
992
+ test 'nodes_shared_key_miss_match with TLS' do
993
+ input_conf = TARGET_CONFIG + %[
994
+ <security>
995
+ self_hostname in.localhost
996
+ shared_key fluentd-sharedkey
997
+ </security>
998
+ <transport tls>
999
+ insecure true
1000
+ </transport>
1001
+ ]
1002
+ target_input_driver = create_target_input_driver(conf: input_conf)
1003
+ output_conf = %[
909
1004
  transport tls
910
- tls_verify_hostname false
1005
+ tls_insecure_mode true
911
1006
  verify_connection_at_startup true
912
- require_ack_response true
913
- ack_response_timeout 5s
914
1007
  <security>
915
1008
  self_hostname localhost
916
1009
  shared_key key_miss_match
917
1010
  </security>
918
- <buffer tag>
919
- flush_mode immediate
920
- retry_type periodic
921
- retry_wait 30s
922
- flush_at_shutdown false # suppress errors in d.instance_shutdown
923
- flush_thread_interval 31s
924
- </buffer>
925
1011
 
926
1012
  <server>
927
1013
  host #{TARGET_HOST}
@@ -930,11 +1016,12 @@ EOL
930
1016
  ]
931
1017
  @d = d = create_driver(output_conf)
932
1018
 
933
- target_input_driver.run(expect_records: 1, timeout: 15) do
934
- assert_raise Fluent::UnrecoverableError do
1019
+ target_input_driver.run(expect_records: 1, timeout: 1) do
1020
+ e = assert_raise Fluent::UnrecoverableError do
935
1021
  d.instance_start
936
1022
  end
937
- d.instance_shutdown
1023
+
1024
+ assert_match(/Failed to establish connection/, e.message)
938
1025
  end
939
1026
  end
940
1027
 
@@ -943,15 +1030,10 @@ EOL
943
1030
  <security>
944
1031
  self_hostname in.localhost
945
1032
  shared_key fluentd-sharedkey
946
- <client>
947
- host 127.0.0.1
948
- </client>
949
1033
  </security>
950
1034
  ]
951
1035
  target_input_driver = create_target_input_driver(conf: input_conf)
952
-
953
1036
  output_conf = %[
954
- send_timeout 51
955
1037
  verify_connection_at_startup true
956
1038
  <security>
957
1039
  self_hostname localhost
@@ -961,18 +1043,14 @@ EOL
961
1043
  name test
962
1044
  host #{TARGET_HOST}
963
1045
  port #{TARGET_PORT}
964
- shared_key fluentd-sharedkey
965
1046
  </server>
966
1047
  ]
967
1048
  @d = d = create_driver(output_conf)
968
1049
 
969
1050
  time = event_time("2011-01-02 13:14:15 UTC")
970
- records = [
971
- {"a" => 1},
972
- {"a" => 2}
973
- ]
1051
+ records = [{ "a" => 1 }, { "a" => 2 }]
974
1052
 
975
- target_input_driver.run(expect_records: 2, timeout: 15) do
1053
+ target_input_driver.run(expect_records: 2, timeout: 3) do
976
1054
  d.run(default_tag: 'test') do
977
1055
  records.each do |record|
978
1056
  d.feed(time, record)
@@ -981,7 +1059,7 @@ EOL
981
1059
  end
982
1060
 
983
1061
  events = target_input_driver.events
984
- assert{ events != [] }
1062
+ assert_false events.empty?
985
1063
  assert_equal(['test', time, records[0]], events[0])
986
1064
  assert_equal(['test', time, records[1]], events[1])
987
1065
  end
@@ -995,7 +1073,7 @@ EOL
995
1073
 
996
1074
  begin
997
1075
  chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
998
- mock.proxy(d.instance).create_transfer_socket(TARGET_HOST, TARGET_PORT, 'test') { |sock| mock(sock).close.once; sock }.twice
1076
+ mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, TARGET_PORT, anything) { |sock| mock(sock).close.once; sock }.twice
999
1077
 
1000
1078
  target_input_driver.run(timeout: 15) do
1001
1079
  d.run(shutdown: false) do
@@ -1036,7 +1114,7 @@ EOL
1036
1114
 
1037
1115
  begin
1038
1116
  chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1039
- mock.proxy(d.instance).create_transfer_socket(TARGET_HOST, TARGET_PORT, 'test') { |sock| mock(sock).close.once; sock }.once
1117
+ mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, TARGET_PORT, anything) { |sock| mock(sock).close.once; sock }.once
1040
1118
 
1041
1119
  target_input_driver.run(timeout: 15) do
1042
1120
  d.run(shutdown: false) do
@@ -1052,7 +1130,7 @@ EOL
1052
1130
  end
1053
1131
 
1054
1132
  sub_test_case 'with require_ack_response' do
1055
- test 'Do not create connection per send_data' do
1133
+ test 'Create connection per send_data' do
1056
1134
  target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
1057
1135
  output_conf = CONFIG + %[
1058
1136
  require_ack_response true
@@ -1064,7 +1142,7 @@ EOL
1064
1142
 
1065
1143
  begin
1066
1144
  chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1067
- mock.proxy(d.instance).create_transfer_socket(TARGET_HOST, TARGET_PORT, 'test') { |sock| mock(sock).close.once; sock }.once
1145
+ mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, TARGET_PORT, anything) { |sock| mock(sock).close.once; sock }.twice
1068
1146
 
1069
1147
  target_input_driver.run(timeout: 15) do
1070
1148
  d.run(shutdown: false) do
@@ -1080,133 +1158,4 @@ EOL
1080
1158
  end
1081
1159
  end
1082
1160
  end
1083
-
1084
- sub_test_case 'SocketCache' do
1085
- sub_test_case 'fetch_or' do
1086
- test 'when gived key does not exist' do
1087
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1088
- sock = mock!.open { 1 }.subject
1089
- assert_equal(1, c.fetch_or { sock.open })
1090
- end
1091
-
1092
- test 'when given key exists' do
1093
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1094
- assert_equal(1, c.fetch_or { 1 })
1095
-
1096
- sock = dont_allow(mock!).open
1097
- assert_equal(1, c.fetch_or { sock.open })
1098
- end
1099
-
1100
- test "when given key's value was expired" do
1101
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(0, Logger.new(nil))
1102
- assert_equal(1, c.fetch_or { 1 })
1103
-
1104
- sock = mock!.open { 1 }.subject
1105
- assert_equal(1, c.fetch_or { sock.open })
1106
- end
1107
- end
1108
-
1109
- test 'revoke' do
1110
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1111
- c.fetch_or { 1 }
1112
- c.revoke
1113
-
1114
- sock = mock!.open { 1 }.subject
1115
- assert_equal(1, c.fetch_or { sock.open })
1116
- end
1117
-
1118
- test 'revoke_by_value' do
1119
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1120
- c.fetch_or { 1 }
1121
- c.revoke_by_value(1)
1122
-
1123
- sock = mock!.open { 1 }.subject
1124
- assert_equal(1, c.fetch_or { sock.open })
1125
- end
1126
-
1127
- sub_test_case 'dec_ref' do
1128
- test 'when value exists in active_socks' do
1129
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1130
- c.fetch_or { 1 }
1131
- c.dec_ref
1132
-
1133
- assert_equal(0, c.instance_variable_get(:@active_socks)[Thread.current.object_id].ref)
1134
- end
1135
-
1136
- test 'when value exists in inactive_socks' do
1137
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1138
- c.fetch_or { 1 }
1139
- c.revoke
1140
- c.dec_ref
1141
- assert_equal(-1, c.instance_variable_get(:@inactive_socks)[Thread.current.object_id].ref)
1142
- end
1143
- end
1144
-
1145
- sub_test_case 'dec_ref_by_value' do
1146
- test 'when value exists in active_socks' do
1147
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1148
- c.fetch_or { 1 }
1149
- c.dec_ref_by_value(1)
1150
-
1151
- assert_equal(0, c.instance_variable_get(:@active_socks)[Thread.current.object_id].ref)
1152
- end
1153
-
1154
- test 'when value exists in inactive_socks' do
1155
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1156
- c.fetch_or { 1 }
1157
- c.revoke
1158
- c.dec_ref_by_value(1)
1159
- assert_equal(-1, c.instance_variable_get(:@inactive_socks)[Thread.current.object_id].ref)
1160
- end
1161
- end
1162
-
1163
- sub_test_case 'clear' do
1164
- test 'when value is in active_socks' do
1165
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1166
- m = mock!.close { 'closed' }.subject
1167
- c.fetch_or { m }
1168
- assert_true(!c.instance_variable_get(:@active_socks).empty?)
1169
-
1170
- c.clear
1171
- assert_true(c.instance_variable_get(:@active_socks).empty?)
1172
- end
1173
-
1174
- test 'when value is in inactive_socks' do
1175
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1176
- m = mock!.close { 'closed' }.subject
1177
- c.fetch_or { m }
1178
- c.revoke
1179
- assert_true(!c.instance_variable_get(:@inactive_socks).empty?)
1180
-
1181
- c.clear
1182
- assert_true(c.instance_variable_get(:@active_socks).empty?)
1183
- end
1184
- end
1185
-
1186
- sub_test_case 'purge_obsolete_socks' do
1187
- test 'delete key in inactive_socks' do
1188
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1189
- m = mock!.close { 'closed' }.subject
1190
- c.fetch_or { m }
1191
- c.revoke
1192
- assert_true(!c.instance_variable_get(:@inactive_socks).empty?)
1193
-
1194
- c.purge_obsolete_socks
1195
- assert_true(c.instance_variable_get(:@active_socks).empty?)
1196
- end
1197
-
1198
- test 'move key from active_socks to inactive_socks' do
1199
- c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
1200
- m = dont_allow(mock!).close
1201
- stub(m).inspect # for log
1202
- c.fetch_or { m }
1203
- assert_true(!c.instance_variable_get(:@active_socks).empty?)
1204
- assert_true(c.instance_variable_get(:@inactive_socks).empty?)
1205
-
1206
- c.purge_obsolete_socks
1207
- assert_true(!c.instance_variable_get(:@active_socks).empty?)
1208
- assert_true(c.instance_variable_get(:@inactive_socks).empty?)
1209
- end
1210
- end
1211
- end
1212
1161
  end