fluentd 1.6.3 → 1.7.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.drone.yml +35 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +2 -0
- data/README.md +5 -1
- data/fluentd.gemspec +1 -1
- data/lib/fluent/clock.rb +4 -0
- data/lib/fluent/compat/output.rb +3 -3
- data/lib/fluent/compat/socket_util.rb +1 -1
- data/lib/fluent/config/element.rb +3 -3
- data/lib/fluent/config/literal_parser.rb +1 -1
- data/lib/fluent/config/section.rb +4 -1
- data/lib/fluent/error.rb +4 -0
- data/lib/fluent/event.rb +28 -24
- data/lib/fluent/event_router.rb +2 -1
- data/lib/fluent/log.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +8 -0
- data/lib/fluent/plugin/bare_output.rb +4 -4
- data/lib/fluent/plugin/buf_file_single.rb +211 -0
- data/lib/fluent/plugin/buffer.rb +62 -63
- data/lib/fluent/plugin/buffer/chunk.rb +21 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +37 -12
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +314 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -1
- data/lib/fluent/plugin/compressable.rb +10 -6
- data/lib/fluent/plugin/filter_grep.rb +2 -2
- data/lib/fluent/plugin/formatter_csv.rb +10 -6
- data/lib/fluent/plugin/in_syslog.rb +10 -3
- data/lib/fluent/plugin/in_tail.rb +7 -2
- data/lib/fluent/plugin/in_tcp.rb +34 -7
- data/lib/fluent/plugin/multi_output.rb +4 -4
- data/lib/fluent/plugin/out_exec_filter.rb +1 -0
- data/lib/fluent/plugin/out_file.rb +13 -3
- data/lib/fluent/plugin/out_forward.rb +126 -588
- data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
- data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
- data/lib/fluent/plugin/out_forward/error.rb +28 -0
- data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +121 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +111 -0
- data/lib/fluent/plugin/out_forward/socket_cache.rb +138 -0
- data/lib/fluent/plugin/out_http.rb +231 -0
- data/lib/fluent/plugin/output.rb +29 -35
- data/lib/fluent/plugin/parser.rb +77 -0
- data/lib/fluent/plugin/parser_csv.rb +75 -0
- data/lib/fluent/plugin_helper/server.rb +1 -1
- data/lib/fluent/plugin_helper/thread.rb +1 -0
- data/lib/fluent/root_agent.rb +1 -1
- data/lib/fluent/time.rb +4 -2
- data/lib/fluent/timezone.rb +21 -7
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_fluentd.rb +1 -1
- data/test/command/test_plugin_generator.rb +18 -2
- data/test/config/test_configurable.rb +78 -40
- data/test/counter/test_store.rb +1 -1
- data/test/helper.rb +1 -0
- data/test/helpers/process_extenstion.rb +33 -0
- data/test/plugin/out_forward/test_ack_handler.rb +101 -0
- data/test/plugin/out_forward/test_connection_manager.rb +145 -0
- data/test/plugin/out_forward/test_handshake_protocol.rb +103 -0
- data/test/plugin/out_forward/test_load_balancer.rb +60 -0
- data/test/plugin/out_forward/test_socket_cache.rb +139 -0
- data/test/plugin/test_buf_file.rb +118 -2
- data/test/plugin/test_buf_file_single.rb +734 -0
- data/test/plugin/test_buffer.rb +4 -48
- data/test/plugin/test_buffer_file_chunk.rb +19 -1
- data/test/plugin/test_buffer_file_single_chunk.rb +620 -0
- data/test/plugin/test_formatter_csv.rb +16 -0
- data/test/plugin/test_in_syslog.rb +56 -6
- data/test/plugin/test_in_tail.rb +1 -1
- data/test/plugin/test_in_tcp.rb +25 -0
- data/test/plugin/test_out_forward.rb +75 -201
- data/test/plugin/test_out_http.rb +352 -0
- data/test/plugin/test_output_as_buffered.rb +27 -24
- data/test/plugin/test_parser.rb +40 -0
- data/test/plugin/test_parser_csv.rb +83 -0
- data/test/plugin_helper/test_record_accessor.rb +1 -1
- data/test/test_time_formatter.rb +140 -121
- metadata +35 -6
@@ -108,4 +108,20 @@ class CsvFormatterTest < ::Test::Unit::TestCase
|
|
108
108
|
d = create_driver('fields' => data)
|
109
109
|
assert_equal %w(one two three), d.instance.fields
|
110
110
|
end
|
111
|
+
|
112
|
+
def test_format_with_multiple_records
|
113
|
+
d = create_driver("fields" => "message,message2")
|
114
|
+
r = {'message' => 'hello', 'message2' => 'fluentd'}
|
115
|
+
|
116
|
+
formatted = d.instance.format(tag, @time, r)
|
117
|
+
assert_equal("\"hello\",\"fluentd\"\n", formatted)
|
118
|
+
|
119
|
+
r = {'message' => 'hey', 'message2' => 'ho'}
|
120
|
+
formatted = d.instance.format(tag, @time, r)
|
121
|
+
assert_equal("\"hey\",\"ho\"\n", formatted)
|
122
|
+
|
123
|
+
r = {'message' => 'longer message', 'message2' => 'longer longer message'}
|
124
|
+
formatted = d.instance.format(tag, @time, r)
|
125
|
+
assert_equal("\"longer message\",\"longer longer message\"\n", formatted)
|
126
|
+
end
|
111
127
|
end
|
@@ -362,9 +362,8 @@ EOS
|
|
362
362
|
end
|
363
363
|
end
|
364
364
|
|
365
|
-
def
|
366
|
-
|
367
|
-
tests = [
|
365
|
+
def create_unmatched_lines_test_case
|
366
|
+
[
|
368
367
|
# valid message
|
369
368
|
{'msg' => '<6>Sep 10 00:00:00 localhost logger: xxx', 'expected' => {'host'=>'localhost', 'ident'=>'logger', 'message'=>'xxx'}},
|
370
369
|
# missing priority
|
@@ -372,6 +371,22 @@ EOS
|
|
372
371
|
# timestamp parsing failure
|
373
372
|
{'msg' => '<6>ZZZ 99 99:99:99 localhost logger: xxx', 'expected' => {'unmatched_line' => '<6>ZZZ 99 99:99:99 localhost logger: xxx'}},
|
374
373
|
]
|
374
|
+
end
|
375
|
+
|
376
|
+
def compare_unmatched_lines_test_result(events, tests, options = {})
|
377
|
+
events.each_index { |i|
|
378
|
+
tests[i]['expected'].each { |k,v|
|
379
|
+
assert_equal v, events[i][2][k], "No key <#{k}> in response or value mismatch"
|
380
|
+
}
|
381
|
+
assert_equal('syslog.unmatched', events[i][0], 'tag does not match syslog.unmatched') unless i==0
|
382
|
+
assert_equal(options[:address], events[i][2]['source_address'], 'response has no source_address or mismatch') if options[:address]
|
383
|
+
assert_equal(options[:hostname], events[i][2]['source_hostname'], 'response has no source_hostname or mismatch') if options[:hostname]
|
384
|
+
}
|
385
|
+
end
|
386
|
+
|
387
|
+
def test_emit_unmatched_lines
|
388
|
+
d = create_driver([CONFIG, 'emit_unmatched_lines true'].join("\n"))
|
389
|
+
tests = create_unmatched_lines_test_case
|
375
390
|
|
376
391
|
d.run(expect_emits: 3) do
|
377
392
|
u = UDPSocket.new
|
@@ -383,9 +398,44 @@ EOS
|
|
383
398
|
end
|
384
399
|
|
385
400
|
assert_equal tests.size, d.events.size
|
386
|
-
|
387
|
-
|
388
|
-
|
401
|
+
compare_unmatched_lines_test_result(d.events, tests)
|
402
|
+
end
|
403
|
+
|
404
|
+
def test_emit_unmatched_lines_with_hostname
|
405
|
+
d = create_driver([CONFIG, 'emit_unmatched_lines true', 'source_hostname_key source_hostname'].join("\n"))
|
406
|
+
tests = create_unmatched_lines_test_case
|
407
|
+
|
408
|
+
hostname = nil
|
409
|
+
d.run(expect_emits: 3) do
|
410
|
+
u = UDPSocket.new
|
411
|
+
u.do_not_reverse_lookup = false
|
412
|
+
u.connect('127.0.0.1', PORT)
|
413
|
+
hostname = u.peeraddr[2]
|
414
|
+
tests.each {|test|
|
415
|
+
u.send(test['msg'], 0)
|
416
|
+
}
|
417
|
+
end
|
418
|
+
|
419
|
+
assert_equal tests.size, d.events.size
|
420
|
+
compare_unmatched_lines_test_result(d.events, tests, {hostname: hostname})
|
421
|
+
end
|
422
|
+
|
423
|
+
def test_emit_unmatched_lines_with_address
|
424
|
+
d = create_driver([CONFIG, 'emit_unmatched_lines true', 'source_address_key source_address'].join("\n"))
|
425
|
+
tests = create_unmatched_lines_test_case
|
426
|
+
|
427
|
+
address = nil
|
428
|
+
d.run(expect_emits: 3) do
|
429
|
+
u = UDPSocket.new
|
430
|
+
u.do_not_reverse_lookup = false
|
431
|
+
u.connect('127.0.0.1', PORT)
|
432
|
+
address = u.peeraddr[3]
|
433
|
+
tests.each {|test|
|
434
|
+
u.send(test['msg'], 0)
|
435
|
+
}
|
389
436
|
end
|
437
|
+
|
438
|
+
assert_equal tests.size, d.events.size
|
439
|
+
compare_unmatched_lines_test_result(d.events, tests, {address: address})
|
390
440
|
end
|
391
441
|
end
|
data/test/plugin/test_in_tail.rb
CHANGED
@@ -1011,7 +1011,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
1011
1011
|
waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 0 }
|
1012
1012
|
|
1013
1013
|
# Previous implementation has an infinite watcher creation bug.
|
1014
|
-
# Following code checks such unexpected bug by
|
1014
|
+
# Following code checks such unexpected bug by counting actual object allocation.
|
1015
1015
|
base_num = count_timer_object
|
1016
1016
|
2.times {
|
1017
1017
|
sleep 1
|
data/test/plugin/test_in_tcp.rb
CHANGED
@@ -162,4 +162,29 @@ class TcpInputTest < Test::Unit::TestCase
|
|
162
162
|
assert event[1].is_a?(Fluent::EventTime)
|
163
163
|
assert_equal address, event[2]['addr']
|
164
164
|
end
|
165
|
+
|
166
|
+
sub_test_case '<extract>' do
|
167
|
+
test 'extract tag from record field' do
|
168
|
+
d = create_driver(BASE_CONFIG + %!
|
169
|
+
<parse>
|
170
|
+
@type json
|
171
|
+
</parse>
|
172
|
+
<extract>
|
173
|
+
tag_key tag
|
174
|
+
</extract>
|
175
|
+
!)
|
176
|
+
d.run(expect_records: 1) do
|
177
|
+
create_tcp_socket('127.0.0.1', PORT) do |sock|
|
178
|
+
data = {'msg' => 'hello', 'tag' => 'helper_test'}
|
179
|
+
sock.send("#{data.to_json}\n", 0)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
assert_equal 1, d.events.size
|
184
|
+
event = d.events[0]
|
185
|
+
assert_equal 'helper_test', event[0]
|
186
|
+
assert event[1].is_a?(Fluent::EventTime)
|
187
|
+
assert_equal 'hello', event[2]['msg']
|
188
|
+
end
|
189
|
+
end
|
165
190
|
end
|
@@ -39,13 +39,11 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
39
39
|
|
40
40
|
def create_driver(conf=CONFIG)
|
41
41
|
Fluent::Test::Driver::Output.new(Fluent::Plugin::ForwardOutput) {
|
42
|
-
attr_reader :
|
42
|
+
attr_reader :sent_chunk_ids, :ack_handler
|
43
43
|
|
44
44
|
def initialize
|
45
45
|
super
|
46
46
|
@sent_chunk_ids = []
|
47
|
-
@response_chunk_ids = []
|
48
|
-
@exceptions = []
|
49
47
|
end
|
50
48
|
|
51
49
|
def try_write(chunk)
|
@@ -53,15 +51,6 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
53
51
|
@sent_chunk_ids << chunk.unique_id
|
54
52
|
retval
|
55
53
|
end
|
56
|
-
|
57
|
-
def read_ack_from_sock(sock, unpacker)
|
58
|
-
retval = super
|
59
|
-
@response_chunk_ids << retval
|
60
|
-
retval
|
61
|
-
rescue => e
|
62
|
-
@exceptions << e
|
63
|
-
raise e
|
64
|
-
end
|
65
54
|
}.configure(conf)
|
66
55
|
end
|
67
56
|
|
@@ -235,14 +224,14 @@ EOL
|
|
235
224
|
node = d.instance.nodes.first
|
236
225
|
stub(node.failure).phi { raise 'Should not be called' }
|
237
226
|
node.tick
|
238
|
-
|
227
|
+
assert_true node.available?
|
239
228
|
end
|
240
229
|
|
241
230
|
test 'phi_failure_detector enabled' do
|
242
231
|
@d = d = create_driver(CONFIG + %[phi_failure_detector true \n phi_threshold 0])
|
243
232
|
node = d.instance.nodes.first
|
244
233
|
node.tick
|
245
|
-
|
234
|
+
assert_false node.available?
|
246
235
|
end
|
247
236
|
|
248
237
|
test 'require_ack_response is disabled in default' do
|
@@ -287,6 +276,7 @@ EOL
|
|
287
276
|
[tag_in_ascii, time, {"a" => 2}],
|
288
277
|
]
|
289
278
|
|
279
|
+
stub(d.instance.ack_handler).read_ack_from_sock(anything).never
|
290
280
|
target_input_driver.run(expect_records: 2) do
|
291
281
|
d.run do
|
292
282
|
emit_events.each do |tag, t, record|
|
@@ -302,8 +292,6 @@ EOL
|
|
302
292
|
assert_equal_event_time(time, events[1][1])
|
303
293
|
assert_equal ['test.ascii', time, emit_events[1][2]], events[1]
|
304
294
|
assert_equal Encoding::UTF_8, events[1][0].encoding
|
305
|
-
|
306
|
-
assert_empty d.instance.exceptions
|
307
295
|
end
|
308
296
|
|
309
297
|
test 'send_with_time_as_integer' do
|
@@ -317,6 +305,8 @@ EOL
|
|
317
305
|
{"a" => 1},
|
318
306
|
{"a" => 2}
|
319
307
|
]
|
308
|
+
|
309
|
+
stub(d.instance.ack_handler).read_ack_from_sock(anything).never
|
320
310
|
target_input_driver.run(expect_records: 2) do
|
321
311
|
d.run(default_tag: 'test') do
|
322
312
|
records.each do |record|
|
@@ -330,8 +320,6 @@ EOL
|
|
330
320
|
assert_equal ['test', time, records[0]], events[0]
|
331
321
|
assert_equal_event_time(time, events[1][1])
|
332
322
|
assert_equal ['test', time, records[1]], events[1]
|
333
|
-
|
334
|
-
assert_empty d.instance.exceptions
|
335
323
|
end
|
336
324
|
|
337
325
|
test 'send_without_time_as_integer' do
|
@@ -348,6 +336,7 @@ EOL
|
|
348
336
|
{"a" => 1},
|
349
337
|
{"a" => 2}
|
350
338
|
]
|
339
|
+
stub(d.instance.ack_handler).read_ack_from_sock(anything).never
|
351
340
|
target_input_driver.run(expect_records: 2) do
|
352
341
|
d.run(default_tag: 'test') do
|
353
342
|
records.each do |record|
|
@@ -361,8 +350,6 @@ EOL
|
|
361
350
|
assert_equal ['test', time, records[0]], events[0]
|
362
351
|
assert_equal_event_time(time, events[1][1])
|
363
352
|
assert_equal ['test', time, records[1]], events[1]
|
364
|
-
|
365
|
-
assert_empty d.instance.exceptions
|
366
353
|
end
|
367
354
|
|
368
355
|
test 'send_comprssed_message_pack_stream_if_compress_is_gzip' do
|
@@ -406,6 +393,8 @@ EOL
|
|
406
393
|
{"a" => 1},
|
407
394
|
{"a" => 2}
|
408
395
|
]
|
396
|
+
# not attempt to receive responses
|
397
|
+
stub(d.instance.ack_handler).read_ack_from_sock(anything).never
|
409
398
|
target_input_driver.run(expect_records: 2) do
|
410
399
|
d.run(default_tag: 'test') do
|
411
400
|
records.each do |record|
|
@@ -417,9 +406,6 @@ EOL
|
|
417
406
|
events = target_input_driver.events
|
418
407
|
assert_equal ['test', time, records[0]], events[0]
|
419
408
|
assert_equal ['test', time, records[1]], events[1]
|
420
|
-
|
421
|
-
assert_empty d.instance.response_chunk_ids # not attempt to receive responses, so it's empty
|
422
|
-
assert_empty d.instance.exceptions
|
423
409
|
end
|
424
410
|
|
425
411
|
test 'send_to_a_node_not_supporting_responses' do
|
@@ -433,6 +419,8 @@ EOL
|
|
433
419
|
{"a" => 1},
|
434
420
|
{"a" => 2}
|
435
421
|
]
|
422
|
+
# not attempt to receive responses
|
423
|
+
stub(d.instance.ack_handler).read_ack_from_sock(anything).never
|
436
424
|
target_input_driver.run(expect_records: 2) do
|
437
425
|
d.run(default_tag: 'test') do
|
438
426
|
records.each do |record|
|
@@ -444,9 +432,6 @@ EOL
|
|
444
432
|
events = target_input_driver.events
|
445
433
|
assert_equal ['test', time, records[0]], events[0]
|
446
434
|
assert_equal ['test', time, records[1]], events[1]
|
447
|
-
|
448
|
-
assert_empty d.instance.response_chunk_ids # not attempt to receive responses, so it's empty
|
449
|
-
assert_empty d.instance.exceptions
|
450
435
|
end
|
451
436
|
|
452
437
|
test 'a node supporting responses' do
|
@@ -465,12 +450,20 @@ EOL
|
|
465
450
|
|
466
451
|
time = event_time("2011-01-02 13:14:15 UTC")
|
467
452
|
|
453
|
+
acked_chunk_ids = []
|
454
|
+
mock.proxy(d.instance.ack_handler).read_ack_from_sock(anything) do |info, success|
|
455
|
+
if success
|
456
|
+
acked_chunk_ids << info.chunk_id
|
457
|
+
end
|
458
|
+
[chunk_id, success]
|
459
|
+
end
|
460
|
+
|
468
461
|
records = [
|
469
462
|
{"a" => 1},
|
470
463
|
{"a" => 2}
|
471
464
|
]
|
472
465
|
target_input_driver.run(expect_records: 2) do
|
473
|
-
d.end_if{
|
466
|
+
d.end_if { acked_chunk_ids.size > 0 }
|
474
467
|
d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
|
475
468
|
d.feed([[time, records[0]], [time,records[1]]])
|
476
469
|
end
|
@@ -480,9 +473,8 @@ EOL
|
|
480
473
|
assert_equal ['test', time, records[0]], events[0]
|
481
474
|
assert_equal ['test', time, records[1]], events[1]
|
482
475
|
|
483
|
-
assert_equal 1,
|
484
|
-
assert_equal d.instance.sent_chunk_ids.first,
|
485
|
-
assert_empty d.instance.exceptions
|
476
|
+
assert_equal 1, acked_chunk_ids.size
|
477
|
+
assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first
|
486
478
|
end
|
487
479
|
|
488
480
|
data('ack true' => true,
|
@@ -555,7 +547,7 @@ EOL
|
|
555
547
|
{"a" => 2}
|
556
548
|
]
|
557
549
|
target_input_driver.end_if{ d.instance.rollback_count > 0 }
|
558
|
-
target_input_driver.end_if{ !node.available }
|
550
|
+
target_input_driver.end_if{ !node.available? }
|
559
551
|
target_input_driver.run(expect_records: 2, timeout: 25) do
|
560
552
|
d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do
|
561
553
|
delayed_commit_timeout_value = d.instance.delayed_commit_timeout
|
@@ -600,7 +592,7 @@ EOL
|
|
600
592
|
{"a" => 2}
|
601
593
|
]
|
602
594
|
target_input_driver.end_if{ d.instance.rollback_count > 0 }
|
603
|
-
target_input_driver.end_if{ !node.available }
|
595
|
+
target_input_driver.end_if{ !node.available? }
|
604
596
|
target_input_driver.run(expect_records: 2, timeout: 25) do
|
605
597
|
d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do
|
606
598
|
delayed_commit_timeout_value = d.instance.delayed_commit_timeout
|
@@ -840,7 +832,7 @@ EOL
|
|
840
832
|
|
841
833
|
stub(node.failure).phi { raise 'Should not be called' }
|
842
834
|
node.tick
|
843
|
-
|
835
|
+
assert_true node.available?
|
844
836
|
end
|
845
837
|
|
846
838
|
test 'heartbeat_type_udp' do
|
@@ -856,7 +848,7 @@ EOL
|
|
856
848
|
assert timers.include?(:out_forward_heartbeat_request)
|
857
849
|
|
858
850
|
mock(usock).send("\0", 0, Socket.pack_sockaddr_in(TARGET_PORT, '127.0.0.1')).once
|
859
|
-
d.instance.send(:
|
851
|
+
d.instance.send(:on_heartbeat_timer)
|
860
852
|
end
|
861
853
|
|
862
854
|
test 'acts_as_secondary' do
|
@@ -882,16 +874,12 @@ EOL
|
|
882
874
|
test 'nodes are not available' do
|
883
875
|
@d = d = create_driver(CONFIG + %[
|
884
876
|
verify_connection_at_startup true
|
885
|
-
<buffer tag>
|
886
|
-
flush_mode immediate
|
887
|
-
retry_type periodic
|
888
|
-
retry_wait 30s
|
889
|
-
flush_at_shutdown false # suppress errors in d.instance_shutdown
|
890
|
-
</buffer>
|
891
877
|
])
|
892
|
-
assert_raise Fluent::UnrecoverableError do
|
878
|
+
e = assert_raise Fluent::UnrecoverableError do
|
893
879
|
d.instance_start
|
894
880
|
end
|
881
|
+
assert_match(/Connection refused/, e.message)
|
882
|
+
|
895
883
|
d.instance_shutdown
|
896
884
|
end
|
897
885
|
|
@@ -904,24 +892,47 @@ EOL
|
|
904
892
|
]
|
905
893
|
target_input_driver = create_target_input_driver(conf: input_conf)
|
906
894
|
output_conf = %[
|
907
|
-
|
908
|
-
|
895
|
+
transport tcp
|
896
|
+
verify_connection_at_startup true
|
897
|
+
<security>
|
898
|
+
self_hostname localhost
|
899
|
+
shared_key key_miss_match
|
900
|
+
</security>
|
901
|
+
|
902
|
+
<server>
|
903
|
+
host #{TARGET_HOST}
|
904
|
+
port #{TARGET_PORT}
|
905
|
+
</server>
|
906
|
+
]
|
907
|
+
@d = d = create_driver(output_conf)
|
908
|
+
|
909
|
+
target_input_driver.run(expect_records: 1, timeout: 1) do
|
910
|
+
e = assert_raise Fluent::UnrecoverableError do
|
911
|
+
d.instance_start
|
912
|
+
end
|
913
|
+
assert_match(/Failed to establish connection/, e.message)
|
914
|
+
end
|
915
|
+
end
|
916
|
+
|
917
|
+
test 'nodes_shared_key_miss_match with TLS' do
|
918
|
+
input_conf = TARGET_CONFIG + %[
|
919
|
+
<security>
|
920
|
+
self_hostname in.localhost
|
921
|
+
shared_key fluentd-sharedkey
|
922
|
+
</security>
|
923
|
+
<transport tls>
|
924
|
+
insecure true
|
925
|
+
</transport>
|
926
|
+
]
|
927
|
+
target_input_driver = create_target_input_driver(conf: input_conf)
|
928
|
+
output_conf = %[
|
909
929
|
transport tls
|
910
|
-
|
930
|
+
tls_insecure_mode true
|
911
931
|
verify_connection_at_startup true
|
912
|
-
require_ack_response true
|
913
|
-
ack_response_timeout 5s
|
914
932
|
<security>
|
915
933
|
self_hostname localhost
|
916
934
|
shared_key key_miss_match
|
917
935
|
</security>
|
918
|
-
<buffer tag>
|
919
|
-
flush_mode immediate
|
920
|
-
retry_type periodic
|
921
|
-
retry_wait 30s
|
922
|
-
flush_at_shutdown false # suppress errors in d.instance_shutdown
|
923
|
-
flush_thread_interval 31s
|
924
|
-
</buffer>
|
925
936
|
|
926
937
|
<server>
|
927
938
|
host #{TARGET_HOST}
|
@@ -930,11 +941,12 @@ EOL
|
|
930
941
|
]
|
931
942
|
@d = d = create_driver(output_conf)
|
932
943
|
|
933
|
-
target_input_driver.run(expect_records: 1, timeout:
|
934
|
-
assert_raise Fluent::UnrecoverableError do
|
944
|
+
target_input_driver.run(expect_records: 1, timeout: 1) do
|
945
|
+
e = assert_raise Fluent::UnrecoverableError do
|
935
946
|
d.instance_start
|
936
947
|
end
|
937
|
-
|
948
|
+
|
949
|
+
assert_match(/Failed to establish connection/, e.message)
|
938
950
|
end
|
939
951
|
end
|
940
952
|
|
@@ -943,15 +955,10 @@ EOL
|
|
943
955
|
<security>
|
944
956
|
self_hostname in.localhost
|
945
957
|
shared_key fluentd-sharedkey
|
946
|
-
<client>
|
947
|
-
host 127.0.0.1
|
948
|
-
</client>
|
949
958
|
</security>
|
950
959
|
]
|
951
960
|
target_input_driver = create_target_input_driver(conf: input_conf)
|
952
|
-
|
953
961
|
output_conf = %[
|
954
|
-
send_timeout 51
|
955
962
|
verify_connection_at_startup true
|
956
963
|
<security>
|
957
964
|
self_hostname localhost
|
@@ -961,18 +968,14 @@ EOL
|
|
961
968
|
name test
|
962
969
|
host #{TARGET_HOST}
|
963
970
|
port #{TARGET_PORT}
|
964
|
-
shared_key fluentd-sharedkey
|
965
971
|
</server>
|
966
972
|
]
|
967
973
|
@d = d = create_driver(output_conf)
|
968
974
|
|
969
975
|
time = event_time("2011-01-02 13:14:15 UTC")
|
970
|
-
records = [
|
971
|
-
{"a" => 1},
|
972
|
-
{"a" => 2}
|
973
|
-
]
|
976
|
+
records = [{ "a" => 1 }, { "a" => 2 }]
|
974
977
|
|
975
|
-
target_input_driver.run(expect_records: 2, timeout:
|
978
|
+
target_input_driver.run(expect_records: 2, timeout: 3) do
|
976
979
|
d.run(default_tag: 'test') do
|
977
980
|
records.each do |record|
|
978
981
|
d.feed(time, record)
|
@@ -981,7 +984,7 @@ EOL
|
|
981
984
|
end
|
982
985
|
|
983
986
|
events = target_input_driver.events
|
984
|
-
|
987
|
+
assert_false events.empty?
|
985
988
|
assert_equal(['test', time, records[0]], events[0])
|
986
989
|
assert_equal(['test', time, records[1]], events[1])
|
987
990
|
end
|
@@ -995,7 +998,7 @@ EOL
|
|
995
998
|
|
996
999
|
begin
|
997
1000
|
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
|
998
|
-
mock.proxy(d.instance).
|
1001
|
+
mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, TARGET_PORT, anything) { |sock| mock(sock).close.once; sock }.twice
|
999
1002
|
|
1000
1003
|
target_input_driver.run(timeout: 15) do
|
1001
1004
|
d.run(shutdown: false) do
|
@@ -1036,7 +1039,7 @@ EOL
|
|
1036
1039
|
|
1037
1040
|
begin
|
1038
1041
|
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
|
1039
|
-
mock.proxy(d.instance).
|
1042
|
+
mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, TARGET_PORT, anything) { |sock| mock(sock).close.once; sock }.once
|
1040
1043
|
|
1041
1044
|
target_input_driver.run(timeout: 15) do
|
1042
1045
|
d.run(shutdown: false) do
|
@@ -1052,7 +1055,7 @@ EOL
|
|
1052
1055
|
end
|
1053
1056
|
|
1054
1057
|
sub_test_case 'with require_ack_response' do
|
1055
|
-
test '
|
1058
|
+
test 'Create connection per send_data' do
|
1056
1059
|
target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
|
1057
1060
|
output_conf = CONFIG + %[
|
1058
1061
|
require_ack_response true
|
@@ -1064,7 +1067,7 @@ EOL
|
|
1064
1067
|
|
1065
1068
|
begin
|
1066
1069
|
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
|
1067
|
-
mock.proxy(d.instance).
|
1070
|
+
mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, TARGET_PORT, anything) { |sock| mock(sock).close.once; sock }.twice
|
1068
1071
|
|
1069
1072
|
target_input_driver.run(timeout: 15) do
|
1070
1073
|
d.run(shutdown: false) do
|
@@ -1080,133 +1083,4 @@ EOL
|
|
1080
1083
|
end
|
1081
1084
|
end
|
1082
1085
|
end
|
1083
|
-
|
1084
|
-
sub_test_case 'SocketCache' do
|
1085
|
-
sub_test_case 'fetch_or' do
|
1086
|
-
test 'when gived key does not exist' do
|
1087
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1088
|
-
sock = mock!.open { 1 }.subject
|
1089
|
-
assert_equal(1, c.fetch_or { sock.open })
|
1090
|
-
end
|
1091
|
-
|
1092
|
-
test 'when given key exists' do
|
1093
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1094
|
-
assert_equal(1, c.fetch_or { 1 })
|
1095
|
-
|
1096
|
-
sock = dont_allow(mock!).open
|
1097
|
-
assert_equal(1, c.fetch_or { sock.open })
|
1098
|
-
end
|
1099
|
-
|
1100
|
-
test "when given key's value was expired" do
|
1101
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(0, Logger.new(nil))
|
1102
|
-
assert_equal(1, c.fetch_or { 1 })
|
1103
|
-
|
1104
|
-
sock = mock!.open { 1 }.subject
|
1105
|
-
assert_equal(1, c.fetch_or { sock.open })
|
1106
|
-
end
|
1107
|
-
end
|
1108
|
-
|
1109
|
-
test 'revoke' do
|
1110
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1111
|
-
c.fetch_or { 1 }
|
1112
|
-
c.revoke
|
1113
|
-
|
1114
|
-
sock = mock!.open { 1 }.subject
|
1115
|
-
assert_equal(1, c.fetch_or { sock.open })
|
1116
|
-
end
|
1117
|
-
|
1118
|
-
test 'revoke_by_value' do
|
1119
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1120
|
-
c.fetch_or { 1 }
|
1121
|
-
c.revoke_by_value(1)
|
1122
|
-
|
1123
|
-
sock = mock!.open { 1 }.subject
|
1124
|
-
assert_equal(1, c.fetch_or { sock.open })
|
1125
|
-
end
|
1126
|
-
|
1127
|
-
sub_test_case 'dec_ref' do
|
1128
|
-
test 'when value exists in active_socks' do
|
1129
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1130
|
-
c.fetch_or { 1 }
|
1131
|
-
c.dec_ref
|
1132
|
-
|
1133
|
-
assert_equal(0, c.instance_variable_get(:@active_socks)[Thread.current.object_id].ref)
|
1134
|
-
end
|
1135
|
-
|
1136
|
-
test 'when value exists in inactive_socks' do
|
1137
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1138
|
-
c.fetch_or { 1 }
|
1139
|
-
c.revoke
|
1140
|
-
c.dec_ref
|
1141
|
-
assert_equal(-1, c.instance_variable_get(:@inactive_socks)[Thread.current.object_id].ref)
|
1142
|
-
end
|
1143
|
-
end
|
1144
|
-
|
1145
|
-
sub_test_case 'dec_ref_by_value' do
|
1146
|
-
test 'when value exists in active_socks' do
|
1147
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1148
|
-
c.fetch_or { 1 }
|
1149
|
-
c.dec_ref_by_value(1)
|
1150
|
-
|
1151
|
-
assert_equal(0, c.instance_variable_get(:@active_socks)[Thread.current.object_id].ref)
|
1152
|
-
end
|
1153
|
-
|
1154
|
-
test 'when value exists in inactive_socks' do
|
1155
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1156
|
-
c.fetch_or { 1 }
|
1157
|
-
c.revoke
|
1158
|
-
c.dec_ref_by_value(1)
|
1159
|
-
assert_equal(-1, c.instance_variable_get(:@inactive_socks)[Thread.current.object_id].ref)
|
1160
|
-
end
|
1161
|
-
end
|
1162
|
-
|
1163
|
-
sub_test_case 'clear' do
|
1164
|
-
test 'when value is in active_socks' do
|
1165
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1166
|
-
m = mock!.close { 'closed' }.subject
|
1167
|
-
c.fetch_or { m }
|
1168
|
-
assert_true(!c.instance_variable_get(:@active_socks).empty?)
|
1169
|
-
|
1170
|
-
c.clear
|
1171
|
-
assert_true(c.instance_variable_get(:@active_socks).empty?)
|
1172
|
-
end
|
1173
|
-
|
1174
|
-
test 'when value is in inactive_socks' do
|
1175
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1176
|
-
m = mock!.close { 'closed' }.subject
|
1177
|
-
c.fetch_or { m }
|
1178
|
-
c.revoke
|
1179
|
-
assert_true(!c.instance_variable_get(:@inactive_socks).empty?)
|
1180
|
-
|
1181
|
-
c.clear
|
1182
|
-
assert_true(c.instance_variable_get(:@active_socks).empty?)
|
1183
|
-
end
|
1184
|
-
end
|
1185
|
-
|
1186
|
-
sub_test_case 'purge_obsolete_socks' do
|
1187
|
-
test 'delete key in inactive_socks' do
|
1188
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1189
|
-
m = mock!.close { 'closed' }.subject
|
1190
|
-
c.fetch_or { m }
|
1191
|
-
c.revoke
|
1192
|
-
assert_true(!c.instance_variable_get(:@inactive_socks).empty?)
|
1193
|
-
|
1194
|
-
c.purge_obsolete_socks
|
1195
|
-
assert_true(c.instance_variable_get(:@active_socks).empty?)
|
1196
|
-
end
|
1197
|
-
|
1198
|
-
test 'move key from active_socks to inactive_socks' do
|
1199
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1200
|
-
m = dont_allow(mock!).close
|
1201
|
-
stub(m).inspect # for log
|
1202
|
-
c.fetch_or { m }
|
1203
|
-
assert_true(!c.instance_variable_get(:@active_socks).empty?)
|
1204
|
-
assert_true(c.instance_variable_get(:@inactive_socks).empty?)
|
1205
|
-
|
1206
|
-
c.purge_obsolete_socks
|
1207
|
-
assert_true(!c.instance_variable_get(:@active_socks).empty?)
|
1208
|
-
assert_true(c.instance_variable_get(:@inactive_socks).empty?)
|
1209
|
-
end
|
1210
|
-
end
|
1211
|
-
end
|
1212
1086
|
end
|