fluentd 1.6.3 → 1.7.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.drone.yml +35 -0
- data/.github/ISSUE_TEMPLATE/bug_report.md +2 -0
- data/CHANGELOG.md +83 -0
- data/README.md +5 -1
- data/fluentd.gemspec +3 -2
- data/lib/fluent/clock.rb +4 -0
- data/lib/fluent/compat/output.rb +3 -3
- data/lib/fluent/compat/socket_util.rb +1 -1
- data/lib/fluent/config/element.rb +3 -3
- data/lib/fluent/config/literal_parser.rb +1 -1
- data/lib/fluent/config/section.rb +4 -1
- data/lib/fluent/error.rb +4 -0
- data/lib/fluent/event.rb +28 -24
- data/lib/fluent/event_router.rb +2 -1
- data/lib/fluent/log.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +8 -0
- data/lib/fluent/plugin/bare_output.rb +4 -4
- data/lib/fluent/plugin/buf_file.rb +10 -1
- data/lib/fluent/plugin/buf_file_single.rb +219 -0
- data/lib/fluent/plugin/buffer.rb +62 -63
- data/lib/fluent/plugin/buffer/chunk.rb +21 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +44 -12
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +314 -0
- data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -1
- data/lib/fluent/plugin/compressable.rb +10 -6
- data/lib/fluent/plugin/filter_grep.rb +2 -2
- data/lib/fluent/plugin/formatter_csv.rb +10 -6
- data/lib/fluent/plugin/in_syslog.rb +10 -3
- data/lib/fluent/plugin/in_tail.rb +7 -2
- data/lib/fluent/plugin/in_tcp.rb +34 -7
- data/lib/fluent/plugin/multi_output.rb +4 -4
- data/lib/fluent/plugin/out_exec_filter.rb +1 -0
- data/lib/fluent/plugin/out_file.rb +13 -3
- data/lib/fluent/plugin/out_forward.rb +144 -588
- data/lib/fluent/plugin/out_forward/ack_handler.rb +161 -0
- data/lib/fluent/plugin/out_forward/connection_manager.rb +113 -0
- data/lib/fluent/plugin/out_forward/error.rb +28 -0
- data/lib/fluent/plugin/out_forward/failure_detector.rb +84 -0
- data/lib/fluent/plugin/out_forward/handshake_protocol.rb +121 -0
- data/lib/fluent/plugin/out_forward/load_balancer.rb +111 -0
- data/lib/fluent/plugin/out_forward/socket_cache.rb +138 -0
- data/lib/fluent/plugin/out_http.rb +231 -0
- data/lib/fluent/plugin/output.rb +29 -35
- data/lib/fluent/plugin/parser.rb +77 -0
- data/lib/fluent/plugin/parser_csv.rb +75 -0
- data/lib/fluent/plugin/parser_syslog.rb +106 -3
- data/lib/fluent/plugin_helper/server.rb +2 -2
- data/lib/fluent/plugin_helper/socket.rb +14 -1
- data/lib/fluent/plugin_helper/thread.rb +1 -0
- data/lib/fluent/root_agent.rb +1 -1
- data/lib/fluent/time.rb +4 -2
- data/lib/fluent/timezone.rb +21 -7
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_fluentd.rb +1 -1
- data/test/command/test_plugin_generator.rb +18 -2
- data/test/config/test_configurable.rb +78 -40
- data/test/counter/test_store.rb +1 -1
- data/test/helper.rb +1 -0
- data/test/helpers/process_extenstion.rb +33 -0
- data/test/plugin/out_forward/test_ack_handler.rb +101 -0
- data/test/plugin/out_forward/test_connection_manager.rb +145 -0
- data/test/plugin/out_forward/test_handshake_protocol.rb +103 -0
- data/test/plugin/out_forward/test_load_balancer.rb +60 -0
- data/test/plugin/out_forward/test_socket_cache.rb +139 -0
- data/test/plugin/test_buf_file.rb +172 -2
- data/test/plugin/test_buf_file_single.rb +801 -0
- data/test/plugin/test_buffer.rb +4 -48
- data/test/plugin/test_buffer_file_chunk.rb +38 -1
- data/test/plugin/test_buffer_file_single_chunk.rb +621 -0
- data/test/plugin/test_buffer_memory_chunk.rb +1 -0
- data/test/plugin/test_formatter_csv.rb +16 -0
- data/test/plugin/test_in_syslog.rb +56 -6
- data/test/plugin/test_in_tail.rb +1 -1
- data/test/plugin/test_in_tcp.rb +25 -0
- data/test/plugin/test_out_forward.rb +150 -201
- data/test/plugin/test_out_http.rb +352 -0
- data/test/plugin/test_output_as_buffered.rb +27 -24
- data/test/plugin/test_parser.rb +40 -0
- data/test/plugin/test_parser_csv.rb +83 -0
- data/test/plugin/test_parser_syslog.rb +118 -19
- data/test/plugin_helper/test_record_accessor.rb +1 -1
- data/test/test_time_formatter.rb +140 -121
- metadata +35 -6
@@ -331,6 +331,7 @@ class BufferMemoryChunkTest < Test::Unit::TestCase
|
|
331
331
|
assert_equal @gzipped_src, c.read(compressed: :gzip)
|
332
332
|
|
333
333
|
io = StringIO.new
|
334
|
+
io.set_encoding(Encoding::ASCII_8BIT)
|
334
335
|
c.write_to(io, compressed: :gzip)
|
335
336
|
assert_equal @gzipped_src, io.string
|
336
337
|
end
|
@@ -108,4 +108,20 @@ class CsvFormatterTest < ::Test::Unit::TestCase
|
|
108
108
|
d = create_driver('fields' => data)
|
109
109
|
assert_equal %w(one two three), d.instance.fields
|
110
110
|
end
|
111
|
+
|
112
|
+
def test_format_with_multiple_records
|
113
|
+
d = create_driver("fields" => "message,message2")
|
114
|
+
r = {'message' => 'hello', 'message2' => 'fluentd'}
|
115
|
+
|
116
|
+
formatted = d.instance.format(tag, @time, r)
|
117
|
+
assert_equal("\"hello\",\"fluentd\"\n", formatted)
|
118
|
+
|
119
|
+
r = {'message' => 'hey', 'message2' => 'ho'}
|
120
|
+
formatted = d.instance.format(tag, @time, r)
|
121
|
+
assert_equal("\"hey\",\"ho\"\n", formatted)
|
122
|
+
|
123
|
+
r = {'message' => 'longer message', 'message2' => 'longer longer message'}
|
124
|
+
formatted = d.instance.format(tag, @time, r)
|
125
|
+
assert_equal("\"longer message\",\"longer longer message\"\n", formatted)
|
126
|
+
end
|
111
127
|
end
|
@@ -362,9 +362,8 @@ EOS
|
|
362
362
|
end
|
363
363
|
end
|
364
364
|
|
365
|
-
def
|
366
|
-
|
367
|
-
tests = [
|
365
|
+
def create_unmatched_lines_test_case
|
366
|
+
[
|
368
367
|
# valid message
|
369
368
|
{'msg' => '<6>Sep 10 00:00:00 localhost logger: xxx', 'expected' => {'host'=>'localhost', 'ident'=>'logger', 'message'=>'xxx'}},
|
370
369
|
# missing priority
|
@@ -372,6 +371,22 @@ EOS
|
|
372
371
|
# timestamp parsing failure
|
373
372
|
{'msg' => '<6>ZZZ 99 99:99:99 localhost logger: xxx', 'expected' => {'unmatched_line' => '<6>ZZZ 99 99:99:99 localhost logger: xxx'}},
|
374
373
|
]
|
374
|
+
end
|
375
|
+
|
376
|
+
def compare_unmatched_lines_test_result(events, tests, options = {})
|
377
|
+
events.each_index { |i|
|
378
|
+
tests[i]['expected'].each { |k,v|
|
379
|
+
assert_equal v, events[i][2][k], "No key <#{k}> in response or value mismatch"
|
380
|
+
}
|
381
|
+
assert_equal('syslog.unmatched', events[i][0], 'tag does not match syslog.unmatched') unless i==0
|
382
|
+
assert_equal(options[:address], events[i][2]['source_address'], 'response has no source_address or mismatch') if options[:address]
|
383
|
+
assert_equal(options[:hostname], events[i][2]['source_hostname'], 'response has no source_hostname or mismatch') if options[:hostname]
|
384
|
+
}
|
385
|
+
end
|
386
|
+
|
387
|
+
def test_emit_unmatched_lines
|
388
|
+
d = create_driver([CONFIG, 'emit_unmatched_lines true'].join("\n"))
|
389
|
+
tests = create_unmatched_lines_test_case
|
375
390
|
|
376
391
|
d.run(expect_emits: 3) do
|
377
392
|
u = UDPSocket.new
|
@@ -383,9 +398,44 @@ EOS
|
|
383
398
|
end
|
384
399
|
|
385
400
|
assert_equal tests.size, d.events.size
|
386
|
-
|
387
|
-
|
388
|
-
|
401
|
+
compare_unmatched_lines_test_result(d.events, tests)
|
402
|
+
end
|
403
|
+
|
404
|
+
def test_emit_unmatched_lines_with_hostname
|
405
|
+
d = create_driver([CONFIG, 'emit_unmatched_lines true', 'source_hostname_key source_hostname'].join("\n"))
|
406
|
+
tests = create_unmatched_lines_test_case
|
407
|
+
|
408
|
+
hostname = nil
|
409
|
+
d.run(expect_emits: 3) do
|
410
|
+
u = UDPSocket.new
|
411
|
+
u.do_not_reverse_lookup = false
|
412
|
+
u.connect('127.0.0.1', PORT)
|
413
|
+
hostname = u.peeraddr[2]
|
414
|
+
tests.each {|test|
|
415
|
+
u.send(test['msg'], 0)
|
416
|
+
}
|
417
|
+
end
|
418
|
+
|
419
|
+
assert_equal tests.size, d.events.size
|
420
|
+
compare_unmatched_lines_test_result(d.events, tests, {hostname: hostname})
|
421
|
+
end
|
422
|
+
|
423
|
+
def test_emit_unmatched_lines_with_address
|
424
|
+
d = create_driver([CONFIG, 'emit_unmatched_lines true', 'source_address_key source_address'].join("\n"))
|
425
|
+
tests = create_unmatched_lines_test_case
|
426
|
+
|
427
|
+
address = nil
|
428
|
+
d.run(expect_emits: 3) do
|
429
|
+
u = UDPSocket.new
|
430
|
+
u.do_not_reverse_lookup = false
|
431
|
+
u.connect('127.0.0.1', PORT)
|
432
|
+
address = u.peeraddr[3]
|
433
|
+
tests.each {|test|
|
434
|
+
u.send(test['msg'], 0)
|
435
|
+
}
|
389
436
|
end
|
437
|
+
|
438
|
+
assert_equal tests.size, d.events.size
|
439
|
+
compare_unmatched_lines_test_result(d.events, tests, {address: address})
|
390
440
|
end
|
391
441
|
end
|
data/test/plugin/test_in_tail.rb
CHANGED
@@ -1011,7 +1011,7 @@ class TailInputTest < Test::Unit::TestCase
|
|
1011
1011
|
waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 0 }
|
1012
1012
|
|
1013
1013
|
# Previous implementation has an infinite watcher creation bug.
|
1014
|
-
# Following code checks such unexpected bug by
|
1014
|
+
# Following code checks such unexpected bug by counting actual object allocation.
|
1015
1015
|
base_num = count_timer_object
|
1016
1016
|
2.times {
|
1017
1017
|
sleep 1
|
data/test/plugin/test_in_tcp.rb
CHANGED
@@ -162,4 +162,29 @@ class TcpInputTest < Test::Unit::TestCase
|
|
162
162
|
assert event[1].is_a?(Fluent::EventTime)
|
163
163
|
assert_equal address, event[2]['addr']
|
164
164
|
end
|
165
|
+
|
166
|
+
sub_test_case '<extract>' do
|
167
|
+
test 'extract tag from record field' do
|
168
|
+
d = create_driver(BASE_CONFIG + %!
|
169
|
+
<parse>
|
170
|
+
@type json
|
171
|
+
</parse>
|
172
|
+
<extract>
|
173
|
+
tag_key tag
|
174
|
+
</extract>
|
175
|
+
!)
|
176
|
+
d.run(expect_records: 1) do
|
177
|
+
create_tcp_socket('127.0.0.1', PORT) do |sock|
|
178
|
+
data = {'msg' => 'hello', 'tag' => 'helper_test'}
|
179
|
+
sock.send("#{data.to_json}\n", 0)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
assert_equal 1, d.events.size
|
184
|
+
event = d.events[0]
|
185
|
+
assert_equal 'helper_test', event[0]
|
186
|
+
assert event[1].is_a?(Fluent::EventTime)
|
187
|
+
assert_equal 'hello', event[2]['msg']
|
188
|
+
end
|
189
|
+
end
|
165
190
|
end
|
@@ -39,13 +39,11 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
39
39
|
|
40
40
|
def create_driver(conf=CONFIG)
|
41
41
|
Fluent::Test::Driver::Output.new(Fluent::Plugin::ForwardOutput) {
|
42
|
-
attr_reader :
|
42
|
+
attr_reader :sent_chunk_ids, :ack_handler
|
43
43
|
|
44
44
|
def initialize
|
45
45
|
super
|
46
46
|
@sent_chunk_ids = []
|
47
|
-
@response_chunk_ids = []
|
48
|
-
@exceptions = []
|
49
47
|
end
|
50
48
|
|
51
49
|
def try_write(chunk)
|
@@ -53,15 +51,6 @@ class ForwardOutputTest < Test::Unit::TestCase
|
|
53
51
|
@sent_chunk_ids << chunk.unique_id
|
54
52
|
retval
|
55
53
|
end
|
56
|
-
|
57
|
-
def read_ack_from_sock(sock, unpacker)
|
58
|
-
retval = super
|
59
|
-
@response_chunk_ids << retval
|
60
|
-
retval
|
61
|
-
rescue => e
|
62
|
-
@exceptions << e
|
63
|
-
raise e
|
64
|
-
end
|
65
54
|
}.configure(conf)
|
66
55
|
end
|
67
56
|
|
@@ -196,6 +185,81 @@ EOL
|
|
196
185
|
assert_equal([dummy_cert_path], d.instance.tls_ca_cert_path)
|
197
186
|
end
|
198
187
|
|
188
|
+
sub_test_case "certstore loading parameters for Windows" do
|
189
|
+
test 'certstore related config parameters' do
|
190
|
+
omit "certstore related values raise error on not Windows" if Fluent.windows?
|
191
|
+
conf = %[
|
192
|
+
send_timeout 5
|
193
|
+
transport tls
|
194
|
+
tls_cert_logical_store_name Root
|
195
|
+
tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b
|
196
|
+
<server>
|
197
|
+
host #{TARGET_HOST}
|
198
|
+
port #{TARGET_PORT}
|
199
|
+
</server>
|
200
|
+
]
|
201
|
+
|
202
|
+
assert_raise(Fluent::ConfigError) do
|
203
|
+
create_driver(conf)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
test 'cert_logical_store_name and tls_cert_thumbprint default values' do
|
208
|
+
conf = %[
|
209
|
+
send_timeout 5
|
210
|
+
transport tls
|
211
|
+
<server>
|
212
|
+
host #{TARGET_HOST}
|
213
|
+
port #{TARGET_PORT}
|
214
|
+
</server>
|
215
|
+
]
|
216
|
+
|
217
|
+
@d = d = create_driver(conf)
|
218
|
+
assert_nil d.instance.tls_cert_logical_store_name
|
219
|
+
assert_nil d.instance.tls_cert_thumbprint
|
220
|
+
end
|
221
|
+
|
222
|
+
data('CA cert' => 'tls_ca_cert_path',
|
223
|
+
'non CA cert' => 'tls_cert_path')
|
224
|
+
test 'specify tls_cert_logical_store_name and tls_cert_path should raise error' do |param|
|
225
|
+
omit "Loading CertStore feature works only Windows" unless Fluent.windows?
|
226
|
+
dummy_cert_path = File.join(TMP_DIR, "dummy_cert.pem")
|
227
|
+
FileUtils.touch(dummy_cert_path)
|
228
|
+
conf = %[
|
229
|
+
send_timeout 5
|
230
|
+
transport tls
|
231
|
+
#{param} #{dummy_cert_path}
|
232
|
+
tls_cert_logical_store_name Root
|
233
|
+
<server>
|
234
|
+
host #{TARGET_HOST}
|
235
|
+
port #{TARGET_PORT}
|
236
|
+
</server>
|
237
|
+
]
|
238
|
+
|
239
|
+
assert_raise(Fluent::ConfigError) do
|
240
|
+
create_driver(conf)
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
test 'configure cert_logical_store_name and tls_cert_thumbprint' do
|
245
|
+
omit "Loading CertStore feature works only Windows" unless Fluent.windows?
|
246
|
+
conf = %[
|
247
|
+
send_timeout 5
|
248
|
+
transport tls
|
249
|
+
tls_cert_logical_store_name Root
|
250
|
+
tls_cert_thumbprint a909502dd82ae41433e6f83886b00d4277a32a7b
|
251
|
+
<server>
|
252
|
+
host #{TARGET_HOST}
|
253
|
+
port #{TARGET_PORT}
|
254
|
+
</server>
|
255
|
+
]
|
256
|
+
|
257
|
+
@d = d = create_driver(conf)
|
258
|
+
assert_equal "Root", d.instance.tls_cert_logical_store_name
|
259
|
+
assert_equal "a909502dd82ae41433e6f83886b00d4277a32a7b", d.instance.tls_cert_thumbprint
|
260
|
+
end
|
261
|
+
end
|
262
|
+
|
199
263
|
test 'compress_default_value' do
|
200
264
|
@d = d = create_driver
|
201
265
|
assert_equal :text, d.instance.compress
|
@@ -235,14 +299,14 @@ EOL
|
|
235
299
|
node = d.instance.nodes.first
|
236
300
|
stub(node.failure).phi { raise 'Should not be called' }
|
237
301
|
node.tick
|
238
|
-
|
302
|
+
assert_true node.available?
|
239
303
|
end
|
240
304
|
|
241
305
|
test 'phi_failure_detector enabled' do
|
242
306
|
@d = d = create_driver(CONFIG + %[phi_failure_detector true \n phi_threshold 0])
|
243
307
|
node = d.instance.nodes.first
|
244
308
|
node.tick
|
245
|
-
|
309
|
+
assert_false node.available?
|
246
310
|
end
|
247
311
|
|
248
312
|
test 'require_ack_response is disabled in default' do
|
@@ -287,6 +351,7 @@ EOL
|
|
287
351
|
[tag_in_ascii, time, {"a" => 2}],
|
288
352
|
]
|
289
353
|
|
354
|
+
stub(d.instance.ack_handler).read_ack_from_sock(anything).never
|
290
355
|
target_input_driver.run(expect_records: 2) do
|
291
356
|
d.run do
|
292
357
|
emit_events.each do |tag, t, record|
|
@@ -302,8 +367,6 @@ EOL
|
|
302
367
|
assert_equal_event_time(time, events[1][1])
|
303
368
|
assert_equal ['test.ascii', time, emit_events[1][2]], events[1]
|
304
369
|
assert_equal Encoding::UTF_8, events[1][0].encoding
|
305
|
-
|
306
|
-
assert_empty d.instance.exceptions
|
307
370
|
end
|
308
371
|
|
309
372
|
test 'send_with_time_as_integer' do
|
@@ -317,6 +380,8 @@ EOL
|
|
317
380
|
{"a" => 1},
|
318
381
|
{"a" => 2}
|
319
382
|
]
|
383
|
+
|
384
|
+
stub(d.instance.ack_handler).read_ack_from_sock(anything).never
|
320
385
|
target_input_driver.run(expect_records: 2) do
|
321
386
|
d.run(default_tag: 'test') do
|
322
387
|
records.each do |record|
|
@@ -330,8 +395,6 @@ EOL
|
|
330
395
|
assert_equal ['test', time, records[0]], events[0]
|
331
396
|
assert_equal_event_time(time, events[1][1])
|
332
397
|
assert_equal ['test', time, records[1]], events[1]
|
333
|
-
|
334
|
-
assert_empty d.instance.exceptions
|
335
398
|
end
|
336
399
|
|
337
400
|
test 'send_without_time_as_integer' do
|
@@ -348,6 +411,7 @@ EOL
|
|
348
411
|
{"a" => 1},
|
349
412
|
{"a" => 2}
|
350
413
|
]
|
414
|
+
stub(d.instance.ack_handler).read_ack_from_sock(anything).never
|
351
415
|
target_input_driver.run(expect_records: 2) do
|
352
416
|
d.run(default_tag: 'test') do
|
353
417
|
records.each do |record|
|
@@ -361,8 +425,6 @@ EOL
|
|
361
425
|
assert_equal ['test', time, records[0]], events[0]
|
362
426
|
assert_equal_event_time(time, events[1][1])
|
363
427
|
assert_equal ['test', time, records[1]], events[1]
|
364
|
-
|
365
|
-
assert_empty d.instance.exceptions
|
366
428
|
end
|
367
429
|
|
368
430
|
test 'send_comprssed_message_pack_stream_if_compress_is_gzip' do
|
@@ -406,6 +468,8 @@ EOL
|
|
406
468
|
{"a" => 1},
|
407
469
|
{"a" => 2}
|
408
470
|
]
|
471
|
+
# not attempt to receive responses
|
472
|
+
stub(d.instance.ack_handler).read_ack_from_sock(anything).never
|
409
473
|
target_input_driver.run(expect_records: 2) do
|
410
474
|
d.run(default_tag: 'test') do
|
411
475
|
records.each do |record|
|
@@ -417,9 +481,6 @@ EOL
|
|
417
481
|
events = target_input_driver.events
|
418
482
|
assert_equal ['test', time, records[0]], events[0]
|
419
483
|
assert_equal ['test', time, records[1]], events[1]
|
420
|
-
|
421
|
-
assert_empty d.instance.response_chunk_ids # not attempt to receive responses, so it's empty
|
422
|
-
assert_empty d.instance.exceptions
|
423
484
|
end
|
424
485
|
|
425
486
|
test 'send_to_a_node_not_supporting_responses' do
|
@@ -433,6 +494,8 @@ EOL
|
|
433
494
|
{"a" => 1},
|
434
495
|
{"a" => 2}
|
435
496
|
]
|
497
|
+
# not attempt to receive responses
|
498
|
+
stub(d.instance.ack_handler).read_ack_from_sock(anything).never
|
436
499
|
target_input_driver.run(expect_records: 2) do
|
437
500
|
d.run(default_tag: 'test') do
|
438
501
|
records.each do |record|
|
@@ -444,9 +507,6 @@ EOL
|
|
444
507
|
events = target_input_driver.events
|
445
508
|
assert_equal ['test', time, records[0]], events[0]
|
446
509
|
assert_equal ['test', time, records[1]], events[1]
|
447
|
-
|
448
|
-
assert_empty d.instance.response_chunk_ids # not attempt to receive responses, so it's empty
|
449
|
-
assert_empty d.instance.exceptions
|
450
510
|
end
|
451
511
|
|
452
512
|
test 'a node supporting responses' do
|
@@ -465,12 +525,20 @@ EOL
|
|
465
525
|
|
466
526
|
time = event_time("2011-01-02 13:14:15 UTC")
|
467
527
|
|
528
|
+
acked_chunk_ids = []
|
529
|
+
mock.proxy(d.instance.ack_handler).read_ack_from_sock(anything) do |info, success|
|
530
|
+
if success
|
531
|
+
acked_chunk_ids << info.chunk_id
|
532
|
+
end
|
533
|
+
[chunk_id, success]
|
534
|
+
end
|
535
|
+
|
468
536
|
records = [
|
469
537
|
{"a" => 1},
|
470
538
|
{"a" => 2}
|
471
539
|
]
|
472
540
|
target_input_driver.run(expect_records: 2) do
|
473
|
-
d.end_if{
|
541
|
+
d.end_if { acked_chunk_ids.size > 0 }
|
474
542
|
d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
|
475
543
|
d.feed([[time, records[0]], [time,records[1]]])
|
476
544
|
end
|
@@ -480,9 +548,8 @@ EOL
|
|
480
548
|
assert_equal ['test', time, records[0]], events[0]
|
481
549
|
assert_equal ['test', time, records[1]], events[1]
|
482
550
|
|
483
|
-
assert_equal 1,
|
484
|
-
assert_equal d.instance.sent_chunk_ids.first,
|
485
|
-
assert_empty d.instance.exceptions
|
551
|
+
assert_equal 1, acked_chunk_ids.size
|
552
|
+
assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first
|
486
553
|
end
|
487
554
|
|
488
555
|
data('ack true' => true,
|
@@ -555,7 +622,7 @@ EOL
|
|
555
622
|
{"a" => 2}
|
556
623
|
]
|
557
624
|
target_input_driver.end_if{ d.instance.rollback_count > 0 }
|
558
|
-
target_input_driver.end_if{ !node.available }
|
625
|
+
target_input_driver.end_if{ !node.available? }
|
559
626
|
target_input_driver.run(expect_records: 2, timeout: 25) do
|
560
627
|
d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do
|
561
628
|
delayed_commit_timeout_value = d.instance.delayed_commit_timeout
|
@@ -600,7 +667,7 @@ EOL
|
|
600
667
|
{"a" => 2}
|
601
668
|
]
|
602
669
|
target_input_driver.end_if{ d.instance.rollback_count > 0 }
|
603
|
-
target_input_driver.end_if{ !node.available }
|
670
|
+
target_input_driver.end_if{ !node.available? }
|
604
671
|
target_input_driver.run(expect_records: 2, timeout: 25) do
|
605
672
|
d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do
|
606
673
|
delayed_commit_timeout_value = d.instance.delayed_commit_timeout
|
@@ -840,7 +907,7 @@ EOL
|
|
840
907
|
|
841
908
|
stub(node.failure).phi { raise 'Should not be called' }
|
842
909
|
node.tick
|
843
|
-
|
910
|
+
assert_true node.available?
|
844
911
|
end
|
845
912
|
|
846
913
|
test 'heartbeat_type_udp' do
|
@@ -856,7 +923,7 @@ EOL
|
|
856
923
|
assert timers.include?(:out_forward_heartbeat_request)
|
857
924
|
|
858
925
|
mock(usock).send("\0", 0, Socket.pack_sockaddr_in(TARGET_PORT, '127.0.0.1')).once
|
859
|
-
d.instance.send(:
|
926
|
+
d.instance.send(:on_heartbeat_timer)
|
860
927
|
end
|
861
928
|
|
862
929
|
test 'acts_as_secondary' do
|
@@ -882,16 +949,12 @@ EOL
|
|
882
949
|
test 'nodes are not available' do
|
883
950
|
@d = d = create_driver(CONFIG + %[
|
884
951
|
verify_connection_at_startup true
|
885
|
-
<buffer tag>
|
886
|
-
flush_mode immediate
|
887
|
-
retry_type periodic
|
888
|
-
retry_wait 30s
|
889
|
-
flush_at_shutdown false # suppress errors in d.instance_shutdown
|
890
|
-
</buffer>
|
891
952
|
])
|
892
|
-
assert_raise Fluent::UnrecoverableError do
|
953
|
+
e = assert_raise Fluent::UnrecoverableError do
|
893
954
|
d.instance_start
|
894
955
|
end
|
956
|
+
assert_match(/Connection refused/, e.message)
|
957
|
+
|
895
958
|
d.instance_shutdown
|
896
959
|
end
|
897
960
|
|
@@ -904,24 +967,47 @@ EOL
|
|
904
967
|
]
|
905
968
|
target_input_driver = create_target_input_driver(conf: input_conf)
|
906
969
|
output_conf = %[
|
907
|
-
|
908
|
-
|
970
|
+
transport tcp
|
971
|
+
verify_connection_at_startup true
|
972
|
+
<security>
|
973
|
+
self_hostname localhost
|
974
|
+
shared_key key_miss_match
|
975
|
+
</security>
|
976
|
+
|
977
|
+
<server>
|
978
|
+
host #{TARGET_HOST}
|
979
|
+
port #{TARGET_PORT}
|
980
|
+
</server>
|
981
|
+
]
|
982
|
+
@d = d = create_driver(output_conf)
|
983
|
+
|
984
|
+
target_input_driver.run(expect_records: 1, timeout: 1) do
|
985
|
+
e = assert_raise Fluent::UnrecoverableError do
|
986
|
+
d.instance_start
|
987
|
+
end
|
988
|
+
assert_match(/Failed to establish connection/, e.message)
|
989
|
+
end
|
990
|
+
end
|
991
|
+
|
992
|
+
test 'nodes_shared_key_miss_match with TLS' do
|
993
|
+
input_conf = TARGET_CONFIG + %[
|
994
|
+
<security>
|
995
|
+
self_hostname in.localhost
|
996
|
+
shared_key fluentd-sharedkey
|
997
|
+
</security>
|
998
|
+
<transport tls>
|
999
|
+
insecure true
|
1000
|
+
</transport>
|
1001
|
+
]
|
1002
|
+
target_input_driver = create_target_input_driver(conf: input_conf)
|
1003
|
+
output_conf = %[
|
909
1004
|
transport tls
|
910
|
-
|
1005
|
+
tls_insecure_mode true
|
911
1006
|
verify_connection_at_startup true
|
912
|
-
require_ack_response true
|
913
|
-
ack_response_timeout 5s
|
914
1007
|
<security>
|
915
1008
|
self_hostname localhost
|
916
1009
|
shared_key key_miss_match
|
917
1010
|
</security>
|
918
|
-
<buffer tag>
|
919
|
-
flush_mode immediate
|
920
|
-
retry_type periodic
|
921
|
-
retry_wait 30s
|
922
|
-
flush_at_shutdown false # suppress errors in d.instance_shutdown
|
923
|
-
flush_thread_interval 31s
|
924
|
-
</buffer>
|
925
1011
|
|
926
1012
|
<server>
|
927
1013
|
host #{TARGET_HOST}
|
@@ -930,11 +1016,12 @@ EOL
|
|
930
1016
|
]
|
931
1017
|
@d = d = create_driver(output_conf)
|
932
1018
|
|
933
|
-
target_input_driver.run(expect_records: 1, timeout:
|
934
|
-
assert_raise Fluent::UnrecoverableError do
|
1019
|
+
target_input_driver.run(expect_records: 1, timeout: 1) do
|
1020
|
+
e = assert_raise Fluent::UnrecoverableError do
|
935
1021
|
d.instance_start
|
936
1022
|
end
|
937
|
-
|
1023
|
+
|
1024
|
+
assert_match(/Failed to establish connection/, e.message)
|
938
1025
|
end
|
939
1026
|
end
|
940
1027
|
|
@@ -943,15 +1030,10 @@ EOL
|
|
943
1030
|
<security>
|
944
1031
|
self_hostname in.localhost
|
945
1032
|
shared_key fluentd-sharedkey
|
946
|
-
<client>
|
947
|
-
host 127.0.0.1
|
948
|
-
</client>
|
949
1033
|
</security>
|
950
1034
|
]
|
951
1035
|
target_input_driver = create_target_input_driver(conf: input_conf)
|
952
|
-
|
953
1036
|
output_conf = %[
|
954
|
-
send_timeout 51
|
955
1037
|
verify_connection_at_startup true
|
956
1038
|
<security>
|
957
1039
|
self_hostname localhost
|
@@ -961,18 +1043,14 @@ EOL
|
|
961
1043
|
name test
|
962
1044
|
host #{TARGET_HOST}
|
963
1045
|
port #{TARGET_PORT}
|
964
|
-
shared_key fluentd-sharedkey
|
965
1046
|
</server>
|
966
1047
|
]
|
967
1048
|
@d = d = create_driver(output_conf)
|
968
1049
|
|
969
1050
|
time = event_time("2011-01-02 13:14:15 UTC")
|
970
|
-
records = [
|
971
|
-
{"a" => 1},
|
972
|
-
{"a" => 2}
|
973
|
-
]
|
1051
|
+
records = [{ "a" => 1 }, { "a" => 2 }]
|
974
1052
|
|
975
|
-
target_input_driver.run(expect_records: 2, timeout:
|
1053
|
+
target_input_driver.run(expect_records: 2, timeout: 3) do
|
976
1054
|
d.run(default_tag: 'test') do
|
977
1055
|
records.each do |record|
|
978
1056
|
d.feed(time, record)
|
@@ -981,7 +1059,7 @@ EOL
|
|
981
1059
|
end
|
982
1060
|
|
983
1061
|
events = target_input_driver.events
|
984
|
-
|
1062
|
+
assert_false events.empty?
|
985
1063
|
assert_equal(['test', time, records[0]], events[0])
|
986
1064
|
assert_equal(['test', time, records[1]], events[1])
|
987
1065
|
end
|
@@ -995,7 +1073,7 @@ EOL
|
|
995
1073
|
|
996
1074
|
begin
|
997
1075
|
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
|
998
|
-
mock.proxy(d.instance).
|
1076
|
+
mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, TARGET_PORT, anything) { |sock| mock(sock).close.once; sock }.twice
|
999
1077
|
|
1000
1078
|
target_input_driver.run(timeout: 15) do
|
1001
1079
|
d.run(shutdown: false) do
|
@@ -1036,7 +1114,7 @@ EOL
|
|
1036
1114
|
|
1037
1115
|
begin
|
1038
1116
|
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
|
1039
|
-
mock.proxy(d.instance).
|
1117
|
+
mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, TARGET_PORT, anything) { |sock| mock(sock).close.once; sock }.once
|
1040
1118
|
|
1041
1119
|
target_input_driver.run(timeout: 15) do
|
1042
1120
|
d.run(shutdown: false) do
|
@@ -1052,7 +1130,7 @@ EOL
|
|
1052
1130
|
end
|
1053
1131
|
|
1054
1132
|
sub_test_case 'with require_ack_response' do
|
1055
|
-
test '
|
1133
|
+
test 'Create connection per send_data' do
|
1056
1134
|
target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
|
1057
1135
|
output_conf = CONFIG + %[
|
1058
1136
|
require_ack_response true
|
@@ -1064,7 +1142,7 @@ EOL
|
|
1064
1142
|
|
1065
1143
|
begin
|
1066
1144
|
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
|
1067
|
-
mock.proxy(d.instance).
|
1145
|
+
mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, TARGET_PORT, anything) { |sock| mock(sock).close.once; sock }.twice
|
1068
1146
|
|
1069
1147
|
target_input_driver.run(timeout: 15) do
|
1070
1148
|
d.run(shutdown: false) do
|
@@ -1080,133 +1158,4 @@ EOL
|
|
1080
1158
|
end
|
1081
1159
|
end
|
1082
1160
|
end
|
1083
|
-
|
1084
|
-
sub_test_case 'SocketCache' do
|
1085
|
-
sub_test_case 'fetch_or' do
|
1086
|
-
test 'when gived key does not exist' do
|
1087
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1088
|
-
sock = mock!.open { 1 }.subject
|
1089
|
-
assert_equal(1, c.fetch_or { sock.open })
|
1090
|
-
end
|
1091
|
-
|
1092
|
-
test 'when given key exists' do
|
1093
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1094
|
-
assert_equal(1, c.fetch_or { 1 })
|
1095
|
-
|
1096
|
-
sock = dont_allow(mock!).open
|
1097
|
-
assert_equal(1, c.fetch_or { sock.open })
|
1098
|
-
end
|
1099
|
-
|
1100
|
-
test "when given key's value was expired" do
|
1101
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(0, Logger.new(nil))
|
1102
|
-
assert_equal(1, c.fetch_or { 1 })
|
1103
|
-
|
1104
|
-
sock = mock!.open { 1 }.subject
|
1105
|
-
assert_equal(1, c.fetch_or { sock.open })
|
1106
|
-
end
|
1107
|
-
end
|
1108
|
-
|
1109
|
-
test 'revoke' do
|
1110
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1111
|
-
c.fetch_or { 1 }
|
1112
|
-
c.revoke
|
1113
|
-
|
1114
|
-
sock = mock!.open { 1 }.subject
|
1115
|
-
assert_equal(1, c.fetch_or { sock.open })
|
1116
|
-
end
|
1117
|
-
|
1118
|
-
test 'revoke_by_value' do
|
1119
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1120
|
-
c.fetch_or { 1 }
|
1121
|
-
c.revoke_by_value(1)
|
1122
|
-
|
1123
|
-
sock = mock!.open { 1 }.subject
|
1124
|
-
assert_equal(1, c.fetch_or { sock.open })
|
1125
|
-
end
|
1126
|
-
|
1127
|
-
sub_test_case 'dec_ref' do
|
1128
|
-
test 'when value exists in active_socks' do
|
1129
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1130
|
-
c.fetch_or { 1 }
|
1131
|
-
c.dec_ref
|
1132
|
-
|
1133
|
-
assert_equal(0, c.instance_variable_get(:@active_socks)[Thread.current.object_id].ref)
|
1134
|
-
end
|
1135
|
-
|
1136
|
-
test 'when value exists in inactive_socks' do
|
1137
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1138
|
-
c.fetch_or { 1 }
|
1139
|
-
c.revoke
|
1140
|
-
c.dec_ref
|
1141
|
-
assert_equal(-1, c.instance_variable_get(:@inactive_socks)[Thread.current.object_id].ref)
|
1142
|
-
end
|
1143
|
-
end
|
1144
|
-
|
1145
|
-
sub_test_case 'dec_ref_by_value' do
|
1146
|
-
test 'when value exists in active_socks' do
|
1147
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1148
|
-
c.fetch_or { 1 }
|
1149
|
-
c.dec_ref_by_value(1)
|
1150
|
-
|
1151
|
-
assert_equal(0, c.instance_variable_get(:@active_socks)[Thread.current.object_id].ref)
|
1152
|
-
end
|
1153
|
-
|
1154
|
-
test 'when value exists in inactive_socks' do
|
1155
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1156
|
-
c.fetch_or { 1 }
|
1157
|
-
c.revoke
|
1158
|
-
c.dec_ref_by_value(1)
|
1159
|
-
assert_equal(-1, c.instance_variable_get(:@inactive_socks)[Thread.current.object_id].ref)
|
1160
|
-
end
|
1161
|
-
end
|
1162
|
-
|
1163
|
-
sub_test_case 'clear' do
|
1164
|
-
test 'when value is in active_socks' do
|
1165
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1166
|
-
m = mock!.close { 'closed' }.subject
|
1167
|
-
c.fetch_or { m }
|
1168
|
-
assert_true(!c.instance_variable_get(:@active_socks).empty?)
|
1169
|
-
|
1170
|
-
c.clear
|
1171
|
-
assert_true(c.instance_variable_get(:@active_socks).empty?)
|
1172
|
-
end
|
1173
|
-
|
1174
|
-
test 'when value is in inactive_socks' do
|
1175
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1176
|
-
m = mock!.close { 'closed' }.subject
|
1177
|
-
c.fetch_or { m }
|
1178
|
-
c.revoke
|
1179
|
-
assert_true(!c.instance_variable_get(:@inactive_socks).empty?)
|
1180
|
-
|
1181
|
-
c.clear
|
1182
|
-
assert_true(c.instance_variable_get(:@active_socks).empty?)
|
1183
|
-
end
|
1184
|
-
end
|
1185
|
-
|
1186
|
-
sub_test_case 'purge_obsolete_socks' do
|
1187
|
-
test 'delete key in inactive_socks' do
|
1188
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1189
|
-
m = mock!.close { 'closed' }.subject
|
1190
|
-
c.fetch_or { m }
|
1191
|
-
c.revoke
|
1192
|
-
assert_true(!c.instance_variable_get(:@inactive_socks).empty?)
|
1193
|
-
|
1194
|
-
c.purge_obsolete_socks
|
1195
|
-
assert_true(c.instance_variable_get(:@active_socks).empty?)
|
1196
|
-
end
|
1197
|
-
|
1198
|
-
test 'move key from active_socks to inactive_socks' do
|
1199
|
-
c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
|
1200
|
-
m = dont_allow(mock!).close
|
1201
|
-
stub(m).inspect # for log
|
1202
|
-
c.fetch_or { m }
|
1203
|
-
assert_true(!c.instance_variable_get(:@active_socks).empty?)
|
1204
|
-
assert_true(c.instance_variable_get(:@inactive_socks).empty?)
|
1205
|
-
|
1206
|
-
c.purge_obsolete_socks
|
1207
|
-
assert_true(!c.instance_variable_get(:@active_socks).empty?)
|
1208
|
-
assert_true(c.instance_variable_get(:@inactive_socks).empty?)
|
1209
|
-
end
|
1210
|
-
end
|
1211
|
-
end
|
1212
1161
|
end
|