fluentd 1.15.3-x64-mingw-ucrt → 1.16.0-x64-mingw-ucrt
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/linux-test.yaml +2 -2
- data/.github/workflows/macos-test.yaml +2 -2
- data/.github/workflows/windows-test.yaml +2 -2
- data/CHANGELOG.md +77 -0
- data/MAINTAINERS.md +2 -0
- data/README.md +0 -1
- data/fluentd.gemspec +2 -2
- data/lib/fluent/command/fluentd.rb +55 -53
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +35 -5
- data/lib/fluent/plugin/base.rb +5 -7
- data/lib/fluent/plugin/buf_file.rb +32 -3
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +21 -0
- data/lib/fluent/plugin/in_tcp.rb +4 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward.rb +2 -2
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +49 -12
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/supervisor.rb +157 -251
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_ctl.rb +1 -1
- data/test/command/test_fluentd.rb +137 -6
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +5 -5
- data/test/config/test_system_config.rb +0 -8
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/test_base.rb +98 -0
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_in_http.rb +2 -3
- data/test/plugin/test_in_monitor_agent.rb +2 -3
- data/test/plugin/test_in_tcp.rb +15 -0
- data/test/plugin/test_out_forward.rb +14 -18
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_output.rb +269 -0
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_server.rb +10 -5
- data/test/test_config.rb +0 -21
- data/test/test_formatter.rb +23 -20
- data/test/test_log.rb +71 -36
- data/test/test_supervisor.rb +277 -282
- metadata +12 -19
- data/.drone.yml +0 -35
- data/.gitlab-ci.yml +0 -103
- data/test/test_logger_initializer.rb +0 -46
@@ -1151,15 +1151,13 @@ class FileBufferTest < Test::Unit::TestCase
|
|
1151
1151
|
|
1152
1152
|
sub_test_case 'there are existing broken file chunks' do
|
1153
1153
|
setup do
|
1154
|
+
@id_output = 'backup_test'
|
1154
1155
|
@bufdir = File.expand_path('../../tmp/broken_buffer_file', __FILE__)
|
1155
|
-
FileUtils.
|
1156
|
+
FileUtils.rm_rf @bufdir rescue nil
|
1157
|
+
FileUtils.mkdir_p @bufdir
|
1156
1158
|
@bufpath = File.join(@bufdir, 'broken_test.*.log')
|
1157
1159
|
|
1158
1160
|
Fluent::Test.setup
|
1159
|
-
@d = FluentPluginFileBufferTest::DummyOutputPlugin.new
|
1160
|
-
@p = Fluent::Plugin::FileBuffer.new
|
1161
|
-
@p.owner = @d
|
1162
|
-
@p.configure(config_element('buffer', '', {'path' => @bufpath}))
|
1163
1161
|
end
|
1164
1162
|
|
1165
1163
|
teardown do
|
@@ -1171,12 +1169,12 @@ class FileBufferTest < Test::Unit::TestCase
|
|
1171
1169
|
@p.close unless @p.closed?
|
1172
1170
|
@p.terminate unless @p.terminated?
|
1173
1171
|
end
|
1174
|
-
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1172
|
+
end
|
1173
|
+
|
1174
|
+
def setup_plugins(buf_conf)
|
1175
|
+
@d = FluentPluginFileBufferTest::DummyOutputPlugin.new
|
1176
|
+
@d.configure(config_element('ROOT', '', {'@id' => @id_output}, [config_element('buffer', '', buf_conf)]))
|
1177
|
+
@p = @d.buffer
|
1180
1178
|
end
|
1181
1179
|
|
1182
1180
|
def create_first_chunk(mode)
|
@@ -1232,44 +1230,85 @@ class FileBufferTest < Test::Unit::TestCase
|
|
1232
1230
|
assert { logs.any? { |log| log.include?(msg) } }
|
1233
1231
|
end
|
1234
1232
|
|
1235
|
-
test '#resume
|
1236
|
-
|
1233
|
+
test '#resume backups staged empty chunk' do
|
1234
|
+
setup_plugins({'path' => @bufpath})
|
1235
|
+
c1id, p1 = create_first_chunk('b')
|
1237
1236
|
File.open(p1, 'wb') { |f| } # create staged empty chunk file
|
1238
1237
|
c2id, _ = create_second_chunk('b')
|
1239
1238
|
|
1240
|
-
@
|
1239
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
1240
|
+
@p.start
|
1241
|
+
end
|
1242
|
+
|
1241
1243
|
compare_staged_chunk(@p.stage, c2id, '2016-04-17 14:01:00 -0700', 3, :staged)
|
1242
1244
|
compare_log(@p, 'staged file chunk is empty')
|
1245
|
+
assert { not File.exist?(p1) }
|
1246
|
+
assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
|
1243
1247
|
end
|
1244
1248
|
|
1245
|
-
test '#resume
|
1249
|
+
test '#resume backups staged broken metadata' do
|
1250
|
+
setup_plugins({'path' => @bufpath})
|
1246
1251
|
c1id, _ = create_first_chunk('b')
|
1247
|
-
|
1252
|
+
c2id, p2 = create_second_chunk('b')
|
1248
1253
|
File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
|
1249
1254
|
|
1250
|
-
@
|
1255
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
1256
|
+
@p.start
|
1257
|
+
end
|
1258
|
+
|
1251
1259
|
compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
|
1252
1260
|
compare_log(@p, 'staged meta file is broken')
|
1261
|
+
assert { not File.exist?(p2) }
|
1262
|
+
assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
|
1253
1263
|
end
|
1254
1264
|
|
1255
|
-
test '#resume
|
1256
|
-
|
1265
|
+
test '#resume backups enqueued empty chunk' do
|
1266
|
+
setup_plugins({'path' => @bufpath})
|
1267
|
+
c1id, p1 = create_first_chunk('q')
|
1257
1268
|
File.open(p1, 'wb') { |f| } # create enqueued empty chunk file
|
1258
1269
|
c2id, _ = create_second_chunk('q')
|
1259
1270
|
|
1260
|
-
@
|
1271
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
1272
|
+
@p.start
|
1273
|
+
end
|
1274
|
+
|
1261
1275
|
compare_queued_chunk(@p.queue, c2id, 3, :queued)
|
1262
1276
|
compare_log(@p, 'enqueued file chunk is empty')
|
1277
|
+
assert { not File.exist?(p1) }
|
1278
|
+
assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
|
1263
1279
|
end
|
1264
1280
|
|
1265
|
-
test '#resume
|
1281
|
+
test '#resume backups enqueued broken metadata' do
|
1282
|
+
setup_plugins({'path' => @bufpath})
|
1266
1283
|
c1id, _ = create_first_chunk('q')
|
1267
|
-
|
1284
|
+
c2id, p2 = create_second_chunk('q')
|
1268
1285
|
File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create enqueued broken meta file
|
1269
1286
|
|
1270
|
-
@
|
1287
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
1288
|
+
@p.start
|
1289
|
+
end
|
1290
|
+
|
1271
1291
|
compare_queued_chunk(@p.queue, c1id, 4, :queued)
|
1272
1292
|
compare_log(@p, 'enqueued meta file is broken')
|
1293
|
+
assert { not File.exist?(p2) }
|
1294
|
+
assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
|
1295
|
+
end
|
1296
|
+
|
1297
|
+
test '#resume throws away broken chunk with disable_chunk_backup' do
|
1298
|
+
setup_plugins({'path' => @bufpath, 'disable_chunk_backup' => true})
|
1299
|
+
c1id, _ = create_first_chunk('b')
|
1300
|
+
c2id, p2 = create_second_chunk('b')
|
1301
|
+
File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
|
1302
|
+
|
1303
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
1304
|
+
@p.start
|
1305
|
+
end
|
1306
|
+
|
1307
|
+
compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
|
1308
|
+
compare_log(@p, 'staged meta file is broken')
|
1309
|
+
compare_log(@p, 'disable_chunk_backup is true')
|
1310
|
+
assert { not File.exist?(p2) }
|
1311
|
+
assert { not File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
|
1273
1312
|
end
|
1274
1313
|
end
|
1275
1314
|
end
|
@@ -830,4 +830,69 @@ class FileSingleBufferTest < Test::Unit::TestCase
|
|
830
830
|
assert_equal :queued, queue[0].state
|
831
831
|
end
|
832
832
|
end
|
833
|
+
|
834
|
+
sub_test_case 'there are existing broken file chunks' do
|
835
|
+
setup do
|
836
|
+
FileUtils.rm_rf(@bufdir) rescue nil
|
837
|
+
FileUtils.mkdir_p(@bufdir)
|
838
|
+
end
|
839
|
+
|
840
|
+
teardown do
|
841
|
+
return unless @p
|
842
|
+
|
843
|
+
@p.stop unless @p.stopped?
|
844
|
+
@p.before_shutdown unless @p.before_shutdown?
|
845
|
+
@p.shutdown unless @p.shutdown?
|
846
|
+
@p.after_shutdown unless @p.after_shutdown?
|
847
|
+
@p.close unless @p.closed?
|
848
|
+
@p.terminate unless @p.terminated?
|
849
|
+
end
|
850
|
+
|
851
|
+
test '#resume backups empty chunk' do
|
852
|
+
id_output = 'backup_test'
|
853
|
+
@d = create_driver(%[
|
854
|
+
@id #{id_output}
|
855
|
+
<buffer tag>
|
856
|
+
@type file_single
|
857
|
+
path #{PATH}
|
858
|
+
</buffer>
|
859
|
+
])
|
860
|
+
@p = @d.instance.buffer
|
861
|
+
|
862
|
+
c1id = Fluent::UniqueId.generate
|
863
|
+
p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
|
864
|
+
File.open(p1, 'wb') { |f| } # create empty chunk file
|
865
|
+
|
866
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
867
|
+
@p.start
|
868
|
+
end
|
869
|
+
|
870
|
+
assert { not File.exist?(p1) }
|
871
|
+
assert { File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
|
872
|
+
end
|
873
|
+
|
874
|
+
test '#resume throws away broken chunk with disable_chunk_backup' do
|
875
|
+
id_output = 'backup_test'
|
876
|
+
@d = create_driver(%[
|
877
|
+
@id #{id_output}
|
878
|
+
<buffer tag>
|
879
|
+
@type file_single
|
880
|
+
path #{PATH}
|
881
|
+
disable_chunk_backup true
|
882
|
+
</buffer>
|
883
|
+
])
|
884
|
+
@p = @d.instance.buffer
|
885
|
+
|
886
|
+
c1id = Fluent::UniqueId.generate
|
887
|
+
p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
|
888
|
+
File.open(p1, 'wb') { |f| } # create empty chunk file
|
889
|
+
|
890
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
891
|
+
@p.start
|
892
|
+
end
|
893
|
+
|
894
|
+
assert { not File.exist?(p1) }
|
895
|
+
assert { not File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
|
896
|
+
end
|
897
|
+
end
|
833
898
|
end
|
data/test/plugin/test_in_http.rb
CHANGED
@@ -7,9 +7,8 @@ require 'timecop'
|
|
7
7
|
class HttpInputTest < Test::Unit::TestCase
|
8
8
|
class << self
|
9
9
|
def startup
|
10
|
-
|
11
|
-
|
12
|
-
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
|
10
|
+
@server = ServerEngine::SocketManager::Server.open
|
11
|
+
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
|
13
12
|
end
|
14
13
|
|
15
14
|
def shutdown
|
@@ -355,14 +355,13 @@ EOC
|
|
355
355
|
|
356
356
|
test "fluentd opts" do
|
357
357
|
d = create_driver
|
358
|
-
opts = Fluent::Supervisor.default_options
|
359
358
|
|
360
359
|
filepath = nil
|
361
360
|
begin
|
362
361
|
FileUtils.mkdir_p(CONFIG_DIR)
|
363
362
|
filepath = File.expand_path('fluentd.conf', CONFIG_DIR)
|
364
363
|
FileUtils.touch(filepath)
|
365
|
-
s = Fluent::Supervisor.new(
|
364
|
+
s = Fluent::Supervisor.new({config_path: filepath})
|
366
365
|
s.configure
|
367
366
|
ensure
|
368
367
|
FileUtils.rm_r(CONFIG_DIR) rescue _
|
@@ -502,7 +501,7 @@ EOC
|
|
502
501
|
v.puts(conf)
|
503
502
|
end
|
504
503
|
|
505
|
-
@supervisor = Fluent::Supervisor.new(
|
504
|
+
@supervisor = Fluent::Supervisor.new({config_path: @filepath})
|
506
505
|
@supervisor.configure
|
507
506
|
ensure
|
508
507
|
FileUtils.rm_r(CONFIG_DIR) rescue _
|
data/test/plugin/test_in_tcp.rb
CHANGED
@@ -156,6 +156,19 @@ class TcpInputTest < Test::Unit::TestCase
|
|
156
156
|
assert_equal hostname, event[2]['host']
|
157
157
|
end
|
158
158
|
|
159
|
+
test "send_keepalive_packet_can_be_enabled" do
|
160
|
+
d = create_driver(base_config + %!
|
161
|
+
format none
|
162
|
+
send_keepalive_packet true
|
163
|
+
!)
|
164
|
+
assert_true d.instance.send_keepalive_packet
|
165
|
+
|
166
|
+
d = create_driver(base_config + %!
|
167
|
+
format none
|
168
|
+
!)
|
169
|
+
assert_false d.instance.send_keepalive_packet
|
170
|
+
end
|
171
|
+
|
159
172
|
test 'source_address_key' do
|
160
173
|
d = create_driver(base_config + %!
|
161
174
|
format none
|
@@ -213,6 +226,8 @@ class TcpInputTest < Test::Unit::TestCase
|
|
213
226
|
|
214
227
|
assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
|
215
228
|
assert_equal 0, d.events.size
|
229
|
+
ensure
|
230
|
+
d.instance_shutdown if d&.instance
|
216
231
|
end
|
217
232
|
end
|
218
233
|
|
@@ -1331,26 +1331,22 @@ EOL
|
|
1331
1331
|
d = create_driver(output_conf)
|
1332
1332
|
d.instance_start
|
1333
1333
|
|
1334
|
-
|
1335
|
-
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1340
|
-
|
1341
|
-
|
1342
|
-
|
1343
|
-
|
1344
|
-
|
1345
|
-
d.
|
1346
|
-
|
1347
|
-
|
1348
|
-
node.send_data('test', chunk) rescue nil
|
1349
|
-
end
|
1334
|
+
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
|
1335
|
+
mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
|
1336
|
+
linger_timeout: anything,
|
1337
|
+
send_timeout: anything,
|
1338
|
+
recv_timeout: anything,
|
1339
|
+
connect_timeout: anything) { |sock|
|
1340
|
+
mock(sock).close.once; sock
|
1341
|
+
}.twice
|
1342
|
+
|
1343
|
+
target_input_driver.run(timeout: 15) do
|
1344
|
+
d.run do
|
1345
|
+
node = d.instance.nodes.first
|
1346
|
+
2.times do
|
1347
|
+
node.send_data('test', chunk) rescue nil
|
1350
1348
|
end
|
1351
1349
|
end
|
1352
|
-
ensure
|
1353
|
-
d.instance_shutdown
|
1354
1350
|
end
|
1355
1351
|
end
|
1356
1352
|
end
|
@@ -378,6 +378,7 @@ class HTTPOutputTest < Test::Unit::TestCase
|
|
378
378
|
password hello?
|
379
379
|
</auth>
|
380
380
|
])
|
381
|
+
d.instance.system_config_override(root_dir: TMP_DIR) # Backup files are generated in TMP_DIR.
|
381
382
|
d.run(default_tag: 'test.http', shutdown: false) do
|
382
383
|
test_events.each { |event|
|
383
384
|
d.feed(event)
|
data/test/plugin/test_output.rb
CHANGED
@@ -803,7 +803,10 @@ class OutputTest < Test::Unit::TestCase
|
|
803
803
|
end
|
804
804
|
|
805
805
|
test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
|
806
|
+
tmp_dir = File.join(__dir__, '../tmp/test_output')
|
807
|
+
|
806
808
|
i = create_output(:delayed)
|
809
|
+
i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
|
807
810
|
try_write_called = false
|
808
811
|
i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
|
809
812
|
|
@@ -820,6 +823,8 @@ class OutputTest < Test::Unit::TestCase
|
|
820
823
|
assert try_write_called
|
821
824
|
|
822
825
|
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
826
|
+
ensure
|
827
|
+
FileUtils.rm_rf(tmp_dir)
|
823
828
|
end
|
824
829
|
|
825
830
|
test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
|
@@ -849,7 +854,10 @@ class OutputTest < Test::Unit::TestCase
|
|
849
854
|
end
|
850
855
|
|
851
856
|
test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
|
857
|
+
tmp_dir = File.join(__dir__, '../tmp/test_output')
|
858
|
+
|
852
859
|
i = create_output(:full)
|
860
|
+
i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
|
853
861
|
write_called = false
|
854
862
|
try_write_called = false
|
855
863
|
i.register(:write){ |chunk| write_called = true }
|
@@ -872,6 +880,8 @@ class OutputTest < Test::Unit::TestCase
|
|
872
880
|
assert try_write_called
|
873
881
|
|
874
882
|
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
883
|
+
ensure
|
884
|
+
FileUtils.rm_rf(tmp_dir)
|
875
885
|
end
|
876
886
|
|
877
887
|
test 'flush_interval is ignored when flush_mode is not interval' do
|
@@ -1062,4 +1072,263 @@ class OutputTest < Test::Unit::TestCase
|
|
1062
1072
|
}
|
1063
1073
|
end
|
1064
1074
|
end
|
1075
|
+
|
1076
|
+
sub_test_case "actual_flush_thread_count" do
|
1077
|
+
data(
|
1078
|
+
"Not buffered",
|
1079
|
+
{
|
1080
|
+
output_type: :sync,
|
1081
|
+
config: config_element(),
|
1082
|
+
expected: 0,
|
1083
|
+
}
|
1084
|
+
)
|
1085
|
+
data(
|
1086
|
+
"Buffered with singile thread",
|
1087
|
+
{
|
1088
|
+
output_type: :full,
|
1089
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
|
1090
|
+
expected: 1,
|
1091
|
+
}
|
1092
|
+
)
|
1093
|
+
data(
|
1094
|
+
"Buffered with multiple threads",
|
1095
|
+
{
|
1096
|
+
output_type: :full,
|
1097
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
|
1098
|
+
expected: 8,
|
1099
|
+
}
|
1100
|
+
)
|
1101
|
+
test "actual_flush_thread_count" do |data|
|
1102
|
+
o = create_output(data[:output_type])
|
1103
|
+
o.configure(data[:config])
|
1104
|
+
assert_equal data[:expected], o.actual_flush_thread_count
|
1105
|
+
end
|
1106
|
+
|
1107
|
+
data(
|
1108
|
+
"Buffered with single thread",
|
1109
|
+
{
|
1110
|
+
output_type: :full,
|
1111
|
+
config: config_element(
|
1112
|
+
"ROOT", "", {},
|
1113
|
+
[
|
1114
|
+
config_element("buffer", "", {}),
|
1115
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1116
|
+
]
|
1117
|
+
),
|
1118
|
+
expected: 1,
|
1119
|
+
}
|
1120
|
+
)
|
1121
|
+
data(
|
1122
|
+
"Buffered with multiple threads",
|
1123
|
+
{
|
1124
|
+
output_type: :full,
|
1125
|
+
config: config_element(
|
1126
|
+
"ROOT", "", {},
|
1127
|
+
[
|
1128
|
+
config_element("buffer", "", {"flush_thread_count" => 8}),
|
1129
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1130
|
+
]
|
1131
|
+
),
|
1132
|
+
expected: 8,
|
1133
|
+
}
|
1134
|
+
)
|
1135
|
+
test "actual_flush_thread_count for secondary" do |data|
|
1136
|
+
primary = create_output(data[:output_type])
|
1137
|
+
primary.configure(data[:config])
|
1138
|
+
assert_equal data[:expected], primary.secondary.actual_flush_thread_count
|
1139
|
+
end
|
1140
|
+
end
|
1141
|
+
|
1142
|
+
sub_test_case "synchronize_path" do
|
1143
|
+
def setup
|
1144
|
+
Dir.mktmpdir do |lock_dir|
|
1145
|
+
ENV['FLUENTD_LOCK_DIR'] = lock_dir
|
1146
|
+
yield
|
1147
|
+
end
|
1148
|
+
end
|
1149
|
+
|
1150
|
+
def assert_worker_lock(lock_path, expect_locked)
|
1151
|
+
# With LOCK_NB set, flock() returns:
|
1152
|
+
# * `false` when the file is already locked.
|
1153
|
+
# * `0` when the file is not locked.
|
1154
|
+
File.open(lock_path, "w") do |f|
|
1155
|
+
if expect_locked
|
1156
|
+
assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)
|
1157
|
+
else
|
1158
|
+
assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)
|
1159
|
+
end
|
1160
|
+
end
|
1161
|
+
end
|
1162
|
+
|
1163
|
+
def assert_thread_lock(output_plugin, expect_locked)
|
1164
|
+
t = Thread.new do
|
1165
|
+
output_plugin.synchronize_path("test") do
|
1166
|
+
end
|
1167
|
+
end
|
1168
|
+
if expect_locked
|
1169
|
+
assert_nil t.join(3)
|
1170
|
+
else
|
1171
|
+
assert_not_nil t.join(3)
|
1172
|
+
end
|
1173
|
+
end
|
1174
|
+
|
1175
|
+
data(
|
1176
|
+
"Not buffered with single worker",
|
1177
|
+
{
|
1178
|
+
output_type: :sync,
|
1179
|
+
config: config_element(),
|
1180
|
+
workers: 1,
|
1181
|
+
expect_worker_lock: false,
|
1182
|
+
expect_thread_lock: false,
|
1183
|
+
}
|
1184
|
+
)
|
1185
|
+
data(
|
1186
|
+
"Not buffered with multiple workers",
|
1187
|
+
{
|
1188
|
+
output_type: :sync,
|
1189
|
+
config: config_element(),
|
1190
|
+
workers: 4,
|
1191
|
+
expect_worker_lock: true,
|
1192
|
+
expect_thread_lock: false,
|
1193
|
+
}
|
1194
|
+
)
|
1195
|
+
data(
|
1196
|
+
"Buffered with single thread and single worker",
|
1197
|
+
{
|
1198
|
+
output_type: :full,
|
1199
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
|
1200
|
+
workers: 1,
|
1201
|
+
expect_worker_lock: false,
|
1202
|
+
expect_thread_lock: false,
|
1203
|
+
}
|
1204
|
+
)
|
1205
|
+
data(
|
1206
|
+
"Buffered with multiple threads and single worker",
|
1207
|
+
{
|
1208
|
+
output_type: :full,
|
1209
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
|
1210
|
+
workers: 1,
|
1211
|
+
expect_worker_lock: false,
|
1212
|
+
expect_thread_lock: true,
|
1213
|
+
}
|
1214
|
+
)
|
1215
|
+
data(
|
1216
|
+
"Buffered with single thread and multiple workers",
|
1217
|
+
{
|
1218
|
+
output_type: :full,
|
1219
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
|
1220
|
+
workers: 4,
|
1221
|
+
expect_worker_lock: true,
|
1222
|
+
expect_thread_lock: false,
|
1223
|
+
}
|
1224
|
+
)
|
1225
|
+
data(
|
1226
|
+
"Buffered with multiple threads and multiple workers",
|
1227
|
+
{
|
1228
|
+
output_type: :full,
|
1229
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
|
1230
|
+
workers: 4,
|
1231
|
+
expect_worker_lock: true,
|
1232
|
+
expect_thread_lock: true,
|
1233
|
+
}
|
1234
|
+
)
|
1235
|
+
test "synchronize_path" do |data|
|
1236
|
+
o = create_output(data[:output_type])
|
1237
|
+
o.configure(data[:config])
|
1238
|
+
o.system_config_override(workers: data[:workers])
|
1239
|
+
|
1240
|
+
test_lock_name = "test_lock_name"
|
1241
|
+
lock_path = o.get_lock_path(test_lock_name)
|
1242
|
+
|
1243
|
+
o.synchronize_path(test_lock_name) do
|
1244
|
+
assert_worker_lock(lock_path, data[:expect_worker_lock])
|
1245
|
+
assert_thread_lock(o, data[:expect_thread_lock])
|
1246
|
+
end
|
1247
|
+
|
1248
|
+
assert_worker_lock(lock_path, false)
|
1249
|
+
assert_thread_lock(o, false)
|
1250
|
+
end
|
1251
|
+
|
1252
|
+
data(
|
1253
|
+
"Buffered with single thread and single worker",
|
1254
|
+
{
|
1255
|
+
output_type: :full,
|
1256
|
+
config: config_element(
|
1257
|
+
"ROOT", "", {},
|
1258
|
+
[
|
1259
|
+
config_element("buffer", "", {}),
|
1260
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1261
|
+
]
|
1262
|
+
),
|
1263
|
+
workers: 1,
|
1264
|
+
expect_worker_lock: false,
|
1265
|
+
expect_thread_lock: false,
|
1266
|
+
}
|
1267
|
+
)
|
1268
|
+
data(
|
1269
|
+
"Buffered with multiple threads and single worker",
|
1270
|
+
{
|
1271
|
+
output_type: :full,
|
1272
|
+
config: config_element(
|
1273
|
+
"ROOT", "", {},
|
1274
|
+
[
|
1275
|
+
config_element("buffer", "", {"flush_thread_count" => 8}),
|
1276
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1277
|
+
]
|
1278
|
+
),
|
1279
|
+
workers: 1,
|
1280
|
+
expect_worker_lock: false,
|
1281
|
+
expect_thread_lock: true,
|
1282
|
+
}
|
1283
|
+
)
|
1284
|
+
data(
|
1285
|
+
"Buffered with single thread and multiple workers",
|
1286
|
+
{
|
1287
|
+
output_type: :full,
|
1288
|
+
config: config_element(
|
1289
|
+
"ROOT", "", {},
|
1290
|
+
[
|
1291
|
+
config_element("buffer", "", {}),
|
1292
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1293
|
+
]
|
1294
|
+
),
|
1295
|
+
workers: 4,
|
1296
|
+
expect_worker_lock: true,
|
1297
|
+
expect_thread_lock: false,
|
1298
|
+
}
|
1299
|
+
)
|
1300
|
+
data(
|
1301
|
+
"Buffered with multiple threads and multiple workers",
|
1302
|
+
{
|
1303
|
+
output_type: :full,
|
1304
|
+
config: config_element(
|
1305
|
+
"ROOT", "", {},
|
1306
|
+
[
|
1307
|
+
config_element("buffer", "", {"flush_thread_count" => 8}),
|
1308
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1309
|
+
]
|
1310
|
+
),
|
1311
|
+
workers: 4,
|
1312
|
+
expect_worker_lock: true,
|
1313
|
+
expect_thread_lock: true,
|
1314
|
+
}
|
1315
|
+
)
|
1316
|
+
test "synchronize_path for secondary" do |data|
|
1317
|
+
primary = create_output(data[:output_type])
|
1318
|
+
primary.configure(data[:config])
|
1319
|
+
secondary = primary.secondary
|
1320
|
+
secondary.system_config_override(workers: data[:workers])
|
1321
|
+
|
1322
|
+
test_lock_name = "test_lock_name"
|
1323
|
+
lock_path = secondary.get_lock_path(test_lock_name)
|
1324
|
+
|
1325
|
+
secondary.synchronize_path(test_lock_name) do
|
1326
|
+
assert_worker_lock(lock_path, data[:expect_worker_lock])
|
1327
|
+
assert_thread_lock(secondary, data[:expect_thread_lock])
|
1328
|
+
end
|
1329
|
+
|
1330
|
+
assert_worker_lock(lock_path, false)
|
1331
|
+
assert_thread_lock(secondary, false)
|
1332
|
+
end
|
1333
|
+
end
|
1065
1334
|
end
|
@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
|
|
28
28
|
if initialize_conf
|
29
29
|
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
|
30
30
|
else
|
31
|
-
|
32
|
-
instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
|
33
|
-
instance.configure(conf)
|
34
|
-
d = Struct.new(:instance).new
|
35
|
-
d.instance = instance
|
36
|
-
d
|
31
|
+
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
|
37
32
|
end
|
38
33
|
end
|
39
34
|
|
@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
|
|
127
127
|
end
|
128
128
|
|
129
129
|
client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
|
130
|
-
reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
|
130
|
+
reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
|
131
131
|
|
132
132
|
resp = nil
|
133
133
|
error = nil
|
@@ -16,11 +16,16 @@ class ServerPluginHelperTest < Test::Unit::TestCase
|
|
16
16
|
|
17
17
|
setup do
|
18
18
|
@port = unused_port
|
19
|
-
|
20
|
-
|
21
|
-
|
19
|
+
if Fluent.windows?
|
20
|
+
@socket_manager_server = ServerEngine::SocketManager::Server.open
|
21
|
+
@socket_manager_path = @socket_manager_server.path
|
22
|
+
else
|
23
|
+
@socket_manager_path = ServerEngine::SocketManager::Server.generate_path
|
24
|
+
if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
|
25
|
+
FileUtils.rm_f @socket_manager_path
|
26
|
+
end
|
27
|
+
@socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
|
22
28
|
end
|
23
|
-
@socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
|
24
29
|
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
|
25
30
|
|
26
31
|
@d = Dummy.new
|
@@ -37,7 +42,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
|
|
37
42
|
(@d.terminated? || @d.terminate) rescue nil
|
38
43
|
|
39
44
|
@socket_manager_server.close
|
40
|
-
if @
|
45
|
+
if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
|
41
46
|
FileUtils.rm_f @socket_manager_path
|
42
47
|
end
|
43
48
|
end
|