fluentd 1.15.3-x64-mingw-ucrt → 1.16.0-x64-mingw-ucrt
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux-test.yaml +2 -2
- data/.github/workflows/macos-test.yaml +2 -2
- data/.github/workflows/windows-test.yaml +2 -2
- data/CHANGELOG.md +77 -0
- data/MAINTAINERS.md +2 -0
- data/README.md +0 -1
- data/fluentd.gemspec +2 -2
- data/lib/fluent/command/fluentd.rb +55 -53
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +35 -5
- data/lib/fluent/plugin/base.rb +5 -7
- data/lib/fluent/plugin/buf_file.rb +32 -3
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +21 -0
- data/lib/fluent/plugin/in_tcp.rb +4 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward.rb +2 -2
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +49 -12
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/supervisor.rb +157 -251
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/version.rb +1 -1
- data/test/command/test_ctl.rb +1 -1
- data/test/command/test_fluentd.rb +137 -6
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +5 -5
- data/test/config/test_system_config.rb +0 -8
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/test_base.rb +98 -0
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_in_http.rb +2 -3
- data/test/plugin/test_in_monitor_agent.rb +2 -3
- data/test/plugin/test_in_tcp.rb +15 -0
- data/test/plugin/test_out_forward.rb +14 -18
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_output.rb +269 -0
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_server.rb +10 -5
- data/test/test_config.rb +0 -21
- data/test/test_formatter.rb +23 -20
- data/test/test_log.rb +71 -36
- data/test/test_supervisor.rb +277 -282
- metadata +12 -19
- data/.drone.yml +0 -35
- data/.gitlab-ci.yml +0 -103
- data/test/test_logger_initializer.rb +0 -46
@@ -1151,15 +1151,13 @@ class FileBufferTest < Test::Unit::TestCase
|
|
1151
1151
|
|
1152
1152
|
sub_test_case 'there are existing broken file chunks' do
|
1153
1153
|
setup do
|
1154
|
+
@id_output = 'backup_test'
|
1154
1155
|
@bufdir = File.expand_path('../../tmp/broken_buffer_file', __FILE__)
|
1155
|
-
FileUtils.
|
1156
|
+
FileUtils.rm_rf @bufdir rescue nil
|
1157
|
+
FileUtils.mkdir_p @bufdir
|
1156
1158
|
@bufpath = File.join(@bufdir, 'broken_test.*.log')
|
1157
1159
|
|
1158
1160
|
Fluent::Test.setup
|
1159
|
-
@d = FluentPluginFileBufferTest::DummyOutputPlugin.new
|
1160
|
-
@p = Fluent::Plugin::FileBuffer.new
|
1161
|
-
@p.owner = @d
|
1162
|
-
@p.configure(config_element('buffer', '', {'path' => @bufpath}))
|
1163
1161
|
end
|
1164
1162
|
|
1165
1163
|
teardown do
|
@@ -1171,12 +1169,12 @@ class FileBufferTest < Test::Unit::TestCase
|
|
1171
1169
|
@p.close unless @p.closed?
|
1172
1170
|
@p.terminate unless @p.terminated?
|
1173
1171
|
end
|
1174
|
-
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1172
|
+
end
|
1173
|
+
|
1174
|
+
def setup_plugins(buf_conf)
|
1175
|
+
@d = FluentPluginFileBufferTest::DummyOutputPlugin.new
|
1176
|
+
@d.configure(config_element('ROOT', '', {'@id' => @id_output}, [config_element('buffer', '', buf_conf)]))
|
1177
|
+
@p = @d.buffer
|
1180
1178
|
end
|
1181
1179
|
|
1182
1180
|
def create_first_chunk(mode)
|
@@ -1232,44 +1230,85 @@ class FileBufferTest < Test::Unit::TestCase
|
|
1232
1230
|
assert { logs.any? { |log| log.include?(msg) } }
|
1233
1231
|
end
|
1234
1232
|
|
1235
|
-
test '#resume
|
1236
|
-
|
1233
|
+
test '#resume backups staged empty chunk' do
|
1234
|
+
setup_plugins({'path' => @bufpath})
|
1235
|
+
c1id, p1 = create_first_chunk('b')
|
1237
1236
|
File.open(p1, 'wb') { |f| } # create staged empty chunk file
|
1238
1237
|
c2id, _ = create_second_chunk('b')
|
1239
1238
|
|
1240
|
-
@
|
1239
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
1240
|
+
@p.start
|
1241
|
+
end
|
1242
|
+
|
1241
1243
|
compare_staged_chunk(@p.stage, c2id, '2016-04-17 14:01:00 -0700', 3, :staged)
|
1242
1244
|
compare_log(@p, 'staged file chunk is empty')
|
1245
|
+
assert { not File.exist?(p1) }
|
1246
|
+
assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
|
1243
1247
|
end
|
1244
1248
|
|
1245
|
-
test '#resume
|
1249
|
+
test '#resume backups staged broken metadata' do
|
1250
|
+
setup_plugins({'path' => @bufpath})
|
1246
1251
|
c1id, _ = create_first_chunk('b')
|
1247
|
-
|
1252
|
+
c2id, p2 = create_second_chunk('b')
|
1248
1253
|
File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
|
1249
1254
|
|
1250
|
-
@
|
1255
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
1256
|
+
@p.start
|
1257
|
+
end
|
1258
|
+
|
1251
1259
|
compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
|
1252
1260
|
compare_log(@p, 'staged meta file is broken')
|
1261
|
+
assert { not File.exist?(p2) }
|
1262
|
+
assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
|
1253
1263
|
end
|
1254
1264
|
|
1255
|
-
test '#resume
|
1256
|
-
|
1265
|
+
test '#resume backups enqueued empty chunk' do
|
1266
|
+
setup_plugins({'path' => @bufpath})
|
1267
|
+
c1id, p1 = create_first_chunk('q')
|
1257
1268
|
File.open(p1, 'wb') { |f| } # create enqueued empty chunk file
|
1258
1269
|
c2id, _ = create_second_chunk('q')
|
1259
1270
|
|
1260
|
-
@
|
1271
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
1272
|
+
@p.start
|
1273
|
+
end
|
1274
|
+
|
1261
1275
|
compare_queued_chunk(@p.queue, c2id, 3, :queued)
|
1262
1276
|
compare_log(@p, 'enqueued file chunk is empty')
|
1277
|
+
assert { not File.exist?(p1) }
|
1278
|
+
assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
|
1263
1279
|
end
|
1264
1280
|
|
1265
|
-
test '#resume
|
1281
|
+
test '#resume backups enqueued broken metadata' do
|
1282
|
+
setup_plugins({'path' => @bufpath})
|
1266
1283
|
c1id, _ = create_first_chunk('q')
|
1267
|
-
|
1284
|
+
c2id, p2 = create_second_chunk('q')
|
1268
1285
|
File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create enqueued broken meta file
|
1269
1286
|
|
1270
|
-
@
|
1287
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
1288
|
+
@p.start
|
1289
|
+
end
|
1290
|
+
|
1271
1291
|
compare_queued_chunk(@p.queue, c1id, 4, :queued)
|
1272
1292
|
compare_log(@p, 'enqueued meta file is broken')
|
1293
|
+
assert { not File.exist?(p2) }
|
1294
|
+
assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
|
1295
|
+
end
|
1296
|
+
|
1297
|
+
test '#resume throws away broken chunk with disable_chunk_backup' do
|
1298
|
+
setup_plugins({'path' => @bufpath, 'disable_chunk_backup' => true})
|
1299
|
+
c1id, _ = create_first_chunk('b')
|
1300
|
+
c2id, p2 = create_second_chunk('b')
|
1301
|
+
File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
|
1302
|
+
|
1303
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
1304
|
+
@p.start
|
1305
|
+
end
|
1306
|
+
|
1307
|
+
compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
|
1308
|
+
compare_log(@p, 'staged meta file is broken')
|
1309
|
+
compare_log(@p, 'disable_chunk_backup is true')
|
1310
|
+
assert { not File.exist?(p2) }
|
1311
|
+
assert { not File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
|
1273
1312
|
end
|
1274
1313
|
end
|
1275
1314
|
end
|
@@ -830,4 +830,69 @@ class FileSingleBufferTest < Test::Unit::TestCase
|
|
830
830
|
assert_equal :queued, queue[0].state
|
831
831
|
end
|
832
832
|
end
|
833
|
+
|
834
|
+
sub_test_case 'there are existing broken file chunks' do
|
835
|
+
setup do
|
836
|
+
FileUtils.rm_rf(@bufdir) rescue nil
|
837
|
+
FileUtils.mkdir_p(@bufdir)
|
838
|
+
end
|
839
|
+
|
840
|
+
teardown do
|
841
|
+
return unless @p
|
842
|
+
|
843
|
+
@p.stop unless @p.stopped?
|
844
|
+
@p.before_shutdown unless @p.before_shutdown?
|
845
|
+
@p.shutdown unless @p.shutdown?
|
846
|
+
@p.after_shutdown unless @p.after_shutdown?
|
847
|
+
@p.close unless @p.closed?
|
848
|
+
@p.terminate unless @p.terminated?
|
849
|
+
end
|
850
|
+
|
851
|
+
test '#resume backups empty chunk' do
|
852
|
+
id_output = 'backup_test'
|
853
|
+
@d = create_driver(%[
|
854
|
+
@id #{id_output}
|
855
|
+
<buffer tag>
|
856
|
+
@type file_single
|
857
|
+
path #{PATH}
|
858
|
+
</buffer>
|
859
|
+
])
|
860
|
+
@p = @d.instance.buffer
|
861
|
+
|
862
|
+
c1id = Fluent::UniqueId.generate
|
863
|
+
p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
|
864
|
+
File.open(p1, 'wb') { |f| } # create empty chunk file
|
865
|
+
|
866
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
867
|
+
@p.start
|
868
|
+
end
|
869
|
+
|
870
|
+
assert { not File.exist?(p1) }
|
871
|
+
assert { File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
|
872
|
+
end
|
873
|
+
|
874
|
+
test '#resume throws away broken chunk with disable_chunk_backup' do
|
875
|
+
id_output = 'backup_test'
|
876
|
+
@d = create_driver(%[
|
877
|
+
@id #{id_output}
|
878
|
+
<buffer tag>
|
879
|
+
@type file_single
|
880
|
+
path #{PATH}
|
881
|
+
disable_chunk_backup true
|
882
|
+
</buffer>
|
883
|
+
])
|
884
|
+
@p = @d.instance.buffer
|
885
|
+
|
886
|
+
c1id = Fluent::UniqueId.generate
|
887
|
+
p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
|
888
|
+
File.open(p1, 'wb') { |f| } # create empty chunk file
|
889
|
+
|
890
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
|
891
|
+
@p.start
|
892
|
+
end
|
893
|
+
|
894
|
+
assert { not File.exist?(p1) }
|
895
|
+
assert { not File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
|
896
|
+
end
|
897
|
+
end
|
833
898
|
end
|
data/test/plugin/test_in_http.rb
CHANGED
@@ -7,9 +7,8 @@ require 'timecop'
|
|
7
7
|
class HttpInputTest < Test::Unit::TestCase
|
8
8
|
class << self
|
9
9
|
def startup
|
10
|
-
|
11
|
-
|
12
|
-
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
|
10
|
+
@server = ServerEngine::SocketManager::Server.open
|
11
|
+
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
|
13
12
|
end
|
14
13
|
|
15
14
|
def shutdown
|
@@ -355,14 +355,13 @@ EOC
|
|
355
355
|
|
356
356
|
test "fluentd opts" do
|
357
357
|
d = create_driver
|
358
|
-
opts = Fluent::Supervisor.default_options
|
359
358
|
|
360
359
|
filepath = nil
|
361
360
|
begin
|
362
361
|
FileUtils.mkdir_p(CONFIG_DIR)
|
363
362
|
filepath = File.expand_path('fluentd.conf', CONFIG_DIR)
|
364
363
|
FileUtils.touch(filepath)
|
365
|
-
s = Fluent::Supervisor.new(
|
364
|
+
s = Fluent::Supervisor.new({config_path: filepath})
|
366
365
|
s.configure
|
367
366
|
ensure
|
368
367
|
FileUtils.rm_r(CONFIG_DIR) rescue _
|
@@ -502,7 +501,7 @@ EOC
|
|
502
501
|
v.puts(conf)
|
503
502
|
end
|
504
503
|
|
505
|
-
@supervisor = Fluent::Supervisor.new(
|
504
|
+
@supervisor = Fluent::Supervisor.new({config_path: @filepath})
|
506
505
|
@supervisor.configure
|
507
506
|
ensure
|
508
507
|
FileUtils.rm_r(CONFIG_DIR) rescue _
|
data/test/plugin/test_in_tcp.rb
CHANGED
@@ -156,6 +156,19 @@ class TcpInputTest < Test::Unit::TestCase
|
|
156
156
|
assert_equal hostname, event[2]['host']
|
157
157
|
end
|
158
158
|
|
159
|
+
test "send_keepalive_packet_can_be_enabled" do
|
160
|
+
d = create_driver(base_config + %!
|
161
|
+
format none
|
162
|
+
send_keepalive_packet true
|
163
|
+
!)
|
164
|
+
assert_true d.instance.send_keepalive_packet
|
165
|
+
|
166
|
+
d = create_driver(base_config + %!
|
167
|
+
format none
|
168
|
+
!)
|
169
|
+
assert_false d.instance.send_keepalive_packet
|
170
|
+
end
|
171
|
+
|
159
172
|
test 'source_address_key' do
|
160
173
|
d = create_driver(base_config + %!
|
161
174
|
format none
|
@@ -213,6 +226,8 @@ class TcpInputTest < Test::Unit::TestCase
|
|
213
226
|
|
214
227
|
assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
|
215
228
|
assert_equal 0, d.events.size
|
229
|
+
ensure
|
230
|
+
d.instance_shutdown if d&.instance
|
216
231
|
end
|
217
232
|
end
|
218
233
|
|
@@ -1331,26 +1331,22 @@ EOL
|
|
1331
1331
|
d = create_driver(output_conf)
|
1332
1332
|
d.instance_start
|
1333
1333
|
|
1334
|
-
|
1335
|
-
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1340
|
-
|
1341
|
-
|
1342
|
-
|
1343
|
-
|
1344
|
-
|
1345
|
-
d.
|
1346
|
-
|
1347
|
-
|
1348
|
-
node.send_data('test', chunk) rescue nil
|
1349
|
-
end
|
1334
|
+
chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
|
1335
|
+
mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
|
1336
|
+
linger_timeout: anything,
|
1337
|
+
send_timeout: anything,
|
1338
|
+
recv_timeout: anything,
|
1339
|
+
connect_timeout: anything) { |sock|
|
1340
|
+
mock(sock).close.once; sock
|
1341
|
+
}.twice
|
1342
|
+
|
1343
|
+
target_input_driver.run(timeout: 15) do
|
1344
|
+
d.run do
|
1345
|
+
node = d.instance.nodes.first
|
1346
|
+
2.times do
|
1347
|
+
node.send_data('test', chunk) rescue nil
|
1350
1348
|
end
|
1351
1349
|
end
|
1352
|
-
ensure
|
1353
|
-
d.instance_shutdown
|
1354
1350
|
end
|
1355
1351
|
end
|
1356
1352
|
end
|
@@ -378,6 +378,7 @@ class HTTPOutputTest < Test::Unit::TestCase
|
|
378
378
|
password hello?
|
379
379
|
</auth>
|
380
380
|
])
|
381
|
+
d.instance.system_config_override(root_dir: TMP_DIR) # Backup files are generated in TMP_DIR.
|
381
382
|
d.run(default_tag: 'test.http', shutdown: false) do
|
382
383
|
test_events.each { |event|
|
383
384
|
d.feed(event)
|
data/test/plugin/test_output.rb
CHANGED
@@ -803,7 +803,10 @@ class OutputTest < Test::Unit::TestCase
|
|
803
803
|
end
|
804
804
|
|
805
805
|
test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
|
806
|
+
tmp_dir = File.join(__dir__, '../tmp/test_output')
|
807
|
+
|
806
808
|
i = create_output(:delayed)
|
809
|
+
i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
|
807
810
|
try_write_called = false
|
808
811
|
i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
|
809
812
|
|
@@ -820,6 +823,8 @@ class OutputTest < Test::Unit::TestCase
|
|
820
823
|
assert try_write_called
|
821
824
|
|
822
825
|
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
826
|
+
ensure
|
827
|
+
FileUtils.rm_rf(tmp_dir)
|
823
828
|
end
|
824
829
|
|
825
830
|
test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
|
@@ -849,7 +854,10 @@ class OutputTest < Test::Unit::TestCase
|
|
849
854
|
end
|
850
855
|
|
851
856
|
test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
|
857
|
+
tmp_dir = File.join(__dir__, '../tmp/test_output')
|
858
|
+
|
852
859
|
i = create_output(:full)
|
860
|
+
i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
|
853
861
|
write_called = false
|
854
862
|
try_write_called = false
|
855
863
|
i.register(:write){ |chunk| write_called = true }
|
@@ -872,6 +880,8 @@ class OutputTest < Test::Unit::TestCase
|
|
872
880
|
assert try_write_called
|
873
881
|
|
874
882
|
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
883
|
+
ensure
|
884
|
+
FileUtils.rm_rf(tmp_dir)
|
875
885
|
end
|
876
886
|
|
877
887
|
test 'flush_interval is ignored when flush_mode is not interval' do
|
@@ -1062,4 +1072,263 @@ class OutputTest < Test::Unit::TestCase
|
|
1062
1072
|
}
|
1063
1073
|
end
|
1064
1074
|
end
|
1075
|
+
|
1076
|
+
sub_test_case "actual_flush_thread_count" do
|
1077
|
+
data(
|
1078
|
+
"Not buffered",
|
1079
|
+
{
|
1080
|
+
output_type: :sync,
|
1081
|
+
config: config_element(),
|
1082
|
+
expected: 0,
|
1083
|
+
}
|
1084
|
+
)
|
1085
|
+
data(
|
1086
|
+
"Buffered with singile thread",
|
1087
|
+
{
|
1088
|
+
output_type: :full,
|
1089
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
|
1090
|
+
expected: 1,
|
1091
|
+
}
|
1092
|
+
)
|
1093
|
+
data(
|
1094
|
+
"Buffered with multiple threads",
|
1095
|
+
{
|
1096
|
+
output_type: :full,
|
1097
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
|
1098
|
+
expected: 8,
|
1099
|
+
}
|
1100
|
+
)
|
1101
|
+
test "actual_flush_thread_count" do |data|
|
1102
|
+
o = create_output(data[:output_type])
|
1103
|
+
o.configure(data[:config])
|
1104
|
+
assert_equal data[:expected], o.actual_flush_thread_count
|
1105
|
+
end
|
1106
|
+
|
1107
|
+
data(
|
1108
|
+
"Buffered with single thread",
|
1109
|
+
{
|
1110
|
+
output_type: :full,
|
1111
|
+
config: config_element(
|
1112
|
+
"ROOT", "", {},
|
1113
|
+
[
|
1114
|
+
config_element("buffer", "", {}),
|
1115
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1116
|
+
]
|
1117
|
+
),
|
1118
|
+
expected: 1,
|
1119
|
+
}
|
1120
|
+
)
|
1121
|
+
data(
|
1122
|
+
"Buffered with multiple threads",
|
1123
|
+
{
|
1124
|
+
output_type: :full,
|
1125
|
+
config: config_element(
|
1126
|
+
"ROOT", "", {},
|
1127
|
+
[
|
1128
|
+
config_element("buffer", "", {"flush_thread_count" => 8}),
|
1129
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1130
|
+
]
|
1131
|
+
),
|
1132
|
+
expected: 8,
|
1133
|
+
}
|
1134
|
+
)
|
1135
|
+
test "actual_flush_thread_count for secondary" do |data|
|
1136
|
+
primary = create_output(data[:output_type])
|
1137
|
+
primary.configure(data[:config])
|
1138
|
+
assert_equal data[:expected], primary.secondary.actual_flush_thread_count
|
1139
|
+
end
|
1140
|
+
end
|
1141
|
+
|
1142
|
+
sub_test_case "synchronize_path" do
|
1143
|
+
def setup
|
1144
|
+
Dir.mktmpdir do |lock_dir|
|
1145
|
+
ENV['FLUENTD_LOCK_DIR'] = lock_dir
|
1146
|
+
yield
|
1147
|
+
end
|
1148
|
+
end
|
1149
|
+
|
1150
|
+
def assert_worker_lock(lock_path, expect_locked)
|
1151
|
+
# With LOCK_NB set, flock() returns:
|
1152
|
+
# * `false` when the file is already locked.
|
1153
|
+
# * `0` when the file is not locked.
|
1154
|
+
File.open(lock_path, "w") do |f|
|
1155
|
+
if expect_locked
|
1156
|
+
assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)
|
1157
|
+
else
|
1158
|
+
assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)
|
1159
|
+
end
|
1160
|
+
end
|
1161
|
+
end
|
1162
|
+
|
1163
|
+
def assert_thread_lock(output_plugin, expect_locked)
|
1164
|
+
t = Thread.new do
|
1165
|
+
output_plugin.synchronize_path("test") do
|
1166
|
+
end
|
1167
|
+
end
|
1168
|
+
if expect_locked
|
1169
|
+
assert_nil t.join(3)
|
1170
|
+
else
|
1171
|
+
assert_not_nil t.join(3)
|
1172
|
+
end
|
1173
|
+
end
|
1174
|
+
|
1175
|
+
data(
|
1176
|
+
"Not buffered with single worker",
|
1177
|
+
{
|
1178
|
+
output_type: :sync,
|
1179
|
+
config: config_element(),
|
1180
|
+
workers: 1,
|
1181
|
+
expect_worker_lock: false,
|
1182
|
+
expect_thread_lock: false,
|
1183
|
+
}
|
1184
|
+
)
|
1185
|
+
data(
|
1186
|
+
"Not buffered with multiple workers",
|
1187
|
+
{
|
1188
|
+
output_type: :sync,
|
1189
|
+
config: config_element(),
|
1190
|
+
workers: 4,
|
1191
|
+
expect_worker_lock: true,
|
1192
|
+
expect_thread_lock: false,
|
1193
|
+
}
|
1194
|
+
)
|
1195
|
+
data(
|
1196
|
+
"Buffered with single thread and single worker",
|
1197
|
+
{
|
1198
|
+
output_type: :full,
|
1199
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
|
1200
|
+
workers: 1,
|
1201
|
+
expect_worker_lock: false,
|
1202
|
+
expect_thread_lock: false,
|
1203
|
+
}
|
1204
|
+
)
|
1205
|
+
data(
|
1206
|
+
"Buffered with multiple threads and single worker",
|
1207
|
+
{
|
1208
|
+
output_type: :full,
|
1209
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
|
1210
|
+
workers: 1,
|
1211
|
+
expect_worker_lock: false,
|
1212
|
+
expect_thread_lock: true,
|
1213
|
+
}
|
1214
|
+
)
|
1215
|
+
data(
|
1216
|
+
"Buffered with single thread and multiple workers",
|
1217
|
+
{
|
1218
|
+
output_type: :full,
|
1219
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
|
1220
|
+
workers: 4,
|
1221
|
+
expect_worker_lock: true,
|
1222
|
+
expect_thread_lock: false,
|
1223
|
+
}
|
1224
|
+
)
|
1225
|
+
data(
|
1226
|
+
"Buffered with multiple threads and multiple workers",
|
1227
|
+
{
|
1228
|
+
output_type: :full,
|
1229
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
|
1230
|
+
workers: 4,
|
1231
|
+
expect_worker_lock: true,
|
1232
|
+
expect_thread_lock: true,
|
1233
|
+
}
|
1234
|
+
)
|
1235
|
+
test "synchronize_path" do |data|
|
1236
|
+
o = create_output(data[:output_type])
|
1237
|
+
o.configure(data[:config])
|
1238
|
+
o.system_config_override(workers: data[:workers])
|
1239
|
+
|
1240
|
+
test_lock_name = "test_lock_name"
|
1241
|
+
lock_path = o.get_lock_path(test_lock_name)
|
1242
|
+
|
1243
|
+
o.synchronize_path(test_lock_name) do
|
1244
|
+
assert_worker_lock(lock_path, data[:expect_worker_lock])
|
1245
|
+
assert_thread_lock(o, data[:expect_thread_lock])
|
1246
|
+
end
|
1247
|
+
|
1248
|
+
assert_worker_lock(lock_path, false)
|
1249
|
+
assert_thread_lock(o, false)
|
1250
|
+
end
|
1251
|
+
|
1252
|
+
data(
|
1253
|
+
"Buffered with single thread and single worker",
|
1254
|
+
{
|
1255
|
+
output_type: :full,
|
1256
|
+
config: config_element(
|
1257
|
+
"ROOT", "", {},
|
1258
|
+
[
|
1259
|
+
config_element("buffer", "", {}),
|
1260
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1261
|
+
]
|
1262
|
+
),
|
1263
|
+
workers: 1,
|
1264
|
+
expect_worker_lock: false,
|
1265
|
+
expect_thread_lock: false,
|
1266
|
+
}
|
1267
|
+
)
|
1268
|
+
data(
|
1269
|
+
"Buffered with multiple threads and single worker",
|
1270
|
+
{
|
1271
|
+
output_type: :full,
|
1272
|
+
config: config_element(
|
1273
|
+
"ROOT", "", {},
|
1274
|
+
[
|
1275
|
+
config_element("buffer", "", {"flush_thread_count" => 8}),
|
1276
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1277
|
+
]
|
1278
|
+
),
|
1279
|
+
workers: 1,
|
1280
|
+
expect_worker_lock: false,
|
1281
|
+
expect_thread_lock: true,
|
1282
|
+
}
|
1283
|
+
)
|
1284
|
+
data(
|
1285
|
+
"Buffered with single thread and multiple workers",
|
1286
|
+
{
|
1287
|
+
output_type: :full,
|
1288
|
+
config: config_element(
|
1289
|
+
"ROOT", "", {},
|
1290
|
+
[
|
1291
|
+
config_element("buffer", "", {}),
|
1292
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1293
|
+
]
|
1294
|
+
),
|
1295
|
+
workers: 4,
|
1296
|
+
expect_worker_lock: true,
|
1297
|
+
expect_thread_lock: false,
|
1298
|
+
}
|
1299
|
+
)
|
1300
|
+
data(
|
1301
|
+
"Buffered with multiple threads and multiple workers",
|
1302
|
+
{
|
1303
|
+
output_type: :full,
|
1304
|
+
config: config_element(
|
1305
|
+
"ROOT", "", {},
|
1306
|
+
[
|
1307
|
+
config_element("buffer", "", {"flush_thread_count" => 8}),
|
1308
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1309
|
+
]
|
1310
|
+
),
|
1311
|
+
workers: 4,
|
1312
|
+
expect_worker_lock: true,
|
1313
|
+
expect_thread_lock: true,
|
1314
|
+
}
|
1315
|
+
)
|
1316
|
+
test "synchronize_path for secondary" do |data|
|
1317
|
+
primary = create_output(data[:output_type])
|
1318
|
+
primary.configure(data[:config])
|
1319
|
+
secondary = primary.secondary
|
1320
|
+
secondary.system_config_override(workers: data[:workers])
|
1321
|
+
|
1322
|
+
test_lock_name = "test_lock_name"
|
1323
|
+
lock_path = secondary.get_lock_path(test_lock_name)
|
1324
|
+
|
1325
|
+
secondary.synchronize_path(test_lock_name) do
|
1326
|
+
assert_worker_lock(lock_path, data[:expect_worker_lock])
|
1327
|
+
assert_thread_lock(secondary, data[:expect_thread_lock])
|
1328
|
+
end
|
1329
|
+
|
1330
|
+
assert_worker_lock(lock_path, false)
|
1331
|
+
assert_thread_lock(secondary, false)
|
1332
|
+
end
|
1333
|
+
end
|
1065
1334
|
end
|
@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
|
|
28
28
|
if initialize_conf
|
29
29
|
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
|
30
30
|
else
|
31
|
-
|
32
|
-
instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
|
33
|
-
instance.configure(conf)
|
34
|
-
d = Struct.new(:instance).new
|
35
|
-
d.instance = instance
|
36
|
-
d
|
31
|
+
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
|
37
32
|
end
|
38
33
|
end
|
39
34
|
|
@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
|
|
127
127
|
end
|
128
128
|
|
129
129
|
client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
|
130
|
-
reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
|
130
|
+
reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
|
131
131
|
|
132
132
|
resp = nil
|
133
133
|
error = nil
|
@@ -16,11 +16,16 @@ class ServerPluginHelperTest < Test::Unit::TestCase
|
|
16
16
|
|
17
17
|
setup do
|
18
18
|
@port = unused_port
|
19
|
-
|
20
|
-
|
21
|
-
|
19
|
+
if Fluent.windows?
|
20
|
+
@socket_manager_server = ServerEngine::SocketManager::Server.open
|
21
|
+
@socket_manager_path = @socket_manager_server.path
|
22
|
+
else
|
23
|
+
@socket_manager_path = ServerEngine::SocketManager::Server.generate_path
|
24
|
+
if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
|
25
|
+
FileUtils.rm_f @socket_manager_path
|
26
|
+
end
|
27
|
+
@socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
|
22
28
|
end
|
23
|
-
@socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
|
24
29
|
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
|
25
30
|
|
26
31
|
@d = Dummy.new
|
@@ -37,7 +42,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
|
|
37
42
|
(@d.terminated? || @d.terminate) rescue nil
|
38
43
|
|
39
44
|
@socket_manager_server.close
|
40
|
-
if @
|
45
|
+
if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
|
41
46
|
FileUtils.rm_f @socket_manager_path
|
42
47
|
end
|
43
48
|
end
|