fluentd 1.15.3-x86-mingw32 → 1.16.2-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
- data/.github/workflows/linux-test.yaml +2 -2
- data/.github/workflows/macos-test.yaml +2 -2
- data/.github/workflows/stale-actions.yml +24 -0
- data/.github/workflows/windows-test.yaml +2 -2
- data/CHANGELOG.md +151 -0
- data/CONTRIBUTING.md +1 -1
- data/MAINTAINERS.md +5 -3
- data/README.md +0 -1
- data/SECURITY.md +5 -9
- data/fluentd.gemspec +3 -3
- data/lib/fluent/command/ctl.rb +2 -2
- data/lib/fluent/command/fluentd.rb +55 -53
- data/lib/fluent/command/plugin_config_formatter.rb +1 -1
- data/lib/fluent/config/dsl.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +2 -2
- data/lib/fluent/counter/server.rb +1 -1
- data/lib/fluent/counter/validator.rb +3 -3
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/event.rb +8 -4
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +44 -5
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +6 -1
- data/lib/fluent/plugin/base.rb +6 -8
- data/lib/fluent/plugin/buf_file.rb +32 -3
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +21 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +8 -8
- data/lib/fluent/plugin/in_sample.rb +1 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
- data/lib/fluent/plugin/in_tail.rb +58 -24
- data/lib/fluent/plugin/in_tcp.rb +47 -2
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward.rb +2 -2
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +50 -13
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin_helper/event_loop.rb +2 -2
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
- data/lib/fluent/plugin_helper/server.rb +8 -0
- data/lib/fluent/plugin_helper/thread.rb +3 -3
- data/lib/fluent/plugin_id.rb +1 -1
- data/lib/fluent/supervisor.rb +157 -251
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/test/helper.rb.erb +0 -1
- data/test/command/test_ctl.rb +1 -1
- data/test/command/test_fluentd.rb +137 -6
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +5 -5
- data/test/config/test_system_config.rb +0 -8
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/in_tail/test_position_file.rb +31 -1
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/test_base.rb +99 -1
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_buffer_chunk.rb +11 -0
- data/test/plugin/test_in_forward.rb +9 -9
- data/test/plugin/test_in_http.rb +2 -3
- data/test/plugin/test_in_monitor_agent.rb +2 -3
- data/test/plugin/test_in_tail.rb +379 -0
- data/test/plugin/test_in_tcp.rb +87 -2
- data/test/plugin/test_in_udp.rb +28 -0
- data/test/plugin/test_in_unix.rb +2 -2
- data/test/plugin/test_multi_output.rb +1 -1
- data/test/plugin/test_out_exec_filter.rb +2 -2
- data/test/plugin/test_out_file.rb +2 -2
- data/test/plugin/test_out_forward.rb +14 -18
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_output.rb +281 -12
- data/test/plugin/test_output_as_buffered.rb +44 -44
- data/test/plugin/test_output_as_buffered_compress.rb +32 -18
- data/test/plugin/test_output_as_buffered_retries.rb +1 -1
- data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin_helper/test_child_process.rb +2 -2
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_server.rb +60 -6
- data/test/test_config.rb +0 -21
- data/test/test_formatter.rb +23 -20
- data/test/test_log.rb +108 -36
- data/test/test_msgpack_factory.rb +32 -0
- data/test/test_supervisor.rb +287 -279
- metadata +15 -21
- data/.drone.yml +0 -35
- data/.gitlab-ci.yml +0 -103
- data/test/test_logger_initializer.rb +0 -46
data/test/plugin/test_output.rb
CHANGED
@@ -447,25 +447,25 @@ class OutputTest < Test::Unit::TestCase
|
|
447
447
|
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
448
448
|
validators = @i.placeholder_validators(:path, "/my/path/${tag}/${username}/file.%Y%m%d_%H%M.log")
|
449
449
|
assert_equal 3, validators.size
|
450
|
-
assert_equal 1, validators.
|
451
|
-
assert_equal 1, validators.
|
452
|
-
assert_equal 1, validators.
|
450
|
+
assert_equal 1, validators.count(&:time?)
|
451
|
+
assert_equal 1, validators.count(&:tag?)
|
452
|
+
assert_equal 1, validators.count(&:keys?)
|
453
453
|
end
|
454
454
|
|
455
455
|
test 'returns validators for time, tag and keys when a plugin is configured with these keys even if a template does not have placeholders' do
|
456
456
|
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,username', {'timekey' => 60})]))
|
457
457
|
validators = @i.placeholder_validators(:path, "/my/path/file.log")
|
458
458
|
assert_equal 3, validators.size
|
459
|
-
assert_equal 1, validators.
|
460
|
-
assert_equal 1, validators.
|
461
|
-
assert_equal 1, validators.
|
459
|
+
assert_equal 1, validators.count(&:time?)
|
460
|
+
assert_equal 1, validators.count(&:tag?)
|
461
|
+
assert_equal 1, validators.count(&:keys?)
|
462
462
|
end
|
463
463
|
|
464
464
|
test 'returns a validator for time if a template has timestamp placeholders' do
|
465
465
|
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
466
466
|
validators = @i.placeholder_validators(:path, "/my/path/file.%Y-%m-%d.log")
|
467
467
|
assert_equal 1, validators.size
|
468
|
-
assert_equal 1, validators.
|
468
|
+
assert_equal 1, validators.count(&:time?)
|
469
469
|
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.%Y-%m-%d.log' has timestamp placeholders, but chunk key 'time' is not configured") do
|
470
470
|
validators.first.validate!
|
471
471
|
end
|
@@ -475,7 +475,7 @@ class OutputTest < Test::Unit::TestCase
|
|
475
475
|
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => '30'})]))
|
476
476
|
validators = @i.placeholder_validators(:path, "/my/path/to/file.log")
|
477
477
|
assert_equal 1, validators.size
|
478
|
-
assert_equal 1, validators.
|
478
|
+
assert_equal 1, validators.count(&:time?)
|
479
479
|
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/to/file.log' doesn't have timestamp placeholders for timekey 30") do
|
480
480
|
validators.first.validate!
|
481
481
|
end
|
@@ -485,7 +485,7 @@ class OutputTest < Test::Unit::TestCase
|
|
485
485
|
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
486
486
|
validators = @i.placeholder_validators(:path, "/my/path/${tag}/file.log")
|
487
487
|
assert_equal 1, validators.size
|
488
|
-
assert_equal 1, validators.
|
488
|
+
assert_equal 1, validators.count(&:tag?)
|
489
489
|
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${tag}/file.log' has tag placeholders, but chunk key 'tag' is not configured") do
|
490
490
|
validators.first.validate!
|
491
491
|
end
|
@@ -495,7 +495,7 @@ class OutputTest < Test::Unit::TestCase
|
|
495
495
|
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
|
496
496
|
validators = @i.placeholder_validators(:path, "/my/path/file.log")
|
497
497
|
assert_equal 1, validators.size
|
498
|
-
assert_equal 1, validators.
|
498
|
+
assert_equal 1, validators.count(&:tag?)
|
499
499
|
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have tag placeholder") do
|
500
500
|
validators.first.validate!
|
501
501
|
end
|
@@ -505,7 +505,7 @@ class OutputTest < Test::Unit::TestCase
|
|
505
505
|
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
|
506
506
|
validators = @i.placeholder_validators(:path, "/my/path/${username}/file.${group}.log")
|
507
507
|
assert_equal 1, validators.size
|
508
|
-
assert_equal 1, validators.
|
508
|
+
assert_equal 1, validators.count(&:keys?)
|
509
509
|
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${username}/file.${group}.log' has placeholders, but chunk keys doesn't have keys group,username") do
|
510
510
|
validators.first.validate!
|
511
511
|
end
|
@@ -515,7 +515,7 @@ class OutputTest < Test::Unit::TestCase
|
|
515
515
|
@i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,group')]))
|
516
516
|
validators = @i.placeholder_validators(:path, "/my/path/file.log")
|
517
517
|
assert_equal 1, validators.size
|
518
|
-
assert_equal 1, validators.
|
518
|
+
assert_equal 1, validators.count(&:keys?)
|
519
519
|
assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys group,username") do
|
520
520
|
validators.first.validate!
|
521
521
|
end
|
@@ -803,7 +803,10 @@ class OutputTest < Test::Unit::TestCase
|
|
803
803
|
end
|
804
804
|
|
805
805
|
test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
|
806
|
+
tmp_dir = File.join(__dir__, '../tmp/test_output')
|
807
|
+
|
806
808
|
i = create_output(:delayed)
|
809
|
+
i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
|
807
810
|
try_write_called = false
|
808
811
|
i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
|
809
812
|
|
@@ -820,6 +823,8 @@ class OutputTest < Test::Unit::TestCase
|
|
820
823
|
assert try_write_called
|
821
824
|
|
822
825
|
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
826
|
+
ensure
|
827
|
+
FileUtils.rm_rf(tmp_dir)
|
823
828
|
end
|
824
829
|
|
825
830
|
test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
|
@@ -849,7 +854,10 @@ class OutputTest < Test::Unit::TestCase
|
|
849
854
|
end
|
850
855
|
|
851
856
|
test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
|
857
|
+
tmp_dir = File.join(__dir__, '../tmp/test_output')
|
858
|
+
|
852
859
|
i = create_output(:full)
|
860
|
+
i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
|
853
861
|
write_called = false
|
854
862
|
try_write_called = false
|
855
863
|
i.register(:write){ |chunk| write_called = true }
|
@@ -872,6 +880,8 @@ class OutputTest < Test::Unit::TestCase
|
|
872
880
|
assert try_write_called
|
873
881
|
|
874
882
|
i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
|
883
|
+
ensure
|
884
|
+
FileUtils.rm_rf(tmp_dir)
|
875
885
|
end
|
876
886
|
|
877
887
|
test 'flush_interval is ignored when flush_mode is not interval' do
|
@@ -1062,4 +1072,263 @@ class OutputTest < Test::Unit::TestCase
|
|
1062
1072
|
}
|
1063
1073
|
end
|
1064
1074
|
end
|
1075
|
+
|
1076
|
+
sub_test_case "actual_flush_thread_count" do
|
1077
|
+
data(
|
1078
|
+
"Not buffered",
|
1079
|
+
{
|
1080
|
+
output_type: :sync,
|
1081
|
+
config: config_element(),
|
1082
|
+
expected: 0,
|
1083
|
+
}
|
1084
|
+
)
|
1085
|
+
data(
|
1086
|
+
"Buffered with singile thread",
|
1087
|
+
{
|
1088
|
+
output_type: :full,
|
1089
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
|
1090
|
+
expected: 1,
|
1091
|
+
}
|
1092
|
+
)
|
1093
|
+
data(
|
1094
|
+
"Buffered with multiple threads",
|
1095
|
+
{
|
1096
|
+
output_type: :full,
|
1097
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
|
1098
|
+
expected: 8,
|
1099
|
+
}
|
1100
|
+
)
|
1101
|
+
test "actual_flush_thread_count" do |data|
|
1102
|
+
o = create_output(data[:output_type])
|
1103
|
+
o.configure(data[:config])
|
1104
|
+
assert_equal data[:expected], o.actual_flush_thread_count
|
1105
|
+
end
|
1106
|
+
|
1107
|
+
data(
|
1108
|
+
"Buffered with single thread",
|
1109
|
+
{
|
1110
|
+
output_type: :full,
|
1111
|
+
config: config_element(
|
1112
|
+
"ROOT", "", {},
|
1113
|
+
[
|
1114
|
+
config_element("buffer", "", {}),
|
1115
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1116
|
+
]
|
1117
|
+
),
|
1118
|
+
expected: 1,
|
1119
|
+
}
|
1120
|
+
)
|
1121
|
+
data(
|
1122
|
+
"Buffered with multiple threads",
|
1123
|
+
{
|
1124
|
+
output_type: :full,
|
1125
|
+
config: config_element(
|
1126
|
+
"ROOT", "", {},
|
1127
|
+
[
|
1128
|
+
config_element("buffer", "", {"flush_thread_count" => 8}),
|
1129
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1130
|
+
]
|
1131
|
+
),
|
1132
|
+
expected: 8,
|
1133
|
+
}
|
1134
|
+
)
|
1135
|
+
test "actual_flush_thread_count for secondary" do |data|
|
1136
|
+
primary = create_output(data[:output_type])
|
1137
|
+
primary.configure(data[:config])
|
1138
|
+
assert_equal data[:expected], primary.secondary.actual_flush_thread_count
|
1139
|
+
end
|
1140
|
+
end
|
1141
|
+
|
1142
|
+
sub_test_case "synchronize_path" do
|
1143
|
+
def setup
|
1144
|
+
Dir.mktmpdir do |lock_dir|
|
1145
|
+
ENV['FLUENTD_LOCK_DIR'] = lock_dir
|
1146
|
+
yield
|
1147
|
+
end
|
1148
|
+
end
|
1149
|
+
|
1150
|
+
def assert_worker_lock(lock_path, expect_locked)
|
1151
|
+
# With LOCK_NB set, flock() returns:
|
1152
|
+
# * `false` when the file is already locked.
|
1153
|
+
# * `0` when the file is not locked.
|
1154
|
+
File.open(lock_path, "w") do |f|
|
1155
|
+
if expect_locked
|
1156
|
+
assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)
|
1157
|
+
else
|
1158
|
+
assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)
|
1159
|
+
end
|
1160
|
+
end
|
1161
|
+
end
|
1162
|
+
|
1163
|
+
def assert_thread_lock(output_plugin, expect_locked)
|
1164
|
+
t = Thread.new do
|
1165
|
+
output_plugin.synchronize_path("test") do
|
1166
|
+
end
|
1167
|
+
end
|
1168
|
+
if expect_locked
|
1169
|
+
assert_nil t.join(3)
|
1170
|
+
else
|
1171
|
+
assert_not_nil t.join(3)
|
1172
|
+
end
|
1173
|
+
end
|
1174
|
+
|
1175
|
+
data(
|
1176
|
+
"Not buffered with single worker",
|
1177
|
+
{
|
1178
|
+
output_type: :sync,
|
1179
|
+
config: config_element(),
|
1180
|
+
workers: 1,
|
1181
|
+
expect_worker_lock: false,
|
1182
|
+
expect_thread_lock: false,
|
1183
|
+
}
|
1184
|
+
)
|
1185
|
+
data(
|
1186
|
+
"Not buffered with multiple workers",
|
1187
|
+
{
|
1188
|
+
output_type: :sync,
|
1189
|
+
config: config_element(),
|
1190
|
+
workers: 4,
|
1191
|
+
expect_worker_lock: true,
|
1192
|
+
expect_thread_lock: false,
|
1193
|
+
}
|
1194
|
+
)
|
1195
|
+
data(
|
1196
|
+
"Buffered with single thread and single worker",
|
1197
|
+
{
|
1198
|
+
output_type: :full,
|
1199
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
|
1200
|
+
workers: 1,
|
1201
|
+
expect_worker_lock: false,
|
1202
|
+
expect_thread_lock: false,
|
1203
|
+
}
|
1204
|
+
)
|
1205
|
+
data(
|
1206
|
+
"Buffered with multiple threads and single worker",
|
1207
|
+
{
|
1208
|
+
output_type: :full,
|
1209
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
|
1210
|
+
workers: 1,
|
1211
|
+
expect_worker_lock: false,
|
1212
|
+
expect_thread_lock: true,
|
1213
|
+
}
|
1214
|
+
)
|
1215
|
+
data(
|
1216
|
+
"Buffered with single thread and multiple workers",
|
1217
|
+
{
|
1218
|
+
output_type: :full,
|
1219
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
|
1220
|
+
workers: 4,
|
1221
|
+
expect_worker_lock: true,
|
1222
|
+
expect_thread_lock: false,
|
1223
|
+
}
|
1224
|
+
)
|
1225
|
+
data(
|
1226
|
+
"Buffered with multiple threads and multiple workers",
|
1227
|
+
{
|
1228
|
+
output_type: :full,
|
1229
|
+
config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
|
1230
|
+
workers: 4,
|
1231
|
+
expect_worker_lock: true,
|
1232
|
+
expect_thread_lock: true,
|
1233
|
+
}
|
1234
|
+
)
|
1235
|
+
test "synchronize_path" do |data|
|
1236
|
+
o = create_output(data[:output_type])
|
1237
|
+
o.configure(data[:config])
|
1238
|
+
o.system_config_override(workers: data[:workers])
|
1239
|
+
|
1240
|
+
test_lock_name = "test_lock_name"
|
1241
|
+
lock_path = o.get_lock_path(test_lock_name)
|
1242
|
+
|
1243
|
+
o.synchronize_path(test_lock_name) do
|
1244
|
+
assert_worker_lock(lock_path, data[:expect_worker_lock])
|
1245
|
+
assert_thread_lock(o, data[:expect_thread_lock])
|
1246
|
+
end
|
1247
|
+
|
1248
|
+
assert_worker_lock(lock_path, false)
|
1249
|
+
assert_thread_lock(o, false)
|
1250
|
+
end
|
1251
|
+
|
1252
|
+
data(
|
1253
|
+
"Buffered with single thread and single worker",
|
1254
|
+
{
|
1255
|
+
output_type: :full,
|
1256
|
+
config: config_element(
|
1257
|
+
"ROOT", "", {},
|
1258
|
+
[
|
1259
|
+
config_element("buffer", "", {}),
|
1260
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1261
|
+
]
|
1262
|
+
),
|
1263
|
+
workers: 1,
|
1264
|
+
expect_worker_lock: false,
|
1265
|
+
expect_thread_lock: false,
|
1266
|
+
}
|
1267
|
+
)
|
1268
|
+
data(
|
1269
|
+
"Buffered with multiple threads and single worker",
|
1270
|
+
{
|
1271
|
+
output_type: :full,
|
1272
|
+
config: config_element(
|
1273
|
+
"ROOT", "", {},
|
1274
|
+
[
|
1275
|
+
config_element("buffer", "", {"flush_thread_count" => 8}),
|
1276
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1277
|
+
]
|
1278
|
+
),
|
1279
|
+
workers: 1,
|
1280
|
+
expect_worker_lock: false,
|
1281
|
+
expect_thread_lock: true,
|
1282
|
+
}
|
1283
|
+
)
|
1284
|
+
data(
|
1285
|
+
"Buffered with single thread and multiple workers",
|
1286
|
+
{
|
1287
|
+
output_type: :full,
|
1288
|
+
config: config_element(
|
1289
|
+
"ROOT", "", {},
|
1290
|
+
[
|
1291
|
+
config_element("buffer", "", {}),
|
1292
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1293
|
+
]
|
1294
|
+
),
|
1295
|
+
workers: 4,
|
1296
|
+
expect_worker_lock: true,
|
1297
|
+
expect_thread_lock: false,
|
1298
|
+
}
|
1299
|
+
)
|
1300
|
+
data(
|
1301
|
+
"Buffered with multiple threads and multiple workers",
|
1302
|
+
{
|
1303
|
+
output_type: :full,
|
1304
|
+
config: config_element(
|
1305
|
+
"ROOT", "", {},
|
1306
|
+
[
|
1307
|
+
config_element("buffer", "", {"flush_thread_count" => 8}),
|
1308
|
+
config_element("secondary", "", {"@type" => "test", "name" => "test"}),
|
1309
|
+
]
|
1310
|
+
),
|
1311
|
+
workers: 4,
|
1312
|
+
expect_worker_lock: true,
|
1313
|
+
expect_thread_lock: true,
|
1314
|
+
}
|
1315
|
+
)
|
1316
|
+
test "synchronize_path for secondary" do |data|
|
1317
|
+
primary = create_output(data[:output_type])
|
1318
|
+
primary.configure(data[:config])
|
1319
|
+
secondary = primary.secondary
|
1320
|
+
secondary.system_config_override(workers: data[:workers])
|
1321
|
+
|
1322
|
+
test_lock_name = "test_lock_name"
|
1323
|
+
lock_path = secondary.get_lock_path(test_lock_name)
|
1324
|
+
|
1325
|
+
secondary.synchronize_path(test_lock_name) do
|
1326
|
+
assert_worker_lock(lock_path, data[:expect_worker_lock])
|
1327
|
+
assert_thread_lock(secondary, data[:expect_thread_lock])
|
1328
|
+
end
|
1329
|
+
|
1330
|
+
assert_worker_lock(lock_path, false)
|
1331
|
+
assert_thread_lock(secondary, false)
|
1332
|
+
end
|
1333
|
+
end
|
1065
1334
|
end
|
@@ -510,7 +510,7 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
510
510
|
logs = @i.log.out.logs.dup
|
511
511
|
@i.start
|
512
512
|
@i.after_start
|
513
|
-
assert{ logs.
|
513
|
+
assert{ logs.count{|log| log.include?('[warn]') } == 0 }
|
514
514
|
end
|
515
515
|
|
516
516
|
test 'a warning reported with 4 chunk keys' do
|
@@ -522,7 +522,7 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
522
522
|
@i.after_start
|
523
523
|
assert_equal ['key1', 'key2', 'key3', 'key4'], @i.chunk_keys
|
524
524
|
|
525
|
-
assert{ logs.
|
525
|
+
assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }
|
526
526
|
end
|
527
527
|
|
528
528
|
test 'a warning reported with 4 chunk keys including "tag"' do
|
@@ -531,7 +531,7 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
531
531
|
logs = @i.log.out.logs.dup
|
532
532
|
@i.start # this calls `log.reset`... capturing logs about configure must be done before this line
|
533
533
|
@i.after_start
|
534
|
-
assert{ logs.
|
534
|
+
assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }
|
535
535
|
end
|
536
536
|
|
537
537
|
test 'time key is not included for warned chunk keys' do
|
@@ -540,7 +540,7 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
540
540
|
logs = @i.log.out.logs.dup
|
541
541
|
@i.start
|
542
542
|
@i.after_start
|
543
|
-
assert{ logs.
|
543
|
+
assert{ logs.count{|log| log.include?('[warn]') } == 0 }
|
544
544
|
end
|
545
545
|
end
|
546
546
|
|
@@ -968,8 +968,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
968
968
|
waiting(4){ sleep 0.1 until ary.size == 3 }
|
969
969
|
|
970
970
|
assert_equal 3, ary.size
|
971
|
-
assert_equal 2, ary.
|
972
|
-
assert_equal 1, ary.
|
971
|
+
assert_equal 2, ary.count{|e| e[0] == "test.tag.1" }
|
972
|
+
assert_equal 1, ary.count{|e| e[0] == "test.tag.2" }
|
973
973
|
|
974
974
|
Timecop.freeze( Time.parse('2016-04-13 14:04:04 +0900') )
|
975
975
|
|
@@ -985,8 +985,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
985
985
|
assert{ @i.buffer.stage.size == 1 && @i.write_count == 2 }
|
986
986
|
|
987
987
|
assert_equal 9, ary.size
|
988
|
-
assert_equal 7, ary.
|
989
|
-
assert_equal 2, ary.
|
988
|
+
assert_equal 7, ary.count{|e| e[0] == "test.tag.1" }
|
989
|
+
assert_equal 2, ary.count{|e| e[0] == "test.tag.2" }
|
990
990
|
|
991
991
|
assert metachecks.all?{|e| e }
|
992
992
|
end
|
@@ -1224,8 +1224,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1224
1224
|
|
1225
1225
|
# events fulfills a chunk (and queued immediately)
|
1226
1226
|
assert_equal 5, ary.size
|
1227
|
-
assert_equal 5, ary.
|
1228
|
-
assert_equal 0, ary.
|
1227
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
1228
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
1229
1229
|
|
1230
1230
|
Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
|
1231
1231
|
|
@@ -1249,8 +1249,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1249
1249
|
assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 }
|
1250
1250
|
|
1251
1251
|
assert_equal 11, ary.size
|
1252
|
-
assert_equal 8, ary.
|
1253
|
-
assert_equal 3, ary.
|
1252
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
1253
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
1254
1254
|
|
1255
1255
|
assert metachecks.all?{|e| e }
|
1256
1256
|
end
|
@@ -1315,8 +1315,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1315
1315
|
|
1316
1316
|
# events fulfills a chunk (and queued immediately)
|
1317
1317
|
assert_equal 5, ary.size
|
1318
|
-
assert_equal 5, ary.
|
1319
|
-
assert_equal 0, ary.
|
1318
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
1319
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
1320
1320
|
|
1321
1321
|
@i.stop
|
1322
1322
|
@i.before_shutdown
|
@@ -1330,8 +1330,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1330
1330
|
assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 3 }
|
1331
1331
|
|
1332
1332
|
assert_equal 11, ary.size
|
1333
|
-
assert_equal 8, ary.
|
1334
|
-
assert_equal 3, ary.
|
1333
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
1334
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
1335
1335
|
|
1336
1336
|
assert metachecks.all?{|e| e }
|
1337
1337
|
end
|
@@ -1435,8 +1435,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1435
1435
|
|
1436
1436
|
# events fulfills a chunk (and queued immediately)
|
1437
1437
|
assert_equal 5, ary.size
|
1438
|
-
assert_equal 5, ary.
|
1439
|
-
assert_equal 0, ary.
|
1438
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
1439
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
1440
1440
|
assert ary[0...5].all?{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
|
1441
1441
|
|
1442
1442
|
Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
|
@@ -1465,11 +1465,11 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1465
1465
|
assert{ @i.buffer.stage.size == 0 && @i.write_count == 4 }
|
1466
1466
|
|
1467
1467
|
assert_equal 11, ary.size
|
1468
|
-
assert_equal 8, ary.
|
1469
|
-
assert_equal 3, ary.
|
1470
|
-
assert_equal 6, ary.
|
1471
|
-
assert_equal 3, ary.
|
1472
|
-
assert_equal 2, ary.
|
1468
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
1469
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
1470
|
+
assert_equal 6, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
|
1471
|
+
assert_equal 3, ary.count{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }
|
1472
|
+
assert_equal 2, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }
|
1473
1473
|
|
1474
1474
|
assert metachecks.all?{|e| e }
|
1475
1475
|
end
|
@@ -1525,8 +1525,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1525
1525
|
|
1526
1526
|
# events fulfills a chunk (and queued immediately)
|
1527
1527
|
assert_equal 5, ary.size
|
1528
|
-
assert_equal 5, ary.
|
1529
|
-
assert_equal 0, ary.
|
1528
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
1529
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
1530
1530
|
|
1531
1531
|
@i.stop
|
1532
1532
|
@i.before_shutdown
|
@@ -1540,11 +1540,11 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1540
1540
|
assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 4 }
|
1541
1541
|
|
1542
1542
|
assert_equal 11, ary.size
|
1543
|
-
assert_equal 8, ary.
|
1544
|
-
assert_equal 3, ary.
|
1545
|
-
assert_equal 6, ary.
|
1546
|
-
assert_equal 3, ary.
|
1547
|
-
assert_equal 2, ary.
|
1543
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
1544
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
1545
|
+
assert_equal 6, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
|
1546
|
+
assert_equal 3, ary.count{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }
|
1547
|
+
assert_equal 2, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }
|
1548
1548
|
|
1549
1549
|
assert metachecks.all?{|e| e }
|
1550
1550
|
end
|
@@ -1683,8 +1683,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1683
1683
|
|
1684
1684
|
# events fulfills a chunk (and queued immediately)
|
1685
1685
|
assert_equal 5, ary.size
|
1686
|
-
assert_equal 5, ary.
|
1687
|
-
assert_equal 0, ary.
|
1686
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
1687
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
1688
1688
|
|
1689
1689
|
assert_equal 1, chunks.size
|
1690
1690
|
assert !chunks.first.empty?
|
@@ -1716,8 +1716,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1716
1716
|
assert{ @i.buffer.dequeued.size == 3 }
|
1717
1717
|
|
1718
1718
|
assert_equal 11, ary.size
|
1719
|
-
assert_equal 8, ary.
|
1720
|
-
assert_equal 3, ary.
|
1719
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
1720
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
1721
1721
|
|
1722
1722
|
assert_equal 3, chunks.size
|
1723
1723
|
assert chunks.all?{|c| !c.empty? }
|
@@ -1802,8 +1802,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1802
1802
|
|
1803
1803
|
# events fulfills a chunk (and queued immediately)
|
1804
1804
|
assert_equal 5, ary.size
|
1805
|
-
assert_equal 5, ary.
|
1806
|
-
assert_equal 0, ary.
|
1805
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
1806
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
1807
1807
|
|
1808
1808
|
assert_equal 1, chunks.size
|
1809
1809
|
assert !chunks.first.empty?
|
@@ -1835,8 +1835,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1835
1835
|
assert{ @i.buffer.dequeued.size == 3 }
|
1836
1836
|
|
1837
1837
|
assert_equal 11, ary.size
|
1838
|
-
assert_equal 8, ary.
|
1839
|
-
assert_equal 3, ary.
|
1838
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
1839
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
1840
1840
|
|
1841
1841
|
assert_equal 3, chunks.size
|
1842
1842
|
assert chunks.all?{|c| !c.empty? }
|
@@ -1892,8 +1892,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1892
1892
|
|
1893
1893
|
assert{ @i.write_count == 7 }
|
1894
1894
|
assert_equal 11, ary.size
|
1895
|
-
assert_equal 8, ary.
|
1896
|
-
assert_equal 3, ary.
|
1895
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
1896
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
1897
1897
|
assert{ chunks.size == 3 }
|
1898
1898
|
assert{ chunks.all?{|c| !c.empty? } }
|
1899
1899
|
|
@@ -1963,8 +1963,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1963
1963
|
|
1964
1964
|
# events fulfills a chunk (and queued immediately)
|
1965
1965
|
assert_equal 5, ary.size
|
1966
|
-
assert_equal 5, ary.
|
1967
|
-
assert_equal 0, ary.
|
1966
|
+
assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
|
1967
|
+
assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
|
1968
1968
|
|
1969
1969
|
assert_equal 1, chunks.size
|
1970
1970
|
assert !chunks.first.empty?
|
@@ -1999,8 +1999,8 @@ class BufferedOutputTest < Test::Unit::TestCase
|
|
1999
1999
|
assert{ @i.rollback_count == 0 }
|
2000
2000
|
|
2001
2001
|
assert_equal 11, ary.size
|
2002
|
-
assert_equal 8, ary.
|
2003
|
-
assert_equal 3, ary.
|
2002
|
+
assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
|
2003
|
+
assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
|
2004
2004
|
|
2005
2005
|
assert{ chunks.size == 3 }
|
2006
2006
|
assert{ chunks.all?{|c| !c.empty? } }
|