fluentd 1.15.2-x86-mingw32 → 1.16.1-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
  4. data/.github/workflows/linux-test.yaml +2 -2
  5. data/.github/workflows/macos-test.yaml +2 -2
  6. data/.github/workflows/stale-actions.yml +11 -9
  7. data/.github/workflows/windows-test.yaml +2 -2
  8. data/CHANGELOG.md +133 -0
  9. data/CONTRIBUTING.md +1 -1
  10. data/MAINTAINERS.md +5 -3
  11. data/README.md +0 -1
  12. data/SECURITY.md +5 -9
  13. data/fluentd.gemspec +2 -2
  14. data/lib/fluent/command/fluentd.rb +55 -64
  15. data/lib/fluent/config/yaml_parser/loader.rb +18 -1
  16. data/lib/fluent/daemon.rb +2 -4
  17. data/lib/fluent/event.rb +2 -2
  18. data/lib/fluent/file_wrapper.rb +137 -0
  19. data/lib/fluent/log/console_adapter.rb +66 -0
  20. data/lib/fluent/log.rb +35 -5
  21. data/lib/fluent/oj_options.rb +1 -2
  22. data/lib/fluent/plugin/base.rb +5 -7
  23. data/lib/fluent/plugin/buf_file.rb +32 -3
  24. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  25. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  26. data/lib/fluent/plugin/buffer.rb +21 -0
  27. data/lib/fluent/plugin/in_tail.rb +1 -6
  28. data/lib/fluent/plugin/in_tcp.rb +47 -2
  29. data/lib/fluent/plugin/out_file.rb +0 -4
  30. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  31. data/lib/fluent/plugin/out_forward.rb +2 -2
  32. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  33. data/lib/fluent/plugin/output.rb +49 -12
  34. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  35. data/lib/fluent/plugin_helper/server.rb +8 -0
  36. data/lib/fluent/supervisor.rb +157 -232
  37. data/lib/fluent/test/driver/base.rb +11 -5
  38. data/lib/fluent/test/driver/filter.rb +4 -0
  39. data/lib/fluent/test/startup_shutdown.rb +6 -8
  40. data/lib/fluent/version.rb +1 -1
  41. data/templates/new_gem/test/helper.rb.erb +0 -1
  42. data/test/command/test_ctl.rb +1 -1
  43. data/test/command/test_fluentd.rb +168 -22
  44. data/test/command/test_plugin_config_formatter.rb +0 -1
  45. data/test/compat/test_parser.rb +5 -5
  46. data/test/config/test_system_config.rb +0 -8
  47. data/test/log/test_console_adapter.rb +110 -0
  48. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  49. data/test/plugin/test_base.rb +98 -0
  50. data/test/plugin/test_buf_file.rb +62 -23
  51. data/test/plugin/test_buf_file_single.rb +65 -0
  52. data/test/plugin/test_in_http.rb +2 -3
  53. data/test/plugin/test_in_monitor_agent.rb +2 -3
  54. data/test/plugin/test_in_tail.rb +105 -103
  55. data/test/plugin/test_in_tcp.rb +87 -2
  56. data/test/plugin/test_in_udp.rb +28 -0
  57. data/test/plugin/test_out_file.rb +3 -2
  58. data/test/plugin/test_out_forward.rb +14 -18
  59. data/test/plugin/test_out_http.rb +1 -0
  60. data/test/plugin/test_output.rb +269 -0
  61. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  62. data/test/plugin/test_parser_regexp.rb +1 -6
  63. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  64. data/test/plugin_helper/test_server.rb +59 -5
  65. data/test/test_config.rb +57 -21
  66. data/test/{plugin/test_file_wrapper.rb → test_file_wrapper.rb} +2 -2
  67. data/test/test_formatter.rb +23 -20
  68. data/test/test_log.rb +85 -40
  69. data/test/test_supervisor.rb +300 -283
  70. metadata +15 -23
  71. data/.drone.yml +0 -35
  72. data/.github/workflows/issue-auto-closer.yml +0 -12
  73. data/.gitlab-ci.yml +0 -103
  74. data/lib/fluent/plugin/file_wrapper.rb +0 -131
  75. data/test/test_logger_initializer.rb +0 -46
@@ -156,6 +156,19 @@ class TcpInputTest < Test::Unit::TestCase
156
156
  assert_equal hostname, event[2]['host']
157
157
  end
158
158
 
159
+ test "send_keepalive_packet_can_be_enabled" do
160
+ d = create_driver(base_config + %!
161
+ format none
162
+ send_keepalive_packet true
163
+ !)
164
+ assert_true d.instance.send_keepalive_packet
165
+
166
+ d = create_driver(base_config + %!
167
+ format none
168
+ !)
169
+ assert_false d.instance.send_keepalive_packet
170
+ end
171
+
159
172
  test 'source_address_key' do
160
173
  d = create_driver(base_config + %!
161
174
  format none
@@ -205,13 +218,13 @@ class TcpInputTest < Test::Unit::TestCase
205
218
  </client>
206
219
  </security>
207
220
  !)
208
- d.run(shutdown: false, expect_records: 1, timeout: 2) do
221
+ d.run(expect_records: 1, timeout: 2) do
209
222
  create_tcp_socket('127.0.0.1', @port) do |sock|
210
223
  sock.send("hello\n", 0)
211
224
  end
212
225
  end
213
226
 
214
- assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
227
+ assert_equal 1, d.logs.count { |l| l =~ /anonymous client/ }
215
228
  assert_equal 0, d.events.size
216
229
  end
217
230
  end
@@ -240,4 +253,76 @@ class TcpInputTest < Test::Unit::TestCase
240
253
  assert_equal 'hello', event[2]['msg']
241
254
  end
242
255
  end
256
+
257
+ sub_test_case "message_length_limit" do
258
+ data("batch_emit", { extract: "" }, keep: true)
259
+ data("single_emit", { extract: "<extract>\ntag_key tag\n</extract>\n" }, keep: true)
260
+ test "drop records exceeding limit" do |data|
261
+ message_length_limit = 10
262
+ d = create_driver(base_config + %!
263
+ message_length_limit #{message_length_limit}
264
+ <parse>
265
+ @type none
266
+ </parse>
267
+ #{data[:extract]}
268
+ !)
269
+ d.run(expect_records: 2, timeout: 10) do
270
+ create_tcp_socket('127.0.0.1', @port) do |sock|
271
+ sock.send("a" * message_length_limit + "\n", 0)
272
+ sock.send("b" * (message_length_limit + 1) + "\n", 0)
273
+ sock.send("c" * (message_length_limit - 1) + "\n", 0)
274
+ end
275
+ end
276
+
277
+ expected_records = [
278
+ "a" * message_length_limit,
279
+ "c" * (message_length_limit - 1)
280
+ ]
281
+ actual_records = d.events.collect do |event|
282
+ event[2]["message"]
283
+ end
284
+
285
+ assert_equal expected_records, actual_records
286
+ end
287
+
288
+ test "clear buffer and discard the subsequent data until the next delimiter" do |data|
289
+ message_length_limit = 12
290
+ d = create_driver(base_config + %!
291
+ message_length_limit #{message_length_limit}
292
+ delimiter ";"
293
+ <parse>
294
+ @type json
295
+ </parse>
296
+ #{data[:extract]}
297
+ !)
298
+ d.run(expect_records: 1, timeout: 10) do
299
+ create_tcp_socket('127.0.0.1', @port) do |sock|
300
+ sock.send('{"message":', 0)
301
+ sock.send('"hello', 0)
302
+ sleep 1 # To make the server read data and clear the buffer here.
303
+ sock.send('world!"};', 0) # This subsequent data must be discarded so that a parsing failure doesn't occur.
304
+ sock.send('{"k":"v"};', 0) # This will succeed to parse.
305
+ end
306
+ end
307
+
308
+ logs = d.logs.collect do |log|
309
+ log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
310
+ end
311
+ actual_records = d.events.collect do |event|
312
+ event[2]
313
+ end
314
+
315
+ assert_equal(
316
+ {
317
+ # Asserting that '[warn]: pattern not matched message="world!\"}"' warning does not occur.
318
+ logs: ['[info]: The buffer size exceeds \'message_length_limit\', cleared: limit=12 size=17 head="{\"message\":\"hello"' + "\n"],
319
+ records: [{"k" => "v"}],
320
+ },
321
+ {
322
+ logs: logs[1..],
323
+ records: actual_records,
324
+ }
325
+ )
326
+ end
327
+ end
243
328
  end
@@ -265,4 +265,32 @@ class UdpInputTest < Test::Unit::TestCase
265
265
  end
266
266
  end
267
267
  end
268
+
269
+ test 'message_length_limit' do
270
+ message_length_limit = 32
271
+ d = create_driver(base_config + %!
272
+ format none
273
+ message_length_limit #{message_length_limit}
274
+ !)
275
+ d.run(expect_records: 3) do
276
+ create_udp_socket('127.0.0.1', @port) do |u|
277
+ 3.times do |i|
278
+ u.send("#{i}" * 40 + "\n", 0)
279
+ end
280
+ end
281
+ end
282
+
283
+ if Fluent.windows?
284
+ expected_records = []
285
+ else
286
+ expected_records = 3.times.collect do |i|
287
+ "#{i}" * message_length_limit
288
+ end
289
+ end
290
+ actual_records = d.events.collect do |event|
291
+ event[2]["message"]
292
+ end
293
+
294
+ assert_equal expected_records, actual_records
295
+ end
268
296
  end
@@ -5,6 +5,7 @@ require 'fileutils'
5
5
  require 'time'
6
6
  require 'timecop'
7
7
  require 'zlib'
8
+ require 'fluent/file_wrapper'
8
9
 
9
10
  class FileOutputTest < Test::Unit::TestCase
10
11
  def setup
@@ -1016,7 +1017,7 @@ class FileOutputTest < Test::Unit::TestCase
1016
1017
 
1017
1018
  test 'returns filepath with index which does not exist yet' do
1018
1019
  5.times do |i|
1019
- File.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
1020
+ Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
1020
1021
  end
1021
1022
  @i.find_filepath_available(File.join(@tmp, "exist_**.log")) do |path|
1022
1023
  assert_equal File.join(@tmp, "exist_5.log"), path
@@ -1025,7 +1026,7 @@ class FileOutputTest < Test::Unit::TestCase
1025
1026
 
1026
1027
  test 'creates lock directory when with_lock is true to exclude operations of other worker process' do
1027
1028
  5.times do |i|
1028
- File.open(File.join(@tmp, "exist_#{i}.log"), 'a')
1029
+ Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a')
1029
1030
  end
1030
1031
  Dir.mkdir(File.join(@tmp, "exist_5.log.lock"))
1031
1032
  @i.find_filepath_available(File.join(@tmp, "exist_**.log"), with_lock: true) do |path|
@@ -1331,26 +1331,22 @@ EOL
1331
1331
  d = create_driver(output_conf)
1332
1332
  d.instance_start
1333
1333
 
1334
- begin
1335
- chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1336
- mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
1337
- linger_timeout: anything,
1338
- send_timeout: anything,
1339
- recv_timeout: anything,
1340
- connect_timeout: anything) { |sock|
1341
- mock(sock).close.once; sock
1342
- }.twice
1343
-
1344
- target_input_driver.run(timeout: 15) do
1345
- d.run(shutdown: false) do
1346
- node = d.instance.nodes.first
1347
- 2.times do
1348
- node.send_data('test', chunk) rescue nil
1349
- end
1334
+ chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
1335
+ mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
1336
+ linger_timeout: anything,
1337
+ send_timeout: anything,
1338
+ recv_timeout: anything,
1339
+ connect_timeout: anything) { |sock|
1340
+ mock(sock).close.once; sock
1341
+ }.twice
1342
+
1343
+ target_input_driver.run(timeout: 15) do
1344
+ d.run do
1345
+ node = d.instance.nodes.first
1346
+ 2.times do
1347
+ node.send_data('test', chunk) rescue nil
1350
1348
  end
1351
1349
  end
1352
- ensure
1353
- d.instance_shutdown
1354
1350
  end
1355
1351
  end
1356
1352
  end
@@ -378,6 +378,7 @@ class HTTPOutputTest < Test::Unit::TestCase
378
378
  password hello?
379
379
  </auth>
380
380
  ])
381
+ d.instance.system_config_override(root_dir: TMP_DIR) # Backup files are generated in TMP_DIR.
381
382
  d.run(default_tag: 'test.http', shutdown: false) do
382
383
  test_events.each { |event|
383
384
  d.feed(event)
@@ -803,7 +803,10 @@ class OutputTest < Test::Unit::TestCase
803
803
  end
804
804
 
805
805
  test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
806
+ tmp_dir = File.join(__dir__, '../tmp/test_output')
807
+
806
808
  i = create_output(:delayed)
809
+ i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
807
810
  try_write_called = false
808
811
  i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
809
812
 
@@ -820,6 +823,8 @@ class OutputTest < Test::Unit::TestCase
820
823
  assert try_write_called
821
824
 
822
825
  i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
826
+ ensure
827
+ FileUtils.rm_rf(tmp_dir)
823
828
  end
824
829
 
825
830
  test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
@@ -849,7 +854,10 @@ class OutputTest < Test::Unit::TestCase
849
854
  end
850
855
 
851
856
  test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
857
+ tmp_dir = File.join(__dir__, '../tmp/test_output')
858
+
852
859
  i = create_output(:full)
860
+ i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
853
861
  write_called = false
854
862
  try_write_called = false
855
863
  i.register(:write){ |chunk| write_called = true }
@@ -872,6 +880,8 @@ class OutputTest < Test::Unit::TestCase
872
880
  assert try_write_called
873
881
 
874
882
  i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
883
+ ensure
884
+ FileUtils.rm_rf(tmp_dir)
875
885
  end
876
886
 
877
887
  test 'flush_interval is ignored when flush_mode is not interval' do
@@ -1062,4 +1072,263 @@ class OutputTest < Test::Unit::TestCase
1062
1072
  }
1063
1073
  end
1064
1074
  end
1075
+
1076
+ sub_test_case "actual_flush_thread_count" do
1077
+ data(
1078
+ "Not buffered",
1079
+ {
1080
+ output_type: :sync,
1081
+ config: config_element(),
1082
+ expected: 0,
1083
+ }
1084
+ )
1085
+ data(
1086
+ "Buffered with singile thread",
1087
+ {
1088
+ output_type: :full,
1089
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
1090
+ expected: 1,
1091
+ }
1092
+ )
1093
+ data(
1094
+ "Buffered with multiple threads",
1095
+ {
1096
+ output_type: :full,
1097
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
1098
+ expected: 8,
1099
+ }
1100
+ )
1101
+ test "actual_flush_thread_count" do |data|
1102
+ o = create_output(data[:output_type])
1103
+ o.configure(data[:config])
1104
+ assert_equal data[:expected], o.actual_flush_thread_count
1105
+ end
1106
+
1107
+ data(
1108
+ "Buffered with single thread",
1109
+ {
1110
+ output_type: :full,
1111
+ config: config_element(
1112
+ "ROOT", "", {},
1113
+ [
1114
+ config_element("buffer", "", {}),
1115
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1116
+ ]
1117
+ ),
1118
+ expected: 1,
1119
+ }
1120
+ )
1121
+ data(
1122
+ "Buffered with multiple threads",
1123
+ {
1124
+ output_type: :full,
1125
+ config: config_element(
1126
+ "ROOT", "", {},
1127
+ [
1128
+ config_element("buffer", "", {"flush_thread_count" => 8}),
1129
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1130
+ ]
1131
+ ),
1132
+ expected: 8,
1133
+ }
1134
+ )
1135
+ test "actual_flush_thread_count for secondary" do |data|
1136
+ primary = create_output(data[:output_type])
1137
+ primary.configure(data[:config])
1138
+ assert_equal data[:expected], primary.secondary.actual_flush_thread_count
1139
+ end
1140
+ end
1141
+
1142
+ sub_test_case "synchronize_path" do
1143
+ def setup
1144
+ Dir.mktmpdir do |lock_dir|
1145
+ ENV['FLUENTD_LOCK_DIR'] = lock_dir
1146
+ yield
1147
+ end
1148
+ end
1149
+
1150
+ def assert_worker_lock(lock_path, expect_locked)
1151
+ # With LOCK_NB set, flock() returns:
1152
+ # * `false` when the file is already locked.
1153
+ # * `0` when the file is not locked.
1154
+ File.open(lock_path, "w") do |f|
1155
+ if expect_locked
1156
+ assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)
1157
+ else
1158
+ assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)
1159
+ end
1160
+ end
1161
+ end
1162
+
1163
+ def assert_thread_lock(output_plugin, expect_locked)
1164
+ t = Thread.new do
1165
+ output_plugin.synchronize_path("test") do
1166
+ end
1167
+ end
1168
+ if expect_locked
1169
+ assert_nil t.join(3)
1170
+ else
1171
+ assert_not_nil t.join(3)
1172
+ end
1173
+ end
1174
+
1175
+ data(
1176
+ "Not buffered with single worker",
1177
+ {
1178
+ output_type: :sync,
1179
+ config: config_element(),
1180
+ workers: 1,
1181
+ expect_worker_lock: false,
1182
+ expect_thread_lock: false,
1183
+ }
1184
+ )
1185
+ data(
1186
+ "Not buffered with multiple workers",
1187
+ {
1188
+ output_type: :sync,
1189
+ config: config_element(),
1190
+ workers: 4,
1191
+ expect_worker_lock: true,
1192
+ expect_thread_lock: false,
1193
+ }
1194
+ )
1195
+ data(
1196
+ "Buffered with single thread and single worker",
1197
+ {
1198
+ output_type: :full,
1199
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
1200
+ workers: 1,
1201
+ expect_worker_lock: false,
1202
+ expect_thread_lock: false,
1203
+ }
1204
+ )
1205
+ data(
1206
+ "Buffered with multiple threads and single worker",
1207
+ {
1208
+ output_type: :full,
1209
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
1210
+ workers: 1,
1211
+ expect_worker_lock: false,
1212
+ expect_thread_lock: true,
1213
+ }
1214
+ )
1215
+ data(
1216
+ "Buffered with single thread and multiple workers",
1217
+ {
1218
+ output_type: :full,
1219
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
1220
+ workers: 4,
1221
+ expect_worker_lock: true,
1222
+ expect_thread_lock: false,
1223
+ }
1224
+ )
1225
+ data(
1226
+ "Buffered with multiple threads and multiple workers",
1227
+ {
1228
+ output_type: :full,
1229
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
1230
+ workers: 4,
1231
+ expect_worker_lock: true,
1232
+ expect_thread_lock: true,
1233
+ }
1234
+ )
1235
+ test "synchronize_path" do |data|
1236
+ o = create_output(data[:output_type])
1237
+ o.configure(data[:config])
1238
+ o.system_config_override(workers: data[:workers])
1239
+
1240
+ test_lock_name = "test_lock_name"
1241
+ lock_path = o.get_lock_path(test_lock_name)
1242
+
1243
+ o.synchronize_path(test_lock_name) do
1244
+ assert_worker_lock(lock_path, data[:expect_worker_lock])
1245
+ assert_thread_lock(o, data[:expect_thread_lock])
1246
+ end
1247
+
1248
+ assert_worker_lock(lock_path, false)
1249
+ assert_thread_lock(o, false)
1250
+ end
1251
+
1252
+ data(
1253
+ "Buffered with single thread and single worker",
1254
+ {
1255
+ output_type: :full,
1256
+ config: config_element(
1257
+ "ROOT", "", {},
1258
+ [
1259
+ config_element("buffer", "", {}),
1260
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1261
+ ]
1262
+ ),
1263
+ workers: 1,
1264
+ expect_worker_lock: false,
1265
+ expect_thread_lock: false,
1266
+ }
1267
+ )
1268
+ data(
1269
+ "Buffered with multiple threads and single worker",
1270
+ {
1271
+ output_type: :full,
1272
+ config: config_element(
1273
+ "ROOT", "", {},
1274
+ [
1275
+ config_element("buffer", "", {"flush_thread_count" => 8}),
1276
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1277
+ ]
1278
+ ),
1279
+ workers: 1,
1280
+ expect_worker_lock: false,
1281
+ expect_thread_lock: true,
1282
+ }
1283
+ )
1284
+ data(
1285
+ "Buffered with single thread and multiple workers",
1286
+ {
1287
+ output_type: :full,
1288
+ config: config_element(
1289
+ "ROOT", "", {},
1290
+ [
1291
+ config_element("buffer", "", {}),
1292
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1293
+ ]
1294
+ ),
1295
+ workers: 4,
1296
+ expect_worker_lock: true,
1297
+ expect_thread_lock: false,
1298
+ }
1299
+ )
1300
+ data(
1301
+ "Buffered with multiple threads and multiple workers",
1302
+ {
1303
+ output_type: :full,
1304
+ config: config_element(
1305
+ "ROOT", "", {},
1306
+ [
1307
+ config_element("buffer", "", {"flush_thread_count" => 8}),
1308
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1309
+ ]
1310
+ ),
1311
+ workers: 4,
1312
+ expect_worker_lock: true,
1313
+ expect_thread_lock: true,
1314
+ }
1315
+ )
1316
+ test "synchronize_path for secondary" do |data|
1317
+ primary = create_output(data[:output_type])
1318
+ primary.configure(data[:config])
1319
+ secondary = primary.secondary
1320
+ secondary.system_config_override(workers: data[:workers])
1321
+
1322
+ test_lock_name = "test_lock_name"
1323
+ lock_path = secondary.get_lock_path(test_lock_name)
1324
+
1325
+ secondary.synchronize_path(test_lock_name) do
1326
+ assert_worker_lock(lock_path, data[:expect_worker_lock])
1327
+ assert_thread_lock(secondary, data[:expect_thread_lock])
1328
+ end
1329
+
1330
+ assert_worker_lock(lock_path, false)
1331
+ assert_thread_lock(secondary, false)
1332
+ end
1333
+ end
1065
1334
  end
@@ -35,6 +35,16 @@ module FluentPluginOutputAsBufferedCompressTest
35
35
  @format ? @format.call(tag, time, record) : [tag, time, record].to_json
36
36
  end
37
37
  end
38
+
39
+ def self.dummy_event_stream
40
+ Fluent::ArrayEventStream.new(
41
+ [
42
+ [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
43
+ [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
44
+ [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
45
+ ]
46
+ )
47
+ end
38
48
  end
39
49
 
40
50
  class BufferedOutputCompressTest < Test::Unit::TestCase
@@ -60,16 +70,6 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
60
70
  end
61
71
  end
62
72
 
63
- def dummy_event_stream
64
- Fluent::ArrayEventStream.new(
65
- [
66
- [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
67
- [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
68
- [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
69
- ]
70
- )
71
- end
72
-
73
73
  TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
74
74
 
75
75
  setup do
@@ -89,20 +89,34 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
89
89
  end
90
90
 
91
91
  data(
92
- handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
93
- handle_stream_with_standard_format: config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
94
- handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
95
- handle_stream_with_standard_format_and_file_chunk: config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
92
+ :buffer_config,
93
+ [
94
+ config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
95
+ config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
96
+ config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
97
+ config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
98
+ ],
96
99
  )
97
- test 'call a standard format when output plugin adds data to chunk' do |buffer_config|
100
+ data(
101
+ :input_es,
102
+ [
103
+ FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
104
+ # If already compressed data is incoming, it must be written as is (i.e. without decompressed).
105
+ # https://github.com/fluent/fluentd/issues/4146
106
+ Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
107
+ ],
108
+ )
109
+ test 'call a standard format when output plugin adds data to chunk' do |data|
110
+ buffer_config = data[:buffer_config]
111
+ es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
112
+
98
113
  @i = create_output(:async)
99
114
  @i.configure(config_element('ROOT','', {}, [buffer_config]))
100
115
  @i.start
101
116
  @i.after_start
102
117
 
103
118
  io = StringIO.new
104
- es = dummy_event_stream
105
- expected = es.map { |e| e }
119
+ expected = es.dup.map { |t, r| [t, r] }
106
120
  compressed_data = ''
107
121
 
108
122
  assert_equal :gzip, @i.buffer.compress
@@ -138,7 +152,7 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
138
152
  @i.after_start
139
153
 
140
154
  io = StringIO.new
141
- es = dummy_event_stream
155
+ es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
142
156
  expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
143
157
  compressed_data = ''
144
158
 
@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
28
28
  if initialize_conf
29
29
  Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
30
30
  else
31
- # Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
32
- instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
33
- instance.configure(conf)
34
- d = Struct.new(:instance).new
35
- d.instance = instance
36
- d
31
+ Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
37
32
  end
38
33
  end
39
34
 
@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
127
127
  end
128
128
 
129
129
  client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
130
- reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
130
+ reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
131
131
 
132
132
  resp = nil
133
133
  error = nil