fluentd 1.13.3 → 1.16.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (179) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/{bug_report.yaml → bug_report.yml} +2 -0
  3. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  4. data/.github/ISSUE_TEMPLATE/{feature_request.yaml → feature_request.yml} +1 -0
  5. data/.github/workflows/stale-actions.yml +11 -9
  6. data/.github/workflows/test.yml +32 -0
  7. data/CHANGELOG.md +490 -10
  8. data/CONTRIBUTING.md +2 -2
  9. data/MAINTAINERS.md +7 -5
  10. data/README.md +3 -23
  11. data/Rakefile +1 -1
  12. data/SECURITY.md +14 -0
  13. data/fluentd.gemspec +7 -8
  14. data/lib/fluent/command/cat.rb +13 -3
  15. data/lib/fluent/command/ctl.rb +6 -3
  16. data/lib/fluent/command/fluentd.rb +73 -65
  17. data/lib/fluent/command/plugin_config_formatter.rb +1 -1
  18. data/lib/fluent/compat/output.rb +9 -6
  19. data/lib/fluent/config/dsl.rb +1 -1
  20. data/lib/fluent/config/error.rb +12 -0
  21. data/lib/fluent/config/literal_parser.rb +2 -2
  22. data/lib/fluent/config/parser.rb +1 -1
  23. data/lib/fluent/config/v1_parser.rb +3 -3
  24. data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
  25. data/lib/fluent/config/yaml_parser/loader.rb +108 -0
  26. data/lib/fluent/config/yaml_parser/parser.rb +166 -0
  27. data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
  28. data/lib/fluent/config/yaml_parser.rb +56 -0
  29. data/lib/fluent/config.rb +14 -1
  30. data/lib/fluent/counter/server.rb +1 -1
  31. data/lib/fluent/counter/validator.rb +3 -3
  32. data/lib/fluent/daemon.rb +2 -4
  33. data/lib/fluent/engine.rb +1 -1
  34. data/lib/fluent/env.rb +4 -0
  35. data/lib/fluent/error.rb +3 -0
  36. data/lib/fluent/event.rb +8 -4
  37. data/lib/fluent/event_router.rb +47 -2
  38. data/lib/fluent/file_wrapper.rb +137 -0
  39. data/lib/fluent/log/console_adapter.rb +66 -0
  40. data/lib/fluent/log.rb +44 -5
  41. data/lib/fluent/match.rb +1 -1
  42. data/lib/fluent/msgpack_factory.rb +6 -1
  43. data/lib/fluent/oj_options.rb +1 -2
  44. data/lib/fluent/plugin/bare_output.rb +49 -8
  45. data/lib/fluent/plugin/base.rb +26 -9
  46. data/lib/fluent/plugin/buf_file.rb +34 -5
  47. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  48. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  49. data/lib/fluent/plugin/buffer.rb +216 -70
  50. data/lib/fluent/plugin/filter.rb +35 -1
  51. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  52. data/lib/fluent/plugin/in_forward.rb +2 -2
  53. data/lib/fluent/plugin/in_http.rb +39 -10
  54. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  55. data/lib/fluent/plugin/in_sample.rb +1 -1
  56. data/lib/fluent/plugin/in_syslog.rb +13 -1
  57. data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
  58. data/lib/fluent/plugin/in_tail/position_file.rb +33 -33
  59. data/lib/fluent/plugin/in_tail.rb +216 -84
  60. data/lib/fluent/plugin/in_tcp.rb +47 -2
  61. data/lib/fluent/plugin/input.rb +39 -1
  62. data/lib/fluent/plugin/metrics.rb +119 -0
  63. data/lib/fluent/plugin/metrics_local.rb +96 -0
  64. data/lib/fluent/plugin/multi_output.rb +43 -6
  65. data/lib/fluent/plugin/out_copy.rb +1 -1
  66. data/lib/fluent/plugin/out_exec_filter.rb +2 -2
  67. data/lib/fluent/plugin/out_file.rb +20 -2
  68. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  69. data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
  70. data/lib/fluent/plugin/out_forward.rb +17 -9
  71. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  72. data/lib/fluent/plugin/output.rb +167 -78
  73. data/lib/fluent/plugin/parser.rb +3 -4
  74. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  75. data/lib/fluent/plugin/parser_json.rb +1 -1
  76. data/lib/fluent/plugin/parser_syslog.rb +1 -1
  77. data/lib/fluent/plugin/storage_local.rb +3 -5
  78. data/lib/fluent/plugin.rb +10 -1
  79. data/lib/fluent/plugin_helper/child_process.rb +3 -0
  80. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  81. data/lib/fluent/plugin_helper/event_loop.rb +2 -2
  82. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  83. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  84. data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
  85. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  86. data/lib/fluent/plugin_helper/server.rb +35 -6
  87. data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
  88. data/lib/fluent/plugin_helper/socket.rb +13 -2
  89. data/lib/fluent/plugin_helper/thread.rb +3 -3
  90. data/lib/fluent/plugin_helper.rb +1 -0
  91. data/lib/fluent/plugin_id.rb +3 -2
  92. data/lib/fluent/registry.rb +2 -1
  93. data/lib/fluent/root_agent.rb +6 -0
  94. data/lib/fluent/rpc.rb +4 -3
  95. data/lib/fluent/supervisor.rb +283 -259
  96. data/lib/fluent/system_config.rb +13 -3
  97. data/lib/fluent/test/driver/base.rb +11 -5
  98. data/lib/fluent/test/driver/filter.rb +4 -0
  99. data/lib/fluent/test/startup_shutdown.rb +6 -8
  100. data/lib/fluent/time.rb +21 -20
  101. data/lib/fluent/version.rb +1 -1
  102. data/lib/fluent/win32api.rb +38 -0
  103. data/lib/fluent/winsvc.rb +5 -8
  104. data/templates/new_gem/test/helper.rb.erb +0 -1
  105. data/test/command/test_cat.rb +31 -2
  106. data/test/command/test_ctl.rb +1 -2
  107. data/test/command/test_fluentd.rb +209 -24
  108. data/test/command/test_plugin_config_formatter.rb +0 -1
  109. data/test/compat/test_parser.rb +6 -6
  110. data/test/config/test_system_config.rb +13 -11
  111. data/test/config/test_types.rb +1 -1
  112. data/test/log/test_console_adapter.rb +110 -0
  113. data/test/plugin/in_tail/test_io_handler.rb +26 -8
  114. data/test/plugin/in_tail/test_position_file.rb +48 -59
  115. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  116. data/test/plugin/out_forward/test_socket_cache.rb +26 -1
  117. data/test/plugin/test_bare_output.rb +14 -1
  118. data/test/plugin/test_base.rb +133 -1
  119. data/test/plugin/test_buf_file.rb +62 -23
  120. data/test/plugin/test_buf_file_single.rb +65 -0
  121. data/test/plugin/test_buffer.rb +267 -3
  122. data/test/plugin/test_buffer_chunk.rb +11 -0
  123. data/test/plugin/test_filter.rb +12 -1
  124. data/test/plugin/test_filter_parser.rb +1 -1
  125. data/test/plugin/test_filter_stdout.rb +2 -2
  126. data/test/plugin/test_in_forward.rb +9 -11
  127. data/test/plugin/test_in_http.rb +65 -3
  128. data/test/plugin/test_in_monitor_agent.rb +216 -11
  129. data/test/plugin/test_in_object_space.rb +9 -3
  130. data/test/plugin/test_in_syslog.rb +35 -0
  131. data/test/plugin/test_in_tail.rb +1393 -385
  132. data/test/plugin/test_in_tcp.rb +87 -2
  133. data/test/plugin/test_in_udp.rb +28 -0
  134. data/test/plugin/test_in_unix.rb +2 -2
  135. data/test/plugin/test_input.rb +12 -1
  136. data/test/plugin/test_metrics.rb +294 -0
  137. data/test/plugin/test_metrics_local.rb +96 -0
  138. data/test/plugin/test_multi_output.rb +25 -1
  139. data/test/plugin/test_out_exec.rb +6 -4
  140. data/test/plugin/test_out_exec_filter.rb +6 -2
  141. data/test/plugin/test_out_file.rb +34 -17
  142. data/test/plugin/test_out_forward.rb +78 -77
  143. data/test/plugin/test_out_http.rb +1 -0
  144. data/test/plugin/test_out_stdout.rb +2 -2
  145. data/test/plugin/test_output.rb +297 -12
  146. data/test/plugin/test_output_as_buffered.rb +44 -44
  147. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  148. data/test/plugin/test_output_as_buffered_retries.rb +54 -7
  149. data/test/plugin/test_output_as_buffered_secondary.rb +4 -4
  150. data/test/plugin/test_parser_regexp.rb +1 -6
  151. data/test/plugin/test_parser_syslog.rb +1 -1
  152. data/test/plugin_helper/test_cert_option.rb +1 -1
  153. data/test/plugin_helper/test_child_process.rb +38 -16
  154. data/test/plugin_helper/test_event_emitter.rb +29 -0
  155. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  156. data/test/plugin_helper/test_metrics.rb +137 -0
  157. data/test/plugin_helper/test_retry_state.rb +602 -38
  158. data/test/plugin_helper/test_server.rb +78 -6
  159. data/test/plugin_helper/test_timer.rb +2 -2
  160. data/test/test_config.rb +191 -24
  161. data/test/test_event_router.rb +17 -0
  162. data/test/test_file_wrapper.rb +53 -0
  163. data/test/test_formatter.rb +24 -21
  164. data/test/test_log.rb +122 -40
  165. data/test/test_msgpack_factory.rb +32 -0
  166. data/test/test_plugin_classes.rb +102 -0
  167. data/test/test_root_agent.rb +30 -1
  168. data/test/test_supervisor.rb +477 -257
  169. data/test/test_time_parser.rb +22 -0
  170. metadata +55 -34
  171. data/.drone.yml +0 -35
  172. data/.github/workflows/issue-auto-closer.yml +0 -12
  173. data/.github/workflows/linux-test.yaml +0 -36
  174. data/.github/workflows/macos-test.yaml +0 -30
  175. data/.github/workflows/windows-test.yaml +0 -46
  176. data/.gitlab-ci.yml +0 -103
  177. data/lib/fluent/plugin/file_wrapper.rb +0 -187
  178. data/test/plugin/test_file_wrapper.rb +0 -126
  179. data/test/test_logger_initializer.rb +0 -46
@@ -223,6 +223,22 @@ class OutputTest < Test::Unit::TestCase
223
223
  assert @i.terminated?
224
224
  end
225
225
 
226
+ test 'can use metrics plugins and fallback methods' do
227
+ @i.configure(config_element())
228
+
229
+ %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics
230
+ write_count_metrics rollback_count_metrics flush_time_count_metrics slow_flush_count_metrics].each do |metric_name|
231
+ assert_true @i.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
232
+ end
233
+
234
+ assert_equal 0, @i.num_errors
235
+ assert_equal 0, @i.emit_count
236
+ assert_equal 0, @i.emit_size
237
+ assert_equal 0, @i.emit_records
238
+ assert_equal 0, @i.write_count
239
+ assert_equal 0, @i.rollback_count
240
+ end
241
+
226
242
  data(:new_api => :chunk,
227
243
  :old_api => :metadata)
228
244
  test '#extract_placeholders does nothing if chunk key is not specified' do |api|
@@ -431,25 +447,25 @@ class OutputTest < Test::Unit::TestCase
431
447
  @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
432
448
  validators = @i.placeholder_validators(:path, "/my/path/${tag}/${username}/file.%Y%m%d_%H%M.log")
433
449
  assert_equal 3, validators.size
434
- assert_equal 1, validators.select(&:time?).size
435
- assert_equal 1, validators.select(&:tag?).size
436
- assert_equal 1, validators.select(&:keys?).size
450
+ assert_equal 1, validators.count(&:time?)
451
+ assert_equal 1, validators.count(&:tag?)
452
+ assert_equal 1, validators.count(&:keys?)
437
453
  end
438
454
 
439
455
  test 'returns validators for time, tag and keys when a plugin is configured with these keys even if a template does not have placeholders' do
440
456
  @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,username', {'timekey' => 60})]))
441
457
  validators = @i.placeholder_validators(:path, "/my/path/file.log")
442
458
  assert_equal 3, validators.size
443
- assert_equal 1, validators.select(&:time?).size
444
- assert_equal 1, validators.select(&:tag?).size
445
- assert_equal 1, validators.select(&:keys?).size
459
+ assert_equal 1, validators.count(&:time?)
460
+ assert_equal 1, validators.count(&:tag?)
461
+ assert_equal 1, validators.count(&:keys?)
446
462
  end
447
463
 
448
464
  test 'returns a validator for time if a template has timestamp placeholders' do
449
465
  @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
450
466
  validators = @i.placeholder_validators(:path, "/my/path/file.%Y-%m-%d.log")
451
467
  assert_equal 1, validators.size
452
- assert_equal 1, validators.select(&:time?).size
468
+ assert_equal 1, validators.count(&:time?)
453
469
  assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.%Y-%m-%d.log' has timestamp placeholders, but chunk key 'time' is not configured") do
454
470
  validators.first.validate!
455
471
  end
@@ -459,7 +475,7 @@ class OutputTest < Test::Unit::TestCase
459
475
  @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => '30'})]))
460
476
  validators = @i.placeholder_validators(:path, "/my/path/to/file.log")
461
477
  assert_equal 1, validators.size
462
- assert_equal 1, validators.select(&:time?).size
478
+ assert_equal 1, validators.count(&:time?)
463
479
  assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/to/file.log' doesn't have timestamp placeholders for timekey 30") do
464
480
  validators.first.validate!
465
481
  end
@@ -469,7 +485,7 @@ class OutputTest < Test::Unit::TestCase
469
485
  @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
470
486
  validators = @i.placeholder_validators(:path, "/my/path/${tag}/file.log")
471
487
  assert_equal 1, validators.size
472
- assert_equal 1, validators.select(&:tag?).size
488
+ assert_equal 1, validators.count(&:tag?)
473
489
  assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${tag}/file.log' has tag placeholders, but chunk key 'tag' is not configured") do
474
490
  validators.first.validate!
475
491
  end
@@ -479,7 +495,7 @@ class OutputTest < Test::Unit::TestCase
479
495
  @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
480
496
  validators = @i.placeholder_validators(:path, "/my/path/file.log")
481
497
  assert_equal 1, validators.size
482
- assert_equal 1, validators.select(&:tag?).size
498
+ assert_equal 1, validators.count(&:tag?)
483
499
  assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have tag placeholder") do
484
500
  validators.first.validate!
485
501
  end
@@ -489,7 +505,7 @@ class OutputTest < Test::Unit::TestCase
489
505
  @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
490
506
  validators = @i.placeholder_validators(:path, "/my/path/${username}/file.${group}.log")
491
507
  assert_equal 1, validators.size
492
- assert_equal 1, validators.select(&:keys?).size
508
+ assert_equal 1, validators.count(&:keys?)
493
509
  assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${username}/file.${group}.log' has placeholders, but chunk keys doesn't have keys group,username") do
494
510
  validators.first.validate!
495
511
  end
@@ -499,7 +515,7 @@ class OutputTest < Test::Unit::TestCase
499
515
  @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,group')]))
500
516
  validators = @i.placeholder_validators(:path, "/my/path/file.log")
501
517
  assert_equal 1, validators.size
502
- assert_equal 1, validators.select(&:keys?).size
518
+ assert_equal 1, validators.count(&:keys?)
503
519
  assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys group,username") do
504
520
  validators.first.validate!
505
521
  end
@@ -787,7 +803,10 @@ class OutputTest < Test::Unit::TestCase
787
803
  end
788
804
 
789
805
  test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
806
+ tmp_dir = File.join(__dir__, '../tmp/test_output')
807
+
790
808
  i = create_output(:delayed)
809
+ i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
791
810
  try_write_called = false
792
811
  i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
793
812
 
@@ -804,6 +823,8 @@ class OutputTest < Test::Unit::TestCase
804
823
  assert try_write_called
805
824
 
806
825
  i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
826
+ ensure
827
+ FileUtils.rm_rf(tmp_dir)
807
828
  end
808
829
 
809
830
  test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
@@ -833,7 +854,10 @@ class OutputTest < Test::Unit::TestCase
833
854
  end
834
855
 
835
856
  test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
857
+ tmp_dir = File.join(__dir__, '../tmp/test_output')
858
+
836
859
  i = create_output(:full)
860
+ i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
837
861
  write_called = false
838
862
  try_write_called = false
839
863
  i.register(:write){ |chunk| write_called = true }
@@ -856,6 +880,8 @@ class OutputTest < Test::Unit::TestCase
856
880
  assert try_write_called
857
881
 
858
882
  i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
883
+ ensure
884
+ FileUtils.rm_rf(tmp_dir)
859
885
  end
860
886
 
861
887
  test 'flush_interval is ignored when flush_mode is not interval' do
@@ -1046,4 +1072,263 @@ class OutputTest < Test::Unit::TestCase
1046
1072
  }
1047
1073
  end
1048
1074
  end
1075
+
1076
+ sub_test_case "actual_flush_thread_count" do
1077
+ data(
1078
+ "Not buffered",
1079
+ {
1080
+ output_type: :sync,
1081
+ config: config_element(),
1082
+ expected: 0,
1083
+ }
1084
+ )
1085
+ data(
1086
+ "Buffered with singile thread",
1087
+ {
1088
+ output_type: :full,
1089
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
1090
+ expected: 1,
1091
+ }
1092
+ )
1093
+ data(
1094
+ "Buffered with multiple threads",
1095
+ {
1096
+ output_type: :full,
1097
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
1098
+ expected: 8,
1099
+ }
1100
+ )
1101
+ test "actual_flush_thread_count" do |data|
1102
+ o = create_output(data[:output_type])
1103
+ o.configure(data[:config])
1104
+ assert_equal data[:expected], o.actual_flush_thread_count
1105
+ end
1106
+
1107
+ data(
1108
+ "Buffered with single thread",
1109
+ {
1110
+ output_type: :full,
1111
+ config: config_element(
1112
+ "ROOT", "", {},
1113
+ [
1114
+ config_element("buffer", "", {}),
1115
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1116
+ ]
1117
+ ),
1118
+ expected: 1,
1119
+ }
1120
+ )
1121
+ data(
1122
+ "Buffered with multiple threads",
1123
+ {
1124
+ output_type: :full,
1125
+ config: config_element(
1126
+ "ROOT", "", {},
1127
+ [
1128
+ config_element("buffer", "", {"flush_thread_count" => 8}),
1129
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1130
+ ]
1131
+ ),
1132
+ expected: 8,
1133
+ }
1134
+ )
1135
+ test "actual_flush_thread_count for secondary" do |data|
1136
+ primary = create_output(data[:output_type])
1137
+ primary.configure(data[:config])
1138
+ assert_equal data[:expected], primary.secondary.actual_flush_thread_count
1139
+ end
1140
+ end
1141
+
1142
+ sub_test_case "synchronize_path" do
1143
+ def setup
1144
+ Dir.mktmpdir do |lock_dir|
1145
+ ENV['FLUENTD_LOCK_DIR'] = lock_dir
1146
+ yield
1147
+ end
1148
+ end
1149
+
1150
+ def assert_worker_lock(lock_path, expect_locked)
1151
+ # With LOCK_NB set, flock() returns:
1152
+ # * `false` when the file is already locked.
1153
+ # * `0` when the file is not locked.
1154
+ File.open(lock_path, "w") do |f|
1155
+ if expect_locked
1156
+ assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)
1157
+ else
1158
+ assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)
1159
+ end
1160
+ end
1161
+ end
1162
+
1163
+ def assert_thread_lock(output_plugin, expect_locked)
1164
+ t = Thread.new do
1165
+ output_plugin.synchronize_path("test") do
1166
+ end
1167
+ end
1168
+ if expect_locked
1169
+ assert_nil t.join(3)
1170
+ else
1171
+ assert_not_nil t.join(3)
1172
+ end
1173
+ end
1174
+
1175
+ data(
1176
+ "Not buffered with single worker",
1177
+ {
1178
+ output_type: :sync,
1179
+ config: config_element(),
1180
+ workers: 1,
1181
+ expect_worker_lock: false,
1182
+ expect_thread_lock: false,
1183
+ }
1184
+ )
1185
+ data(
1186
+ "Not buffered with multiple workers",
1187
+ {
1188
+ output_type: :sync,
1189
+ config: config_element(),
1190
+ workers: 4,
1191
+ expect_worker_lock: true,
1192
+ expect_thread_lock: false,
1193
+ }
1194
+ )
1195
+ data(
1196
+ "Buffered with single thread and single worker",
1197
+ {
1198
+ output_type: :full,
1199
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
1200
+ workers: 1,
1201
+ expect_worker_lock: false,
1202
+ expect_thread_lock: false,
1203
+ }
1204
+ )
1205
+ data(
1206
+ "Buffered with multiple threads and single worker",
1207
+ {
1208
+ output_type: :full,
1209
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
1210
+ workers: 1,
1211
+ expect_worker_lock: false,
1212
+ expect_thread_lock: true,
1213
+ }
1214
+ )
1215
+ data(
1216
+ "Buffered with single thread and multiple workers",
1217
+ {
1218
+ output_type: :full,
1219
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
1220
+ workers: 4,
1221
+ expect_worker_lock: true,
1222
+ expect_thread_lock: false,
1223
+ }
1224
+ )
1225
+ data(
1226
+ "Buffered with multiple threads and multiple workers",
1227
+ {
1228
+ output_type: :full,
1229
+ config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
1230
+ workers: 4,
1231
+ expect_worker_lock: true,
1232
+ expect_thread_lock: true,
1233
+ }
1234
+ )
1235
+ test "synchronize_path" do |data|
1236
+ o = create_output(data[:output_type])
1237
+ o.configure(data[:config])
1238
+ o.system_config_override(workers: data[:workers])
1239
+
1240
+ test_lock_name = "test_lock_name"
1241
+ lock_path = o.get_lock_path(test_lock_name)
1242
+
1243
+ o.synchronize_path(test_lock_name) do
1244
+ assert_worker_lock(lock_path, data[:expect_worker_lock])
1245
+ assert_thread_lock(o, data[:expect_thread_lock])
1246
+ end
1247
+
1248
+ assert_worker_lock(lock_path, false)
1249
+ assert_thread_lock(o, false)
1250
+ end
1251
+
1252
+ data(
1253
+ "Buffered with single thread and single worker",
1254
+ {
1255
+ output_type: :full,
1256
+ config: config_element(
1257
+ "ROOT", "", {},
1258
+ [
1259
+ config_element("buffer", "", {}),
1260
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1261
+ ]
1262
+ ),
1263
+ workers: 1,
1264
+ expect_worker_lock: false,
1265
+ expect_thread_lock: false,
1266
+ }
1267
+ )
1268
+ data(
1269
+ "Buffered with multiple threads and single worker",
1270
+ {
1271
+ output_type: :full,
1272
+ config: config_element(
1273
+ "ROOT", "", {},
1274
+ [
1275
+ config_element("buffer", "", {"flush_thread_count" => 8}),
1276
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1277
+ ]
1278
+ ),
1279
+ workers: 1,
1280
+ expect_worker_lock: false,
1281
+ expect_thread_lock: true,
1282
+ }
1283
+ )
1284
+ data(
1285
+ "Buffered with single thread and multiple workers",
1286
+ {
1287
+ output_type: :full,
1288
+ config: config_element(
1289
+ "ROOT", "", {},
1290
+ [
1291
+ config_element("buffer", "", {}),
1292
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1293
+ ]
1294
+ ),
1295
+ workers: 4,
1296
+ expect_worker_lock: true,
1297
+ expect_thread_lock: false,
1298
+ }
1299
+ )
1300
+ data(
1301
+ "Buffered with multiple threads and multiple workers",
1302
+ {
1303
+ output_type: :full,
1304
+ config: config_element(
1305
+ "ROOT", "", {},
1306
+ [
1307
+ config_element("buffer", "", {"flush_thread_count" => 8}),
1308
+ config_element("secondary", "", {"@type" => "test", "name" => "test"}),
1309
+ ]
1310
+ ),
1311
+ workers: 4,
1312
+ expect_worker_lock: true,
1313
+ expect_thread_lock: true,
1314
+ }
1315
+ )
1316
+ test "synchronize_path for secondary" do |data|
1317
+ primary = create_output(data[:output_type])
1318
+ primary.configure(data[:config])
1319
+ secondary = primary.secondary
1320
+ secondary.system_config_override(workers: data[:workers])
1321
+
1322
+ test_lock_name = "test_lock_name"
1323
+ lock_path = secondary.get_lock_path(test_lock_name)
1324
+
1325
+ secondary.synchronize_path(test_lock_name) do
1326
+ assert_worker_lock(lock_path, data[:expect_worker_lock])
1327
+ assert_thread_lock(secondary, data[:expect_thread_lock])
1328
+ end
1329
+
1330
+ assert_worker_lock(lock_path, false)
1331
+ assert_thread_lock(secondary, false)
1332
+ end
1333
+ end
1049
1334
  end
@@ -510,7 +510,7 @@ class BufferedOutputTest < Test::Unit::TestCase
510
510
  logs = @i.log.out.logs.dup
511
511
  @i.start
512
512
  @i.after_start
513
- assert{ logs.select{|log| log.include?('[warn]') }.size == 0 }
513
+ assert{ logs.count{|log| log.include?('[warn]') } == 0 }
514
514
  end
515
515
 
516
516
  test 'a warning reported with 4 chunk keys' do
@@ -522,7 +522,7 @@ class BufferedOutputTest < Test::Unit::TestCase
522
522
  @i.after_start
523
523
  assert_equal ['key1', 'key2', 'key3', 'key4'], @i.chunk_keys
524
524
 
525
- assert{ logs.select{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') }.size == 1 }
525
+ assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }
526
526
  end
527
527
 
528
528
  test 'a warning reported with 4 chunk keys including "tag"' do
@@ -531,7 +531,7 @@ class BufferedOutputTest < Test::Unit::TestCase
531
531
  logs = @i.log.out.logs.dup
532
532
  @i.start # this calls `log.reset`... capturing logs about configure must be done before this line
533
533
  @i.after_start
534
- assert{ logs.select{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') }.size == 1 }
534
+ assert{ logs.count{|log| log.include?('[warn]: many chunk keys specified, and it may cause too many chunks on your system.') } == 1 }
535
535
  end
536
536
 
537
537
  test 'time key is not included for warned chunk keys' do
@@ -540,7 +540,7 @@ class BufferedOutputTest < Test::Unit::TestCase
540
540
  logs = @i.log.out.logs.dup
541
541
  @i.start
542
542
  @i.after_start
543
- assert{ logs.select{|log| log.include?('[warn]') }.size == 0 }
543
+ assert{ logs.count{|log| log.include?('[warn]') } == 0 }
544
544
  end
545
545
  end
546
546
 
@@ -968,8 +968,8 @@ class BufferedOutputTest < Test::Unit::TestCase
968
968
  waiting(4){ sleep 0.1 until ary.size == 3 }
969
969
 
970
970
  assert_equal 3, ary.size
971
- assert_equal 2, ary.select{|e| e[0] == "test.tag.1" }.size
972
- assert_equal 1, ary.select{|e| e[0] == "test.tag.2" }.size
971
+ assert_equal 2, ary.count{|e| e[0] == "test.tag.1" }
972
+ assert_equal 1, ary.count{|e| e[0] == "test.tag.2" }
973
973
 
974
974
  Timecop.freeze( Time.parse('2016-04-13 14:04:04 +0900') )
975
975
 
@@ -985,8 +985,8 @@ class BufferedOutputTest < Test::Unit::TestCase
985
985
  assert{ @i.buffer.stage.size == 1 && @i.write_count == 2 }
986
986
 
987
987
  assert_equal 9, ary.size
988
- assert_equal 7, ary.select{|e| e[0] == "test.tag.1" }.size
989
- assert_equal 2, ary.select{|e| e[0] == "test.tag.2" }.size
988
+ assert_equal 7, ary.count{|e| e[0] == "test.tag.1" }
989
+ assert_equal 2, ary.count{|e| e[0] == "test.tag.2" }
990
990
 
991
991
  assert metachecks.all?{|e| e }
992
992
  end
@@ -1224,8 +1224,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1224
1224
 
1225
1225
  # events fulfills a chunk (and queued immediately)
1226
1226
  assert_equal 5, ary.size
1227
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1228
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1227
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1228
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1229
1229
 
1230
1230
  Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
1231
1231
 
@@ -1249,8 +1249,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1249
1249
  assert{ @i.buffer.stage.size == 0 && @i.write_count == 3 }
1250
1250
 
1251
1251
  assert_equal 11, ary.size
1252
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1253
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1252
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1253
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1254
1254
 
1255
1255
  assert metachecks.all?{|e| e }
1256
1256
  end
@@ -1315,8 +1315,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1315
1315
 
1316
1316
  # events fulfills a chunk (and queued immediately)
1317
1317
  assert_equal 5, ary.size
1318
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1319
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1318
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1319
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1320
1320
 
1321
1321
  @i.stop
1322
1322
  @i.before_shutdown
@@ -1330,8 +1330,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1330
1330
  assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 3 }
1331
1331
 
1332
1332
  assert_equal 11, ary.size
1333
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1334
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1333
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1334
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1335
1335
 
1336
1336
  assert metachecks.all?{|e| e }
1337
1337
  end
@@ -1435,8 +1435,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1435
1435
 
1436
1436
  # events fulfills a chunk (and queued immediately)
1437
1437
  assert_equal 5, ary.size
1438
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1439
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1438
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1439
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1440
1440
  assert ary[0...5].all?{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
1441
1441
 
1442
1442
  Timecop.freeze( Time.parse('2016-04-13 14:04:09 +0900') )
@@ -1465,11 +1465,11 @@ class BufferedOutputTest < Test::Unit::TestCase
1465
1465
  assert{ @i.buffer.stage.size == 0 && @i.write_count == 4 }
1466
1466
 
1467
1467
  assert_equal 11, ary.size
1468
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1469
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1470
- assert_equal 6, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }.size
1471
- assert_equal 3, ary.select{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }.size
1472
- assert_equal 2, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }.size
1468
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1469
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1470
+ assert_equal 6, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
1471
+ assert_equal 3, ary.count{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }
1472
+ assert_equal 2, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }
1473
1473
 
1474
1474
  assert metachecks.all?{|e| e }
1475
1475
  end
@@ -1525,8 +1525,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1525
1525
 
1526
1526
  # events fulfills a chunk (and queued immediately)
1527
1527
  assert_equal 5, ary.size
1528
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1529
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1528
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1529
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1530
1530
 
1531
1531
  @i.stop
1532
1532
  @i.before_shutdown
@@ -1540,11 +1540,11 @@ class BufferedOutputTest < Test::Unit::TestCase
1540
1540
  assert{ @i.buffer.stage.size == 0 && @i.buffer.queue.size == 0 && @i.write_count == 4 }
1541
1541
 
1542
1542
  assert_equal 11, ary.size
1543
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1544
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1545
- assert_equal 6, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }.size
1546
- assert_equal 3, ary.select{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }.size
1547
- assert_equal 2, ary.select{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }.size
1543
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1544
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1545
+ assert_equal 6, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "a" }
1546
+ assert_equal 3, ary.count{|e| e[2]["name"] == "yyy" && e[2]["service"] == "a" }
1547
+ assert_equal 2, ary.count{|e| e[2]["name"] == "xxx" && e[2]["service"] == "b" }
1548
1548
 
1549
1549
  assert metachecks.all?{|e| e }
1550
1550
  end
@@ -1683,8 +1683,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1683
1683
 
1684
1684
  # events fulfills a chunk (and queued immediately)
1685
1685
  assert_equal 5, ary.size
1686
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1687
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1686
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1687
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1688
1688
 
1689
1689
  assert_equal 1, chunks.size
1690
1690
  assert !chunks.first.empty?
@@ -1716,8 +1716,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1716
1716
  assert{ @i.buffer.dequeued.size == 3 }
1717
1717
 
1718
1718
  assert_equal 11, ary.size
1719
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1720
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1719
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1720
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1721
1721
 
1722
1722
  assert_equal 3, chunks.size
1723
1723
  assert chunks.all?{|c| !c.empty? }
@@ -1802,8 +1802,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1802
1802
 
1803
1803
  # events fulfills a chunk (and queued immediately)
1804
1804
  assert_equal 5, ary.size
1805
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1806
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1805
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1806
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1807
1807
 
1808
1808
  assert_equal 1, chunks.size
1809
1809
  assert !chunks.first.empty?
@@ -1835,8 +1835,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1835
1835
  assert{ @i.buffer.dequeued.size == 3 }
1836
1836
 
1837
1837
  assert_equal 11, ary.size
1838
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1839
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1838
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1839
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1840
1840
 
1841
1841
  assert_equal 3, chunks.size
1842
1842
  assert chunks.all?{|c| !c.empty? }
@@ -1892,8 +1892,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1892
1892
 
1893
1893
  assert{ @i.write_count == 7 }
1894
1894
  assert_equal 11, ary.size
1895
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
1896
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
1895
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
1896
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
1897
1897
  assert{ chunks.size == 3 }
1898
1898
  assert{ chunks.all?{|c| !c.empty? } }
1899
1899
 
@@ -1963,8 +1963,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1963
1963
 
1964
1964
  # events fulfills a chunk (and queued immediately)
1965
1965
  assert_equal 5, ary.size
1966
- assert_equal 5, ary.select{|e| e[0] == "test.tag.1" }.size
1967
- assert_equal 0, ary.select{|e| e[0] == "test.tag.2" }.size
1966
+ assert_equal 5, ary.count{|e| e[0] == "test.tag.1" }
1967
+ assert_equal 0, ary.count{|e| e[0] == "test.tag.2" }
1968
1968
 
1969
1969
  assert_equal 1, chunks.size
1970
1970
  assert !chunks.first.empty?
@@ -1999,8 +1999,8 @@ class BufferedOutputTest < Test::Unit::TestCase
1999
1999
  assert{ @i.rollback_count == 0 }
2000
2000
 
2001
2001
  assert_equal 11, ary.size
2002
- assert_equal 8, ary.select{|e| e[0] == "test.tag.1" }.size
2003
- assert_equal 3, ary.select{|e| e[0] == "test.tag.2" }.size
2002
+ assert_equal 8, ary.count{|e| e[0] == "test.tag.1" }
2003
+ assert_equal 3, ary.count{|e| e[0] == "test.tag.2" }
2004
2004
 
2005
2005
  assert{ chunks.size == 3 }
2006
2006
  assert{ chunks.all?{|c| !c.empty? } }