fluentd 1.16.11 → 1.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. checksums.yaml +4 -4
  2. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +50 -0
  3. data/.github/DISCUSSION_TEMPLATE/q-a.yml +47 -0
  4. data/.github/workflows/test-ruby-head.yml +31 -0
  5. data/.github/workflows/test.yml +6 -38
  6. data/CHANGELOG.md +33 -122
  7. data/README.md +4 -1
  8. data/Rakefile +1 -1
  9. data/SECURITY.md +4 -6
  10. data/fluentd.gemspec +5 -11
  11. data/lib/fluent/command/binlog_reader.rb +1 -1
  12. data/lib/fluent/command/fluentd.rb +1 -1
  13. data/lib/fluent/compat/formatter.rb +0 -6
  14. data/lib/fluent/config/configure_proxy.rb +2 -2
  15. data/lib/fluent/config/parser.rb +3 -15
  16. data/lib/fluent/config/types.rb +1 -1
  17. data/lib/fluent/config/v1_parser.rb +1 -1
  18. data/lib/fluent/config/yaml_parser/parser.rb +0 -4
  19. data/lib/fluent/configurable.rb +2 -2
  20. data/lib/fluent/counter/mutex_hash.rb +1 -1
  21. data/lib/fluent/fluent_log_event_router.rb +0 -2
  22. data/lib/fluent/plugin/buf_file.rb +1 -1
  23. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  24. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
  25. data/lib/fluent/plugin/filter_parser.rb +26 -8
  26. data/lib/fluent/plugin/formatter_csv.rb +4 -18
  27. data/lib/fluent/plugin/in_http.rb +17 -52
  28. data/lib/fluent/plugin/in_tail.rb +35 -3
  29. data/lib/fluent/plugin/out_file.rb +0 -8
  30. data/lib/fluent/plugin/out_forward.rb +0 -10
  31. data/lib/fluent/plugin/out_http.rb +125 -13
  32. data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
  33. data/lib/fluent/plugin/parser_json.rb +34 -9
  34. data/lib/fluent/plugin/parser_msgpack.rb +24 -3
  35. data/lib/fluent/plugin_helper/metrics.rb +2 -2
  36. data/lib/fluent/plugin_helper/server.rb +1 -4
  37. data/lib/fluent/registry.rb +6 -6
  38. data/lib/fluent/supervisor.rb +3 -3
  39. data/lib/fluent/test/output_test.rb +1 -1
  40. data/lib/fluent/test.rb +2 -5
  41. data/lib/fluent/unique_id.rb +1 -1
  42. data/lib/fluent/version.rb +1 -1
  43. data/lib/fluent/winsvc.rb +8 -38
  44. data/test/command/test_cat.rb +2 -2
  45. data/test/command/test_fluentd.rb +11 -58
  46. data/test/config/test_dsl.rb +1 -8
  47. data/test/config/test_element.rb +2 -2
  48. data/test/config/test_plugin_configuration.rb +6 -6
  49. data/test/helper.rb +7 -27
  50. data/test/log/test_console_adapter.rb +10 -3
  51. data/test/plugin/data/log_numeric/01.log +0 -0
  52. data/test/plugin/data/log_numeric/02.log +0 -0
  53. data/test/plugin/data/log_numeric/12.log +0 -0
  54. data/test/plugin/data/log_numeric/14.log +0 -0
  55. data/test/plugin/in_tail/test_io_handler.rb +14 -13
  56. data/test/plugin/in_tail/test_position_file.rb +7 -6
  57. data/test/plugin/out_forward/test_ack_handler.rb +3 -3
  58. data/test/plugin/out_forward/test_socket_cache.rb +3 -3
  59. data/test/plugin/test_buffer.rb +2 -2
  60. data/test/plugin/test_filter_grep.rb +1 -1
  61. data/test/plugin/test_filter_stdout.rb +4 -4
  62. data/test/plugin/test_formatter_hash.rb +2 -2
  63. data/test/plugin/test_in_forward.rb +1 -2
  64. data/test/plugin/test_in_http.rb +23 -1
  65. data/test/plugin/test_in_monitor_agent.rb +6 -6
  66. data/test/plugin/test_in_object_space.rb +0 -4
  67. data/test/plugin/test_in_syslog.rb +18 -25
  68. data/test/plugin/test_in_tail.rb +153 -5
  69. data/test/plugin/test_in_tcp.rb +1 -1
  70. data/test/plugin/test_in_udp.rb +10 -16
  71. data/test/plugin/test_out_exec_filter.rb +7 -12
  72. data/test/plugin/test_out_file.rb +2 -22
  73. data/test/plugin/test_out_forward.rb +37 -61
  74. data/test/plugin/test_out_http.rb +128 -0
  75. data/test/plugin/test_out_stdout.rb +3 -3
  76. data/test/plugin/test_out_stream.rb +1 -1
  77. data/test/plugin/test_output.rb +1 -1
  78. data/test/plugin/test_output_as_buffered.rb +2 -2
  79. data/test/plugin/test_output_as_buffered_retries.rb +2 -2
  80. data/test/plugin/test_owned_by.rb +0 -1
  81. data/test/plugin/test_parser_csv.rb +1 -1
  82. data/test/plugin/test_parser_json.rb +106 -31
  83. data/test/plugin/test_parser_msgpack.rb +127 -0
  84. data/test/plugin/test_storage.rb +0 -1
  85. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  86. data/test/plugin_helper/test_server.rb +41 -83
  87. data/test/plugin_helper/test_socket.rb +1 -1
  88. data/test/test_config.rb +0 -6
  89. data/test/test_configdsl.rb +1 -8
  90. data/test/test_event_router.rb +2 -2
  91. data/test/test_plugin_helper.rb +1 -1
  92. data/test/test_supervisor.rb +21 -32
  93. data/test/test_tls.rb +1 -1
  94. metadata +66 -38
  95. data/test/scripts/windows_service_test.ps1 +0 -73
@@ -214,9 +214,6 @@ class SupervisorTest < ::Test::Unit::TestCase
214
214
  def test_cont_in_main_process_signal_handlers
215
215
  omit "Windows cannot handle signals" if Fluent.windows?
216
216
 
217
- # https://github.com/fluent/fluentd/issues/4063
218
- GC.start
219
-
220
217
  sv = Fluent::Supervisor.new({})
221
218
  sv.send(:install_main_process_signal_handlers)
222
219
 
@@ -232,9 +229,6 @@ class SupervisorTest < ::Test::Unit::TestCase
232
229
  def test_term_cont_in_main_process_signal_handlers
233
230
  omit "Windows cannot handle signals" if Fluent.windows?
234
231
 
235
- # https://github.com/fluent/fluentd/issues/4063
236
- GC.start
237
-
238
232
  create_debug_dummy_logger
239
233
 
240
234
  sv = Fluent::Supervisor.new({})
@@ -302,9 +296,6 @@ class SupervisorTest < ::Test::Unit::TestCase
302
296
  def test_cont_in_supervisor_signal_handler
303
297
  omit "Windows cannot handle signals" if Fluent.windows?
304
298
 
305
- # https://github.com/fluent/fluentd/issues/4063
306
- GC.start
307
-
308
299
  server = DummyServer.new
309
300
  server.install_supervisor_signal_handlers
310
301
 
@@ -320,9 +311,6 @@ class SupervisorTest < ::Test::Unit::TestCase
320
311
  def test_term_cont_in_supervisor_signal_handler
321
312
  omit "Windows cannot handle signals" if Fluent.windows?
322
313
 
323
- # https://github.com/fluent/fluentd/issues/4063
324
- GC.start
325
-
326
314
  server = DummyServer.new
327
315
  server.install_supervisor_signal_handlers
328
316
 
@@ -353,7 +341,6 @@ class SupervisorTest < ::Test::Unit::TestCase
353
341
  event = Win32::Event.open("TestFluentdEvent")
354
342
  event.set
355
343
  event.close
356
- sleep 1.0 # Wait for dumping
357
344
  ensure
358
345
  server.stop_windows_event_thread
359
346
  end
@@ -380,7 +367,6 @@ class SupervisorTest < ::Test::Unit::TestCase
380
367
  event = Win32::Event.open("TestFluentdEvent_USR1")
381
368
  event.set
382
369
  event.close
383
- sleep 1.0 # Wait for dumping
384
370
  ensure
385
371
  server.stop_windows_event_thread
386
372
  end
@@ -404,31 +390,34 @@ class SupervisorTest < ::Test::Unit::TestCase
404
390
  def test_supervisor_event_dump_windows
405
391
  omit "Only for Windows, alternative to UNIX signals" unless Fluent.windows?
406
392
 
407
- # https://github.com/fluent/fluentd/issues/4063
408
- GC.start
409
-
410
- ENV['SIGDUMP_PATH'] = @tmp_dir + "/sigdump.log"
411
-
412
393
  server = DummyServer.new
413
394
  def server.config
414
395
  {:signame => "TestFluentdEvent"}
415
396
  end
416
397
  server.install_windows_event_handler
417
398
 
418
- begin
419
- sleep 0.1 # Wait for starting windows event thread
420
- event = Win32::Event.open("TestFluentdEvent_CONT")
421
- event.set
422
- event.close
423
- sleep 1.0 # Wait for dumping
424
- ensure
425
- server.stop_windows_event_thread
399
+ assert_rr do
400
+ # Have to use mock because `Sigdump.dump` seems to be somehow incompatible with RR.
401
+ # The `mock(server).restart(true) { nil }` line in `test_rpc_server_windows` cause the next error.
402
+ # Failure: test_supervisor_event_dump_windows(SupervisorTest):
403
+ # class()
404
+ # Called 0 times.
405
+ # Expected 1 times.
406
+ # .../Ruby26-x64/lib/ruby/gems/2.6.0/gems/sigdump-0.2.4/lib/sigdump.rb:74:in `block in dump_object_count'
407
+ # 73: ObjectSpace.each_object {|o|
408
+ # 74: c = o.class <-- HERE!
409
+ mock(Sigdump).dump(anything)
410
+
411
+ begin
412
+ sleep 0.1 # Wait for starting windows event thread
413
+ event = Win32::Event.open("TestFluentdEvent_CONT")
414
+ event.set
415
+ event.close
416
+ sleep 1.0 # Wait for dumping
417
+ ensure
418
+ server.stop_windows_event_thread
419
+ end
426
420
  end
427
-
428
- result_filepaths = Dir.glob("#{@tmp_dir}/*")
429
- assert {result_filepaths.length > 0}
430
- ensure
431
- ENV.delete('SIGDUMP_PATH')
432
421
  end
433
422
 
434
423
  data(:ipv4 => ["0.0.0.0", "127.0.0.1", false],
data/test/test_tls.rb CHANGED
@@ -9,7 +9,7 @@ class UniqueIdTest < Test::Unit::TestCase
9
9
  TEST_TLS1_2_CASES = {
10
10
  'New TLS v1.2' => :'TLS1_2',
11
11
  'Old TLS v1.2' => :'TLSv1_2'
12
- }
12
+ }
13
13
  TEST_TLS_CASES = TEST_TLS1_1_CASES.merge(TEST_TLS1_2_CASES)
14
14
 
15
15
  sub_test_case 'constants' do
metadata CHANGED
@@ -1,13 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.16.11
4
+ version: 1.17.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
+ autorequire:
8
9
  bindir: bin
9
10
  cert_chain: []
10
- date: 1980-01-02 00:00:00.000000000 Z
11
+ date: 2024-04-30 00:00:00.000000000 Z
11
12
  dependencies:
12
13
  - !ruby/object:Gem::Dependency
13
14
  name: bundler
@@ -199,20 +200,6 @@ dependencies:
199
200
  - - "~>"
200
201
  - !ruby/object:Gem::Version
201
202
  version: '1.4'
202
- - !ruby/object:Gem::Dependency
203
- name: console
204
- requirement: !ruby/object:Gem::Requirement
205
- requirements:
206
- - - "<"
207
- - !ruby/object:Gem::Version
208
- version: '1.24'
209
- type: :runtime
210
- prerelease: false
211
- version_requirements: !ruby/object:Gem::Requirement
212
- requirements:
213
- - - "<"
214
- - !ruby/object:Gem::Version
215
- version: '1.24'
216
203
  - !ruby/object:Gem::Dependency
217
204
  name: base64
218
205
  requirement: !ruby/object:Gem::Requirement
@@ -255,20 +242,6 @@ dependencies:
255
242
  - - "~>"
256
243
  - !ruby/object:Gem::Version
257
244
  version: '2.2'
258
- - !ruby/object:Gem::Dependency
259
- name: logger
260
- requirement: !ruby/object:Gem::Requirement
261
- requirements:
262
- - - "~>"
263
- - !ruby/object:Gem::Version
264
- version: '1.6'
265
- type: :runtime
266
- prerelease: false
267
- version_requirements: !ruby/object:Gem::Requirement
268
- requirements:
269
- - - "~>"
270
- - !ruby/object:Gem::Version
271
- version: '1.6'
272
245
  - !ruby/object:Gem::Dependency
273
246
  name: rake
274
247
  requirement: !ruby/object:Gem::Requirement
@@ -343,16 +316,16 @@ dependencies:
343
316
  name: timecop
344
317
  requirement: !ruby/object:Gem::Requirement
345
318
  requirements:
346
- - - "<"
319
+ - - "~>"
347
320
  - !ruby/object:Gem::Version
348
- version: 0.9.9
321
+ version: '0.9'
349
322
  type: :development
350
323
  prerelease: false
351
324
  version_requirements: !ruby/object:Gem::Requirement
352
325
  requirements:
353
- - - "<"
326
+ - - "~>"
354
327
  - !ruby/object:Gem::Version
355
- version: 0.9.9
328
+ version: '0.9'
356
329
  - !ruby/object:Gem::Dependency
357
330
  name: test-unit
358
331
  requirement: !ruby/object:Gem::Requirement
@@ -429,6 +402,48 @@ dependencies:
429
402
  - - ">="
430
403
  - !ruby/object:Gem::Version
431
404
  version: 0.50.0
405
+ - !ruby/object:Gem::Dependency
406
+ name: aws-sigv4
407
+ requirement: !ruby/object:Gem::Requirement
408
+ requirements:
409
+ - - "~>"
410
+ - !ruby/object:Gem::Version
411
+ version: '1.8'
412
+ type: :development
413
+ prerelease: false
414
+ version_requirements: !ruby/object:Gem::Requirement
415
+ requirements:
416
+ - - "~>"
417
+ - !ruby/object:Gem::Version
418
+ version: '1.8'
419
+ - !ruby/object:Gem::Dependency
420
+ name: aws-sdk-core
421
+ requirement: !ruby/object:Gem::Requirement
422
+ requirements:
423
+ - - "~>"
424
+ - !ruby/object:Gem::Version
425
+ version: '3.191'
426
+ type: :development
427
+ prerelease: false
428
+ version_requirements: !ruby/object:Gem::Requirement
429
+ requirements:
430
+ - - "~>"
431
+ - !ruby/object:Gem::Version
432
+ version: '3.191'
433
+ - !ruby/object:Gem::Dependency
434
+ name: rexml
435
+ requirement: !ruby/object:Gem::Requirement
436
+ requirements:
437
+ - - "~>"
438
+ - !ruby/object:Gem::Version
439
+ version: '3.2'
440
+ type: :development
441
+ prerelease: false
442
+ version_requirements: !ruby/object:Gem::Requirement
443
+ requirements:
444
+ - - "~>"
445
+ - !ruby/object:Gem::Version
446
+ version: '3.2'
432
447
  description: Fluentd is an open source data collector designed to scale and simplify
433
448
  log management. It can collect, process and ship many kinds of data in near real-time.
434
449
  email:
@@ -448,12 +463,15 @@ extensions: []
448
463
  extra_rdoc_files: []
449
464
  files:
450
465
  - ".deepsource.toml"
466
+ - ".github/DISCUSSION_TEMPLATE/q-a-japanese.yml"
467
+ - ".github/DISCUSSION_TEMPLATE/q-a.yml"
451
468
  - ".github/ISSUE_TEMPLATE.md"
452
469
  - ".github/ISSUE_TEMPLATE/bug_report.yml"
453
470
  - ".github/ISSUE_TEMPLATE/config.yml"
454
471
  - ".github/ISSUE_TEMPLATE/feature_request.yml"
455
472
  - ".github/PULL_REQUEST_TEMPLATE.md"
456
473
  - ".github/workflows/stale-actions.yml"
474
+ - ".github/workflows/test-ruby-head.yml"
457
475
  - ".github/workflows/test.yml"
458
476
  - ".gitignore"
459
477
  - ADOPTERS.md
@@ -825,6 +843,10 @@ files:
825
843
  - test/plugin/data/log/foo/bar.log
826
844
  - test/plugin/data/log/foo/bar2
827
845
  - test/plugin/data/log/test.log
846
+ - test/plugin/data/log_numeric/01.log
847
+ - test/plugin/data/log_numeric/02.log
848
+ - test/plugin/data/log_numeric/12.log
849
+ - test/plugin/data/log_numeric/14.log
828
850
  - test/plugin/data/sd_file/config
829
851
  - test/plugin/data/sd_file/config.json
830
852
  - test/plugin/data/sd_file/config.yaml
@@ -909,6 +931,7 @@ files:
909
931
  - test/plugin/test_parser_csv.rb
910
932
  - test/plugin/test_parser_json.rb
911
933
  - test/plugin/test_parser_labeled_tsv.rb
934
+ - test/plugin/test_parser_msgpack.rb
912
935
  - test/plugin/test_parser_multiline.rb
913
936
  - test/plugin/test_parser_nginx.rb
914
937
  - test/plugin/test_parser_none.rb
@@ -972,7 +995,6 @@ files:
972
995
  - test/scripts/fluent/plugin/out_test.rb
973
996
  - test/scripts/fluent/plugin/out_test2.rb
974
997
  - test/scripts/fluent/plugin/parser_known.rb
975
- - test/scripts/windows_service_test.ps1
976
998
  - test/test_capability.rb
977
999
  - test/test_clock.rb
978
1000
  - test/test_config.rb
@@ -1011,6 +1033,7 @@ homepage: https://www.fluentd.org/
1011
1033
  licenses:
1012
1034
  - Apache-2.0
1013
1035
  metadata: {}
1036
+ post_install_message:
1014
1037
  rdoc_options: []
1015
1038
  require_paths:
1016
1039
  - lib
@@ -1018,14 +1041,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
1018
1041
  requirements:
1019
1042
  - - ">="
1020
1043
  - !ruby/object:Gem::Version
1021
- version: '2.4'
1044
+ version: '2.7'
1022
1045
  required_rubygems_version: !ruby/object:Gem::Requirement
1023
1046
  requirements:
1024
1047
  - - ">="
1025
1048
  - !ruby/object:Gem::Version
1026
1049
  version: '0'
1027
1050
  requirements: []
1028
- rubygems_version: 3.6.9
1051
+ rubygems_version: 3.4.13
1052
+ signing_key:
1029
1053
  specification_version: 4
1030
1054
  summary: Fluentd event collector
1031
1055
  test_files:
@@ -1067,6 +1091,10 @@ test_files:
1067
1091
  - test/plugin/data/log/foo/bar.log
1068
1092
  - test/plugin/data/log/foo/bar2
1069
1093
  - test/plugin/data/log/test.log
1094
+ - test/plugin/data/log_numeric/01.log
1095
+ - test/plugin/data/log_numeric/02.log
1096
+ - test/plugin/data/log_numeric/12.log
1097
+ - test/plugin/data/log_numeric/14.log
1070
1098
  - test/plugin/data/sd_file/config
1071
1099
  - test/plugin/data/sd_file/config.json
1072
1100
  - test/plugin/data/sd_file/config.yaml
@@ -1151,6 +1179,7 @@ test_files:
1151
1179
  - test/plugin/test_parser_csv.rb
1152
1180
  - test/plugin/test_parser_json.rb
1153
1181
  - test/plugin/test_parser_labeled_tsv.rb
1182
+ - test/plugin/test_parser_msgpack.rb
1154
1183
  - test/plugin/test_parser_multiline.rb
1155
1184
  - test/plugin/test_parser_nginx.rb
1156
1185
  - test/plugin/test_parser_none.rb
@@ -1214,7 +1243,6 @@ test_files:
1214
1243
  - test/scripts/fluent/plugin/out_test.rb
1215
1244
  - test/scripts/fluent/plugin/out_test2.rb
1216
1245
  - test/scripts/fluent/plugin/parser_known.rb
1217
- - test/scripts/windows_service_test.ps1
1218
1246
  - test/test_capability.rb
1219
1247
  - test/test_clock.rb
1220
1248
  - test/test_config.rb
@@ -1,73 +0,0 @@
1
- $ErrorActionPreference = "Stop"
2
- Set-PSDebug -Trace 1
3
-
4
- $default_conf_path = (Resolve-Path fluent.conf).Path
5
- $current_path = (Get-Location).Path
6
- $log_path = "$current_path/fluentd.log"
7
-
8
- ruby bin/fluentd --reg-winsvc i --reg-winsvc-fluentdopt "-c '$default_conf_path' -o '$log_path'"
9
-
10
- # Test: must not start automatically
11
- if ((Get-Service fluentdwinsvc).Status -ne "Stopped") {
12
- Write-Error "The service should not start automatically."
13
- }
14
-
15
- Start-Service fluentdwinsvc
16
- Start-Sleep 30
17
-
18
- # Test: the service should be running after started
19
- if ((Get-Service fluentdwinsvc).Status -ne "Running") {
20
- Write-Error "The service should be running after started."
21
- }
22
-
23
- # Test: no warn/error/fatal logs
24
- Get-ChildItem "*.log" | %{
25
- Get-Content $_
26
- if (Select-String -Path $_ -Pattern "[warn]", "[error]", "[fatal]" -SimpleMatch -Quiet) {
27
- Select-String -Path $_ -Pattern "[warn]", "[error]", "[fatal]" -SimpleMatch
28
- Write-Error "There are abnormal level logs in ${_}:"
29
- }
30
- }
31
-
32
- Stop-Service fluentdwinsvc
33
- Start-Sleep 10 # Somehow it is possible that some processes stay alive for a while. (This could be not good behavior...)
34
-
35
- # Test: status after stopped
36
- if ((Get-Service fluentdwinsvc).Status -ne "Stopped") {
37
- Write-Error "The service should be in 'Stopped' status after stopped."
38
- }
39
- # Test: all Ruby processes should stop
40
- $ruby_processes = Get-Process -name ruby -ErrorAction SilentlyContinue
41
- if ($ruby_processes.Count -ne 0) {
42
- Write-Output $ruby_processes
43
- Write-Error "All Ruby processes should stop."
44
- }
45
-
46
- # Test: service should stop when the supervisor fails to launch
47
- # https://github.com/fluent/fluentd/pull/4909
48
- $test_setting = @'
49
- <source>
50
- @type sample
51
- @id DUPLICATED_ID
52
- tag test
53
- </source>
54
- <match test>
55
- @type stdout
56
- @id DUPLICATED_ID
57
- </match>
58
- '@
59
- Add-Content -Path "duplicated_id.conf" -Encoding UTF8 -Value $test_setting
60
- ruby bin/fluentd --reg-winsvc-fluentdopt "-c '$current_path/duplicated_id.conf' -o '$log_path'"
61
- Start-Service fluentdwinsvc
62
- Start-Sleep 30
63
- if ((Get-Service fluentdwinsvc).Status -ne "Stopped") {
64
- Write-Error "The service should be in 'Stopped' status when the supervisor fails to launch."
65
- }
66
- $ruby_processes = Get-Process -name ruby -ErrorAction SilentlyContinue
67
- if ($ruby_processes.Count -ne 0) {
68
- Write-Output $ruby_processes
69
- Write-Error "All Ruby processes should stop."
70
- }
71
-
72
- ruby bin/fluentd --reg-winsvc u
73
- Remove-Item $log_path