fluentd 1.16.5 → 1.17.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (268) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +88 -0
  3. data/README.md +2 -1
  4. data/Rakefile +1 -1
  5. data/SECURITY.md +2 -2
  6. data/fluent.conf +14 -14
  7. data/lib/fluent/command/binlog_reader.rb +1 -1
  8. data/lib/fluent/command/cap_ctl.rb +4 -4
  9. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  10. data/lib/fluent/compat/propagate_default.rb +4 -4
  11. data/lib/fluent/config/configure_proxy.rb +2 -2
  12. data/lib/fluent/config/types.rb +1 -1
  13. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  14. data/lib/fluent/configurable.rb +2 -2
  15. data/lib/fluent/counter/mutex_hash.rb +1 -1
  16. data/lib/fluent/fluent_log_event_router.rb +0 -2
  17. data/lib/fluent/log/console_adapter.rb +4 -2
  18. data/lib/fluent/plugin/buf_file.rb +1 -1
  19. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  20. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
  21. data/lib/fluent/plugin/filter_parser.rb +26 -8
  22. data/lib/fluent/plugin/in_exec.rb +14 -2
  23. data/lib/fluent/plugin/in_http.rb +19 -54
  24. data/lib/fluent/plugin/in_sample.rb +13 -7
  25. data/lib/fluent/plugin/in_tail.rb +99 -25
  26. data/lib/fluent/plugin/out_copy.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +8 -0
  28. data/lib/fluent/plugin/out_http.rb +137 -13
  29. data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
  30. data/lib/fluent/plugin/parser_json.rb +26 -17
  31. data/lib/fluent/plugin/parser_msgpack.rb +24 -3
  32. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  33. data/lib/fluent/plugin_helper/metrics.rb +2 -2
  34. data/lib/fluent/registry.rb +6 -6
  35. data/lib/fluent/test/output_test.rb +1 -1
  36. data/lib/fluent/unique_id.rb +1 -1
  37. data/lib/fluent/version.rb +1 -1
  38. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  39. metadata +109 -459
  40. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  41. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  42. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  43. data/.github/ISSUE_TEMPLATE.md +0 -17
  44. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  45. data/.github/workflows/stale-actions.yml +0 -24
  46. data/.github/workflows/test.yml +0 -32
  47. data/.gitignore +0 -30
  48. data/Gemfile +0 -9
  49. data/fluentd.gemspec +0 -54
  50. data/test/command/test_binlog_reader.rb +0 -362
  51. data/test/command/test_ca_generate.rb +0 -70
  52. data/test/command/test_cap_ctl.rb +0 -100
  53. data/test/command/test_cat.rb +0 -128
  54. data/test/command/test_ctl.rb +0 -56
  55. data/test/command/test_fluentd.rb +0 -1291
  56. data/test/command/test_plugin_config_formatter.rb +0 -397
  57. data/test/command/test_plugin_generator.rb +0 -109
  58. data/test/compat/test_calls_super.rb +0 -166
  59. data/test/compat/test_parser.rb +0 -92
  60. data/test/config/assertions.rb +0 -42
  61. data/test/config/test_config_parser.rb +0 -551
  62. data/test/config/test_configurable.rb +0 -1784
  63. data/test/config/test_configure_proxy.rb +0 -604
  64. data/test/config/test_dsl.rb +0 -415
  65. data/test/config/test_element.rb +0 -518
  66. data/test/config/test_literal_parser.rb +0 -309
  67. data/test/config/test_plugin_configuration.rb +0 -56
  68. data/test/config/test_section.rb +0 -191
  69. data/test/config/test_system_config.rb +0 -195
  70. data/test/config/test_types.rb +0 -408
  71. data/test/counter/test_client.rb +0 -563
  72. data/test/counter/test_error.rb +0 -44
  73. data/test/counter/test_mutex_hash.rb +0 -179
  74. data/test/counter/test_server.rb +0 -589
  75. data/test/counter/test_store.rb +0 -258
  76. data/test/counter/test_validator.rb +0 -137
  77. data/test/helper.rb +0 -155
  78. data/test/helpers/fuzzy_assert.rb +0 -89
  79. data/test/helpers/process_extenstion.rb +0 -33
  80. data/test/log/test_console_adapter.rb +0 -110
  81. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  82. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  83. data/test/plugin/data/2010/01/20100102.log +0 -0
  84. data/test/plugin/data/log/bar +0 -0
  85. data/test/plugin/data/log/foo/bar.log +0 -0
  86. data/test/plugin/data/log/foo/bar2 +0 -0
  87. data/test/plugin/data/log/test.log +0 -0
  88. data/test/plugin/data/sd_file/config +0 -11
  89. data/test/plugin/data/sd_file/config.json +0 -17
  90. data/test/plugin/data/sd_file/config.yaml +0 -11
  91. data/test/plugin/data/sd_file/config.yml +0 -11
  92. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  93. data/test/plugin/in_tail/test_fifo.rb +0 -121
  94. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  95. data/test/plugin/in_tail/test_position_file.rb +0 -346
  96. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  97. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  98. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  99. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  100. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  101. data/test/plugin/test_bare_output.rb +0 -131
  102. data/test/plugin/test_base.rb +0 -247
  103. data/test/plugin/test_buf_file.rb +0 -1314
  104. data/test/plugin/test_buf_file_single.rb +0 -898
  105. data/test/plugin/test_buf_memory.rb +0 -42
  106. data/test/plugin/test_buffer.rb +0 -1493
  107. data/test/plugin/test_buffer_chunk.rb +0 -209
  108. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  109. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  110. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  111. data/test/plugin/test_compressable.rb +0 -87
  112. data/test/plugin/test_file_util.rb +0 -96
  113. data/test/plugin/test_filter.rb +0 -368
  114. data/test/plugin/test_filter_grep.rb +0 -697
  115. data/test/plugin/test_filter_parser.rb +0 -731
  116. data/test/plugin/test_filter_record_transformer.rb +0 -577
  117. data/test/plugin/test_filter_stdout.rb +0 -207
  118. data/test/plugin/test_formatter_csv.rb +0 -136
  119. data/test/plugin/test_formatter_hash.rb +0 -38
  120. data/test/plugin/test_formatter_json.rb +0 -61
  121. data/test/plugin/test_formatter_ltsv.rb +0 -70
  122. data/test/plugin/test_formatter_msgpack.rb +0 -28
  123. data/test/plugin/test_formatter_out_file.rb +0 -116
  124. data/test/plugin/test_formatter_single_value.rb +0 -44
  125. data/test/plugin/test_formatter_tsv.rb +0 -76
  126. data/test/plugin/test_in_debug_agent.rb +0 -49
  127. data/test/plugin/test_in_exec.rb +0 -261
  128. data/test/plugin/test_in_forward.rb +0 -1178
  129. data/test/plugin/test_in_gc_stat.rb +0 -62
  130. data/test/plugin/test_in_http.rb +0 -1102
  131. data/test/plugin/test_in_monitor_agent.rb +0 -922
  132. data/test/plugin/test_in_object_space.rb +0 -66
  133. data/test/plugin/test_in_sample.rb +0 -190
  134. data/test/plugin/test_in_syslog.rb +0 -505
  135. data/test/plugin/test_in_tail.rb +0 -3288
  136. data/test/plugin/test_in_tcp.rb +0 -328
  137. data/test/plugin/test_in_udp.rb +0 -296
  138. data/test/plugin/test_in_unix.rb +0 -181
  139. data/test/plugin/test_input.rb +0 -137
  140. data/test/plugin/test_metadata.rb +0 -89
  141. data/test/plugin/test_metrics.rb +0 -294
  142. data/test/plugin/test_metrics_local.rb +0 -96
  143. data/test/plugin/test_multi_output.rb +0 -204
  144. data/test/plugin/test_out_copy.rb +0 -308
  145. data/test/plugin/test_out_exec.rb +0 -312
  146. data/test/plugin/test_out_exec_filter.rb +0 -606
  147. data/test/plugin/test_out_file.rb +0 -1038
  148. data/test/plugin/test_out_forward.rb +0 -1349
  149. data/test/plugin/test_out_http.rb +0 -429
  150. data/test/plugin/test_out_null.rb +0 -105
  151. data/test/plugin/test_out_relabel.rb +0 -28
  152. data/test/plugin/test_out_roundrobin.rb +0 -146
  153. data/test/plugin/test_out_secondary_file.rb +0 -458
  154. data/test/plugin/test_out_stdout.rb +0 -205
  155. data/test/plugin/test_out_stream.rb +0 -103
  156. data/test/plugin/test_output.rb +0 -1334
  157. data/test/plugin/test_output_as_buffered.rb +0 -2024
  158. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  159. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  160. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  161. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  162. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  163. data/test/plugin/test_output_as_standard.rb +0 -374
  164. data/test/plugin/test_owned_by.rb +0 -35
  165. data/test/plugin/test_parser.rb +0 -399
  166. data/test/plugin/test_parser_apache.rb +0 -42
  167. data/test/plugin/test_parser_apache2.rb +0 -47
  168. data/test/plugin/test_parser_apache_error.rb +0 -45
  169. data/test/plugin/test_parser_csv.rb +0 -200
  170. data/test/plugin/test_parser_json.rb +0 -138
  171. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  172. data/test/plugin/test_parser_multiline.rb +0 -111
  173. data/test/plugin/test_parser_nginx.rb +0 -88
  174. data/test/plugin/test_parser_none.rb +0 -52
  175. data/test/plugin/test_parser_regexp.rb +0 -284
  176. data/test/plugin/test_parser_syslog.rb +0 -650
  177. data/test/plugin/test_parser_tsv.rb +0 -122
  178. data/test/plugin/test_sd_file.rb +0 -228
  179. data/test/plugin/test_sd_srv.rb +0 -230
  180. data/test/plugin/test_storage.rb +0 -167
  181. data/test/plugin/test_storage_local.rb +0 -335
  182. data/test/plugin/test_string_util.rb +0 -26
  183. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  184. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  185. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  186. data/test/plugin_helper/data/cert/cert.pem +0 -19
  187. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  188. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  189. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  191. data/test/plugin_helper/data/cert/empty.pem +0 -0
  192. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  193. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  194. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  195. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  196. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  197. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  198. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  199. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  200. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  201. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  202. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  203. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  204. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  205. data/test/plugin_helper/http_server/test_app.rb +0 -65
  206. data/test/plugin_helper/http_server/test_route.rb +0 -32
  207. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  208. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  209. data/test/plugin_helper/test_cert_option.rb +0 -25
  210. data/test/plugin_helper/test_child_process.rb +0 -862
  211. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  212. data/test/plugin_helper/test_event_emitter.rb +0 -80
  213. data/test/plugin_helper/test_event_loop.rb +0 -52
  214. data/test/plugin_helper/test_extract.rb +0 -194
  215. data/test/plugin_helper/test_formatter.rb +0 -255
  216. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  217. data/test/plugin_helper/test_inject.rb +0 -561
  218. data/test/plugin_helper/test_metrics.rb +0 -137
  219. data/test/plugin_helper/test_parser.rb +0 -264
  220. data/test/plugin_helper/test_record_accessor.rb +0 -238
  221. data/test/plugin_helper/test_retry_state.rb +0 -1006
  222. data/test/plugin_helper/test_server.rb +0 -1895
  223. data/test/plugin_helper/test_service_discovery.rb +0 -165
  224. data/test/plugin_helper/test_socket.rb +0 -146
  225. data/test/plugin_helper/test_storage.rb +0 -542
  226. data/test/plugin_helper/test_thread.rb +0 -164
  227. data/test/plugin_helper/test_timer.rb +0 -130
  228. data/test/scripts/exec_script.rb +0 -32
  229. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  230. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  231. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  232. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  233. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  234. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  235. data/test/test_capability.rb +0 -74
  236. data/test/test_clock.rb +0 -164
  237. data/test/test_config.rb +0 -369
  238. data/test/test_configdsl.rb +0 -148
  239. data/test/test_daemonizer.rb +0 -91
  240. data/test/test_engine.rb +0 -203
  241. data/test/test_event.rb +0 -531
  242. data/test/test_event_router.rb +0 -348
  243. data/test/test_event_time.rb +0 -199
  244. data/test/test_file_wrapper.rb +0 -53
  245. data/test/test_filter.rb +0 -121
  246. data/test/test_fluent_log_event_router.rb +0 -99
  247. data/test/test_formatter.rb +0 -369
  248. data/test/test_input.rb +0 -31
  249. data/test/test_log.rb +0 -1076
  250. data/test/test_match.rb +0 -148
  251. data/test/test_mixin.rb +0 -351
  252. data/test/test_msgpack_factory.rb +0 -50
  253. data/test/test_oj_options.rb +0 -55
  254. data/test/test_output.rb +0 -278
  255. data/test/test_plugin.rb +0 -251
  256. data/test/test_plugin_classes.rb +0 -370
  257. data/test/test_plugin_helper.rb +0 -81
  258. data/test/test_plugin_id.rb +0 -119
  259. data/test/test_process.rb +0 -14
  260. data/test/test_root_agent.rb +0 -951
  261. data/test/test_static_config_analysis.rb +0 -177
  262. data/test/test_supervisor.rb +0 -821
  263. data/test/test_test_drivers.rb +0 -136
  264. data/test/test_time_formatter.rb +0 -301
  265. data/test/test_time_parser.rb +0 -362
  266. data/test/test_tls.rb +0 -65
  267. data/test/test_unique_id.rb +0 -47
  268. data/test/test_variable_store.rb +0 -65
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6557ce41f087cb6d7e0b7e8fa76d10259b8aa3538b917cb0b51bd4836cbdc01b
4
- data.tar.gz: 062f08fb0fd8c9b6a8d0e5ff0f2619a94bc7b07de4d94d466c0a20c03459d8f3
3
+ metadata.gz: 4921c240e6dc91b8104f572134c47656f30172a0336bf8fc499b583f99ca316e
4
+ data.tar.gz: fff090306c0ec88d17c36732f9c074afdf40c30e00e8012abfc70f7a7b2888dd
5
5
  SHA512:
6
- metadata.gz: 42c8786c9abbcb0144fc755492b41b2d6b1eb1dc77b5ffa3bce568554be262733659be42972b6b676dda52581924f29fc5976087368f865a629085a06264be83
7
- data.tar.gz: ab0b41ca7db17f7968c512433efe185fb3228856fb53730af1888ed7ee6459962298607c2a87f1d3b722c51fe637a0638f119cf0722fb0a3087c0cf45e91ced8
6
+ metadata.gz: ff35533952604174b264e9c48fbe953d880770870865993c14f89ffe730f0c09272b408d62e02779f999ab73d8fa02027aa60f14cf523f96cc87924937e0ce64
7
+ data.tar.gz: c1d9f699c770f8c522176bd365700284d63c2f4cf4859925cc730b4b27b04396eca649432494085d2891863a9232ee21f7f97250b7816d0da211b2067d8d3ff7
data/CHANGELOG.md CHANGED
@@ -1,3 +1,91 @@
1
+ # v1.17
2
+
3
+ ## Release v1.17.1 - 2024/08/19
4
+
5
+ ### Enhancement
6
+
7
+ * yaml_parser: Support $log_level element
8
+ https://github.com/fluent/fluentd/pull/4482
9
+ * out_file: Add warn message for symlink_path setting
10
+ https://github.com/fluent/fluentd/pull/4502
11
+ * out_http: Add `compress gzip` option
12
+ https://github.com/fluent/fluentd/pull/4528
13
+ * in_exec: Add `encoding` option to handle non-ascii characters
14
+ https://github.com/fluent/fluentd/pull/4533
15
+ * in_tail: Add throttling metrics
16
+ https://github.com/fluent/fluentd/pull/4578
17
+ * compat: Improve method call performance
18
+ https://github.com/fluent/fluentd/pull/4588
19
+ * in_sample: Add `reuse_record` parameter to reuse the sample data
20
+ https://github.com/fluent/fluentd/pull/4586
21
+ * `in_sample` has changed to copy sample data by default to avoid the impact of destructive changes by subsequent plugins.
22
+ * This increases the load when generating large amounts of sample data.
23
+ * You can use this new parameter to have the same performance as before.
24
+
25
+ ### Bug Fix
26
+
27
+ * logger: Fix LoadError with console gem v1.25
28
+ https://github.com/fluent/fluentd/pull/4492
29
+ * parser_json: Fix wrong LoadError warning
30
+ https://github.com/fluent/fluentd/pull/4522
31
+ * in_tail: Fix an issue where a large single line could consume a large amount of memory even though `max_line_size` is set
32
+ https://github.com/fluent/fluentd/pull/4530
33
+
34
+ ### Misc
35
+
36
+ * Comment out inappropriate default configuration about out_forward
37
+ https://github.com/fluent/fluentd/pull/4523
38
+ * gemspec: Remove unnecessary files from released gem
39
+ https://github.com/fluent/fluentd/pull/4534
40
+ * plugin-generator: Update gemspec to remove unnecessary files
41
+ https://github.com/fluent/fluentd/pull/4535
42
+ * Suppress non-parenthesis warnings
43
+ https://github.com/fluent/fluentd/pull/4594
44
+ * Fix FrozenError in http_server plugin helper
45
+ https://github.com/fluent/fluentd/pull/4598
46
+ * Add logger gem dependency for Ruby 3.5
47
+ https://github.com/fluent/fluentd/pull/4589
48
+
49
+ ## Release v1.17.0 - 2024/04/30
50
+
51
+ ### Enhancement
52
+
53
+ * in_http: Recognize CSP reports as JSON data
54
+ https://github.com/fluent/fluentd/pull/4282
55
+ * out_http: Add option to reuse connections
56
+ https://github.com/fluent/fluentd/pull/4330
57
+ * in_tail: Expand glob capability for square brackets and one character matcher
58
+ https://github.com/fluent/fluentd/pull/4401
59
+ * out_http: Support AWS Signature Version 4 authentication
60
+ https://github.com/fluent/fluentd/pull/4459
61
+
62
+ ### Bug Fix
63
+
64
+ * Make sure `parser_json` and `parser_msgpack` return `Hash`.
65
+ Make `parser_json` and `parser_msgpack` accept only `Hash` or `Array` of `Hash`.
66
+ https://github.com/fluent/fluentd/pull/4474
67
+ * filter_parser: Add error event for multiple parsed results
68
+ https://github.com/fluent/fluentd/pull/4478
69
+
70
+ ### Misc
71
+
72
+ * Raise minimum required ruby version
73
+ https://github.com/fluent/fluentd/pull/4288
74
+ * Require missing dependent gems as of Ruby 3.4-dev
75
+ https://github.com/fluent/fluentd/pull/4411
76
+ * Minor code refactoring
77
+ https://github.com/fluent/fluentd/pull/4294
78
+ https://github.com/fluent/fluentd/pull/4299
79
+ https://github.com/fluent/fluentd/pull/4302
80
+ https://github.com/fluent/fluentd/pull/4320
81
+ * CI fixes
82
+ https://github.com/fluent/fluentd/pull/4369
83
+ https://github.com/fluent/fluentd/pull/4433
84
+ https://github.com/fluent/fluentd/pull/4452
85
+ https://github.com/fluent/fluentd/pull/4477
86
+ * github: unify YAML file extension to .yml
87
+ https://github.com/fluent/fluentd/pull/4429
88
+
1
89
  # v1.16
2
90
 
3
91
  ## Release v1.16.5 - 2024/03/27
data/README.md CHANGED
@@ -6,6 +6,7 @@ Fluentd: Open-Source Log Collector
6
6
  [![Testing on macOS](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
7
7
  [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
8
8
  [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
9
+ [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/fluent/fluentd/badge)](https://scorecard.dev/viewer/?uri=github.com/fluent/fluentd)
9
10
 
10
11
  [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
11
12
 
@@ -29,7 +30,7 @@ Fluentd: Open-Source Log Collector
29
30
 
30
31
  ### Prerequisites
31
32
 
32
- - Ruby 2.4 or later
33
+ - Ruby 2.7 or later
33
34
  - git
34
35
 
35
36
  `git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.
data/Rakefile CHANGED
@@ -13,7 +13,7 @@ task test: [:base_test]
13
13
  namespace :build do
14
14
  desc 'Build gems for all platforms'
15
15
  task :all do
16
- Bundler.with_clean_env do
16
+ Bundler.with_original_env do
17
17
  %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
18
18
  ENV['GEM_BUILD_FAKE_PLATFORM'] = name
19
19
  Rake::Task["build"].execute
data/SECURITY.md CHANGED
@@ -4,9 +4,9 @@
4
4
 
5
5
  | Version | Supported |
6
6
  | ------- | ------------------ |
7
+ | 1.17.x | :white_check_mark: |
7
8
  | 1.16.x | :white_check_mark: |
8
- | 1.15.x | :white_check_mark: |
9
- | <= 1.14.x | :x: |
9
+ | <= 1.15.x | :x: |
10
10
 
11
11
  ## Reporting a Vulnerability
12
12
 
data/fluent.conf CHANGED
@@ -81,20 +81,20 @@
81
81
  @id stdout_output
82
82
  </match>
83
83
 
84
- # match tag=system.** and forward to another fluent server
85
- <match system.**>
86
- @type forward
87
- @id forward_output
88
-
89
- <server>
90
- host 192.168.0.11
91
- </server>
92
- <secondary>
93
- <server>
94
- host 192.168.0.12
95
- </server>
96
- </secondary>
97
- </match>
84
+ ## match tag=system.** and forward to another fluent server
85
+ #<match system.**>
86
+ # @type forward
87
+ # @id forward_output
88
+ #
89
+ # <server>
90
+ # host 192.168.0.11
91
+ # </server>
92
+ # <secondary>
93
+ # <server>
94
+ # host 192.168.0.12
95
+ # </server>
96
+ # </secondary>
97
+ #</match>
98
98
 
99
99
  ## match tag=myapp.** and forward and write to file
100
100
  #<match myapp.**>
@@ -130,7 +130,7 @@ module BinlogReaderCommand
130
130
  private
131
131
 
132
132
  def configure_option_parser
133
- @options.merge!(config_params: {})
133
+ @options[:config_params] = {}
134
134
 
135
135
  @opt_parser.banner = "Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file"
136
136
 
@@ -76,10 +76,10 @@ module Fluent
76
76
  parse_options!(@argv)
77
77
 
78
78
  target_file = if !!@opts[:target_file]
79
- @opts[:target_file]
80
- else
81
- File.readlink("/proc/self/exe")
82
- end
79
+ @opts[:target_file]
80
+ else
81
+ File.readlink("/proc/self/exe")
82
+ end
83
83
 
84
84
  if @opts[:clear_capabilities]
85
85
  clear_capabilities(@opts, target_file)
@@ -41,7 +41,7 @@ module Fluent
41
41
  def start
42
42
  super
43
43
  unless self.started?
44
- @@_super_start[self.class].bind(self).call
44
+ @@_super_start[self.class].bind_call(self)
45
45
  # #super will reset logdev (especially in test), so this warn should be after calling it
46
46
  log.warn "super was not called in #start: called it forcedly", plugin: self.class
47
47
  end
@@ -51,7 +51,7 @@ module Fluent
51
51
  super
52
52
  unless self.before_shutdown?
53
53
  log.warn "super was not called in #before_shutdown: calling it forcedly", plugin: self.class
54
- @@_super_before_shutdown[self.class].bind(self).call
54
+ @@_super_before_shutdown[self.class].bind_call(self)
55
55
  end
56
56
  end
57
57
 
@@ -68,7 +68,7 @@ module Fluent
68
68
  super
69
69
  unless self.shutdown?
70
70
  log.warn "super was not called in #shutdown: calling it forcedly", plugin: self.class
71
- @@_super_shutdown[self.class].bind(self).call
71
+ @@_super_shutdown[self.class].bind_call(self)
72
72
  end
73
73
  end
74
74
  end
@@ -33,25 +33,25 @@ module Fluent
33
33
  CONFIGURABLE_CLASS_METHODS = Fluent::Configurable::ClassMethods
34
34
 
35
35
  def config_param(name, type = nil, **kwargs, &block)
36
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).bind(self).call(name, type, **kwargs, &block)
36
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).bind_call(self, name, type, **kwargs, &block)
37
37
  pparams = propagate_default_params
38
38
  if kwargs.has_key?(:default) && pparams[name.to_s]
39
39
  newer = pparams[name.to_s].to_sym
40
40
  overridden_default_value = kwargs[:default]
41
41
 
42
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind(self).call(:buffer) do
42
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do
43
43
  config_set_default newer, overridden_default_value
44
44
  end
45
45
  end
46
46
  end
47
47
 
48
48
  def config_set_default(name, defval)
49
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).bind(self).call(name, defval)
49
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).bind_call(self, name, defval)
50
50
  pparams = propagate_default_params
51
51
  if pparams[name.to_s]
52
52
  newer = pparams[name.to_s].to_sym
53
53
 
54
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind(self).call(:buffer) do
54
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do
55
55
  self.config_set_default newer, defval
56
56
  end
57
57
  end
@@ -201,7 +201,7 @@ module Fluent
201
201
 
202
202
  def overwrite_defaults(other) # other is owner plugin's corresponding proxy
203
203
  self.defaults = self.defaults.merge(other.defaults)
204
- self.sections.keys.each do |section_key|
204
+ self.sections.each_key do |section_key|
205
205
  if other.sections.has_key?(section_key)
206
206
  self.sections[section_key].overwrite_defaults(other.sections[section_key])
207
207
  end
@@ -274,7 +274,7 @@ module Fluent
274
274
  option_value_type!(name, opts, :deprecated, String)
275
275
  option_value_type!(name, opts, :obsoleted, String)
276
276
  if type == :enum
277
- if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?{|v| v.is_a?(Symbol) }
277
+ if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?(Symbol)
278
278
  raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
279
279
  end
280
280
  end
@@ -123,7 +123,7 @@ module Fluent
123
123
 
124
124
  s = val.to_sym
125
125
  list = opts[:list]
126
- raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?{|v| v.is_a? Symbol }
126
+ raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?(Symbol)
127
127
  unless list.include?(s)
128
128
  raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
129
129
  end
@@ -138,6 +138,10 @@ module Fluent
138
138
  sb.add_line('@id', v)
139
139
  end
140
140
 
141
+ if (v = config.delete('$log_level'))
142
+ sb.add_line('@log_level', v)
143
+ end
144
+
141
145
  config.each do |key, val|
142
146
  if val.is_a?(Array)
143
147
  val.each do |v|
@@ -31,13 +31,13 @@ module Fluent
31
31
  super
32
32
  # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
33
33
  proxy = self.class.merged_configure_proxy
34
- proxy.params.keys.each do |name|
34
+ proxy.params.each_key do |name|
35
35
  next if name.to_s.start_with?('@')
36
36
  if proxy.defaults.has_key?(name)
37
37
  instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
38
38
  end
39
39
  end
40
- proxy.sections.keys.each do |name|
40
+ proxy.sections.each_key do |name|
41
41
  next if name.to_s.start_with?('@')
42
42
  subproxy = proxy.sections[name]
43
43
  if subproxy.multi?
@@ -54,7 +54,7 @@ module Fluent
54
54
  if mutex.try_lock
55
55
  locks[key] = mutex
56
56
  else
57
- locks.values.each(&:unlock)
57
+ locks.each_value(&:unlock)
58
58
  locks = {} # flush locked keys
59
59
  break
60
60
  end
@@ -47,8 +47,6 @@ module Fluent
47
47
  # it's not suppressed in default event router for non-log-event events
48
48
  log_event_router.suppress_missing_match!
49
49
 
50
- log_event_router = log_event_router
51
-
52
50
  unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
53
51
  unless unmatched_tags.empty?
54
52
  $log.warn "match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)", tags: unmatched_tags
@@ -14,14 +14,16 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'console/terminal/logger'
17
+ require 'console'
18
18
 
19
19
  module Fluent
20
20
  class Log
21
21
  # Async gem which is used by http_server helper switched logger mechanism to
22
22
  # Console gem which isn't complatible with Ruby's standard Logger (since
23
23
  # v1.17). This class adapts it to Fluentd's logger mechanism.
24
- class ConsoleAdapter < Console::Terminal::Logger
24
+ class ConsoleAdapter < Gem::Version.new(Console::VERSION) >= Gem::Version.new("1.25") ?
25
+ Console::Output::Terminal : Console::Terminal::Logger
26
+
25
27
  def self.wrap(logger)
26
28
  _, level = Console::Logger::LEVELS.find { |key, value|
27
29
  if logger.level <= 0
@@ -56,7 +56,7 @@ module Fluent
56
56
 
57
57
  @variable_store = Fluent::VariableStore.fetch_or_build(:buf_file)
58
58
 
59
- multi_workers_configured = owner.system_config.workers > 1 ? true : false
59
+ multi_workers_configured = owner.system_config.workers > 1
60
60
 
61
61
  using_plugin_root_dir = false
62
62
  unless @path
@@ -399,7 +399,7 @@ module Fluent
399
399
  end
400
400
 
401
401
  if chunk.slice(0, 2) == BUFFER_HEADER
402
- size = chunk.slice(2, 4).unpack('N').first
402
+ size = chunk.slice(2, 4).unpack1('N')
403
403
  if size
404
404
  return Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
405
405
  end
@@ -238,17 +238,16 @@ module Fluent
238
238
  callback.call(file) if callback
239
239
  end
240
240
 
241
- URI_PARSER = URI::Parser.new
242
241
  ESCAPE_REGEXP = /[^-_.a-zA-Z0-9]/n
243
242
 
244
243
  def encode_key(metadata)
245
244
  k = @key ? metadata.variables[@key] : metadata.tag
246
245
  k ||= ''
247
- URI_PARSER.escape(k, ESCAPE_REGEXP)
246
+ URI::DEFAULT_PARSER.escape(k, ESCAPE_REGEXP)
248
247
  end
249
248
 
250
249
  def decode_key(key)
251
- URI_PARSER.unescape(key)
250
+ URI::DEFAULT_PARSER.unescape(key)
252
251
  end
253
252
 
254
253
  def create_new_chunk(path, metadata, perm)
@@ -70,6 +70,13 @@ module Fluent::Plugin
70
70
  end
71
71
  end
72
72
  begin
73
+ # Note: https://github.com/fluent/fluentd/issues/4100
74
+ # If the parser returns multiple records from one raw_value,
75
+ # this returns only the first one record.
76
+ # This should be fixed in the future version.
77
+ result_time = nil
78
+ result_record = nil
79
+
73
80
  @parser.parse(raw_value) do |t, values|
74
81
  if values
75
82
  t = if @reserve_time
@@ -79,20 +86,31 @@ module Fluent::Plugin
79
86
  end
80
87
  @accessor.delete(record) if @remove_key_name_field
81
88
  r = handle_parsed(tag, record, t, values)
82
- return t, r
89
+
90
+ if result_record.nil?
91
+ result_time = t
92
+ result_record = r
93
+ else
94
+ if @emit_invalid_record_to_error
95
+ router.emit_error_event(tag, t, r, Fluent::Plugin::Parser::ParserError.new(
96
+ "Could not emit the event. The parser returned multiple results, but currently filter_parser plugin only returns the first parsed result. Raw data: '#{raw_value}'"
97
+ ))
98
+ end
99
+ end
83
100
  else
84
101
  if @emit_invalid_record_to_error
85
102
  router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
86
103
  end
87
- if @reserve_data
88
- t = time
89
- r = handle_parsed(tag, record, time, {})
90
- return t, r
91
- else
92
- return FAILED_RESULT
93
- end
104
+
105
+ next unless @reserve_data
106
+ next unless result_record.nil?
107
+
108
+ result_time = time
109
+ result_record = handle_parsed(tag, record, time, {})
94
110
  end
95
111
  end
112
+
113
+ return result_time, result_record
96
114
  rescue Fluent::Plugin::Parser::ParserError => e
97
115
  if @emit_invalid_record_to_error
98
116
  raise e
@@ -45,6 +45,8 @@ module Fluent::Plugin
45
45
  config_param :run_interval, :time, default: nil
46
46
  desc 'The default block size to read if parser requires partial read.'
47
47
  config_param :read_block_size, :size, default: 10240 # 10k
48
+ desc 'The encoding to receive the result of the command, especially for non-ascii characters.'
49
+ config_param :encoding, :string, default: nil
48
50
 
49
51
  attr_reader :parser
50
52
 
@@ -63,9 +65,16 @@ module Fluent::Plugin
63
65
  if !@tag && (!@extract_config || !@extract_config.tag_key)
64
66
  raise Fluent::ConfigError, "'tag' or 'tag_key' option is required on exec input"
65
67
  end
68
+ validate_encoding(@encoding) if @encoding
66
69
  @parser = parser_create
67
70
  end
68
71
 
72
+ def validate_encoding(encoding)
73
+ Encoding.find(encoding)
74
+ rescue ArgumentError => e
75
+ raise Fluent::ConfigError, e.message
76
+ end
77
+
69
78
  def multi_workers_ready?
70
79
  true
71
80
  end
@@ -73,10 +82,13 @@ module Fluent::Plugin
73
82
  def start
74
83
  super
75
84
 
85
+ options = { mode: [@connect_mode] }
86
+ options[:external_encoding] = @encoding if @encoding
87
+
76
88
  if @run_interval
77
- child_process_execute(:exec_input, @command, interval: @run_interval, mode: [@connect_mode], &method(:run))
89
+ child_process_execute(:exec_input, @command, interval: @run_interval, **options, &method(:run))
78
90
  else
79
- child_process_execute(:exec_input, @command, immediate: true, mode: [@connect_mode], &method(:run))
91
+ child_process_execute(:exec_input, @command, immediate: true, **options, &method(:run))
80
92
  end
81
93
  end
82
94
 
@@ -203,54 +203,24 @@ module Fluent::Plugin
203
203
  begin
204
204
  path = path_info[1..-1] # remove /
205
205
  tag = path.split('/').join('.')
206
- record_time, record = parse_params(params)
207
206
 
208
- # Skip nil record
209
- if record.nil?
210
- log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
211
- if @respond_with_empty_img
212
- return RESPONSE_IMG
213
- else
214
- if @use_204_response
215
- return RESPONSE_204
216
- else
217
- return RESPONSE_200
218
- end
207
+ mes = Fluent::MultiEventStream.new
208
+ parse_params(params) do |record_time, record|
209
+ if record.nil?
210
+ log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
211
+ next
219
212
  end
220
- end
221
213
 
222
- mes = nil
223
- # Support batched requests
224
- if record.is_a?(Array)
225
- mes = Fluent::MultiEventStream.new
226
- record.each do |single_record|
227
- add_params_to_record(single_record, params)
228
-
229
- if param_time = params['time']
230
- param_time = param_time.to_f
231
- single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
232
- elsif @custom_parser
233
- single_time = @custom_parser.parse_time(single_record)
234
- single_time, single_record = @custom_parser.convert_values(single_time, single_record)
235
- else
236
- single_time = convert_time_field(single_record)
237
- end
238
-
239
- mes.add(single_time, single_record)
240
- end
241
- else
242
214
  add_params_to_record(record, params)
243
215
 
244
216
  time = if param_time = params['time']
245
217
  param_time = param_time.to_f
246
218
  param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
247
219
  else
248
- if record_time.nil?
249
- convert_time_field(record)
250
- else
251
- record_time
252
- end
220
+ record_time.nil? ? convert_time_field(record) : record_time
253
221
  end
222
+
223
+ mes.add(time, record)
254
224
  end
255
225
  rescue => e
256
226
  if @dump_error_log
@@ -261,11 +231,7 @@ module Fluent::Plugin
261
231
 
262
232
  # TODO server error
263
233
  begin
264
- if mes
265
- router.emit_stream(tag, mes)
266
- else
267
- router.emit(tag, time, record)
268
- end
234
+ router.emit_stream(tag, mes) unless mes.empty?
269
235
  rescue => e
270
236
  if @dump_error_log
271
237
  log.error "failed to emit data", error: e
@@ -308,20 +274,18 @@ module Fluent::Plugin
308
274
  def parse_params_default(params)
309
275
  if msgpack = params['msgpack']
310
276
  @parser_msgpack.parse(msgpack) do |_time, record|
311
- return nil, record
277
+ yield nil, record
312
278
  end
313
279
  elsif js = params['json']
314
280
  @parser_json.parse(js) do |_time, record|
315
- return nil, record
281
+ yield nil, record
316
282
  end
317
283
  elsif ndjson = params['ndjson']
318
- events = []
319
284
  ndjson.split(/\r?\n/).each do |js|
320
285
  @parser_json.parse(js) do |_time, record|
321
- events.push(record)
286
+ yield nil, record
322
287
  end
323
288
  end
324
- return nil, events
325
289
  else
326
290
  raise "'json', 'ndjson' or 'msgpack' parameter is required"
327
291
  end
@@ -329,10 +293,9 @@ module Fluent::Plugin
329
293
 
330
294
  def parse_params_with_parser(params)
331
295
  if content = params[EVENT_RECORD_PARAMETER]
332
- @custom_parser.parse(content) { |time, record|
333
- raise "Received event is not #{@format_name}: #{content}" if record.nil?
334
- return time, record
335
- }
296
+ @custom_parser.parse(content) do |time, record|
297
+ yield time, record
298
+ end
336
299
  else
337
300
  raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
338
301
  end
@@ -485,7 +448,7 @@ module Fluent::Plugin
485
448
  # Azure App Service sends GET requests for health checking purpose.
486
449
  # Respond with `200 OK` to accommodate it.
487
450
  def handle_get_request
488
- return send_response_and_close(RES_200_STATUS, {}, "")
451
+ return send_response_and_close(RES_200_STATUS, {}, "")
489
452
  end
490
453
 
491
454
  # Web browsers can send an OPTIONS request before performing POST
@@ -573,6 +536,8 @@ module Fluent::Plugin
573
536
  params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
574
537
  elsif /^application\/json/.match?(@content_type)
575
538
  params['json'] = @body
539
+ elsif /^application\/csp-report/.match?(@content_type)
540
+ params['json'] = @body
576
541
  elsif /^application\/msgpack/.match?(@content_type)
577
542
  params['msgpack'] = @body
578
543
  elsif /^application\/x-ndjson/.match?(@content_type)
@@ -580,7 +545,7 @@ module Fluent::Plugin
580
545
  end
581
546
  path_info = uri.path
582
547
 
583
- if (@add_query_params)
548
+ if (@add_query_params)
584
549
 
585
550
  query_params = WEBrick::HTTPUtils.parse_query(uri.query)
586
551