fluentd 1.17.0-x86-mingw32 → 1.17.1-x86-mingw32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cc7924012e09f9568cbda96bbb4d1449074a4db721795d74900261b984b04047
4
- data.tar.gz: 8c3e8f5ef86237b5c18943ffbf393188338f9dbd3dbcac3d9fb321973ba3b43e
3
+ metadata.gz: 903b5b473134dbb508109e0eef063284372ebaa63a04af9811506811c244e769
4
+ data.tar.gz: f0967403164bbdb9353de96663fe823acac35c890a55db11bebd78b2456d4762
5
5
  SHA512:
6
- metadata.gz: 7998a2f12349a767e7952dcf3a17a108cfb0f2e2063524862d97b83159012e8b753e46d2e4893e235e6a4bb1d43533f3068dc955601533fddf839f75edcca096
7
- data.tar.gz: '09dac9716859404ddb3b563408cab32921c894300f2701b3e7aff89a608be154d0090e321adbbed9e0615459b06a4be3038f3e5b16206e624a0e1e770970c803'
6
+ metadata.gz: dd316e8374119e53e528294d07ff114812c720d08b1bc7101c4cce99fe4a07878c36f834c9edc410536d25248e47f53f36cd80daeb3e9950050c63ee143e7692
7
+ data.tar.gz: 2d08385d2e0014ae058778b4137db8a1cc2462c33383c29c4d2877d6fe351ea40d4ac0ee2aed87c5d15c8ca139087e89aea350279e2404fac8639ec5176f9a15
data/CHANGELOG.md CHANGED
@@ -1,5 +1,51 @@
1
1
  # v1.17
2
2
 
3
+ ## Release v1.17.1 - 2024/08/19
4
+
5
+ ### Enhancement
6
+
7
+ * yaml_parser: Support $log_level element
8
+ https://github.com/fluent/fluentd/pull/4482
9
+ * out_file: Add warn message for symlink_path setting
10
+ https://github.com/fluent/fluentd/pull/4502
11
+ * out_http: Add `compress gzip` option
12
+ https://github.com/fluent/fluentd/pull/4528
13
+ * in_exec: Add `encoding` option to handle non-ascii characters
14
+ https://github.com/fluent/fluentd/pull/4533
15
+ * in_tail: Add throttling metrics
16
+ https://github.com/fluent/fluentd/pull/4578
17
+ * compat: Improve method call performance
18
+ https://github.com/fluent/fluentd/pull/4588
19
+ * in_sample: Add `reuse_record` parameter to reuse the sample data
20
+ https://github.com/fluent/fluentd/pull/4586
21
+ * `in_sample` has changed to copy sample data by default to avoid the impact of destructive changes by subsequent plugins.
22
+ * This increases the load when generating large amounts of sample data.
23
+ * You can use this new parameter to have the same performance as before.
24
+
25
+ ### Bug Fix
26
+
27
+ * logger: Fix LoadError with console gem v1.25
28
+ https://github.com/fluent/fluentd/pull/4492
29
+ * parser_json: Fix wrong LoadError warning
30
+ https://github.com/fluent/fluentd/pull/4522
31
+ * in_tail: Fix an issue where a large single line could consume a large amount of memory even though `max_line_size` is set
32
+ https://github.com/fluent/fluentd/pull/4530
33
+
34
+ ### Misc
35
+
36
+ * Comment out inappropriate default configuration about out_forward
37
+ https://github.com/fluent/fluentd/pull/4523
38
+ * gemspec: Remove unnecessary files from released gem
39
+ https://github.com/fluent/fluentd/pull/4534
40
+ * plugin-generator: Update gemspec to remove unnecessary files
41
+ https://github.com/fluent/fluentd/pull/4535
42
+ * Suppress non-parenthesis warnings
43
+ https://github.com/fluent/fluentd/pull/4594
44
+ * Fix FrozenError in http_server plugin helper
45
+ https://github.com/fluent/fluentd/pull/4598
46
+ * Add logger gem dependency for Ruby 3.5
47
+ https://github.com/fluent/fluentd/pull/4589
48
+
3
49
  ## Release v1.17.0 - 2024/04/30
4
50
 
5
51
  ### Enhancement
data/README.md CHANGED
@@ -6,6 +6,7 @@ Fluentd: Open-Source Log Collector
6
6
  [![Testing on macOS](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
7
7
  [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
8
8
  [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
9
+ [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/fluent/fluentd/badge)](https://scorecard.dev/viewer/?uri=github.com/fluent/fluentd)
9
10
 
10
11
  [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
11
12
 
data/SECURITY.md CHANGED
@@ -4,9 +4,9 @@
4
4
 
5
5
  | Version | Supported |
6
6
  | ------- | ------------------ |
7
+ | 1.17.x | :white_check_mark: |
7
8
  | 1.16.x | :white_check_mark: |
8
- | 1.15.x | :white_check_mark: |
9
- | <= 1.14.x | :x: |
9
+ | <= 1.15.x | :x: |
10
10
 
11
11
  ## Reporting a Vulnerability
12
12
 
data/fluent.conf CHANGED
@@ -81,20 +81,20 @@
81
81
  @id stdout_output
82
82
  </match>
83
83
 
84
- # match tag=system.** and forward to another fluent server
85
- <match system.**>
86
- @type forward
87
- @id forward_output
88
-
89
- <server>
90
- host 192.168.0.11
91
- </server>
92
- <secondary>
93
- <server>
94
- host 192.168.0.12
95
- </server>
96
- </secondary>
97
- </match>
84
+ ## match tag=system.** and forward to another fluent server
85
+ #<match system.**>
86
+ # @type forward
87
+ # @id forward_output
88
+ #
89
+ # <server>
90
+ # host 192.168.0.11
91
+ # </server>
92
+ # <secondary>
93
+ # <server>
94
+ # host 192.168.0.12
95
+ # </server>
96
+ # </secondary>
97
+ #</match>
98
98
 
99
99
  ## match tag=myapp.** and forward and write to file
100
100
  #<match myapp.**>
@@ -76,10 +76,10 @@ module Fluent
76
76
  parse_options!(@argv)
77
77
 
78
78
  target_file = if !!@opts[:target_file]
79
- @opts[:target_file]
80
- else
81
- File.readlink("/proc/self/exe")
82
- end
79
+ @opts[:target_file]
80
+ else
81
+ File.readlink("/proc/self/exe")
82
+ end
83
83
 
84
84
  if @opts[:clear_capabilities]
85
85
  clear_capabilities(@opts, target_file)
@@ -41,7 +41,7 @@ module Fluent
41
41
  def start
42
42
  super
43
43
  unless self.started?
44
- @@_super_start[self.class].bind(self).call
44
+ @@_super_start[self.class].bind_call(self)
45
45
  # #super will reset logdev (especially in test), so this warn should be after calling it
46
46
  log.warn "super was not called in #start: called it forcedly", plugin: self.class
47
47
  end
@@ -51,7 +51,7 @@ module Fluent
51
51
  super
52
52
  unless self.before_shutdown?
53
53
  log.warn "super was not called in #before_shutdown: calling it forcedly", plugin: self.class
54
- @@_super_before_shutdown[self.class].bind(self).call
54
+ @@_super_before_shutdown[self.class].bind_call(self)
55
55
  end
56
56
  end
57
57
 
@@ -68,7 +68,7 @@ module Fluent
68
68
  super
69
69
  unless self.shutdown?
70
70
  log.warn "super was not called in #shutdown: calling it forcedly", plugin: self.class
71
- @@_super_shutdown[self.class].bind(self).call
71
+ @@_super_shutdown[self.class].bind_call(self)
72
72
  end
73
73
  end
74
74
  end
@@ -33,25 +33,25 @@ module Fluent
33
33
  CONFIGURABLE_CLASS_METHODS = Fluent::Configurable::ClassMethods
34
34
 
35
35
  def config_param(name, type = nil, **kwargs, &block)
36
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).bind(self).call(name, type, **kwargs, &block)
36
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).bind_call(self, name, type, **kwargs, &block)
37
37
  pparams = propagate_default_params
38
38
  if kwargs.has_key?(:default) && pparams[name.to_s]
39
39
  newer = pparams[name.to_s].to_sym
40
40
  overridden_default_value = kwargs[:default]
41
41
 
42
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind(self).call(:buffer) do
42
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do
43
43
  config_set_default newer, overridden_default_value
44
44
  end
45
45
  end
46
46
  end
47
47
 
48
48
  def config_set_default(name, defval)
49
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).bind(self).call(name, defval)
49
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).bind_call(self, name, defval)
50
50
  pparams = propagate_default_params
51
51
  if pparams[name.to_s]
52
52
  newer = pparams[name.to_s].to_sym
53
53
 
54
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind(self).call(:buffer) do
54
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do
55
55
  self.config_set_default newer, defval
56
56
  end
57
57
  end
@@ -138,6 +138,10 @@ module Fluent
138
138
  sb.add_line('@id', v)
139
139
  end
140
140
 
141
+ if (v = config.delete('$log_level'))
142
+ sb.add_line('@log_level', v)
143
+ end
144
+
141
145
  config.each do |key, val|
142
146
  if val.is_a?(Array)
143
147
  val.each do |v|
@@ -14,14 +14,16 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'console/terminal/logger'
17
+ require 'console'
18
18
 
19
19
  module Fluent
20
20
  class Log
21
21
  # Async gem which is used by http_server helper switched logger mechanism to
22
22
  # Console gem which isn't complatible with Ruby's standard Logger (since
23
23
  # v1.17). This class adapts it to Fluentd's logger mechanism.
24
- class ConsoleAdapter < Console::Terminal::Logger
24
+ class ConsoleAdapter < Gem::Version.new(Console::VERSION) >= Gem::Version.new("1.25") ?
25
+ Console::Output::Terminal : Console::Terminal::Logger
26
+
25
27
  def self.wrap(logger)
26
28
  _, level = Console::Logger::LEVELS.find { |key, value|
27
29
  if logger.level <= 0
@@ -45,6 +45,8 @@ module Fluent::Plugin
45
45
  config_param :run_interval, :time, default: nil
46
46
  desc 'The default block size to read if parser requires partial read.'
47
47
  config_param :read_block_size, :size, default: 10240 # 10k
48
+ desc 'The encoding to receive the result of the command, especially for non-ascii characters.'
49
+ config_param :encoding, :string, default: nil
48
50
 
49
51
  attr_reader :parser
50
52
 
@@ -63,9 +65,16 @@ module Fluent::Plugin
63
65
  if !@tag && (!@extract_config || !@extract_config.tag_key)
64
66
  raise Fluent::ConfigError, "'tag' or 'tag_key' option is required on exec input"
65
67
  end
68
+ validate_encoding(@encoding) if @encoding
66
69
  @parser = parser_create
67
70
  end
68
71
 
72
+ def validate_encoding(encoding)
73
+ Encoding.find(encoding)
74
+ rescue ArgumentError => e
75
+ raise Fluent::ConfigError, e.message
76
+ end
77
+
69
78
  def multi_workers_ready?
70
79
  true
71
80
  end
@@ -73,10 +82,13 @@ module Fluent::Plugin
73
82
  def start
74
83
  super
75
84
 
85
+ options = { mode: [@connect_mode] }
86
+ options[:external_encoding] = @encoding if @encoding
87
+
76
88
  if @run_interval
77
- child_process_execute(:exec_input, @command, interval: @run_interval, mode: [@connect_mode], &method(:run))
89
+ child_process_execute(:exec_input, @command, interval: @run_interval, **options, &method(:run))
78
90
  else
79
- child_process_execute(:exec_input, @command, immediate: true, mode: [@connect_mode], &method(:run))
91
+ child_process_execute(:exec_input, @command, immediate: true, **options, &method(:run))
80
92
  end
81
93
  end
82
94
 
@@ -448,7 +448,7 @@ module Fluent::Plugin
448
448
  # Azure App Service sends GET requests for health checking purpose.
449
449
  # Respond with `200 OK` to accommodate it.
450
450
  def handle_get_request
451
- return send_response_and_close(RES_200_STATUS, {}, "")
451
+ return send_response_and_close(RES_200_STATUS, {}, "")
452
452
  end
453
453
 
454
454
  # Web browsers can send an OPTIONS request before performing POST
@@ -39,6 +39,8 @@ module Fluent::Plugin
39
39
  config_param :auto_increment_key, :string, default: nil
40
40
  desc "The boolean to suspend-and-resume incremental value after restart"
41
41
  config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
42
+ desc "Reuse the sample data to reduce the load when sending large amounts of data. You can enable it if filter does not do destructive change"
43
+ config_param :reuse_record, :bool, default: false
42
44
  desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
43
45
  config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
44
46
  begin
@@ -117,15 +119,19 @@ module Fluent::Plugin
117
119
  end
118
120
  end
119
121
 
120
- def generate
121
- d = @sample[@sample_index]
122
- unless d
123
- @sample_index = 0
124
- d = @sample[@sample_index]
125
- end
122
+ def next_sample
123
+ d = @reuse_record ? @sample[@sample_index] : @sample[@sample_index].dup
126
124
  @sample_index += 1
125
+ return d if d
126
+
127
+ @sample_index = 0
128
+ next_sample
129
+ end
130
+
131
+ def generate
132
+ d = next_sample
127
133
  if @auto_increment_key
128
- d = d.dup
134
+ d = d.dup if @reuse_record
129
135
  d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
130
136
  end
131
137
  d
@@ -36,7 +36,7 @@ module Fluent::Plugin
36
36
  helpers :timer, :event_loop, :parser, :compat_parameters
37
37
 
38
38
  RESERVED_CHARS = ['/', '*', '%'].freeze
39
- MetricsInfo = Struct.new(:opened, :closed, :rotated)
39
+ MetricsInfo = Struct.new(:opened, :closed, :rotated, :throttled)
40
40
 
41
41
  class WatcherSetupError < StandardError
42
42
  def initialize(msg)
@@ -208,7 +208,8 @@ module Fluent::Plugin
208
208
  opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
209
209
  closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
210
210
  rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
211
- @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
211
+ throttling_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_throttled_total", help_text: "Total number of times throttling occurs per file when throttling enabled")
212
+ @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics, throttling_metrics)
212
213
  end
213
214
 
214
215
  def configure_tag
@@ -566,7 +567,7 @@ module Fluent::Plugin
566
567
  if @follow_inodes && new_inode.nil?
567
568
  # nil inode means the file disappeared, so we only need to stop it.
568
569
  @tails.delete(tail_watcher.path)
569
- # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
570
+ # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
570
571
  # Because of this problem, log duplication can occur during `rotate_wait`.
571
572
  # Need to set `rotate_wait 0` for a workaround.
572
573
  # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.
@@ -696,14 +697,6 @@ module Fluent::Plugin
696
697
 
697
698
  # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
698
699
  def receive_lines(lines, tail_watcher)
699
- lines = lines.reject do |line|
700
- skip_line = @max_line_size ? line.bytesize > @max_line_size : false
701
- if skip_line
702
- log.warn "received line length is longer than #{@max_line_size}"
703
- log.debug "skipped line: #{line.chomp}"
704
- end
705
- skip_line
706
- end
707
700
  es = @receive_handler.call(lines, tail_watcher)
708
701
  unless es.empty?
709
702
  tag = if @tag_prefix || @tag_suffix
@@ -801,6 +794,7 @@ module Fluent::Plugin
801
794
  'opened_file_count' => @metrics.opened.get,
802
795
  'closed_file_count' => @metrics.closed.get,
803
796
  'rotated_file_count' => @metrics.rotated.get,
797
+ 'throttled_log_count' => @metrics.throttled.get,
804
798
  })
805
799
  }
806
800
  stats
@@ -819,6 +813,7 @@ module Fluent::Plugin
819
813
  from_encoding: @from_encoding,
820
814
  encoding: @encoding,
821
815
  metrics: @metrics,
816
+ max_line_size: @max_line_size,
822
817
  &method(:receive_lines)
823
818
  )
824
819
  end
@@ -1011,15 +1006,19 @@ module Fluent::Plugin
1011
1006
  end
1012
1007
 
1013
1008
  class FIFO
1014
- def initialize(from_encoding, encoding)
1009
+ def initialize(from_encoding, encoding, log, max_line_size=nil)
1015
1010
  @from_encoding = from_encoding
1016
1011
  @encoding = encoding
1017
1012
  @need_enc = from_encoding != encoding
1018
1013
  @buffer = ''.force_encoding(from_encoding)
1019
1014
  @eol = "\n".encode(from_encoding).freeze
1015
+ @max_line_size = max_line_size
1016
+ @skip_current_line = false
1017
+ @skipping_current_line_bytesize = 0
1018
+ @log = log
1020
1019
  end
1021
1020
 
1022
- attr_reader :from_encoding, :encoding, :buffer
1021
+ attr_reader :from_encoding, :encoding, :buffer, :max_line_size
1023
1022
 
1024
1023
  def <<(chunk)
1025
1024
  # Although "chunk" is most likely transient besides String#force_encoding itself
@@ -1051,6 +1050,7 @@ module Fluent::Plugin
1051
1050
 
1052
1051
  def read_lines(lines)
1053
1052
  idx = @buffer.index(@eol)
1053
+ has_skipped_line = false
1054
1054
 
1055
1055
  until idx.nil?
1056
1056
  # Using freeze and slice is faster than slice!
@@ -1059,11 +1059,47 @@ module Fluent::Plugin
1059
1059
  rbuf = @buffer.slice(0, idx + 1)
1060
1060
  @buffer = @buffer.slice(idx + 1, @buffer.size)
1061
1061
  idx = @buffer.index(@eol)
1062
+
1063
+ is_long_line = @max_line_size && (
1064
+ @skip_current_line || rbuf.bytesize > @max_line_size
1065
+ )
1066
+
1067
+ if is_long_line
1068
+ @log.warn "received line length is longer than #{@max_line_size}"
1069
+ if @skip_current_line
1070
+ @log.debug("The continuing line is finished. Finally discarded data: ") { convert(rbuf).chomp }
1071
+ else
1072
+ @log.debug("skipped line: ") { convert(rbuf).chomp }
1073
+ end
1074
+ has_skipped_line = true
1075
+ @skip_current_line = false
1076
+ @skipping_current_line_bytesize = 0
1077
+ next
1078
+ end
1079
+
1062
1080
  lines << convert(rbuf)
1063
1081
  end
1082
+
1083
+ is_long_current_line = @max_line_size && (
1084
+ @skip_current_line || @buffer.bytesize > @max_line_size
1085
+ )
1086
+
1087
+ if is_long_current_line
1088
+ @log.debug(
1089
+ "The continuing current line length is longer than #{@max_line_size}." +
1090
+ " The received data will be discarded until this line is finished." +
1091
+ " Discarded data: "
1092
+ ) { convert(@buffer).chomp }
1093
+ @skip_current_line = true
1094
+ @skipping_current_line_bytesize += @buffer.bytesize
1095
+ @buffer.clear
1096
+ end
1097
+
1098
+ return has_skipped_line
1064
1099
  end
1065
1100
 
1066
- def bytesize
1101
+ def reading_bytesize
1102
+ return @skipping_current_line_bytesize if @skip_current_line
1067
1103
  @buffer.bytesize
1068
1104
  end
1069
1105
  end
@@ -1074,14 +1110,14 @@ module Fluent::Plugin
1074
1110
 
1075
1111
  attr_accessor :shutdown_timeout
1076
1112
 
1077
- def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
1113
+ def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, max_line_size: nil, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
1078
1114
  @watcher = watcher
1079
1115
  @path = path
1080
1116
  @read_lines_limit = read_lines_limit
1081
1117
  @read_bytes_limit_per_second = read_bytes_limit_per_second
1082
1118
  @receive_lines = receive_lines
1083
1119
  @open_on_every_update = open_on_every_update
1084
- @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT)
1120
+ @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT, log, max_line_size)
1085
1121
  @iobuf = ''.force_encoding('ASCII-8BIT')
1086
1122
  @lines = []
1087
1123
  @io = nil
@@ -1158,12 +1194,15 @@ module Fluent::Plugin
1158
1194
  end
1159
1195
 
1160
1196
  def handle_notify
1161
- return if limit_bytes_per_second_reached?
1162
- return if group_watcher&.limit_lines_reached?(@path)
1197
+ if limit_bytes_per_second_reached? || group_watcher&.limit_lines_reached?(@path)
1198
+ @metrics.throttled.inc
1199
+ return
1200
+ end
1163
1201
 
1164
1202
  with_io do |io|
1165
1203
  begin
1166
1204
  read_more = false
1205
+ has_skipped_line = false
1167
1206
 
1168
1207
  if !io.nil? && @lines.empty?
1169
1208
  begin
@@ -1177,7 +1216,7 @@ module Fluent::Plugin
1177
1216
  @fifo << data
1178
1217
 
1179
1218
  n_lines_before_read = @lines.size
1180
- @fifo.read_lines(@lines)
1219
+ has_skipped_line = @fifo.read_lines(@lines) || has_skipped_line
1181
1220
  group_watcher&.update_lines_read(@path, @lines.size - n_lines_before_read)
1182
1221
 
1183
1222
  group_watcher_limit = group_watcher&.limit_lines_reached?(@path)
@@ -1185,6 +1224,7 @@ module Fluent::Plugin
1185
1224
 
1186
1225
  if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now?
1187
1226
  # Just get out from tailing loop.
1227
+ @metrics.throttled.inc if group_watcher_limit || limit_bytes_per_second_reached?
1188
1228
  read_more = false
1189
1229
  break
1190
1230
  end
@@ -1200,9 +1240,11 @@ module Fluent::Plugin
1200
1240
  end
1201
1241
  end
1202
1242
 
1203
- unless @lines.empty?
1243
+ if @lines.empty?
1244
+ @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize) if has_skipped_line
1245
+ else
1204
1246
  if @receive_lines.call(@lines, @watcher)
1205
- @watcher.pe.update_pos(io.pos - @fifo.bytesize)
1247
+ @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize)
1206
1248
  @lines.clear
1207
1249
  else
1208
1250
  read_more = false
@@ -1214,12 +1256,12 @@ module Fluent::Plugin
1214
1256
 
1215
1257
  def open
1216
1258
  io = Fluent::FileWrapper.open(@path)
1217
- io.seek(@watcher.pe.read_pos + @fifo.bytesize)
1259
+ io.seek(@watcher.pe.read_pos + @fifo.reading_bytesize)
1218
1260
  @metrics.opened.inc
1219
1261
  io
1220
1262
  rescue RangeError
1221
1263
  io.close if io
1222
- raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
1264
+ raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.reading_bytesize.to_s(16)}"
1223
1265
  rescue Errno::EACCES => e
1224
1266
  @log.warn "#{e}"
1225
1267
  nil
@@ -89,7 +89,7 @@ module Fluent::Plugin
89
89
 
90
90
  case @copy_mode
91
91
  when :no_copy
92
- nil
92
+ nil
93
93
  when :shallow
94
94
  Proc.new { |es| es.dup }
95
95
  when :deep
@@ -172,6 +172,14 @@ module Fluent::Plugin
172
172
  log.warn "symlink_path is unavailable on Windows platform. disabled."
173
173
  @symlink_path = nil
174
174
  else
175
+ placeholder_validators(:symlink_path, @symlink_path).reject{ |v| v.type == :time }.each do |v|
176
+ begin
177
+ v.validate!
178
+ rescue Fluent::ConfigError => e
179
+ log.warn "#{e}. This means multiple chunks are competing for a single symlink_path, so some logs may not be taken from the symlink."
180
+ end
181
+ end
182
+
175
183
  @buffer.extend SymlinkBufferMixin
176
184
  @buffer.symlink_path = @symlink_path
177
185
  @buffer.output_plugin_for_symlink = self
@@ -55,6 +55,8 @@ module Fluent::Plugin
55
55
  config_param :headers, :hash, default: nil
56
56
  desc 'Additional placeholder based headers for HTTP request'
57
57
  config_param :headers_from_placeholders, :hash, default: nil
58
+ desc 'Compress HTTP request body'
59
+ config_param :compress, :enum, list: [:text, :gzip], default: :text
58
60
 
59
61
  desc 'The connection open timeout in seconds'
60
62
  config_param :open_timeout, :integer, default: nil
@@ -290,6 +292,9 @@ module Fluent::Plugin
290
292
  req[k] = extract_placeholders(v, chunk)
291
293
  end
292
294
  end
295
+ if @compress == :gzip
296
+ req['Content-Encoding'] = "gzip"
297
+ end
293
298
  req['Content-Type'] = @content_type
294
299
  end
295
300
 
@@ -323,8 +328,15 @@ module Fluent::Plugin
323
328
  Net::HTTP::Put.new(uri.request_uri)
324
329
  end
325
330
  set_headers(req, uri, chunk)
331
+
326
332
  req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
327
333
 
334
+ if @compress == :gzip
335
+ gz = Zlib::GzipWriter.new(StringIO.new)
336
+ gz << req.body
337
+ req.body = gz.close.string
338
+ end
339
+
328
340
  # At least one authentication method requires the body and other headers, so the order of this call matters
329
341
  set_auth(req, uri)
330
342
  req
@@ -50,23 +50,15 @@ module Fluent
50
50
  def configure_json_parser(name)
51
51
  case name
52
52
  when :oj
53
- raise LoadError unless Fluent::OjOptions.available?
54
- [Oj.method(:load), Oj::ParseError]
53
+ return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?
54
+
55
+ log&.info "Oj is not installed, and failing back to Yajl for json parser"
56
+ configure_json_parser(:yajl)
55
57
  when :json then [JSON.method(:load), JSON::ParserError]
56
58
  when :yajl then [Yajl.method(:load), Yajl::ParseError]
57
59
  else
58
60
  raise "BUG: unknown json parser specified: #{name}"
59
61
  end
60
- rescue LoadError => ex
61
- name = :yajl
62
- if log
63
- if /\boj\z/.match?(ex.message)
64
- log.info "Oj is not installed, and failing back to Yajl for json parser"
65
- else
66
- log.warn ex.message
67
- end
68
- end
69
- retry
70
62
  end
71
63
 
72
64
  def parse(text)
@@ -78,7 +78,7 @@ module Fluent
78
78
  HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
79
79
  define_method(name) do |path, app = nil, &block|
80
80
  unless path.end_with?('/')
81
- path << '/'
81
+ path += '/'
82
82
  end
83
83
 
84
84
  if (block && app) || (!block && !app)
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.17.0'
19
+ VERSION = '1.17.1'
20
20
 
21
21
  end
@@ -12,12 +12,13 @@ Gem::Specification.new do |spec|
12
12
  spec.homepage = "TODO: Put your gem's website or public repo URL here."
13
13
  spec.license = "<%= @license.name %>"
14
14
 
15
- test_files, files = `git ls-files -z`.split("\x0").partition do |f|
16
- f.match(%r{^(test|spec|features)/})
15
+ spec.files = Dir.chdir(__dir__) do
16
+ `git ls-files -z`.split("\x0").reject do |f|
17
+ (File.expand_path(f) == __FILE__) ||
18
+ f.start_with?(*%w[test/ spec/ features/ .git .circleci appveyor Gemfile])
19
+ end
17
20
  end
18
- spec.files = files
19
- spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
20
- spec.test_files = test_files
21
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
21
22
  spec.require_paths = ["lib"]
22
23
 
23
24
  spec.add_development_dependency "bundler", "~> <%= bundler_version %>"