fluentd 1.17.0-x64-mingw32 → 1.17.1-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.
Files changed (259) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +46 -0
  3. data/README.md +1 -0
  4. data/SECURITY.md +2 -2
  5. data/fluent.conf +14 -14
  6. data/lib/fluent/command/cap_ctl.rb +4 -4
  7. data/lib/fluent/compat/call_super_mixin.rb +3 -3
  8. data/lib/fluent/compat/propagate_default.rb +4 -4
  9. data/lib/fluent/config/yaml_parser/parser.rb +4 -0
  10. data/lib/fluent/log/console_adapter.rb +4 -2
  11. data/lib/fluent/plugin/in_exec.rb +14 -2
  12. data/lib/fluent/plugin/in_http.rb +1 -1
  13. data/lib/fluent/plugin/in_sample.rb +13 -7
  14. data/lib/fluent/plugin/in_tail.rb +65 -23
  15. data/lib/fluent/plugin/out_copy.rb +1 -1
  16. data/lib/fluent/plugin/out_file.rb +8 -0
  17. data/lib/fluent/plugin/out_http.rb +12 -0
  18. data/lib/fluent/plugin/parser_json.rb +4 -12
  19. data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
  20. data/lib/fluent/version.rb +1 -1
  21. data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
  22. metadata +25 -472
  23. data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
  24. data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
  25. data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
  26. data/.github/ISSUE_TEMPLATE/config.yml +0 -5
  27. data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
  28. data/.github/ISSUE_TEMPLATE.md +0 -17
  29. data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  30. data/.github/workflows/stale-actions.yml +0 -24
  31. data/.github/workflows/test-ruby-head.yml +0 -31
  32. data/.github/workflows/test.yml +0 -32
  33. data/.gitignore +0 -30
  34. data/Gemfile +0 -9
  35. data/fluentd.gemspec +0 -62
  36. data/test/command/test_binlog_reader.rb +0 -362
  37. data/test/command/test_ca_generate.rb +0 -70
  38. data/test/command/test_cap_ctl.rb +0 -100
  39. data/test/command/test_cat.rb +0 -128
  40. data/test/command/test_ctl.rb +0 -56
  41. data/test/command/test_fluentd.rb +0 -1291
  42. data/test/command/test_plugin_config_formatter.rb +0 -397
  43. data/test/command/test_plugin_generator.rb +0 -109
  44. data/test/compat/test_calls_super.rb +0 -166
  45. data/test/compat/test_parser.rb +0 -92
  46. data/test/config/assertions.rb +0 -42
  47. data/test/config/test_config_parser.rb +0 -551
  48. data/test/config/test_configurable.rb +0 -1784
  49. data/test/config/test_configure_proxy.rb +0 -604
  50. data/test/config/test_dsl.rb +0 -415
  51. data/test/config/test_element.rb +0 -518
  52. data/test/config/test_literal_parser.rb +0 -309
  53. data/test/config/test_plugin_configuration.rb +0 -56
  54. data/test/config/test_section.rb +0 -191
  55. data/test/config/test_system_config.rb +0 -195
  56. data/test/config/test_types.rb +0 -408
  57. data/test/counter/test_client.rb +0 -563
  58. data/test/counter/test_error.rb +0 -44
  59. data/test/counter/test_mutex_hash.rb +0 -179
  60. data/test/counter/test_server.rb +0 -589
  61. data/test/counter/test_store.rb +0 -258
  62. data/test/counter/test_validator.rb +0 -137
  63. data/test/helper.rb +0 -155
  64. data/test/helpers/fuzzy_assert.rb +0 -89
  65. data/test/helpers/process_extenstion.rb +0 -33
  66. data/test/log/test_console_adapter.rb +0 -117
  67. data/test/plugin/data/2010/01/20100102-030405.log +0 -0
  68. data/test/plugin/data/2010/01/20100102-030406.log +0 -0
  69. data/test/plugin/data/2010/01/20100102.log +0 -0
  70. data/test/plugin/data/log/bar +0 -0
  71. data/test/plugin/data/log/foo/bar.log +0 -0
  72. data/test/plugin/data/log/foo/bar2 +0 -0
  73. data/test/plugin/data/log/test.log +0 -0
  74. data/test/plugin/data/log_numeric/01.log +0 -0
  75. data/test/plugin/data/log_numeric/02.log +0 -0
  76. data/test/plugin/data/log_numeric/12.log +0 -0
  77. data/test/plugin/data/log_numeric/14.log +0 -0
  78. data/test/plugin/data/sd_file/config +0 -11
  79. data/test/plugin/data/sd_file/config.json +0 -17
  80. data/test/plugin/data/sd_file/config.yaml +0 -11
  81. data/test/plugin/data/sd_file/config.yml +0 -11
  82. data/test/plugin/data/sd_file/invalid_config.yml +0 -7
  83. data/test/plugin/in_tail/test_fifo.rb +0 -121
  84. data/test/plugin/in_tail/test_io_handler.rb +0 -150
  85. data/test/plugin/in_tail/test_position_file.rb +0 -346
  86. data/test/plugin/out_forward/test_ack_handler.rb +0 -140
  87. data/test/plugin/out_forward/test_connection_manager.rb +0 -145
  88. data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
  89. data/test/plugin/out_forward/test_load_balancer.rb +0 -106
  90. data/test/plugin/out_forward/test_socket_cache.rb +0 -174
  91. data/test/plugin/test_bare_output.rb +0 -131
  92. data/test/plugin/test_base.rb +0 -247
  93. data/test/plugin/test_buf_file.rb +0 -1314
  94. data/test/plugin/test_buf_file_single.rb +0 -898
  95. data/test/plugin/test_buf_memory.rb +0 -42
  96. data/test/plugin/test_buffer.rb +0 -1493
  97. data/test/plugin/test_buffer_chunk.rb +0 -209
  98. data/test/plugin/test_buffer_file_chunk.rb +0 -871
  99. data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
  100. data/test/plugin/test_buffer_memory_chunk.rb +0 -339
  101. data/test/plugin/test_compressable.rb +0 -87
  102. data/test/plugin/test_file_util.rb +0 -96
  103. data/test/plugin/test_filter.rb +0 -368
  104. data/test/plugin/test_filter_grep.rb +0 -697
  105. data/test/plugin/test_filter_parser.rb +0 -731
  106. data/test/plugin/test_filter_record_transformer.rb +0 -577
  107. data/test/plugin/test_filter_stdout.rb +0 -207
  108. data/test/plugin/test_formatter_csv.rb +0 -136
  109. data/test/plugin/test_formatter_hash.rb +0 -38
  110. data/test/plugin/test_formatter_json.rb +0 -61
  111. data/test/plugin/test_formatter_ltsv.rb +0 -70
  112. data/test/plugin/test_formatter_msgpack.rb +0 -28
  113. data/test/plugin/test_formatter_out_file.rb +0 -116
  114. data/test/plugin/test_formatter_single_value.rb +0 -44
  115. data/test/plugin/test_formatter_tsv.rb +0 -76
  116. data/test/plugin/test_in_debug_agent.rb +0 -49
  117. data/test/plugin/test_in_exec.rb +0 -261
  118. data/test/plugin/test_in_forward.rb +0 -1178
  119. data/test/plugin/test_in_gc_stat.rb +0 -62
  120. data/test/plugin/test_in_http.rb +0 -1124
  121. data/test/plugin/test_in_monitor_agent.rb +0 -922
  122. data/test/plugin/test_in_object_space.rb +0 -66
  123. data/test/plugin/test_in_sample.rb +0 -190
  124. data/test/plugin/test_in_syslog.rb +0 -505
  125. data/test/plugin/test_in_tail.rb +0 -3429
  126. data/test/plugin/test_in_tcp.rb +0 -328
  127. data/test/plugin/test_in_udp.rb +0 -296
  128. data/test/plugin/test_in_unix.rb +0 -181
  129. data/test/plugin/test_input.rb +0 -137
  130. data/test/plugin/test_metadata.rb +0 -89
  131. data/test/plugin/test_metrics.rb +0 -294
  132. data/test/plugin/test_metrics_local.rb +0 -96
  133. data/test/plugin/test_multi_output.rb +0 -204
  134. data/test/plugin/test_out_copy.rb +0 -308
  135. data/test/plugin/test_out_exec.rb +0 -312
  136. data/test/plugin/test_out_exec_filter.rb +0 -606
  137. data/test/plugin/test_out_file.rb +0 -1038
  138. data/test/plugin/test_out_forward.rb +0 -1349
  139. data/test/plugin/test_out_http.rb +0 -557
  140. data/test/plugin/test_out_null.rb +0 -105
  141. data/test/plugin/test_out_relabel.rb +0 -28
  142. data/test/plugin/test_out_roundrobin.rb +0 -146
  143. data/test/plugin/test_out_secondary_file.rb +0 -458
  144. data/test/plugin/test_out_stdout.rb +0 -205
  145. data/test/plugin/test_out_stream.rb +0 -103
  146. data/test/plugin/test_output.rb +0 -1334
  147. data/test/plugin/test_output_as_buffered.rb +0 -2024
  148. data/test/plugin/test_output_as_buffered_backup.rb +0 -363
  149. data/test/plugin/test_output_as_buffered_compress.rb +0 -179
  150. data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
  151. data/test/plugin/test_output_as_buffered_retries.rb +0 -966
  152. data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
  153. data/test/plugin/test_output_as_standard.rb +0 -374
  154. data/test/plugin/test_owned_by.rb +0 -34
  155. data/test/plugin/test_parser.rb +0 -399
  156. data/test/plugin/test_parser_apache.rb +0 -42
  157. data/test/plugin/test_parser_apache2.rb +0 -47
  158. data/test/plugin/test_parser_apache_error.rb +0 -45
  159. data/test/plugin/test_parser_csv.rb +0 -200
  160. data/test/plugin/test_parser_json.rb +0 -244
  161. data/test/plugin/test_parser_labeled_tsv.rb +0 -160
  162. data/test/plugin/test_parser_msgpack.rb +0 -127
  163. data/test/plugin/test_parser_multiline.rb +0 -111
  164. data/test/plugin/test_parser_nginx.rb +0 -88
  165. data/test/plugin/test_parser_none.rb +0 -52
  166. data/test/plugin/test_parser_regexp.rb +0 -284
  167. data/test/plugin/test_parser_syslog.rb +0 -650
  168. data/test/plugin/test_parser_tsv.rb +0 -122
  169. data/test/plugin/test_sd_file.rb +0 -228
  170. data/test/plugin/test_sd_srv.rb +0 -230
  171. data/test/plugin/test_storage.rb +0 -166
  172. data/test/plugin/test_storage_local.rb +0 -335
  173. data/test/plugin/test_string_util.rb +0 -26
  174. data/test/plugin_helper/data/cert/cert-key.pem +0 -27
  175. data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
  176. data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
  177. data/test/plugin_helper/data/cert/cert.pem +0 -19
  178. data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
  179. data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
  180. data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
  181. data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
  182. data/test/plugin_helper/data/cert/empty.pem +0 -0
  183. data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
  184. data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
  185. data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
  186. data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
  187. data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
  188. data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
  189. data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
  190. data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
  191. data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
  192. data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
  193. data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
  194. data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
  195. data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
  196. data/test/plugin_helper/http_server/test_app.rb +0 -65
  197. data/test/plugin_helper/http_server/test_route.rb +0 -32
  198. data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
  199. data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
  200. data/test/plugin_helper/test_cert_option.rb +0 -25
  201. data/test/plugin_helper/test_child_process.rb +0 -862
  202. data/test/plugin_helper/test_compat_parameters.rb +0 -358
  203. data/test/plugin_helper/test_event_emitter.rb +0 -80
  204. data/test/plugin_helper/test_event_loop.rb +0 -52
  205. data/test/plugin_helper/test_extract.rb +0 -194
  206. data/test/plugin_helper/test_formatter.rb +0 -255
  207. data/test/plugin_helper/test_http_server_helper.rb +0 -372
  208. data/test/plugin_helper/test_inject.rb +0 -561
  209. data/test/plugin_helper/test_metrics.rb +0 -137
  210. data/test/plugin_helper/test_parser.rb +0 -264
  211. data/test/plugin_helper/test_record_accessor.rb +0 -238
  212. data/test/plugin_helper/test_retry_state.rb +0 -1006
  213. data/test/plugin_helper/test_server.rb +0 -1895
  214. data/test/plugin_helper/test_service_discovery.rb +0 -165
  215. data/test/plugin_helper/test_socket.rb +0 -146
  216. data/test/plugin_helper/test_storage.rb +0 -542
  217. data/test/plugin_helper/test_thread.rb +0 -164
  218. data/test/plugin_helper/test_timer.rb +0 -130
  219. data/test/scripts/exec_script.rb +0 -32
  220. data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
  221. data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
  222. data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
  223. data/test/scripts/fluent/plugin/out_test.rb +0 -81
  224. data/test/scripts/fluent/plugin/out_test2.rb +0 -80
  225. data/test/scripts/fluent/plugin/parser_known.rb +0 -4
  226. data/test/test_capability.rb +0 -74
  227. data/test/test_clock.rb +0 -164
  228. data/test/test_config.rb +0 -369
  229. data/test/test_configdsl.rb +0 -148
  230. data/test/test_daemonizer.rb +0 -91
  231. data/test/test_engine.rb +0 -203
  232. data/test/test_event.rb +0 -531
  233. data/test/test_event_router.rb +0 -348
  234. data/test/test_event_time.rb +0 -199
  235. data/test/test_file_wrapper.rb +0 -53
  236. data/test/test_filter.rb +0 -121
  237. data/test/test_fluent_log_event_router.rb +0 -99
  238. data/test/test_formatter.rb +0 -369
  239. data/test/test_input.rb +0 -31
  240. data/test/test_log.rb +0 -1076
  241. data/test/test_match.rb +0 -148
  242. data/test/test_mixin.rb +0 -351
  243. data/test/test_msgpack_factory.rb +0 -50
  244. data/test/test_oj_options.rb +0 -55
  245. data/test/test_output.rb +0 -278
  246. data/test/test_plugin.rb +0 -251
  247. data/test/test_plugin_classes.rb +0 -370
  248. data/test/test_plugin_helper.rb +0 -81
  249. data/test/test_plugin_id.rb +0 -119
  250. data/test/test_process.rb +0 -14
  251. data/test/test_root_agent.rb +0 -951
  252. data/test/test_static_config_analysis.rb +0 -177
  253. data/test/test_supervisor.rb +0 -821
  254. data/test/test_test_drivers.rb +0 -136
  255. data/test/test_time_formatter.rb +0 -301
  256. data/test/test_time_parser.rb +0 -362
  257. data/test/test_tls.rb +0 -65
  258. data/test/test_unique_id.rb +0 -47
  259. data/test/test_variable_store.rb +0 -65
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e73288c9df7fa99e4865e553ce418ad0ce0214ac97eaab3bd260b01b200dc7cb
4
- data.tar.gz: 8c3e8f5ef86237b5c18943ffbf393188338f9dbd3dbcac3d9fb321973ba3b43e
3
+ metadata.gz: 7e55a7c45b8027074365502256a34130db0adf0671d8439a914969740f2a26a3
4
+ data.tar.gz: f0967403164bbdb9353de96663fe823acac35c890a55db11bebd78b2456d4762
5
5
  SHA512:
6
- metadata.gz: 7856e88fb46cb823a43f59f8034a7d7c4b3f7c53b7fde03118b85a02fe4c3eff0fc6207c95411ba7229217aba7f8825411589424082e35d8847cc7a1380d1320
7
- data.tar.gz: '09dac9716859404ddb3b563408cab32921c894300f2701b3e7aff89a608be154d0090e321adbbed9e0615459b06a4be3038f3e5b16206e624a0e1e770970c803'
6
+ metadata.gz: 38fa3fe620def4afd7bd9ef2fdbfc035f1b098d8151fd26b05920866acb0a6a66d5a75dec94c94d66544571a8e1e05a9c5d64dd2c83f1145e8735054ff4628cf
7
+ data.tar.gz: 2d08385d2e0014ae058778b4137db8a1cc2462c33383c29c4d2877d6fe351ea40d4ac0ee2aed87c5d15c8ca139087e89aea350279e2404fac8639ec5176f9a15
data/CHANGELOG.md CHANGED
@@ -1,5 +1,51 @@
1
1
  # v1.17
2
2
 
3
+ ## Release v1.17.1 - 2024/08/19
4
+
5
+ ### Enhancement
6
+
7
+ * yaml_parser: Support $log_level element
8
+ https://github.com/fluent/fluentd/pull/4482
9
+ * out_file: Add warn message for symlink_path setting
10
+ https://github.com/fluent/fluentd/pull/4502
11
+ * out_http: Add `compress gzip` option
12
+ https://github.com/fluent/fluentd/pull/4528
13
+ * in_exec: Add `encoding` option to handle non-ascii characters
14
+ https://github.com/fluent/fluentd/pull/4533
15
+ * in_tail: Add throttling metrics
16
+ https://github.com/fluent/fluentd/pull/4578
17
+ * compat: Improve method call performance
18
+ https://github.com/fluent/fluentd/pull/4588
19
+ * in_sample: Add `reuse_record` parameter to reuse the sample data
20
+ https://github.com/fluent/fluentd/pull/4586
21
+ * `in_sample` has changed to copy sample data by default to avoid the impact of destructive changes by subsequent plugins.
22
+ * This increases the load when generating large amounts of sample data.
23
+ * You can use this new parameter to have the same performance as before.
24
+
25
+ ### Bug Fix
26
+
27
+ * logger: Fix LoadError with console gem v1.25
28
+ https://github.com/fluent/fluentd/pull/4492
29
+ * parser_json: Fix wrong LoadError warning
30
+ https://github.com/fluent/fluentd/pull/4522
31
+ * in_tail: Fix an issue where a large single line could consume a large amount of memory even though `max_line_size` is set
32
+ https://github.com/fluent/fluentd/pull/4530
33
+
34
+ ### Misc
35
+
36
+ * Comment out inappropriate default configuration about out_forward
37
+ https://github.com/fluent/fluentd/pull/4523
38
+ * gemspec: Remove unnecessary files from released gem
39
+ https://github.com/fluent/fluentd/pull/4534
40
+ * plugin-generator: Update gemspec to remove unnecessary files
41
+ https://github.com/fluent/fluentd/pull/4535
42
+ * Suppress non-parenthesis warnings
43
+ https://github.com/fluent/fluentd/pull/4594
44
+ * Fix FrozenError in http_server plugin helper
45
+ https://github.com/fluent/fluentd/pull/4598
46
+ * Add logger gem dependency for Ruby 3.5
47
+ https://github.com/fluent/fluentd/pull/4589
48
+
3
49
  ## Release v1.17.0 - 2024/04/30
4
50
 
5
51
  ### Enhancement
data/README.md CHANGED
@@ -6,6 +6,7 @@ Fluentd: Open-Source Log Collector
6
6
  [![Testing on macOS](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
7
7
  [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
8
8
  [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
9
+ [![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/fluent/fluentd/badge)](https://scorecard.dev/viewer/?uri=github.com/fluent/fluentd)
9
10
 
10
11
  [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
11
12
 
data/SECURITY.md CHANGED
@@ -4,9 +4,9 @@
4
4
 
5
5
  | Version | Supported |
6
6
  | ------- | ------------------ |
7
+ | 1.17.x | :white_check_mark: |
7
8
  | 1.16.x | :white_check_mark: |
8
- | 1.15.x | :white_check_mark: |
9
- | <= 1.14.x | :x: |
9
+ | <= 1.15.x | :x: |
10
10
 
11
11
  ## Reporting a Vulnerability
12
12
 
data/fluent.conf CHANGED
@@ -81,20 +81,20 @@
81
81
  @id stdout_output
82
82
  </match>
83
83
 
84
- # match tag=system.** and forward to another fluent server
85
- <match system.**>
86
- @type forward
87
- @id forward_output
88
-
89
- <server>
90
- host 192.168.0.11
91
- </server>
92
- <secondary>
93
- <server>
94
- host 192.168.0.12
95
- </server>
96
- </secondary>
97
- </match>
84
+ ## match tag=system.** and forward to another fluent server
85
+ #<match system.**>
86
+ # @type forward
87
+ # @id forward_output
88
+ #
89
+ # <server>
90
+ # host 192.168.0.11
91
+ # </server>
92
+ # <secondary>
93
+ # <server>
94
+ # host 192.168.0.12
95
+ # </server>
96
+ # </secondary>
97
+ #</match>
98
98
 
99
99
  ## match tag=myapp.** and forward and write to file
100
100
  #<match myapp.**>
@@ -76,10 +76,10 @@ module Fluent
76
76
  parse_options!(@argv)
77
77
 
78
78
  target_file = if !!@opts[:target_file]
79
- @opts[:target_file]
80
- else
81
- File.readlink("/proc/self/exe")
82
- end
79
+ @opts[:target_file]
80
+ else
81
+ File.readlink("/proc/self/exe")
82
+ end
83
83
 
84
84
  if @opts[:clear_capabilities]
85
85
  clear_capabilities(@opts, target_file)
@@ -41,7 +41,7 @@ module Fluent
41
41
  def start
42
42
  super
43
43
  unless self.started?
44
- @@_super_start[self.class].bind(self).call
44
+ @@_super_start[self.class].bind_call(self)
45
45
  # #super will reset logdev (especially in test), so this warn should be after calling it
46
46
  log.warn "super was not called in #start: called it forcedly", plugin: self.class
47
47
  end
@@ -51,7 +51,7 @@ module Fluent
51
51
  super
52
52
  unless self.before_shutdown?
53
53
  log.warn "super was not called in #before_shutdown: calling it forcedly", plugin: self.class
54
- @@_super_before_shutdown[self.class].bind(self).call
54
+ @@_super_before_shutdown[self.class].bind_call(self)
55
55
  end
56
56
  end
57
57
 
@@ -68,7 +68,7 @@ module Fluent
68
68
  super
69
69
  unless self.shutdown?
70
70
  log.warn "super was not called in #shutdown: calling it forcedly", plugin: self.class
71
- @@_super_shutdown[self.class].bind(self).call
71
+ @@_super_shutdown[self.class].bind_call(self)
72
72
  end
73
73
  end
74
74
  end
@@ -33,25 +33,25 @@ module Fluent
33
33
  CONFIGURABLE_CLASS_METHODS = Fluent::Configurable::ClassMethods
34
34
 
35
35
  def config_param(name, type = nil, **kwargs, &block)
36
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).bind(self).call(name, type, **kwargs, &block)
36
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).bind_call(self, name, type, **kwargs, &block)
37
37
  pparams = propagate_default_params
38
38
  if kwargs.has_key?(:default) && pparams[name.to_s]
39
39
  newer = pparams[name.to_s].to_sym
40
40
  overridden_default_value = kwargs[:default]
41
41
 
42
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind(self).call(:buffer) do
42
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do
43
43
  config_set_default newer, overridden_default_value
44
44
  end
45
45
  end
46
46
  end
47
47
 
48
48
  def config_set_default(name, defval)
49
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).bind(self).call(name, defval)
49
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).bind_call(self, name, defval)
50
50
  pparams = propagate_default_params
51
51
  if pparams[name.to_s]
52
52
  newer = pparams[name.to_s].to_sym
53
53
 
54
- CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind(self).call(:buffer) do
54
+ CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do
55
55
  self.config_set_default newer, defval
56
56
  end
57
57
  end
@@ -138,6 +138,10 @@ module Fluent
138
138
  sb.add_line('@id', v)
139
139
  end
140
140
 
141
+ if (v = config.delete('$log_level'))
142
+ sb.add_line('@log_level', v)
143
+ end
144
+
141
145
  config.each do |key, val|
142
146
  if val.is_a?(Array)
143
147
  val.each do |v|
@@ -14,14 +14,16 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
- require 'console/terminal/logger'
17
+ require 'console'
18
18
 
19
19
  module Fluent
20
20
  class Log
21
21
  # Async gem which is used by http_server helper switched logger mechanism to
22
22
  # Console gem which isn't complatible with Ruby's standard Logger (since
23
23
  # v1.17). This class adapts it to Fluentd's logger mechanism.
24
- class ConsoleAdapter < Console::Terminal::Logger
24
+ class ConsoleAdapter < Gem::Version.new(Console::VERSION) >= Gem::Version.new("1.25") ?
25
+ Console::Output::Terminal : Console::Terminal::Logger
26
+
25
27
  def self.wrap(logger)
26
28
  _, level = Console::Logger::LEVELS.find { |key, value|
27
29
  if logger.level <= 0
@@ -45,6 +45,8 @@ module Fluent::Plugin
45
45
  config_param :run_interval, :time, default: nil
46
46
  desc 'The default block size to read if parser requires partial read.'
47
47
  config_param :read_block_size, :size, default: 10240 # 10k
48
+ desc 'The encoding to receive the result of the command, especially for non-ascii characters.'
49
+ config_param :encoding, :string, default: nil
48
50
 
49
51
  attr_reader :parser
50
52
 
@@ -63,9 +65,16 @@ module Fluent::Plugin
63
65
  if !@tag && (!@extract_config || !@extract_config.tag_key)
64
66
  raise Fluent::ConfigError, "'tag' or 'tag_key' option is required on exec input"
65
67
  end
68
+ validate_encoding(@encoding) if @encoding
66
69
  @parser = parser_create
67
70
  end
68
71
 
72
+ def validate_encoding(encoding)
73
+ Encoding.find(encoding)
74
+ rescue ArgumentError => e
75
+ raise Fluent::ConfigError, e.message
76
+ end
77
+
69
78
  def multi_workers_ready?
70
79
  true
71
80
  end
@@ -73,10 +82,13 @@ module Fluent::Plugin
73
82
  def start
74
83
  super
75
84
 
85
+ options = { mode: [@connect_mode] }
86
+ options[:external_encoding] = @encoding if @encoding
87
+
76
88
  if @run_interval
77
- child_process_execute(:exec_input, @command, interval: @run_interval, mode: [@connect_mode], &method(:run))
89
+ child_process_execute(:exec_input, @command, interval: @run_interval, **options, &method(:run))
78
90
  else
79
- child_process_execute(:exec_input, @command, immediate: true, mode: [@connect_mode], &method(:run))
91
+ child_process_execute(:exec_input, @command, immediate: true, **options, &method(:run))
80
92
  end
81
93
  end
82
94
 
@@ -448,7 +448,7 @@ module Fluent::Plugin
448
448
  # Azure App Service sends GET requests for health checking purpose.
449
449
  # Respond with `200 OK` to accommodate it.
450
450
  def handle_get_request
451
- return send_response_and_close(RES_200_STATUS, {}, "")
451
+ return send_response_and_close(RES_200_STATUS, {}, "")
452
452
  end
453
453
 
454
454
  # Web browsers can send an OPTIONS request before performing POST
@@ -39,6 +39,8 @@ module Fluent::Plugin
39
39
  config_param :auto_increment_key, :string, default: nil
40
40
  desc "The boolean to suspend-and-resume incremental value after restart"
41
41
  config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
42
+ desc "Reuse the sample data to reduce the load when sending large amounts of data. You can enable it if filter does not do destructive change"
43
+ config_param :reuse_record, :bool, default: false
42
44
  desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
43
45
  config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
44
46
  begin
@@ -117,15 +119,19 @@ module Fluent::Plugin
117
119
  end
118
120
  end
119
121
 
120
- def generate
121
- d = @sample[@sample_index]
122
- unless d
123
- @sample_index = 0
124
- d = @sample[@sample_index]
125
- end
122
+ def next_sample
123
+ d = @reuse_record ? @sample[@sample_index] : @sample[@sample_index].dup
126
124
  @sample_index += 1
125
+ return d if d
126
+
127
+ @sample_index = 0
128
+ next_sample
129
+ end
130
+
131
+ def generate
132
+ d = next_sample
127
133
  if @auto_increment_key
128
- d = d.dup
134
+ d = d.dup if @reuse_record
129
135
  d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
130
136
  end
131
137
  d
@@ -36,7 +36,7 @@ module Fluent::Plugin
36
36
  helpers :timer, :event_loop, :parser, :compat_parameters
37
37
 
38
38
  RESERVED_CHARS = ['/', '*', '%'].freeze
39
- MetricsInfo = Struct.new(:opened, :closed, :rotated)
39
+ MetricsInfo = Struct.new(:opened, :closed, :rotated, :throttled)
40
40
 
41
41
  class WatcherSetupError < StandardError
42
42
  def initialize(msg)
@@ -208,7 +208,8 @@ module Fluent::Plugin
208
208
  opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
209
209
  closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
210
210
  rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
211
- @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
211
+ throttling_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_throttled_total", help_text: "Total number of times throttling occurs per file when throttling enabled")
212
+ @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics, throttling_metrics)
212
213
  end
213
214
 
214
215
  def configure_tag
@@ -566,7 +567,7 @@ module Fluent::Plugin
566
567
  if @follow_inodes && new_inode.nil?
567
568
  # nil inode means the file disappeared, so we only need to stop it.
568
569
  @tails.delete(tail_watcher.path)
569
- # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
570
+ # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
570
571
  # Because of this problem, log duplication can occur during `rotate_wait`.
571
572
  # Need to set `rotate_wait 0` for a workaround.
572
573
  # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.
@@ -696,14 +697,6 @@ module Fluent::Plugin
696
697
 
697
698
  # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
698
699
  def receive_lines(lines, tail_watcher)
699
- lines = lines.reject do |line|
700
- skip_line = @max_line_size ? line.bytesize > @max_line_size : false
701
- if skip_line
702
- log.warn "received line length is longer than #{@max_line_size}"
703
- log.debug "skipped line: #{line.chomp}"
704
- end
705
- skip_line
706
- end
707
700
  es = @receive_handler.call(lines, tail_watcher)
708
701
  unless es.empty?
709
702
  tag = if @tag_prefix || @tag_suffix
@@ -801,6 +794,7 @@ module Fluent::Plugin
801
794
  'opened_file_count' => @metrics.opened.get,
802
795
  'closed_file_count' => @metrics.closed.get,
803
796
  'rotated_file_count' => @metrics.rotated.get,
797
+ 'throttled_log_count' => @metrics.throttled.get,
804
798
  })
805
799
  }
806
800
  stats
@@ -819,6 +813,7 @@ module Fluent::Plugin
819
813
  from_encoding: @from_encoding,
820
814
  encoding: @encoding,
821
815
  metrics: @metrics,
816
+ max_line_size: @max_line_size,
822
817
  &method(:receive_lines)
823
818
  )
824
819
  end
@@ -1011,15 +1006,19 @@ module Fluent::Plugin
1011
1006
  end
1012
1007
 
1013
1008
  class FIFO
1014
- def initialize(from_encoding, encoding)
1009
+ def initialize(from_encoding, encoding, log, max_line_size=nil)
1015
1010
  @from_encoding = from_encoding
1016
1011
  @encoding = encoding
1017
1012
  @need_enc = from_encoding != encoding
1018
1013
  @buffer = ''.force_encoding(from_encoding)
1019
1014
  @eol = "\n".encode(from_encoding).freeze
1015
+ @max_line_size = max_line_size
1016
+ @skip_current_line = false
1017
+ @skipping_current_line_bytesize = 0
1018
+ @log = log
1020
1019
  end
1021
1020
 
1022
- attr_reader :from_encoding, :encoding, :buffer
1021
+ attr_reader :from_encoding, :encoding, :buffer, :max_line_size
1023
1022
 
1024
1023
  def <<(chunk)
1025
1024
  # Although "chunk" is most likely transient besides String#force_encoding itself
@@ -1051,6 +1050,7 @@ module Fluent::Plugin
1051
1050
 
1052
1051
  def read_lines(lines)
1053
1052
  idx = @buffer.index(@eol)
1053
+ has_skipped_line = false
1054
1054
 
1055
1055
  until idx.nil?
1056
1056
  # Using freeze and slice is faster than slice!
@@ -1059,11 +1059,47 @@ module Fluent::Plugin
1059
1059
  rbuf = @buffer.slice(0, idx + 1)
1060
1060
  @buffer = @buffer.slice(idx + 1, @buffer.size)
1061
1061
  idx = @buffer.index(@eol)
1062
+
1063
+ is_long_line = @max_line_size && (
1064
+ @skip_current_line || rbuf.bytesize > @max_line_size
1065
+ )
1066
+
1067
+ if is_long_line
1068
+ @log.warn "received line length is longer than #{@max_line_size}"
1069
+ if @skip_current_line
1070
+ @log.debug("The continuing line is finished. Finally discarded data: ") { convert(rbuf).chomp }
1071
+ else
1072
+ @log.debug("skipped line: ") { convert(rbuf).chomp }
1073
+ end
1074
+ has_skipped_line = true
1075
+ @skip_current_line = false
1076
+ @skipping_current_line_bytesize = 0
1077
+ next
1078
+ end
1079
+
1062
1080
  lines << convert(rbuf)
1063
1081
  end
1082
+
1083
+ is_long_current_line = @max_line_size && (
1084
+ @skip_current_line || @buffer.bytesize > @max_line_size
1085
+ )
1086
+
1087
+ if is_long_current_line
1088
+ @log.debug(
1089
+ "The continuing current line length is longer than #{@max_line_size}." +
1090
+ " The received data will be discarded until this line is finished." +
1091
+ " Discarded data: "
1092
+ ) { convert(@buffer).chomp }
1093
+ @skip_current_line = true
1094
+ @skipping_current_line_bytesize += @buffer.bytesize
1095
+ @buffer.clear
1096
+ end
1097
+
1098
+ return has_skipped_line
1064
1099
  end
1065
1100
 
1066
- def bytesize
1101
+ def reading_bytesize
1102
+ return @skipping_current_line_bytesize if @skip_current_line
1067
1103
  @buffer.bytesize
1068
1104
  end
1069
1105
  end
@@ -1074,14 +1110,14 @@ module Fluent::Plugin
1074
1110
 
1075
1111
  attr_accessor :shutdown_timeout
1076
1112
 
1077
- def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
1113
+ def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, max_line_size: nil, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
1078
1114
  @watcher = watcher
1079
1115
  @path = path
1080
1116
  @read_lines_limit = read_lines_limit
1081
1117
  @read_bytes_limit_per_second = read_bytes_limit_per_second
1082
1118
  @receive_lines = receive_lines
1083
1119
  @open_on_every_update = open_on_every_update
1084
- @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT)
1120
+ @fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT, log, max_line_size)
1085
1121
  @iobuf = ''.force_encoding('ASCII-8BIT')
1086
1122
  @lines = []
1087
1123
  @io = nil
@@ -1158,12 +1194,15 @@ module Fluent::Plugin
1158
1194
  end
1159
1195
 
1160
1196
  def handle_notify
1161
- return if limit_bytes_per_second_reached?
1162
- return if group_watcher&.limit_lines_reached?(@path)
1197
+ if limit_bytes_per_second_reached? || group_watcher&.limit_lines_reached?(@path)
1198
+ @metrics.throttled.inc
1199
+ return
1200
+ end
1163
1201
 
1164
1202
  with_io do |io|
1165
1203
  begin
1166
1204
  read_more = false
1205
+ has_skipped_line = false
1167
1206
 
1168
1207
  if !io.nil? && @lines.empty?
1169
1208
  begin
@@ -1177,7 +1216,7 @@ module Fluent::Plugin
1177
1216
  @fifo << data
1178
1217
 
1179
1218
  n_lines_before_read = @lines.size
1180
- @fifo.read_lines(@lines)
1219
+ has_skipped_line = @fifo.read_lines(@lines) || has_skipped_line
1181
1220
  group_watcher&.update_lines_read(@path, @lines.size - n_lines_before_read)
1182
1221
 
1183
1222
  group_watcher_limit = group_watcher&.limit_lines_reached?(@path)
@@ -1185,6 +1224,7 @@ module Fluent::Plugin
1185
1224
 
1186
1225
  if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now?
1187
1226
  # Just get out from tailing loop.
1227
+ @metrics.throttled.inc if group_watcher_limit || limit_bytes_per_second_reached?
1188
1228
  read_more = false
1189
1229
  break
1190
1230
  end
@@ -1200,9 +1240,11 @@ module Fluent::Plugin
1200
1240
  end
1201
1241
  end
1202
1242
 
1203
- unless @lines.empty?
1243
+ if @lines.empty?
1244
+ @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize) if has_skipped_line
1245
+ else
1204
1246
  if @receive_lines.call(@lines, @watcher)
1205
- @watcher.pe.update_pos(io.pos - @fifo.bytesize)
1247
+ @watcher.pe.update_pos(io.pos - @fifo.reading_bytesize)
1206
1248
  @lines.clear
1207
1249
  else
1208
1250
  read_more = false
@@ -1214,12 +1256,12 @@ module Fluent::Plugin
1214
1256
 
1215
1257
  def open
1216
1258
  io = Fluent::FileWrapper.open(@path)
1217
- io.seek(@watcher.pe.read_pos + @fifo.bytesize)
1259
+ io.seek(@watcher.pe.read_pos + @fifo.reading_bytesize)
1218
1260
  @metrics.opened.inc
1219
1261
  io
1220
1262
  rescue RangeError
1221
1263
  io.close if io
1222
- raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
1264
+ raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.reading_bytesize.to_s(16)}"
1223
1265
  rescue Errno::EACCES => e
1224
1266
  @log.warn "#{e}"
1225
1267
  nil
@@ -89,7 +89,7 @@ module Fluent::Plugin
89
89
 
90
90
  case @copy_mode
91
91
  when :no_copy
92
- nil
92
+ nil
93
93
  when :shallow
94
94
  Proc.new { |es| es.dup }
95
95
  when :deep
@@ -172,6 +172,14 @@ module Fluent::Plugin
172
172
  log.warn "symlink_path is unavailable on Windows platform. disabled."
173
173
  @symlink_path = nil
174
174
  else
175
+ placeholder_validators(:symlink_path, @symlink_path).reject{ |v| v.type == :time }.each do |v|
176
+ begin
177
+ v.validate!
178
+ rescue Fluent::ConfigError => e
179
+ log.warn "#{e}. This means multiple chunks are competing for a single symlink_path, so some logs may not be taken from the symlink."
180
+ end
181
+ end
182
+
175
183
  @buffer.extend SymlinkBufferMixin
176
184
  @buffer.symlink_path = @symlink_path
177
185
  @buffer.output_plugin_for_symlink = self
@@ -55,6 +55,8 @@ module Fluent::Plugin
55
55
  config_param :headers, :hash, default: nil
56
56
  desc 'Additional placeholder based headers for HTTP request'
57
57
  config_param :headers_from_placeholders, :hash, default: nil
58
+ desc 'Compress HTTP request body'
59
+ config_param :compress, :enum, list: [:text, :gzip], default: :text
58
60
 
59
61
  desc 'The connection open timeout in seconds'
60
62
  config_param :open_timeout, :integer, default: nil
@@ -290,6 +292,9 @@ module Fluent::Plugin
290
292
  req[k] = extract_placeholders(v, chunk)
291
293
  end
292
294
  end
295
+ if @compress == :gzip
296
+ req['Content-Encoding'] = "gzip"
297
+ end
293
298
  req['Content-Type'] = @content_type
294
299
  end
295
300
 
@@ -323,8 +328,15 @@ module Fluent::Plugin
323
328
  Net::HTTP::Put.new(uri.request_uri)
324
329
  end
325
330
  set_headers(req, uri, chunk)
331
+
326
332
  req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
327
333
 
334
+ if @compress == :gzip
335
+ gz = Zlib::GzipWriter.new(StringIO.new)
336
+ gz << req.body
337
+ req.body = gz.close.string
338
+ end
339
+
328
340
  # At least one authentication method requires the body and other headers, so the order of this call matters
329
341
  set_auth(req, uri)
330
342
  req
@@ -50,23 +50,15 @@ module Fluent
50
50
  def configure_json_parser(name)
51
51
  case name
52
52
  when :oj
53
- raise LoadError unless Fluent::OjOptions.available?
54
- [Oj.method(:load), Oj::ParseError]
53
+ return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?
54
+
55
+ log&.info "Oj is not installed, and failing back to Yajl for json parser"
56
+ configure_json_parser(:yajl)
55
57
  when :json then [JSON.method(:load), JSON::ParserError]
56
58
  when :yajl then [Yajl.method(:load), Yajl::ParseError]
57
59
  else
58
60
  raise "BUG: unknown json parser specified: #{name}"
59
61
  end
60
- rescue LoadError => ex
61
- name = :yajl
62
- if log
63
- if /\boj\z/.match?(ex.message)
64
- log.info "Oj is not installed, and failing back to Yajl for json parser"
65
- else
66
- log.warn ex.message
67
- end
68
- end
69
- retry
70
62
  end
71
63
 
72
64
  def parse(text)
@@ -78,7 +78,7 @@ module Fluent
78
78
  HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
79
79
  define_method(name) do |path, app = nil, &block|
80
80
  unless path.end_with?('/')
81
- path << '/'
81
+ path += '/'
82
82
  end
83
83
 
84
84
  if (block && app) || (!block && !app)
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.17.0'
19
+ VERSION = '1.17.1'
20
20
 
21
21
  end
@@ -12,12 +12,13 @@ Gem::Specification.new do |spec|
12
12
  spec.homepage = "TODO: Put your gem's website or public repo URL here."
13
13
  spec.license = "<%= @license.name %>"
14
14
 
15
- test_files, files = `git ls-files -z`.split("\x0").partition do |f|
16
- f.match(%r{^(test|spec|features)/})
15
+ spec.files = Dir.chdir(__dir__) do
16
+ `git ls-files -z`.split("\x0").reject do |f|
17
+ (File.expand_path(f) == __FILE__) ||
18
+ f.start_with?(*%w[test/ spec/ features/ .git .circleci appveyor Gemfile])
19
+ end
17
20
  end
18
- spec.files = files
19
- spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
20
- spec.test_files = test_files
21
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
21
22
  spec.require_paths = ["lib"]
22
23
 
23
24
  spec.add_development_dependency "bundler", "~> <%= bundler_version %>"