fluentd 1.17.0-x86-mingw32 → 1.17.1-x86-mingw32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -0
- data/README.md +1 -0
- data/SECURITY.md +2 -2
- data/fluent.conf +14 -14
- data/lib/fluent/command/cap_ctl.rb +4 -4
- data/lib/fluent/compat/call_super_mixin.rb +3 -3
- data/lib/fluent/compat/propagate_default.rb +4 -4
- data/lib/fluent/config/yaml_parser/parser.rb +4 -0
- data/lib/fluent/log/console_adapter.rb +4 -2
- data/lib/fluent/plugin/in_exec.rb +14 -2
- data/lib/fluent/plugin/in_http.rb +1 -1
- data/lib/fluent/plugin/in_sample.rb +13 -7
- data/lib/fluent/plugin/in_tail.rb +65 -23
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_file.rb +8 -0
- data/lib/fluent/plugin/out_http.rb +12 -0
- data/lib/fluent/plugin/parser_json.rb +4 -12
- data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
- metadata +25 -472
- data/.github/DISCUSSION_TEMPLATE/q-a-japanese.yml +0 -50
- data/.github/DISCUSSION_TEMPLATE/q-a.yml +0 -47
- data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
- data/.github/ISSUE_TEMPLATE/config.yml +0 -5
- data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
- data/.github/ISSUE_TEMPLATE.md +0 -17
- data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
- data/.github/workflows/stale-actions.yml +0 -24
- data/.github/workflows/test-ruby-head.yml +0 -31
- data/.github/workflows/test.yml +0 -32
- data/.gitignore +0 -30
- data/Gemfile +0 -9
- data/fluentd.gemspec +0 -62
- data/test/command/test_binlog_reader.rb +0 -362
- data/test/command/test_ca_generate.rb +0 -70
- data/test/command/test_cap_ctl.rb +0 -100
- data/test/command/test_cat.rb +0 -128
- data/test/command/test_ctl.rb +0 -56
- data/test/command/test_fluentd.rb +0 -1291
- data/test/command/test_plugin_config_formatter.rb +0 -397
- data/test/command/test_plugin_generator.rb +0 -109
- data/test/compat/test_calls_super.rb +0 -166
- data/test/compat/test_parser.rb +0 -92
- data/test/config/assertions.rb +0 -42
- data/test/config/test_config_parser.rb +0 -551
- data/test/config/test_configurable.rb +0 -1784
- data/test/config/test_configure_proxy.rb +0 -604
- data/test/config/test_dsl.rb +0 -415
- data/test/config/test_element.rb +0 -518
- data/test/config/test_literal_parser.rb +0 -309
- data/test/config/test_plugin_configuration.rb +0 -56
- data/test/config/test_section.rb +0 -191
- data/test/config/test_system_config.rb +0 -195
- data/test/config/test_types.rb +0 -408
- data/test/counter/test_client.rb +0 -563
- data/test/counter/test_error.rb +0 -44
- data/test/counter/test_mutex_hash.rb +0 -179
- data/test/counter/test_server.rb +0 -589
- data/test/counter/test_store.rb +0 -258
- data/test/counter/test_validator.rb +0 -137
- data/test/helper.rb +0 -155
- data/test/helpers/fuzzy_assert.rb +0 -89
- data/test/helpers/process_extenstion.rb +0 -33
- data/test/log/test_console_adapter.rb +0 -117
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/data/log_numeric/01.log +0 -0
- data/test/plugin/data/log_numeric/02.log +0 -0
- data/test/plugin/data/log_numeric/12.log +0 -0
- data/test/plugin/data/log_numeric/14.log +0 -0
- data/test/plugin/data/sd_file/config +0 -11
- data/test/plugin/data/sd_file/config.json +0 -17
- data/test/plugin/data/sd_file/config.yaml +0 -11
- data/test/plugin/data/sd_file/config.yml +0 -11
- data/test/plugin/data/sd_file/invalid_config.yml +0 -7
- data/test/plugin/in_tail/test_fifo.rb +0 -121
- data/test/plugin/in_tail/test_io_handler.rb +0 -150
- data/test/plugin/in_tail/test_position_file.rb +0 -346
- data/test/plugin/out_forward/test_ack_handler.rb +0 -140
- data/test/plugin/out_forward/test_connection_manager.rb +0 -145
- data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
- data/test/plugin/out_forward/test_load_balancer.rb +0 -106
- data/test/plugin/out_forward/test_socket_cache.rb +0 -174
- data/test/plugin/test_bare_output.rb +0 -131
- data/test/plugin/test_base.rb +0 -247
- data/test/plugin/test_buf_file.rb +0 -1314
- data/test/plugin/test_buf_file_single.rb +0 -898
- data/test/plugin/test_buf_memory.rb +0 -42
- data/test/plugin/test_buffer.rb +0 -1493
- data/test/plugin/test_buffer_chunk.rb +0 -209
- data/test/plugin/test_buffer_file_chunk.rb +0 -871
- data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
- data/test/plugin/test_buffer_memory_chunk.rb +0 -339
- data/test/plugin/test_compressable.rb +0 -87
- data/test/plugin/test_file_util.rb +0 -96
- data/test/plugin/test_filter.rb +0 -368
- data/test/plugin/test_filter_grep.rb +0 -697
- data/test/plugin/test_filter_parser.rb +0 -731
- data/test/plugin/test_filter_record_transformer.rb +0 -577
- data/test/plugin/test_filter_stdout.rb +0 -207
- data/test/plugin/test_formatter_csv.rb +0 -136
- data/test/plugin/test_formatter_hash.rb +0 -38
- data/test/plugin/test_formatter_json.rb +0 -61
- data/test/plugin/test_formatter_ltsv.rb +0 -70
- data/test/plugin/test_formatter_msgpack.rb +0 -28
- data/test/plugin/test_formatter_out_file.rb +0 -116
- data/test/plugin/test_formatter_single_value.rb +0 -44
- data/test/plugin/test_formatter_tsv.rb +0 -76
- data/test/plugin/test_in_debug_agent.rb +0 -49
- data/test/plugin/test_in_exec.rb +0 -261
- data/test/plugin/test_in_forward.rb +0 -1178
- data/test/plugin/test_in_gc_stat.rb +0 -62
- data/test/plugin/test_in_http.rb +0 -1124
- data/test/plugin/test_in_monitor_agent.rb +0 -922
- data/test/plugin/test_in_object_space.rb +0 -66
- data/test/plugin/test_in_sample.rb +0 -190
- data/test/plugin/test_in_syslog.rb +0 -505
- data/test/plugin/test_in_tail.rb +0 -3429
- data/test/plugin/test_in_tcp.rb +0 -328
- data/test/plugin/test_in_udp.rb +0 -296
- data/test/plugin/test_in_unix.rb +0 -181
- data/test/plugin/test_input.rb +0 -137
- data/test/plugin/test_metadata.rb +0 -89
- data/test/plugin/test_metrics.rb +0 -294
- data/test/plugin/test_metrics_local.rb +0 -96
- data/test/plugin/test_multi_output.rb +0 -204
- data/test/plugin/test_out_copy.rb +0 -308
- data/test/plugin/test_out_exec.rb +0 -312
- data/test/plugin/test_out_exec_filter.rb +0 -606
- data/test/plugin/test_out_file.rb +0 -1038
- data/test/plugin/test_out_forward.rb +0 -1349
- data/test/plugin/test_out_http.rb +0 -557
- data/test/plugin/test_out_null.rb +0 -105
- data/test/plugin/test_out_relabel.rb +0 -28
- data/test/plugin/test_out_roundrobin.rb +0 -146
- data/test/plugin/test_out_secondary_file.rb +0 -458
- data/test/plugin/test_out_stdout.rb +0 -205
- data/test/plugin/test_out_stream.rb +0 -103
- data/test/plugin/test_output.rb +0 -1334
- data/test/plugin/test_output_as_buffered.rb +0 -2024
- data/test/plugin/test_output_as_buffered_backup.rb +0 -363
- data/test/plugin/test_output_as_buffered_compress.rb +0 -179
- data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
- data/test/plugin/test_output_as_buffered_retries.rb +0 -966
- data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
- data/test/plugin/test_output_as_standard.rb +0 -374
- data/test/plugin/test_owned_by.rb +0 -34
- data/test/plugin/test_parser.rb +0 -399
- data/test/plugin/test_parser_apache.rb +0 -42
- data/test/plugin/test_parser_apache2.rb +0 -47
- data/test/plugin/test_parser_apache_error.rb +0 -45
- data/test/plugin/test_parser_csv.rb +0 -200
- data/test/plugin/test_parser_json.rb +0 -244
- data/test/plugin/test_parser_labeled_tsv.rb +0 -160
- data/test/plugin/test_parser_msgpack.rb +0 -127
- data/test/plugin/test_parser_multiline.rb +0 -111
- data/test/plugin/test_parser_nginx.rb +0 -88
- data/test/plugin/test_parser_none.rb +0 -52
- data/test/plugin/test_parser_regexp.rb +0 -284
- data/test/plugin/test_parser_syslog.rb +0 -650
- data/test/plugin/test_parser_tsv.rb +0 -122
- data/test/plugin/test_sd_file.rb +0 -228
- data/test/plugin/test_sd_srv.rb +0 -230
- data/test/plugin/test_storage.rb +0 -166
- data/test/plugin/test_storage_local.rb +0 -335
- data/test/plugin/test_string_util.rb +0 -26
- data/test/plugin_helper/data/cert/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
- data/test/plugin_helper/data/cert/cert.pem +0 -19
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
- data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
- data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
- data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
- data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
- data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
- data/test/plugin_helper/http_server/test_app.rb +0 -65
- data/test/plugin_helper/http_server/test_route.rb +0 -32
- data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
- data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
- data/test/plugin_helper/test_cert_option.rb +0 -25
- data/test/plugin_helper/test_child_process.rb +0 -862
- data/test/plugin_helper/test_compat_parameters.rb +0 -358
- data/test/plugin_helper/test_event_emitter.rb +0 -80
- data/test/plugin_helper/test_event_loop.rb +0 -52
- data/test/plugin_helper/test_extract.rb +0 -194
- data/test/plugin_helper/test_formatter.rb +0 -255
- data/test/plugin_helper/test_http_server_helper.rb +0 -372
- data/test/plugin_helper/test_inject.rb +0 -561
- data/test/plugin_helper/test_metrics.rb +0 -137
- data/test/plugin_helper/test_parser.rb +0 -264
- data/test/plugin_helper/test_record_accessor.rb +0 -238
- data/test/plugin_helper/test_retry_state.rb +0 -1006
- data/test/plugin_helper/test_server.rb +0 -1895
- data/test/plugin_helper/test_service_discovery.rb +0 -165
- data/test/plugin_helper/test_socket.rb +0 -146
- data/test/plugin_helper/test_storage.rb +0 -542
- data/test/plugin_helper/test_thread.rb +0 -164
- data/test/plugin_helper/test_timer.rb +0 -130
- data/test/scripts/exec_script.rb +0 -32
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
- data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
- data/test/scripts/fluent/plugin/out_test.rb +0 -81
- data/test/scripts/fluent/plugin/out_test2.rb +0 -80
- data/test/scripts/fluent/plugin/parser_known.rb +0 -4
- data/test/test_capability.rb +0 -74
- data/test/test_clock.rb +0 -164
- data/test/test_config.rb +0 -369
- data/test/test_configdsl.rb +0 -148
- data/test/test_daemonizer.rb +0 -91
- data/test/test_engine.rb +0 -203
- data/test/test_event.rb +0 -531
- data/test/test_event_router.rb +0 -348
- data/test/test_event_time.rb +0 -199
- data/test/test_file_wrapper.rb +0 -53
- data/test/test_filter.rb +0 -121
- data/test/test_fluent_log_event_router.rb +0 -99
- data/test/test_formatter.rb +0 -369
- data/test/test_input.rb +0 -31
- data/test/test_log.rb +0 -1076
- data/test/test_match.rb +0 -148
- data/test/test_mixin.rb +0 -351
- data/test/test_msgpack_factory.rb +0 -50
- data/test/test_oj_options.rb +0 -55
- data/test/test_output.rb +0 -278
- data/test/test_plugin.rb +0 -251
- data/test/test_plugin_classes.rb +0 -370
- data/test/test_plugin_helper.rb +0 -81
- data/test/test_plugin_id.rb +0 -119
- data/test/test_process.rb +0 -14
- data/test/test_root_agent.rb +0 -951
- data/test/test_static_config_analysis.rb +0 -177
- data/test/test_supervisor.rb +0 -821
- data/test/test_test_drivers.rb +0 -136
- data/test/test_time_formatter.rb +0 -301
- data/test/test_time_parser.rb +0 -362
- data/test/test_tls.rb +0 -65
- data/test/test_unique_id.rb +0 -47
- data/test/test_variable_store.rb +0 -65
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 903b5b473134dbb508109e0eef063284372ebaa63a04af9811506811c244e769
|
4
|
+
data.tar.gz: f0967403164bbdb9353de96663fe823acac35c890a55db11bebd78b2456d4762
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dd316e8374119e53e528294d07ff114812c720d08b1bc7101c4cce99fe4a07878c36f834c9edc410536d25248e47f53f36cd80daeb3e9950050c63ee143e7692
|
7
|
+
data.tar.gz: 2d08385d2e0014ae058778b4137db8a1cc2462c33383c29c4d2877d6fe351ea40d4ac0ee2aed87c5d15c8ca139087e89aea350279e2404fac8639ec5176f9a15
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,51 @@
|
|
1
1
|
# v1.17
|
2
2
|
|
3
|
+
## Release v1.17.1 - 2024/08/19
|
4
|
+
|
5
|
+
### Enhancement
|
6
|
+
|
7
|
+
* yaml_parser: Support $log_level element
|
8
|
+
https://github.com/fluent/fluentd/pull/4482
|
9
|
+
* out_file: Add warn message for symlink_path setting
|
10
|
+
https://github.com/fluent/fluentd/pull/4502
|
11
|
+
* out_http: Add `compress gzip` option
|
12
|
+
https://github.com/fluent/fluentd/pull/4528
|
13
|
+
* in_exec: Add `encoding` option to handle non-ascii characters
|
14
|
+
https://github.com/fluent/fluentd/pull/4533
|
15
|
+
* in_tail: Add throttling metrics
|
16
|
+
https://github.com/fluent/fluentd/pull/4578
|
17
|
+
* compat: Improve method call performance
|
18
|
+
https://github.com/fluent/fluentd/pull/4588
|
19
|
+
* in_sample: Add `reuse_record` parameter to reuse the sample data
|
20
|
+
https://github.com/fluent/fluentd/pull/4586
|
21
|
+
* `in_sample` has changed to copy sample data by default to avoid the impact of destructive changes by subsequent plugins.
|
22
|
+
* This increases the load when generating large amounts of sample data.
|
23
|
+
* You can use this new parameter to have the same performance as before.
|
24
|
+
|
25
|
+
### Bug Fix
|
26
|
+
|
27
|
+
* logger: Fix LoadError with console gem v1.25
|
28
|
+
https://github.com/fluent/fluentd/pull/4492
|
29
|
+
* parser_json: Fix wrong LoadError warning
|
30
|
+
https://github.com/fluent/fluentd/pull/4522
|
31
|
+
* in_tail: Fix an issue where a large single line could consume a large amount of memory even though `max_line_size` is set
|
32
|
+
https://github.com/fluent/fluentd/pull/4530
|
33
|
+
|
34
|
+
### Misc
|
35
|
+
|
36
|
+
* Comment out inappropriate default configuration about out_forward
|
37
|
+
https://github.com/fluent/fluentd/pull/4523
|
38
|
+
* gemspec: Remove unnecessary files from released gem
|
39
|
+
https://github.com/fluent/fluentd/pull/4534
|
40
|
+
* plugin-generator: Update gemspec to remove unnecessary files
|
41
|
+
https://github.com/fluent/fluentd/pull/4535
|
42
|
+
* Suppress non-parenthesis warnings
|
43
|
+
https://github.com/fluent/fluentd/pull/4594
|
44
|
+
* Fix FrozenError in http_server plugin helper
|
45
|
+
https://github.com/fluent/fluentd/pull/4598
|
46
|
+
* Add logger gem dependency for Ruby 3.5
|
47
|
+
https://github.com/fluent/fluentd/pull/4589
|
48
|
+
|
3
49
|
## Release v1.17.0 - 2024/04/30
|
4
50
|
|
5
51
|
### Enhancement
|
data/README.md
CHANGED
@@ -6,6 +6,7 @@ Fluentd: Open-Source Log Collector
|
|
6
6
|
[](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
|
7
7
|
[](https://codeclimate.com/github/fluent/fluentd)
|
8
8
|
[](https://bestpractices.coreinfrastructure.org/projects/1189)
|
9
|
+
[](https://scorecard.dev/viewer/?uri=github.com/fluent/fluentd)
|
9
10
|
|
10
11
|
[Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
|
11
12
|
|
data/SECURITY.md
CHANGED
data/fluent.conf
CHANGED
@@ -81,20 +81,20 @@
|
|
81
81
|
@id stdout_output
|
82
82
|
</match>
|
83
83
|
|
84
|
-
|
85
|
-
|
86
|
-
@type forward
|
87
|
-
@id forward_output
|
88
|
-
|
89
|
-
<server>
|
90
|
-
host 192.168.0.11
|
91
|
-
</server>
|
92
|
-
<secondary>
|
93
|
-
<server>
|
94
|
-
host 192.168.0.12
|
95
|
-
</server>
|
96
|
-
</secondary>
|
97
|
-
|
84
|
+
## match tag=system.** and forward to another fluent server
|
85
|
+
#<match system.**>
|
86
|
+
# @type forward
|
87
|
+
# @id forward_output
|
88
|
+
#
|
89
|
+
# <server>
|
90
|
+
# host 192.168.0.11
|
91
|
+
# </server>
|
92
|
+
# <secondary>
|
93
|
+
# <server>
|
94
|
+
# host 192.168.0.12
|
95
|
+
# </server>
|
96
|
+
# </secondary>
|
97
|
+
#</match>
|
98
98
|
|
99
99
|
## match tag=myapp.** and forward and write to file
|
100
100
|
#<match myapp.**>
|
@@ -76,10 +76,10 @@ module Fluent
|
|
76
76
|
parse_options!(@argv)
|
77
77
|
|
78
78
|
target_file = if !!@opts[:target_file]
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
79
|
+
@opts[:target_file]
|
80
|
+
else
|
81
|
+
File.readlink("/proc/self/exe")
|
82
|
+
end
|
83
83
|
|
84
84
|
if @opts[:clear_capabilities]
|
85
85
|
clear_capabilities(@opts, target_file)
|
@@ -41,7 +41,7 @@ module Fluent
|
|
41
41
|
def start
|
42
42
|
super
|
43
43
|
unless self.started?
|
44
|
-
@@_super_start[self.class].
|
44
|
+
@@_super_start[self.class].bind_call(self)
|
45
45
|
# #super will reset logdev (especially in test), so this warn should be after calling it
|
46
46
|
log.warn "super was not called in #start: called it forcedly", plugin: self.class
|
47
47
|
end
|
@@ -51,7 +51,7 @@ module Fluent
|
|
51
51
|
super
|
52
52
|
unless self.before_shutdown?
|
53
53
|
log.warn "super was not called in #before_shutdown: calling it forcedly", plugin: self.class
|
54
|
-
@@_super_before_shutdown[self.class].
|
54
|
+
@@_super_before_shutdown[self.class].bind_call(self)
|
55
55
|
end
|
56
56
|
end
|
57
57
|
|
@@ -68,7 +68,7 @@ module Fluent
|
|
68
68
|
super
|
69
69
|
unless self.shutdown?
|
70
70
|
log.warn "super was not called in #shutdown: calling it forcedly", plugin: self.class
|
71
|
-
@@_super_shutdown[self.class].
|
71
|
+
@@_super_shutdown[self.class].bind_call(self)
|
72
72
|
end
|
73
73
|
end
|
74
74
|
end
|
@@ -33,25 +33,25 @@ module Fluent
|
|
33
33
|
CONFIGURABLE_CLASS_METHODS = Fluent::Configurable::ClassMethods
|
34
34
|
|
35
35
|
def config_param(name, type = nil, **kwargs, &block)
|
36
|
-
CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).
|
36
|
+
CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).bind_call(self, name, type, **kwargs, &block)
|
37
37
|
pparams = propagate_default_params
|
38
38
|
if kwargs.has_key?(:default) && pparams[name.to_s]
|
39
39
|
newer = pparams[name.to_s].to_sym
|
40
40
|
overridden_default_value = kwargs[:default]
|
41
41
|
|
42
|
-
CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).
|
42
|
+
CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do
|
43
43
|
config_set_default newer, overridden_default_value
|
44
44
|
end
|
45
45
|
end
|
46
46
|
end
|
47
47
|
|
48
48
|
def config_set_default(name, defval)
|
49
|
-
CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).
|
49
|
+
CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).bind_call(self, name, defval)
|
50
50
|
pparams = propagate_default_params
|
51
51
|
if pparams[name.to_s]
|
52
52
|
newer = pparams[name.to_s].to_sym
|
53
53
|
|
54
|
-
CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).
|
54
|
+
CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do
|
55
55
|
self.config_set_default newer, defval
|
56
56
|
end
|
57
57
|
end
|
@@ -14,14 +14,16 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
-
require 'console
|
17
|
+
require 'console'
|
18
18
|
|
19
19
|
module Fluent
|
20
20
|
class Log
|
21
21
|
# Async gem which is used by http_server helper switched logger mechanism to
|
22
22
|
# Console gem which isn't complatible with Ruby's standard Logger (since
|
23
23
|
# v1.17). This class adapts it to Fluentd's logger mechanism.
|
24
|
-
class ConsoleAdapter < Console::
|
24
|
+
class ConsoleAdapter < Gem::Version.new(Console::VERSION) >= Gem::Version.new("1.25") ?
|
25
|
+
Console::Output::Terminal : Console::Terminal::Logger
|
26
|
+
|
25
27
|
def self.wrap(logger)
|
26
28
|
_, level = Console::Logger::LEVELS.find { |key, value|
|
27
29
|
if logger.level <= 0
|
@@ -45,6 +45,8 @@ module Fluent::Plugin
|
|
45
45
|
config_param :run_interval, :time, default: nil
|
46
46
|
desc 'The default block size to read if parser requires partial read.'
|
47
47
|
config_param :read_block_size, :size, default: 10240 # 10k
|
48
|
+
desc 'The encoding to receive the result of the command, especially for non-ascii characters.'
|
49
|
+
config_param :encoding, :string, default: nil
|
48
50
|
|
49
51
|
attr_reader :parser
|
50
52
|
|
@@ -63,9 +65,16 @@ module Fluent::Plugin
|
|
63
65
|
if !@tag && (!@extract_config || !@extract_config.tag_key)
|
64
66
|
raise Fluent::ConfigError, "'tag' or 'tag_key' option is required on exec input"
|
65
67
|
end
|
68
|
+
validate_encoding(@encoding) if @encoding
|
66
69
|
@parser = parser_create
|
67
70
|
end
|
68
71
|
|
72
|
+
def validate_encoding(encoding)
|
73
|
+
Encoding.find(encoding)
|
74
|
+
rescue ArgumentError => e
|
75
|
+
raise Fluent::ConfigError, e.message
|
76
|
+
end
|
77
|
+
|
69
78
|
def multi_workers_ready?
|
70
79
|
true
|
71
80
|
end
|
@@ -73,10 +82,13 @@ module Fluent::Plugin
|
|
73
82
|
def start
|
74
83
|
super
|
75
84
|
|
85
|
+
options = { mode: [@connect_mode] }
|
86
|
+
options[:external_encoding] = @encoding if @encoding
|
87
|
+
|
76
88
|
if @run_interval
|
77
|
-
child_process_execute(:exec_input, @command, interval: @run_interval,
|
89
|
+
child_process_execute(:exec_input, @command, interval: @run_interval, **options, &method(:run))
|
78
90
|
else
|
79
|
-
child_process_execute(:exec_input, @command, immediate: true,
|
91
|
+
child_process_execute(:exec_input, @command, immediate: true, **options, &method(:run))
|
80
92
|
end
|
81
93
|
end
|
82
94
|
|
@@ -448,7 +448,7 @@ module Fluent::Plugin
|
|
448
448
|
# Azure App Service sends GET requests for health checking purpose.
|
449
449
|
# Respond with `200 OK` to accommodate it.
|
450
450
|
def handle_get_request
|
451
|
-
|
451
|
+
return send_response_and_close(RES_200_STATUS, {}, "")
|
452
452
|
end
|
453
453
|
|
454
454
|
# Web browsers can send an OPTIONS request before performing POST
|
@@ -39,6 +39,8 @@ module Fluent::Plugin
|
|
39
39
|
config_param :auto_increment_key, :string, default: nil
|
40
40
|
desc "The boolean to suspend-and-resume incremental value after restart"
|
41
41
|
config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
|
42
|
+
desc "Reuse the sample data to reduce the load when sending large amounts of data. You can enable it if filter does not do destructive change"
|
43
|
+
config_param :reuse_record, :bool, default: false
|
42
44
|
desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
|
43
45
|
config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
|
44
46
|
begin
|
@@ -117,15 +119,19 @@ module Fluent::Plugin
|
|
117
119
|
end
|
118
120
|
end
|
119
121
|
|
120
|
-
def
|
121
|
-
d = @sample[@sample_index]
|
122
|
-
unless d
|
123
|
-
@sample_index = 0
|
124
|
-
d = @sample[@sample_index]
|
125
|
-
end
|
122
|
+
def next_sample
|
123
|
+
d = @reuse_record ? @sample[@sample_index] : @sample[@sample_index].dup
|
126
124
|
@sample_index += 1
|
125
|
+
return d if d
|
126
|
+
|
127
|
+
@sample_index = 0
|
128
|
+
next_sample
|
129
|
+
end
|
130
|
+
|
131
|
+
def generate
|
132
|
+
d = next_sample
|
127
133
|
if @auto_increment_key
|
128
|
-
d = d.dup
|
134
|
+
d = d.dup if @reuse_record
|
129
135
|
d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
|
130
136
|
end
|
131
137
|
d
|
@@ -36,7 +36,7 @@ module Fluent::Plugin
|
|
36
36
|
helpers :timer, :event_loop, :parser, :compat_parameters
|
37
37
|
|
38
38
|
RESERVED_CHARS = ['/', '*', '%'].freeze
|
39
|
-
MetricsInfo = Struct.new(:opened, :closed, :rotated)
|
39
|
+
MetricsInfo = Struct.new(:opened, :closed, :rotated, :throttled)
|
40
40
|
|
41
41
|
class WatcherSetupError < StandardError
|
42
42
|
def initialize(msg)
|
@@ -208,7 +208,8 @@ module Fluent::Plugin
|
|
208
208
|
opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
|
209
209
|
closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
|
210
210
|
rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
|
211
|
-
|
211
|
+
throttling_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_throttled_total", help_text: "Total number of times throttling occurs per file when throttling enabled")
|
212
|
+
@metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics, throttling_metrics)
|
212
213
|
end
|
213
214
|
|
214
215
|
def configure_tag
|
@@ -566,7 +567,7 @@ module Fluent::Plugin
|
|
566
567
|
if @follow_inodes && new_inode.nil?
|
567
568
|
# nil inode means the file disappeared, so we only need to stop it.
|
568
569
|
@tails.delete(tail_watcher.path)
|
569
|
-
# https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
|
570
|
+
# https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632
|
570
571
|
# Because of this problem, log duplication can occur during `rotate_wait`.
|
571
572
|
# Need to set `rotate_wait 0` for a workaround.
|
572
573
|
# Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.
|
@@ -696,14 +697,6 @@ module Fluent::Plugin
|
|
696
697
|
|
697
698
|
# @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
|
698
699
|
def receive_lines(lines, tail_watcher)
|
699
|
-
lines = lines.reject do |line|
|
700
|
-
skip_line = @max_line_size ? line.bytesize > @max_line_size : false
|
701
|
-
if skip_line
|
702
|
-
log.warn "received line length is longer than #{@max_line_size}"
|
703
|
-
log.debug "skipped line: #{line.chomp}"
|
704
|
-
end
|
705
|
-
skip_line
|
706
|
-
end
|
707
700
|
es = @receive_handler.call(lines, tail_watcher)
|
708
701
|
unless es.empty?
|
709
702
|
tag = if @tag_prefix || @tag_suffix
|
@@ -801,6 +794,7 @@ module Fluent::Plugin
|
|
801
794
|
'opened_file_count' => @metrics.opened.get,
|
802
795
|
'closed_file_count' => @metrics.closed.get,
|
803
796
|
'rotated_file_count' => @metrics.rotated.get,
|
797
|
+
'throttled_log_count' => @metrics.throttled.get,
|
804
798
|
})
|
805
799
|
}
|
806
800
|
stats
|
@@ -819,6 +813,7 @@ module Fluent::Plugin
|
|
819
813
|
from_encoding: @from_encoding,
|
820
814
|
encoding: @encoding,
|
821
815
|
metrics: @metrics,
|
816
|
+
max_line_size: @max_line_size,
|
822
817
|
&method(:receive_lines)
|
823
818
|
)
|
824
819
|
end
|
@@ -1011,15 +1006,19 @@ module Fluent::Plugin
|
|
1011
1006
|
end
|
1012
1007
|
|
1013
1008
|
class FIFO
|
1014
|
-
def initialize(from_encoding, encoding)
|
1009
|
+
def initialize(from_encoding, encoding, log, max_line_size=nil)
|
1015
1010
|
@from_encoding = from_encoding
|
1016
1011
|
@encoding = encoding
|
1017
1012
|
@need_enc = from_encoding != encoding
|
1018
1013
|
@buffer = ''.force_encoding(from_encoding)
|
1019
1014
|
@eol = "\n".encode(from_encoding).freeze
|
1015
|
+
@max_line_size = max_line_size
|
1016
|
+
@skip_current_line = false
|
1017
|
+
@skipping_current_line_bytesize = 0
|
1018
|
+
@log = log
|
1020
1019
|
end
|
1021
1020
|
|
1022
|
-
attr_reader :from_encoding, :encoding, :buffer
|
1021
|
+
attr_reader :from_encoding, :encoding, :buffer, :max_line_size
|
1023
1022
|
|
1024
1023
|
def <<(chunk)
|
1025
1024
|
# Although "chunk" is most likely transient besides String#force_encoding itself
|
@@ -1051,6 +1050,7 @@ module Fluent::Plugin
|
|
1051
1050
|
|
1052
1051
|
def read_lines(lines)
|
1053
1052
|
idx = @buffer.index(@eol)
|
1053
|
+
has_skipped_line = false
|
1054
1054
|
|
1055
1055
|
until idx.nil?
|
1056
1056
|
# Using freeze and slice is faster than slice!
|
@@ -1059,11 +1059,47 @@ module Fluent::Plugin
|
|
1059
1059
|
rbuf = @buffer.slice(0, idx + 1)
|
1060
1060
|
@buffer = @buffer.slice(idx + 1, @buffer.size)
|
1061
1061
|
idx = @buffer.index(@eol)
|
1062
|
+
|
1063
|
+
is_long_line = @max_line_size && (
|
1064
|
+
@skip_current_line || rbuf.bytesize > @max_line_size
|
1065
|
+
)
|
1066
|
+
|
1067
|
+
if is_long_line
|
1068
|
+
@log.warn "received line length is longer than #{@max_line_size}"
|
1069
|
+
if @skip_current_line
|
1070
|
+
@log.debug("The continuing line is finished. Finally discarded data: ") { convert(rbuf).chomp }
|
1071
|
+
else
|
1072
|
+
@log.debug("skipped line: ") { convert(rbuf).chomp }
|
1073
|
+
end
|
1074
|
+
has_skipped_line = true
|
1075
|
+
@skip_current_line = false
|
1076
|
+
@skipping_current_line_bytesize = 0
|
1077
|
+
next
|
1078
|
+
end
|
1079
|
+
|
1062
1080
|
lines << convert(rbuf)
|
1063
1081
|
end
|
1082
|
+
|
1083
|
+
is_long_current_line = @max_line_size && (
|
1084
|
+
@skip_current_line || @buffer.bytesize > @max_line_size
|
1085
|
+
)
|
1086
|
+
|
1087
|
+
if is_long_current_line
|
1088
|
+
@log.debug(
|
1089
|
+
"The continuing current line length is longer than #{@max_line_size}." +
|
1090
|
+
" The received data will be discarded until this line is finished." +
|
1091
|
+
" Discarded data: "
|
1092
|
+
) { convert(@buffer).chomp }
|
1093
|
+
@skip_current_line = true
|
1094
|
+
@skipping_current_line_bytesize += @buffer.bytesize
|
1095
|
+
@buffer.clear
|
1096
|
+
end
|
1097
|
+
|
1098
|
+
return has_skipped_line
|
1064
1099
|
end
|
1065
1100
|
|
1066
|
-
def
|
1101
|
+
def reading_bytesize
|
1102
|
+
return @skipping_current_line_bytesize if @skip_current_line
|
1067
1103
|
@buffer.bytesize
|
1068
1104
|
end
|
1069
1105
|
end
|
@@ -1074,14 +1110,14 @@ module Fluent::Plugin
|
|
1074
1110
|
|
1075
1111
|
attr_accessor :shutdown_timeout
|
1076
1112
|
|
1077
|
-
def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
|
1113
|
+
def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, max_line_size: nil, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
|
1078
1114
|
@watcher = watcher
|
1079
1115
|
@path = path
|
1080
1116
|
@read_lines_limit = read_lines_limit
|
1081
1117
|
@read_bytes_limit_per_second = read_bytes_limit_per_second
|
1082
1118
|
@receive_lines = receive_lines
|
1083
1119
|
@open_on_every_update = open_on_every_update
|
1084
|
-
@fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT)
|
1120
|
+
@fifo = FIFO.new(from_encoding || Encoding::ASCII_8BIT, encoding || Encoding::ASCII_8BIT, log, max_line_size)
|
1085
1121
|
@iobuf = ''.force_encoding('ASCII-8BIT')
|
1086
1122
|
@lines = []
|
1087
1123
|
@io = nil
|
@@ -1158,12 +1194,15 @@ module Fluent::Plugin
|
|
1158
1194
|
end
|
1159
1195
|
|
1160
1196
|
def handle_notify
|
1161
|
-
|
1162
|
-
|
1197
|
+
if limit_bytes_per_second_reached? || group_watcher&.limit_lines_reached?(@path)
|
1198
|
+
@metrics.throttled.inc
|
1199
|
+
return
|
1200
|
+
end
|
1163
1201
|
|
1164
1202
|
with_io do |io|
|
1165
1203
|
begin
|
1166
1204
|
read_more = false
|
1205
|
+
has_skipped_line = false
|
1167
1206
|
|
1168
1207
|
if !io.nil? && @lines.empty?
|
1169
1208
|
begin
|
@@ -1177,7 +1216,7 @@ module Fluent::Plugin
|
|
1177
1216
|
@fifo << data
|
1178
1217
|
|
1179
1218
|
n_lines_before_read = @lines.size
|
1180
|
-
@fifo.read_lines(@lines)
|
1219
|
+
has_skipped_line = @fifo.read_lines(@lines) || has_skipped_line
|
1181
1220
|
group_watcher&.update_lines_read(@path, @lines.size - n_lines_before_read)
|
1182
1221
|
|
1183
1222
|
group_watcher_limit = group_watcher&.limit_lines_reached?(@path)
|
@@ -1185,6 +1224,7 @@ module Fluent::Plugin
|
|
1185
1224
|
|
1186
1225
|
if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now?
|
1187
1226
|
# Just get out from tailing loop.
|
1227
|
+
@metrics.throttled.inc if group_watcher_limit || limit_bytes_per_second_reached?
|
1188
1228
|
read_more = false
|
1189
1229
|
break
|
1190
1230
|
end
|
@@ -1200,9 +1240,11 @@ module Fluent::Plugin
|
|
1200
1240
|
end
|
1201
1241
|
end
|
1202
1242
|
|
1203
|
-
|
1243
|
+
if @lines.empty?
|
1244
|
+
@watcher.pe.update_pos(io.pos - @fifo.reading_bytesize) if has_skipped_line
|
1245
|
+
else
|
1204
1246
|
if @receive_lines.call(@lines, @watcher)
|
1205
|
-
@watcher.pe.update_pos(io.pos - @fifo.
|
1247
|
+
@watcher.pe.update_pos(io.pos - @fifo.reading_bytesize)
|
1206
1248
|
@lines.clear
|
1207
1249
|
else
|
1208
1250
|
read_more = false
|
@@ -1214,12 +1256,12 @@ module Fluent::Plugin
|
|
1214
1256
|
|
1215
1257
|
def open
|
1216
1258
|
io = Fluent::FileWrapper.open(@path)
|
1217
|
-
io.seek(@watcher.pe.read_pos + @fifo.
|
1259
|
+
io.seek(@watcher.pe.read_pos + @fifo.reading_bytesize)
|
1218
1260
|
@metrics.opened.inc
|
1219
1261
|
io
|
1220
1262
|
rescue RangeError
|
1221
1263
|
io.close if io
|
1222
|
-
raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.
|
1264
|
+
raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.reading_bytesize.to_s(16)}"
|
1223
1265
|
rescue Errno::EACCES => e
|
1224
1266
|
@log.warn "#{e}"
|
1225
1267
|
nil
|
@@ -172,6 +172,14 @@ module Fluent::Plugin
|
|
172
172
|
log.warn "symlink_path is unavailable on Windows platform. disabled."
|
173
173
|
@symlink_path = nil
|
174
174
|
else
|
175
|
+
placeholder_validators(:symlink_path, @symlink_path).reject{ |v| v.type == :time }.each do |v|
|
176
|
+
begin
|
177
|
+
v.validate!
|
178
|
+
rescue Fluent::ConfigError => e
|
179
|
+
log.warn "#{e}. This means multiple chunks are competing for a single symlink_path, so some logs may not be taken from the symlink."
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
175
183
|
@buffer.extend SymlinkBufferMixin
|
176
184
|
@buffer.symlink_path = @symlink_path
|
177
185
|
@buffer.output_plugin_for_symlink = self
|
@@ -55,6 +55,8 @@ module Fluent::Plugin
|
|
55
55
|
config_param :headers, :hash, default: nil
|
56
56
|
desc 'Additional placeholder based headers for HTTP request'
|
57
57
|
config_param :headers_from_placeholders, :hash, default: nil
|
58
|
+
desc 'Compress HTTP request body'
|
59
|
+
config_param :compress, :enum, list: [:text, :gzip], default: :text
|
58
60
|
|
59
61
|
desc 'The connection open timeout in seconds'
|
60
62
|
config_param :open_timeout, :integer, default: nil
|
@@ -290,6 +292,9 @@ module Fluent::Plugin
|
|
290
292
|
req[k] = extract_placeholders(v, chunk)
|
291
293
|
end
|
292
294
|
end
|
295
|
+
if @compress == :gzip
|
296
|
+
req['Content-Encoding'] = "gzip"
|
297
|
+
end
|
293
298
|
req['Content-Type'] = @content_type
|
294
299
|
end
|
295
300
|
|
@@ -323,8 +328,15 @@ module Fluent::Plugin
|
|
323
328
|
Net::HTTP::Put.new(uri.request_uri)
|
324
329
|
end
|
325
330
|
set_headers(req, uri, chunk)
|
331
|
+
|
326
332
|
req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
|
327
333
|
|
334
|
+
if @compress == :gzip
|
335
|
+
gz = Zlib::GzipWriter.new(StringIO.new)
|
336
|
+
gz << req.body
|
337
|
+
req.body = gz.close.string
|
338
|
+
end
|
339
|
+
|
328
340
|
# At least one authentication method requires the body and other headers, so the order of this call matters
|
329
341
|
set_auth(req, uri)
|
330
342
|
req
|
@@ -50,23 +50,15 @@ module Fluent
|
|
50
50
|
def configure_json_parser(name)
|
51
51
|
case name
|
52
52
|
when :oj
|
53
|
-
|
54
|
-
|
53
|
+
return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?
|
54
|
+
|
55
|
+
log&.info "Oj is not installed, and failing back to Yajl for json parser"
|
56
|
+
configure_json_parser(:yajl)
|
55
57
|
when :json then [JSON.method(:load), JSON::ParserError]
|
56
58
|
when :yajl then [Yajl.method(:load), Yajl::ParseError]
|
57
59
|
else
|
58
60
|
raise "BUG: unknown json parser specified: #{name}"
|
59
61
|
end
|
60
|
-
rescue LoadError => ex
|
61
|
-
name = :yajl
|
62
|
-
if log
|
63
|
-
if /\boj\z/.match?(ex.message)
|
64
|
-
log.info "Oj is not installed, and failing back to Yajl for json parser"
|
65
|
-
else
|
66
|
-
log.warn ex.message
|
67
|
-
end
|
68
|
-
end
|
69
|
-
retry
|
70
62
|
end
|
71
63
|
|
72
64
|
def parse(text)
|
data/lib/fluent/version.rb
CHANGED
@@ -12,12 +12,13 @@ Gem::Specification.new do |spec|
|
|
12
12
|
spec.homepage = "TODO: Put your gem's website or public repo URL here."
|
13
13
|
spec.license = "<%= @license.name %>"
|
14
14
|
|
15
|
-
|
16
|
-
|
15
|
+
spec.files = Dir.chdir(__dir__) do
|
16
|
+
`git ls-files -z`.split("\x0").reject do |f|
|
17
|
+
(File.expand_path(f) == __FILE__) ||
|
18
|
+
f.start_with?(*%w[test/ spec/ features/ .git .circleci appveyor Gemfile])
|
19
|
+
end
|
17
20
|
end
|
18
|
-
spec.
|
19
|
-
spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
|
20
|
-
spec.test_files = test_files
|
21
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
21
22
|
spec.require_paths = ["lib"]
|
22
23
|
|
23
24
|
spec.add_development_dependency "bundler", "~> <%= bundler_version %>"
|