fluentd 1.16.5 → 1.17.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +88 -0
- data/README.md +2 -1
- data/Rakefile +1 -1
- data/SECURITY.md +2 -2
- data/fluent.conf +14 -14
- data/lib/fluent/command/binlog_reader.rb +1 -1
- data/lib/fluent/command/cap_ctl.rb +4 -4
- data/lib/fluent/compat/call_super_mixin.rb +3 -3
- data/lib/fluent/compat/propagate_default.rb +4 -4
- data/lib/fluent/config/configure_proxy.rb +2 -2
- data/lib/fluent/config/types.rb +1 -1
- data/lib/fluent/config/yaml_parser/parser.rb +4 -0
- data/lib/fluent/configurable.rb +2 -2
- data/lib/fluent/counter/mutex_hash.rb +1 -1
- data/lib/fluent/fluent_log_event_router.rb +0 -2
- data/lib/fluent/log/console_adapter.rb +4 -2
- data/lib/fluent/plugin/buf_file.rb +1 -1
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -3
- data/lib/fluent/plugin/filter_parser.rb +26 -8
- data/lib/fluent/plugin/in_exec.rb +14 -2
- data/lib/fluent/plugin/in_http.rb +19 -54
- data/lib/fluent/plugin/in_sample.rb +13 -7
- data/lib/fluent/plugin/in_tail.rb +99 -25
- data/lib/fluent/plugin/out_copy.rb +1 -1
- data/lib/fluent/plugin/out_file.rb +8 -0
- data/lib/fluent/plugin/out_http.rb +137 -13
- data/lib/fluent/plugin/owned_by_mixin.rb +0 -1
- data/lib/fluent/plugin/parser_json.rb +26 -17
- data/lib/fluent/plugin/parser_msgpack.rb +24 -3
- data/lib/fluent/plugin_helper/http_server/server.rb +1 -1
- data/lib/fluent/plugin_helper/metrics.rb +2 -2
- data/lib/fluent/registry.rb +6 -6
- data/lib/fluent/test/output_test.rb +1 -1
- data/lib/fluent/unique_id.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/fluent-plugin.gemspec.erb +6 -5
- metadata +109 -459
- data/.github/ISSUE_TEMPLATE/bug_report.yml +0 -71
- data/.github/ISSUE_TEMPLATE/config.yml +0 -5
- data/.github/ISSUE_TEMPLATE/feature_request.yml +0 -39
- data/.github/ISSUE_TEMPLATE.md +0 -17
- data/.github/PULL_REQUEST_TEMPLATE.md +0 -14
- data/.github/workflows/stale-actions.yml +0 -24
- data/.github/workflows/test.yml +0 -32
- data/.gitignore +0 -30
- data/Gemfile +0 -9
- data/fluentd.gemspec +0 -54
- data/test/command/test_binlog_reader.rb +0 -362
- data/test/command/test_ca_generate.rb +0 -70
- data/test/command/test_cap_ctl.rb +0 -100
- data/test/command/test_cat.rb +0 -128
- data/test/command/test_ctl.rb +0 -56
- data/test/command/test_fluentd.rb +0 -1291
- data/test/command/test_plugin_config_formatter.rb +0 -397
- data/test/command/test_plugin_generator.rb +0 -109
- data/test/compat/test_calls_super.rb +0 -166
- data/test/compat/test_parser.rb +0 -92
- data/test/config/assertions.rb +0 -42
- data/test/config/test_config_parser.rb +0 -551
- data/test/config/test_configurable.rb +0 -1784
- data/test/config/test_configure_proxy.rb +0 -604
- data/test/config/test_dsl.rb +0 -415
- data/test/config/test_element.rb +0 -518
- data/test/config/test_literal_parser.rb +0 -309
- data/test/config/test_plugin_configuration.rb +0 -56
- data/test/config/test_section.rb +0 -191
- data/test/config/test_system_config.rb +0 -195
- data/test/config/test_types.rb +0 -408
- data/test/counter/test_client.rb +0 -563
- data/test/counter/test_error.rb +0 -44
- data/test/counter/test_mutex_hash.rb +0 -179
- data/test/counter/test_server.rb +0 -589
- data/test/counter/test_store.rb +0 -258
- data/test/counter/test_validator.rb +0 -137
- data/test/helper.rb +0 -155
- data/test/helpers/fuzzy_assert.rb +0 -89
- data/test/helpers/process_extenstion.rb +0 -33
- data/test/log/test_console_adapter.rb +0 -110
- data/test/plugin/data/2010/01/20100102-030405.log +0 -0
- data/test/plugin/data/2010/01/20100102-030406.log +0 -0
- data/test/plugin/data/2010/01/20100102.log +0 -0
- data/test/plugin/data/log/bar +0 -0
- data/test/plugin/data/log/foo/bar.log +0 -0
- data/test/plugin/data/log/foo/bar2 +0 -0
- data/test/plugin/data/log/test.log +0 -0
- data/test/plugin/data/sd_file/config +0 -11
- data/test/plugin/data/sd_file/config.json +0 -17
- data/test/plugin/data/sd_file/config.yaml +0 -11
- data/test/plugin/data/sd_file/config.yml +0 -11
- data/test/plugin/data/sd_file/invalid_config.yml +0 -7
- data/test/plugin/in_tail/test_fifo.rb +0 -121
- data/test/plugin/in_tail/test_io_handler.rb +0 -150
- data/test/plugin/in_tail/test_position_file.rb +0 -346
- data/test/plugin/out_forward/test_ack_handler.rb +0 -140
- data/test/plugin/out_forward/test_connection_manager.rb +0 -145
- data/test/plugin/out_forward/test_handshake_protocol.rb +0 -112
- data/test/plugin/out_forward/test_load_balancer.rb +0 -106
- data/test/plugin/out_forward/test_socket_cache.rb +0 -174
- data/test/plugin/test_bare_output.rb +0 -131
- data/test/plugin/test_base.rb +0 -247
- data/test/plugin/test_buf_file.rb +0 -1314
- data/test/plugin/test_buf_file_single.rb +0 -898
- data/test/plugin/test_buf_memory.rb +0 -42
- data/test/plugin/test_buffer.rb +0 -1493
- data/test/plugin/test_buffer_chunk.rb +0 -209
- data/test/plugin/test_buffer_file_chunk.rb +0 -871
- data/test/plugin/test_buffer_file_single_chunk.rb +0 -611
- data/test/plugin/test_buffer_memory_chunk.rb +0 -339
- data/test/plugin/test_compressable.rb +0 -87
- data/test/plugin/test_file_util.rb +0 -96
- data/test/plugin/test_filter.rb +0 -368
- data/test/plugin/test_filter_grep.rb +0 -697
- data/test/plugin/test_filter_parser.rb +0 -731
- data/test/plugin/test_filter_record_transformer.rb +0 -577
- data/test/plugin/test_filter_stdout.rb +0 -207
- data/test/plugin/test_formatter_csv.rb +0 -136
- data/test/plugin/test_formatter_hash.rb +0 -38
- data/test/plugin/test_formatter_json.rb +0 -61
- data/test/plugin/test_formatter_ltsv.rb +0 -70
- data/test/plugin/test_formatter_msgpack.rb +0 -28
- data/test/plugin/test_formatter_out_file.rb +0 -116
- data/test/plugin/test_formatter_single_value.rb +0 -44
- data/test/plugin/test_formatter_tsv.rb +0 -76
- data/test/plugin/test_in_debug_agent.rb +0 -49
- data/test/plugin/test_in_exec.rb +0 -261
- data/test/plugin/test_in_forward.rb +0 -1178
- data/test/plugin/test_in_gc_stat.rb +0 -62
- data/test/plugin/test_in_http.rb +0 -1102
- data/test/plugin/test_in_monitor_agent.rb +0 -922
- data/test/plugin/test_in_object_space.rb +0 -66
- data/test/plugin/test_in_sample.rb +0 -190
- data/test/plugin/test_in_syslog.rb +0 -505
- data/test/plugin/test_in_tail.rb +0 -3288
- data/test/plugin/test_in_tcp.rb +0 -328
- data/test/plugin/test_in_udp.rb +0 -296
- data/test/plugin/test_in_unix.rb +0 -181
- data/test/plugin/test_input.rb +0 -137
- data/test/plugin/test_metadata.rb +0 -89
- data/test/plugin/test_metrics.rb +0 -294
- data/test/plugin/test_metrics_local.rb +0 -96
- data/test/plugin/test_multi_output.rb +0 -204
- data/test/plugin/test_out_copy.rb +0 -308
- data/test/plugin/test_out_exec.rb +0 -312
- data/test/plugin/test_out_exec_filter.rb +0 -606
- data/test/plugin/test_out_file.rb +0 -1038
- data/test/plugin/test_out_forward.rb +0 -1349
- data/test/plugin/test_out_http.rb +0 -429
- data/test/plugin/test_out_null.rb +0 -105
- data/test/plugin/test_out_relabel.rb +0 -28
- data/test/plugin/test_out_roundrobin.rb +0 -146
- data/test/plugin/test_out_secondary_file.rb +0 -458
- data/test/plugin/test_out_stdout.rb +0 -205
- data/test/plugin/test_out_stream.rb +0 -103
- data/test/plugin/test_output.rb +0 -1334
- data/test/plugin/test_output_as_buffered.rb +0 -2024
- data/test/plugin/test_output_as_buffered_backup.rb +0 -363
- data/test/plugin/test_output_as_buffered_compress.rb +0 -179
- data/test/plugin/test_output_as_buffered_overflow.rb +0 -250
- data/test/plugin/test_output_as_buffered_retries.rb +0 -966
- data/test/plugin/test_output_as_buffered_secondary.rb +0 -882
- data/test/plugin/test_output_as_standard.rb +0 -374
- data/test/plugin/test_owned_by.rb +0 -35
- data/test/plugin/test_parser.rb +0 -399
- data/test/plugin/test_parser_apache.rb +0 -42
- data/test/plugin/test_parser_apache2.rb +0 -47
- data/test/plugin/test_parser_apache_error.rb +0 -45
- data/test/plugin/test_parser_csv.rb +0 -200
- data/test/plugin/test_parser_json.rb +0 -138
- data/test/plugin/test_parser_labeled_tsv.rb +0 -160
- data/test/plugin/test_parser_multiline.rb +0 -111
- data/test/plugin/test_parser_nginx.rb +0 -88
- data/test/plugin/test_parser_none.rb +0 -52
- data/test/plugin/test_parser_regexp.rb +0 -284
- data/test/plugin/test_parser_syslog.rb +0 -650
- data/test/plugin/test_parser_tsv.rb +0 -122
- data/test/plugin/test_sd_file.rb +0 -228
- data/test/plugin/test_sd_srv.rb +0 -230
- data/test/plugin/test_storage.rb +0 -167
- data/test/plugin/test_storage_local.rb +0 -335
- data/test/plugin/test_string_util.rb +0 -26
- data/test/plugin_helper/data/cert/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert-with-CRLF.pem +0 -19
- data/test/plugin_helper/data/cert/cert-with-no-newline.pem +0 -19
- data/test/plugin_helper/data/cert/cert.pem +0 -19
- data/test/plugin_helper/data/cert/cert_chains/ca-cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert_chains/ca-cert.pem +0 -20
- data/test/plugin_helper/data/cert/cert_chains/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/cert_chains/cert.pem +0 -40
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/data/cert/generate_cert.rb +0 -125
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/with_ca/ca-cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/with_ca/ca-cert-pass.pem +0 -20
- data/test/plugin_helper/data/cert/with_ca/ca-cert.pem +0 -20
- data/test/plugin_helper/data/cert/with_ca/cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/with_ca/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/with_ca/cert-pass.pem +0 -21
- data/test/plugin_helper/data/cert/with_ca/cert.pem +0 -21
- data/test/plugin_helper/data/cert/without_ca/cert-key-pass.pem +0 -30
- data/test/plugin_helper/data/cert/without_ca/cert-key.pem +0 -27
- data/test/plugin_helper/data/cert/without_ca/cert-pass.pem +0 -20
- data/test/plugin_helper/data/cert/without_ca/cert.pem +0 -20
- data/test/plugin_helper/http_server/test_app.rb +0 -65
- data/test/plugin_helper/http_server/test_route.rb +0 -32
- data/test/plugin_helper/service_discovery/test_manager.rb +0 -93
- data/test/plugin_helper/service_discovery/test_round_robin_balancer.rb +0 -21
- data/test/plugin_helper/test_cert_option.rb +0 -25
- data/test/plugin_helper/test_child_process.rb +0 -862
- data/test/plugin_helper/test_compat_parameters.rb +0 -358
- data/test/plugin_helper/test_event_emitter.rb +0 -80
- data/test/plugin_helper/test_event_loop.rb +0 -52
- data/test/plugin_helper/test_extract.rb +0 -194
- data/test/plugin_helper/test_formatter.rb +0 -255
- data/test/plugin_helper/test_http_server_helper.rb +0 -372
- data/test/plugin_helper/test_inject.rb +0 -561
- data/test/plugin_helper/test_metrics.rb +0 -137
- data/test/plugin_helper/test_parser.rb +0 -264
- data/test/plugin_helper/test_record_accessor.rb +0 -238
- data/test/plugin_helper/test_retry_state.rb +0 -1006
- data/test/plugin_helper/test_server.rb +0 -1895
- data/test/plugin_helper/test_service_discovery.rb +0 -165
- data/test/plugin_helper/test_socket.rb +0 -146
- data/test/plugin_helper/test_storage.rb +0 -542
- data/test/plugin_helper/test_thread.rb +0 -164
- data/test/plugin_helper/test_timer.rb +0 -130
- data/test/scripts/exec_script.rb +0 -32
- data/test/scripts/fluent/plugin/formatter1/formatter_test1.rb +0 -7
- data/test/scripts/fluent/plugin/formatter2/formatter_test2.rb +0 -7
- data/test/scripts/fluent/plugin/formatter_known.rb +0 -8
- data/test/scripts/fluent/plugin/out_test.rb +0 -81
- data/test/scripts/fluent/plugin/out_test2.rb +0 -80
- data/test/scripts/fluent/plugin/parser_known.rb +0 -4
- data/test/test_capability.rb +0 -74
- data/test/test_clock.rb +0 -164
- data/test/test_config.rb +0 -369
- data/test/test_configdsl.rb +0 -148
- data/test/test_daemonizer.rb +0 -91
- data/test/test_engine.rb +0 -203
- data/test/test_event.rb +0 -531
- data/test/test_event_router.rb +0 -348
- data/test/test_event_time.rb +0 -199
- data/test/test_file_wrapper.rb +0 -53
- data/test/test_filter.rb +0 -121
- data/test/test_fluent_log_event_router.rb +0 -99
- data/test/test_formatter.rb +0 -369
- data/test/test_input.rb +0 -31
- data/test/test_log.rb +0 -1076
- data/test/test_match.rb +0 -148
- data/test/test_mixin.rb +0 -351
- data/test/test_msgpack_factory.rb +0 -50
- data/test/test_oj_options.rb +0 -55
- data/test/test_output.rb +0 -278
- data/test/test_plugin.rb +0 -251
- data/test/test_plugin_classes.rb +0 -370
- data/test/test_plugin_helper.rb +0 -81
- data/test/test_plugin_id.rb +0 -119
- data/test/test_process.rb +0 -14
- data/test/test_root_agent.rb +0 -951
- data/test/test_static_config_analysis.rb +0 -177
- data/test/test_supervisor.rb +0 -821
- data/test/test_test_drivers.rb +0 -136
- data/test/test_time_formatter.rb +0 -301
- data/test/test_time_parser.rb +0 -362
- data/test/test_tls.rb +0 -65
- data/test/test_unique_id.rb +0 -47
- data/test/test_variable_store.rb +0 -65
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4921c240e6dc91b8104f572134c47656f30172a0336bf8fc499b583f99ca316e
|
4
|
+
data.tar.gz: fff090306c0ec88d17c36732f9c074afdf40c30e00e8012abfc70f7a7b2888dd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ff35533952604174b264e9c48fbe953d880770870865993c14f89ffe730f0c09272b408d62e02779f999ab73d8fa02027aa60f14cf523f96cc87924937e0ce64
|
7
|
+
data.tar.gz: c1d9f699c770f8c522176bd365700284d63c2f4cf4859925cc730b4b27b04396eca649432494085d2891863a9232ee21f7f97250b7816d0da211b2067d8d3ff7
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,91 @@
|
|
1
|
+
# v1.17
|
2
|
+
|
3
|
+
## Release v1.17.1 - 2024/08/19
|
4
|
+
|
5
|
+
### Enhancement
|
6
|
+
|
7
|
+
* yaml_parser: Support $log_level element
|
8
|
+
https://github.com/fluent/fluentd/pull/4482
|
9
|
+
* out_file: Add warn message for symlink_path setting
|
10
|
+
https://github.com/fluent/fluentd/pull/4502
|
11
|
+
* out_http: Add `compress gzip` option
|
12
|
+
https://github.com/fluent/fluentd/pull/4528
|
13
|
+
* in_exec: Add `encoding` option to handle non-ascii characters
|
14
|
+
https://github.com/fluent/fluentd/pull/4533
|
15
|
+
* in_tail: Add throttling metrics
|
16
|
+
https://github.com/fluent/fluentd/pull/4578
|
17
|
+
* compat: Improve method call performance
|
18
|
+
https://github.com/fluent/fluentd/pull/4588
|
19
|
+
* in_sample: Add `reuse_record` parameter to reuse the sample data
|
20
|
+
https://github.com/fluent/fluentd/pull/4586
|
21
|
+
* `in_sample` has changed to copy sample data by default to avoid the impact of destructive changes by subsequent plugins.
|
22
|
+
* This increases the load when generating large amounts of sample data.
|
23
|
+
* You can use this new parameter to have the same performance as before.
|
24
|
+
|
25
|
+
### Bug Fix
|
26
|
+
|
27
|
+
* logger: Fix LoadError with console gem v1.25
|
28
|
+
https://github.com/fluent/fluentd/pull/4492
|
29
|
+
* parser_json: Fix wrong LoadError warning
|
30
|
+
https://github.com/fluent/fluentd/pull/4522
|
31
|
+
* in_tail: Fix an issue where a large single line could consume a large amount of memory even though `max_line_size` is set
|
32
|
+
https://github.com/fluent/fluentd/pull/4530
|
33
|
+
|
34
|
+
### Misc
|
35
|
+
|
36
|
+
* Comment out inappropriate default configuration about out_forward
|
37
|
+
https://github.com/fluent/fluentd/pull/4523
|
38
|
+
* gemspec: Remove unnecessary files from released gem
|
39
|
+
https://github.com/fluent/fluentd/pull/4534
|
40
|
+
* plugin-generator: Update gemspec to remove unnecessary files
|
41
|
+
https://github.com/fluent/fluentd/pull/4535
|
42
|
+
* Suppress non-parenthesis warnings
|
43
|
+
https://github.com/fluent/fluentd/pull/4594
|
44
|
+
* Fix FrozenError in http_server plugin helper
|
45
|
+
https://github.com/fluent/fluentd/pull/4598
|
46
|
+
* Add logger gem dependency for Ruby 3.5
|
47
|
+
https://github.com/fluent/fluentd/pull/4589
|
48
|
+
|
49
|
+
## Release v1.17.0 - 2024/04/30
|
50
|
+
|
51
|
+
### Enhancement
|
52
|
+
|
53
|
+
* in_http: Recognize CSP reports as JSON data
|
54
|
+
https://github.com/fluent/fluentd/pull/4282
|
55
|
+
* out_http: Add option to reuse connections
|
56
|
+
https://github.com/fluent/fluentd/pull/4330
|
57
|
+
* in_tail: Expand glob capability for square brackets and one character matcher
|
58
|
+
https://github.com/fluent/fluentd/pull/4401
|
59
|
+
* out_http: Support AWS Signature Version 4 authentication
|
60
|
+
https://github.com/fluent/fluentd/pull/4459
|
61
|
+
|
62
|
+
### Bug Fix
|
63
|
+
|
64
|
+
* Make sure `parser_json` and `parser_msgpack` return `Hash`.
|
65
|
+
Make `parser_json` and `parser_msgpack` accept only `Hash` or `Array` of `Hash`.
|
66
|
+
https://github.com/fluent/fluentd/pull/4474
|
67
|
+
* filter_parser: Add error event for multiple parsed results
|
68
|
+
https://github.com/fluent/fluentd/pull/4478
|
69
|
+
|
70
|
+
### Misc
|
71
|
+
|
72
|
+
* Raise minimum required ruby version
|
73
|
+
https://github.com/fluent/fluentd/pull/4288
|
74
|
+
* Require missing dependent gems as of Ruby 3.4-dev
|
75
|
+
https://github.com/fluent/fluentd/pull/4411
|
76
|
+
* Minor code refactoring
|
77
|
+
https://github.com/fluent/fluentd/pull/4294
|
78
|
+
https://github.com/fluent/fluentd/pull/4299
|
79
|
+
https://github.com/fluent/fluentd/pull/4302
|
80
|
+
https://github.com/fluent/fluentd/pull/4320
|
81
|
+
* CI fixes
|
82
|
+
https://github.com/fluent/fluentd/pull/4369
|
83
|
+
https://github.com/fluent/fluentd/pull/4433
|
84
|
+
https://github.com/fluent/fluentd/pull/4452
|
85
|
+
https://github.com/fluent/fluentd/pull/4477
|
86
|
+
* github: unify YAML file extension to .yml
|
87
|
+
https://github.com/fluent/fluentd/pull/4429
|
88
|
+
|
1
89
|
# v1.16
|
2
90
|
|
3
91
|
## Release v1.16.5 - 2024/03/27
|
data/README.md
CHANGED
@@ -6,6 +6,7 @@ Fluentd: Open-Source Log Collector
|
|
6
6
|
[![Testing on macOS](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
|
7
7
|
[![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
|
8
8
|
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
|
9
|
+
[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/fluent/fluentd/badge)](https://scorecard.dev/viewer/?uri=github.com/fluent/fluentd)
|
9
10
|
|
10
11
|
[Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
|
11
12
|
|
@@ -29,7 +30,7 @@ Fluentd: Open-Source Log Collector
|
|
29
30
|
|
30
31
|
### Prerequisites
|
31
32
|
|
32
|
-
- Ruby 2.
|
33
|
+
- Ruby 2.7 or later
|
33
34
|
- git
|
34
35
|
|
35
36
|
`git` should be in `PATH`. On Windows, you can use `Github for Windows` and `GitShell` for easy setup.
|
data/Rakefile
CHANGED
@@ -13,7 +13,7 @@ task test: [:base_test]
|
|
13
13
|
namespace :build do
|
14
14
|
desc 'Build gems for all platforms'
|
15
15
|
task :all do
|
16
|
-
Bundler.
|
16
|
+
Bundler.with_original_env do
|
17
17
|
%w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
|
18
18
|
ENV['GEM_BUILD_FAKE_PLATFORM'] = name
|
19
19
|
Rake::Task["build"].execute
|
data/SECURITY.md
CHANGED
data/fluent.conf
CHANGED
@@ -81,20 +81,20 @@
|
|
81
81
|
@id stdout_output
|
82
82
|
</match>
|
83
83
|
|
84
|
-
|
85
|
-
|
86
|
-
@type forward
|
87
|
-
@id forward_output
|
88
|
-
|
89
|
-
<server>
|
90
|
-
host 192.168.0.11
|
91
|
-
</server>
|
92
|
-
<secondary>
|
93
|
-
<server>
|
94
|
-
host 192.168.0.12
|
95
|
-
</server>
|
96
|
-
</secondary>
|
97
|
-
|
84
|
+
## match tag=system.** and forward to another fluent server
|
85
|
+
#<match system.**>
|
86
|
+
# @type forward
|
87
|
+
# @id forward_output
|
88
|
+
#
|
89
|
+
# <server>
|
90
|
+
# host 192.168.0.11
|
91
|
+
# </server>
|
92
|
+
# <secondary>
|
93
|
+
# <server>
|
94
|
+
# host 192.168.0.12
|
95
|
+
# </server>
|
96
|
+
# </secondary>
|
97
|
+
#</match>
|
98
98
|
|
99
99
|
## match tag=myapp.** and forward and write to file
|
100
100
|
#<match myapp.**>
|
@@ -130,7 +130,7 @@ module BinlogReaderCommand
|
|
130
130
|
private
|
131
131
|
|
132
132
|
def configure_option_parser
|
133
|
-
@options
|
133
|
+
@options[:config_params] = {}
|
134
134
|
|
135
135
|
@opt_parser.banner = "Usage: fluent-binlog-reader #{self.class.to_s.split('::').last.downcase} [options] file"
|
136
136
|
|
@@ -76,10 +76,10 @@ module Fluent
|
|
76
76
|
parse_options!(@argv)
|
77
77
|
|
78
78
|
target_file = if !!@opts[:target_file]
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
79
|
+
@opts[:target_file]
|
80
|
+
else
|
81
|
+
File.readlink("/proc/self/exe")
|
82
|
+
end
|
83
83
|
|
84
84
|
if @opts[:clear_capabilities]
|
85
85
|
clear_capabilities(@opts, target_file)
|
@@ -41,7 +41,7 @@ module Fluent
|
|
41
41
|
def start
|
42
42
|
super
|
43
43
|
unless self.started?
|
44
|
-
@@_super_start[self.class].
|
44
|
+
@@_super_start[self.class].bind_call(self)
|
45
45
|
# #super will reset logdev (especially in test), so this warn should be after calling it
|
46
46
|
log.warn "super was not called in #start: called it forcedly", plugin: self.class
|
47
47
|
end
|
@@ -51,7 +51,7 @@ module Fluent
|
|
51
51
|
super
|
52
52
|
unless self.before_shutdown?
|
53
53
|
log.warn "super was not called in #before_shutdown: calling it forcedly", plugin: self.class
|
54
|
-
@@_super_before_shutdown[self.class].
|
54
|
+
@@_super_before_shutdown[self.class].bind_call(self)
|
55
55
|
end
|
56
56
|
end
|
57
57
|
|
@@ -68,7 +68,7 @@ module Fluent
|
|
68
68
|
super
|
69
69
|
unless self.shutdown?
|
70
70
|
log.warn "super was not called in #shutdown: calling it forcedly", plugin: self.class
|
71
|
-
@@_super_shutdown[self.class].
|
71
|
+
@@_super_shutdown[self.class].bind_call(self)
|
72
72
|
end
|
73
73
|
end
|
74
74
|
end
|
@@ -33,25 +33,25 @@ module Fluent
|
|
33
33
|
CONFIGURABLE_CLASS_METHODS = Fluent::Configurable::ClassMethods
|
34
34
|
|
35
35
|
def config_param(name, type = nil, **kwargs, &block)
|
36
|
-
CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).
|
36
|
+
CONFIGURABLE_CLASS_METHODS.instance_method(:config_param).bind_call(self, name, type, **kwargs, &block)
|
37
37
|
pparams = propagate_default_params
|
38
38
|
if kwargs.has_key?(:default) && pparams[name.to_s]
|
39
39
|
newer = pparams[name.to_s].to_sym
|
40
40
|
overridden_default_value = kwargs[:default]
|
41
41
|
|
42
|
-
CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).
|
42
|
+
CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do
|
43
43
|
config_set_default newer, overridden_default_value
|
44
44
|
end
|
45
45
|
end
|
46
46
|
end
|
47
47
|
|
48
48
|
def config_set_default(name, defval)
|
49
|
-
CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).
|
49
|
+
CONFIGURABLE_CLASS_METHODS.instance_method(:config_set_default).bind_call(self, name, defval)
|
50
50
|
pparams = propagate_default_params
|
51
51
|
if pparams[name.to_s]
|
52
52
|
newer = pparams[name.to_s].to_sym
|
53
53
|
|
54
|
-
CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).
|
54
|
+
CONFIGURABLE_CLASS_METHODS.instance_method(:config_section).bind_call(self, :buffer) do
|
55
55
|
self.config_set_default newer, defval
|
56
56
|
end
|
57
57
|
end
|
@@ -201,7 +201,7 @@ module Fluent
|
|
201
201
|
|
202
202
|
def overwrite_defaults(other) # other is owner plugin's corresponding proxy
|
203
203
|
self.defaults = self.defaults.merge(other.defaults)
|
204
|
-
self.sections.
|
204
|
+
self.sections.each_key do |section_key|
|
205
205
|
if other.sections.has_key?(section_key)
|
206
206
|
self.sections[section_key].overwrite_defaults(other.sections[section_key])
|
207
207
|
end
|
@@ -274,7 +274,7 @@ module Fluent
|
|
274
274
|
option_value_type!(name, opts, :deprecated, String)
|
275
275
|
option_value_type!(name, opts, :obsoleted, String)
|
276
276
|
if type == :enum
|
277
|
-
if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?
|
277
|
+
if !opts.has_key?(:list) || !opts[:list].is_a?(Array) || opts[:list].empty? || !opts[:list].all?(Symbol)
|
278
278
|
raise ArgumentError, "#{name}: enum parameter requires :list of Symbols"
|
279
279
|
end
|
280
280
|
end
|
data/lib/fluent/config/types.rb
CHANGED
@@ -123,7 +123,7 @@ module Fluent
|
|
123
123
|
|
124
124
|
s = val.to_sym
|
125
125
|
list = opts[:list]
|
126
|
-
raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?
|
126
|
+
raise "Plugin BUG: config type 'enum' requires :list of symbols" unless list.is_a?(Array) && list.all?(Symbol)
|
127
127
|
unless list.include?(s)
|
128
128
|
raise ConfigError, "valid options are #{list.join(',')} but got #{val}"
|
129
129
|
end
|
data/lib/fluent/configurable.rb
CHANGED
@@ -31,13 +31,13 @@ module Fluent
|
|
31
31
|
super
|
32
32
|
# to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
|
33
33
|
proxy = self.class.merged_configure_proxy
|
34
|
-
proxy.params.
|
34
|
+
proxy.params.each_key do |name|
|
35
35
|
next if name.to_s.start_with?('@')
|
36
36
|
if proxy.defaults.has_key?(name)
|
37
37
|
instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
|
38
38
|
end
|
39
39
|
end
|
40
|
-
proxy.sections.
|
40
|
+
proxy.sections.each_key do |name|
|
41
41
|
next if name.to_s.start_with?('@')
|
42
42
|
subproxy = proxy.sections[name]
|
43
43
|
if subproxy.multi?
|
@@ -47,8 +47,6 @@ module Fluent
|
|
47
47
|
# it's not suppressed in default event router for non-log-event events
|
48
48
|
log_event_router.suppress_missing_match!
|
49
49
|
|
50
|
-
log_event_router = log_event_router
|
51
|
-
|
52
50
|
unmatched_tags = Fluent::Log.event_tags.select { |t| !log_event_router.match?(t) }
|
53
51
|
unless unmatched_tags.empty?
|
54
52
|
$log.warn "match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored)", tags: unmatched_tags
|
@@ -14,14 +14,16 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
-
require 'console
|
17
|
+
require 'console'
|
18
18
|
|
19
19
|
module Fluent
|
20
20
|
class Log
|
21
21
|
# Async gem which is used by http_server helper switched logger mechanism to
|
22
22
|
# Console gem which isn't complatible with Ruby's standard Logger (since
|
23
23
|
# v1.17). This class adapts it to Fluentd's logger mechanism.
|
24
|
-
class ConsoleAdapter < Console::
|
24
|
+
class ConsoleAdapter < Gem::Version.new(Console::VERSION) >= Gem::Version.new("1.25") ?
|
25
|
+
Console::Output::Terminal : Console::Terminal::Logger
|
26
|
+
|
25
27
|
def self.wrap(logger)
|
26
28
|
_, level = Console::Logger::LEVELS.find { |key, value|
|
27
29
|
if logger.level <= 0
|
@@ -56,7 +56,7 @@ module Fluent
|
|
56
56
|
|
57
57
|
@variable_store = Fluent::VariableStore.fetch_or_build(:buf_file)
|
58
58
|
|
59
|
-
multi_workers_configured = owner.system_config.workers > 1
|
59
|
+
multi_workers_configured = owner.system_config.workers > 1
|
60
60
|
|
61
61
|
using_plugin_root_dir = false
|
62
62
|
unless @path
|
@@ -399,7 +399,7 @@ module Fluent
|
|
399
399
|
end
|
400
400
|
|
401
401
|
if chunk.slice(0, 2) == BUFFER_HEADER
|
402
|
-
size = chunk.slice(2, 4).
|
402
|
+
size = chunk.slice(2, 4).unpack1('N')
|
403
403
|
if size
|
404
404
|
return Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(chunk.slice(6, size)).read rescue nil
|
405
405
|
end
|
@@ -238,17 +238,16 @@ module Fluent
|
|
238
238
|
callback.call(file) if callback
|
239
239
|
end
|
240
240
|
|
241
|
-
URI_PARSER = URI::Parser.new
|
242
241
|
ESCAPE_REGEXP = /[^-_.a-zA-Z0-9]/n
|
243
242
|
|
244
243
|
def encode_key(metadata)
|
245
244
|
k = @key ? metadata.variables[@key] : metadata.tag
|
246
245
|
k ||= ''
|
247
|
-
|
246
|
+
URI::DEFAULT_PARSER.escape(k, ESCAPE_REGEXP)
|
248
247
|
end
|
249
248
|
|
250
249
|
def decode_key(key)
|
251
|
-
|
250
|
+
URI::DEFAULT_PARSER.unescape(key)
|
252
251
|
end
|
253
252
|
|
254
253
|
def create_new_chunk(path, metadata, perm)
|
@@ -70,6 +70,13 @@ module Fluent::Plugin
|
|
70
70
|
end
|
71
71
|
end
|
72
72
|
begin
|
73
|
+
# Note: https://github.com/fluent/fluentd/issues/4100
|
74
|
+
# If the parser returns multiple records from one raw_value,
|
75
|
+
# this returns only the first one record.
|
76
|
+
# This should be fixed in the future version.
|
77
|
+
result_time = nil
|
78
|
+
result_record = nil
|
79
|
+
|
73
80
|
@parser.parse(raw_value) do |t, values|
|
74
81
|
if values
|
75
82
|
t = if @reserve_time
|
@@ -79,20 +86,31 @@ module Fluent::Plugin
|
|
79
86
|
end
|
80
87
|
@accessor.delete(record) if @remove_key_name_field
|
81
88
|
r = handle_parsed(tag, record, t, values)
|
82
|
-
|
89
|
+
|
90
|
+
if result_record.nil?
|
91
|
+
result_time = t
|
92
|
+
result_record = r
|
93
|
+
else
|
94
|
+
if @emit_invalid_record_to_error
|
95
|
+
router.emit_error_event(tag, t, r, Fluent::Plugin::Parser::ParserError.new(
|
96
|
+
"Could not emit the event. The parser returned multiple results, but currently filter_parser plugin only returns the first parsed result. Raw data: '#{raw_value}'"
|
97
|
+
))
|
98
|
+
end
|
99
|
+
end
|
83
100
|
else
|
84
101
|
if @emit_invalid_record_to_error
|
85
102
|
router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
|
86
103
|
end
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
end
|
104
|
+
|
105
|
+
next unless @reserve_data
|
106
|
+
next unless result_record.nil?
|
107
|
+
|
108
|
+
result_time = time
|
109
|
+
result_record = handle_parsed(tag, record, time, {})
|
94
110
|
end
|
95
111
|
end
|
112
|
+
|
113
|
+
return result_time, result_record
|
96
114
|
rescue Fluent::Plugin::Parser::ParserError => e
|
97
115
|
if @emit_invalid_record_to_error
|
98
116
|
raise e
|
@@ -45,6 +45,8 @@ module Fluent::Plugin
|
|
45
45
|
config_param :run_interval, :time, default: nil
|
46
46
|
desc 'The default block size to read if parser requires partial read.'
|
47
47
|
config_param :read_block_size, :size, default: 10240 # 10k
|
48
|
+
desc 'The encoding to receive the result of the command, especially for non-ascii characters.'
|
49
|
+
config_param :encoding, :string, default: nil
|
48
50
|
|
49
51
|
attr_reader :parser
|
50
52
|
|
@@ -63,9 +65,16 @@ module Fluent::Plugin
|
|
63
65
|
if !@tag && (!@extract_config || !@extract_config.tag_key)
|
64
66
|
raise Fluent::ConfigError, "'tag' or 'tag_key' option is required on exec input"
|
65
67
|
end
|
68
|
+
validate_encoding(@encoding) if @encoding
|
66
69
|
@parser = parser_create
|
67
70
|
end
|
68
71
|
|
72
|
+
def validate_encoding(encoding)
|
73
|
+
Encoding.find(encoding)
|
74
|
+
rescue ArgumentError => e
|
75
|
+
raise Fluent::ConfigError, e.message
|
76
|
+
end
|
77
|
+
|
69
78
|
def multi_workers_ready?
|
70
79
|
true
|
71
80
|
end
|
@@ -73,10 +82,13 @@ module Fluent::Plugin
|
|
73
82
|
def start
|
74
83
|
super
|
75
84
|
|
85
|
+
options = { mode: [@connect_mode] }
|
86
|
+
options[:external_encoding] = @encoding if @encoding
|
87
|
+
|
76
88
|
if @run_interval
|
77
|
-
child_process_execute(:exec_input, @command, interval: @run_interval,
|
89
|
+
child_process_execute(:exec_input, @command, interval: @run_interval, **options, &method(:run))
|
78
90
|
else
|
79
|
-
child_process_execute(:exec_input, @command, immediate: true,
|
91
|
+
child_process_execute(:exec_input, @command, immediate: true, **options, &method(:run))
|
80
92
|
end
|
81
93
|
end
|
82
94
|
|
@@ -203,54 +203,24 @@ module Fluent::Plugin
|
|
203
203
|
begin
|
204
204
|
path = path_info[1..-1] # remove /
|
205
205
|
tag = path.split('/').join('.')
|
206
|
-
record_time, record = parse_params(params)
|
207
206
|
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
else
|
214
|
-
if @use_204_response
|
215
|
-
return RESPONSE_204
|
216
|
-
else
|
217
|
-
return RESPONSE_200
|
218
|
-
end
|
207
|
+
mes = Fluent::MultiEventStream.new
|
208
|
+
parse_params(params) do |record_time, record|
|
209
|
+
if record.nil?
|
210
|
+
log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
|
211
|
+
next
|
219
212
|
end
|
220
|
-
end
|
221
213
|
|
222
|
-
mes = nil
|
223
|
-
# Support batched requests
|
224
|
-
if record.is_a?(Array)
|
225
|
-
mes = Fluent::MultiEventStream.new
|
226
|
-
record.each do |single_record|
|
227
|
-
add_params_to_record(single_record, params)
|
228
|
-
|
229
|
-
if param_time = params['time']
|
230
|
-
param_time = param_time.to_f
|
231
|
-
single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
|
232
|
-
elsif @custom_parser
|
233
|
-
single_time = @custom_parser.parse_time(single_record)
|
234
|
-
single_time, single_record = @custom_parser.convert_values(single_time, single_record)
|
235
|
-
else
|
236
|
-
single_time = convert_time_field(single_record)
|
237
|
-
end
|
238
|
-
|
239
|
-
mes.add(single_time, single_record)
|
240
|
-
end
|
241
|
-
else
|
242
214
|
add_params_to_record(record, params)
|
243
215
|
|
244
216
|
time = if param_time = params['time']
|
245
217
|
param_time = param_time.to_f
|
246
218
|
param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
|
247
219
|
else
|
248
|
-
|
249
|
-
convert_time_field(record)
|
250
|
-
else
|
251
|
-
record_time
|
252
|
-
end
|
220
|
+
record_time.nil? ? convert_time_field(record) : record_time
|
253
221
|
end
|
222
|
+
|
223
|
+
mes.add(time, record)
|
254
224
|
end
|
255
225
|
rescue => e
|
256
226
|
if @dump_error_log
|
@@ -261,11 +231,7 @@ module Fluent::Plugin
|
|
261
231
|
|
262
232
|
# TODO server error
|
263
233
|
begin
|
264
|
-
|
265
|
-
router.emit_stream(tag, mes)
|
266
|
-
else
|
267
|
-
router.emit(tag, time, record)
|
268
|
-
end
|
234
|
+
router.emit_stream(tag, mes) unless mes.empty?
|
269
235
|
rescue => e
|
270
236
|
if @dump_error_log
|
271
237
|
log.error "failed to emit data", error: e
|
@@ -308,20 +274,18 @@ module Fluent::Plugin
|
|
308
274
|
def parse_params_default(params)
|
309
275
|
if msgpack = params['msgpack']
|
310
276
|
@parser_msgpack.parse(msgpack) do |_time, record|
|
311
|
-
|
277
|
+
yield nil, record
|
312
278
|
end
|
313
279
|
elsif js = params['json']
|
314
280
|
@parser_json.parse(js) do |_time, record|
|
315
|
-
|
281
|
+
yield nil, record
|
316
282
|
end
|
317
283
|
elsif ndjson = params['ndjson']
|
318
|
-
events = []
|
319
284
|
ndjson.split(/\r?\n/).each do |js|
|
320
285
|
@parser_json.parse(js) do |_time, record|
|
321
|
-
|
286
|
+
yield nil, record
|
322
287
|
end
|
323
288
|
end
|
324
|
-
return nil, events
|
325
289
|
else
|
326
290
|
raise "'json', 'ndjson' or 'msgpack' parameter is required"
|
327
291
|
end
|
@@ -329,10 +293,9 @@ module Fluent::Plugin
|
|
329
293
|
|
330
294
|
def parse_params_with_parser(params)
|
331
295
|
if content = params[EVENT_RECORD_PARAMETER]
|
332
|
-
@custom_parser.parse(content)
|
333
|
-
|
334
|
-
|
335
|
-
}
|
296
|
+
@custom_parser.parse(content) do |time, record|
|
297
|
+
yield time, record
|
298
|
+
end
|
336
299
|
else
|
337
300
|
raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
|
338
301
|
end
|
@@ -485,7 +448,7 @@ module Fluent::Plugin
|
|
485
448
|
# Azure App Service sends GET requests for health checking purpose.
|
486
449
|
# Respond with `200 OK` to accommodate it.
|
487
450
|
def handle_get_request
|
488
|
-
|
451
|
+
return send_response_and_close(RES_200_STATUS, {}, "")
|
489
452
|
end
|
490
453
|
|
491
454
|
# Web browsers can send an OPTIONS request before performing POST
|
@@ -573,6 +536,8 @@ module Fluent::Plugin
|
|
573
536
|
params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
|
574
537
|
elsif /^application\/json/.match?(@content_type)
|
575
538
|
params['json'] = @body
|
539
|
+
elsif /^application\/csp-report/.match?(@content_type)
|
540
|
+
params['json'] = @body
|
576
541
|
elsif /^application\/msgpack/.match?(@content_type)
|
577
542
|
params['msgpack'] = @body
|
578
543
|
elsif /^application\/x-ndjson/.match?(@content_type)
|
@@ -580,7 +545,7 @@ module Fluent::Plugin
|
|
580
545
|
end
|
581
546
|
path_info = uri.path
|
582
547
|
|
583
|
-
if (@add_query_params)
|
548
|
+
if (@add_query_params)
|
584
549
|
|
585
550
|
query_params = WEBrick::HTTPUtils.parse_query(uri.query)
|
586
551
|
|