fluentd 1.14.3-x86-mingw32 → 1.14.6-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/config.yml +2 -2
- data/.github/workflows/linux-test.yaml +1 -1
- data/.github/workflows/macos-test.yaml +5 -1
- data/.github/workflows/windows-test.yaml +6 -6
- data/CHANGELOG.md +79 -16
- data/CONTRIBUTING.md +1 -1
- data/MAINTAINERS.md +2 -2
- data/README.md +2 -23
- data/Rakefile +1 -1
- data/fluentd.gemspec +2 -1
- data/lib/fluent/command/fluentd.rb +4 -0
- data/lib/fluent/config/error.rb +12 -0
- data/lib/fluent/env.rb +4 -0
- data/lib/fluent/event_router.rb +19 -1
- data/lib/fluent/plugin/bare_output.rb +1 -1
- data/lib/fluent/plugin/base.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +43 -23
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +11 -1
- data/lib/fluent/plugin/in_tail.rb +10 -0
- data/lib/fluent/plugin/out_file.rb +13 -1
- data/lib/fluent/plugin/output.rb +41 -32
- data/lib/fluent/plugin/parser.rb +3 -4
- data/lib/fluent/plugin_helper/retry_state.rb +14 -4
- data/lib/fluent/plugin_helper/server.rb +21 -4
- data/lib/fluent/plugin_helper/socket.rb +13 -2
- data/lib/fluent/registry.rb +2 -1
- data/lib/fluent/rpc.rb +4 -3
- data/lib/fluent/supervisor.rb +5 -2
- data/lib/fluent/version.rb +1 -1
- data/test/compat/test_parser.rb +1 -1
- data/test/plugin/test_bare_output.rb +1 -1
- data/test/plugin/test_buffer.rb +77 -0
- data/test/plugin/test_filter.rb +1 -1
- data/test/plugin/test_filter_parser.rb +1 -1
- data/test/plugin/test_filter_stdout.rb +2 -2
- data/test/plugin/test_in_forward.rb +0 -2
- data/test/plugin/test_in_http.rb +23 -0
- data/test/plugin/test_in_tail.rb +35 -0
- data/test/plugin/test_input.rb +1 -1
- data/test/plugin/test_out_exec.rb +6 -4
- data/test/plugin/test_out_file.rb +29 -13
- data/test/plugin/test_out_stdout.rb +2 -2
- data/test/plugin/test_output_as_buffered_retries.rb +53 -6
- data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
- data/test/plugin_helper/test_retry_state.rb +602 -38
- data/test/plugin_helper/test_server.rb +18 -0
- data/test/plugin_helper/test_timer.rb +2 -2
- data/test/test_event_router.rb +17 -0
- data/test/test_formatter.rb +1 -1
- data/test/test_supervisor.rb +41 -6
- metadata +19 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: aeb3485cb50937c6242fd3406af5e31fcf8e8e64743d201ddbaed2b9d7c3c9f5
|
|
4
|
+
data.tar.gz: 679a45104420e99c66525884c7a274972a1875abc008d097b786080c9e01833f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bda6562dc55e1387d78a97d01f0da289baa00dbd360dbb4401c12b5a690e7e809c8bc6f11d33353e282215beda96e168791899ffc6d65c09f6c863912196a85b
|
|
7
|
+
data.tar.gz: cdeacccd982235e7baf9e47fd6407d9855bad5face3775ac9c87220b26eb5998d550b1d8cb2f8b425005fb34e239645d118f2cb5afda06103a41be71da934fed
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
blank_issues_enabled: false
|
|
2
2
|
contact_links:
|
|
3
3
|
- name: Ask a Question
|
|
4
|
-
url: https://
|
|
5
|
-
about: I have questions about Fluentd and plugins. Please ask and answer questions at https://
|
|
4
|
+
url: https://github.com/fluent/fluentd/discussions
|
|
5
|
+
about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions
|
|
@@ -13,9 +13,13 @@ jobs:
|
|
|
13
13
|
strategy:
|
|
14
14
|
fail-fast: false
|
|
15
15
|
matrix:
|
|
16
|
-
ruby-version: ['
|
|
16
|
+
ruby-version: ['3.1', '3.0', '2.7']
|
|
17
17
|
os: [macos-latest]
|
|
18
18
|
experimental: [true]
|
|
19
|
+
include:
|
|
20
|
+
- ruby-version: head
|
|
21
|
+
os: macos-latest
|
|
22
|
+
experimental: true
|
|
19
23
|
|
|
20
24
|
name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
|
|
21
25
|
steps:
|
|
@@ -18,9 +18,9 @@ jobs:
|
|
|
18
18
|
- windows-latest
|
|
19
19
|
experimental: [false]
|
|
20
20
|
include:
|
|
21
|
-
- ruby-version: '3.0.
|
|
21
|
+
- ruby-version: '3.0.3'
|
|
22
22
|
os: windows-latest
|
|
23
|
-
experimental:
|
|
23
|
+
experimental: false
|
|
24
24
|
# On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
|
|
25
25
|
# error code. In addition, we have to specify the path of fiddle by RUBYLIB
|
|
26
26
|
# because RubyInstaller loads Ruby's bundled fiddle before initializing gem.
|
|
@@ -28,7 +28,7 @@ jobs:
|
|
|
28
28
|
# * https://github.com/ruby/fiddle/issues/72
|
|
29
29
|
# * https://bugs.ruby-lang.org/issues/17813
|
|
30
30
|
# * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
|
|
31
|
-
ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.
|
|
31
|
+
ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.3/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.1.0/lib
|
|
32
32
|
|
|
33
33
|
name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
|
|
34
34
|
steps:
|
|
@@ -37,9 +37,9 @@ jobs:
|
|
|
37
37
|
uses: ruby/setup-ruby@v1
|
|
38
38
|
with:
|
|
39
39
|
ruby-version: ${{ matrix.ruby-version }}
|
|
40
|
-
- name: Add Fiddle 1.0
|
|
41
|
-
if: ${{ matrix.ruby-version == '3.0.
|
|
42
|
-
run: gem install fiddle --version 1.0
|
|
40
|
+
- name: Add Fiddle 1.1.0
|
|
41
|
+
if: ${{ matrix.ruby-version == '3.0.3' }}
|
|
42
|
+
run: gem install fiddle --version 1.1.0
|
|
43
43
|
- name: Install dependencies
|
|
44
44
|
run: ridk exec bundle install
|
|
45
45
|
- name: Run tests
|
data/CHANGELOG.md
CHANGED
|
@@ -1,4 +1,79 @@
|
|
|
1
|
-
# v1.14
|
|
1
|
+
# v1.14
|
|
2
|
+
|
|
3
|
+
## Release v1.14.6 - 2022/03/31
|
|
4
|
+
|
|
5
|
+
### Enhancement
|
|
6
|
+
|
|
7
|
+
* Enable server plugins to specify socket-option `SO_LINGER`
|
|
8
|
+
https://github.com/fluent/fluentd/pull/3644
|
|
9
|
+
* Add `--umask` command line parameter
|
|
10
|
+
https://github.com/fluent/fluentd/pull/3671
|
|
11
|
+
https://github.com/fluent/fluentd/pull/3679
|
|
12
|
+
|
|
13
|
+
### Bug fixes
|
|
14
|
+
|
|
15
|
+
* Fix metric name typo
|
|
16
|
+
https://github.com/fluent/fluentd/pull/3630
|
|
17
|
+
https://github.com/fluent/fluentd/pull/3673
|
|
18
|
+
* Apply modifications in pipeline to the records being passed to `@ERROR` label
|
|
19
|
+
https://github.com/fluent/fluentd/pull/3631
|
|
20
|
+
* Fix wrong calculation of retry interval
|
|
21
|
+
https://github.com/fluent/fluentd/pull/3640
|
|
22
|
+
https://github.com/fluent/fluentd/pull/3649
|
|
23
|
+
https://github.com/fluent/fluentd/pull/3685
|
|
24
|
+
https://github.com/fluent/fluentd/pull/3686
|
|
25
|
+
* Support IPv6 address for `rpc_endpoint` in `system` config
|
|
26
|
+
https://github.com/fluent/fluentd/pull/3641
|
|
27
|
+
|
|
28
|
+
### Misc
|
|
29
|
+
|
|
30
|
+
* CI: Support Ruby 3.1 except Windows
|
|
31
|
+
https://github.com/fluent/fluentd/pull/3619
|
|
32
|
+
* Switch to GitHub Discussions
|
|
33
|
+
https://github.com/fluent/fluentd/pull/3654
|
|
34
|
+
* Fix CHANGELOG.md heading styles
|
|
35
|
+
https://github.com/fluent/fluentd/pull/3648
|
|
36
|
+
* Declare `null_value_pattern` as `regexp`
|
|
37
|
+
https://github.com/fluent/fluentd/pull/3650
|
|
38
|
+
|
|
39
|
+
## Release v1.14.5 - 2022/02/09
|
|
40
|
+
|
|
41
|
+
### Enhancement
|
|
42
|
+
|
|
43
|
+
* Add support for "application/x-ndjson" to `in_http`
|
|
44
|
+
https://github.com/fluent/fluentd/pull/3616
|
|
45
|
+
* Add support for ucrt binary for Windows
|
|
46
|
+
https://github.com/fluent/fluentd/pull/3613
|
|
47
|
+
|
|
48
|
+
### Bug fixes
|
|
49
|
+
|
|
50
|
+
* Don't retry when `retry_max_times == 0`
|
|
51
|
+
https://github.com/fluent/fluentd/pull/3608
|
|
52
|
+
* Fix hang-up issue during TLS handshake in `out_forward`
|
|
53
|
+
https://github.com/fluent/fluentd/pull/3601
|
|
54
|
+
* Bump up required ServerEngine to v2.2.5
|
|
55
|
+
https://github.com/fluent/fluentd/pull/3599
|
|
56
|
+
* Fix "invalid byte sequence is replaced" warning on Kubernetes
|
|
57
|
+
https://github.com/fluent/fluentd/pull/3596
|
|
58
|
+
* Fix "ArgumentError: unknown keyword: :logger" on Windows with Ruby 3.1
|
|
59
|
+
https://github.com/fluent/fluentd/pull/3592
|
|
60
|
+
|
|
61
|
+
## Release v1.14.4 - 2022/01/06
|
|
62
|
+
|
|
63
|
+
### Enhancement
|
|
64
|
+
|
|
65
|
+
* `in_tail`: Add option to skip long lines (`max_line_size`)
|
|
66
|
+
https://github.com/fluent/fluentd/pull/3565
|
|
67
|
+
|
|
68
|
+
### Bug fix
|
|
69
|
+
|
|
70
|
+
* Incorrect BufferChunkOverflowError when each event size is < `chunk_limit_size`
|
|
71
|
+
https://github.com/fluent/fluentd/pull/3560
|
|
72
|
+
* On macOS with Ruby 2.7/3.0, `out_file` fails to write events if `append` is true.
|
|
73
|
+
https://github.com/fluent/fluentd/pull/3579
|
|
74
|
+
* test: Fix unstable test cases
|
|
75
|
+
https://github.com/fluent/fluentd/pull/3574
|
|
76
|
+
https://github.com/fluent/fluentd/pull/3577
|
|
2
77
|
|
|
3
78
|
## Release v1.14.3 - 2021/11/26
|
|
4
79
|
|
|
@@ -28,8 +103,6 @@
|
|
|
28
103
|
newer version is required to implement additional `fluent-ctl` commands.
|
|
29
104
|
https://github.com/fluent/fluentd/pull/3556
|
|
30
105
|
|
|
31
|
-
# v1.14.2
|
|
32
|
-
|
|
33
106
|
## Release v1.14.2 - 2021/10/29
|
|
34
107
|
|
|
35
108
|
IMPORTANT: This release contain the fix for CVE-2021-41186 -
|
|
@@ -41,6 +114,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
41
114
|
### Enhancement
|
|
42
115
|
|
|
43
116
|
* fluent-cat: Add `--event-time` option to send specified event time for testing.
|
|
117
|
+
https://github.com/fluent/fluentd/pull/3528
|
|
44
118
|
|
|
45
119
|
### Bug fix
|
|
46
120
|
|
|
@@ -49,8 +123,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
49
123
|
* Fixed ReDoS vulnerability in parser_apache2.
|
|
50
124
|
This vulnerability is caused by a certain pattern of a broken apache log.
|
|
51
125
|
|
|
52
|
-
# v1.14.1
|
|
53
|
-
|
|
54
126
|
## Release v1.14.1 - 2021/09/29
|
|
55
127
|
|
|
56
128
|
### Enhancement
|
|
@@ -80,8 +152,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
80
152
|
next_retry_seconds -> next_retry_time)
|
|
81
153
|
https://github.com/fluent/fluentd/pull/3518
|
|
82
154
|
|
|
83
|
-
# v1.14.0
|
|
84
|
-
|
|
85
155
|
## Release v1.14.0 - 2021/08/30
|
|
86
156
|
|
|
87
157
|
### Enhancement
|
|
@@ -124,7 +194,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
124
194
|
https://github.com/fluent/fluentd/pull/3479
|
|
125
195
|
https://github.com/fluent/fluentd/pull/3484
|
|
126
196
|
|
|
127
|
-
# v1.13
|
|
197
|
+
# v1.13
|
|
128
198
|
|
|
129
199
|
## Release v1.13.3 - 2021/07/27
|
|
130
200
|
|
|
@@ -146,10 +216,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
146
216
|
* Remove needless spaces in a sample config file
|
|
147
217
|
https://github.com/fluent/fluentd/pull/3456
|
|
148
218
|
|
|
149
|
-
### Enhancement
|
|
150
|
-
|
|
151
|
-
# v1.13.2
|
|
152
|
-
|
|
153
219
|
## Release v1.13.2 - 2021/07/12
|
|
154
220
|
|
|
155
221
|
### Enhancement
|
|
@@ -185,8 +251,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
185
251
|
by security scanning tools.
|
|
186
252
|
https://github.com/fluent/fluentd/pull/3450
|
|
187
253
|
|
|
188
|
-
# v1.13.1
|
|
189
|
-
|
|
190
254
|
## Release v1.13.1 - 2021/06/25
|
|
191
255
|
|
|
192
256
|
### Bug fix
|
|
@@ -211,8 +275,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
211
275
|
* CI: Dropped to run CI for Ruby 2.5
|
|
212
276
|
https://github.com/fluent/fluentd/pull/3412
|
|
213
277
|
|
|
214
|
-
# v1.13
|
|
215
|
-
|
|
216
278
|
## Release v1.13.0 - 2021/05/29
|
|
217
279
|
|
|
218
280
|
### Enhancement
|
|
@@ -260,6 +322,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
260
322
|
### Bug fix
|
|
261
323
|
|
|
262
324
|
* in_tail: Fix a bug that refresh_watcher fails to handle file rotations
|
|
325
|
+
https://github.com/fluent/fluentd/pull/3393
|
|
263
326
|
|
|
264
327
|
## Release v1.12.3 - 2021/04/23
|
|
265
328
|
|
data/CONTRIBUTING.md
CHANGED
|
@@ -6,7 +6,7 @@ We'd love your contribution. Here are the guidelines!
|
|
|
6
6
|
|
|
7
7
|
RESOURCES of [Official site](https://www.fluentd.org/) and [Fluentd documentation](https://docs.fluentd.org/) may help you.
|
|
8
8
|
|
|
9
|
-
If you have further questions about Fluentd and plugins, please direct these to [Community Forum](https://
|
|
9
|
+
If you have further questions about Fluentd and plugins, please direct these to [Community Forum](https://github.com/fluent/fluentd/discussions).
|
|
10
10
|
Don't use Github issue for asking questions. Here are examples:
|
|
11
11
|
|
|
12
12
|
- I installed xxx plugin but it doesn't work. Why?
|
data/MAINTAINERS.md
CHANGED
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
- [Naotoshi Seo](https://github.com/sonots), [ZOZO Technologies](https://tech.zozo.com/en/)
|
|
4
4
|
- [Okkez](https://github.com/okkez)
|
|
5
|
-
- [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://
|
|
5
|
+
- [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://calyptia.com/)
|
|
6
6
|
- [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)
|
|
7
7
|
- [Satoshi Tagomori](https://github.com/tagomoris), [Treasure Data](https://www.treasuredata.com/)
|
|
8
|
-
- [Eduardo Silva](https://github.com/edsiper), [
|
|
8
|
+
- [Eduardo Silva](https://github.com/edsiper), [Calyptia](https://calyptia/)
|
|
9
9
|
- [Fujimoto Seiji](https://github.com/fujimots), [ClearCode](https://www.clear-code.com/)
|
|
10
10
|
- [Takuro Ashie](https://github.com/ashie), [ClearCode](https://www.clear-code.com/)
|
|
11
11
|
- [Kentaro Hayashi](https://github.com/kenhys), [ClearCode](https://www.clear-code.com/)
|
data/README.md
CHANGED
|
@@ -1,17 +1,12 @@
|
|
|
1
1
|
Fluentd: Open-Source Log Collector
|
|
2
2
|
===================================
|
|
3
|
-
GitHub Actions:
|
|
4
3
|
|
|
5
4
|
[](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml)
|
|
6
5
|
[](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml)
|
|
7
6
|
[](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
|
|
7
|
+
[](https://cloud.drone.io/fluent/fluentd)
|
|
8
8
|
[](https://codeclimate.com/github/fluent/fluentd)
|
|
9
9
|
[](https://bestpractices.coreinfrastructure.org/projects/1189)
|
|
10
|
-
[](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
|
|
11
|
-
|
|
12
|
-
Drone CI for Arm64:
|
|
13
|
-
|
|
14
|
-
[](https://cloud.drone.io/fluent/fluentd)
|
|
15
10
|
|
|
16
11
|
[Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
|
|
17
12
|
|
|
@@ -19,16 +14,6 @@ Drone CI for Arm64:
|
|
|
19
14
|
<img src="https://www.fluentd.org/images/fluentd-architecture.png" width="500px"/>
|
|
20
15
|
</p>
|
|
21
16
|
|
|
22
|
-
An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time recorded at occurrence of an event. Record is a JSON object.
|
|
23
|
-
|
|
24
|
-
## Example Use Cases
|
|
25
|
-
|
|
26
|
-
Use Case | Description | Diagram
|
|
27
|
-
-------- | ------------|:---------:
|
|
28
|
-
Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/images/recipes/elasticsearch-s3-fluentd.png" height="150"/>
|
|
29
|
-
Syslog Alerting | Fluentd can "grep" for events and send out alerts. | <img src="https://www.fluentd.org/images/syslog-fluentd-alert.png" height="100"/>
|
|
30
|
-
Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/images/datasources/asynchronous_logging.png" height="150"/>
|
|
31
|
-
|
|
32
17
|
## Quick Start
|
|
33
18
|
|
|
34
19
|
$ gem install fluentd
|
|
@@ -66,18 +51,12 @@ You can run specified test via `TEST` environment variable:
|
|
|
66
51
|
$ bundle exec rake test TEST=test/test_specified_path.rb
|
|
67
52
|
$ bundle exec rake test TEST=test/test_*.rb
|
|
68
53
|
|
|
69
|
-
## Fluentd UI: Admin GUI
|
|
70
|
-
|
|
71
|
-
[Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.
|
|
72
|
-
|
|
73
|
-
<p align="center"><img width="500" src="https://www.fluentd.org/images/blog/fluentd-ui.gif"/></p>
|
|
74
|
-
|
|
75
54
|
## More Information
|
|
76
55
|
|
|
77
56
|
- Website: https://www.fluentd.org/
|
|
78
57
|
- Documentation: https://docs.fluentd.org/
|
|
79
58
|
- Project repository: https://github.com/fluent
|
|
80
|
-
- Discussion: https://
|
|
59
|
+
- Discussion: https://github.com/fluent/fluentd/discussions
|
|
81
60
|
- Slack / Community: https://slack.fluentd.org
|
|
82
61
|
- Newsletters: https://www.fluentd.org/newsletter
|
|
83
62
|
- Author: [Sadayuki Furuhashi](https://github.com/frsyuki)
|
data/Rakefile
CHANGED
|
@@ -14,7 +14,7 @@ namespace :build do
|
|
|
14
14
|
desc 'Build gems for all platforms'
|
|
15
15
|
task :all do
|
|
16
16
|
Bundler.with_clean_env do
|
|
17
|
-
%w[ruby x86-mingw32 x64-mingw32].each do |name|
|
|
17
|
+
%w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
|
|
18
18
|
ENV['GEM_BUILD_FAKE_PLATFORM'] = name
|
|
19
19
|
Rake::Task["build"].execute
|
|
20
20
|
end
|
data/fluentd.gemspec
CHANGED
|
@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
|
|
|
22
22
|
gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
|
|
23
23
|
gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
|
|
24
24
|
gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
|
|
25
|
-
gem.add_runtime_dependency("serverengine", [">= 2.2.
|
|
25
|
+
gem.add_runtime_dependency("serverengine", [">= 2.2.5", "< 3.0.0"])
|
|
26
26
|
gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
|
|
27
27
|
gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
|
|
28
28
|
gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
|
|
@@ -51,5 +51,6 @@ Gem::Specification.new do |gem|
|
|
|
51
51
|
gem.add_development_dependency("test-unit", ["~> 3.3"])
|
|
52
52
|
gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
|
|
53
53
|
gem.add_development_dependency("oj", [">= 2.14", "< 4"])
|
|
54
|
+
gem.add_development_dependency("async", "~> 1.23")
|
|
54
55
|
gem.add_development_dependency("async-http", ">= 0.50.0")
|
|
55
56
|
end
|
data/lib/fluent/config/error.rb
CHANGED
|
@@ -29,4 +29,16 @@ module Fluent
|
|
|
29
29
|
|
|
30
30
|
class SetDefault < Exception
|
|
31
31
|
end
|
|
32
|
+
|
|
33
|
+
class NotFoundPluginError < ConfigError
|
|
34
|
+
attr_reader :type, :kind
|
|
35
|
+
|
|
36
|
+
def initialize(msg, type: nil, kind: nil)
|
|
37
|
+
@msg = msg
|
|
38
|
+
@type = type
|
|
39
|
+
@kind = kind
|
|
40
|
+
|
|
41
|
+
super(msg)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
32
44
|
end
|
data/lib/fluent/env.rb
CHANGED
data/lib/fluent/event_router.rb
CHANGED
|
@@ -116,6 +116,8 @@ module Fluent
|
|
|
116
116
|
if callback = find_callback
|
|
117
117
|
callback.call(es)
|
|
118
118
|
end
|
|
119
|
+
rescue Pipeline::OutputError => e
|
|
120
|
+
@emit_error_handler.handle_emits_error(tag, e.processed_es, e.internal_error)
|
|
119
121
|
rescue => e
|
|
120
122
|
@emit_error_handler.handle_emits_error(tag, es, e)
|
|
121
123
|
end
|
|
@@ -161,6 +163,17 @@ module Fluent
|
|
|
161
163
|
private
|
|
162
164
|
|
|
163
165
|
class Pipeline
|
|
166
|
+
|
|
167
|
+
class OutputError < StandardError
|
|
168
|
+
attr_reader :internal_error
|
|
169
|
+
attr_reader :processed_es
|
|
170
|
+
|
|
171
|
+
def initialize(internal_error, processed_es)
|
|
172
|
+
@internal_error = internal_error
|
|
173
|
+
@processed_es = processed_es
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
|
|
164
177
|
def initialize
|
|
165
178
|
@filters = []
|
|
166
179
|
@output = nil
|
|
@@ -178,7 +191,12 @@ module Fluent
|
|
|
178
191
|
|
|
179
192
|
def emit_events(tag, es)
|
|
180
193
|
processed = @optimizer.filter_stream(tag, es)
|
|
181
|
-
|
|
194
|
+
|
|
195
|
+
begin
|
|
196
|
+
@output.emit_events(tag, processed)
|
|
197
|
+
rescue => e
|
|
198
|
+
raise OutputError.new(e, processed)
|
|
199
|
+
end
|
|
182
200
|
end
|
|
183
201
|
|
|
184
202
|
class FilterOptimizer
|
|
@@ -70,7 +70,7 @@ module Fluent
|
|
|
70
70
|
super
|
|
71
71
|
|
|
72
72
|
@num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "num_errors", help_text: "Number of count num errors")
|
|
73
|
-
@emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "
|
|
73
|
+
@emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_count", help_text: "Number of count emits")
|
|
74
74
|
@emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of emit records")
|
|
75
75
|
@emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_size", help_text: "Total size of emit events")
|
|
76
76
|
@enable_size_metrics = !!system_config.enable_size_metrics
|
data/lib/fluent/plugin/base.rb
CHANGED
|
@@ -72,8 +72,8 @@ module Fluent
|
|
|
72
72
|
|
|
73
73
|
def string_safe_encoding(str)
|
|
74
74
|
unless str.valid_encoding?
|
|
75
|
-
log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
|
|
76
75
|
str = str.scrub('?')
|
|
76
|
+
log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
|
|
77
77
|
end
|
|
78
78
|
yield str
|
|
79
79
|
end
|
data/lib/fluent/plugin/buffer.rb
CHANGED
|
@@ -767,24 +767,37 @@ module Fluent
|
|
|
767
767
|
raise ShouldRetry unless chunk.writable?
|
|
768
768
|
staged_chunk_used = true if chunk.staged?
|
|
769
769
|
|
|
770
|
-
original_bytesize = chunk.bytesize
|
|
770
|
+
original_bytesize = committed_bytesize = chunk.bytesize
|
|
771
771
|
begin
|
|
772
772
|
while writing_splits_index < splits.size
|
|
773
773
|
split = splits[writing_splits_index]
|
|
774
|
-
formatted_split = format ? format.call(split) :
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
774
|
+
formatted_split = format ? format.call(split) : nil
|
|
775
|
+
|
|
776
|
+
if split.size == 1 # Check BufferChunkOverflowError
|
|
777
|
+
determined_bytesize = nil
|
|
778
|
+
if @compress != :text
|
|
779
|
+
determined_bytesize = nil
|
|
780
|
+
elsif formatted_split
|
|
781
|
+
determined_bytesize = formatted_split.bytesize
|
|
782
|
+
elsif split.first.respond_to?(:bytesize)
|
|
783
|
+
determined_bytesize = split.first.bytesize
|
|
784
|
+
end
|
|
785
|
+
|
|
786
|
+
if determined_bytesize && determined_bytesize > @chunk_limit_size
|
|
787
|
+
# It is a obvious case that BufferChunkOverflowError should be raised here.
|
|
788
|
+
# But if it raises here, already processed 'split' or
|
|
789
|
+
# the proceeding 'split' will be lost completely.
|
|
790
|
+
# So it is a last resort to delay raising such a exception
|
|
791
|
+
errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
|
|
792
|
+
writing_splits_index += 1
|
|
793
|
+
next
|
|
794
|
+
end
|
|
795
|
+
|
|
796
|
+
if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
|
|
797
|
+
# The split will (might) cause size over so keep already processed
|
|
798
|
+
# 'split' content here (allow performance regression a bit).
|
|
780
799
|
chunk.commit
|
|
781
|
-
|
|
782
|
-
big_record_size = formatted_split.bytesize
|
|
783
|
-
if chunk.bytesize + big_record_size > @chunk_limit_size
|
|
784
|
-
errors << "a #{big_record_size} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
|
|
785
|
-
writing_splits_index += 1
|
|
786
|
-
next
|
|
787
|
-
end
|
|
800
|
+
committed_bytesize = chunk.bytesize
|
|
788
801
|
end
|
|
789
802
|
end
|
|
790
803
|
|
|
@@ -793,19 +806,26 @@ module Fluent
|
|
|
793
806
|
else
|
|
794
807
|
chunk.append(split, compress: @compress)
|
|
795
808
|
end
|
|
809
|
+
adding_bytes = chunk.bytesize - committed_bytesize
|
|
796
810
|
|
|
797
811
|
if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
|
|
798
|
-
adding_bytes = chunk.instance_eval { @adding_bytes } || "N/A" # 3rd party might not have 'adding_bytes'
|
|
799
812
|
chunk.rollback
|
|
813
|
+
committed_bytesize = chunk.bytesize
|
|
800
814
|
|
|
801
|
-
if split.size == 1
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
815
|
+
if split.size == 1 # Check BufferChunkOverflowError again
|
|
816
|
+
if adding_bytes > @chunk_limit_size
|
|
817
|
+
errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
|
|
818
|
+
writing_splits_index += 1
|
|
819
|
+
next
|
|
820
|
+
else
|
|
821
|
+
# As already processed content is kept after rollback, then unstaged chunk should be queued.
|
|
822
|
+
# After that, re-process current split again.
|
|
823
|
+
# New chunk should be allocated, to do it, modify @stage and so on.
|
|
824
|
+
synchronize { @stage.delete(modified_metadata) }
|
|
825
|
+
staged_chunk_used = false
|
|
826
|
+
chunk.unstaged!
|
|
827
|
+
break
|
|
828
|
+
end
|
|
809
829
|
end
|
|
810
830
|
|
|
811
831
|
if chunk_size_full?(chunk) || split.size == 1
|
|
@@ -40,7 +40,7 @@ module Fluent::Plugin
|
|
|
40
40
|
config_param :backlog, :integer, default: nil
|
|
41
41
|
# SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
|
|
42
42
|
desc 'The timeout time used to set linger option.'
|
|
43
|
-
config_param :linger_timeout, :integer, default:
|
|
43
|
+
config_param :linger_timeout, :integer, default: nil, deprecated: "use transport directive"
|
|
44
44
|
# This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
|
|
45
45
|
config_param :blocking_timeout, :time, default: 0.5
|
|
46
46
|
desc 'Try to resolve hostname from IP addresses or not.'
|
|
@@ -314,8 +314,16 @@ module Fluent::Plugin
|
|
|
314
314
|
@parser_json.parse(js) do |_time, record|
|
|
315
315
|
return nil, record
|
|
316
316
|
end
|
|
317
|
+
elsif ndjson = params['ndjson']
|
|
318
|
+
events = []
|
|
319
|
+
ndjson.split(/\r?\n/).each do |js|
|
|
320
|
+
@parser_json.parse(js) do |_time, record|
|
|
321
|
+
events.push(record)
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
return nil, events
|
|
317
325
|
else
|
|
318
|
-
raise "'json' or 'msgpack' parameter is required"
|
|
326
|
+
raise "'json', 'ndjson' or 'msgpack' parameter is required"
|
|
319
327
|
end
|
|
320
328
|
end
|
|
321
329
|
|
|
@@ -567,6 +575,8 @@ module Fluent::Plugin
|
|
|
567
575
|
params['json'] = @body
|
|
568
576
|
elsif @content_type =~ /^application\/msgpack/
|
|
569
577
|
params['msgpack'] = @body
|
|
578
|
+
elsif @content_type =~ /^application\/x-ndjson/
|
|
579
|
+
params['ndjson'] = @body
|
|
570
580
|
end
|
|
571
581
|
path_info = uri.path
|
|
572
582
|
|
|
@@ -113,6 +113,8 @@ module Fluent::Plugin
|
|
|
113
113
|
config_param :path_timezone, :string, default: nil
|
|
114
114
|
desc 'Follow inodes instead of following file names. Guarantees more stable delivery and allows to use * in path pattern with rotating files'
|
|
115
115
|
config_param :follow_inodes, :bool, default: false
|
|
116
|
+
desc 'Maximum length of line. The longer line is just skipped.'
|
|
117
|
+
config_param :max_line_size, :size, default: nil
|
|
116
118
|
|
|
117
119
|
config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
|
|
118
120
|
config_argument :usage, :string, default: 'in_tail_parser'
|
|
@@ -594,6 +596,14 @@ module Fluent::Plugin
|
|
|
594
596
|
|
|
595
597
|
# @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
|
|
596
598
|
def receive_lines(lines, tail_watcher)
|
|
599
|
+
lines = lines.reject do |line|
|
|
600
|
+
skip_line = @max_line_size ? line.bytesize > @max_line_size : false
|
|
601
|
+
if skip_line
|
|
602
|
+
log.warn "received line length is longer than #{@max_line_size}"
|
|
603
|
+
log.debug "skipped line: #{line.chomp}"
|
|
604
|
+
end
|
|
605
|
+
skip_line
|
|
606
|
+
end
|
|
597
607
|
es = @receive_handler.call(lines, tail_watcher)
|
|
598
608
|
unless es.empty?
|
|
599
609
|
tag = if @tag_prefix || @tag_suffix
|
|
@@ -181,6 +181,13 @@ module Fluent::Plugin
|
|
|
181
181
|
@dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
|
|
182
182
|
@file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
|
|
183
183
|
@need_lock = system_config.workers > 1
|
|
184
|
+
|
|
185
|
+
# https://github.com/fluent/fluentd/issues/3569
|
|
186
|
+
@need_ruby_on_macos_workaround = false
|
|
187
|
+
if @append && Fluent.macos?
|
|
188
|
+
condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
|
|
189
|
+
@need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
|
|
190
|
+
end
|
|
184
191
|
end
|
|
185
192
|
|
|
186
193
|
def multi_workers_ready?
|
|
@@ -223,7 +230,12 @@ module Fluent::Plugin
|
|
|
223
230
|
|
|
224
231
|
def write_without_compression(path, chunk)
|
|
225
232
|
File.open(path, "ab", @file_perm) do |f|
|
|
226
|
-
|
|
233
|
+
if @need_ruby_on_macos_workaround
|
|
234
|
+
content = chunk.read()
|
|
235
|
+
f.puts content
|
|
236
|
+
else
|
|
237
|
+
chunk.write_to(f)
|
|
238
|
+
end
|
|
227
239
|
end
|
|
228
240
|
end
|
|
229
241
|
|