fluentd 1.14.4-x86-mingw32 → 1.15.0-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/config.yml +2 -2
- data/.github/workflows/linux-test.yaml +1 -1
- data/.github/workflows/macos-test.yaml +5 -1
- data/.github/workflows/windows-test.yaml +9 -6
- data/CHANGELOG.md +115 -19
- data/CONTRIBUTING.md +1 -1
- data/MAINTAINERS.md +2 -2
- data/README.md +2 -23
- data/Rakefile +1 -1
- data/fluentd.gemspec +3 -1
- data/lib/fluent/command/ctl.rb +4 -1
- data/lib/fluent/command/fluentd.rb +14 -0
- data/lib/fluent/config/error.rb +12 -0
- data/lib/fluent/config/literal_parser.rb +2 -2
- data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
- data/lib/fluent/config/yaml_parser/loader.rb +91 -0
- data/lib/fluent/config/yaml_parser/parser.rb +166 -0
- data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
- data/lib/fluent/config/yaml_parser.rb +56 -0
- data/lib/fluent/config.rb +14 -1
- data/lib/fluent/event_router.rb +19 -1
- data/lib/fluent/plugin/bare_output.rb +1 -1
- data/lib/fluent/plugin/base.rb +1 -1
- data/lib/fluent/plugin/file_wrapper.rb +52 -107
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +11 -1
- data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +1 -15
- data/lib/fluent/plugin/in_tail.rb +66 -47
- data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
- data/lib/fluent/plugin/output.rb +43 -33
- data/lib/fluent/plugin/parser.rb +3 -4
- data/lib/fluent/plugin/parser_syslog.rb +1 -1
- data/lib/fluent/plugin_helper/retry_state.rb +14 -4
- data/lib/fluent/plugin_helper/server.rb +23 -4
- data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
- data/lib/fluent/plugin_helper/socket.rb +13 -2
- data/lib/fluent/registry.rb +2 -1
- data/lib/fluent/rpc.rb +4 -3
- data/lib/fluent/supervisor.rb +114 -27
- data/lib/fluent/system_config.rb +2 -1
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +2 -0
- data/test/command/test_ctl.rb +0 -1
- data/test/command/test_fluentd.rb +33 -0
- data/test/compat/test_parser.rb +1 -1
- data/test/config/test_system_config.rb +3 -1
- data/test/config/test_types.rb +1 -1
- data/test/plugin/in_tail/test_io_handler.rb +14 -4
- data/test/plugin/in_tail/test_position_file.rb +0 -63
- data/test/plugin/out_forward/test_socket_cache.rb +26 -1
- data/test/plugin/test_file_wrapper.rb +0 -68
- data/test/plugin/test_filter_parser.rb +1 -1
- data/test/plugin/test_filter_stdout.rb +2 -2
- data/test/plugin/test_in_forward.rb +0 -2
- data/test/plugin/test_in_http.rb +23 -0
- data/test/plugin/test_in_object_space.rb +9 -3
- data/test/plugin/test_in_syslog.rb +1 -1
- data/test/plugin/test_in_tail.rb +629 -353
- data/test/plugin/test_out_forward.rb +30 -20
- data/test/plugin/test_out_stdout.rb +2 -2
- data/test/plugin/test_output_as_buffered_retries.rb +53 -6
- data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
- data/test/plugin/test_parser_syslog.rb +1 -1
- data/test/plugin_helper/test_cert_option.rb +1 -1
- data/test/plugin_helper/test_child_process.rb +16 -4
- data/test/plugin_helper/test_retry_state.rb +602 -38
- data/test/plugin_helper/test_server.rb +18 -0
- data/test/test_config.rb +135 -4
- data/test/test_event_router.rb +17 -0
- data/test/test_formatter.rb +1 -1
- data/test/test_supervisor.rb +196 -6
- metadata +39 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 941e114cec61513919b4fbd711bfc2bb58254a75d8f5f76028027f432cc89116
|
|
4
|
+
data.tar.gz: 8ec3c41476caf98e3a82ef624786d1fb9900361480326476141d8a0eb58c19d4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 923a7cc3e4449082c7e9de20e6fb39cd9d2e42f481bece900e779a8a711d961b81c5fe38ff34f907473ef8a35e9ddff1d33d6e9b42d63377c147ddd88c9ba368
|
|
7
|
+
data.tar.gz: 2f4e5a26e4ab6d82784aad47784c3371c0b59920222301d3ef5ec105807e6bcd58c6153d2af0316842b1f62ca9f68aaa7c7f811f0b8b8d54e1d7bbef101ed1a4
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
blank_issues_enabled: false
|
|
2
2
|
contact_links:
|
|
3
3
|
- name: Ask a Question
|
|
4
|
-
url: https://
|
|
5
|
-
about: I have questions about Fluentd and plugins. Please ask and answer questions at https://
|
|
4
|
+
url: https://github.com/fluent/fluentd/discussions
|
|
5
|
+
about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions
|
|
@@ -13,9 +13,13 @@ jobs:
|
|
|
13
13
|
strategy:
|
|
14
14
|
fail-fast: false
|
|
15
15
|
matrix:
|
|
16
|
-
ruby-version: ['
|
|
16
|
+
ruby-version: ['3.1', '3.0', '2.7']
|
|
17
17
|
os: [macos-latest]
|
|
18
18
|
experimental: [true]
|
|
19
|
+
include:
|
|
20
|
+
- ruby-version: head
|
|
21
|
+
os: macos-latest
|
|
22
|
+
experimental: true
|
|
19
23
|
|
|
20
24
|
name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
|
|
21
25
|
steps:
|
|
@@ -13,14 +13,17 @@ jobs:
|
|
|
13
13
|
strategy:
|
|
14
14
|
fail-fast: false
|
|
15
15
|
matrix:
|
|
16
|
-
ruby-version: ['
|
|
16
|
+
ruby-version: ['3.1', '2.7']
|
|
17
17
|
os:
|
|
18
18
|
- windows-latest
|
|
19
19
|
experimental: [false]
|
|
20
20
|
include:
|
|
21
|
-
- ruby-version:
|
|
21
|
+
- ruby-version: head
|
|
22
22
|
os: windows-latest
|
|
23
23
|
experimental: true
|
|
24
|
+
- ruby-version: '3.0.3'
|
|
25
|
+
os: windows-latest
|
|
26
|
+
experimental: false
|
|
24
27
|
# On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
|
|
25
28
|
# error code. In addition, we have to specify the path of fiddle by RUBYLIB
|
|
26
29
|
# because RubyInstaller loads Ruby's bundled fiddle before initializing gem.
|
|
@@ -28,7 +31,7 @@ jobs:
|
|
|
28
31
|
# * https://github.com/ruby/fiddle/issues/72
|
|
29
32
|
# * https://bugs.ruby-lang.org/issues/17813
|
|
30
33
|
# * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
|
|
31
|
-
ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.
|
|
34
|
+
ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.3/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.1.0/lib
|
|
32
35
|
|
|
33
36
|
name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
|
|
34
37
|
steps:
|
|
@@ -37,9 +40,9 @@ jobs:
|
|
|
37
40
|
uses: ruby/setup-ruby@v1
|
|
38
41
|
with:
|
|
39
42
|
ruby-version: ${{ matrix.ruby-version }}
|
|
40
|
-
- name: Add Fiddle 1.0
|
|
41
|
-
if: ${{ matrix.ruby-version == '3.0.
|
|
42
|
-
run: gem install fiddle --version 1.0
|
|
43
|
+
- name: Add Fiddle 1.1.0
|
|
44
|
+
if: ${{ matrix.ruby-version == '3.0.3' }}
|
|
45
|
+
run: gem install fiddle --version 1.1.0
|
|
43
46
|
- name: Install dependencies
|
|
44
47
|
run: ridk exec bundle install
|
|
45
48
|
- name: Run tests
|
data/CHANGELOG.md
CHANGED
|
@@ -1,4 +1,114 @@
|
|
|
1
|
-
# v1.
|
|
1
|
+
# v1.15
|
|
2
|
+
|
|
3
|
+
## Release v1.15.0 - 2022/06/29
|
|
4
|
+
|
|
5
|
+
### Enhancement
|
|
6
|
+
|
|
7
|
+
* in_tail: Add log throttling in files based on group rules
|
|
8
|
+
https://github.com/fluent/fluentd/pull/3535
|
|
9
|
+
https://github.com/fluent/fluentd/pull/3771
|
|
10
|
+
* Add `dump` command to fluent-ctl
|
|
11
|
+
https://github.com/fluent/fluentd/pull/3680
|
|
12
|
+
* Handle YAML configuration format on configuration file
|
|
13
|
+
https://github.com/fluent/fluentd/pull/3712
|
|
14
|
+
* Add `restart_worker_interval` parameter in `<system>` directive to set
|
|
15
|
+
interval to restart workers that has stopped for some reason.
|
|
16
|
+
https://github.com/fluent/fluentd/pull/3768
|
|
17
|
+
|
|
18
|
+
### Bug fixes
|
|
19
|
+
|
|
20
|
+
* out_forward: Fix to update timeout of cached sockets
|
|
21
|
+
https://github.com/fluent/fluentd/pull/3711
|
|
22
|
+
* in_tail: Fix a possible crash on file rotation when `follow_inodes true`
|
|
23
|
+
https://github.com/fluent/fluentd/pull/3754
|
|
24
|
+
* output: Fix a possible crash of flush thread
|
|
25
|
+
https://github.com/fluent/fluentd/pull/3755
|
|
26
|
+
* in_tail: Fix crash bugs on Ruby 3.1 on Windows
|
|
27
|
+
https://github.com/fluent/fluentd/pull/3766
|
|
28
|
+
* in_tail: Fix a bug that in_tail cannot open non-ascii path on Windows
|
|
29
|
+
https://github.com/fluent/fluentd/pull/3774
|
|
30
|
+
* Fix a bug that fluentd doesn't release its own log file even after rotated by
|
|
31
|
+
external tools
|
|
32
|
+
https://github.com/fluent/fluentd/pull/3782
|
|
33
|
+
|
|
34
|
+
### Misc
|
|
35
|
+
|
|
36
|
+
* in_tail: Simplify TargetInfo related code
|
|
37
|
+
https://github.com/fluent/fluentd/pull/3489
|
|
38
|
+
* Fix a wrong issue number in CHANGELOG
|
|
39
|
+
https://github.com/fluent/fluentd/pull/3700
|
|
40
|
+
* server helper: Add comments to linger_timeout behavior about Windows
|
|
41
|
+
https://github.com/fluent/fluentd/pull/3701
|
|
42
|
+
* service_discovery: Fix typo
|
|
43
|
+
https://github.com/fluent/fluentd/pull/3724
|
|
44
|
+
* test: Fix unstable tests and warnings
|
|
45
|
+
https://github.com/fluent/fluentd/pull/3745
|
|
46
|
+
https://github.com/fluent/fluentd/pull/3753
|
|
47
|
+
https://github.com/fluent/fluentd/pull/3767
|
|
48
|
+
https://github.com/fluent/fluentd/pull/3783
|
|
49
|
+
https://github.com/fluent/fluentd/pull/3784
|
|
50
|
+
https://github.com/fluent/fluentd/pull/3785
|
|
51
|
+
https://github.com/fluent/fluentd/pull/3787
|
|
52
|
+
|
|
53
|
+
# v1.14
|
|
54
|
+
|
|
55
|
+
## Release v1.14.6 - 2022/03/31
|
|
56
|
+
|
|
57
|
+
### Enhancement
|
|
58
|
+
|
|
59
|
+
* Enable server plugins to specify socket-option `SO_LINGER`
|
|
60
|
+
https://github.com/fluent/fluentd/pull/3644
|
|
61
|
+
* Add `--umask` command line parameter
|
|
62
|
+
https://github.com/fluent/fluentd/pull/3671
|
|
63
|
+
https://github.com/fluent/fluentd/pull/3679
|
|
64
|
+
|
|
65
|
+
### Bug fixes
|
|
66
|
+
|
|
67
|
+
* Fix metric name typo
|
|
68
|
+
https://github.com/fluent/fluentd/pull/3630
|
|
69
|
+
https://github.com/fluent/fluentd/pull/3673
|
|
70
|
+
* Apply modifications in pipeline to the records being passed to `@ERROR` label
|
|
71
|
+
https://github.com/fluent/fluentd/pull/3631
|
|
72
|
+
* Fix wrong calculation of retry interval
|
|
73
|
+
https://github.com/fluent/fluentd/pull/3640
|
|
74
|
+
https://github.com/fluent/fluentd/pull/3649
|
|
75
|
+
https://github.com/fluent/fluentd/pull/3685
|
|
76
|
+
https://github.com/fluent/fluentd/pull/3686
|
|
77
|
+
* Support IPv6 address for `rpc_endpoint` in `system` config
|
|
78
|
+
https://github.com/fluent/fluentd/pull/3641
|
|
79
|
+
|
|
80
|
+
### Misc
|
|
81
|
+
|
|
82
|
+
* CI: Support Ruby 3.1 except Windows
|
|
83
|
+
https://github.com/fluent/fluentd/pull/3619
|
|
84
|
+
* Switch to GitHub Discussions
|
|
85
|
+
https://github.com/fluent/fluentd/pull/3654
|
|
86
|
+
* Fix CHANGELOG.md heading styles
|
|
87
|
+
https://github.com/fluent/fluentd/pull/3648
|
|
88
|
+
* Declare `null_value_pattern` as `regexp`
|
|
89
|
+
https://github.com/fluent/fluentd/pull/3650
|
|
90
|
+
|
|
91
|
+
## Release v1.14.5 - 2022/02/09
|
|
92
|
+
|
|
93
|
+
### Enhancement
|
|
94
|
+
|
|
95
|
+
* Add support for "application/x-ndjson" to `in_http`
|
|
96
|
+
https://github.com/fluent/fluentd/pull/3616
|
|
97
|
+
* Add support for ucrt binary for Windows
|
|
98
|
+
https://github.com/fluent/fluentd/pull/3613
|
|
99
|
+
|
|
100
|
+
### Bug fixes
|
|
101
|
+
|
|
102
|
+
* Don't retry when `retry_max_times == 0`
|
|
103
|
+
https://github.com/fluent/fluentd/pull/3608
|
|
104
|
+
* Fix hang-up issue during TLS handshake in `out_forward`
|
|
105
|
+
https://github.com/fluent/fluentd/pull/3601
|
|
106
|
+
* Bump up required ServerEngine to v2.2.5
|
|
107
|
+
https://github.com/fluent/fluentd/pull/3599
|
|
108
|
+
* Fix "invalid byte sequence is replaced" warning on Kubernetes
|
|
109
|
+
https://github.com/fluent/fluentd/pull/3596
|
|
110
|
+
* Fix "ArgumentError: unknown keyword: :logger" on Windows with Ruby 3.1
|
|
111
|
+
https://github.com/fluent/fluentd/pull/3592
|
|
2
112
|
|
|
3
113
|
## Release v1.14.4 - 2022/01/06
|
|
4
114
|
|
|
@@ -17,8 +127,6 @@
|
|
|
17
127
|
https://github.com/fluent/fluentd/pull/3574
|
|
18
128
|
https://github.com/fluent/fluentd/pull/3577
|
|
19
129
|
|
|
20
|
-
# v1.14.3
|
|
21
|
-
|
|
22
130
|
## Release v1.14.3 - 2021/11/26
|
|
23
131
|
|
|
24
132
|
### Enhancement
|
|
@@ -47,8 +155,6 @@
|
|
|
47
155
|
newer version is required to implement additional `fluent-ctl` commands.
|
|
48
156
|
https://github.com/fluent/fluentd/pull/3556
|
|
49
157
|
|
|
50
|
-
# v1.14.2
|
|
51
|
-
|
|
52
158
|
## Release v1.14.2 - 2021/10/29
|
|
53
159
|
|
|
54
160
|
IMPORTANT: This release contain the fix for CVE-2021-41186 -
|
|
@@ -60,6 +166,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
60
166
|
### Enhancement
|
|
61
167
|
|
|
62
168
|
* fluent-cat: Add `--event-time` option to send specified event time for testing.
|
|
169
|
+
https://github.com/fluent/fluentd/pull/3528
|
|
63
170
|
|
|
64
171
|
### Bug fix
|
|
65
172
|
|
|
@@ -68,8 +175,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
68
175
|
* Fixed ReDoS vulnerability in parser_apache2.
|
|
69
176
|
This vulnerability is caused by a certain pattern of a broken apache log.
|
|
70
177
|
|
|
71
|
-
# v1.14.1
|
|
72
|
-
|
|
73
178
|
## Release v1.14.1 - 2021/09/29
|
|
74
179
|
|
|
75
180
|
### Enhancement
|
|
@@ -99,8 +204,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
99
204
|
next_retry_seconds -> next_retry_time)
|
|
100
205
|
https://github.com/fluent/fluentd/pull/3518
|
|
101
206
|
|
|
102
|
-
# v1.14.0
|
|
103
|
-
|
|
104
207
|
## Release v1.14.0 - 2021/08/30
|
|
105
208
|
|
|
106
209
|
### Enhancement
|
|
@@ -143,7 +246,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
143
246
|
https://github.com/fluent/fluentd/pull/3479
|
|
144
247
|
https://github.com/fluent/fluentd/pull/3484
|
|
145
248
|
|
|
146
|
-
# v1.13
|
|
249
|
+
# v1.13
|
|
147
250
|
|
|
148
251
|
## Release v1.13.3 - 2021/07/27
|
|
149
252
|
|
|
@@ -165,10 +268,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
165
268
|
* Remove needless spaces in a sample config file
|
|
166
269
|
https://github.com/fluent/fluentd/pull/3456
|
|
167
270
|
|
|
168
|
-
### Enhancement
|
|
169
|
-
|
|
170
|
-
# v1.13.2
|
|
171
|
-
|
|
172
271
|
## Release v1.13.2 - 2021/07/12
|
|
173
272
|
|
|
174
273
|
### Enhancement
|
|
@@ -204,8 +303,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
204
303
|
by security scanning tools.
|
|
205
304
|
https://github.com/fluent/fluentd/pull/3450
|
|
206
305
|
|
|
207
|
-
# v1.13.1
|
|
208
|
-
|
|
209
306
|
## Release v1.13.1 - 2021/06/25
|
|
210
307
|
|
|
211
308
|
### Bug fix
|
|
@@ -230,8 +327,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
230
327
|
* CI: Dropped to run CI for Ruby 2.5
|
|
231
328
|
https://github.com/fluent/fluentd/pull/3412
|
|
232
329
|
|
|
233
|
-
# v1.13
|
|
234
|
-
|
|
235
330
|
## Release v1.13.0 - 2021/05/29
|
|
236
331
|
|
|
237
332
|
### Enhancement
|
|
@@ -279,6 +374,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
279
374
|
### Bug fix
|
|
280
375
|
|
|
281
376
|
* in_tail: Fix a bug that refresh_watcher fails to handle file rotations
|
|
377
|
+
https://github.com/fluent/fluentd/pull/3393
|
|
282
378
|
|
|
283
379
|
## Release v1.12.3 - 2021/04/23
|
|
284
380
|
|
|
@@ -399,7 +495,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
|
399
495
|
### New feature
|
|
400
496
|
|
|
401
497
|
* in_tail: Add `follow_inode` to support log rotation with wild card
|
|
402
|
-
https://github.com/fluent/fluentd/pull/
|
|
498
|
+
https://github.com/fluent/fluentd/pull/3182
|
|
403
499
|
* in_tail: Handle linux capability
|
|
404
500
|
https://github.com/fluent/fluentd/pull/3155
|
|
405
501
|
https://github.com/fluent/fluentd/pull/3162
|
data/CONTRIBUTING.md
CHANGED
|
@@ -6,7 +6,7 @@ We'd love your contribution. Here are the guidelines!
|
|
|
6
6
|
|
|
7
7
|
RESOURCES of [Official site](https://www.fluentd.org/) and [Fluentd documentation](https://docs.fluentd.org/) may help you.
|
|
8
8
|
|
|
9
|
-
If you have further questions about Fluentd and plugins, please direct these to [Community Forum](https://
|
|
9
|
+
If you have further questions about Fluentd and plugins, please direct these to [Community Forum](https://github.com/fluent/fluentd/discussions).
|
|
10
10
|
Don't use Github issue for asking questions. Here are examples:
|
|
11
11
|
|
|
12
12
|
- I installed xxx plugin but it doesn't work. Why?
|
data/MAINTAINERS.md
CHANGED
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
- [Naotoshi Seo](https://github.com/sonots), [ZOZO Technologies](https://tech.zozo.com/en/)
|
|
4
4
|
- [Okkez](https://github.com/okkez)
|
|
5
|
-
- [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://
|
|
5
|
+
- [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://calyptia.com/)
|
|
6
6
|
- [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)
|
|
7
7
|
- [Satoshi Tagomori](https://github.com/tagomoris), [Treasure Data](https://www.treasuredata.com/)
|
|
8
|
-
- [Eduardo Silva](https://github.com/edsiper), [
|
|
8
|
+
- [Eduardo Silva](https://github.com/edsiper), [Calyptia](https://calyptia/)
|
|
9
9
|
- [Fujimoto Seiji](https://github.com/fujimots), [ClearCode](https://www.clear-code.com/)
|
|
10
10
|
- [Takuro Ashie](https://github.com/ashie), [ClearCode](https://www.clear-code.com/)
|
|
11
11
|
- [Kentaro Hayashi](https://github.com/kenhys), [ClearCode](https://www.clear-code.com/)
|
data/README.md
CHANGED
|
@@ -1,17 +1,12 @@
|
|
|
1
1
|
Fluentd: Open-Source Log Collector
|
|
2
2
|
===================================
|
|
3
|
-
GitHub Actions:
|
|
4
3
|
|
|
5
4
|
[](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml)
|
|
6
5
|
[](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml)
|
|
7
6
|
[](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
|
|
7
|
+
[](https://cloud.drone.io/fluent/fluentd)
|
|
8
8
|
[](https://codeclimate.com/github/fluent/fluentd)
|
|
9
9
|
[](https://bestpractices.coreinfrastructure.org/projects/1189)
|
|
10
|
-
[](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
|
|
11
|
-
|
|
12
|
-
Drone CI for Arm64:
|
|
13
|
-
|
|
14
|
-
[](https://cloud.drone.io/fluent/fluentd)
|
|
15
10
|
|
|
16
11
|
[Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
|
|
17
12
|
|
|
@@ -19,16 +14,6 @@ Drone CI for Arm64:
|
|
|
19
14
|
<img src="https://www.fluentd.org/images/fluentd-architecture.png" width="500px"/>
|
|
20
15
|
</p>
|
|
21
16
|
|
|
22
|
-
An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time recorded at occurrence of an event. Record is a JSON object.
|
|
23
|
-
|
|
24
|
-
## Example Use Cases
|
|
25
|
-
|
|
26
|
-
Use Case | Description | Diagram
|
|
27
|
-
-------- | ------------|:---------:
|
|
28
|
-
Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/images/recipes/elasticsearch-s3-fluentd.png" height="150"/>
|
|
29
|
-
Syslog Alerting | Fluentd can "grep" for events and send out alerts. | <img src="https://www.fluentd.org/images/syslog-fluentd-alert.png" height="100"/>
|
|
30
|
-
Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/images/datasources/asynchronous_logging.png" height="150"/>
|
|
31
|
-
|
|
32
17
|
## Quick Start
|
|
33
18
|
|
|
34
19
|
$ gem install fluentd
|
|
@@ -66,18 +51,12 @@ You can run specified test via `TEST` environment variable:
|
|
|
66
51
|
$ bundle exec rake test TEST=test/test_specified_path.rb
|
|
67
52
|
$ bundle exec rake test TEST=test/test_*.rb
|
|
68
53
|
|
|
69
|
-
## Fluentd UI: Admin GUI
|
|
70
|
-
|
|
71
|
-
[Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.
|
|
72
|
-
|
|
73
|
-
<p align="center"><img width="500" src="https://www.fluentd.org/images/blog/fluentd-ui.gif"/></p>
|
|
74
|
-
|
|
75
54
|
## More Information
|
|
76
55
|
|
|
77
56
|
- Website: https://www.fluentd.org/
|
|
78
57
|
- Documentation: https://docs.fluentd.org/
|
|
79
58
|
- Project repository: https://github.com/fluent
|
|
80
|
-
- Discussion: https://
|
|
59
|
+
- Discussion: https://github.com/fluent/fluentd/discussions
|
|
81
60
|
- Slack / Community: https://slack.fluentd.org
|
|
82
61
|
- Newsletters: https://www.fluentd.org/newsletter
|
|
83
62
|
- Author: [Sadayuki Furuhashi](https://github.com/frsyuki)
|
data/Rakefile
CHANGED
|
@@ -14,7 +14,7 @@ namespace :build do
|
|
|
14
14
|
desc 'Build gems for all platforms'
|
|
15
15
|
task :all do
|
|
16
16
|
Bundler.with_clean_env do
|
|
17
|
-
%w[ruby x86-mingw32 x64-mingw32].each do |name|
|
|
17
|
+
%w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
|
|
18
18
|
ENV['GEM_BUILD_FAKE_PLATFORM'] = name
|
|
19
19
|
Rake::Task["build"].execute
|
|
20
20
|
end
|
data/fluentd.gemspec
CHANGED
|
@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
|
|
|
22
22
|
gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
|
|
23
23
|
gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
|
|
24
24
|
gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
|
|
25
|
-
gem.add_runtime_dependency("serverengine", [">= 2.
|
|
25
|
+
gem.add_runtime_dependency("serverengine", [">= 2.3.0", "< 3.0.0"])
|
|
26
26
|
gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
|
|
27
27
|
gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
|
|
28
28
|
gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
|
|
@@ -38,6 +38,7 @@ Gem::Specification.new do |gem|
|
|
|
38
38
|
gem.add_runtime_dependency("win32-service", ["~> 2.3.0"])
|
|
39
39
|
gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
|
|
40
40
|
gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
|
|
41
|
+
gem.add_runtime_dependency("windows-api", ["~> 0.4.5"])
|
|
41
42
|
gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
|
|
42
43
|
gem.add_runtime_dependency("certstore_c", ["~> 0.1.7"])
|
|
43
44
|
end
|
|
@@ -51,5 +52,6 @@ Gem::Specification.new do |gem|
|
|
|
51
52
|
gem.add_development_dependency("test-unit", ["~> 3.3"])
|
|
52
53
|
gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
|
|
53
54
|
gem.add_development_dependency("oj", [">= 2.14", "< 4"])
|
|
55
|
+
gem.add_development_dependency("async", "~> 1.23")
|
|
54
56
|
gem.add_development_dependency("async-http", ">= 0.50.0")
|
|
55
57
|
end
|
data/lib/fluent/command/ctl.rb
CHANGED
|
@@ -36,6 +36,7 @@ module Fluent
|
|
|
36
36
|
restart: "HUP",
|
|
37
37
|
flush: "USR1",
|
|
38
38
|
reload: "USR2",
|
|
39
|
+
dump: "CONT",
|
|
39
40
|
}
|
|
40
41
|
WINSVC_CONTROL_CODE_MAP = {
|
|
41
42
|
shutdown: SERVICE_CONTROL_STOP,
|
|
@@ -44,6 +45,7 @@ module Fluent
|
|
|
44
45
|
restart: 128,
|
|
45
46
|
flush: 129,
|
|
46
47
|
reload: SERVICE_CONTROL_PARAMCHANGE,
|
|
48
|
+
dump: 130,
|
|
47
49
|
}
|
|
48
50
|
else
|
|
49
51
|
COMMAND_MAP = {
|
|
@@ -51,6 +53,7 @@ module Fluent
|
|
|
51
53
|
restart: :HUP,
|
|
52
54
|
flush: :USR1,
|
|
53
55
|
reload: :USR2,
|
|
56
|
+
dump: :CONT,
|
|
54
57
|
}
|
|
55
58
|
end
|
|
56
59
|
|
|
@@ -146,7 +149,7 @@ module Fluent
|
|
|
146
149
|
event = Win32::Event.open("#{prefix}#{suffix}")
|
|
147
150
|
event.set
|
|
148
151
|
event.close
|
|
149
|
-
rescue Errno::ENOENT
|
|
152
|
+
rescue Errno::ENOENT
|
|
150
153
|
puts "Error: Cannot find the fluentd process with the event name: \"#{prefix}\""
|
|
151
154
|
end
|
|
152
155
|
end
|
|
@@ -81,6 +81,10 @@ op.on('--group GROUP', "change group") {|s|
|
|
|
81
81
|
opts[:chgroup] = s
|
|
82
82
|
}
|
|
83
83
|
|
|
84
|
+
op.on('--umask UMASK', "change umask") {|s|
|
|
85
|
+
opts[:chumask] = s
|
|
86
|
+
}
|
|
87
|
+
|
|
84
88
|
op.on('-o', '--log PATH', "log file path") {|s|
|
|
85
89
|
opts[:log_path] = s
|
|
86
90
|
}
|
|
@@ -122,6 +126,16 @@ op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {
|
|
|
122
126
|
opts[:without_source] = b
|
|
123
127
|
}
|
|
124
128
|
|
|
129
|
+
op.on('--config-file-type VALU', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s|
|
|
130
|
+
if (s == 'yaml') || (s == 'yml')
|
|
131
|
+
opts[:config_file_type] = s.to_sym
|
|
132
|
+
elsif (s == 'guess')
|
|
133
|
+
opts[:config_file_type] = s.to_sym
|
|
134
|
+
else
|
|
135
|
+
usage '--config-file-type accepts yaml/yml or guess'
|
|
136
|
+
end
|
|
137
|
+
}
|
|
138
|
+
|
|
125
139
|
op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b|
|
|
126
140
|
opts[:use_v1_config] = b
|
|
127
141
|
}
|
data/lib/fluent/config/error.rb
CHANGED
|
@@ -29,4 +29,16 @@ module Fluent
|
|
|
29
29
|
|
|
30
30
|
class SetDefault < Exception
|
|
31
31
|
end
|
|
32
|
+
|
|
33
|
+
class NotFoundPluginError < ConfigError
|
|
34
|
+
attr_reader :type, :kind
|
|
35
|
+
|
|
36
|
+
def initialize(msg, type: nil, kind: nil)
|
|
37
|
+
@msg = msg
|
|
38
|
+
@type = type
|
|
39
|
+
@kind = kind
|
|
40
|
+
|
|
41
|
+
super(msg)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
32
44
|
end
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Fluentd
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
module Fluent
|
|
18
|
+
module Config
|
|
19
|
+
module YamlParser
|
|
20
|
+
module FluentValue
|
|
21
|
+
JsonValue = Struct.new(:val) do
|
|
22
|
+
def to_s
|
|
23
|
+
val.to_json
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def to_element
|
|
27
|
+
to_s
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
StringValue = Struct.new(:val, :context) do
|
|
32
|
+
def to_s
|
|
33
|
+
context.instance_eval("\"#{val}\"")
|
|
34
|
+
rescue Fluent::SetNil => _
|
|
35
|
+
''
|
|
36
|
+
rescue Fluent::SetDefault => _
|
|
37
|
+
':default'
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def to_element
|
|
41
|
+
to_s
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Fluentd
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
#
|
|
16
|
+
|
|
17
|
+
require 'psych'
|
|
18
|
+
require 'json'
|
|
19
|
+
require 'fluent/config/error'
|
|
20
|
+
require 'fluent/config/yaml_parser/fluent_value'
|
|
21
|
+
|
|
22
|
+
# Based on https://github.com/eagletmt/hako/blob/34cdde06fe8f3aeafd794be830180c3cedfbb4dc/lib/hako/yaml_loader.rb
|
|
23
|
+
|
|
24
|
+
module Fluent
|
|
25
|
+
module Config
|
|
26
|
+
module YamlParser
|
|
27
|
+
class Loader
|
|
28
|
+
INCLUDE_TAG = 'tag:include'.freeze
|
|
29
|
+
FLUENT_JSON_TAG = 'tag:fluent/json'.freeze
|
|
30
|
+
FLUENT_STR_TAG = 'tag:fluent/s'.freeze
|
|
31
|
+
SHOVEL = '<<'.freeze
|
|
32
|
+
|
|
33
|
+
def initialize(context = Kernel.binding)
|
|
34
|
+
@context = context
|
|
35
|
+
@current_path = nil
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @param [String] path
|
|
39
|
+
# @return [Hash]
|
|
40
|
+
def load(path)
|
|
41
|
+
class_loader = Psych::ClassLoader.new
|
|
42
|
+
scanner = Psych::ScalarScanner.new(class_loader)
|
|
43
|
+
|
|
44
|
+
visitor = Visitor.new(scanner, class_loader)
|
|
45
|
+
|
|
46
|
+
visitor._register_domain(INCLUDE_TAG) do |_, val|
|
|
47
|
+
load(path.parent.join(val))
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
visitor._register_domain(FLUENT_JSON_TAG) do |_, val|
|
|
51
|
+
Fluent::Config::YamlParser::FluentValue::JsonValue.new(val)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
visitor._register_domain(FLUENT_STR_TAG) do |_, val|
|
|
55
|
+
Fluent::Config::YamlParser::FluentValue::StringValue.new(val, @context)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
path.open do |f|
|
|
59
|
+
visitor.accept(Psych.parse(f))
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
class Visitor < Psych::Visitors::ToRuby
|
|
64
|
+
def initialize(scanner, class_loader)
|
|
65
|
+
super(scanner, class_loader)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def _register_domain(name, &block)
|
|
69
|
+
@domain_types.merge!({ name => [name, block] })
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def revive_hash(hash, o)
|
|
73
|
+
super(hash, o).tap do |r|
|
|
74
|
+
if r[SHOVEL].is_a?(Hash)
|
|
75
|
+
h2 = {}
|
|
76
|
+
r.each do |k, v|
|
|
77
|
+
if k == SHOVEL
|
|
78
|
+
h2.merge!(v)
|
|
79
|
+
else
|
|
80
|
+
h2[k] = v
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
r.replace(h2)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|