fluentd 1.14.5 → 1.14.6
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/config.yml +2 -2
- data/.github/workflows/linux-test.yaml +1 -1
- data/.github/workflows/macos-test.yaml +5 -1
- data/.github/workflows/windows-test.yaml +6 -6
- data/CHANGELOG.md +40 -20
- data/CONTRIBUTING.md +1 -1
- data/MAINTAINERS.md +2 -2
- data/README.md +1 -1
- data/lib/fluent/command/fluentd.rb +4 -0
- data/lib/fluent/event_router.rb +19 -1
- data/lib/fluent/plugin/bare_output.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/output.rb +41 -36
- data/lib/fluent/plugin/parser.rb +3 -4
- data/lib/fluent/plugin_helper/retry_state.rb +14 -4
- data/lib/fluent/plugin_helper/server.rb +21 -4
- data/lib/fluent/rpc.rb +4 -3
- data/lib/fluent/supervisor.rb +5 -2
- data/lib/fluent/version.rb +1 -1
- data/test/plugin/test_in_forward.rb +0 -2
- data/test/plugin/test_output_as_buffered_retries.rb +7 -7
- data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
- data/test/plugin_helper/test_retry_state.rb +602 -38
- data/test/plugin_helper/test_server.rb +18 -0
- data/test/test_event_router.rb +17 -0
- data/test/test_supervisor.rb +41 -6
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4c1188a4d037f788a1fbeec731c7117fef237cd4036cd5a5e5b891bba297d184
|
4
|
+
data.tar.gz: f3a1b383fd8d1ee4e5b431c92c83007b58c26f8d91bf1c9b0ce911662935c4e7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9816373aac361da5cbd010e0720982f3f09ab59846e53ade46eccd225b7fc0201f8d02ddb009f3ffb813cfb41d92bd3360edbb7f5ea55ac644f46ad678521d21
|
7
|
+
data.tar.gz: d92e47b57424f825252d3fcc3204812d2747e634d98762a7843a6d68f1386258fc6302793311d21d82c3801e5f71f0fc30a4af1dc5464e2411a4d65b652b960b
|
@@ -1,5 +1,5 @@
|
|
1
1
|
blank_issues_enabled: false
|
2
2
|
contact_links:
|
3
3
|
- name: Ask a Question
|
4
|
-
url: https://
|
5
|
-
about: I have questions about Fluentd and plugins. Please ask and answer questions at https://
|
4
|
+
url: https://github.com/fluent/fluentd/discussions
|
5
|
+
about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions
|
@@ -13,9 +13,13 @@ jobs:
|
|
13
13
|
strategy:
|
14
14
|
fail-fast: false
|
15
15
|
matrix:
|
16
|
-
ruby-version: ['
|
16
|
+
ruby-version: ['3.1', '3.0', '2.7']
|
17
17
|
os: [macos-latest]
|
18
18
|
experimental: [true]
|
19
|
+
include:
|
20
|
+
- ruby-version: head
|
21
|
+
os: macos-latest
|
22
|
+
experimental: true
|
19
23
|
|
20
24
|
name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
|
21
25
|
steps:
|
@@ -18,9 +18,9 @@ jobs:
|
|
18
18
|
- windows-latest
|
19
19
|
experimental: [false]
|
20
20
|
include:
|
21
|
-
- ruby-version: '3.0.
|
21
|
+
- ruby-version: '3.0.3'
|
22
22
|
os: windows-latest
|
23
|
-
experimental:
|
23
|
+
experimental: false
|
24
24
|
# On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
|
25
25
|
# error code. In addition, we have to specify the path of fiddle by RUBYLIB
|
26
26
|
# because RubyInstaller loads Ruby's bundled fiddle before initializing gem.
|
@@ -28,7 +28,7 @@ jobs:
|
|
28
28
|
# * https://github.com/ruby/fiddle/issues/72
|
29
29
|
# * https://bugs.ruby-lang.org/issues/17813
|
30
30
|
# * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
|
31
|
-
ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.
|
31
|
+
ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.3/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.1.0/lib
|
32
32
|
|
33
33
|
name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
|
34
34
|
steps:
|
@@ -37,9 +37,9 @@ jobs:
|
|
37
37
|
uses: ruby/setup-ruby@v1
|
38
38
|
with:
|
39
39
|
ruby-version: ${{ matrix.ruby-version }}
|
40
|
-
- name: Add Fiddle 1.0
|
41
|
-
if: ${{ matrix.ruby-version == '3.0.
|
42
|
-
run: gem install fiddle --version 1.0
|
40
|
+
- name: Add Fiddle 1.1.0
|
41
|
+
if: ${{ matrix.ruby-version == '3.0.3' }}
|
42
|
+
run: gem install fiddle --version 1.1.0
|
43
43
|
- name: Install dependencies
|
44
44
|
run: ridk exec bundle install
|
45
45
|
- name: Run tests
|
data/CHANGELOG.md
CHANGED
@@ -1,4 +1,40 @@
|
|
1
|
-
# v1.14
|
1
|
+
# v1.14
|
2
|
+
|
3
|
+
## Release v1.14.6 - 2022/03/31
|
4
|
+
|
5
|
+
### Enhancement
|
6
|
+
|
7
|
+
* Enable server plugins to specify socket-option `SO_LINGER`
|
8
|
+
https://github.com/fluent/fluentd/pull/3644
|
9
|
+
* Add `--umask` command line parameter
|
10
|
+
https://github.com/fluent/fluentd/pull/3671
|
11
|
+
https://github.com/fluent/fluentd/pull/3679
|
12
|
+
|
13
|
+
### Bug fixes
|
14
|
+
|
15
|
+
* Fix metric name typo
|
16
|
+
https://github.com/fluent/fluentd/pull/3630
|
17
|
+
https://github.com/fluent/fluentd/pull/3673
|
18
|
+
* Apply modifications in pipeline to the records being passed to `@ERROR` label
|
19
|
+
https://github.com/fluent/fluentd/pull/3631
|
20
|
+
* Fix wrong calculation of retry interval
|
21
|
+
https://github.com/fluent/fluentd/pull/3640
|
22
|
+
https://github.com/fluent/fluentd/pull/3649
|
23
|
+
https://github.com/fluent/fluentd/pull/3685
|
24
|
+
https://github.com/fluent/fluentd/pull/3686
|
25
|
+
* Support IPv6 address for `rpc_endpoint` in `system` config
|
26
|
+
https://github.com/fluent/fluentd/pull/3641
|
27
|
+
|
28
|
+
### Misc
|
29
|
+
|
30
|
+
* CI: Support Ruby 3.1 except Windows
|
31
|
+
https://github.com/fluent/fluentd/pull/3619
|
32
|
+
* Switch to GitHub Discussions
|
33
|
+
https://github.com/fluent/fluentd/pull/3654
|
34
|
+
* Fix CHANGELOG.md heading styles
|
35
|
+
https://github.com/fluent/fluentd/pull/3648
|
36
|
+
* Declare `null_value_pattern` as `regexp`
|
37
|
+
https://github.com/fluent/fluentd/pull/3650
|
2
38
|
|
3
39
|
## Release v1.14.5 - 2022/02/09
|
4
40
|
|
@@ -22,8 +58,6 @@
|
|
22
58
|
* Fix "ArgumentError: unknown keyword: :logger" on Windows with Ruby 3.1
|
23
59
|
https://github.com/fluent/fluentd/pull/3592
|
24
60
|
|
25
|
-
# v1.14.4
|
26
|
-
|
27
61
|
## Release v1.14.4 - 2022/01/06
|
28
62
|
|
29
63
|
### Enhancement
|
@@ -41,8 +75,6 @@
|
|
41
75
|
https://github.com/fluent/fluentd/pull/3574
|
42
76
|
https://github.com/fluent/fluentd/pull/3577
|
43
77
|
|
44
|
-
# v1.14.3
|
45
|
-
|
46
78
|
## Release v1.14.3 - 2021/11/26
|
47
79
|
|
48
80
|
### Enhancement
|
@@ -71,8 +103,6 @@
|
|
71
103
|
newer version is required to implement additional `fluent-ctl` commands.
|
72
104
|
https://github.com/fluent/fluentd/pull/3556
|
73
105
|
|
74
|
-
# v1.14.2
|
75
|
-
|
76
106
|
## Release v1.14.2 - 2021/10/29
|
77
107
|
|
78
108
|
IMPORTANT: This release contain the fix for CVE-2021-41186 -
|
@@ -84,6 +114,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
84
114
|
### Enhancement
|
85
115
|
|
86
116
|
* fluent-cat: Add `--event-time` option to send specified event time for testing.
|
117
|
+
https://github.com/fluent/fluentd/pull/3528
|
87
118
|
|
88
119
|
### Bug fix
|
89
120
|
|
@@ -92,8 +123,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
92
123
|
* Fixed ReDoS vulnerability in parser_apache2.
|
93
124
|
This vulnerability is caused by a certain pattern of a broken apache log.
|
94
125
|
|
95
|
-
# v1.14.1
|
96
|
-
|
97
126
|
## Release v1.14.1 - 2021/09/29
|
98
127
|
|
99
128
|
### Enhancement
|
@@ -123,8 +152,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
123
152
|
next_retry_seconds -> next_retry_time)
|
124
153
|
https://github.com/fluent/fluentd/pull/3518
|
125
154
|
|
126
|
-
# v1.14.0
|
127
|
-
|
128
155
|
## Release v1.14.0 - 2021/08/30
|
129
156
|
|
130
157
|
### Enhancement
|
@@ -167,7 +194,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
167
194
|
https://github.com/fluent/fluentd/pull/3479
|
168
195
|
https://github.com/fluent/fluentd/pull/3484
|
169
196
|
|
170
|
-
# v1.13
|
197
|
+
# v1.13
|
171
198
|
|
172
199
|
## Release v1.13.3 - 2021/07/27
|
173
200
|
|
@@ -189,10 +216,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
189
216
|
* Remove needless spaces in a sample config file
|
190
217
|
https://github.com/fluent/fluentd/pull/3456
|
191
218
|
|
192
|
-
### Enhancement
|
193
|
-
|
194
|
-
# v1.13.2
|
195
|
-
|
196
219
|
## Release v1.13.2 - 2021/07/12
|
197
220
|
|
198
221
|
### Enhancement
|
@@ -228,8 +251,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
228
251
|
by security scanning tools.
|
229
252
|
https://github.com/fluent/fluentd/pull/3450
|
230
253
|
|
231
|
-
# v1.13.1
|
232
|
-
|
233
254
|
## Release v1.13.1 - 2021/06/25
|
234
255
|
|
235
256
|
### Bug fix
|
@@ -254,8 +275,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
254
275
|
* CI: Dropped to run CI for Ruby 2.5
|
255
276
|
https://github.com/fluent/fluentd/pull/3412
|
256
277
|
|
257
|
-
# v1.13
|
258
|
-
|
259
278
|
## Release v1.13.0 - 2021/05/29
|
260
279
|
|
261
280
|
### Enhancement
|
@@ -303,6 +322,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
|
|
303
322
|
### Bug fix
|
304
323
|
|
305
324
|
* in_tail: Fix a bug that refresh_watcher fails to handle file rotations
|
325
|
+
https://github.com/fluent/fluentd/pull/3393
|
306
326
|
|
307
327
|
## Release v1.12.3 - 2021/04/23
|
308
328
|
|
data/CONTRIBUTING.md
CHANGED
@@ -6,7 +6,7 @@ We'd love your contribution. Here are the guidelines!
|
|
6
6
|
|
7
7
|
RESOURCES of [Official site](https://www.fluentd.org/) and [Fluentd documentation](https://docs.fluentd.org/) may help you.
|
8
8
|
|
9
|
-
If you have further questions about Fluentd and plugins, please direct these to [Community Forum](https://
|
9
|
+
If you have further questions about Fluentd and plugins, please direct these to [Community Forum](https://github.com/fluent/fluentd/discussions).
|
10
10
|
Don't use Github issue for asking questions. Here are examples:
|
11
11
|
|
12
12
|
- I installed xxx plugin but it doesn't work. Why?
|
data/MAINTAINERS.md
CHANGED
@@ -2,10 +2,10 @@
|
|
2
2
|
|
3
3
|
- [Naotoshi Seo](https://github.com/sonots), [ZOZO Technologies](https://tech.zozo.com/en/)
|
4
4
|
- [Okkez](https://github.com/okkez)
|
5
|
-
- [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://
|
5
|
+
- [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://calyptia.com/)
|
6
6
|
- [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)
|
7
7
|
- [Satoshi Tagomori](https://github.com/tagomoris), [Treasure Data](https://www.treasuredata.com/)
|
8
|
-
- [Eduardo Silva](https://github.com/edsiper), [
|
8
|
+
- [Eduardo Silva](https://github.com/edsiper), [Calyptia](https://calyptia/)
|
9
9
|
- [Fujimoto Seiji](https://github.com/fujimots), [ClearCode](https://www.clear-code.com/)
|
10
10
|
- [Takuro Ashie](https://github.com/ashie), [ClearCode](https://www.clear-code.com/)
|
11
11
|
- [Kentaro Hayashi](https://github.com/kenhys), [ClearCode](https://www.clear-code.com/)
|
data/README.md
CHANGED
@@ -56,7 +56,7 @@ You can run specified test via `TEST` environment variable:
|
|
56
56
|
- Website: https://www.fluentd.org/
|
57
57
|
- Documentation: https://docs.fluentd.org/
|
58
58
|
- Project repository: https://github.com/fluent
|
59
|
-
- Discussion: https://
|
59
|
+
- Discussion: https://github.com/fluent/fluentd/discussions
|
60
60
|
- Slack / Community: https://slack.fluentd.org
|
61
61
|
- Newsletters: https://www.fluentd.org/newsletter
|
62
62
|
- Author: [Sadayuki Furuhashi](https://github.com/frsyuki)
|
data/lib/fluent/event_router.rb
CHANGED
@@ -116,6 +116,8 @@ module Fluent
|
|
116
116
|
if callback = find_callback
|
117
117
|
callback.call(es)
|
118
118
|
end
|
119
|
+
rescue Pipeline::OutputError => e
|
120
|
+
@emit_error_handler.handle_emits_error(tag, e.processed_es, e.internal_error)
|
119
121
|
rescue => e
|
120
122
|
@emit_error_handler.handle_emits_error(tag, es, e)
|
121
123
|
end
|
@@ -161,6 +163,17 @@ module Fluent
|
|
161
163
|
private
|
162
164
|
|
163
165
|
class Pipeline
|
166
|
+
|
167
|
+
class OutputError < StandardError
|
168
|
+
attr_reader :internal_error
|
169
|
+
attr_reader :processed_es
|
170
|
+
|
171
|
+
def initialize(internal_error, processed_es)
|
172
|
+
@internal_error = internal_error
|
173
|
+
@processed_es = processed_es
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
164
177
|
def initialize
|
165
178
|
@filters = []
|
166
179
|
@output = nil
|
@@ -178,7 +191,12 @@ module Fluent
|
|
178
191
|
|
179
192
|
def emit_events(tag, es)
|
180
193
|
processed = @optimizer.filter_stream(tag, es)
|
181
|
-
|
194
|
+
|
195
|
+
begin
|
196
|
+
@output.emit_events(tag, processed)
|
197
|
+
rescue => e
|
198
|
+
raise OutputError.new(e, processed)
|
199
|
+
end
|
182
200
|
end
|
183
201
|
|
184
202
|
class FilterOptimizer
|
@@ -70,7 +70,7 @@ module Fluent
|
|
70
70
|
super
|
71
71
|
|
72
72
|
@num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "num_errors", help_text: "Number of count num errors")
|
73
|
-
@emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "
|
73
|
+
@emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_count", help_text: "Number of count emits")
|
74
74
|
@emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of emit records")
|
75
75
|
@emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_size", help_text: "Total size of emit events")
|
76
76
|
@enable_size_metrics = !!system_config.enable_size_metrics
|
@@ -40,7 +40,7 @@ module Fluent::Plugin
|
|
40
40
|
config_param :backlog, :integer, default: nil
|
41
41
|
# SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
|
42
42
|
desc 'The timeout time used to set linger option.'
|
43
|
-
config_param :linger_timeout, :integer, default:
|
43
|
+
config_param :linger_timeout, :integer, default: nil, deprecated: "use transport directive"
|
44
44
|
# This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
|
45
45
|
config_param :blocking_timeout, :time, default: 0.5
|
46
46
|
desc 'Try to resolve hostname from IP addresses or not.'
|
data/lib/fluent/plugin/output.rb
CHANGED
@@ -273,7 +273,7 @@ module Fluent
|
|
273
273
|
super
|
274
274
|
|
275
275
|
@num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
|
276
|
-
@emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "
|
276
|
+
@emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_count", help_text: "Number of count emits")
|
277
277
|
@emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
|
278
278
|
@emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
|
279
279
|
@write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
|
@@ -1275,52 +1275,57 @@ module Fluent
|
|
1275
1275
|
|
1276
1276
|
unless @retry
|
1277
1277
|
@retry = retry_state(@buffer_config.retry_randomize)
|
1278
|
+
|
1278
1279
|
if @retry.limit?
|
1279
|
-
|
1280
|
-
|
1281
|
-
|
1282
|
-
log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
|
1283
|
-
log.warn_backtrace error.backtrace
|
1284
|
-
end
|
1285
|
-
return
|
1280
|
+
handle_limit_reached(error)
|
1281
|
+
elsif error
|
1282
|
+
log_retry_error(error, chunk_id_hex, using_secondary)
|
1286
1283
|
end
|
1284
|
+
|
1285
|
+
return
|
1287
1286
|
end
|
1288
1287
|
|
1289
1288
|
# @retry exists
|
1290
1289
|
|
1291
|
-
|
1292
|
-
|
1293
|
-
|
1294
|
-
|
1295
|
-
log.error msg, retry_times: @retry.steps, records: records, error: error
|
1296
|
-
log.error_backtrace error.backtrace
|
1297
|
-
end
|
1298
|
-
@buffer.clear_queue!
|
1299
|
-
log.debug "buffer queue cleared"
|
1300
|
-
@retry = nil
|
1290
|
+
# Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
|
1291
|
+
# @retry.step is called almost as many times as the number of flush threads in a short time.
|
1292
|
+
if Time.now >= @retry.next_time
|
1293
|
+
@retry.step
|
1301
1294
|
else
|
1302
|
-
#
|
1303
|
-
|
1304
|
-
|
1305
|
-
|
1306
|
-
|
1307
|
-
|
1308
|
-
|
1309
|
-
if error
|
1310
|
-
if using_secondary
|
1311
|
-
msg = "failed to flush the buffer with secondary output."
|
1312
|
-
log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
|
1313
|
-
log.warn_backtrace error.backtrace
|
1314
|
-
else
|
1315
|
-
msg = "failed to flush the buffer."
|
1316
|
-
log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
|
1317
|
-
log.warn_backtrace error.backtrace
|
1318
|
-
end
|
1319
|
-
end
|
1295
|
+
@retry.recalc_next_time # to prevent all flush threads from retrying at the same time
|
1296
|
+
end
|
1297
|
+
|
1298
|
+
if @retry.limit?
|
1299
|
+
handle_limit_reached(error)
|
1300
|
+
elsif error
|
1301
|
+
log_retry_error(error, chunk_id_hex, using_secondary)
|
1320
1302
|
end
|
1321
1303
|
end
|
1322
1304
|
end
|
1323
1305
|
|
1306
|
+
def log_retry_error(error, chunk_id_hex, using_secondary)
|
1307
|
+
return unless error
|
1308
|
+
if using_secondary
|
1309
|
+
msg = "failed to flush the buffer with secondary output."
|
1310
|
+
else
|
1311
|
+
msg = "failed to flush the buffer."
|
1312
|
+
end
|
1313
|
+
log.warn(msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error)
|
1314
|
+
log.warn_backtrace(error.backtrace)
|
1315
|
+
end
|
1316
|
+
|
1317
|
+
def handle_limit_reached(error)
|
1318
|
+
if error
|
1319
|
+
records = @buffer.queued_records
|
1320
|
+
msg = "Hit limit for retries. dropping all chunks in the buffer queue."
|
1321
|
+
log.error msg, retry_times: @retry.steps, records: records, error: error
|
1322
|
+
log.error_backtrace error.backtrace
|
1323
|
+
end
|
1324
|
+
@buffer.clear_queue!
|
1325
|
+
log.debug "buffer queue cleared"
|
1326
|
+
@retry = nil
|
1327
|
+
end
|
1328
|
+
|
1324
1329
|
def retry_state(randomize)
|
1325
1330
|
if @secondary
|
1326
1331
|
retry_state_create(
|
data/lib/fluent/plugin/parser.rb
CHANGED
@@ -89,7 +89,7 @@ module Fluent
|
|
89
89
|
# : format[, timezone]
|
90
90
|
|
91
91
|
config_param :time_key, :string, default: nil
|
92
|
-
config_param :null_value_pattern, :
|
92
|
+
config_param :null_value_pattern, :regexp, default: nil
|
93
93
|
config_param :null_empty_string, :bool, default: false
|
94
94
|
config_param :estimate_current_event, :bool, default: true
|
95
95
|
config_param :keep_time_key, :bool, default: false
|
@@ -115,9 +115,8 @@ module Fluent
|
|
115
115
|
super
|
116
116
|
|
117
117
|
@time_parser = time_parser_create
|
118
|
-
@null_value_regexp = @null_value_pattern && Regexp.new(@null_value_pattern)
|
119
118
|
@type_converters = build_type_converters(@types)
|
120
|
-
@execute_convert_values = @type_converters || @
|
119
|
+
@execute_convert_values = @type_converters || @null_value_pattern || @null_empty_string
|
121
120
|
@timeout_checker = if @timeout
|
122
121
|
class << self
|
123
122
|
alias_method :parse_orig, :parse
|
@@ -220,7 +219,7 @@ module Fluent
|
|
220
219
|
return time, record
|
221
220
|
end
|
222
221
|
|
223
|
-
def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @
|
222
|
+
def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_pattern)
|
224
223
|
null_empty_string && value.empty? || null_value_regexp && string_safe_encoding(value){|s| null_value_regexp.match(s) }
|
225
224
|
end
|
226
225
|
|
@@ -44,6 +44,8 @@ module Fluent
|
|
44
44
|
|
45
45
|
@timeout = timeout
|
46
46
|
@timeout_at = @start + timeout
|
47
|
+
@has_reached_timeout = false
|
48
|
+
@has_timed_out = false
|
47
49
|
@current = :primary
|
48
50
|
|
49
51
|
if randomize_width < 0 || randomize_width > 0.5
|
@@ -98,7 +100,7 @@ module Fluent
|
|
98
100
|
naive
|
99
101
|
end
|
100
102
|
elsif @current == :secondary
|
101
|
-
naive = naive_next_time(@steps - @secondary_transition_steps
|
103
|
+
naive = naive_next_time(@steps - @secondary_transition_steps)
|
102
104
|
if naive >= @timeout_at
|
103
105
|
@timeout_at
|
104
106
|
else
|
@@ -123,7 +125,15 @@ module Fluent
|
|
123
125
|
@current = :secondary
|
124
126
|
@secondary_transition_steps = @steps
|
125
127
|
end
|
128
|
+
|
126
129
|
@next_time = calc_next_time
|
130
|
+
|
131
|
+
if @has_reached_timeout
|
132
|
+
@has_timed_out = @next_time >= @timeout_at
|
133
|
+
else
|
134
|
+
@has_reached_timeout = @next_time >= @timeout_at
|
135
|
+
end
|
136
|
+
|
127
137
|
nil
|
128
138
|
end
|
129
139
|
|
@@ -135,7 +145,7 @@ module Fluent
|
|
135
145
|
if @forever
|
136
146
|
false
|
137
147
|
else
|
138
|
-
@
|
148
|
+
@has_timed_out || !!(@max_steps && @steps >= @max_steps)
|
139
149
|
end
|
140
150
|
end
|
141
151
|
end
|
@@ -165,7 +175,7 @@ module Fluent
|
|
165
175
|
end
|
166
176
|
|
167
177
|
def calc_interval(num)
|
168
|
-
interval = raw_interval(num
|
178
|
+
interval = raw_interval(num)
|
169
179
|
if @max_interval && interval > @max_interval
|
170
180
|
@max_interval
|
171
181
|
else
|
@@ -175,7 +185,7 @@ module Fluent
|
|
175
185
|
# Calculate previous finite value to avoid inf related errors. If this re-computing is heavy, use cache.
|
176
186
|
until interval.finite?
|
177
187
|
num -= 1
|
178
|
-
interval = raw_interval(num
|
188
|
+
interval = raw_interval(num)
|
179
189
|
end
|
180
190
|
interval
|
181
191
|
end
|
@@ -80,8 +80,8 @@ module Fluent
|
|
80
80
|
raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
|
81
81
|
raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
|
82
82
|
|
83
|
-
if proto == :tcp || proto == :tls
|
84
|
-
socket_options[:linger_timeout] ||= 0
|
83
|
+
if proto == :tcp || proto == :tls
|
84
|
+
socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
|
85
85
|
end
|
86
86
|
|
87
87
|
socket_option_validate!(proto, **socket_options)
|
@@ -132,8 +132,8 @@ module Fluent
|
|
132
132
|
raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
|
133
133
|
raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
|
134
134
|
|
135
|
-
if proto == :tcp || proto == :tls
|
136
|
-
socket_options[:linger_timeout] ||= 0
|
135
|
+
if proto == :tcp || proto == :tls
|
136
|
+
socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
|
137
137
|
end
|
138
138
|
|
139
139
|
unless socket
|
@@ -263,6 +263,23 @@ module Fluent
|
|
263
263
|
include Fluent::Configurable
|
264
264
|
config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
|
265
265
|
config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp
|
266
|
+
|
267
|
+
### Socket Params ###
|
268
|
+
|
269
|
+
# SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src.
|
270
|
+
# Set positive value if needing to send FIN on closing.
|
271
|
+
# NOTE:
|
272
|
+
# Socket-options can be specified from each plugin as needed, so most of them is not defined here for now.
|
273
|
+
# This is because there is no positive reason to do so.
|
274
|
+
# `linger_timeout` option in particular needs to be defined here
|
275
|
+
# although it can be specified from each plugin as well.
|
276
|
+
# This is because this helper fixes the default value to `0` for its own reason
|
277
|
+
# and it has a critical effect on the behavior.
|
278
|
+
desc 'The timeout time used to set linger option.'
|
279
|
+
config_param :linger_timeout, :integer, default: 0
|
280
|
+
|
281
|
+
### TLS Params ###
|
282
|
+
|
266
283
|
config_param :version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
|
267
284
|
config_param :min_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
|
268
285
|
config_param :max_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
|
data/lib/fluent/rpc.rb
CHANGED
@@ -20,9 +20,10 @@ module Fluent
|
|
20
20
|
module RPC
|
21
21
|
class Server
|
22
22
|
def initialize(endpoint, log)
|
23
|
-
|
24
|
-
|
25
|
-
@
|
23
|
+
m = endpoint.match(/^\[?(?<host>[0-9a-zA-Z:\-\.]+)\]?:(?<port>[0-9]+)$/)
|
24
|
+
raise Fluent::ConfigError, "Invalid rpc_endpoint: #{endpoint}" unless m
|
25
|
+
@bind = m[:host]
|
26
|
+
@port = m[:port]
|
26
27
|
@log = log
|
27
28
|
|
28
29
|
@server = WEBrick::HTTPServer.new(
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -397,6 +397,7 @@ module Fluent
|
|
397
397
|
log_path = params['log_path']
|
398
398
|
chuser = params['chuser']
|
399
399
|
chgroup = params['chgroup']
|
400
|
+
chumask = params['chumask']
|
400
401
|
log_rotate_age = params['log_rotate_age']
|
401
402
|
log_rotate_size = params['log_rotate_size']
|
402
403
|
|
@@ -436,7 +437,7 @@ module Fluent
|
|
436
437
|
logger_initializer: logger_initializer,
|
437
438
|
chuser: chuser,
|
438
439
|
chgroup: chgroup,
|
439
|
-
chumask:
|
440
|
+
chumask: chumask,
|
440
441
|
suppress_repeated_stacktrace: suppress_repeated_stacktrace,
|
441
442
|
ignore_repeated_log_interval: ignore_repeated_log_interval,
|
442
443
|
ignore_same_log_interval: ignore_same_log_interval,
|
@@ -565,6 +566,7 @@ module Fluent
|
|
565
566
|
setup_path: nil,
|
566
567
|
chuser: nil,
|
567
568
|
chgroup: nil,
|
569
|
+
chumask: "0",
|
568
570
|
root_dir: nil,
|
569
571
|
suppress_interval: 0,
|
570
572
|
suppress_repeated_stacktrace: true,
|
@@ -603,6 +605,7 @@ module Fluent
|
|
603
605
|
@plugin_dirs = opt[:plugin_dirs]
|
604
606
|
@chgroup = opt[:chgroup]
|
605
607
|
@chuser = opt[:chuser]
|
608
|
+
@chumask = opt[:chumask]
|
606
609
|
|
607
610
|
@log_rotate_age = opt[:log_rotate_age]
|
608
611
|
@log_rotate_size = opt[:log_rotate_size]
|
@@ -709,7 +712,7 @@ module Fluent
|
|
709
712
|
create_socket_manager if @standalone_worker
|
710
713
|
if @standalone_worker
|
711
714
|
ServerEngine::Privilege.change(@chuser, @chgroup)
|
712
|
-
File.umask(
|
715
|
+
File.umask(@chumask.to_i(8))
|
713
716
|
end
|
714
717
|
MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
|
715
718
|
Fluent::Engine.init(@system_config)
|
data/lib/fluent/version.rb
CHANGED
@@ -68,7 +68,6 @@ class ForwardInputTest < Test::Unit::TestCase
|
|
68
68
|
@d = d = create_driver
|
69
69
|
assert_equal @port, d.instance.port
|
70
70
|
assert_equal '127.0.0.1', d.instance.bind
|
71
|
-
assert_equal 0, d.instance.linger_timeout
|
72
71
|
assert_equal 0.5, d.instance.blocking_timeout
|
73
72
|
assert !d.instance.backlog
|
74
73
|
end
|
@@ -77,7 +76,6 @@ class ForwardInputTest < Test::Unit::TestCase
|
|
77
76
|
@d = d = create_driver(config_auth)
|
78
77
|
assert_equal @port, d.instance.port
|
79
78
|
assert_equal '127.0.0.1', d.instance.bind
|
80
|
-
assert_equal 0, d.instance.linger_timeout
|
81
79
|
assert !d.instance.backlog
|
82
80
|
|
83
81
|
assert d.instance.security
|