fluentd 1.14.3-x64-mingw32 → 1.14.6-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/config.yml +2 -2
  3. data/.github/workflows/linux-test.yaml +1 -1
  4. data/.github/workflows/macos-test.yaml +5 -1
  5. data/.github/workflows/windows-test.yaml +6 -6
  6. data/CHANGELOG.md +79 -16
  7. data/CONTRIBUTING.md +1 -1
  8. data/MAINTAINERS.md +2 -2
  9. data/README.md +2 -23
  10. data/Rakefile +1 -1
  11. data/fluentd.gemspec +2 -1
  12. data/lib/fluent/command/fluentd.rb +4 -0
  13. data/lib/fluent/config/error.rb +12 -0
  14. data/lib/fluent/env.rb +4 -0
  15. data/lib/fluent/event_router.rb +19 -1
  16. data/lib/fluent/plugin/bare_output.rb +1 -1
  17. data/lib/fluent/plugin/base.rb +1 -1
  18. data/lib/fluent/plugin/buffer.rb +43 -23
  19. data/lib/fluent/plugin/in_forward.rb +1 -1
  20. data/lib/fluent/plugin/in_http.rb +11 -1
  21. data/lib/fluent/plugin/in_tail.rb +10 -0
  22. data/lib/fluent/plugin/out_file.rb +13 -1
  23. data/lib/fluent/plugin/output.rb +41 -32
  24. data/lib/fluent/plugin/parser.rb +3 -4
  25. data/lib/fluent/plugin_helper/retry_state.rb +14 -4
  26. data/lib/fluent/plugin_helper/server.rb +21 -4
  27. data/lib/fluent/plugin_helper/socket.rb +13 -2
  28. data/lib/fluent/registry.rb +2 -1
  29. data/lib/fluent/rpc.rb +4 -3
  30. data/lib/fluent/supervisor.rb +5 -2
  31. data/lib/fluent/version.rb +1 -1
  32. data/test/compat/test_parser.rb +1 -1
  33. data/test/plugin/test_bare_output.rb +1 -1
  34. data/test/plugin/test_buffer.rb +77 -0
  35. data/test/plugin/test_filter.rb +1 -1
  36. data/test/plugin/test_filter_parser.rb +1 -1
  37. data/test/plugin/test_filter_stdout.rb +2 -2
  38. data/test/plugin/test_in_forward.rb +0 -2
  39. data/test/plugin/test_in_http.rb +23 -0
  40. data/test/plugin/test_in_tail.rb +35 -0
  41. data/test/plugin/test_input.rb +1 -1
  42. data/test/plugin/test_out_exec.rb +6 -4
  43. data/test/plugin/test_out_file.rb +29 -13
  44. data/test/plugin/test_out_stdout.rb +2 -2
  45. data/test/plugin/test_output_as_buffered_retries.rb +53 -6
  46. data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
  47. data/test/plugin_helper/test_retry_state.rb +602 -38
  48. data/test/plugin_helper/test_server.rb +18 -0
  49. data/test/plugin_helper/test_timer.rb +2 -2
  50. data/test/test_event_router.rb +17 -0
  51. data/test/test_formatter.rb +1 -1
  52. data/test/test_supervisor.rb +41 -6
  53. metadata +19 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 394015dc219a38edeae834c955b2d6e9a5139920c49caa349765c4b0d376a12e
4
- data.tar.gz: 5b4dbb5aae91e85043e295f037fac15f877a084ee5c6289d09a0e8b4b7cf0c9c
3
+ metadata.gz: 0437b57bc07c96e97f0fd6f12bcb1e8fdfa2bda366c887aaad5ad99679221988
4
+ data.tar.gz: 679a45104420e99c66525884c7a274972a1875abc008d097b786080c9e01833f
5
5
  SHA512:
6
- metadata.gz: 6241decd6c828af6e8bd6f1522517d54a00757b50bd7bb9746dbfe0bb766c22e94da31a714c1cee4376392f4c22a403da7073fed11c2610a2f0e2e455b632510
7
- data.tar.gz: 902ef5dcd2289d72da71495877ec1b8337477f177b81b7c8002a2fa7beec11a2571957539044e16f2bc7c5a6a6be32dbd72e6867d2056a03ff41e216d5a2af22
6
+ metadata.gz: 8cc3ff86580a0779fcdc27588ca715b1d6b4c2c1c68600066b76ae52882486e4aef2f46160402d0fc56e5ad7831a490dbcad0a0e3376af7627ed6472c83e16c5
7
+ data.tar.gz: cdeacccd982235e7baf9e47fd6407d9855bad5face3775ac9c87220b26eb5998d550b1d8cb2f8b425005fb34e239645d118f2cb5afda06103a41be71da934fed
@@ -1,5 +1,5 @@
1
1
  blank_issues_enabled: false
2
2
  contact_links:
3
3
  - name: Ask a Question
4
- url: https://discuss.fluentd.org/
5
- about: I have questions about Fluentd and plugins. Please ask and answer questions at https://discuss.fluentd.org/.
4
+ url: https://github.com/fluent/fluentd/discussions
5
+ about: I have questions about Fluentd and plugins. Please ask and answer questions at https://github.com/fluent/fluentd/discussions
@@ -13,7 +13,7 @@ jobs:
13
13
  strategy:
14
14
  fail-fast: false
15
15
  matrix:
16
- ruby-version: ['3.0', '2.7', '2.6']
16
+ ruby-version: ['3.1', '3.0', '2.7', '2.6']
17
17
  os: [ubuntu-latest]
18
18
  experimental: [false]
19
19
  include:
@@ -13,9 +13,13 @@ jobs:
13
13
  strategy:
14
14
  fail-fast: false
15
15
  matrix:
16
- ruby-version: ['head', '2.7']
16
+ ruby-version: ['3.1', '3.0', '2.7']
17
17
  os: [macos-latest]
18
18
  experimental: [true]
19
+ include:
20
+ - ruby-version: head
21
+ os: macos-latest
22
+ experimental: true
19
23
 
20
24
  name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
21
25
  steps:
@@ -18,9 +18,9 @@ jobs:
18
18
  - windows-latest
19
19
  experimental: [false]
20
20
  include:
21
- - ruby-version: '3.0.2'
21
+ - ruby-version: '3.0.3'
22
22
  os: windows-latest
23
- experimental: true
23
+ experimental: false
24
24
  # On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
25
25
  # error code. In addition, we have to specify the path of fiddle by RUBYLIB
26
26
  # because RubyInstaller loads Ruby's bundled fiddle before initializing gem.
@@ -28,7 +28,7 @@ jobs:
28
28
  # * https://github.com/ruby/fiddle/issues/72
29
29
  # * https://bugs.ruby-lang.org/issues/17813
30
30
  # * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
31
- ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.2/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
31
+ ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.3/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.1.0/lib
32
32
 
33
33
  name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
34
34
  steps:
@@ -37,9 +37,9 @@ jobs:
37
37
  uses: ruby/setup-ruby@v1
38
38
  with:
39
39
  ruby-version: ${{ matrix.ruby-version }}
40
- - name: Add Fiddle 1.0.8
41
- if: ${{ matrix.ruby-version == '3.0.2' }}
42
- run: gem install fiddle --version 1.0.8
40
+ - name: Add Fiddle 1.1.0
41
+ if: ${{ matrix.ruby-version == '3.0.3' }}
42
+ run: gem install fiddle --version 1.1.0
43
43
  - name: Install dependencies
44
44
  run: ridk exec bundle install
45
45
  - name: Run tests
data/CHANGELOG.md CHANGED
@@ -1,4 +1,79 @@
1
- # v1.14.3
1
+ # v1.14
2
+
3
+ ## Release v1.14.6 - 2022/03/31
4
+
5
+ ### Enhancement
6
+
7
+ * Enable server plugins to specify socket-option `SO_LINGER`
8
+ https://github.com/fluent/fluentd/pull/3644
9
+ * Add `--umask` command line parameter
10
+ https://github.com/fluent/fluentd/pull/3671
11
+ https://github.com/fluent/fluentd/pull/3679
12
+
13
+ ### Bug fixes
14
+
15
+ * Fix metric name typo
16
+ https://github.com/fluent/fluentd/pull/3630
17
+ https://github.com/fluent/fluentd/pull/3673
18
+ * Apply modifications in pipeline to the records being passed to `@ERROR` label
19
+ https://github.com/fluent/fluentd/pull/3631
20
+ * Fix wrong calculation of retry interval
21
+ https://github.com/fluent/fluentd/pull/3640
22
+ https://github.com/fluent/fluentd/pull/3649
23
+ https://github.com/fluent/fluentd/pull/3685
24
+ https://github.com/fluent/fluentd/pull/3686
25
+ * Support IPv6 address for `rpc_endpoint` in `system` config
26
+ https://github.com/fluent/fluentd/pull/3641
27
+
28
+ ### Misc
29
+
30
+ * CI: Support Ruby 3.1 except Windows
31
+ https://github.com/fluent/fluentd/pull/3619
32
+ * Switch to GitHub Discussions
33
+ https://github.com/fluent/fluentd/pull/3654
34
+ * Fix CHANGELOG.md heading styles
35
+ https://github.com/fluent/fluentd/pull/3648
36
+ * Declare `null_value_pattern` as `regexp`
37
+ https://github.com/fluent/fluentd/pull/3650
38
+
39
+ ## Release v1.14.5 - 2022/02/09
40
+
41
+ ### Enhancement
42
+
43
+ * Add support for "application/x-ndjson" to `in_http`
44
+ https://github.com/fluent/fluentd/pull/3616
45
+ * Add support for ucrt binary for Windows
46
+ https://github.com/fluent/fluentd/pull/3613
47
+
48
+ ### Bug fixes
49
+
50
+ * Don't retry when `retry_max_times == 0`
51
+ https://github.com/fluent/fluentd/pull/3608
52
+ * Fix hang-up issue during TLS handshake in `out_forward`
53
+ https://github.com/fluent/fluentd/pull/3601
54
+ * Bump up required ServerEngine to v2.2.5
55
+ https://github.com/fluent/fluentd/pull/3599
56
+ * Fix "invalid byte sequence is replaced" warning on Kubernetes
57
+ https://github.com/fluent/fluentd/pull/3596
58
+ * Fix "ArgumentError: unknown keyword: :logger" on Windows with Ruby 3.1
59
+ https://github.com/fluent/fluentd/pull/3592
60
+
61
+ ## Release v1.14.4 - 2022/01/06
62
+
63
+ ### Enhancement
64
+
65
+ * `in_tail`: Add option to skip long lines (`max_line_size`)
66
+ https://github.com/fluent/fluentd/pull/3565
67
+
68
+ ### Bug fix
69
+
70
+ * Incorrect BufferChunkOverflowError when each event size is < `chunk_limit_size`
71
+ https://github.com/fluent/fluentd/pull/3560
72
+ * On macOS with Ruby 2.7/3.0, `out_file` fails to write events if `append` is true.
73
+ https://github.com/fluent/fluentd/pull/3579
74
+ * test: Fix unstable test cases
75
+ https://github.com/fluent/fluentd/pull/3574
76
+ https://github.com/fluent/fluentd/pull/3577
2
77
 
3
78
  ## Release v1.14.3 - 2021/11/26
4
79
 
@@ -28,8 +103,6 @@
28
103
  newer version is required to implement additional `fluent-ctl` commands.
29
104
  https://github.com/fluent/fluentd/pull/3556
30
105
 
31
- # v1.14.2
32
-
33
106
  ## Release v1.14.2 - 2021/10/29
34
107
 
35
108
  IMPORTANT: This release contain the fix for CVE-2021-41186 -
@@ -41,6 +114,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
41
114
  ### Enhancement
42
115
 
43
116
  * fluent-cat: Add `--event-time` option to send specified event time for testing.
117
+ https://github.com/fluent/fluentd/pull/3528
44
118
 
45
119
  ### Bug fix
46
120
 
@@ -49,8 +123,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
49
123
  * Fixed ReDoS vulnerability in parser_apache2.
50
124
  This vulnerability is caused by a certain pattern of a broken apache log.
51
125
 
52
- # v1.14.1
53
-
54
126
  ## Release v1.14.1 - 2021/09/29
55
127
 
56
128
  ### Enhancement
@@ -80,8 +152,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
80
152
  next_retry_seconds -> next_retry_time)
81
153
  https://github.com/fluent/fluentd/pull/3518
82
154
 
83
- # v1.14.0
84
-
85
155
  ## Release v1.14.0 - 2021/08/30
86
156
 
87
157
  ### Enhancement
@@ -124,7 +194,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
124
194
  https://github.com/fluent/fluentd/pull/3479
125
195
  https://github.com/fluent/fluentd/pull/3484
126
196
 
127
- # v1.13.3
197
+ # v1.13
128
198
 
129
199
  ## Release v1.13.3 - 2021/07/27
130
200
 
@@ -146,10 +216,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
146
216
  * Remove needless spaces in a sample config file
147
217
  https://github.com/fluent/fluentd/pull/3456
148
218
 
149
- ### Enhancement
150
-
151
- # v1.13.2
152
-
153
219
  ## Release v1.13.2 - 2021/07/12
154
220
 
155
221
  ### Enhancement
@@ -185,8 +251,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
185
251
  by security scanning tools.
186
252
  https://github.com/fluent/fluentd/pull/3450
187
253
 
188
- # v1.13.1
189
-
190
254
  ## Release v1.13.1 - 2021/06/25
191
255
 
192
256
  ### Bug fix
@@ -211,8 +275,6 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
211
275
  * CI: Dropped to run CI for Ruby 2.5
212
276
  https://github.com/fluent/fluentd/pull/3412
213
277
 
214
- # v1.13
215
-
216
278
  ## Release v1.13.0 - 2021/05/29
217
279
 
218
280
  ### Enhancement
@@ -260,6 +322,7 @@ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
260
322
  ### Bug fix
261
323
 
262
324
  * in_tail: Fix a bug that refresh_watcher fails to handle file rotations
325
+ https://github.com/fluent/fluentd/pull/3393
263
326
 
264
327
  ## Release v1.12.3 - 2021/04/23
265
328
 
data/CONTRIBUTING.md CHANGED
@@ -6,7 +6,7 @@ We'd love your contribution. Here are the guidelines!
6
6
 
7
7
  RESOURCES of [Official site](https://www.fluentd.org/) and [Fluentd documentation](https://docs.fluentd.org/) may help you.
8
8
 
9
- If you have further questions about Fluentd and plugins, please direct these to [Community Forum](https://discuss.fluentd.org/).
9
+ If you have further questions about Fluentd and plugins, please direct these to [Community Forum](https://github.com/fluent/fluentd/discussions).
10
10
  Don't use Github issue for asking questions. Here are examples:
11
11
 
12
12
  - I installed xxx plugin but it doesn't work. Why?
data/MAINTAINERS.md CHANGED
@@ -2,10 +2,10 @@
2
2
 
3
3
  - [Naotoshi Seo](https://github.com/sonots), [ZOZO Technologies](https://tech.zozo.com/en/)
4
4
  - [Okkez](https://github.com/okkez)
5
- - [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://www.calyptia.com/)
5
+ - [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://calyptia.com/)
6
6
  - [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)
7
7
  - [Satoshi Tagomori](https://github.com/tagomoris), [Treasure Data](https://www.treasuredata.com/)
8
- - [Eduardo Silva](https://github.com/edsiper), [Arm Treasure Data](https://www.treasuredata.com/)
8
+ - [Eduardo Silva](https://github.com/edsiper), [Calyptia](https://calyptia/)
9
9
  - [Fujimoto Seiji](https://github.com/fujimots), [ClearCode](https://www.clear-code.com/)
10
10
  - [Takuro Ashie](https://github.com/ashie), [ClearCode](https://www.clear-code.com/)
11
11
  - [Kentaro Hayashi](https://github.com/kenhys), [ClearCode](https://www.clear-code.com/)
data/README.md CHANGED
@@ -1,17 +1,12 @@
1
1
  Fluentd: Open-Source Log Collector
2
2
  ===================================
3
- GitHub Actions:
4
3
 
5
4
  [![Testing on Ubuntu](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml)
6
5
  [![Testing on Windows](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml)
7
6
  [![Testing on macOS](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
7
+ [![Testing on Arm64](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)
8
8
  [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
9
9
  [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
10
- [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd.svg?type=shield)](https://app.fossa.io/projects/git%2Bhttps%3A%2F%2Fgithub.com%2Ffluent%2Ffluentd?ref=badge_shield)
11
-
12
- Drone CI for Arm64:
13
-
14
- [![pipeline status](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)
15
10
 
16
11
  [Fluentd](https://www.fluentd.org/) collects events from various data sources and writes them to files, RDBMS, NoSQL, IaaS, SaaS, Hadoop and so on. Fluentd helps you unify your logging infrastructure (Learn more about the [Unified Logging Layer](https://www.fluentd.org/blog/unified-logging-layer)).
17
12
 
@@ -19,16 +14,6 @@ Drone CI for Arm64:
19
14
  <img src="https://www.fluentd.org/images/fluentd-architecture.png" width="500px"/>
20
15
  </p>
21
16
 
22
- An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time recorded at occurrence of an event. Record is a JSON object.
23
-
24
- ## Example Use Cases
25
-
26
- Use Case | Description | Diagram
27
- -------- | ------------|:---------:
28
- Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/images/recipes/elasticsearch-s3-fluentd.png" height="150"/>
29
- Syslog Alerting | Fluentd can "grep" for events and send out alerts. | <img src="https://www.fluentd.org/images/syslog-fluentd-alert.png" height="100"/>
30
- Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/images/datasources/asynchronous_logging.png" height="150"/>
31
-
32
17
  ## Quick Start
33
18
 
34
19
  $ gem install fluentd
@@ -66,18 +51,12 @@ You can run specified test via `TEST` environment variable:
66
51
  $ bundle exec rake test TEST=test/test_specified_path.rb
67
52
  $ bundle exec rake test TEST=test/test_*.rb
68
53
 
69
- ## Fluentd UI: Admin GUI
70
-
71
- [Fluentd UI](https://github.com/fluent/fluentd-ui) is a graphical user interface to start/stop/configure Fluentd.
72
-
73
- <p align="center"><img width="500" src="https://www.fluentd.org/images/blog/fluentd-ui.gif"/></p>
74
-
75
54
  ## More Information
76
55
 
77
56
  - Website: https://www.fluentd.org/
78
57
  - Documentation: https://docs.fluentd.org/
79
58
  - Project repository: https://github.com/fluent
80
- - Discussion: https://discuss.fluentd.org/
59
+ - Discussion: https://github.com/fluent/fluentd/discussions
81
60
  - Slack / Community: https://slack.fluentd.org
82
61
  - Newsletters: https://www.fluentd.org/newsletter
83
62
  - Author: [Sadayuki Furuhashi](https://github.com/frsyuki)
data/Rakefile CHANGED
@@ -14,7 +14,7 @@ namespace :build do
14
14
  desc 'Build gems for all platforms'
15
15
  task :all do
16
16
  Bundler.with_clean_env do
17
- %w[ruby x86-mingw32 x64-mingw32].each do |name|
17
+ %w[ruby x86-mingw32 x64-mingw32 x64-mingw-ucrt].each do |name|
18
18
  ENV['GEM_BUILD_FAKE_PLATFORM'] = name
19
19
  Rake::Task["build"].execute
20
20
  end
data/fluentd.gemspec CHANGED
@@ -22,7 +22,7 @@ Gem::Specification.new do |gem|
22
22
  gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
23
23
  gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
24
24
  gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
25
- gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
25
+ gem.add_runtime_dependency("serverengine", [">= 2.2.5", "< 3.0.0"])
26
26
  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
27
27
  gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
28
28
  gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
@@ -51,5 +51,6 @@ Gem::Specification.new do |gem|
51
51
  gem.add_development_dependency("test-unit", ["~> 3.3"])
52
52
  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
53
53
  gem.add_development_dependency("oj", [">= 2.14", "< 4"])
54
+ gem.add_development_dependency("async", "~> 1.23")
54
55
  gem.add_development_dependency("async-http", ">= 0.50.0")
55
56
  end
@@ -81,6 +81,10 @@ op.on('--group GROUP', "change group") {|s|
81
81
  opts[:chgroup] = s
82
82
  }
83
83
 
84
+ op.on('--umask UMASK', "change umask") {|s|
85
+ opts[:chumask] = s
86
+ }
87
+
84
88
  op.on('-o', '--log PATH', "log file path") {|s|
85
89
  opts[:log_path] = s
86
90
  }
@@ -29,4 +29,16 @@ module Fluent
29
29
 
30
30
  class SetDefault < Exception
31
31
  end
32
+
33
+ class NotFoundPluginError < ConfigError
34
+ attr_reader :type, :kind
35
+
36
+ def initialize(msg, type: nil, kind: nil)
37
+ @msg = msg
38
+ @type = type
39
+ @kind = kind
40
+
41
+ super(msg)
42
+ end
43
+ end
32
44
  end
data/lib/fluent/env.rb CHANGED
@@ -33,4 +33,8 @@ module Fluent
33
33
  def self.linux?
34
34
  /linux/ === RUBY_PLATFORM
35
35
  end
36
+
37
+ def self.macos?
38
+ /darwin/ =~ RUBY_PLATFORM
39
+ end
36
40
  end
@@ -116,6 +116,8 @@ module Fluent
116
116
  if callback = find_callback
117
117
  callback.call(es)
118
118
  end
119
+ rescue Pipeline::OutputError => e
120
+ @emit_error_handler.handle_emits_error(tag, e.processed_es, e.internal_error)
119
121
  rescue => e
120
122
  @emit_error_handler.handle_emits_error(tag, es, e)
121
123
  end
@@ -161,6 +163,17 @@ module Fluent
161
163
  private
162
164
 
163
165
  class Pipeline
166
+
167
+ class OutputError < StandardError
168
+ attr_reader :internal_error
169
+ attr_reader :processed_es
170
+
171
+ def initialize(internal_error, processed_es)
172
+ @internal_error = internal_error
173
+ @processed_es = processed_es
174
+ end
175
+ end
176
+
164
177
  def initialize
165
178
  @filters = []
166
179
  @output = nil
@@ -178,7 +191,12 @@ module Fluent
178
191
 
179
192
  def emit_events(tag, es)
180
193
  processed = @optimizer.filter_stream(tag, es)
181
- @output.emit_events(tag, processed)
194
+
195
+ begin
196
+ @output.emit_events(tag, processed)
197
+ rescue => e
198
+ raise OutputError.new(e, processed)
199
+ end
182
200
  end
183
201
 
184
202
  class FilterOptimizer
@@ -70,7 +70,7 @@ module Fluent
70
70
  super
71
71
 
72
72
  @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "num_errors", help_text: "Number of count num errors")
73
- @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of count emits")
73
+ @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_count", help_text: "Number of count emits")
74
74
  @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of emit records")
75
75
  @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_size", help_text: "Total size of emit events")
76
76
  @enable_size_metrics = !!system_config.enable_size_metrics
@@ -72,8 +72,8 @@ module Fluent
72
72
 
73
73
  def string_safe_encoding(str)
74
74
  unless str.valid_encoding?
75
- log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
76
75
  str = str.scrub('?')
76
+ log.info "invalid byte sequence is replaced in `#{str}`" if self.respond_to?(:log)
77
77
  end
78
78
  yield str
79
79
  end
@@ -767,24 +767,37 @@ module Fluent
767
767
  raise ShouldRetry unless chunk.writable?
768
768
  staged_chunk_used = true if chunk.staged?
769
769
 
770
- original_bytesize = chunk.bytesize
770
+ original_bytesize = committed_bytesize = chunk.bytesize
771
771
  begin
772
772
  while writing_splits_index < splits.size
773
773
  split = splits[writing_splits_index]
774
- formatted_split = format ? format.call(split) : split.first
775
- if split.size == 1 && original_bytesize == 0
776
- if format == nil && @compress != :text
777
- # The actual size of chunk is not determined until after chunk.append.
778
- # so, keep already processed 'split' content here.
779
- # (allow performance regression a bit)
774
+ formatted_split = format ? format.call(split) : nil
775
+
776
+ if split.size == 1 # Check BufferChunkOverflowError
777
+ determined_bytesize = nil
778
+ if @compress != :text
779
+ determined_bytesize = nil
780
+ elsif formatted_split
781
+ determined_bytesize = formatted_split.bytesize
782
+ elsif split.first.respond_to?(:bytesize)
783
+ determined_bytesize = split.first.bytesize
784
+ end
785
+
786
+ if determined_bytesize && determined_bytesize > @chunk_limit_size
787
+ # It is a obvious case that BufferChunkOverflowError should be raised here.
788
+ # But if it raises here, already processed 'split' or
789
+ # the proceeding 'split' will be lost completely.
790
+ # So it is a last resort to delay raising such a exception
791
+ errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
792
+ writing_splits_index += 1
793
+ next
794
+ end
795
+
796
+ if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
797
+ # The split will (might) cause size over so keep already processed
798
+ # 'split' content here (allow performance regression a bit).
780
799
  chunk.commit
781
- else
782
- big_record_size = formatted_split.bytesize
783
- if chunk.bytesize + big_record_size > @chunk_limit_size
784
- errors << "a #{big_record_size} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
785
- writing_splits_index += 1
786
- next
787
- end
800
+ committed_bytesize = chunk.bytesize
788
801
  end
789
802
  end
790
803
 
@@ -793,19 +806,26 @@ module Fluent
793
806
  else
794
807
  chunk.append(split, compress: @compress)
795
808
  end
809
+ adding_bytes = chunk.bytesize - committed_bytesize
796
810
 
797
811
  if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
798
- adding_bytes = chunk.instance_eval { @adding_bytes } || "N/A" # 3rd party might not have 'adding_bytes'
799
812
  chunk.rollback
813
+ committed_bytesize = chunk.bytesize
800
814
 
801
- if split.size == 1 && original_bytesize == 0
802
- # It is obviously case that BufferChunkOverflowError should be raised here,
803
- # but if it raises here, already processed 'split' or
804
- # the proceeding 'split' will be lost completely.
805
- # so it is a last resort to delay raising such a exception
806
- errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
807
- writing_splits_index += 1
808
- next
815
+ if split.size == 1 # Check BufferChunkOverflowError again
816
+ if adding_bytes > @chunk_limit_size
817
+ errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
818
+ writing_splits_index += 1
819
+ next
820
+ else
821
+ # As already processed content is kept after rollback, then unstaged chunk should be queued.
822
+ # After that, re-process current split again.
823
+ # New chunk should be allocated, to do it, modify @stage and so on.
824
+ synchronize { @stage.delete(modified_metadata) }
825
+ staged_chunk_used = false
826
+ chunk.unstaged!
827
+ break
828
+ end
809
829
  end
810
830
 
811
831
  if chunk_size_full?(chunk) || split.size == 1
@@ -40,7 +40,7 @@ module Fluent::Plugin
40
40
  config_param :backlog, :integer, default: nil
41
41
  # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src
42
42
  desc 'The timeout time used to set linger option.'
43
- config_param :linger_timeout, :integer, default: 0
43
+ config_param :linger_timeout, :integer, default: nil, deprecated: "use transport directive"
44
44
  # This option is for Cool.io's loop wait timeout to avoid loop stuck at shutdown. Almost users don't need to change this value.
45
45
  config_param :blocking_timeout, :time, default: 0.5
46
46
  desc 'Try to resolve hostname from IP addresses or not.'
@@ -314,8 +314,16 @@ module Fluent::Plugin
314
314
  @parser_json.parse(js) do |_time, record|
315
315
  return nil, record
316
316
  end
317
+ elsif ndjson = params['ndjson']
318
+ events = []
319
+ ndjson.split(/\r?\n/).each do |js|
320
+ @parser_json.parse(js) do |_time, record|
321
+ events.push(record)
322
+ end
323
+ end
324
+ return nil, events
317
325
  else
318
- raise "'json' or 'msgpack' parameter is required"
326
+ raise "'json', 'ndjson' or 'msgpack' parameter is required"
319
327
  end
320
328
  end
321
329
 
@@ -567,6 +575,8 @@ module Fluent::Plugin
567
575
  params['json'] = @body
568
576
  elsif @content_type =~ /^application\/msgpack/
569
577
  params['msgpack'] = @body
578
+ elsif @content_type =~ /^application\/x-ndjson/
579
+ params['ndjson'] = @body
570
580
  end
571
581
  path_info = uri.path
572
582
 
@@ -113,6 +113,8 @@ module Fluent::Plugin
113
113
  config_param :path_timezone, :string, default: nil
114
114
  desc 'Follow inodes instead of following file names. Guarantees more stable delivery and allows to use * in path pattern with rotating files'
115
115
  config_param :follow_inodes, :bool, default: false
116
+ desc 'Maximum length of line. The longer line is just skipped.'
117
+ config_param :max_line_size, :size, default: nil
116
118
 
117
119
  config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
118
120
  config_argument :usage, :string, default: 'in_tail_parser'
@@ -594,6 +596,14 @@ module Fluent::Plugin
594
596
 
595
597
  # @return true if no error or unrecoverable error happens in emit action. false if got BufferOverflowError
596
598
  def receive_lines(lines, tail_watcher)
599
+ lines = lines.reject do |line|
600
+ skip_line = @max_line_size ? line.bytesize > @max_line_size : false
601
+ if skip_line
602
+ log.warn "received line length is longer than #{@max_line_size}"
603
+ log.debug "skipped line: #{line.chomp}"
604
+ end
605
+ skip_line
606
+ end
597
607
  es = @receive_handler.call(lines, tail_watcher)
598
608
  unless es.empty?
599
609
  tag = if @tag_prefix || @tag_suffix
@@ -181,6 +181,13 @@ module Fluent::Plugin
181
181
  @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
182
182
  @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
183
183
  @need_lock = system_config.workers > 1
184
+
185
+ # https://github.com/fluent/fluentd/issues/3569
186
+ @need_ruby_on_macos_workaround = false
187
+ if @append && Fluent.macos?
188
+ condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
189
+ @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
190
+ end
184
191
  end
185
192
 
186
193
  def multi_workers_ready?
@@ -223,7 +230,12 @@ module Fluent::Plugin
223
230
 
224
231
  def write_without_compression(path, chunk)
225
232
  File.open(path, "ab", @file_perm) do |f|
226
- chunk.write_to(f)
233
+ if @need_ruby_on_macos_workaround
234
+ content = chunk.read()
235
+ f.puts content
236
+ else
237
+ chunk.write_to(f)
238
+ end
227
239
  end
228
240
  end
229
241