fluentd 1.15.2 → 1.16.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
  4. data/.github/workflows/linux-test.yaml +2 -2
  5. data/.github/workflows/macos-test.yaml +2 -2
  6. data/.github/workflows/stale-actions.yml +11 -9
  7. data/.github/workflows/windows-test.yaml +2 -2
  8. data/CHANGELOG.md +133 -0
  9. data/CONTRIBUTING.md +1 -1
  10. data/MAINTAINERS.md +5 -3
  11. data/README.md +0 -1
  12. data/SECURITY.md +5 -9
  13. data/fluentd.gemspec +2 -2
  14. data/lib/fluent/command/fluentd.rb +55 -64
  15. data/lib/fluent/config/yaml_parser/loader.rb +18 -1
  16. data/lib/fluent/daemon.rb +2 -4
  17. data/lib/fluent/event.rb +2 -2
  18. data/lib/fluent/file_wrapper.rb +137 -0
  19. data/lib/fluent/log/console_adapter.rb +66 -0
  20. data/lib/fluent/log.rb +35 -5
  21. data/lib/fluent/oj_options.rb +1 -2
  22. data/lib/fluent/plugin/base.rb +5 -7
  23. data/lib/fluent/plugin/buf_file.rb +32 -3
  24. data/lib/fluent/plugin/buf_file_single.rb +32 -3
  25. data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
  26. data/lib/fluent/plugin/buffer.rb +21 -0
  27. data/lib/fluent/plugin/in_tail.rb +1 -6
  28. data/lib/fluent/plugin/in_tcp.rb +47 -2
  29. data/lib/fluent/plugin/out_file.rb +0 -4
  30. data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
  31. data/lib/fluent/plugin/out_forward.rb +2 -2
  32. data/lib/fluent/plugin/out_secondary_file.rb +39 -22
  33. data/lib/fluent/plugin/output.rb +49 -12
  34. data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
  35. data/lib/fluent/plugin_helper/server.rb +8 -0
  36. data/lib/fluent/supervisor.rb +157 -232
  37. data/lib/fluent/test/driver/base.rb +11 -5
  38. data/lib/fluent/test/driver/filter.rb +4 -0
  39. data/lib/fluent/test/startup_shutdown.rb +6 -8
  40. data/lib/fluent/version.rb +1 -1
  41. data/templates/new_gem/test/helper.rb.erb +0 -1
  42. data/test/command/test_ctl.rb +1 -1
  43. data/test/command/test_fluentd.rb +168 -22
  44. data/test/command/test_plugin_config_formatter.rb +0 -1
  45. data/test/compat/test_parser.rb +5 -5
  46. data/test/config/test_system_config.rb +0 -8
  47. data/test/log/test_console_adapter.rb +110 -0
  48. data/test/plugin/out_forward/test_ack_handler.rb +39 -0
  49. data/test/plugin/test_base.rb +98 -0
  50. data/test/plugin/test_buf_file.rb +62 -23
  51. data/test/plugin/test_buf_file_single.rb +65 -0
  52. data/test/plugin/test_in_http.rb +2 -3
  53. data/test/plugin/test_in_monitor_agent.rb +2 -3
  54. data/test/plugin/test_in_tail.rb +105 -103
  55. data/test/plugin/test_in_tcp.rb +87 -2
  56. data/test/plugin/test_in_udp.rb +28 -0
  57. data/test/plugin/test_out_file.rb +3 -2
  58. data/test/plugin/test_out_forward.rb +14 -18
  59. data/test/plugin/test_out_http.rb +1 -0
  60. data/test/plugin/test_output.rb +269 -0
  61. data/test/plugin/test_output_as_buffered_compress.rb +32 -18
  62. data/test/plugin/test_parser_regexp.rb +1 -6
  63. data/test/plugin_helper/test_http_server_helper.rb +1 -1
  64. data/test/plugin_helper/test_server.rb +59 -5
  65. data/test/test_config.rb +57 -21
  66. data/test/{plugin/test_file_wrapper.rb → test_file_wrapper.rb} +2 -2
  67. data/test/test_formatter.rb +23 -20
  68. data/test/test_log.rb +85 -40
  69. data/test/test_supervisor.rb +300 -283
  70. metadata +15 -23
  71. data/.drone.yml +0 -35
  72. data/.github/workflows/issue-auto-closer.yml +0 -12
  73. data/.gitlab-ci.yml +0 -103
  74. data/lib/fluent/plugin/file_wrapper.rb +0 -131
  75. data/test/test_logger_initializer.rb +0 -46
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: af4e72332c4c1128c66e129579a50d0edd10b2bbeae16d805ed57f13cf425ae2
4
- data.tar.gz: de907aa30f8d0352a871b8fb6cc938ffdffef7012910dbb99cc3986b69b6929e
3
+ metadata.gz: e88bbbd20e75035745c46d2a6d86395113244ebdd773071c10dd1093b20216db
4
+ data.tar.gz: cbf603733cd993bfe412328c75ffb82fd068a1779eb2a2750f0caca84f7da845
5
5
  SHA512:
6
- metadata.gz: 908bc3de0f3d959d520e91cd62e6191f9b16c333769b8d1965fd7d0a8bdb585ad203d86fd5061b5d0e45cf7db33e81c2f44dd3f45cb81ee6c28dc287838f349f
7
- data.tar.gz: c3e4b97d24ffb3f699d6d0d8003ae8bdcb1b83b2d0c983ae725f98024935cbc39742016efdbd106387cd8b0880dcc57ca837bde0820a568a301a8937337638b6
6
+ metadata.gz: a9b58aefd54f71282d1bbe40d3ed6def9cd45c4b82e4b5c069004af89530cde348fa5e8af63767a13813aea0e6bac7627d9bf86df4e910117f4d17d4e565b767
7
+ data.tar.gz: 64e0de5f1ffa604a0b7e1087145f62fa8de0dc766bcedf370f037531a2d0f817516bb91d7c3eb45aaa9b02471cf1617f7868754b559faa7eac02f0f641f16925
data/.github/ISSUE_TEMPLATE/bug_report.yaml CHANGED
@@ -1,5 +1,6 @@
1
1
  name: Bug Report
2
2
  description: Create a report with a procedure for reproducing the bug
3
+ labels: "waiting-for-triage"
3
4
  body:
4
5
  - type: markdown
5
6
  attributes:
data/.github/ISSUE_TEMPLATE/feature_request.yaml CHANGED
@@ -1,5 +1,6 @@
1
1
  name: Feature request
2
2
  description: Suggest an idea for this project
3
+ labels: "waiting-for-triage"
3
4
  body:
4
5
  - type: markdown
5
6
  attributes:
data/.github/workflows/linux-test.yaml CHANGED
@@ -13,7 +13,7 @@ jobs:
13
13
  strategy:
14
14
  fail-fast: false
15
15
  matrix:
16
- ruby-version: ['3.1', '3.0', '2.7']
16
+ ruby-version: ['3.2', '3.1', '3.0', '2.7']
17
17
  os: [ubuntu-latest]
18
18
  experimental: [false]
19
19
  include:
@@ -23,7 +23,7 @@ jobs:
23
23
 
24
24
  name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
25
25
  steps:
26
- - uses: actions/checkout@v2
26
+ - uses: actions/checkout@v3
27
27
  - name: Set up Ruby
28
28
  uses: ruby/setup-ruby@v1
29
29
  with:
data/.github/workflows/macos-test.yaml CHANGED
@@ -13,7 +13,7 @@ jobs:
13
13
  strategy:
14
14
  fail-fast: false
15
15
  matrix:
16
- ruby-version: ['3.1', '3.0', '2.7']
16
+ ruby-version: ['3.2', '3.1', '3.0', '2.7']
17
17
  os: [macos-latest]
18
18
  experimental: [true]
19
19
  include:
@@ -23,7 +23,7 @@ jobs:
23
23
 
24
24
  name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
25
25
  steps:
26
- - uses: actions/checkout@v2
26
+ - uses: actions/checkout@v3
27
27
  - name: Set up Ruby
28
28
  uses: ruby/setup-ruby@v1
29
29
  with:
data/.github/workflows/stale-actions.yml CHANGED
@@ -7,16 +7,18 @@ jobs:
7
7
  stale:
8
8
  runs-on: ubuntu-latest
9
9
  steps:
10
- - uses: actions/stale@v3
10
+ - uses: actions/stale@v8
11
11
  with:
12
12
  repo-token: ${{ secrets.GITHUB_TOKEN }}
13
- days-before-stale: 90
14
- days-before-close: 30
15
- stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
16
- stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
17
- close-issue-message: "This issue was automatically closed because of stale in 30 days"
18
- close-pr-message: "This PR was automatically closed because of stale in 30 days"
13
+ days-before-stale: 30
14
+ days-before-close: 7
15
+ stale-issue-message: "This issue has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this issue will be closed in 7 days"
16
+ stale-pr-message: "This PR has been automatically marked as stale because it has been open 30 days with no activity. Remove stale label or comment or this PR will be closed in 7 days"
17
+ close-issue-message: "This issue was automatically closed because of stale in 7 days"
18
+ close-pr-message: "This PR was automatically closed because of stale in 7 days"
19
19
  stale-pr-label: "stale"
20
20
  stale-issue-label: "stale"
21
- exempt-issue-labels: "bug,enhancement,feature request,pending,work_in_progress,v1,v2"
22
- exempt-pr-labels: "bug,enhancement,feature request,pending,work_in_progress,v1,v2"
21
+ exempt-issue-labels: "waiting-for-triage,bug,enhancement,feature request,pending,work_in_progress,v1,v2"
22
+ exempt-pr-labels: "waiting-for-triage,bug,enhancement,feature request,pending,work_in_progress,v1,v2"
23
+ exempt-all-assignees: true
24
+ exempt-all-milestones: true
data/.github/workflows/windows-test.yaml CHANGED
@@ -13,7 +13,7 @@ jobs:
13
13
  strategy:
14
14
  fail-fast: false
15
15
  matrix:
16
- ruby-version: ['3.1', '2.7']
16
+ ruby-version: ['3.2', '3.1', '2.7']
17
17
  os:
18
18
  - windows-latest
19
19
  experimental: [false]
@@ -35,7 +35,7 @@ jobs:
35
35
 
36
36
  name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
37
37
  steps:
38
- - uses: actions/checkout@v2
38
+ - uses: actions/checkout@v3
39
39
  - name: Set up Ruby
40
40
  uses: ruby/setup-ruby@v1
41
41
  with:
data/CHANGELOG.md CHANGED
@@ -1,5 +1,138 @@
1
+ # v1.16
2
+
3
+ ## Release v1.16.1 - 2023/04/17
4
+
5
+ ### Enhancement
6
+
7
+ * in_tcp: Add `message_length_limit` to drop large incoming data
8
+ https://github.com/fluent/fluentd/pull/4137
9
+
10
+ ### Bug Fix
11
+
12
+ * Fix NameError of `SecondaryFileOutput` when setting secondary other than
13
+ `out_secondary_file`
14
+ https://github.com/fluent/fluentd/pull/4124
15
+ * Server helper: Suppress error of `UDPServer` over `max_bytes` on Windows
16
+ https://github.com/fluent/fluentd/pull/4131
17
+ * Buffer: Fix that `compress` setting causes unexpected error when receiving
18
+ already compressed MessagePack
19
+ https://github.com/fluent/fluentd/pull/4147
20
+
21
+ ### Misc
22
+
23
+ * Update MAINTAINERS.md
24
+ https://github.com/fluent/fluentd/pull/4119
25
+ * Update security policy
26
+ https://github.com/fluent/fluentd/pull/4123
27
+ * Plugin template: Remove unnecessary code
28
+ https://github.com/fluent/fluentd/pull/4128
29
+ * Revive issue auto closer
30
+ https://github.com/fluent/fluentd/pull/4116
31
+ * Fix a link for the repository of td-agent
32
+ https://github.com/fluent/fluentd/pull/4145
33
+ * in_udp: add test of message_length_limit
34
+ https://github.com/fluent/fluentd/pull/4117
35
+ * Fix a typo of an argument of `Fluent::EventStream#each`
36
+ https://github.com/fluent/fluentd/pull/4148
37
+ * Test in_tcp: Fix undesirable way to assert logs
38
+ https://github.com/fluent/fluentd/pull/4138
39
+
40
+ ## Release v1.16.0 - 2023/03/29
41
+
42
+ ### Enhancement
43
+
44
+ * in_tcp: Add `send_keepalive_packet` option
45
+ https://github.com/fluent/fluentd/pull/3961
46
+ * buffer: backup broken file chunk
47
+ https://github.com/fluent/fluentd/pull/4025
48
+ * Add warning messages for restoring buffer with `flush_at_shutdown true`
49
+ https://github.com/fluent/fluentd/pull/4027
50
+ * Add logs for time period of restored buffer possibly broken
51
+ https://github.com/fluent/fluentd/pull/4028
52
+
53
+ ### Bug Fix
54
+
55
+ * http_server_helper: Fix format of log messages originating from Async gem
56
+ https://github.com/fluent/fluentd/pull/3987
57
+ * Change to not generate a sigdump file after receiving a `SIGTERM` signal on
58
+ non-Windows
59
+ https://github.com/fluent/fluentd/pull/4034
60
+ https://github.com/fluent/fluentd/pull/4043
61
+ * out_forward: fix error of ack handling conflict on stopping with
62
+ `require_ack_response` enabled
63
+ https://github.com/fluent/fluentd/pull/4030
64
+ * Fix problem that some `system` configs are not reflected
65
+ https://github.com/fluent/fluentd/pull/4064
66
+ https://github.com/fluent/fluentd/pull/4065
67
+ https://github.com/fluent/fluentd/pull/4086
68
+ https://github.com/fluent/fluentd/pull/4090
69
+ https://github.com/fluent/fluentd/pull/4096
70
+ * Fix bug that the logger outputs some initial log messages without applying
71
+ some settings such as `format`
72
+ https://github.com/fluent/fluentd/pull/4091
73
+ * Windows: Fix a bug that the wrong log file is reopened with log rotate setting
74
+ when flushing or graceful reloading
75
+ https://github.com/fluent/fluentd/pull/4054
76
+ * Fix race condition of out_secondary_file
77
+ https://github.com/fluent/fluentd/pull/4081
78
+ * Suppress warning using different secondary for out_secondary_file
79
+ https://github.com/fluent/fluentd/pull/4087
80
+ * Fix value of `system_config.workers` at `run_configure`.
81
+ Change argument type of `Fluent::Plugin::Base::configure()` to
82
+ `Fluent::Config::Element` only.
83
+ https://github.com/fluent/fluentd/pull/4066
84
+ * Fix bug that Fluentd sometimes tries to use an unavailable port and fails to
85
+ start on Windows
86
+ https://github.com/fluent/fluentd/pull/4092
87
+
88
+ ### Misc
89
+
90
+ * Add method for testing `filtered_with_time`
91
+ https://github.com/fluent/fluentd/pull/3899
92
+ * Replace `$$` with `Process.pid`
93
+ https://github.com/fluent/fluentd/pull/4040
94
+ * Relax required webric gem version
95
+ https://github.com/fluent/fluentd/pull/4061
96
+ * CI fixes to support Ruby 3.2
97
+ https://github.com/fluent/fluentd/pull/3968
98
+ https://github.com/fluent/fluentd/pull/3996
99
+ https://github.com/fluent/fluentd/pull/3997
100
+ * Other CI fixes
101
+ https://github.com/fluent/fluentd/pull/3969
102
+ https://github.com/fluent/fluentd/pull/3990
103
+ https://github.com/fluent/fluentd/pull/4013
104
+ https://github.com/fluent/fluentd/pull/4033
105
+ https://github.com/fluent/fluentd/pull/4044
106
+ https://github.com/fluent/fluentd/pull/4050
107
+ https://github.com/fluent/fluentd/pull/4062
108
+ https://github.com/fluent/fluentd/pull/4074
109
+ https://github.com/fluent/fluentd/pull/4082
110
+ https://github.com/fluent/fluentd/pull/4085
111
+ * Update MAINTAINERS.md
112
+ https://github.com/fluent/fluentd/pull/4026
113
+ https://github.com/fluent/fluentd/pull/4069
114
+
1
115
  # v1.15
2
116
 
117
+ ## Release v1.15.3 - 2022/11/02
118
+
119
+ ### Bug Fix
120
+
121
+ * Support glob for `!include` directive in YAML config format
122
+ https://github.com/fluent/fluentd/pull/3917
123
+ * Remove meaningless oj options
124
+ https://github.com/fluent/fluentd/pull/3929
125
+ * Fix log initializer to correctly create per-process files on Windows
126
+ https://github.com/fluent/fluentd/pull/3939
127
+ * out_file: Fix the multi-worker check with `<worker 0-N>` directive
128
+ https://github.com/fluent/fluentd/pull/3942
129
+
130
+ ### Misc
131
+
132
+ * Fix broken tests on Ruby 3.2
133
+ https://github.com/fluent/fluentd/pull/3883
134
+ https://github.com/fluent/fluentd/pull/3922
135
+
3
136
  ## Release v1.15.2 - 2022/08/22
4
137
 
5
138
  ### Enhancement
data/CONTRIBUTING.md CHANGED
@@ -27,7 +27,7 @@ submitting an issue to Fluentd. Even better you can submit a Pull Request with a
27
27
  * **Documentation**: Use [fluentd documentation](https://github.com/fluent/fluentd-docs-gitbook) repository.
28
28
 
29
29
  If you find a bug of 3rd party plugins, please submit an issue to each plugin repository.
30
- And use [omnibus-td-agent](https://github.com/treasure-data/omnibus-td-agent) repository for td-agent related issues.
30
+ And use [fluent-package-builder](https://github.com/fluent/fluent-package-builder) repository for td-agent related issues.
31
31
 
32
32
  Note: Before report the issue, check latest version first. Sometimes users report fixed bug with older version.
33
33
 
data/MAINTAINERS.md CHANGED
@@ -3,9 +3,11 @@
3
3
  - [Naotoshi Seo](https://github.com/sonots), [ZOZO Technologies](https://tech.zozo.com/en/)
4
4
  - [Okkez](https://github.com/okkez)
5
5
  - [Hiroshi Hatake](https://github.com/cosmo0920), [Calyptia](https://calyptia.com/)
6
- - [Masahiro Nakagawa](https://github.com/repeatedly), [Treasure Data](https://www.treasuredata.com/)
7
- - [Satoshi Tagomori](https://github.com/tagomoris), [Treasure Data](https://www.treasuredata.com/)
6
+ - [Masahiro Nakagawa](https://github.com/repeatedly)
7
+ - [Satoshi Tagomori](https://github.com/tagomoris)
8
+ - [Toru Takahashi](https://github.com/toru-takahashi), [Treasure Data](https://www.treasuredata.com/)
8
9
  - [Eduardo Silva](https://github.com/edsiper), [Calyptia](https://calyptia/)
9
- - [Fujimoto Seiji](https://github.com/fujimots), [ClearCode](https://www.clear-code.com/)
10
+ - [Fujimoto Seiji](https://github.com/fujimots)
10
11
  - [Takuro Ashie](https://github.com/ashie), [ClearCode](https://www.clear-code.com/)
11
12
  - [Kentaro Hayashi](https://github.com/kenhys), [ClearCode](https://www.clear-code.com/)
13
+ - [Daijiro Fukuda](https://github.com/daipom), [ClearCode](https://www.clear-code.com/)
data/README.md CHANGED
@@ -4,7 +4,6 @@ Fluentd: Open-Source Log Collector
4
4
  [![Testing on Ubuntu](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/linux-test.yaml)
5
5
  [![Testing on Windows](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/windows-test.yaml)
6
6
  [![Testing on macOS](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml/badge.svg?branch=master)](https://github.com/fluent/fluentd/actions/workflows/macos-test.yaml)
7
- [![Testing on Arm64](https://cloud.drone.io/api/badges/fluent/fluentd/status.svg?branch=master)](https://cloud.drone.io/fluent/fluentd)
8
7
  [![Code Climate](https://codeclimate.com/github/fluent/fluentd/badges/gpa.svg)](https://codeclimate.com/github/fluent/fluentd)
9
8
  [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1189/badge)](https://bestpractices.coreinfrastructure.org/projects/1189)
10
9
 
data/SECURITY.md CHANGED
@@ -4,15 +4,11 @@
4
4
 
5
5
  | Version | Supported |
6
6
  | ------- | ------------------ |
7
- | 1.14.x | :white_check_mark: |
8
- | <= 1.13.x | :x: |
7
+ | 1.16.x | :white_check_mark: |
8
+ | 1.15.x | :white_check_mark: |
9
+ | <= 1.14.x | :x: |
9
10
 
10
11
  ## Reporting a Vulnerability
11
12
 
12
- Please contact to current active maintainers. (in alphabetical order)
13
-
14
- * ashie@clear-code.com
15
- * fujimoto@clear-code.com
16
- * hatake@calyptia.com
17
- * hayashi@clear-code.com
18
-
13
+ Please post your vulnerability report from the following page:
14
+ https://github.com/fluent/fluentd/security/advisories
data/fluentd.gemspec CHANGED
@@ -22,13 +22,13 @@ Gem::Specification.new do |gem|
22
22
  gem.add_runtime_dependency("msgpack", [">= 1.3.1", "< 2.0.0"])
23
23
  gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
24
24
  gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
25
- gem.add_runtime_dependency("serverengine", [">= 2.3.0", "< 3.0.0"])
25
+ gem.add_runtime_dependency("serverengine", [">= 2.3.2", "< 3.0.0"])
26
26
  gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
27
27
  gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
28
28
  gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
29
29
  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
30
30
  gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
31
- gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
31
+ gem.add_runtime_dependency("webrick", ["~> 1.4"])
32
32
 
33
33
  # build gem for a certain platform. see also Rakefile
34
34
  fake_platform = ENV['GEM_BUILD_FAKE_PLATFORM'].to_s
data/lib/fluent/command/fluentd.rb CHANGED
@@ -26,26 +26,27 @@ $fluentdargv = Marshal.load(Marshal.dump(ARGV))
26
26
  op = OptionParser.new
27
27
  op.version = Fluent::VERSION
28
28
 
29
- opts = Fluent::Supervisor.default_options
29
+ default_opts = Fluent::Supervisor.default_options
30
+ cmd_opts = {}
30
31
 
31
32
  op.on('-s', "--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]", "install sample configuration file to the directory") {|s|
32
- opts[:setup_path] = s || File.dirname(Fluent::DEFAULT_CONFIG_PATH)
33
+ cmd_opts[:setup_path] = s || File.dirname(Fluent::DEFAULT_CONFIG_PATH)
33
34
  }
34
35
 
35
36
  op.on('-c', '--config PATH', "config file path (default: #{Fluent::DEFAULT_CONFIG_PATH})") {|s|
36
- opts[:config_path] = s
37
+ cmd_opts[:config_path] = s
37
38
  }
38
39
 
39
40
  op.on('--dry-run', "Check fluentd setup is correct or not", TrueClass) {|b|
40
- opts[:dry_run] = b
41
+ cmd_opts[:dry_run] = b
41
42
  }
42
43
 
43
44
  op.on('--show-plugin-config=PLUGIN', "[DEPRECATED] Show PLUGIN configuration and exit(ex: input:dummy)") {|plugin|
44
- opts[:show_plugin_config] = plugin
45
+ cmd_opts[:show_plugin_config] = plugin
45
46
  }
46
47
 
47
48
  op.on('-p', '--plugin DIR', "add plugin directory") {|s|
48
- opts[:plugin_dirs] << s
49
+ (cmd_opts[:plugin_dirs] ||= []) << s
49
50
  }
50
51
 
51
52
  op.on('-I PATH', "add library path") {|s|
@@ -53,48 +54,48 @@ op.on('-I PATH', "add library path") {|s|
53
54
  }
54
55
 
55
56
  op.on('-r NAME', "load library") {|s|
56
- opts[:libs] << s
57
+ (cmd_opts[:libs] ||= []) << s
57
58
  }
58
59
 
59
60
  op.on('-d', '--daemon PIDFILE', "daemonize fluent process") {|s|
60
- opts[:daemonize] = s
61
+ cmd_opts[:daemonize] = s
61
62
  }
62
63
 
63
64
  op.on('--under-supervisor', "run fluent worker under supervisor (this option is NOT for users)") {
64
- opts[:supervise] = false
65
+ cmd_opts[:supervise] = false
65
66
  }
66
67
 
67
68
  op.on('--no-supervisor', "run fluent worker without supervisor") {
68
- opts[:supervise] = false
69
- opts[:standalone_worker] = true
69
+ cmd_opts[:supervise] = false
70
+ cmd_opts[:standalone_worker] = true
70
71
  }
71
72
 
72
73
  op.on('--workers NUM', "specify the number of workers under supervisor") { |i|
73
- opts[:workers] = i.to_i
74
+ cmd_opts[:workers] = i.to_i
74
75
  }
75
76
 
76
77
  op.on('--user USER', "change user") {|s|
77
- opts[:chuser] = s
78
+ cmd_opts[:chuser] = s
78
79
  }
79
80
 
80
81
  op.on('--group GROUP', "change group") {|s|
81
- opts[:chgroup] = s
82
+ cmd_opts[:chgroup] = s
82
83
  }
83
84
 
84
85
  op.on('--umask UMASK', "change umask") {|s|
85
- opts[:chumask] = s
86
+ cmd_opts[:chumask] = s
86
87
  }
87
88
 
88
89
  op.on('-o', '--log PATH', "log file path") {|s|
89
- opts[:log_path] = s
90
+ cmd_opts[:log_path] = s
90
91
  }
91
92
 
92
93
  op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
93
94
  if Fluent::Log::LOG_ROTATE_AGE.include?(age)
94
- opts[:log_rotate_age] = age
95
+ cmd_opts[:log_rotate_age] = age
95
96
  else
96
97
  begin
97
- opts[:log_rotate_age] = Integer(age)
98
+ cmd_opts[:log_rotate_age] = Integer(age)
98
99
  rescue TypeError, ArgumentError
99
100
  usage "log-rotate-age should be #{ROTATE_AGE.join(', ')} or a number"
100
101
  end
@@ -102,129 +103,129 @@ op.on('--log-rotate-age AGE', 'generations to keep rotated log files') {|age|
102
103
  }
103
104
 
104
105
  op.on('--log-rotate-size BYTES', 'sets the byte size to rotate log files') {|s|
105
- opts[:log_rotate_size] = s.to_i
106
+ cmd_opts[:log_rotate_size] = s.to_i
106
107
  }
107
108
 
108
109
  op.on('--log-event-verbose', 'enable log events during process startup/shutdown') {|b|
109
- opts[:log_event_verbose] = b
110
+ cmd_opts[:log_event_verbose] = b
110
111
  }
111
112
 
112
113
  op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to the config file on-the-fly") {|s|
113
- opts[:inline_config] = s
114
+ cmd_opts[:inline_config] = s
114
115
  }
115
116
 
116
117
  op.on('--emit-error-log-interval SECONDS', "suppress interval seconds of emit error logs") {|s|
117
- opts[:suppress_interval] = s.to_i
118
+ cmd_opts[:suppress_interval] = s.to_i
118
119
  }
119
120
 
120
121
  op.on('--suppress-repeated-stacktrace [VALUE]', "suppress repeated stacktrace", TrueClass) {|b|
121
122
  b = true if b.nil?
122
- opts[:suppress_repeated_stacktrace] = b
123
+ cmd_opts[:suppress_repeated_stacktrace] = b
123
124
  }
124
125
 
125
126
  op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {|b|
126
- opts[:without_source] = b
127
+ cmd_opts[:without_source] = b
127
128
  }
128
129
 
129
130
  op.on('--config-file-type VALU', 'guessing file type of fluentd configuration. yaml/yml or guess') { |s|
130
131
  if (s == 'yaml') || (s == 'yml')
131
- opts[:config_file_type] = s.to_sym
132
+ cmd_opts[:config_file_type] = s.to_sym
132
133
  elsif (s == 'guess')
133
- opts[:config_file_type] = s.to_sym
134
+ cmd_opts[:config_file_type] = s.to_sym
134
135
  else
135
136
  usage '--config-file-type accepts yaml/yml or guess'
136
137
  end
137
138
  }
138
139
 
139
140
  op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b|
140
- opts[:use_v1_config] = b
141
+ cmd_opts[:use_v1_config] = b
141
142
  }
142
143
 
143
144
  op.on('--use-v0-config', "Use v0 configuration format", TrueClass) {|b|
144
- opts[:use_v1_config] = !b
145
+ cmd_opts[:use_v1_config] = !b
145
146
  }
146
147
 
147
148
  op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
148
- opts[:strict_config_value] = b
149
+ cmd_opts[:strict_config_value] = b
149
150
  }
150
151
 
151
152
  op.on('--enable-input-metrics', "Enable input plugin metrics on fluentd", TrueClass) {|b|
152
- opts[:enable_input_metrics] = b
153
+ cmd_opts[:enable_input_metrics] = b
153
154
  }
154
155
 
155
156
  op.on('--enable-size-metrics', "Enable plugin record size metrics on fluentd", TrueClass) {|b|
156
- opts[:enable_size_metrics] = b
157
+ cmd_opts[:enable_size_metrics] = b
157
158
  }
158
159
 
159
160
  op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
160
- if b
161
- opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max
162
- end
161
+ return unless b
162
+ cur_level = cmd_opts.fetch(:log_level, default_opts[:log_level])
163
+ cmd_opts[:log_level] = [cur_level - 1, Fluent::Log::LEVEL_TRACE].max
163
164
  }
164
165
 
165
166
  op.on('-q', '--quiet', "decrease verbose level (-q: warn, -qq: error)", TrueClass) {|b|
166
- if b
167
- opts[:log_level] = [opts[:log_level] + 1, Fluent::Log::LEVEL_ERROR].min
168
- end
167
+ return unless b
168
+ cur_level = cmd_opts.fetch(:log_level, default_opts[:log_level])
169
+ cmd_opts[:log_level] = [cur_level + 1, Fluent::Log::LEVEL_ERROR].min
169
170
  }
170
171
 
171
172
  op.on('--suppress-config-dump', "suppress config dumping when fluentd starts", TrueClass) {|b|
172
- opts[:suppress_config_dump] = b
173
+ cmd_opts[:suppress_config_dump] = b
173
174
  }
174
175
 
175
176
  op.on('-g', '--gemfile GEMFILE', "Gemfile path") {|s|
176
- opts[:gemfile] = s
177
+ cmd_opts[:gemfile] = s
177
178
  }
178
179
 
179
180
  op.on('-G', '--gem-path GEM_INSTALL_PATH', "Gemfile install path (default: $(dirname $gemfile)/vendor/bundle)") {|s|
180
- opts[:gem_install_path] = s
181
+ cmd_opts[:gem_install_path] = s
181
182
  }
182
183
 
183
184
  op.on('--conf-encoding ENCODING', "specify configuration file encoding") { |s|
184
- opts[:conf_encoding] = s
185
+ cmd_opts[:conf_encoding] = s
185
186
  }
186
187
 
187
188
  op.on('--disable-shared-socket', "Don't open shared socket for multiple workers") { |b|
188
- opts[:disable_shared_socket] = b
189
+ cmd_opts[:disable_shared_socket] = b
189
190
  }
190
191
 
191
192
  if Fluent.windows?
192
- opts.merge!(
193
+ cmd_opts.merge!(
193
194
  :winsvc_name => 'fluentdwinsvc',
194
195
  :winsvc_display_name => 'Fluentd Windows Service',
195
196
  :winsvc_desc => 'Fluentd is an event collector system.',
196
197
  )
197
198
 
198
199
  op.on('-x', '--signame INTSIGNAME', "an object name which is used for Windows Service signal (Windows only)") {|s|
199
- opts[:signame] = s
200
+ cmd_opts[:signame] = s
200
201
  }
201
202
 
202
203
  op.on('--reg-winsvc MODE', "install/uninstall as Windows Service. (i: install, u: uninstall) (Windows only)") {|s|
203
- opts[:regwinsvc] = s
204
+ cmd_opts[:regwinsvc] = s
204
205
  }
205
206
 
206
207
  op.on('--[no-]reg-winsvc-auto-start', "Automatically start the Windows Service at boot. (only effective with '--reg-winsvc i') (Windows only)") {|s|
207
- opts[:regwinsvcautostart] = s
208
+ cmd_opts[:regwinsvcautostart] = s
208
209
  }
209
210
 
210
211
  op.on('--[no-]reg-winsvc-delay-start', "Automatically start the Windows Service at boot with delay. (only effective with '--reg-winsvc i' and '--reg-winsvc-auto-start') (Windows only)") {|s|
211
- opts[:regwinsvcdelaystart] = s
212
+ cmd_opts[:regwinsvcdelaystart] = s
212
213
  }
213
214
 
214
215
  op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option parameters for Windows Service. (Windows only)") {|s|
215
- opts[:fluentdopt] = s
216
+ cmd_opts[:fluentdopt] = s
216
217
  }
217
218
 
218
219
  op.on('--winsvc-name NAME', "The Windows Service name to run as (Windows only)") {|s|
219
- opts[:winsvc_name] = s
220
+ cmd_opts[:winsvc_name] = s
220
221
  }
221
222
 
222
223
  op.on('--winsvc-display-name DISPLAY_NAME', "The Windows Service display name (Windows only)") {|s|
223
- opts[:winsvc_display_name] = s
224
+ cmd_opts[:winsvc_display_name] = s
224
225
  }
225
226
 
226
227
  op.on('--winsvc-desc DESC', "The Windows Service description (Windows only)") {|s|
227
- opts[:winsvc_desc] = s
228
+ cmd_opts[:winsvc_desc] = s
228
229
  }
229
230
  end
230
231
 
@@ -247,6 +248,7 @@ rescue
247
248
  usage $!.to_s
248
249
  end
249
250
 
251
+ opts = default_opts.merge(cmd_opts)
250
252
 
251
253
  ##
252
254
  ## Bundler injection
@@ -345,18 +347,7 @@ end
345
347
  exit 0 if early_exit
346
348
 
347
349
  if opts[:supervise]
348
- if Fluent.windows?
349
- if opts[:log_path] && opts[:log_path] != "-"
350
- if opts[:log_rotate_age] || opts[:log_rotate_size]
351
- require 'pathname'
352
-
353
- log_path = Pathname(opts[:log_path]).sub_ext("-supervisor#{Pathname(opts[:log_path]).extname}").to_s
354
- opts[:log_path] = log_path
355
- end
356
- end
357
- end
358
-
359
- supervisor = Fluent::Supervisor.new(opts)
350
+ supervisor = Fluent::Supervisor.new(cmd_opts)
360
351
  supervisor.configure(supervisor: true)
361
352
  supervisor.run_supervisor(dry_run: opts[:dry_run])
362
353
  else
@@ -364,7 +355,7 @@ else
364
355
  puts "Error: multi workers is not supported with --no-supervisor"
365
356
  exit 2
366
357
  end
367
- worker = Fluent::Supervisor.new(opts)
358
+ worker = Fluent::Supervisor.new(cmd_opts)
368
359
  worker.configure
369
360
 
370
361
  if opts[:daemonize] && opts[:standalone_worker]
data/lib/fluent/config/yaml_parser/loader.rb CHANGED
@@ -44,7 +44,7 @@ module Fluent
44
44
  visitor = Visitor.new(scanner, class_loader)
45
45
 
46
46
  visitor._register_domain(INCLUDE_TAG) do |_, val|
47
- load(path.parent.join(val))
47
+ eval_include(Pathname.new(val), path.parent)
48
48
  end
49
49
 
50
50
  visitor._register_domain(FLUENT_JSON_TAG) do |_, val|
@@ -60,6 +60,23 @@ module Fluent
60
60
  end
61
61
  end
62
62
 
63
+ def eval_include(path, parent)
64
+ if path.relative?
65
+ pattern = parent.join(path)
66
+ else
67
+ pattern = path
68
+ end
69
+ result = []
70
+ Dir.glob(pattern).sort.each do |path|
71
+ result.concat(load(Pathname.new(path)))
72
+ end
73
+ result
74
+ rescue SystemCallError => e
75
+ parse_error = ConfigParseError.new("include error #{path} - #{e}")
76
+ parse_error.set_backtrace(e.backtrace)
77
+ raise parse_error
78
+ end
79
+
63
80
  class Visitor < Psych::Visitors::ToRuby
64
81
  def initialize(scanner, class_loader)
65
82
  super(scanner, class_loader)