fluentd 1.13.1-x64-mingw32 → 1.14.1-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.yaml +69 -0
  3. data/.github/ISSUE_TEMPLATE/feature_request.yaml +38 -0
  4. data/.github/workflows/windows-test.yaml +3 -3
  5. data/CHANGELOG.md +136 -0
  6. data/README.md +2 -2
  7. data/example/v0_12_filter.conf +2 -2
  8. data/fluentd.gemspec +1 -1
  9. data/lib/fluent/command/fluentd.rb +8 -0
  10. data/lib/fluent/command/plugin_generator.rb +15 -5
  11. data/lib/fluent/compat/output.rb +9 -6
  12. data/lib/fluent/config/parser.rb +1 -1
  13. data/lib/fluent/config/types.rb +15 -0
  14. data/lib/fluent/config/v1_parser.rb +4 -3
  15. data/lib/fluent/config.rb +1 -1
  16. data/lib/fluent/env.rb +2 -1
  17. data/lib/fluent/event_router.rb +28 -1
  18. data/lib/fluent/oj_options.rb +62 -0
  19. data/lib/fluent/plugin/bare_output.rb +49 -8
  20. data/lib/fluent/plugin/buffer.rb +84 -22
  21. data/lib/fluent/plugin/file_wrapper.rb +22 -0
  22. data/lib/fluent/plugin/filter.rb +35 -1
  23. data/lib/fluent/plugin/formatter.rb +1 -0
  24. data/lib/fluent/plugin/formatter_json.rb +9 -7
  25. data/lib/fluent/plugin/in_http.rb +21 -2
  26. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  27. data/lib/fluent/plugin/in_syslog.rb +13 -1
  28. data/lib/fluent/plugin/in_tail/position_file.rb +20 -18
  29. data/lib/fluent/plugin/in_tail.rb +77 -6
  30. data/lib/fluent/plugin/input.rb +39 -1
  31. data/lib/fluent/plugin/metrics.rb +119 -0
  32. data/lib/fluent/plugin/metrics_local.rb +96 -0
  33. data/lib/fluent/plugin/multi_output.rb +43 -6
  34. data/lib/fluent/plugin/out_copy.rb +1 -1
  35. data/lib/fluent/plugin/out_forward.rb +15 -7
  36. data/lib/fluent/plugin/output.rb +77 -36
  37. data/lib/fluent/plugin/parser_json.rb +2 -3
  38. data/lib/fluent/plugin.rb +10 -1
  39. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  40. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  41. data/lib/fluent/plugin_helper/server.rb +4 -2
  42. data/lib/fluent/plugin_helper.rb +1 -0
  43. data/lib/fluent/root_agent.rb +6 -0
  44. data/lib/fluent/supervisor.rb +2 -0
  45. data/lib/fluent/system_config.rb +9 -1
  46. data/lib/fluent/test/driver/storage.rb +30 -0
  47. data/lib/fluent/version.rb +1 -1
  48. data/templates/new_gem/lib/fluent/plugin/storage.rb.erb +40 -0
  49. data/templates/new_gem/test/plugin/test_storage.rb.erb +18 -0
  50. data/test/command/test_plugin_generator.rb +2 -1
  51. data/test/config/test_system_config.rb +6 -0
  52. data/test/config/test_types.rb +7 -0
  53. data/test/plugin/in_tail/test_io_handler.rb +12 -4
  54. data/test/plugin/in_tail/test_position_file.rb +48 -8
  55. data/test/plugin/test_bare_output.rb +13 -0
  56. data/test/plugin/test_buffer.rb +8 -2
  57. data/test/plugin/test_file_wrapper.rb +11 -0
  58. data/test/plugin/test_filter.rb +11 -0
  59. data/test/plugin/test_in_http.rb +40 -0
  60. data/test/plugin/test_in_monitor_agent.rb +214 -8
  61. data/test/plugin/test_in_syslog.rb +35 -0
  62. data/test/plugin/test_in_tail.rb +157 -29
  63. data/test/plugin/test_input.rb +11 -0
  64. data/test/plugin/test_metrics.rb +294 -0
  65. data/test/plugin/test_metrics_local.rb +96 -0
  66. data/test/plugin/test_multi_output.rb +25 -1
  67. data/test/plugin/test_output.rb +16 -0
  68. data/test/plugin_helper/test_event_emitter.rb +29 -0
  69. data/test/plugin_helper/test_metrics.rb +137 -0
  70. data/test/test_event_time.rb +2 -2
  71. data/test/test_oj_options.rb +55 -0
  72. data/test/test_plugin_classes.rb +102 -0
  73. data/test/test_root_agent.rb +30 -1
  74. metadata +21 -6
  75. data/.github/ISSUE_TEMPLATE/bug_report.md +0 -40
  76. data/.github/ISSUE_TEMPLATE/feature_request.md +0 -23
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7ab139d0823e71c7f395abb8fa59453f18477df1e3b54fe1da0bb7a7e9175729
4
- data.tar.gz: 693676284c38b07e959db0f571919654feca14bdc3488861d599722ddddd95a0
3
+ metadata.gz: 1b95a24ab25f46ab88d9e5bfc09a0f4374dbbb606523fade2d3dd0fb175227e2
4
+ data.tar.gz: 6569c24837680722e497009210e7bf64b1b3a50c9fe4b4ef4d6a89db8cbb0faf
5
5
  SHA512:
6
- metadata.gz: 80598188d2e509113db73f13703655c369e9e54f16a8349201085b4779489910b48ff601ab53542d45bcc9d4bfe71573c953f4ccfac9598ef562eb24cce9ad06
7
- data.tar.gz: d0dab564fd15451fbf664818a85cdde1724da2b5677f20ca4d9a4627ee57c5eb8f228bb1b07e954fa3c385978615278ba67e585aa409a8d5ddaaa4884075ca5e
6
+ metadata.gz: b06c10a7461f156e7da183c18e2ebe47d8e35d2dda2fd61ca607ddac6daa084c9b3e367952927211598a856b08e12c6aaccfe0f7be34c7a23acaac87bec0b858
7
+ data.tar.gz: 31dac402718a9e78416f2481704eec0ac35c154006c14f73c60c62149dc83bf53d55d3aad1b94a57f9dd45ae158b4eac979542de4e2d90ff41339881c16175e0
@@ -0,0 +1,69 @@
1
+ name: Bug Report
2
+ description: Create a report with a procedure for reproducing the bug
3
+ body:
4
+ - type: markdown
5
+ attributes:
6
+ value: |
7
+ Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.
8
+ - type: textarea
9
+ id: description
10
+ attributes:
11
+ label: Describe the bug
12
+ description: A clear and concise description of what the bug is
13
+ validations:
14
+ required: true
15
+ - type: textarea
16
+ id: reproduce
17
+ attributes:
18
+ label: To Reproduce
19
+ description: Steps to reproduce the behavior
20
+ validations:
21
+ required: true
22
+ - type: textarea
23
+ id: expected
24
+ attributes:
25
+ label: Expected behavior
26
+ description: A clear and concise description of what you expected to happen
27
+ validations:
28
+ required: true
29
+ - type: textarea
30
+ id: environment
31
+ attributes:
32
+ label: Your Environment
33
+ description: |
34
+ - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
35
+ - Operating system: `cat /etc/os-release`
36
+ - Kernel version: `uname -r`
37
+
38
+ Tip: If you hit the problem with older fluentd version, try latest version first.
39
+ value: |
40
+ - Fluentd version:
41
+ - TD Agent version:
42
+ - Operating system:
43
+ - Kernel version:
44
+ render: markdown
45
+ validations:
46
+ required: true
47
+ - type: textarea
48
+ id: configuration
49
+ attributes:
50
+ label: Your Configuration
51
+ description: |
52
+ Write your configuration here. Minimum reproducible fluentd.conf is recommended.
53
+ validations:
54
+ required: true
55
+ - type: textarea
56
+ id: logs
57
+ attributes:
58
+ label: Your Error Log
59
+ description: Write your ALL error log here
60
+ render: shell
61
+ validations:
62
+ required: true
63
+ - type: textarea
64
+ id: addtional-context
65
+ attributes:
66
+ label: Additional context
67
+ description: Add any other context about the problem here.
68
+ validations:
69
+ required: false
@@ -0,0 +1,38 @@
1
+ name: Feature request
2
+ description: Suggest an idea for this project
3
+ body:
4
+ - type: markdown
5
+ attributes:
6
+ value: |
7
+ Check [CONTRIBUTING guideline](https://github.com/fluent/fluentd/blob/master/CONTRIBUTING.md) first and here is the list to help us investigate the problem.
8
+ - type: textarea
9
+ id: description
10
+ attributes:
11
+ label: Is your feature request related to a problem? Please describe.
12
+ description: |
13
+ A clear and concise description of what the problem is.
14
+ Ex. I'm always frustrated when [...]
15
+ validations:
16
+ required: true
17
+ - type: textarea
18
+ id: solution
19
+ attributes:
20
+ label: Describe the solution you'd like
21
+ description: A clear and concise description of what you want to happen.
22
+ validations:
23
+ required: true
24
+ - type: textarea
25
+ id: alternative
26
+ attributes:
27
+ label: Describe alternatives you've considered
28
+ description: A clear and concise description of any alternative solutions or features you've considered.
29
+ validations:
30
+ required: true
31
+ - type: textarea
32
+ id: addtional-context
33
+ attributes:
34
+ label: Additional context
35
+ description: Add any other context or screenshots about the feature request here.
36
+ validations:
37
+ required: false
38
+
@@ -18,7 +18,7 @@ jobs:
18
18
  - windows-latest
19
19
  experimental: [false]
20
20
  include:
21
- - ruby-version: '3.0.1'
21
+ - ruby-version: '3.0.2'
22
22
  os: windows-latest
23
23
  experimental: true
24
24
  # On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
@@ -28,7 +28,7 @@ jobs:
28
28
  # * https://github.com/ruby/fiddle/issues/72
29
29
  # * https://bugs.ruby-lang.org/issues/17813
30
30
  # * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
31
- ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.1/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
31
+ ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.2/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
32
32
 
33
33
  name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
34
34
  steps:
@@ -38,7 +38,7 @@ jobs:
38
38
  with:
39
39
  ruby-version: ${{ matrix.ruby-version }}
40
40
  - name: Add Fiddle 1.0.8
41
- if: ${{ matrix.ruby-version == '3.0.1' }}
41
+ if: ${{ matrix.ruby-version == '3.0.2' }}
42
42
  run: gem install fiddle --version 1.0.8
43
43
  - name: Install dependencies
44
44
  run: ridk exec bundle install
data/CHANGELOG.md CHANGED
@@ -1,3 +1,139 @@
1
+ # v1.14.1
2
+
3
+ ## Release v1.14.1 - 2021/09/29
4
+
5
+ ### Enhancement
6
+
7
+ * in_tail: Added file related metrics.
8
+ These metrics should be collected same as fluent-bit's in_tail.
9
+ https://github.com/fluent/fluentd/pull/3504
10
+ * out_forward: Changed to use metrics mechanism for node statistics
11
+ https://github.com/fluent/fluentd/pull/3506
12
+
13
+ ### Bug fix
14
+
15
+ * in_tail: Fixed a crash bug that it raise undefined method of eof? error.
16
+ This error may happen only when `read_bytes_limit_per_second` was specified.
17
+ https://github.com/fluent/fluentd/pull/3500
18
+ * out_forward: Fixed a bug that node statistics information is not included correctly.
19
+ https://github.com/fluent/fluentd/pull/3503
20
+ https://github.com/fluent/fluentd/pull/3507
21
+ * Fixed a error when using `@include` directive
22
+ It was occurred when http/https scheme URI is used in `@include` directive with Ruby 3.
23
+ https://github.com/fluent/fluentd/pull/3517
24
+ * out_copy: Fixed to suppress a wrong warning for `ignore_if_prev_success`
25
+ It didn't work even if a user set it.
26
+ https://github.com/fluent/fluentd/pull/3515
27
+ * Fixed not to output nanoseconds field of next retry time in warning log
28
+ Then, inappropriate labels in log are also fixed. (retry_time -> retry_times,
29
+ next_retry_seconds -> next_retry_time)
30
+ https://github.com/fluent/fluentd/pull/3518
31
+
32
+ # v1.14.0
33
+
34
+ ## Release v1.14.0 - 2021/08/30
35
+
36
+ ### Enhancement
37
+
38
+ * Added `enable_input_metrics`, `enable_size_metrics` system
39
+ configuration parameter
40
+ This feature might need to pay higher CPU cost, so input event metrics
41
+ features are disabled by default. These features are also enabled by
42
+ `--enable-input-metrics`,`--enable-size-metrics` command line
43
+ option.
44
+ https://github.com/fluent/fluentd/pull/3440
45
+ * Added reserved word `@ROOT` for getting root router.
46
+ This is incompatible change. Do not use `@ROOT` for label name.
47
+ https://github.com/fluent/fluentd/pull/3358
48
+ * in_syslog: Added `send_keepalive_packet` option
49
+ https://github.com/fluent/fluentd/pull/3474
50
+ * in_http: Added `cors_allow_credentials` option.
51
+ This option tells browsers whether to expose the response to
52
+ frontend when the credentials mode is "include".
53
+ https://github.com/fluent/fluentd/pull/3481
54
+ https://github.com/fluent/fluentd/pull/3491
55
+
56
+ ### Bug fix
57
+
58
+ * in_tail: Fixed a bug that deleted paths are not removed
59
+ from pos file by file compaction at start up
60
+ https://github.com/fluent/fluentd/pull/3467
61
+ * in_tail: Revived a warning message of retrying unaccessible file
62
+ https://github.com/fluent/fluentd/pull/3478
63
+ * TLSServer: Fixed a crash bug on logging peer host name errors
64
+ https://github.com/fluent/fluentd/pull/3483
65
+
66
+ ### Misc
67
+
68
+ * Added metrics plugin mechanism
69
+ The implementations is changed to use metrics plugin.
70
+ In the future, 3rd party plugin will be able to handle these metrics.
71
+ https://github.com/fluent/fluentd/pull/3471
72
+ https://github.com/fluent/fluentd/pull/3473
73
+ https://github.com/fluent/fluentd/pull/3479
74
+ https://github.com/fluent/fluentd/pull/3484
75
+
76
+ # v1.13.3
77
+
78
+ ## Release v1.13.3 - 2021/07/27
79
+
80
+ ### Bug fix
81
+
82
+ * in_tail: Care DeletePending state on Windows
83
+ https://github.com/fluent/fluentd/pull/3457
84
+ https://github.com/fluent/fluentd/pull/3460
85
+ * in_tail: Fix some pos_file bugs.
86
+ Avoid deleting pos_file entries unexpectedly when both
87
+ `pos_file_compaction_interval` and `follow_inode` are enabled.
88
+ Use `bytesize` instead of `size` for path length.
89
+ https://github.com/fluent/fluentd/pull/3459
90
+ * in_tail: Fix detecting rotation twice on `follow_inode`.
91
+ https://github.com/fluent/fluentd/pull/3466
92
+
93
+ ### Misc
94
+
95
+ * Remove needless spaces in a sample config file
96
+ https://github.com/fluent/fluentd/pull/3456
97
+
98
+ ### Enhancement
99
+
100
+ # v1.13.2
101
+
102
+ ## Release v1.13.2 - 2021/07/12
103
+
104
+ ### Enhancement
105
+
106
+ * fluent-plugin-generate: Storage plugin was supported.
107
+ https://github.com/fluent/fluentd/pull/3426
108
+ * parser_json: Added support to customize configuration of oj options.
109
+ Use `FLUENT_OJ_OPTION_BIGDECIMAL_LOAD`, `FLUENT_OJ_OPTION_MAX_NESTING`,
110
+ `FLUENT_OJ_OPTION_MODE`, and `FLUENT_OJ_OPTION_USE_TO_JSON` environment
111
+ variable to configure it.
112
+ https://github.com/fluent/fluentd/pull/3315
113
+
114
+ ### Bug fix
115
+
116
+ * binlog_reader: Fixed a crash bug by missing "fluent/env" dependency.
117
+ https://github.com/fluent/fluentd/pull/3443
118
+ * Fixed a crash bug on outputting log at the early stage when parsing
119
+ config file. This is a regression since v1.13.0. If you use invalid
120
+ '@' prefix parameter, remove it as a workaround.
121
+ https://github.com/fluent/fluentd/pull/3451
122
+ * in_tail: Fixed a bug that when rotation is occurred, remaining lines
123
+ will be discarded if the throttling feature is enabled.
124
+ https://github.com/fluent/fluentd/pull/3390
125
+ * fluent-plugin-generate: Fixed a crash bug during gemspec generation.
126
+ It was unexpectedly introduced by #3305, thus this bug was a
127
+ regression since 1.12.3.
128
+ https://github.com/fluent/fluentd/pull/3444
129
+
130
+ ### Misc
131
+
132
+ * Fixed the runtime dependency version of http_parse.rb to 0.7.0.
133
+ It was fixed because false positive detection is occurred frequently
134
+ by security scanning tools.
135
+ https://github.com/fluent/fluentd/pull/3450
136
+
1
137
  # v1.13.1
2
138
 
3
139
  ## Release v1.13.1 - 2021/06/25
data/README.md CHANGED
@@ -25,9 +25,9 @@ An event consists of *tag*, *time* and *record*. Tag is a string separated with
25
25
 
26
26
  Use Case | Description | Diagram
27
27
  -------- | ------------|:---------:
28
- Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/assets/img/recipes/elasticsearch-s3-fluentd.png" height="150"/>
28
+ Centralizing Apache/Nginx Server Logs | Fluentd can be used to tail access/error logs and transport them reliably to remote systems. | <img src="https://www.fluentd.org/images/recipes/elasticsearch-s3-fluentd.png" height="150"/>
29
29
  Syslog Alerting | Fluentd can "grep" for events and send out alerts. | <img src="https://www.fluentd.org/images/syslog-fluentd-alert.png" height="100"/>
30
- Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/assets/img/datasources/asynchronous_logging.png" height="150"/>
30
+ Mobile/Web Application Logging | Fluentd can function as middleware to enable asynchronous, scalable logging for user action events. | <img src="https://www.fluentd.org/images/datasources/asynchronous_logging.png" height="150"/>
31
31
 
32
32
  ## Quick Start
33
33
 
@@ -52,10 +52,10 @@
52
52
  </match>
53
53
 
54
54
  # Composing two filters. For all events with the tag foo.bar,
55
- #
55
+ #
56
56
  # 1. The first filter filters out all events that has the field "hello"
57
57
  # 2. Then, for those events, we downcase the value of the "name" field.
58
- #
58
+ #
59
59
  # - {"name":"SADA", "hello":100} gets filtered out
60
60
  # - {"name":"SADA"} becomes {"name":"sada"}
61
61
  # - {"last_name":"FURUHASHI"} throws an error because it has no field called "name"
data/fluentd.gemspec CHANGED
@@ -23,7 +23,7 @@ Gem::Specification.new do |gem|
23
23
  gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
24
24
  gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
25
25
  gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
26
- gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.7.0"])
26
+ gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.8.0"])
27
27
  gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
28
28
  gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
29
29
  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
@@ -134,6 +134,14 @@ op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
134
134
  opts[:strict_config_value] = b
135
135
  }
136
136
 
137
+ op.on('--enable-input-metrics', "Enable input plugin metrics on fluentd", TrueClass) {|b|
138
+ opts[:enable_input_metrics] = b
139
+ }
140
+
141
+ op.on('--enable-size-metrics', "Enable plugin record size metrics on fluentd", TrueClass) {|b|
142
+ opts[:enable_size_metrics] = b
143
+ }
144
+
137
145
  op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
138
146
  if b
139
147
  opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max
@@ -27,7 +27,7 @@ class FluentPluginGenerator
27
27
  attr_reader :type, :name
28
28
  attr_reader :license_name
29
29
 
30
- SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter"]
30
+ SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter", "storage"]
31
31
 
32
32
  def initialize(argv = ARGV)
33
33
  @argv = argv
@@ -164,8 +164,13 @@ BANNER
164
164
  end
165
165
 
166
166
  def locked_gem_version(gem_name)
167
- d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
168
- d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
167
+ if File.exist?(lock_file_path)
168
+ d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
169
+ d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
170
+ else
171
+ # fallback even though Fluentd is installed without bundler
172
+ Gem::Specification.find_by_name(gem_name).version.version
173
+ end
169
174
  end
170
175
 
171
176
  def rake_version
@@ -177,8 +182,13 @@ BANNER
177
182
  end
178
183
 
179
184
  def bundler_version
180
- d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
181
- d.locked_gems.bundler_version.version
185
+ if File.exist?(lock_file_path)
186
+ d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
187
+ d.locked_gems.bundler_version.version
188
+ else
189
+ # fallback even though Fluentd is installed without bundler
190
+ Gem::Specification.find_by_name("bundler").version.version
191
+ end
182
192
  end
183
193
 
184
194
  def class_name
@@ -310,7 +310,7 @@ module Fluent
310
310
  # original implementation of v0.12 BufferedOutput
311
311
  def emit(tag, es, chain, key="")
312
312
  # this method will not be used except for the case that plugin calls super
313
- @emit_count += 1
313
+ @emit_count_metrics.inc
314
314
  data = format_stream(tag, es)
315
315
  if @buffer.emit(key, data, chain)
316
316
  submit_flush
@@ -337,14 +337,14 @@ module Fluent
337
337
  # because v0.12 BufferedOutput may overrides #format_stream, but original #handle_stream_simple method doesn't consider about it
338
338
  def handle_stream_simple(tag, es, enqueue: false)
339
339
  if @overrides_emit
340
- current_emit_count = @emit_count
340
+ current_emit_count = @emit_count_metrics.get
341
341
  size = es.size
342
342
  key = data = nil
343
343
  begin
344
344
  emit(tag, es, NULL_OUTPUT_CHAIN)
345
345
  key, data = self.last_emit_via_buffer
346
346
  ensure
347
- @emit_count = current_emit_count
347
+ @emit_count_metrics.set(current_emit_count)
348
348
  self.last_emit_via_buffer = nil
349
349
  end
350
350
  # on-the-fly key assignment can be done, and it's not configurable if Plugin#emit does it dynamically
@@ -352,7 +352,8 @@ module Fluent
352
352
  write_guard do
353
353
  @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
354
354
  end
355
- @counter_mutex.synchronize{ @emit_records += size }
355
+ @emit_records_metrics.add(es.size)
356
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
356
357
  return [meta]
357
358
  end
358
359
 
@@ -363,7 +364,8 @@ module Fluent
363
364
  write_guard do
364
365
  @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
365
366
  end
366
- @counter_mutex.synchronize{ @emit_records += size }
367
+ @emit_records_metrics.add(es.size)
368
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
367
369
  return [meta]
368
370
  end
369
371
 
@@ -373,7 +375,8 @@ module Fluent
373
375
  write_guard do
374
376
  @buffer.write({meta => data}, enqueue: enqueue)
375
377
  end
376
- @counter_mutex.synchronize{ @emit_records += size }
378
+ @emit_records_metrics.add(es.size)
379
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
377
380
  [meta]
378
381
  end
379
382
 
@@ -93,7 +93,7 @@ module Fluent
93
93
  basepath = '/'
94
94
  fname = path
95
95
  require 'open-uri'
96
- open(uri) {|f|
96
+ URI.open(uri) {|f|
97
97
  Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
98
98
  }
99
99
  end
@@ -20,6 +20,10 @@ require 'fluent/config/error'
20
20
 
21
21
  module Fluent
22
22
  module Config
23
+ def self.reformatted_value(type, val, opts = {}, name = nil)
24
+ REFORMAT_VALUE.call(type, val, opts, name)
25
+ end
26
+
23
27
  def self.size_value(str, opts = {}, name = nil)
24
28
  return nil if str.nil?
25
29
 
@@ -104,6 +108,16 @@ module Fluent
104
108
  Config.string_value(val, opts, name)
105
109
  }
106
110
 
111
+ def self.symbol_value(val, opts = {}, name = nil)
112
+ return nil if val.nil? || val.empty?
113
+
114
+ val.delete_prefix(":").to_sym
115
+ end
116
+
117
+ SYMBOL_TYPE = Proc.new { |val, opts = {}, name = nil|
118
+ Config.symbol_value(val, opts, name)
119
+ }
120
+
107
121
  def self.enum_value(val, opts = {}, name = nil)
108
122
  return nil if val.nil?
109
123
 
@@ -176,6 +190,7 @@ module Fluent
176
190
  when :bool then Config.bool_value(value, opts, name)
177
191
  when :time then Config.time_value(value, opts, name)
178
192
  when :regexp then Config.regexp_value(value, opts, name)
193
+ when :symbol then Config.symbol_value(value, opts, name)
179
194
  else
180
195
  raise "unknown type in REFORMAT: #{type}"
181
196
  end
@@ -37,6 +37,7 @@ module Fluent
37
37
  super(strscan, eval_context)
38
38
  @include_basepath = include_basepath
39
39
  @fname = fname
40
+ @logger = defined?($log) ? $log : nil
40
41
  end
41
42
 
42
43
  def parse!
@@ -99,7 +100,7 @@ module Fluent
99
100
 
100
101
  elsif root_element && skip(/(\@include|include)#{SPACING}/)
101
102
  if !prev_match.start_with?('@')
102
- $log.warn "'include' is deprecated. Use '@include' instead"
103
+ @logger.warn "'include' is deprecated. Use '@include' instead" if @logger
103
104
  end
104
105
  parse_include(attrs, elems)
105
106
 
@@ -123,7 +124,7 @@ module Fluent
123
124
  parse_error! "'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}"
124
125
  else
125
126
  # TODO: This is for backward compatibility. It will throw an error in the future.
126
- $log.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}"
127
+ @logger.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}" if @logger
127
128
  end
128
129
  end
129
130
 
@@ -171,7 +172,7 @@ module Fluent
171
172
  require 'open-uri'
172
173
  basepath = '/'
173
174
  fname = path
174
- data = open(uri) { |f| f.read }
175
+ data = URI.open(uri) { |f| f.read }
175
176
  data.force_encoding('UTF-8')
176
177
  ss = StringScanner.new(data)
177
178
  V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
data/lib/fluent/config.rb CHANGED
@@ -62,7 +62,7 @@ module Fluent
62
62
  Parser.parse(str, fname, basepath)
63
63
  when :ruby
64
64
  require 'fluent/config/dsl'
65
- $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file")
65
+ $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file") if $log
66
66
  Config::DSL::Parser.parse(str, File.join(basepath, fname))
67
67
  else
68
68
  raise "[BUG] unknown configuration parser specification:'#{parser}'"
data/lib/fluent/env.rb CHANGED
@@ -15,13 +15,14 @@
15
15
  #
16
16
 
17
17
  require 'serverengine/utils'
18
+ require 'fluent/oj_options'
18
19
 
19
20
  module Fluent
20
21
  DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
21
22
  DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
22
23
  DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
23
24
  DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
24
- DEFAULT_OJ_OPTIONS = {bigdecimal_load: :float, mode: :compat, use_to_json: true}
25
+ DEFAULT_OJ_OPTIONS = Fluent::OjOptions.load_env
25
26
  DEFAULT_DIR_PERMISSION = 0755
26
27
  DEFAULT_FILE_PERMISSION = 0644
27
28
 
@@ -47,6 +47,8 @@ module Fluent
47
47
  @match_cache = MatchCache.new
48
48
  @default_collector = default_collector
49
49
  @emit_error_handler = emit_error_handler
50
+ @metric_callbacks = {}
51
+ @caller_plugin_id = nil
50
52
  end
51
53
 
52
54
  attr_accessor :default_collector
@@ -83,6 +85,22 @@ module Fluent
83
85
  @match_rules << Rule.new(pattern, collector)
84
86
  end
85
87
 
88
+ def add_metric_callbacks(caller_plugin_id, callback)
89
+ @metric_callbacks[caller_plugin_id] = callback
90
+ end
91
+
92
+ def caller_plugin_id=(caller_plugin_id)
93
+ @caller_plugin_id = caller_plugin_id
94
+ end
95
+
96
+ def find_callback
97
+ if @caller_plugin_id
98
+ @metric_callbacks[@caller_plugin_id]
99
+ else
100
+ nil
101
+ end
102
+ end
103
+
86
104
  def emit(tag, time, record)
87
105
  unless record.nil?
88
106
  emit_stream(tag, OneEventStream.new(time, record))
@@ -95,6 +113,9 @@ module Fluent
95
113
 
96
114
  def emit_stream(tag, es)
97
115
  match(tag).emit_events(tag, es)
116
+ if callback = find_callback
117
+ callback.call(es)
118
+ end
98
119
  rescue => e
99
120
  @emit_error_handler.handle_emits_error(tag, es, e)
100
121
  end
@@ -175,7 +196,11 @@ module Fluent
175
196
  if optimizable?
176
197
  optimized_filter_stream(tag, es)
177
198
  else
178
- @filters.reduce(es) { |acc, filter| filter.filter_stream(tag, acc) }
199
+ @filters.reduce(es) { |acc, filter|
200
+ filtered_es = filter.filter_stream(tag, acc)
201
+ filter.measure_metrics(filtered_es)
202
+ filtered_es
203
+ }
179
204
  end
180
205
  end
181
206
 
@@ -193,6 +218,7 @@ module Fluent
193
218
  begin
194
219
  filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
195
220
  throw :break_loop unless filtered_record && filtered_time
221
+ filter.measure_metrics(OneEventStream.new(time, record))
196
222
  rescue => e
197
223
  filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
198
224
  end
@@ -200,6 +226,7 @@ module Fluent
200
226
  begin
201
227
  filtered_record = filter.filter(tag, filtered_time, filtered_record)
202
228
  throw :break_loop unless filtered_record
229
+ filter.measure_metrics(OneEventStream.new(time, record))
203
230
  rescue => e
204
231
  filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
205
232
  end