fluentd 1.13.3-x64-mingw32 → 1.14.3-x64-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.drone.yml +6 -6
  3. data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
  4. data/.github/workflows/windows-test.yaml +3 -3
  5. data/CHANGELOG.md +126 -0
  6. data/README.md +2 -0
  7. data/SECURITY.md +18 -0
  8. data/fluentd.gemspec +3 -3
  9. data/lib/fluent/command/cat.rb +13 -3
  10. data/lib/fluent/command/fluentd.rb +8 -0
  11. data/lib/fluent/compat/output.rb +9 -6
  12. data/lib/fluent/config/parser.rb +1 -1
  13. data/lib/fluent/config/v1_parser.rb +1 -1
  14. data/lib/fluent/event_router.rb +28 -1
  15. data/lib/fluent/plugin/bare_output.rb +49 -8
  16. data/lib/fluent/plugin/buf_file.rb +2 -2
  17. data/lib/fluent/plugin/buffer.rb +123 -27
  18. data/lib/fluent/plugin/filter.rb +35 -1
  19. data/lib/fluent/plugin/in_http.rb +21 -2
  20. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  21. data/lib/fluent/plugin/in_syslog.rb +13 -1
  22. data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
  23. data/lib/fluent/plugin/in_tail.rb +47 -8
  24. data/lib/fluent/plugin/input.rb +39 -1
  25. data/lib/fluent/plugin/metrics.rb +119 -0
  26. data/lib/fluent/plugin/metrics_local.rb +96 -0
  27. data/lib/fluent/plugin/multi_output.rb +43 -6
  28. data/lib/fluent/plugin/out_copy.rb +1 -1
  29. data/lib/fluent/plugin/out_forward.rb +15 -7
  30. data/lib/fluent/plugin/output.rb +80 -38
  31. data/lib/fluent/plugin/parser_apache2.rb +1 -1
  32. data/lib/fluent/plugin/storage_local.rb +3 -5
  33. data/lib/fluent/plugin.rb +10 -1
  34. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  35. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  36. data/lib/fluent/plugin_helper/server.rb +4 -2
  37. data/lib/fluent/plugin_helper.rb +1 -0
  38. data/lib/fluent/plugin_id.rb +2 -1
  39. data/lib/fluent/root_agent.rb +6 -0
  40. data/lib/fluent/supervisor.rb +4 -2
  41. data/lib/fluent/system_config.rb +9 -1
  42. data/lib/fluent/time.rb +21 -20
  43. data/lib/fluent/version.rb +1 -1
  44. data/test/command/test_cat.rb +31 -2
  45. data/test/config/test_system_config.rb +6 -0
  46. data/test/plugin/in_tail/test_io_handler.rb +12 -4
  47. data/test/plugin/in_tail/test_position_file.rb +26 -4
  48. data/test/plugin/test_bare_output.rb +13 -0
  49. data/test/plugin/test_buffer.rb +80 -3
  50. data/test/plugin/test_filter.rb +11 -0
  51. data/test/plugin/test_in_http.rb +40 -0
  52. data/test/plugin/test_in_monitor_agent.rb +214 -8
  53. data/test/plugin/test_in_syslog.rb +35 -0
  54. data/test/plugin/test_in_tail.rb +83 -35
  55. data/test/plugin/test_input.rb +11 -0
  56. data/test/plugin/test_metrics.rb +294 -0
  57. data/test/plugin/test_metrics_local.rb +96 -0
  58. data/test/plugin/test_multi_output.rb +25 -1
  59. data/test/plugin/test_out_exec_filter.rb +4 -0
  60. data/test/plugin/test_output.rb +16 -0
  61. data/test/plugin_helper/test_child_process.rb +9 -9
  62. data/test/plugin_helper/test_event_emitter.rb +29 -0
  63. data/test/plugin_helper/test_metrics.rb +137 -0
  64. data/test/test_plugin_classes.rb +102 -0
  65. data/test/test_root_agent.rb +30 -1
  66. data/test/test_time_parser.rb +22 -0
  67. metadata +18 -8
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 504fb7ee1a236fd83f1562020ce5cdea6207c9b3af0ec2d7bfa1ea17f0214c03
4
- data.tar.gz: 5868dca822982fe4501beba7c2637f647334a8ff187b097aca2805be1c6eb16d
3
+ metadata.gz: 394015dc219a38edeae834c955b2d6e9a5139920c49caa349765c4b0d376a12e
4
+ data.tar.gz: 5b4dbb5aae91e85043e295f037fac15f877a084ee5c6289d09a0e8b4b7cf0c9c
5
5
  SHA512:
6
- metadata.gz: 36e1633f9e1c12fdd1c228ae11c747d335efe0c2ad4d12ab666f8674964b5d0bc2a28b233bd71f1101d2ff45e970a9893b33209ef22ebb82baf36eb0afb37621
7
- data.tar.gz: 817231dbb280c9fd18fe09eb8fc2912d6de761be28cd5394b8b96b03435f5555903caea124d2d1edd98c0ba8a4c8460f85e9057127ef2fee1a8147d6e680b071
6
+ metadata.gz: 6241decd6c828af6e8bd6f1522517d54a00757b50bd7bb9746dbfe0bb766c22e94da31a714c1cee4376392f4c22a403da7073fed11c2610a2f0e2e455b632510
7
+ data.tar.gz: 902ef5dcd2289d72da71495877ec1b8337477f177b81b7c8002a2fa7beec11a2571957539044e16f2bc7c5a6a6be32dbd72e6867d2056a03ff41e216d5a2af22
data/.drone.yml CHANGED
@@ -1,13 +1,13 @@
1
1
  kind: pipeline
2
- name: fluentd-test-arm64-2-6-3
2
+ name: fluentd-test-arm64-3-0
3
3
 
4
4
  platform:
5
5
  os: linux
6
6
  arch: arm64
7
7
 
8
8
  steps:
9
- - name: fluentd-test-arm64-2-6-3
10
- image: arm64v8/ruby:2.6.3
9
+ - name: fluentd-test-arm64-3-0
10
+ image: arm64v8/ruby:3.0
11
11
  commands:
12
12
  - apt update
13
13
  - apt -y install libgmp3-dev
@@ -17,15 +17,15 @@ steps:
17
17
  - bundle exec rake test
18
18
  ---
19
19
  kind: pipeline
20
- name: fluentd-test-arm64-latest
20
+ name: fluentd-test-arm64-2-7
21
21
 
22
22
  platform:
23
23
  os: linux
24
24
  arch: arm64
25
25
 
26
26
  steps:
27
- - name: fluentd-test-arm64-latest
28
- image: arm64v8/ruby:latest
27
+ - name: fluentd-test-arm64-2-7
28
+ image: arm64v8/ruby:2.7
29
29
  commands:
30
30
  - apt update
31
31
  - apt -y install libgmp3-dev
@@ -50,6 +50,7 @@ body:
50
50
  label: Your Configuration
51
51
  description: |
52
52
  Write your configuration here. Minimum reproducible fluentd.conf is recommended.
53
+ render: apache
53
54
  validations:
54
55
  required: true
55
56
  - type: textarea
@@ -18,7 +18,7 @@ jobs:
18
18
  - windows-latest
19
19
  experimental: [false]
20
20
  include:
21
- - ruby-version: '3.0.1'
21
+ - ruby-version: '3.0.2'
22
22
  os: windows-latest
23
23
  experimental: true
24
24
  # On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
@@ -28,7 +28,7 @@ jobs:
28
28
  # * https://github.com/ruby/fiddle/issues/72
29
29
  # * https://bugs.ruby-lang.org/issues/17813
30
30
  # * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
31
- ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.1/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
31
+ ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.2/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.0.8/lib
32
32
 
33
33
  name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
34
34
  steps:
@@ -38,7 +38,7 @@ jobs:
38
38
  with:
39
39
  ruby-version: ${{ matrix.ruby-version }}
40
40
  - name: Add Fiddle 1.0.8
41
- if: ${{ matrix.ruby-version == '3.0.1' }}
41
+ if: ${{ matrix.ruby-version == '3.0.2' }}
42
42
  run: gem install fiddle --version 1.0.8
43
43
  - name: Install dependencies
44
44
  run: ridk exec bundle install
data/CHANGELOG.md CHANGED
@@ -1,3 +1,129 @@
1
+ # v1.14.3
2
+
3
+ ## Release v1.14.3 - 2021/11/26
4
+
5
+ ### Enhancement
6
+
7
+ * Changed to accept `http_parser.rb` 0.8.0.
8
+ `http_parser.rb` 0.8.0 is ready for Ractor.
9
+ https://github.com/fluent/fluentd/pull/3544
10
+
11
+ ### Bug fix
12
+
13
+ * in_tail: Fixed a bug that no new logs are read when
14
+ `enable_stat_watcher true` and `enable_watch_timer false` is set.
15
+ https://github.com/fluent/fluentd/pull/3541
16
+ * in_tail: Fixed a bug that the beginning and initial lines are lost
17
+ after startup when `read_from_head false` and path includes wildcard '*'.
18
+ https://github.com/fluent/fluentd/pull/3542
19
+ * Fixed a bug that processing messages were lost when
20
+ BufferChunkOverflowError was thrown even though only a specific
21
+ message size exceeds chunk_limit_size.
22
+ https://github.com/fluent/fluentd/pull/3553
23
+ https://github.com/fluent/fluentd/pull/3562
24
+
25
+ ### Misc
26
+
27
+ * Bump up required version of `win32-service` gem.
28
+ newer version is required to implement additional `fluent-ctl` commands.
29
+ https://github.com/fluent/fluentd/pull/3556
30
+
31
+ # v1.14.2
32
+
33
+ ## Release v1.14.2 - 2021/10/29
34
+
35
+ IMPORTANT: This release contain the fix for CVE-2021-41186 -
36
+ ReDoS vulnerability in `parser_apache2`.
37
+ This vulnerability is affected from Fluentd v0.14.14 to v1.14.1.
38
+ We recommend to upgrade Fluentd to v1.14.2 or use patched version of
39
+ `parser_apache2` plugin.
40
+
41
+ ### Enhancement
42
+
43
+ * fluent-cat: Add `--event-time` option to send specified event time for testing.
44
+
45
+ ### Bug fix
46
+
47
+ * Fixed to generate correct epoch timestamp even after switching Daylight Saving Time
48
+ https://github.com/fluent/fluentd/pull/3524
49
+ * Fixed ReDoS vulnerability in parser_apache2.
50
+ This vulnerability is caused by a certain pattern of a broken apache log.
51
+
52
+ # v1.14.1
53
+
54
+ ## Release v1.14.1 - 2021/09/29
55
+
56
+ ### Enhancement
57
+
58
+ * in_tail: Added file related metrics.
59
+ These metrics should be collected same as fluent-bit's in_tail.
60
+ https://github.com/fluent/fluentd/pull/3504
61
+ * out_forward: Changed to use metrics mechanism for node statistics
62
+ https://github.com/fluent/fluentd/pull/3506
63
+
64
+ ### Bug fix
65
+
66
+ * in_tail: Fixed a crash bug that it raise undefined method of eof? error.
67
+ This error may happen only when `read_bytes_limit_per_second` was specified.
68
+ https://github.com/fluent/fluentd/pull/3500
69
+ * out_forward: Fixed a bug that node statistics information is not included correctly.
70
+ https://github.com/fluent/fluentd/pull/3503
71
+ https://github.com/fluent/fluentd/pull/3507
72
+ * Fixed a error when using `@include` directive
73
+ It was occurred when http/https scheme URI is used in `@include` directive with Ruby 3.
74
+ https://github.com/fluent/fluentd/pull/3517
75
+ * out_copy: Fixed to suppress a wrong warning for `ignore_if_prev_success`
76
+ It didn't work even if a user set it.
77
+ https://github.com/fluent/fluentd/pull/3515
78
+ * Fixed not to output nanoseconds field of next retry time in warning log
79
+ Then, inappropriate labels in log are also fixed. (retry_time -> retry_times,
80
+ next_retry_seconds -> next_retry_time)
81
+ https://github.com/fluent/fluentd/pull/3518
82
+
83
+ # v1.14.0
84
+
85
+ ## Release v1.14.0 - 2021/08/30
86
+
87
+ ### Enhancement
88
+
89
+ * Added `enable_input_metrics`, `enable_size_metrics` system
90
+ configuration parameter
91
+ This feature might need to pay higher CPU cost, so input event metrics
92
+ features are disabled by default. These features are also enabled by
93
+ `--enable-input-metrics`,`--enable-size-metrics` command line
94
+ option.
95
+ https://github.com/fluent/fluentd/pull/3440
96
+ * Added reserved word `@ROOT` for getting root router.
97
+ This is incompatible change. Do not use `@ROOT` for label name.
98
+ https://github.com/fluent/fluentd/pull/3358
99
+ * in_syslog: Added `send_keepalive_packet` option
100
+ https://github.com/fluent/fluentd/pull/3474
101
+ * in_http: Added `cors_allow_credentials` option.
102
+ This option tells browsers whether to expose the response to
103
+ frontend when the credentials mode is "include".
104
+ https://github.com/fluent/fluentd/pull/3481
105
+ https://github.com/fluent/fluentd/pull/3491
106
+
107
+ ### Bug fix
108
+
109
+ * in_tail: Fixed a bug that deleted paths are not removed
110
+ from pos file by file compaction at start up
111
+ https://github.com/fluent/fluentd/pull/3467
112
+ * in_tail: Revived a warning message of retrying unaccessible file
113
+ https://github.com/fluent/fluentd/pull/3478
114
+ * TLSServer: Fixed a crash bug on logging peer host name errors
115
+ https://github.com/fluent/fluentd/pull/3483
116
+
117
+ ### Misc
118
+
119
+ * Added metrics plugin mechanism
120
+ The implementations is changed to use metrics plugin.
121
+ In the future, 3rd party plugin will be able to handle these metrics.
122
+ https://github.com/fluent/fluentd/pull/3471
123
+ https://github.com/fluent/fluentd/pull/3473
124
+ https://github.com/fluent/fluentd/pull/3479
125
+ https://github.com/fluent/fluentd/pull/3484
126
+
1
127
  # v1.13.3
2
128
 
3
129
  ## Release v1.13.3 - 2021/07/27
data/README.md CHANGED
@@ -88,6 +88,8 @@ You can run specified test via `TEST` environment variable:
88
88
 
89
89
  A third party security audit was performed by Cure53, you can see the full report [here](docs/SECURITY_AUDIT.pdf).
90
90
 
91
+ See [SECURITY](SECURITY.md) to contact us about vulnerability.
92
+
91
93
  ## Contributors:
92
94
 
93
95
  Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).
data/SECURITY.md ADDED
@@ -0,0 +1,18 @@
1
+ # Security Policy
2
+
3
+ ## Supported Versions
4
+
5
+ | Version | Supported |
6
+ | ------- | ------------------ |
7
+ | 1.14.x | :white_check_mark: |
8
+ | <= 1.13.x | :x: |
9
+
10
+ ## Reporting a Vulnerability
11
+
12
+ Please contact to current active maintainers. (in alphabetical order)
13
+
14
+ * ashie@clear-code.com
15
+ * fujimoto@clear-code.com
16
+ * hatake@calyptia.com
17
+ * hayashi@clear-code.com
18
+
data/fluentd.gemspec CHANGED
@@ -23,11 +23,11 @@ Gem::Specification.new do |gem|
23
23
  gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
24
24
  gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
25
25
  gem.add_runtime_dependency("serverengine", [">= 2.2.2", "< 3.0.0"])
26
- gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.8.0"])
26
+ gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
27
27
  gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
28
28
  gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
29
29
  gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
30
- gem.add_runtime_dependency("strptime", [">= 0.2.2", "< 1.0.0"])
30
+ gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])
31
31
  gem.add_runtime_dependency("webrick", [">= 1.4.2", "< 1.8.0"])
32
32
 
33
33
  # build gem for a certain platform. see also Rakefile
@@ -35,7 +35,7 @@ Gem::Specification.new do |gem|
35
35
  gem.platform = fake_platform unless fake_platform.empty?
36
36
  if /mswin|mingw/ =~ fake_platform || (/mswin|mingw/ =~ RUBY_PLATFORM && fake_platform.empty?)
37
37
  gem.add_runtime_dependency("win32-api", [">= 1.10", "< 2.0.0"])
38
- gem.add_runtime_dependency("win32-service", ["~> 2.2.0"])
38
+ gem.add_runtime_dependency("win32-service", ["~> 2.3.0"])
39
39
  gem.add_runtime_dependency("win32-ipc", ["~> 0.7.0"])
40
40
  gem.add_runtime_dependency("win32-event", ["~> 0.6.3"])
41
41
  gem.add_runtime_dependency("windows-pr", ["~> 1.2.6"])
@@ -35,6 +35,7 @@ format = 'json'
35
35
  message_key = 'message'
36
36
  time_as_integer = false
37
37
  retry_limit = 5
38
+ event_time = nil
38
39
 
39
40
  op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
40
41
  port = i
@@ -80,6 +81,10 @@ op.on('--retry-limit N', "Specify the number of retry limit (default: #{retry_li
80
81
  retry_limit = n
81
82
  }
82
83
 
84
+ op.on('--event-time TIME_STRING', "Specify the time expression string (default: nil)") {|v|
85
+ event_time = v
86
+ }
87
+
83
88
  singleton_class.module_eval do
84
89
  define_method(:usage) do |msg|
85
90
  puts op.to_s
@@ -134,7 +139,7 @@ class Writer
134
139
  end
135
140
  end
136
141
 
137
- def initialize(tag, connector, time_as_integer: false, retry_limit: 5)
142
+ def initialize(tag, connector, time_as_integer: false, retry_limit: 5, event_time: nil)
138
143
  @tag = tag
139
144
  @connector = connector
140
145
  @socket = false
@@ -148,6 +153,7 @@ class Writer
148
153
  @retry_wait = 1
149
154
  @retry_limit = retry_limit
150
155
  @time_as_integer = time_as_integer
156
+ @event_time = event_time
151
157
 
152
158
  super()
153
159
  end
@@ -166,7 +172,11 @@ class Writer
166
172
  end
167
173
  end
168
174
 
169
- time = Fluent::EventTime.now
175
+ time = if @event_time
176
+ Fluent::EventTime.parse(@event_time)
177
+ else
178
+ Fluent::EventTime.now
179
+ end
170
180
  time = time.to_i if @time_as_integer
171
181
  entry = if secondary_record?(record)
172
182
  # Even though secondary contains Fluent::EventTime in record,
@@ -309,7 +319,7 @@ else
309
319
  }
310
320
  end
311
321
 
312
- w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit)
322
+ w = Writer.new(tag, connector, time_as_integer: time_as_integer, retry_limit: retry_limit, event_time: event_time)
313
323
  w.start
314
324
 
315
325
  case format
@@ -134,6 +134,14 @@ op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
134
134
  opts[:strict_config_value] = b
135
135
  }
136
136
 
137
+ op.on('--enable-input-metrics', "Enable input plugin metrics on fluentd", TrueClass) {|b|
138
+ opts[:enable_input_metrics] = b
139
+ }
140
+
141
+ op.on('--enable-size-metrics', "Enable plugin record size metrics on fluentd", TrueClass) {|b|
142
+ opts[:enable_size_metrics] = b
143
+ }
144
+
137
145
  op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
138
146
  if b
139
147
  opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max
@@ -310,7 +310,7 @@ module Fluent
310
310
  # original implementation of v0.12 BufferedOutput
311
311
  def emit(tag, es, chain, key="")
312
312
  # this method will not be used except for the case that plugin calls super
313
- @emit_count += 1
313
+ @emit_count_metrics.inc
314
314
  data = format_stream(tag, es)
315
315
  if @buffer.emit(key, data, chain)
316
316
  submit_flush
@@ -337,14 +337,14 @@ module Fluent
337
337
  # because v0.12 BufferedOutput may overrides #format_stream, but original #handle_stream_simple method doesn't consider about it
338
338
  def handle_stream_simple(tag, es, enqueue: false)
339
339
  if @overrides_emit
340
- current_emit_count = @emit_count
340
+ current_emit_count = @emit_count_metrics.get
341
341
  size = es.size
342
342
  key = data = nil
343
343
  begin
344
344
  emit(tag, es, NULL_OUTPUT_CHAIN)
345
345
  key, data = self.last_emit_via_buffer
346
346
  ensure
347
- @emit_count = current_emit_count
347
+ @emit_count_metrics.set(current_emit_count)
348
348
  self.last_emit_via_buffer = nil
349
349
  end
350
350
  # on-the-fly key assignment can be done, and it's not configurable if Plugin#emit does it dynamically
@@ -352,7 +352,8 @@ module Fluent
352
352
  write_guard do
353
353
  @buffer.write({meta => data}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
354
354
  end
355
- @counter_mutex.synchronize{ @emit_records += size }
355
+ @emit_records_metrics.add(es.size)
356
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
356
357
  return [meta]
357
358
  end
358
359
 
@@ -363,7 +364,8 @@ module Fluent
363
364
  write_guard do
364
365
  @buffer.write({meta => bulk}, format: ->(_data){ _data }, size: ->(){ size }, enqueue: enqueue)
365
366
  end
366
- @counter_mutex.synchronize{ @emit_records += size }
367
+ @emit_records_metrics.add(es.size)
368
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
367
369
  return [meta]
368
370
  end
369
371
 
@@ -373,7 +375,8 @@ module Fluent
373
375
  write_guard do
374
376
  @buffer.write({meta => data}, enqueue: enqueue)
375
377
  end
376
- @counter_mutex.synchronize{ @emit_records += size }
378
+ @emit_records_metrics.add(es.size)
379
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
377
380
  [meta]
378
381
  end
379
382
 
@@ -93,7 +93,7 @@ module Fluent
93
93
  basepath = '/'
94
94
  fname = path
95
95
  require 'open-uri'
96
- open(uri) {|f|
96
+ URI.open(uri) {|f|
97
97
  Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
98
98
  }
99
99
  end
@@ -172,7 +172,7 @@ module Fluent
172
172
  require 'open-uri'
173
173
  basepath = '/'
174
174
  fname = path
175
- data = open(uri) { |f| f.read }
175
+ data = URI.open(uri) { |f| f.read }
176
176
  data.force_encoding('UTF-8')
177
177
  ss = StringScanner.new(data)
178
178
  V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
@@ -47,6 +47,8 @@ module Fluent
47
47
  @match_cache = MatchCache.new
48
48
  @default_collector = default_collector
49
49
  @emit_error_handler = emit_error_handler
50
+ @metric_callbacks = {}
51
+ @caller_plugin_id = nil
50
52
  end
51
53
 
52
54
  attr_accessor :default_collector
@@ -83,6 +85,22 @@ module Fluent
83
85
  @match_rules << Rule.new(pattern, collector)
84
86
  end
85
87
 
88
+ def add_metric_callbacks(caller_plugin_id, callback)
89
+ @metric_callbacks[caller_plugin_id] = callback
90
+ end
91
+
92
+ def caller_plugin_id=(caller_plugin_id)
93
+ @caller_plugin_id = caller_plugin_id
94
+ end
95
+
96
+ def find_callback
97
+ if @caller_plugin_id
98
+ @metric_callbacks[@caller_plugin_id]
99
+ else
100
+ nil
101
+ end
102
+ end
103
+
86
104
  def emit(tag, time, record)
87
105
  unless record.nil?
88
106
  emit_stream(tag, OneEventStream.new(time, record))
@@ -95,6 +113,9 @@ module Fluent
95
113
 
96
114
  def emit_stream(tag, es)
97
115
  match(tag).emit_events(tag, es)
116
+ if callback = find_callback
117
+ callback.call(es)
118
+ end
98
119
  rescue => e
99
120
  @emit_error_handler.handle_emits_error(tag, es, e)
100
121
  end
@@ -175,7 +196,11 @@ module Fluent
175
196
  if optimizable?
176
197
  optimized_filter_stream(tag, es)
177
198
  else
178
- @filters.reduce(es) { |acc, filter| filter.filter_stream(tag, acc) }
199
+ @filters.reduce(es) { |acc, filter|
200
+ filtered_es = filter.filter_stream(tag, acc)
201
+ filter.measure_metrics(filtered_es)
202
+ filtered_es
203
+ }
179
204
  end
180
205
  end
181
206
 
@@ -193,6 +218,7 @@ module Fluent
193
218
  begin
194
219
  filtered_time, filtered_record = filter.filter_with_time(tag, filtered_time, filtered_record)
195
220
  throw :break_loop unless filtered_record && filtered_time
221
+ filter.measure_metrics(OneEventStream.new(time, record))
196
222
  rescue => e
197
223
  filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
198
224
  end
@@ -200,6 +226,7 @@ module Fluent
200
226
  begin
201
227
  filtered_record = filter.filter(tag, filtered_time, filtered_record)
202
228
  throw :break_loop unless filtered_record
229
+ filter.measure_metrics(OneEventStream.new(time, record))
203
230
  rescue => e
204
231
  filter.router.emit_error_event(tag, filtered_time, filtered_record, e)
205
232
  end
@@ -23,37 +23,78 @@ require 'fluent/plugin_helper'
23
23
  module Fluent
24
24
  module Plugin
25
25
  class BareOutput < Base
26
+ include PluginHelper::Mixin # for metrics
27
+
26
28
  # DO NOT USE THIS plugin for normal output plugin. Use Output instead.
27
29
  # This output plugin base class is only for meta-output plugins
28
30
  # which cannot be implemented on MultiOutput.
29
31
  # E.g,: forest, config-expander
30
32
 
33
+ helpers_internal :metrics
34
+
31
35
  include PluginId
32
36
  include PluginLoggerMixin
33
37
  include PluginHelper::Mixin
34
38
 
35
- attr_reader :num_errors, :emit_count, :emit_records
36
-
37
39
  def process(tag, es)
38
40
  raise NotImplementedError, "BUG: output plugins MUST implement this method"
39
41
  end
40
42
 
43
+ def num_errors
44
+ @num_errors_metrics.get
45
+ end
46
+
47
+ def emit_count
48
+ @emit_count_metrics.get
49
+ end
50
+
51
+ def emit_size
52
+ @emit_size_metrics.get
53
+ end
54
+
55
+ def emit_records
56
+ @emit_records_metrics.get
57
+ end
58
+
41
59
  def initialize
42
60
  super
43
61
  @counter_mutex = Mutex.new
44
62
  # TODO: well organized counters
45
- @num_errors = 0
46
- @emit_count = 0
47
- @emit_records = 0
63
+ @num_errors_metrics = nil
64
+ @emit_count_metrics = nil
65
+ @emit_records_metrics = nil
66
+ @emit_size_metrics = nil
67
+ end
68
+
69
+ def configure(conf)
70
+ super
71
+
72
+ @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "num_errors", help_text: "Number of count num errors")
73
+ @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of count emits")
74
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_records", help_text: "Number of emit records")
75
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "bare_output", name: "emit_size", help_text: "Total size of emit events")
76
+ @enable_size_metrics = !!system_config.enable_size_metrics
77
+ end
78
+
79
+ def statistics
80
+ stats = {
81
+ 'num_errors' => @num_errors_metrics.get,
82
+ 'emit_records' => @emit_records_metrics.get,
83
+ 'emit_count' => @emit_count_metrics.get,
84
+ 'emit_size' => @emit_size_metrics.get,
85
+ }
86
+
87
+ { 'bare_output' => stats }
48
88
  end
49
89
 
50
90
  def emit_sync(tag, es)
51
- @counter_mutex.synchronize{ @emit_count += 1 }
91
+ @emit_count_metrics.inc
52
92
  begin
53
93
  process(tag, es)
54
- @counter_mutex.synchronize{ @emit_records += es.size }
94
+ @emit_records_metrics.add(es.size)
95
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
55
96
  rescue
56
- @counter_mutex.synchronize{ @num_errors += 1 }
97
+ @num_errors_metrics.inc
57
98
  raise
58
99
  end
59
100
  end
@@ -39,8 +39,8 @@ module Fluent
39
39
  config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
40
40
  config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
41
41
 
42
- config_param :file_permission, :string, default: nil # '0644'
43
- config_param :dir_permission, :string, default: nil # '0755'
42
+ config_param :file_permission, :string, default: nil # '0644' (Fluent::DEFAULT_FILE_PERMISSION)
43
+ config_param :dir_permission, :string, default: nil # '0755' (Fluent::DEFAULT_DIR_PERMISSION)
44
44
 
45
45
  def initialize
46
46
  super