fluentd 1.13.3-x86-mingw32 → 1.14.0-x86-mingw32

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/windows-test.yaml +3 -3
  3. data/CHANGELOG.md +44 -0
  4. data/lib/fluent/command/fluentd.rb +8 -0
  5. data/lib/fluent/compat/output.rb +9 -6
  6. data/lib/fluent/event_router.rb +28 -1
  7. data/lib/fluent/plugin/bare_output.rb +49 -8
  8. data/lib/fluent/plugin/buffer.rb +84 -22
  9. data/lib/fluent/plugin/filter.rb +35 -1
  10. data/lib/fluent/plugin/in_http.rb +21 -2
  11. data/lib/fluent/plugin/in_monitor_agent.rb +4 -2
  12. data/lib/fluent/plugin/in_syslog.rb +13 -1
  13. data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
  14. data/lib/fluent/plugin/in_tail.rb +4 -1
  15. data/lib/fluent/plugin/input.rb +39 -1
  16. data/lib/fluent/plugin/metrics.rb +119 -0
  17. data/lib/fluent/plugin/metrics_local.rb +96 -0
  18. data/lib/fluent/plugin/multi_output.rb +43 -6
  19. data/lib/fluent/plugin/output.rb +74 -33
  20. data/lib/fluent/plugin.rb +10 -1
  21. data/lib/fluent/plugin_helper/event_emitter.rb +8 -1
  22. data/lib/fluent/plugin_helper/metrics.rb +129 -0
  23. data/lib/fluent/plugin_helper/server.rb +4 -2
  24. data/lib/fluent/plugin_helper.rb +1 -0
  25. data/lib/fluent/root_agent.rb +6 -0
  26. data/lib/fluent/supervisor.rb +2 -0
  27. data/lib/fluent/system_config.rb +9 -1
  28. data/lib/fluent/version.rb +1 -1
  29. data/test/config/test_system_config.rb +6 -0
  30. data/test/plugin/in_tail/test_position_file.rb +26 -4
  31. data/test/plugin/test_bare_output.rb +13 -0
  32. data/test/plugin/test_buffer.rb +8 -2
  33. data/test/plugin/test_filter.rb +11 -0
  34. data/test/plugin/test_in_http.rb +40 -0
  35. data/test/plugin/test_in_monitor_agent.rb +214 -8
  36. data/test/plugin/test_in_syslog.rb +35 -0
  37. data/test/plugin/test_in_tail.rb +9 -26
  38. data/test/plugin/test_input.rb +11 -0
  39. data/test/plugin/test_metrics.rb +294 -0
  40. data/test/plugin/test_metrics_local.rb +96 -0
  41. data/test/plugin/test_multi_output.rb +25 -1
  42. data/test/plugin/test_output.rb +16 -0
  43. data/test/plugin_helper/test_event_emitter.rb +29 -0
  44. data/test/plugin_helper/test_metrics.rb +137 -0
  45. data/test/test_plugin_classes.rb +102 -0
  46. data/test/test_root_agent.rb +30 -1
  47. metadata +11 -2
@@ -28,13 +28,47 @@ module Fluent
28
28
  include PluginLoggerMixin
29
29
  include PluginHelper::Mixin
30
30
 
31
- helpers_internal :event_emitter
31
+ helpers_internal :event_emitter, :metrics
32
32
 
33
33
  attr_reader :has_filter_with_time
34
34
 
35
35
  def initialize
36
36
  super
37
37
  @has_filter_with_time = has_filter_with_time?
38
+ @emit_records_metrics = nil
39
+ @emit_size_metrics = nil
40
+ @counter_mutex = Mutex.new
41
+ @enable_size_metrics = false
42
+ end
43
+
44
+ def emit_records
45
+ @emit_records_metrics.get
46
+ end
47
+
48
+ def emit_size
49
+ @emit_size_metrics.get
50
+ end
51
+
52
+ def configure(conf)
53
+ super
54
+
55
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "filter", name: "emit_records", help_text: "Number of count emit records")
56
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "filter", name: "emit_size", help_text: "Total size of emit events")
57
+ @enable_size_metrics = !!system_config.enable_size_metrics
58
+ end
59
+
60
+ def statistics
61
+ stats = {
62
+ 'emit_records' => @emit_records_metrics.get,
63
+ 'emit_size' => @emit_size_metrics.get,
64
+ }
65
+
66
+ { 'filter' => stats }
67
+ end
68
+
69
+ def measure_metrics(es)
70
+ @emit_records_metrics.add(es.size)
71
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
38
72
  end
39
73
 
40
74
  def filter(tag, time, record)
@@ -74,6 +74,8 @@ module Fluent::Plugin
74
74
  config_param :blocking_timeout, :time, default: 0.5
75
75
  desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'
76
76
  config_param :cors_allow_origins, :array, default: nil
77
+ desc 'Tells browsers whether to expose the response to frontend when the credentials mode is "include".'
78
+ config_param :cors_allow_credentials, :bool, default: false
77
79
  desc 'Respond with empty gif image of 1x1 pixel.'
78
80
  config_param :respond_with_empty_img, :bool, default: false
79
81
  desc 'Respond status code with 204.'
@@ -112,6 +114,12 @@ module Fluent::Plugin
112
114
 
113
115
  super
114
116
 
117
+ if @cors_allow_credentials
118
+ if @cors_allow_origins.nil? || @cors_allow_origins.include?('*')
119
+ raise Fluent::ConfigError, "Cannot enable cors_allow_credentials without specific origins"
120
+ end
121
+ end
122
+
115
123
  m = if @parser_configs.first['@type'] == 'in_http'
116
124
  @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
117
125
  @parser_msgpack.time_key = nil
@@ -279,7 +287,10 @@ module Fluent::Plugin
279
287
  private
280
288
 
281
289
  def on_server_connect(conn)
282
- handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins, @add_query_params)
290
+ handler = Handler.new(conn, @km, method(:on_request),
291
+ @body_size_limit, @format_name, log,
292
+ @cors_allow_origins, @cors_allow_credentials,
293
+ @add_query_params)
283
294
 
284
295
  conn.on(:data) do |data|
285
296
  handler.on_read(data)
@@ -356,7 +367,8 @@ module Fluent::Plugin
356
367
  class Handler
357
368
  attr_reader :content_type
358
369
 
359
- def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins, add_query_params)
370
+ def initialize(io, km, callback, body_size_limit, format_name, log,
371
+ cors_allow_origins, cors_allow_credentials, add_query_params)
360
372
  @io = io
361
373
  @km = km
362
374
  @callback = callback
@@ -365,6 +377,7 @@ module Fluent::Plugin
365
377
  @format_name = format_name
366
378
  @log = log
367
379
  @cors_allow_origins = cors_allow_origins
380
+ @cors_allow_credentials = cors_allow_credentials
368
381
  @idle = 0
369
382
  @add_query_params = add_query_params
370
383
  @km.add(self)
@@ -491,6 +504,9 @@ module Fluent::Plugin
491
504
  send_response_and_close(RES_200_STATUS, header, "")
492
505
  elsif include_cors_allow_origin
493
506
  header["Access-Control-Allow-Origin"] = @origin
507
+ if @cors_allow_credentials
508
+ header["Access-Control-Allow-Credentials"] = true
509
+ end
494
510
  send_response_and_close(RES_200_STATUS, header, "")
495
511
  else
496
512
  send_response_and_close(RES_403_STATUS, {}, "")
@@ -576,6 +592,9 @@ module Fluent::Plugin
576
592
  header['Access-Control-Allow-Origin'] = '*'
577
593
  elsif include_cors_allow_origin
578
594
  header['Access-Control-Allow-Origin'] = @origin
595
+ if @cors_allow_credentials
596
+ header["Access-Control-Allow-Credentials"] = true
597
+ end
579
598
  end
580
599
  end
581
600
 
@@ -238,7 +238,7 @@ module Fluent::Plugin
238
238
  'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },
239
239
  'buffer_timekeys' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.timekeys },
240
240
  'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size + @buffer.queue_size },
241
- 'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
241
+ 'retry_count' => ->(){ respond_to?(:num_errors) ? num_errors : nil },
242
242
  }
243
243
 
244
244
  def all_plugins
@@ -335,7 +335,9 @@ module Fluent::Plugin
335
335
  }
336
336
 
337
337
  if pe.respond_to?(:statistics)
338
- obj.merge!(pe.statistics['output'] || {})
338
+ obj.merge!(pe.statistics.dig('output') || {})
339
+ obj.merge!(pe.statistics.dig('filter') || {})
340
+ obj.merge!(pe.statistics.dig('input') || {})
339
341
  end
340
342
 
341
343
  obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)
@@ -89,6 +89,8 @@ module Fluent::Plugin
89
89
  config_param :source_hostname_key, :string, default: nil
90
90
  desc 'Try to resolve hostname from IP addresses or not.'
91
91
  config_param :resolve_hostname, :bool, default: nil
92
+ desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
93
+ config_param :send_keepalive_packet, :bool, default: false
92
94
  desc 'The field name of source address of sender.'
93
95
  config_param :source_address_key, :string, default: nil
94
96
  desc 'The field name of the severity.'
@@ -143,6 +145,11 @@ module Fluent::Plugin
143
145
  end
144
146
 
145
147
  @_event_loop_run_timeout = @blocking_timeout
148
+
149
+ protocol = @protocol_type || @transport_config.protocol
150
+ if @send_keepalive_packet && protocol == :udp
151
+ raise Fluent::ConfigError, "send_keepalive_packet is available for tcp/tls"
152
+ end
146
153
  end
147
154
 
148
155
  def multi_workers_ready?
@@ -173,7 +180,12 @@ module Fluent::Plugin
173
180
 
174
181
  delimiter = octet_count_frame ? " " : @delimiter
175
182
  delimiter_size = delimiter.size
176
- server_create_connection(tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
183
+ server_create_connection(
184
+ tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port,
185
+ bind: @bind,
186
+ resolve_name: @resolve_hostname,
187
+ send_keepalive_packet: @send_keepalive_packet
188
+ ) do |conn|
177
189
  conn.data do |data|
178
190
  buffer = conn.buffer
179
191
  buffer << data
@@ -158,7 +158,7 @@ module Fluent::Plugin
158
158
  end
159
159
  end
160
160
 
161
- entries = remove_deleted_files_entries(entries, existing_targets) if @follow_inodes
161
+ entries = remove_deleted_files_entries(entries, existing_targets)
162
162
  entries
163
163
  end
164
164
 
@@ -1063,7 +1063,10 @@ module Fluent::Plugin
1063
1063
  rescue RangeError
1064
1064
  io.close if io
1065
1065
  raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
1066
- rescue Errno::ENOENT, Errno::EACCES
1066
+ rescue Errno::EACCES => e
1067
+ @log.warn "#{e}"
1068
+ nil
1069
+ rescue Errno::ENOENT
1067
1070
  nil
1068
1071
  end
1069
1072
 
@@ -27,7 +27,45 @@ module Fluent
27
27
  include PluginLoggerMixin
28
28
  include PluginHelper::Mixin
29
29
 
30
- helpers_internal :event_emitter
30
+ helpers_internal :event_emitter, :metrics
31
+
32
+ def initialize
33
+ super
34
+ @emit_records_metrics = nil
35
+ @emit_size_metrics = nil
36
+ @counter_mutex = Mutex.new
37
+ @enable_size_metrics = false
38
+ end
39
+
40
+ def emit_records
41
+ @emit_records_metrics.get
42
+ end
43
+
44
+ def emit_size
45
+ @emit_size_metrics.get
46
+ end
47
+
48
+ def configure(conf)
49
+ super
50
+
51
+ @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_records", help_text: "Number of count emit records")
52
+ @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_size", help_text: "Total size of emit events")
53
+ @enable_size_metrics = !!system_config.enable_size_metrics
54
+ end
55
+
56
+ def statistics
57
+ stats = {
58
+ 'emit_records' => @emit_records_metrics.get,
59
+ 'emit_size' => @emit_size_metrics.get,
60
+ }
61
+
62
+ { 'input' => stats }
63
+ end
64
+
65
+ def metric_callback(es)
66
+ @emit_records_metrics.add(es.size)
67
+ @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
68
+ end
31
69
 
32
70
  def multi_workers_ready?
33
71
  false
@@ -0,0 +1,119 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'socket'
18
+
19
+ require 'fluent/plugin/base'
20
+
21
+ require 'fluent/log'
22
+ require 'fluent/unique_id'
23
+ require 'fluent/plugin_id'
24
+
25
+ module Fluent
26
+ module Plugin
27
+ class Metrics < Base
28
+ include PluginId
29
+ include PluginLoggerMixin
30
+ include UniqueId::Mixin
31
+
32
+ DEFAULT_TYPE = 'local'
33
+
34
+ configured_in :metrics
35
+
36
+ config_param :default_labels, :hash, default: {agent: "Fluentd", hostname: "#{Socket.gethostname}"}
37
+ config_param :labels, :hash, default: {}
38
+
39
+ attr_reader :use_gauge_metric
40
+ attr_reader :has_methods_for_gauge, :has_methods_for_counter
41
+
42
+ def initialize
43
+ super
44
+
45
+ @has_methods_for_counter = false
46
+ @has_methods_for_gauge = false
47
+ @use_gauge_metric = false
48
+ end
49
+
50
+ def configure(conf)
51
+ super
52
+
53
+ if use_gauge_metric
54
+ @has_methods_for_gauge = has_methods_for_gauge?
55
+ else
56
+ @has_methods_for_counter = has_methods_for_counter?
57
+ end
58
+ end
59
+
60
+ # Some metrics should be counted by gauge.
61
+ # ref: https://prometheus.io/docs/concepts/metric_types/#gauge
62
+ def use_gauge_metric=(use_gauge_metric=false)
63
+ @use_gauge_metric = use_gauge_metric
64
+ end
65
+
66
+ def create(namespace:, subsystem:,name:,help_text:,labels: {})
67
+ # This API is for cmetrics type.
68
+ end
69
+
70
+ def get
71
+ raise NotImplementedError, "Implement this method in child class"
72
+ end
73
+
74
+ def inc
75
+ raise NotImplementedError, "Implement this method in child class"
76
+ end
77
+
78
+ def dec
79
+ raise NotImplementedError, "Implement this method in child class"
80
+ end
81
+
82
+ def add(value)
83
+ raise NotImplementedError, "Implement this method in child class"
84
+ end
85
+
86
+ def sub(value)
87
+ raise NotImplementedError, "Implement this method in child class"
88
+ end
89
+
90
+ def set(value)
91
+ raise NotImplementedError, "Implement this method in child class"
92
+ end
93
+
94
+ private
95
+
96
+ def has_methods_for_counter?
97
+ implemented_methods = self.class.instance_methods(false)
98
+
99
+ if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
100
+ [:set].all?{|e| self.class.method_defined?(e)}
101
+ true
102
+ else
103
+ raise "BUG: metrics plugin on counter mode MUST implement `get`, `inc`, `add` methods. And aliased `set` methods should be aliased from another method"
104
+ end
105
+ end
106
+
107
+ def has_methods_for_gauge?
108
+ implemented_methods = self.class.instance_methods(false)
109
+
110
+ if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
111
+ [:set, :dec, :sub].all?{|e| self.class.method_defined?(e)}
112
+ true
113
+ else
114
+ raise "BUG: metrics plugin on gauge mode MUST implement `get`, `inc`, and `add` methods. And `dec`, `sub`, and `set` methods should be aliased from other methods"
115
+ end
116
+ end
117
+ end
118
+ end
119
+ end
@@ -0,0 +1,96 @@
1
+ #
2
+ # Fluentd
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ #
16
+
17
+ require 'fluent/plugin'
18
+ require 'fluent/plugin/metrics'
19
+
20
+ module Fluent
21
+ module Plugin
22
+ class LocalMetrics < Metrics
23
+ Fluent::Plugin.register_metrics('local', self)
24
+
25
+ def initialize
26
+ super
27
+ @store = 0
28
+ @monitor = Monitor.new
29
+ end
30
+
31
+ def configure(conf)
32
+ super
33
+
34
+ if use_gauge_metric
35
+ class << self
36
+ alias_method :dec, :dec_gauge
37
+ alias_method :set, :set_gauge
38
+ alias_method :sub, :sub_gauge
39
+ end
40
+ else
41
+ class << self
42
+ alias_method :set, :set_counter
43
+ end
44
+ end
45
+ end
46
+
47
+ def multi_workers_ready?
48
+ true
49
+ end
50
+
51
+ def get
52
+ @monitor.synchronize do
53
+ @store
54
+ end
55
+ end
56
+
57
+ def inc
58
+ @monitor.synchronize do
59
+ @store += 1
60
+ end
61
+ end
62
+
63
+ def dec_gauge
64
+ @monitor.synchronize do
65
+ @store -= 1
66
+ end
67
+ end
68
+
69
+ def add(value)
70
+ @monitor.synchronize do
71
+ @store += value
72
+ end
73
+ end
74
+
75
+ def sub_gauge(value)
76
+ @monitor.synchronize do
77
+ @store -= value
78
+ end
79
+ end
80
+
81
+ def set_counter(value)
82
+ return if @store > value
83
+
84
+ @monitor.synchronize do
85
+ @store = value
86
+ end
87
+ end
88
+
89
+ def set_gauge(value)
90
+ @monitor.synchronize do
91
+ @store = value
92
+ end
93
+ end
94
+ end
95
+ end
96
+ end