fluentd 1.19.1 → 1.19.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +60 -0
  3. data/Rakefile +7 -0
  4. data/lib/fluent/command/fluentd.rb +1 -0
  5. data/lib/fluent/config/v1_parser.rb +13 -5
  6. data/lib/fluent/config/yaml_parser/loader.rb +7 -2
  7. data/lib/fluent/config/yaml_parser.rb +2 -2
  8. data/lib/fluent/config.rb +5 -5
  9. data/lib/fluent/engine.rb +1 -1
  10. data/lib/fluent/event.rb +2 -1
  11. data/lib/fluent/plugin/buf_file.rb +3 -3
  12. data/lib/fluent/plugin/buf_file_single.rb +2 -2
  13. data/lib/fluent/plugin/buf_memory.rb +1 -1
  14. data/lib/fluent/plugin/buffer/chunk.rb +2 -1
  15. data/lib/fluent/plugin/buffer/file_chunk.rb +2 -2
  16. data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -2
  17. data/lib/fluent/plugin/buffer/memory_chunk.rb +1 -1
  18. data/lib/fluent/plugin/buffer.rb +3 -0
  19. data/lib/fluent/plugin/compressable.rb +7 -62
  20. data/lib/fluent/plugin/extractor.rb +121 -0
  21. data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
  22. data/lib/fluent/plugin/in_debug_agent.rb +1 -1
  23. data/lib/fluent/plugin/in_forward.rb +3 -1
  24. data/lib/fluent/plugin/in_http.rb +8 -4
  25. data/lib/fluent/plugin/in_monitor_agent.rb +19 -22
  26. data/lib/fluent/plugin/in_tail.rb +1 -1
  27. data/lib/fluent/plugin/out_file.rb +10 -0
  28. data/lib/fluent/plugin/out_forward/socket_cache.rb +45 -10
  29. data/lib/fluent/plugin/out_forward.rb +10 -0
  30. data/lib/fluent/plugin/out_http.rb +31 -1
  31. data/lib/fluent/plugin/output.rb +45 -1
  32. data/lib/fluent/plugin/parser_csv.rb +5 -0
  33. data/lib/fluent/plugin/storage_local.rb +2 -2
  34. data/lib/fluent/plugin_helper/http_server/app.rb +2 -0
  35. data/lib/fluent/supervisor.rb +30 -4
  36. data/lib/fluent/test/base.rb +5 -0
  37. data/lib/fluent/version.rb +1 -1
  38. metadata +34 -9
  39. data/.deepsource.toml +0 -13
@@ -14,6 +14,7 @@
14
14
  # limitations under the License.
15
15
  #
16
16
 
17
+ require 'fluent/plugin/extractor'
17
18
  require 'fluent/plugin/input'
18
19
  require 'fluent/plugin/parser'
19
20
  require 'fluent/event'
@@ -64,6 +65,8 @@ module Fluent::Plugin
64
65
  config_param :bind, :string, default: '0.0.0.0'
65
66
  desc 'The size limit of the POSTed element. Default is 32MB.'
66
67
  config_param :body_size_limit, :size, default: 32*1024*1024 # TODO default
68
+ desc 'The size limit of the decompressed element.'
69
+ config_param :decompression_size_limit, :size, default: 256*1024*1024 # TODO default
67
70
  desc 'The timeout limit for keeping the connection alive.'
68
71
  config_param :keepalive_timeout, :time, default: 10 # TODO default
69
72
  config_param :backlog, :integer, default: nil
@@ -259,7 +262,7 @@ module Fluent::Plugin
259
262
 
260
263
  def on_server_connect(conn)
261
264
  handler = Handler.new(conn, @km, method(:on_request),
262
- @body_size_limit, @format_name, log,
265
+ @body_size_limit, @decompression_size_limit, @format_name, log,
263
266
  @cors_allow_origins, @cors_allow_credentials,
264
267
  @add_query_params)
265
268
 
@@ -343,12 +346,13 @@ module Fluent::Plugin
343
346
  class Handler
344
347
  attr_reader :content_type
345
348
 
346
- def initialize(io, km, callback, body_size_limit, format_name, log,
349
+ def initialize(io, km, callback, body_size_limit, decompression_size_limit, format_name, log,
347
350
  cors_allow_origins, cors_allow_credentials, add_query_params)
348
351
  @io = io
349
352
  @km = km
350
353
  @callback = callback
351
354
  @body_size_limit = body_size_limit
355
+ @decompression_size_limit = decompression_size_limit
352
356
  @next_close = false
353
357
  @format_name = format_name
354
358
  @log = log
@@ -518,9 +522,9 @@ module Fluent::Plugin
518
522
  # For now, we only support 'gzip' and 'deflate'.
519
523
  begin
520
524
  if @content_encoding == 'gzip'.freeze
521
- @body = Zlib::GzipReader.new(StringIO.new(@body)).read
525
+ @body = Extractor.decompress_gzip(@body, limit: @decompression_size_limit)
522
526
  elsif @content_encoding == 'deflate'.freeze
523
- @body = Zlib::Inflate.inflate(@body)
527
+ @body = Extractor.decompress_deflate(@body, limit: @decompression_size_limit)
524
528
  end
525
529
  rescue
526
530
  @log.warn 'fails to decode payload', error: $!.to_s
@@ -37,9 +37,11 @@ module Fluent::Plugin
37
37
  desc 'Determine the rate to emit internal metrics as events.'
38
38
  config_param :emit_interval, :time, default: 60
39
39
  desc 'Determine whether to include the config information.'
40
- config_param :include_config, :bool, default: true
40
+ config_param :include_config, :bool, default: false
41
41
  desc 'Determine whether to include the retry information.'
42
- config_param :include_retry, :bool, default: true
42
+ config_param :include_retry, :bool, default: false
43
+ desc 'Determine whether to include the debug information.'
44
+ config_param :include_debug_info, :bool, default: false
43
45
 
44
46
  class APIHandler
45
47
  def initialize(agent)
@@ -151,28 +153,23 @@ module Fluent::Plugin
151
153
  # parse ?=query string
152
154
  qs.merge!(req.query || {})
153
155
 
154
- # if ?debug=1 is set, set :with_debug_info for get_monitor_info
155
- # and :pretty_json for render_json_error
156
- opts = { query: qs }
157
- if qs['debug'.freeze].first
158
- opts[:with_debug_info] = true
159
- opts[:pretty_json] = true
160
- end
161
-
162
- if ivars = qs['with_ivars'.freeze].first
163
- opts[:ivars] = ivars.split(',')
164
- end
156
+ opts = {
157
+ query: qs,
158
+ with_config: @agent.include_config,
159
+ with_retry: @agent.include_retry
160
+ }
165
161
 
166
- if with_config = qs['with_config'.freeze].first
167
- opts[:with_config] = Fluent::Config.bool_value(with_config)
168
- else
169
- opts[:with_config] = @agent.include_config
170
- end
162
+ if @agent.include_debug_info
163
+ # if ?debug=1 is set, set :with_debug_info for get_monitor_info
164
+ # and :pretty_json for render_json_error
165
+ if qs['debug'.freeze].first
166
+ opts[:with_debug_info] = true
167
+ opts[:pretty_json] = true
168
+ end
171
169
 
172
- if with_retry = qs['with_retry'.freeze].first
173
- opts[:with_retry] = Fluent::Config.bool_value(with_retry)
174
- else
175
- opts[:with_retry] = @agent.include_retry
170
+ if ivars = qs['with_ivars'.freeze].first
171
+ opts[:ivars] = ivars.split(',')
172
+ end
176
173
  end
177
174
 
178
175
  opts
@@ -59,6 +59,7 @@ module Fluent::Plugin
59
59
  @shutdown_start_time = nil
60
60
  @metrics = nil
61
61
  @startup = true
62
+ @capability = Fluent::Capability.new(:current_process)
62
63
  end
63
64
 
64
65
  desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -195,7 +196,6 @@ module Fluent::Plugin
195
196
  @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
196
197
  # parser is already created by parser helper
197
198
  @parser = parser_create(usage: parser_config['usage'] || @parser_configs.first.usage)
198
- @capability = Fluent::Capability.new(:current_process)
199
199
  if @read_bytes_limit_per_second > 0
200
200
  if !@enable_watch_timer
201
201
  raise Fluent::ConfigError, "Need to enable watch timer when using log throttling feature"
@@ -117,7 +117,17 @@ module Fluent::Plugin
117
117
  configured_time_slice_format = conf['time_slice_format']
118
118
 
119
119
  if conf.elements(name: 'buffer').empty?
120
+ # no <buffer> section, default time chunk key and timekey (1d) will be used.
121
+ log.warn "default timekey interval (1d) will be used because of missing <buffer> section. To change the output frequency, please modify the timekey value"
122
+
120
123
  conf.add_element('buffer', 'time')
124
+ else
125
+ unless conf.elements(name: 'buffer').first.has_key?('timekey')
126
+ if conf.elements(name: 'buffer').first.arg != "[]"
127
+ # with <buffer> section (except <buffer []>), and no timekey
128
+ log.warn "default timekey interval (1d) will be used. To change the output frequency, please modify the timekey value"
129
+ end
130
+ end
121
131
  end
122
132
  buffer_conf = conf.elements(name: 'buffer').first
123
133
  # Fluent::PluginId#configure is not called yet, so we can't use #plugin_root_dir here.
@@ -31,20 +31,26 @@ module Fluent::Plugin
31
31
  end
32
32
 
33
33
  def checkout_or(key)
34
+ obsolete_sockets = []
35
+
34
36
  @mutex.synchronize do
35
- tsock = pick_socket(key)
37
+ tsock, obsolete_sockets = pick_socket(key)
36
38
 
37
39
  if tsock
38
- tsock.sock
40
+ return tsock.sock
39
41
  else
40
42
  sock = yield
41
43
  new_tsock = TimedSocket.new(timeout, key, sock)
42
44
  @log.debug("connect new socket #{new_tsock}")
43
45
 
44
46
  @inflight_sockets[sock] = new_tsock
45
- new_tsock.sock
47
+ return new_tsock.sock
46
48
  end
47
49
  end
50
+ ensure
51
+ obsolete_sockets.each do |sock|
52
+ sock.sock.close rescue nil
53
+ end
48
54
  end
49
55
 
50
56
  def checkin(sock)
@@ -117,17 +123,32 @@ module Fluent::Plugin
117
123
  # this method is not thread safe
118
124
  def pick_socket(key)
119
125
  if @available_sockets[key].empty?
120
- return nil
126
+ return nil, []
121
127
  end
122
128
 
123
129
  t = Time.now
124
- if (s = @available_sockets[key].find { |sock| !expired_socket?(sock, time: t) })
125
- @inflight_sockets[s.sock] = @available_sockets[key].delete(s)
126
- s.timeout = timeout
127
- s
128
- else
129
- nil
130
+ selected = nil
131
+ remaining = []
132
+ obsolete_sockets = []
133
+
134
+ @available_sockets[key].each do |sock|
135
+ if expired_socket?(sock, time: t) || unavailable_socket?(sock.sock)
136
+ obsolete_sockets << sock
137
+ elsif selected.nil?
138
+ selected = sock
139
+ else
140
+ remaining << sock
141
+ end
142
+ end
143
+
144
+ @available_sockets[key] = remaining
145
+
146
+ if selected
147
+ @inflight_sockets[selected.sock] = selected
148
+ selected.timeout = timeout
130
149
  end
150
+
151
+ [selected, obsolete_sockets]
131
152
  end
132
153
 
133
154
  def timeout
@@ -137,6 +158,20 @@ module Fluent::Plugin
137
158
  def expired_socket?(sock, time: Time.now)
138
159
  sock.timeout ? sock.timeout < time : false
139
160
  end
161
+
162
+ def unavailable_socket?(sock)
163
+ return sock.closed? if sock.respond_to?(:closed?) && sock.closed?
164
+
165
+ io = if sock.respond_to?(:to_io)
166
+ sock.to_io
167
+ elsif sock.is_a?(IO)
168
+ sock
169
+ end
170
+
171
+ io ? !!IO.select([io], nil, nil, 0) : false
172
+ rescue IOError, SystemCallError
173
+ true
174
+ end
140
175
  end
141
176
  end
142
177
  end
@@ -620,7 +620,17 @@ module Fluent::Plugin
620
620
  end
621
621
 
622
622
  def establish_connection(sock, ri)
623
+ start_time = Fluent::Clock.now
624
+ timeout = @sender.hard_timeout
625
+
623
626
  while ri.state != :established
627
+ # Check for timeout to prevent infinite loop
628
+ if Fluent::Clock.now - start_time > timeout
629
+ @log.warn "handshake timeout after #{timeout}s", host: @host, port: @port
630
+ disable!
631
+ break
632
+ end
633
+
624
634
  begin
625
635
  # TODO: On Ruby 2.2 or earlier, read_nonblock doesn't work expectedly.
626
636
  # We need rewrite around here using new socket/server plugin helper.
@@ -17,6 +17,7 @@
17
17
  require 'net/http'
18
18
  require 'uri'
19
19
  require 'openssl'
20
+ require 'securerandom'
20
21
  require 'fluent/tls'
21
22
  require 'fluent/plugin/output'
22
23
  require 'fluent/plugin_helper/socket'
@@ -58,6 +59,9 @@ module Fluent::Plugin
58
59
  desc 'Compress HTTP request body'
59
60
  config_param :compress, :enum, list: [:text, :gzip], default: :text
60
61
 
62
+ desc 'Allowed hosts list for dynamic endpoints'
63
+ config_param :allowed_hosts, :array, default: []
64
+
61
65
  desc 'The connection open timeout in seconds'
62
66
  config_param :open_timeout, :integer, default: nil
63
67
  desc 'The read timeout in seconds'
@@ -106,6 +110,11 @@ module Fluent::Plugin
106
110
  config_param :aws_role_arn, :string, default: nil
107
111
  end
108
112
 
113
+ # To prevent URI::InvalidURIError, we replace Fluentd placeholders with a dummy string.
114
+ # We use the ".invalid" TLD (RFC 2606) to ensure it is RFC-compliant for URI parsing,
115
+ # while guaranteeing it will never conflict with a real-world hostname.
116
+ REPLACED_ENDPOINT_PLACEHOLDER = "#{SecureRandom.uuid}.invalid".freeze
117
+
109
118
  def connection_cache_id_thread_key
110
119
  "#{plugin_id}_connection_cache_id"
111
120
  end
@@ -146,6 +155,15 @@ module Fluent::Plugin
146
155
  @retryable_response_codes = [503]
147
156
  end
148
157
 
158
+ begin
159
+ # Replace all Fluentd placeholder syntaxes (${...} or %{...})
160
+ endpoint = @endpoint.gsub(%r([$%]{[^}]+}), REPLACED_ENDPOINT_PLACEHOLDER)
161
+ # If @endpoint has placeholder as host name, then, @endpoint_host == REPLACED_ENDPOINT_PLACEHOLDER
162
+ @endpoint_host = URI.parse(endpoint).host
163
+ rescue URI::InvalidURIError => e
164
+ raise Fluent::ConfigError, "Invalid endpoint URI: #{@endpoint} (#{e.message})"
165
+ end
166
+
149
167
  @http_opt = setup_http_option
150
168
  @proxy_uri = URI.parse(@proxy) if @proxy
151
169
  @formatter = formatter_create
@@ -278,7 +296,19 @@ module Fluent::Plugin
278
296
 
279
297
  def parse_endpoint(chunk)
280
298
  endpoint = extract_placeholders(@endpoint, chunk)
281
- URI.parse(endpoint)
299
+ uri = URI.parse(endpoint)
300
+
301
+ if @endpoint_host != uri.host
302
+ if @allowed_hosts.empty?
303
+ raise Fluent::UnrecoverableError, "allowed_hosts is strictly required when using placeholders in the endpoint host"
304
+ end
305
+
306
+ unless @allowed_hosts.include?(uri.host)
307
+ raise Fluent::UnrecoverableError, "Not allowed host: #{uri.host}"
308
+ end
309
+ end
310
+
311
+ uri
282
312
  end
283
313
 
284
314
  def set_headers(req, uri, chunk)
@@ -44,6 +44,8 @@ module Fluent
44
44
  CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
45
45
  CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[-?\d+\])?)\}/
46
46
  CHUNK_ID_PLACEHOLDER_PATTERN = /\$\{chunk_id\}/
47
+ INVALID_PATH_COMPONENT_PATTERN = %r{\.\.[/\\]|^[/\\]}
48
+ PARENT_DIRECTORY_PATTERN = %r{\.\.[/\\]}
47
49
 
48
50
  CHUNKING_FIELD_WARN_NUM = 4
49
51
 
@@ -372,6 +374,13 @@ module Fluent
372
374
  buf_type = Plugin.lookup_type_from_class(@buffer.class)
373
375
  log.warn "'flush_at_shutdown' is false, and buffer plugin '#{buf_type}' is not persistent buffer."
374
376
  log.warn "your configuration will lose buffered data at shutdown. please confirm your configuration again."
377
+ else
378
+ if Fluent.windows? && @buffer.persistent?
379
+ service_timeout = read_wait_to_kill_service_timeout
380
+ if service_timeout && service_timeout <= 5000 # default value might varies on windows client/server
381
+ log.warn "your WaitToKillServiceTimeout=#{service_timeout} registry configuration seems too short. Recommend to extend the value of 'HKLM\\SYSTEM\\CurrentControlSet\\Control\\WaitToKillServiceTimeout' to prevent buffer corruption from a forced shutdown"
382
+ end
383
+ end
375
384
  end
376
385
 
377
386
  if (@flush_mode != :interval) && buffer_conf.has_key?('flush_interval')
@@ -418,6 +427,22 @@ module Fluent
418
427
  self
419
428
  end
420
429
 
430
+ def read_wait_to_kill_service_timeout
431
+ if Fluent.windows?
432
+ begin
433
+ require "win32/registry"
434
+ Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Control",
435
+ Win32::Registry::KEY_READ) do |reg|
436
+ reg["WaitToKillServiceTimeout"].to_i
437
+ end
438
+ rescue => e
439
+ log.warn "'flush_at_shutdown' is true, but can't check WaitToKillServiceTimeout registry configuration", error: e
440
+ end
441
+ else
442
+ nil # not supported
443
+ end
444
+ end
445
+
421
446
  def keep_buffer_config_compat
422
447
  # Need this to call `@buffer_config.disable_chunk_backup` just as before,
423
448
  # since some plugins may use this option in this way.
@@ -802,9 +827,17 @@ module Fluent
802
827
  # ${tag}, ${tag[0]}, ${tag[1]}, ... , ${tag[-2]}, ${tag[-1]}
803
828
  if @chunk_key_tag
804
829
  if str.include?('${tag}')
830
+ if metadata.tag.match?(INVALID_PATH_COMPONENT_PATTERN)
831
+ raise Fluent::UnrecoverableError, "Invalid path component detected in tag: #{metadata.tag}"
832
+ end
833
+
805
834
  rvalue = rvalue.gsub('${tag}', metadata.tag)
806
835
  end
807
836
  if CHUNK_TAG_PLACEHOLDER_PATTERN.match?(str)
837
+ if metadata.tag.match?(INVALID_PATH_COMPONENT_PATTERN)
838
+ raise Fluent::UnrecoverableError, "Invalid path component detected in tag: #{metadata.tag}"
839
+ end
840
+
808
841
  hash = {}
809
842
  tag_parts = metadata.tag.split('.')
810
843
  tag_parts.each_with_index do |part, i|
@@ -835,10 +868,21 @@ module Fluent
835
868
  end
836
869
 
837
870
  rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN) do |matched|
838
- hash.fetch(matched) do
871
+ replace = hash.fetch(matched) do
839
872
  log.warn "chunk key placeholder '#{matched[2..-2]}' not replaced. template:#{str}"
840
873
  ''
841
874
  end
875
+ if replace.to_s.match?(INVALID_PATH_COMPONENT_PATTERN)
876
+ raise Fluent::UnrecoverableError, "Invalid path component detected in #{matched}: #{replace}"
877
+ end
878
+
879
+ replace
880
+ end
881
+ # Check if the number of parent directory components (../) has increased due to variable substitution
882
+ if rvalue.match?(PARENT_DIRECTORY_PATTERN)
883
+ if rvalue.scan(PARENT_DIRECTORY_PATTERN).size > str.scan(PARENT_DIRECTORY_PATTERN).size
884
+ raise Fluent::UnrecoverableError, "Invalid path component detected, replaced to: #{rvalue}"
885
+ end
842
886
  end
843
887
  end
844
888
 
@@ -47,6 +47,11 @@ module Fluent
47
47
 
48
48
  def parse(text, &block)
49
49
  values = CSV.parse_line(text, col_sep: @delimiter)
50
+ unless values
51
+ yield nil, nil
52
+ return
53
+ end
54
+
50
55
  r = Hash[@keys.zip(values)]
51
56
  time, record = convert_values(parse_time(r), r)
52
57
  yield time, record
@@ -85,7 +85,7 @@ module Fluent
85
85
  if File.exist?(@path)
86
86
  raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
87
87
  begin
88
- data = File.open(@path, 'r:utf-8') { |io| io.read }
88
+ data = File.open(@path, 'r:utf-8:utf-8') { |io| io.read }
89
89
  if data.empty?
90
90
  log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
91
91
  return
@@ -113,7 +113,7 @@ module Fluent
113
113
  return if @on_memory
114
114
  return unless File.exist?(@path)
115
115
  begin
116
- json_string = File.open(@path, 'r:utf-8'){ |io| io.read }
116
+ json_string = File.open(@path, 'r:utf-8:utf-8'){ |io| io.read }
117
117
  json = JSON.parse(json_string)
118
118
  unless json.is_a?(Hash)
119
119
  log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
@@ -71,6 +71,8 @@ module Fluent
71
71
  path
72
72
  end
73
73
  @router.route!(name, canonical_path, req)
74
+ ensure
75
+ request.body&.close
74
76
  end
75
77
  end
76
78
  end
@@ -18,6 +18,7 @@ require 'fileutils'
18
18
  require 'open3'
19
19
  require 'pathname'
20
20
  require 'find'
21
+ require 'set'
21
22
 
22
23
  require 'fluent/config'
23
24
  require 'fluent/counter'
@@ -796,21 +797,37 @@ module Fluent
796
797
  $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
797
798
  @inline_config = STDIN.read
798
799
  end
800
+ parsed_files = Set.new
799
801
  @conf = Fluent::Config.build(
800
802
  config_path: @config_path,
801
803
  encoding: @conf_encoding,
802
804
  additional_config: @inline_config,
803
805
  use_v1_config: @use_v1_config,
804
806
  type: @config_file_type,
807
+ on_file_parsed: ->(path) { parsed_files << path },
805
808
  )
806
809
  @system_config = build_system_config(@conf)
807
810
 
808
811
  $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
809
812
 
810
- build_additional_configurations do |additional_conf|
813
+ build_additional_configurations(parsed_files) do |additional_conf|
811
814
  @conf += additional_conf
812
815
  end
813
816
 
817
+ if Fluent.windows?
818
+ @conf.elements.each do |element|
819
+ next unless element.name == 'source'
820
+ if element['@type'] == 'tail' && element['pos_file']
821
+ $log.warn("Recommend adding #{File.dirname(element['pos_file'])} to the exclusion path of your antivirus software on Windows")
822
+ else
823
+ storage = element.elements.find { |v| v.name == 'storage' and v['@type'] == 'local' }
824
+ if storage
825
+ $log.warn("Recommend adding #{File.dirname(storage['path'])} to the exclusion path of your antivirus software on Windows")
826
+ end
827
+ end
828
+ end
829
+ end
830
+
814
831
  @libs.each do |lib|
815
832
  require lib
816
833
  end
@@ -854,6 +871,7 @@ module Fluent
854
871
  additional_config: @inline_config,
855
872
  use_v1_config: @use_v1_config,
856
873
  type: @config_file_type,
874
+ on_file_parsed: nil,
857
875
  )
858
876
  system_config = build_system_config(conf)
859
877
 
@@ -1088,15 +1106,17 @@ module Fluent
1088
1106
  $log.debug('worker got SIGUSR2')
1089
1107
 
1090
1108
  begin
1109
+ parsed_files = Set.new
1091
1110
  conf = Fluent::Config.build(
1092
1111
  config_path: @config_path,
1093
1112
  encoding: @conf_encoding,
1094
1113
  additional_config: @inline_config,
1095
1114
  use_v1_config: @use_v1_config,
1096
1115
  type: @config_file_type,
1116
+ on_file_parsed: ->(path) { parsed_files << path },
1097
1117
  )
1098
1118
 
1099
- build_additional_configurations do |additional_conf|
1119
+ build_additional_configurations(parsed_files) do |additional_conf|
1100
1120
  conf += additional_conf
1101
1121
  end
1102
1122
 
@@ -1206,7 +1226,7 @@ module Fluent
1206
1226
  system_config
1207
1227
  end
1208
1228
 
1209
- def build_additional_configurations
1229
+ def build_additional_configurations(parsed_files)
1210
1230
  if @system_config.config_include_dir&.empty?
1211
1231
  $log.info :supervisor, 'configuration include directory is disabled'
1212
1232
  return
@@ -1218,11 +1238,17 @@ module Fluent
1218
1238
  next unless supported_suffixes.include?(File.extname(path))
1219
1239
  # NOTE: both types of normal config (.conf) and YAML will be loaded.
1220
1240
  # Thus, it does not care whether @config_path is .conf or .yml.
1241
+ if parsed_files.include?(path)
1242
+ $log.info :supervisor, 'skip auto loading, it was already loaded', path: path
1243
+ next
1244
+ end
1245
+
1221
1246
  $log.info :supervisor, 'loading additional configuration file', path: path
1222
1247
  yield Fluent::Config.build(config_path: path,
1223
1248
  encoding: @conf_encoding,
1224
1249
  use_v1_config: @use_v1_config,
1225
- type: :guess)
1250
+ type: :guess,
1251
+ on_file_parsed: nil)
1226
1252
  end
1227
1253
  rescue Errno::ENOENT
1228
1254
  $log.info :supervisor, 'inaccessible include directory was specified', path: @system_config.config_include_dir
@@ -70,7 +70,12 @@ module Fluent
70
70
  num_waits.times { sleep 0.05 }
71
71
  return yield
72
72
  ensure
73
+ @instance.stop
74
+ @instance.before_shutdown
73
75
  @instance.shutdown
76
+ @instance.after_shutdown
77
+ @instance.close
78
+ @instance.terminate
74
79
  end
75
80
  end
76
81
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.19.1'
19
+ VERSION = '1.19.3'
20
20
 
21
21
  end
metadata CHANGED
@@ -1,13 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.19.1
4
+ version: 1.19.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
+ autorequire:
8
9
  bindir: bin
9
10
  cert_chain: []
10
- date: 1980-01-02 00:00:00.000000000 Z
11
+ date: 2026-06-25 00:00:00.000000000 Z
11
12
  dependencies:
12
13
  - !ruby/object:Gem::Dependency
13
14
  name: bundler
@@ -220,9 +221,6 @@ dependencies:
220
221
  - - "~>"
221
222
  - !ruby/object:Gem::Version
222
223
  version: '1.0'
223
- - - "<"
224
- - !ruby/object:Gem::Version
225
- version: 1.1.0
226
224
  type: :runtime
227
225
  prerelease: false
228
226
  version_requirements: !ruby/object:Gem::Requirement
@@ -230,9 +228,20 @@ dependencies:
230
228
  - - "~>"
231
229
  - !ruby/object:Gem::Version
232
230
  version: '1.0'
233
- - - "<"
231
+ - !ruby/object:Gem::Dependency
232
+ name: net-http
233
+ requirement: !ruby/object:Gem::Requirement
234
+ requirements:
235
+ - - "~>"
236
+ - !ruby/object:Gem::Version
237
+ version: '0.8'
238
+ type: :runtime
239
+ prerelease: false
240
+ version_requirements: !ruby/object:Gem::Requirement
241
+ requirements:
242
+ - - "~>"
234
243
  - !ruby/object:Gem::Version
235
- version: 1.1.0
244
+ version: '0.8'
236
245
  - !ruby/object:Gem::Dependency
237
246
  name: async-http
238
247
  requirement: !ruby/object:Gem::Requirement
@@ -303,6 +312,20 @@ dependencies:
303
312
  - - "~>"
304
313
  - !ruby/object:Gem::Version
305
314
  version: '1.6'
315
+ - !ruby/object:Gem::Dependency
316
+ name: ostruct
317
+ requirement: !ruby/object:Gem::Requirement
318
+ requirements:
319
+ - - "~>"
320
+ - !ruby/object:Gem::Version
321
+ version: '0.6'
322
+ type: :runtime
323
+ prerelease: false
324
+ version_requirements: !ruby/object:Gem::Requirement
325
+ requirements:
326
+ - - "~>"
327
+ - !ruby/object:Gem::Version
328
+ version: '0.6'
306
329
  - !ruby/object:Gem::Dependency
307
330
  name: rake
308
331
  requirement: !ruby/object:Gem::Requirement
@@ -509,7 +532,6 @@ executables:
509
532
  extensions: []
510
533
  extra_rdoc_files: []
511
534
  files:
512
- - ".deepsource.toml"
513
535
  - ".rubocop.yml"
514
536
  - ADOPTERS.md
515
537
  - AUTHORS
@@ -669,6 +691,7 @@ files:
669
691
  - lib/fluent/plugin/buffer/memory_chunk.rb
670
692
  - lib/fluent/plugin/compressable.rb
671
693
  - lib/fluent/plugin/exec_util.rb
694
+ - lib/fluent/plugin/extractor.rb
672
695
  - lib/fluent/plugin/file_util.rb
673
696
  - lib/fluent/plugin/filter.rb
674
697
  - lib/fluent/plugin/filter_grep.rb
@@ -847,6 +870,7 @@ metadata:
847
870
  source_code_uri: https://github.com/fluent/fluentd
848
871
  changelog_uri: https://github.com/fluent/fluentd/blob/master/CHANGELOG.md
849
872
  bug_tracker_uri: https://github.com/fluent/fluentd/issues
873
+ post_install_message:
850
874
  rdoc_options: []
851
875
  require_paths:
852
876
  - lib
@@ -861,7 +885,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
861
885
  - !ruby/object:Gem::Version
862
886
  version: '0'
863
887
  requirements: []
864
- rubygems_version: 3.6.8
888
+ rubygems_version: 3.5.22
889
+ signing_key:
865
890
  specification_version: 4
866
891
  summary: Fluentd event collector
867
892
  test_files: []