fluentd 1.19.1 → 1.19.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +60 -0
- data/Rakefile +7 -0
- data/lib/fluent/command/fluentd.rb +1 -0
- data/lib/fluent/config/v1_parser.rb +13 -5
- data/lib/fluent/config/yaml_parser/loader.rb +7 -2
- data/lib/fluent/config/yaml_parser.rb +2 -2
- data/lib/fluent/config.rb +5 -5
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/event.rb +2 -1
- data/lib/fluent/plugin/buf_file.rb +3 -3
- data/lib/fluent/plugin/buf_file_single.rb +2 -2
- data/lib/fluent/plugin/buf_memory.rb +1 -1
- data/lib/fluent/plugin/buffer/chunk.rb +2 -1
- data/lib/fluent/plugin/buffer/file_chunk.rb +2 -2
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +2 -2
- data/lib/fluent/plugin/buffer/memory_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +3 -0
- data/lib/fluent/plugin/compressable.rb +7 -62
- data/lib/fluent/plugin/extractor.rb +121 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_debug_agent.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +3 -1
- data/lib/fluent/plugin/in_http.rb +8 -4
- data/lib/fluent/plugin/in_monitor_agent.rb +19 -22
- data/lib/fluent/plugin/in_tail.rb +1 -1
- data/lib/fluent/plugin/out_file.rb +10 -0
- data/lib/fluent/plugin/out_forward/socket_cache.rb +45 -10
- data/lib/fluent/plugin/out_forward.rb +10 -0
- data/lib/fluent/plugin/out_http.rb +31 -1
- data/lib/fluent/plugin/output.rb +45 -1
- data/lib/fluent/plugin/parser_csv.rb +5 -0
- data/lib/fluent/plugin/storage_local.rb +2 -2
- data/lib/fluent/plugin_helper/http_server/app.rb +2 -0
- data/lib/fluent/supervisor.rb +30 -4
- data/lib/fluent/test/base.rb +5 -0
- data/lib/fluent/version.rb +1 -1
- metadata +34 -9
- data/.deepsource.toml +0 -13
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
# limitations under the License.
|
|
15
15
|
#
|
|
16
16
|
|
|
17
|
+
require 'fluent/plugin/extractor'
|
|
17
18
|
require 'fluent/plugin/input'
|
|
18
19
|
require 'fluent/plugin/parser'
|
|
19
20
|
require 'fluent/event'
|
|
@@ -64,6 +65,8 @@ module Fluent::Plugin
|
|
|
64
65
|
config_param :bind, :string, default: '0.0.0.0'
|
|
65
66
|
desc 'The size limit of the POSTed element. Default is 32MB.'
|
|
66
67
|
config_param :body_size_limit, :size, default: 32*1024*1024 # TODO default
|
|
68
|
+
desc 'The size limit of the decompressed element.'
|
|
69
|
+
config_param :decompression_size_limit, :size, default: 256*1024*1024 # TODO default
|
|
67
70
|
desc 'The timeout limit for keeping the connection alive.'
|
|
68
71
|
config_param :keepalive_timeout, :time, default: 10 # TODO default
|
|
69
72
|
config_param :backlog, :integer, default: nil
|
|
@@ -259,7 +262,7 @@ module Fluent::Plugin
|
|
|
259
262
|
|
|
260
263
|
def on_server_connect(conn)
|
|
261
264
|
handler = Handler.new(conn, @km, method(:on_request),
|
|
262
|
-
@body_size_limit, @format_name, log,
|
|
265
|
+
@body_size_limit, @decompression_size_limit, @format_name, log,
|
|
263
266
|
@cors_allow_origins, @cors_allow_credentials,
|
|
264
267
|
@add_query_params)
|
|
265
268
|
|
|
@@ -343,12 +346,13 @@ module Fluent::Plugin
|
|
|
343
346
|
class Handler
|
|
344
347
|
attr_reader :content_type
|
|
345
348
|
|
|
346
|
-
def initialize(io, km, callback, body_size_limit, format_name, log,
|
|
349
|
+
def initialize(io, km, callback, body_size_limit, decompression_size_limit, format_name, log,
|
|
347
350
|
cors_allow_origins, cors_allow_credentials, add_query_params)
|
|
348
351
|
@io = io
|
|
349
352
|
@km = km
|
|
350
353
|
@callback = callback
|
|
351
354
|
@body_size_limit = body_size_limit
|
|
355
|
+
@decompression_size_limit = decompression_size_limit
|
|
352
356
|
@next_close = false
|
|
353
357
|
@format_name = format_name
|
|
354
358
|
@log = log
|
|
@@ -518,9 +522,9 @@ module Fluent::Plugin
|
|
|
518
522
|
# For now, we only support 'gzip' and 'deflate'.
|
|
519
523
|
begin
|
|
520
524
|
if @content_encoding == 'gzip'.freeze
|
|
521
|
-
@body =
|
|
525
|
+
@body = Extractor.decompress_gzip(@body, limit: @decompression_size_limit)
|
|
522
526
|
elsif @content_encoding == 'deflate'.freeze
|
|
523
|
-
@body =
|
|
527
|
+
@body = Extractor.decompress_deflate(@body, limit: @decompression_size_limit)
|
|
524
528
|
end
|
|
525
529
|
rescue
|
|
526
530
|
@log.warn 'fails to decode payload', error: $!.to_s
|
|
@@ -37,9 +37,11 @@ module Fluent::Plugin
|
|
|
37
37
|
desc 'Determine the rate to emit internal metrics as events.'
|
|
38
38
|
config_param :emit_interval, :time, default: 60
|
|
39
39
|
desc 'Determine whether to include the config information.'
|
|
40
|
-
config_param :include_config, :bool, default:
|
|
40
|
+
config_param :include_config, :bool, default: false
|
|
41
41
|
desc 'Determine whether to include the retry information.'
|
|
42
|
-
config_param :include_retry, :bool, default:
|
|
42
|
+
config_param :include_retry, :bool, default: false
|
|
43
|
+
desc 'Determine whether to include the debug information.'
|
|
44
|
+
config_param :include_debug_info, :bool, default: false
|
|
43
45
|
|
|
44
46
|
class APIHandler
|
|
45
47
|
def initialize(agent)
|
|
@@ -151,28 +153,23 @@ module Fluent::Plugin
|
|
|
151
153
|
# parse ?=query string
|
|
152
154
|
qs.merge!(req.query || {})
|
|
153
155
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
opts[:pretty_json] = true
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
if ivars = qs['with_ivars'.freeze].first
|
|
163
|
-
opts[:ivars] = ivars.split(',')
|
|
164
|
-
end
|
|
156
|
+
opts = {
|
|
157
|
+
query: qs,
|
|
158
|
+
with_config: @agent.include_config,
|
|
159
|
+
with_retry: @agent.include_retry
|
|
160
|
+
}
|
|
165
161
|
|
|
166
|
-
if
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
162
|
+
if @agent.include_debug_info
|
|
163
|
+
# if ?debug=1 is set, set :with_debug_info for get_monitor_info
|
|
164
|
+
# and :pretty_json for render_json_error
|
|
165
|
+
if qs['debug'.freeze].first
|
|
166
|
+
opts[:with_debug_info] = true
|
|
167
|
+
opts[:pretty_json] = true
|
|
168
|
+
end
|
|
171
169
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
opts[:with_retry] = @agent.include_retry
|
|
170
|
+
if ivars = qs['with_ivars'.freeze].first
|
|
171
|
+
opts[:ivars] = ivars.split(',')
|
|
172
|
+
end
|
|
176
173
|
end
|
|
177
174
|
|
|
178
175
|
opts
|
|
@@ -59,6 +59,7 @@ module Fluent::Plugin
|
|
|
59
59
|
@shutdown_start_time = nil
|
|
60
60
|
@metrics = nil
|
|
61
61
|
@startup = true
|
|
62
|
+
@capability = Fluent::Capability.new(:current_process)
|
|
62
63
|
end
|
|
63
64
|
|
|
64
65
|
desc 'The paths to read. Multiple paths can be specified, separated by comma.'
|
|
@@ -195,7 +196,6 @@ module Fluent::Plugin
|
|
|
195
196
|
@dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
|
|
196
197
|
# parser is already created by parser helper
|
|
197
198
|
@parser = parser_create(usage: parser_config['usage'] || @parser_configs.first.usage)
|
|
198
|
-
@capability = Fluent::Capability.new(:current_process)
|
|
199
199
|
if @read_bytes_limit_per_second > 0
|
|
200
200
|
if !@enable_watch_timer
|
|
201
201
|
raise Fluent::ConfigError, "Need to enable watch timer when using log throttling feature"
|
|
@@ -117,7 +117,17 @@ module Fluent::Plugin
|
|
|
117
117
|
configured_time_slice_format = conf['time_slice_format']
|
|
118
118
|
|
|
119
119
|
if conf.elements(name: 'buffer').empty?
|
|
120
|
+
# no <buffer> section, default time chunk key and timekey (1d) will be used.
|
|
121
|
+
log.warn "default timekey interval (1d) will be used because of missing <buffer> section. To change the output frequency, please modify the timekey value"
|
|
122
|
+
|
|
120
123
|
conf.add_element('buffer', 'time')
|
|
124
|
+
else
|
|
125
|
+
unless conf.elements(name: 'buffer').first.has_key?('timekey')
|
|
126
|
+
if conf.elements(name: 'buffer').first.arg != "[]"
|
|
127
|
+
# with <buffer> section (except <buffer []>), and no timekey
|
|
128
|
+
log.warn "default timekey interval (1d) will be used. To change the output frequency, please modify the timekey value"
|
|
129
|
+
end
|
|
130
|
+
end
|
|
121
131
|
end
|
|
122
132
|
buffer_conf = conf.elements(name: 'buffer').first
|
|
123
133
|
# Fluent::PluginId#configure is not called yet, so we can't use #plugin_root_dir here.
|
|
@@ -31,20 +31,26 @@ module Fluent::Plugin
|
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
def checkout_or(key)
|
|
34
|
+
obsolete_sockets = []
|
|
35
|
+
|
|
34
36
|
@mutex.synchronize do
|
|
35
|
-
tsock = pick_socket(key)
|
|
37
|
+
tsock, obsolete_sockets = pick_socket(key)
|
|
36
38
|
|
|
37
39
|
if tsock
|
|
38
|
-
tsock.sock
|
|
40
|
+
return tsock.sock
|
|
39
41
|
else
|
|
40
42
|
sock = yield
|
|
41
43
|
new_tsock = TimedSocket.new(timeout, key, sock)
|
|
42
44
|
@log.debug("connect new socket #{new_tsock}")
|
|
43
45
|
|
|
44
46
|
@inflight_sockets[sock] = new_tsock
|
|
45
|
-
new_tsock.sock
|
|
47
|
+
return new_tsock.sock
|
|
46
48
|
end
|
|
47
49
|
end
|
|
50
|
+
ensure
|
|
51
|
+
obsolete_sockets.each do |sock|
|
|
52
|
+
sock.sock.close rescue nil
|
|
53
|
+
end
|
|
48
54
|
end
|
|
49
55
|
|
|
50
56
|
def checkin(sock)
|
|
@@ -117,17 +123,32 @@ module Fluent::Plugin
|
|
|
117
123
|
# this method is not thread safe
|
|
118
124
|
def pick_socket(key)
|
|
119
125
|
if @available_sockets[key].empty?
|
|
120
|
-
return nil
|
|
126
|
+
return nil, []
|
|
121
127
|
end
|
|
122
128
|
|
|
123
129
|
t = Time.now
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
+
selected = nil
|
|
131
|
+
remaining = []
|
|
132
|
+
obsolete_sockets = []
|
|
133
|
+
|
|
134
|
+
@available_sockets[key].each do |sock|
|
|
135
|
+
if expired_socket?(sock, time: t) || unavailable_socket?(sock.sock)
|
|
136
|
+
obsolete_sockets << sock
|
|
137
|
+
elsif selected.nil?
|
|
138
|
+
selected = sock
|
|
139
|
+
else
|
|
140
|
+
remaining << sock
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
@available_sockets[key] = remaining
|
|
145
|
+
|
|
146
|
+
if selected
|
|
147
|
+
@inflight_sockets[selected.sock] = selected
|
|
148
|
+
selected.timeout = timeout
|
|
130
149
|
end
|
|
150
|
+
|
|
151
|
+
[selected, obsolete_sockets]
|
|
131
152
|
end
|
|
132
153
|
|
|
133
154
|
def timeout
|
|
@@ -137,6 +158,20 @@ module Fluent::Plugin
|
|
|
137
158
|
def expired_socket?(sock, time: Time.now)
|
|
138
159
|
sock.timeout ? sock.timeout < time : false
|
|
139
160
|
end
|
|
161
|
+
|
|
162
|
+
def unavailable_socket?(sock)
|
|
163
|
+
return sock.closed? if sock.respond_to?(:closed?) && sock.closed?
|
|
164
|
+
|
|
165
|
+
io = if sock.respond_to?(:to_io)
|
|
166
|
+
sock.to_io
|
|
167
|
+
elsif sock.is_a?(IO)
|
|
168
|
+
sock
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
io ? !!IO.select([io], nil, nil, 0) : false
|
|
172
|
+
rescue IOError, SystemCallError
|
|
173
|
+
true
|
|
174
|
+
end
|
|
140
175
|
end
|
|
141
176
|
end
|
|
142
177
|
end
|
|
@@ -620,7 +620,17 @@ module Fluent::Plugin
|
|
|
620
620
|
end
|
|
621
621
|
|
|
622
622
|
def establish_connection(sock, ri)
|
|
623
|
+
start_time = Fluent::Clock.now
|
|
624
|
+
timeout = @sender.hard_timeout
|
|
625
|
+
|
|
623
626
|
while ri.state != :established
|
|
627
|
+
# Check for timeout to prevent infinite loop
|
|
628
|
+
if Fluent::Clock.now - start_time > timeout
|
|
629
|
+
@log.warn "handshake timeout after #{timeout}s", host: @host, port: @port
|
|
630
|
+
disable!
|
|
631
|
+
break
|
|
632
|
+
end
|
|
633
|
+
|
|
624
634
|
begin
|
|
625
635
|
# TODO: On Ruby 2.2 or earlier, read_nonblock doesn't work expectedly.
|
|
626
636
|
# We need rewrite around here using new socket/server plugin helper.
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
require 'net/http'
|
|
18
18
|
require 'uri'
|
|
19
19
|
require 'openssl'
|
|
20
|
+
require 'securerandom'
|
|
20
21
|
require 'fluent/tls'
|
|
21
22
|
require 'fluent/plugin/output'
|
|
22
23
|
require 'fluent/plugin_helper/socket'
|
|
@@ -58,6 +59,9 @@ module Fluent::Plugin
|
|
|
58
59
|
desc 'Compress HTTP request body'
|
|
59
60
|
config_param :compress, :enum, list: [:text, :gzip], default: :text
|
|
60
61
|
|
|
62
|
+
desc 'Allowed hosts list for dynamic endpoints'
|
|
63
|
+
config_param :allowed_hosts, :array, default: []
|
|
64
|
+
|
|
61
65
|
desc 'The connection open timeout in seconds'
|
|
62
66
|
config_param :open_timeout, :integer, default: nil
|
|
63
67
|
desc 'The read timeout in seconds'
|
|
@@ -106,6 +110,11 @@ module Fluent::Plugin
|
|
|
106
110
|
config_param :aws_role_arn, :string, default: nil
|
|
107
111
|
end
|
|
108
112
|
|
|
113
|
+
# To prevent URI::InvalidURIError, we replace Fluentd placeholders with a dummy string.
|
|
114
|
+
# We use the ".invalid" TLD (RFC 2606) to ensure it is RFC-compliant for URI parsing,
|
|
115
|
+
# while guaranteeing it will never conflict with a real-world hostname.
|
|
116
|
+
REPLACED_ENDPOINT_PLACEHOLDER = "#{SecureRandom.uuid}.invalid".freeze
|
|
117
|
+
|
|
109
118
|
def connection_cache_id_thread_key
|
|
110
119
|
"#{plugin_id}_connection_cache_id"
|
|
111
120
|
end
|
|
@@ -146,6 +155,15 @@ module Fluent::Plugin
|
|
|
146
155
|
@retryable_response_codes = [503]
|
|
147
156
|
end
|
|
148
157
|
|
|
158
|
+
begin
|
|
159
|
+
# Replace all Fluentd placeholder syntaxes (${...} or %{...})
|
|
160
|
+
endpoint = @endpoint.gsub(%r([$%]{[^}]+}), REPLACED_ENDPOINT_PLACEHOLDER)
|
|
161
|
+
# If @endpoint has placeholder as host name, then, @endpoint_host == REPLACED_ENDPOINT_PLACEHOLDER
|
|
162
|
+
@endpoint_host = URI.parse(endpoint).host
|
|
163
|
+
rescue URI::InvalidURIError => e
|
|
164
|
+
raise Fluent::ConfigError, "Invalid endpoint URI: #{@endpoint} (#{e.message})"
|
|
165
|
+
end
|
|
166
|
+
|
|
149
167
|
@http_opt = setup_http_option
|
|
150
168
|
@proxy_uri = URI.parse(@proxy) if @proxy
|
|
151
169
|
@formatter = formatter_create
|
|
@@ -278,7 +296,19 @@ module Fluent::Plugin
|
|
|
278
296
|
|
|
279
297
|
def parse_endpoint(chunk)
|
|
280
298
|
endpoint = extract_placeholders(@endpoint, chunk)
|
|
281
|
-
URI.parse(endpoint)
|
|
299
|
+
uri = URI.parse(endpoint)
|
|
300
|
+
|
|
301
|
+
if @endpoint_host != uri.host
|
|
302
|
+
if @allowed_hosts.empty?
|
|
303
|
+
raise Fluent::UnrecoverableError, "allowed_hosts is strictly required when using placeholders in the endpoint host"
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
unless @allowed_hosts.include?(uri.host)
|
|
307
|
+
raise Fluent::UnrecoverableError, "Not allowed host: #{uri.host}"
|
|
308
|
+
end
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
uri
|
|
282
312
|
end
|
|
283
313
|
|
|
284
314
|
def set_headers(req, uri, chunk)
|
data/lib/fluent/plugin/output.rb
CHANGED
|
@@ -44,6 +44,8 @@ module Fluent
|
|
|
44
44
|
CHUNK_KEY_PLACEHOLDER_PATTERN = /\$\{([-_.@$a-zA-Z0-9]+)\}/
|
|
45
45
|
CHUNK_TAG_PLACEHOLDER_PATTERN = /\$\{(tag(?:\[-?\d+\])?)\}/
|
|
46
46
|
CHUNK_ID_PLACEHOLDER_PATTERN = /\$\{chunk_id\}/
|
|
47
|
+
INVALID_PATH_COMPONENT_PATTERN = %r{\.\.[/\\]|^[/\\]}
|
|
48
|
+
PARENT_DIRECTORY_PATTERN = %r{\.\.[/\\]}
|
|
47
49
|
|
|
48
50
|
CHUNKING_FIELD_WARN_NUM = 4
|
|
49
51
|
|
|
@@ -372,6 +374,13 @@ module Fluent
|
|
|
372
374
|
buf_type = Plugin.lookup_type_from_class(@buffer.class)
|
|
373
375
|
log.warn "'flush_at_shutdown' is false, and buffer plugin '#{buf_type}' is not persistent buffer."
|
|
374
376
|
log.warn "your configuration will lose buffered data at shutdown. please confirm your configuration again."
|
|
377
|
+
else
|
|
378
|
+
if Fluent.windows? && @buffer.persistent?
|
|
379
|
+
service_timeout = read_wait_to_kill_service_timeout
|
|
380
|
+
if service_timeout && service_timeout <= 5000 # default value might varies on windows client/server
|
|
381
|
+
log.warn "your WaitToKillServiceTimeout=#{service_timeout} registry configuration seems too short. Recommend to extend the value of 'HKLM\\SYSTEM\\CurrentControlSet\\Control\\WaitToKillServiceTimeout' to prevent buffer corruption from a forced shutdown"
|
|
382
|
+
end
|
|
383
|
+
end
|
|
375
384
|
end
|
|
376
385
|
|
|
377
386
|
if (@flush_mode != :interval) && buffer_conf.has_key?('flush_interval')
|
|
@@ -418,6 +427,22 @@ module Fluent
|
|
|
418
427
|
self
|
|
419
428
|
end
|
|
420
429
|
|
|
430
|
+
def read_wait_to_kill_service_timeout
|
|
431
|
+
if Fluent.windows?
|
|
432
|
+
begin
|
|
433
|
+
require "win32/registry"
|
|
434
|
+
Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Control",
|
|
435
|
+
Win32::Registry::KEY_READ) do |reg|
|
|
436
|
+
reg["WaitToKillServiceTimeout"].to_i
|
|
437
|
+
end
|
|
438
|
+
rescue => e
|
|
439
|
+
log.warn "'flush_at_shutdown' is true, but can't check WaitToKillServiceTimeout registry configuration", error: e
|
|
440
|
+
end
|
|
441
|
+
else
|
|
442
|
+
nil # not supported
|
|
443
|
+
end
|
|
444
|
+
end
|
|
445
|
+
|
|
421
446
|
def keep_buffer_config_compat
|
|
422
447
|
# Need this to call `@buffer_config.disable_chunk_backup` just as before,
|
|
423
448
|
# since some plugins may use this option in this way.
|
|
@@ -802,9 +827,17 @@ module Fluent
|
|
|
802
827
|
# ${tag}, ${tag[0]}, ${tag[1]}, ... , ${tag[-2]}, ${tag[-1]}
|
|
803
828
|
if @chunk_key_tag
|
|
804
829
|
if str.include?('${tag}')
|
|
830
|
+
if metadata.tag.match?(INVALID_PATH_COMPONENT_PATTERN)
|
|
831
|
+
raise Fluent::UnrecoverableError, "Invalid path component detected in tag: #{metadata.tag}"
|
|
832
|
+
end
|
|
833
|
+
|
|
805
834
|
rvalue = rvalue.gsub('${tag}', metadata.tag)
|
|
806
835
|
end
|
|
807
836
|
if CHUNK_TAG_PLACEHOLDER_PATTERN.match?(str)
|
|
837
|
+
if metadata.tag.match?(INVALID_PATH_COMPONENT_PATTERN)
|
|
838
|
+
raise Fluent::UnrecoverableError, "Invalid path component detected in tag: #{metadata.tag}"
|
|
839
|
+
end
|
|
840
|
+
|
|
808
841
|
hash = {}
|
|
809
842
|
tag_parts = metadata.tag.split('.')
|
|
810
843
|
tag_parts.each_with_index do |part, i|
|
|
@@ -835,10 +868,21 @@ module Fluent
|
|
|
835
868
|
end
|
|
836
869
|
|
|
837
870
|
rvalue = rvalue.gsub(CHUNK_KEY_PLACEHOLDER_PATTERN) do |matched|
|
|
838
|
-
hash.fetch(matched) do
|
|
871
|
+
replace = hash.fetch(matched) do
|
|
839
872
|
log.warn "chunk key placeholder '#{matched[2..-2]}' not replaced. template:#{str}"
|
|
840
873
|
''
|
|
841
874
|
end
|
|
875
|
+
if replace.to_s.match?(INVALID_PATH_COMPONENT_PATTERN)
|
|
876
|
+
raise Fluent::UnrecoverableError, "Invalid path component detected in #{matched}: #{replace}"
|
|
877
|
+
end
|
|
878
|
+
|
|
879
|
+
replace
|
|
880
|
+
end
|
|
881
|
+
# Check if the number of parent directory components (../) has increased due to variable substitution
|
|
882
|
+
if rvalue.match?(PARENT_DIRECTORY_PATTERN)
|
|
883
|
+
if rvalue.scan(PARENT_DIRECTORY_PATTERN).size > str.scan(PARENT_DIRECTORY_PATTERN).size
|
|
884
|
+
raise Fluent::UnrecoverableError, "Invalid path component detected, replaced to: #{rvalue}"
|
|
885
|
+
end
|
|
842
886
|
end
|
|
843
887
|
end
|
|
844
888
|
|
|
@@ -47,6 +47,11 @@ module Fluent
|
|
|
47
47
|
|
|
48
48
|
def parse(text, &block)
|
|
49
49
|
values = CSV.parse_line(text, col_sep: @delimiter)
|
|
50
|
+
unless values
|
|
51
|
+
yield nil, nil
|
|
52
|
+
return
|
|
53
|
+
end
|
|
54
|
+
|
|
50
55
|
r = Hash[@keys.zip(values)]
|
|
51
56
|
time, record = convert_values(parse_time(r), r)
|
|
52
57
|
yield time, record
|
|
@@ -85,7 +85,7 @@ module Fluent
|
|
|
85
85
|
if File.exist?(@path)
|
|
86
86
|
raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
|
|
87
87
|
begin
|
|
88
|
-
data = File.open(@path, 'r:utf-8') { |io| io.read }
|
|
88
|
+
data = File.open(@path, 'r:utf-8:utf-8') { |io| io.read }
|
|
89
89
|
if data.empty?
|
|
90
90
|
log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
|
|
91
91
|
return
|
|
@@ -113,7 +113,7 @@ module Fluent
|
|
|
113
113
|
return if @on_memory
|
|
114
114
|
return unless File.exist?(@path)
|
|
115
115
|
begin
|
|
116
|
-
json_string = File.open(@path, 'r:utf-8'){ |io| io.read }
|
|
116
|
+
json_string = File.open(@path, 'r:utf-8:utf-8'){ |io| io.read }
|
|
117
117
|
json = JSON.parse(json_string)
|
|
118
118
|
unless json.is_a?(Hash)
|
|
119
119
|
log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
|
data/lib/fluent/supervisor.rb
CHANGED
|
@@ -18,6 +18,7 @@ require 'fileutils'
|
|
|
18
18
|
require 'open3'
|
|
19
19
|
require 'pathname'
|
|
20
20
|
require 'find'
|
|
21
|
+
require 'set'
|
|
21
22
|
|
|
22
23
|
require 'fluent/config'
|
|
23
24
|
require 'fluent/counter'
|
|
@@ -796,21 +797,37 @@ module Fluent
|
|
|
796
797
|
$log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
|
|
797
798
|
@inline_config = STDIN.read
|
|
798
799
|
end
|
|
800
|
+
parsed_files = Set.new
|
|
799
801
|
@conf = Fluent::Config.build(
|
|
800
802
|
config_path: @config_path,
|
|
801
803
|
encoding: @conf_encoding,
|
|
802
804
|
additional_config: @inline_config,
|
|
803
805
|
use_v1_config: @use_v1_config,
|
|
804
806
|
type: @config_file_type,
|
|
807
|
+
on_file_parsed: ->(path) { parsed_files << path },
|
|
805
808
|
)
|
|
806
809
|
@system_config = build_system_config(@conf)
|
|
807
810
|
|
|
808
811
|
$log.info :supervisor, 'parsing config file is succeeded', path: @config_path
|
|
809
812
|
|
|
810
|
-
build_additional_configurations do |additional_conf|
|
|
813
|
+
build_additional_configurations(parsed_files) do |additional_conf|
|
|
811
814
|
@conf += additional_conf
|
|
812
815
|
end
|
|
813
816
|
|
|
817
|
+
if Fluent.windows?
|
|
818
|
+
@conf.elements.each do |element|
|
|
819
|
+
next unless element.name == 'source'
|
|
820
|
+
if element['@type'] == 'tail' && element['pos_file']
|
|
821
|
+
$log.warn("Recommend adding #{File.dirname(element['pos_file'])} to the exclusion path of your antivirus software on Windows")
|
|
822
|
+
else
|
|
823
|
+
storage = element.elements.find { |v| v.name == 'storage' and v['@type'] == 'local' }
|
|
824
|
+
if storage
|
|
825
|
+
$log.warn("Recommend adding #{File.dirname(storage['path'])} to the exclusion path of your antivirus software on Windows")
|
|
826
|
+
end
|
|
827
|
+
end
|
|
828
|
+
end
|
|
829
|
+
end
|
|
830
|
+
|
|
814
831
|
@libs.each do |lib|
|
|
815
832
|
require lib
|
|
816
833
|
end
|
|
@@ -854,6 +871,7 @@ module Fluent
|
|
|
854
871
|
additional_config: @inline_config,
|
|
855
872
|
use_v1_config: @use_v1_config,
|
|
856
873
|
type: @config_file_type,
|
|
874
|
+
on_file_parsed: nil,
|
|
857
875
|
)
|
|
858
876
|
system_config = build_system_config(conf)
|
|
859
877
|
|
|
@@ -1088,15 +1106,17 @@ module Fluent
|
|
|
1088
1106
|
$log.debug('worker got SIGUSR2')
|
|
1089
1107
|
|
|
1090
1108
|
begin
|
|
1109
|
+
parsed_files = Set.new
|
|
1091
1110
|
conf = Fluent::Config.build(
|
|
1092
1111
|
config_path: @config_path,
|
|
1093
1112
|
encoding: @conf_encoding,
|
|
1094
1113
|
additional_config: @inline_config,
|
|
1095
1114
|
use_v1_config: @use_v1_config,
|
|
1096
1115
|
type: @config_file_type,
|
|
1116
|
+
on_file_parsed: ->(path) { parsed_files << path },
|
|
1097
1117
|
)
|
|
1098
1118
|
|
|
1099
|
-
build_additional_configurations do |additional_conf|
|
|
1119
|
+
build_additional_configurations(parsed_files) do |additional_conf|
|
|
1100
1120
|
conf += additional_conf
|
|
1101
1121
|
end
|
|
1102
1122
|
|
|
@@ -1206,7 +1226,7 @@ module Fluent
|
|
|
1206
1226
|
system_config
|
|
1207
1227
|
end
|
|
1208
1228
|
|
|
1209
|
-
def build_additional_configurations
|
|
1229
|
+
def build_additional_configurations(parsed_files)
|
|
1210
1230
|
if @system_config.config_include_dir&.empty?
|
|
1211
1231
|
$log.info :supervisor, 'configuration include directory is disabled'
|
|
1212
1232
|
return
|
|
@@ -1218,11 +1238,17 @@ module Fluent
|
|
|
1218
1238
|
next unless supported_suffixes.include?(File.extname(path))
|
|
1219
1239
|
# NOTE: both types of normal config (.conf) and YAML will be loaded.
|
|
1220
1240
|
# Thus, it does not care whether @config_path is .conf or .yml.
|
|
1241
|
+
if parsed_files.include?(path)
|
|
1242
|
+
$log.info :supervisor, 'skip auto loading, it was already loaded', path: path
|
|
1243
|
+
next
|
|
1244
|
+
end
|
|
1245
|
+
|
|
1221
1246
|
$log.info :supervisor, 'loading additional configuration file', path: path
|
|
1222
1247
|
yield Fluent::Config.build(config_path: path,
|
|
1223
1248
|
encoding: @conf_encoding,
|
|
1224
1249
|
use_v1_config: @use_v1_config,
|
|
1225
|
-
type: :guess
|
|
1250
|
+
type: :guess,
|
|
1251
|
+
on_file_parsed: nil)
|
|
1226
1252
|
end
|
|
1227
1253
|
rescue Errno::ENOENT
|
|
1228
1254
|
$log.info :supervisor, 'inaccessible include directory was specified', path: @system_config.config_include_dir
|
data/lib/fluent/test/base.rb
CHANGED
data/lib/fluent/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluentd
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.19.
|
|
4
|
+
version: 1.19.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sadayuki Furuhashi
|
|
8
|
+
autorequire:
|
|
8
9
|
bindir: bin
|
|
9
10
|
cert_chain: []
|
|
10
|
-
date:
|
|
11
|
+
date: 2026-06-25 00:00:00.000000000 Z
|
|
11
12
|
dependencies:
|
|
12
13
|
- !ruby/object:Gem::Dependency
|
|
13
14
|
name: bundler
|
|
@@ -220,9 +221,6 @@ dependencies:
|
|
|
220
221
|
- - "~>"
|
|
221
222
|
- !ruby/object:Gem::Version
|
|
222
223
|
version: '1.0'
|
|
223
|
-
- - "<"
|
|
224
|
-
- !ruby/object:Gem::Version
|
|
225
|
-
version: 1.1.0
|
|
226
224
|
type: :runtime
|
|
227
225
|
prerelease: false
|
|
228
226
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -230,9 +228,20 @@ dependencies:
|
|
|
230
228
|
- - "~>"
|
|
231
229
|
- !ruby/object:Gem::Version
|
|
232
230
|
version: '1.0'
|
|
233
|
-
|
|
231
|
+
- !ruby/object:Gem::Dependency
|
|
232
|
+
name: net-http
|
|
233
|
+
requirement: !ruby/object:Gem::Requirement
|
|
234
|
+
requirements:
|
|
235
|
+
- - "~>"
|
|
236
|
+
- !ruby/object:Gem::Version
|
|
237
|
+
version: '0.8'
|
|
238
|
+
type: :runtime
|
|
239
|
+
prerelease: false
|
|
240
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
241
|
+
requirements:
|
|
242
|
+
- - "~>"
|
|
234
243
|
- !ruby/object:Gem::Version
|
|
235
|
-
version:
|
|
244
|
+
version: '0.8'
|
|
236
245
|
- !ruby/object:Gem::Dependency
|
|
237
246
|
name: async-http
|
|
238
247
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -303,6 +312,20 @@ dependencies:
|
|
|
303
312
|
- - "~>"
|
|
304
313
|
- !ruby/object:Gem::Version
|
|
305
314
|
version: '1.6'
|
|
315
|
+
- !ruby/object:Gem::Dependency
|
|
316
|
+
name: ostruct
|
|
317
|
+
requirement: !ruby/object:Gem::Requirement
|
|
318
|
+
requirements:
|
|
319
|
+
- - "~>"
|
|
320
|
+
- !ruby/object:Gem::Version
|
|
321
|
+
version: '0.6'
|
|
322
|
+
type: :runtime
|
|
323
|
+
prerelease: false
|
|
324
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
325
|
+
requirements:
|
|
326
|
+
- - "~>"
|
|
327
|
+
- !ruby/object:Gem::Version
|
|
328
|
+
version: '0.6'
|
|
306
329
|
- !ruby/object:Gem::Dependency
|
|
307
330
|
name: rake
|
|
308
331
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -509,7 +532,6 @@ executables:
|
|
|
509
532
|
extensions: []
|
|
510
533
|
extra_rdoc_files: []
|
|
511
534
|
files:
|
|
512
|
-
- ".deepsource.toml"
|
|
513
535
|
- ".rubocop.yml"
|
|
514
536
|
- ADOPTERS.md
|
|
515
537
|
- AUTHORS
|
|
@@ -669,6 +691,7 @@ files:
|
|
|
669
691
|
- lib/fluent/plugin/buffer/memory_chunk.rb
|
|
670
692
|
- lib/fluent/plugin/compressable.rb
|
|
671
693
|
- lib/fluent/plugin/exec_util.rb
|
|
694
|
+
- lib/fluent/plugin/extractor.rb
|
|
672
695
|
- lib/fluent/plugin/file_util.rb
|
|
673
696
|
- lib/fluent/plugin/filter.rb
|
|
674
697
|
- lib/fluent/plugin/filter_grep.rb
|
|
@@ -847,6 +870,7 @@ metadata:
|
|
|
847
870
|
source_code_uri: https://github.com/fluent/fluentd
|
|
848
871
|
changelog_uri: https://github.com/fluent/fluentd/blob/master/CHANGELOG.md
|
|
849
872
|
bug_tracker_uri: https://github.com/fluent/fluentd/issues
|
|
873
|
+
post_install_message:
|
|
850
874
|
rdoc_options: []
|
|
851
875
|
require_paths:
|
|
852
876
|
- lib
|
|
@@ -861,7 +885,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
861
885
|
- !ruby/object:Gem::Version
|
|
862
886
|
version: '0'
|
|
863
887
|
requirements: []
|
|
864
|
-
rubygems_version: 3.
|
|
888
|
+
rubygems_version: 3.5.22
|
|
889
|
+
signing_key:
|
|
865
890
|
specification_version: 4
|
|
866
891
|
summary: Fluentd event collector
|
|
867
892
|
test_files: []
|