fluentd 1.17.1-x64-mingw-ucrt → 1.19.0-x64-mingw-ucrt
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +116 -0
- data/CHANGELOG.md +293 -16
- data/MAINTAINERS.md +8 -2
- data/README.md +3 -7
- data/Rakefile +2 -0
- data/SECURITY.md +5 -3
- data/lib/fluent/command/cap_ctl.rb +2 -2
- data/lib/fluent/command/fluentd.rb +13 -3
- data/lib/fluent/compat/formatter.rb +6 -0
- data/lib/fluent/compat/socket_util.rb +2 -2
- data/lib/fluent/config/configure_proxy.rb +1 -1
- data/lib/fluent/config/element.rb +2 -2
- data/lib/fluent/config/literal_parser.rb +12 -5
- data/lib/fluent/config/parser.rb +15 -3
- data/lib/fluent/config/section.rb +2 -2
- data/lib/fluent/config/types.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +3 -3
- data/lib/fluent/counter/store.rb +1 -1
- data/lib/fluent/engine.rb +50 -34
- data/lib/fluent/env.rb +6 -2
- data/lib/fluent/event.rb +7 -6
- data/lib/fluent/event_router.rb +2 -2
- data/lib/fluent/log/console_adapter.rb +5 -7
- data/lib/fluent/log.rb +23 -0
- data/lib/fluent/plugin/bare_output.rb +0 -16
- data/lib/fluent/plugin/base.rb +2 -2
- data/lib/fluent/plugin/buf_file.rb +15 -1
- data/lib/fluent/plugin/buf_file_single.rb +15 -1
- data/lib/fluent/plugin/buffer/chunk.rb +74 -10
- data/lib/fluent/plugin/buffer/file_chunk.rb +9 -5
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +3 -3
- data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -2
- data/lib/fluent/plugin/buffer.rb +34 -6
- data/lib/fluent/plugin/compressable.rb +68 -22
- data/lib/fluent/plugin/filter.rb +0 -8
- data/lib/fluent/plugin/filter_parser.rb +27 -51
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/formatter_csv.rb +18 -4
- data/lib/fluent/plugin/formatter_json.rb +7 -4
- data/lib/fluent/plugin/formatter_out_file.rb +5 -2
- data/lib/fluent/plugin/in_forward.rb +9 -5
- data/lib/fluent/plugin/in_http.rb +14 -4
- data/lib/fluent/plugin/in_monitor_agent.rb +4 -8
- data/lib/fluent/plugin/in_syslog.rb +4 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
- data/lib/fluent/plugin/in_tail.rb +80 -57
- data/lib/fluent/plugin/in_tcp.rb +6 -2
- data/lib/fluent/plugin/in_udp.rb +11 -2
- data/lib/fluent/plugin/input.rb +4 -8
- data/lib/fluent/plugin/multi_output.rb +1 -17
- data/lib/fluent/plugin/out_buffer.rb +40 -0
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/out_file.rb +37 -30
- data/lib/fluent/plugin/out_forward/connection_manager.rb +2 -2
- data/lib/fluent/plugin/out_forward.rb +23 -13
- data/lib/fluent/plugin/out_http.rb +1 -1
- data/lib/fluent/plugin/out_secondary_file.rb +2 -2
- data/lib/fluent/plugin/out_stdout.rb +10 -3
- data/lib/fluent/plugin/out_stream.rb +3 -3
- data/lib/fluent/plugin/output.rb +26 -35
- data/lib/fluent/plugin/owned_by_mixin.rb +2 -2
- data/lib/fluent/plugin/parser.rb +3 -3
- data/lib/fluent/plugin/parser_json.rb +3 -3
- data/lib/fluent/plugin/sd_file.rb +2 -2
- data/lib/fluent/plugin/storage_local.rb +8 -4
- data/lib/fluent/plugin.rb +1 -1
- data/lib/fluent/plugin_helper/cert_option.rb +8 -0
- data/lib/fluent/plugin_helper/child_process.rb +2 -2
- data/lib/fluent/plugin_helper/event_emitter.rb +12 -0
- data/lib/fluent/plugin_helper/http_server/request.rb +13 -2
- data/lib/fluent/plugin_helper/http_server/server.rb +14 -8
- data/lib/fluent/plugin_helper/http_server.rb +1 -8
- data/lib/fluent/plugin_helper/metrics.rb +7 -0
- data/lib/fluent/plugin_helper/server.rb +13 -1
- data/lib/fluent/plugin_helper/service_discovery.rb +1 -1
- data/lib/fluent/plugin_helper/socket_option.rb +2 -2
- data/lib/fluent/plugin_helper/storage.rb +1 -1
- data/lib/fluent/plugin_id.rb +3 -3
- data/lib/fluent/root_agent.rb +117 -21
- data/lib/fluent/source_only_buffer_agent.rb +102 -0
- data/lib/fluent/static_config_analysis.rb +3 -2
- data/lib/fluent/supervisor.rb +258 -39
- data/lib/fluent/system_config.rb +27 -6
- data/lib/fluent/test/base.rb +1 -1
- data/lib/fluent/test/driver/base.rb +2 -2
- data/lib/fluent/test/filter_test.rb +2 -2
- data/lib/fluent/test/formatter_test.rb +1 -1
- data/lib/fluent/test/helpers.rb +4 -0
- data/lib/fluent/test/input_test.rb +2 -2
- data/lib/fluent/test/output_test.rb +4 -4
- data/lib/fluent/test/parser_test.rb +1 -1
- data/lib/fluent/tls.rb +24 -0
- data/lib/fluent/variable_store.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +38 -8
- metadata +99 -28
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +0 -92
- data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +0 -52
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +0 -58
data/lib/fluent/plugin/parser.rb
CHANGED
@@ -29,7 +29,7 @@ module Fluent
|
|
29
29
|
class Parser < Base
|
30
30
|
class TimeoutChecker
|
31
31
|
# This implementation now uses mutex because parser is typically used in input.
|
32
|
-
# If this has a performance issue under high
|
32
|
+
# If this has a performance issue under high concurrent, use concurrent-ruby's map instead.
|
33
33
|
def initialize(timeout)
|
34
34
|
@map = {}
|
35
35
|
@flag = ServerEngine::BlockingFlag.new
|
@@ -46,8 +46,8 @@ module Fluent
|
|
46
46
|
@map.keys.each { |th|
|
47
47
|
time = @map[th]
|
48
48
|
if now - time > @timeout
|
49
|
-
th.raise UncatchableError, "parsing timed out"
|
50
49
|
@map.delete(th)
|
50
|
+
th.raise UncatchableError, "parsing timed out"
|
51
51
|
end
|
52
52
|
}
|
53
53
|
}
|
@@ -220,7 +220,7 @@ module Fluent
|
|
220
220
|
end
|
221
221
|
|
222
222
|
def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_pattern)
|
223
|
-
null_empty_string && value.empty? || null_value_regexp && string_safe_encoding(value){|s| null_value_regexp.match(s) }
|
223
|
+
null_empty_string && value.empty? || null_value_regexp && string_safe_encoding(value){|s| null_value_regexp.match?(s) }
|
224
224
|
end
|
225
225
|
|
226
226
|
TRUTHY_VALUES = ['true', 'yes', '1']
|
@@ -52,9 +52,9 @@ module Fluent
|
|
52
52
|
when :oj
|
53
53
|
return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?
|
54
54
|
|
55
|
-
log&.info "Oj is not installed, and failing back to
|
56
|
-
configure_json_parser(:
|
57
|
-
when :json then [JSON.method(:
|
55
|
+
log&.info "Oj is not installed, and failing back to JSON for json parser"
|
56
|
+
configure_json_parser(:json)
|
57
|
+
when :json then [JSON.method(:parse), JSON::ParserError]
|
58
58
|
when :yajl then [Yajl.method(:load), Yajl::ParseError]
|
59
59
|
else
|
60
60
|
raise "BUG: unknown json parser specified: #{name}"
|
@@ -27,7 +27,7 @@ module Fluent
|
|
27
27
|
Plugin.register_sd('file', self)
|
28
28
|
|
29
29
|
DEFAULT_FILE_TYPE = :yaml
|
30
|
-
|
30
|
+
DEFAULT_WEIGHT = 60
|
31
31
|
DEFAULT_SD_FILE_PATH = ENV['DEFAULT_SD_FILE_PATH'] || '/etc/fluent/sd.yaml'
|
32
32
|
|
33
33
|
helpers :event_loop
|
@@ -126,7 +126,7 @@ module Fluent
|
|
126
126
|
s.fetch('host'),
|
127
127
|
s.fetch('port'),
|
128
128
|
s['name'],
|
129
|
-
s.fetch('weight',
|
129
|
+
s.fetch('weight', DEFAULT_WEIGHT),
|
130
130
|
s['standby'],
|
131
131
|
s['username'],
|
132
132
|
s['password'],
|
@@ -19,7 +19,7 @@ require 'fluent/plugin'
|
|
19
19
|
require 'fluent/plugin/storage'
|
20
20
|
|
21
21
|
require 'fileutils'
|
22
|
-
require '
|
22
|
+
require 'json'
|
23
23
|
|
24
24
|
module Fluent
|
25
25
|
module Plugin
|
@@ -90,7 +90,7 @@ module Fluent
|
|
90
90
|
log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
|
91
91
|
return
|
92
92
|
end
|
93
|
-
data =
|
93
|
+
data = JSON.parse(data)
|
94
94
|
raise Fluent::ConfigError, "Invalid contents (not object) in plugin storage file: '#{@path}'" unless data.is_a?(Hash)
|
95
95
|
rescue => e
|
96
96
|
log.error "failed to read data from plugin storage file", path: @path, error: e
|
@@ -114,7 +114,7 @@ module Fluent
|
|
114
114
|
return unless File.exist?(@path)
|
115
115
|
begin
|
116
116
|
json_string = File.open(@path, 'r:utf-8'){ |io| io.read }
|
117
|
-
json =
|
117
|
+
json = JSON.parse(json_string)
|
118
118
|
unless json.is_a?(Hash)
|
119
119
|
log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
|
120
120
|
log.debug "broken content", content: json_string
|
@@ -130,7 +130,11 @@ module Fluent
|
|
130
130
|
return if @on_memory
|
131
131
|
tmp_path = @path + '.tmp.' + Fluent::UniqueId.hex(Fluent::UniqueId.generate)
|
132
132
|
begin
|
133
|
-
|
133
|
+
if @pretty_print
|
134
|
+
json_string = JSON.pretty_generate(@store)
|
135
|
+
else
|
136
|
+
json_string = JSON.generate(@store)
|
137
|
+
end
|
134
138
|
File.open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
|
135
139
|
File.rename(tmp_path, @path)
|
136
140
|
rescue => e
|
data/lib/fluent/plugin.rb
CHANGED
@@ -175,7 +175,7 @@ module Fluent
|
|
175
175
|
else
|
176
176
|
raise Fluent::ConfigError, "#{kind} plugin '#{type}' is not a Class nor callable (without arguments)."
|
177
177
|
end
|
178
|
-
if parent && impl.respond_to?(
|
178
|
+
if parent && impl.respond_to?(:owner=)
|
179
179
|
impl.owner = parent
|
180
180
|
end
|
181
181
|
impl.extend FeatureAvailabilityChecker
|
@@ -33,6 +33,14 @@ module Fluent
|
|
33
33
|
|
34
34
|
if conf.client_cert_auth
|
35
35
|
ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
|
36
|
+
else
|
37
|
+
ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
38
|
+
end
|
39
|
+
|
40
|
+
if conf.ensure_fips
|
41
|
+
unless OpenSSL.fips_mode
|
42
|
+
raise Fluent::ConfigError, "Cannot enable FIPS compliant mode. OpenSSL FIPS configuration is disabled"
|
43
|
+
end
|
36
44
|
end
|
37
45
|
|
38
46
|
ctx.ca_file = conf.ca_path
|
@@ -143,7 +143,7 @@ module Fluent
|
|
143
143
|
@_child_process_mutex.synchronize{ @_child_process_processes.keys }.each do |pid|
|
144
144
|
process_info = @_child_process_processes[pid]
|
145
145
|
next if !process_info
|
146
|
-
process_info.writeio
|
146
|
+
process_info.writeio&.close rescue nil
|
147
147
|
end
|
148
148
|
|
149
149
|
super
|
@@ -183,7 +183,7 @@ module Fluent
|
|
183
183
|
|
184
184
|
living_process_exist = true
|
185
185
|
|
186
|
-
process_info.killed_at ||= Fluent::Clock.now # for
|
186
|
+
process_info.killed_at ||= Fluent::Clock.now # for irregular case (e.g., created after shutdown)
|
187
187
|
timeout_at = process_info.killed_at + @_child_process_kill_timeout
|
188
188
|
now = Fluent::Clock.now
|
189
189
|
next if now < timeout_at
|
@@ -26,6 +26,9 @@ module Fluent
|
|
26
26
|
|
27
27
|
def router
|
28
28
|
@_event_emitter_used_actually = true
|
29
|
+
|
30
|
+
return Engine.root_agent.source_only_router if @_event_emitter_force_source_only_router
|
31
|
+
|
29
32
|
if @_event_emitter_lazy_init
|
30
33
|
@router = @primary_instance.router
|
31
34
|
end
|
@@ -48,6 +51,14 @@ module Fluent
|
|
48
51
|
@_event_emitter_used_actually
|
49
52
|
end
|
50
53
|
|
54
|
+
def event_emitter_apply_source_only
|
55
|
+
@_event_emitter_force_source_only_router = true
|
56
|
+
end
|
57
|
+
|
58
|
+
def event_emitter_cancel_source_only
|
59
|
+
@_event_emitter_force_source_only_router = false
|
60
|
+
end
|
61
|
+
|
51
62
|
def event_emitter_router(label_name)
|
52
63
|
if label_name
|
53
64
|
if label_name == "@ROOT"
|
@@ -72,6 +83,7 @@ module Fluent
|
|
72
83
|
super
|
73
84
|
@_event_emitter_used_actually = false
|
74
85
|
@_event_emitter_lazy_init = false
|
86
|
+
@_event_emitter_force_source_only_router = false
|
75
87
|
@router = nil
|
76
88
|
end
|
77
89
|
|
@@ -14,6 +14,7 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
+
require 'uri'
|
17
18
|
require 'async/http/protocol'
|
18
19
|
require 'fluent/plugin_helper/http_server/methods'
|
19
20
|
|
@@ -29,12 +30,22 @@ module Fluent
|
|
29
30
|
@path, @query_string = path.split('?', 2)
|
30
31
|
end
|
31
32
|
|
33
|
+
def headers
|
34
|
+
@request.headers
|
35
|
+
end
|
36
|
+
|
32
37
|
def query
|
33
|
-
|
38
|
+
if @query_string
|
39
|
+
hash = Hash.new { |h, k| h[k] = [] }
|
40
|
+
# For compatibility with CGI.parse
|
41
|
+
URI.decode_www_form(@query_string).each_with_object(hash) do |(key, value), h|
|
42
|
+
h[key] << value
|
43
|
+
end
|
44
|
+
end
|
34
45
|
end
|
35
46
|
|
36
47
|
def body
|
37
|
-
@request.body
|
48
|
+
@request.body&.read
|
38
49
|
end
|
39
50
|
end
|
40
51
|
end
|
@@ -39,7 +39,8 @@ module Fluent
|
|
39
39
|
scheme = tls_context ? 'https' : 'http'
|
40
40
|
@uri = URI("#{scheme}://#{@addr}:#{@port}").to_s
|
41
41
|
@router = Router.new(default_app)
|
42
|
-
@
|
42
|
+
@server_task = nil
|
43
|
+
Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)
|
43
44
|
|
44
45
|
opts = if tls_context
|
45
46
|
{ ssl_context: tls_context }
|
@@ -54,25 +55,30 @@ module Fluent
|
|
54
55
|
end
|
55
56
|
|
56
57
|
def start(notify = nil)
|
58
|
+
Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)
|
57
59
|
@logger.debug("Start async HTTP server listening #{@uri}")
|
58
|
-
task = @reactor.run do
|
59
|
-
@server.run
|
60
60
|
|
61
|
+
Async do |task|
|
62
|
+
Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)
|
63
|
+
@server_task = task.async do
|
64
|
+
Console.logger = Fluent::Log::ConsoleAdapter.wrap(@logger)
|
65
|
+
@server.run
|
66
|
+
end
|
61
67
|
if notify
|
62
68
|
notify.push(:ready)
|
63
69
|
end
|
70
|
+
|
71
|
+
@server_task_queue = ::Thread::Queue.new
|
72
|
+
@server_task_queue.pop
|
73
|
+
@server_task&.stop
|
64
74
|
end
|
65
75
|
|
66
|
-
task.stop
|
67
76
|
@logger.debug('Finished HTTP server')
|
68
77
|
end
|
69
78
|
|
70
79
|
def stop
|
71
80
|
@logger.debug('closing HTTP server')
|
72
|
-
|
73
|
-
if @reactor
|
74
|
-
@reactor.stop
|
75
|
-
end
|
81
|
+
@server_task_queue.push(:stop)
|
76
82
|
end
|
77
83
|
|
78
84
|
HttpServer::Methods::ALL.map { |e| e.downcase.to_sym }.each do |name|
|
@@ -14,16 +14,9 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
-
begin
|
18
|
-
require 'async'
|
19
|
-
require 'fluent/plugin_helper/http_server/server'
|
20
|
-
rescue LoadError => _
|
21
|
-
require 'fluent/plugin_helper/http_server/compat/server'
|
22
|
-
Fluent::PluginHelper::HttpServer::Server = Fluent::PluginHelper::HttpServer::Compat::Server
|
23
|
-
end
|
24
|
-
|
25
17
|
require 'fluent/plugin_helper/thread'
|
26
18
|
require 'fluent/plugin_helper/server' # For Server::ServerTransportParams
|
19
|
+
require 'fluent/plugin_helper/http_server/server'
|
27
20
|
require 'fluent/plugin_helper/http_server/ssl_context_builder'
|
28
21
|
|
29
22
|
module Fluent
|
@@ -72,6 +72,13 @@ module Fluent
|
|
72
72
|
|
73
73
|
@_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics
|
74
74
|
|
75
|
+
# define the getter method for the calling instance.
|
76
|
+
singleton_class.module_eval do
|
77
|
+
unless method_defined?(name)
|
78
|
+
define_method(name) { metrics.get }
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
75
82
|
metrics
|
76
83
|
end
|
77
84
|
|
@@ -84,6 +84,8 @@ module Fluent
|
|
84
84
|
socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
|
85
85
|
end
|
86
86
|
|
87
|
+
socket_options[:receive_buffer_size] ||= @transport_config&.receive_buffer_size
|
88
|
+
|
87
89
|
socket_option_validate!(proto, **socket_options)
|
88
90
|
socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
|
89
91
|
|
@@ -136,6 +138,8 @@ module Fluent
|
|
136
138
|
socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
|
137
139
|
end
|
138
140
|
|
141
|
+
socket_options[:receive_buffer_size] ||= @transport_config&.receive_buffer_size
|
142
|
+
|
139
143
|
unless socket
|
140
144
|
socket_option_validate!(proto, **socket_options)
|
141
145
|
socket_option_setter = ->(sock){ socket_option_set(sock, **socket_options) }
|
@@ -247,6 +251,7 @@ module Fluent
|
|
247
251
|
:generate_cert_country, :generate_cert_state, :generate_cert_state,
|
248
252
|
:generate_cert_locality, :generate_cert_common_name,
|
249
253
|
:generate_cert_expiration, :generate_cert_digest,
|
254
|
+
:ensure_fips,
|
250
255
|
]
|
251
256
|
|
252
257
|
def server_create_transport_section_object(opts)
|
@@ -266,6 +271,9 @@ module Fluent
|
|
266
271
|
|
267
272
|
### Socket Params ###
|
268
273
|
|
274
|
+
desc "The max size of socket receive buffer. SO_RCVBUF"
|
275
|
+
config_param :receive_buffer_size, :size, default: nil
|
276
|
+
|
269
277
|
# SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src.
|
270
278
|
# Set positive value if needing to send FIN on closing on non-Windows.
|
271
279
|
# (On Windows, Fluentd can send FIN with zero `linger_timeout` since Fluentd doesn't set 0 to SO_LINGER on Windows.
|
@@ -287,6 +295,7 @@ module Fluent
|
|
287
295
|
config_param :max_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
|
288
296
|
config_param :ciphers, :string, default: Fluent::TLS::CIPHERS_DEFAULT
|
289
297
|
config_param :insecure, :bool, default: false
|
298
|
+
config_param :ensure_fips, :bool, default: false
|
290
299
|
|
291
300
|
# Cert signed by public CA
|
292
301
|
config_param :ca_path, :string, default: nil
|
@@ -347,7 +356,10 @@ module Fluent
|
|
347
356
|
end
|
348
357
|
|
349
358
|
def shutdown
|
350
|
-
@_server_connections
|
359
|
+
# When it invokes conn.cose, it reduces elements in @_server_connections by close_callback,
|
360
|
+
# and it reduces the number of loops. This prevents the connection closing.
|
361
|
+
# So, it requires invoking #dup to avoid the problem.
|
362
|
+
@_server_connections.dup.each do |conn|
|
351
363
|
conn.close rescue nil
|
352
364
|
end
|
353
365
|
|
@@ -91,7 +91,7 @@ module Fluent
|
|
91
91
|
end
|
92
92
|
|
93
93
|
# @param title [Symbol] the thread name. this value should be unique.
|
94
|
-
# @param configurations [Hash] hash which must has
|
94
|
+
# @param configurations [Hash] hash which must has discovery_service type and its configuration like `{ type: :static, conf: <Fluent::Config::Element> }`
|
95
95
|
# @param load_balancer [Object] object which has two methods #rebalance and #select_service
|
96
96
|
# @param custom_build_method [Proc]
|
97
97
|
def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)
|
@@ -54,8 +54,8 @@ module Fluent
|
|
54
54
|
if Fluent.windows?
|
55
55
|
# To prevent closing socket forcibly on Windows,
|
56
56
|
# this options shouldn't be set up when linger_timeout equals to 0 (including nil).
|
57
|
-
# This unintended behavior always
|
58
|
-
# This
|
57
|
+
# This unintended behavior always occurs on Windows when linger_timeout.to_i == 0.
|
58
|
+
# This unintended behavior causes "Errno::ECONNRESET: An existing connection was forcibly
|
59
59
|
# closed by the remote host." on Windows.
|
60
60
|
if linger_timeout.to_i > 0
|
61
61
|
if linger_timeout >= 2**16
|
data/lib/fluent/plugin_id.rb
CHANGED
@@ -30,7 +30,7 @@ module Fluent
|
|
30
30
|
end
|
31
31
|
|
32
32
|
def configure(conf)
|
33
|
-
@_plugin_id_variable_store = Fluent::VariableStore.fetch_or_build(:
|
33
|
+
@_plugin_id_variable_store = Fluent::VariableStore.fetch_or_build(:plugin_id, default_value: Set.new)
|
34
34
|
@id = conf['@id']
|
35
35
|
@_id_configured = !!@id # plugin id is explicitly configured by users (or not)
|
36
36
|
if @id
|
@@ -57,13 +57,13 @@ module Fluent
|
|
57
57
|
end
|
58
58
|
|
59
59
|
def plugin_id_configured?
|
60
|
-
if instance_variable_defined?(
|
60
|
+
if instance_variable_defined?(:@_id_configured)
|
61
61
|
@_id_configured
|
62
62
|
end
|
63
63
|
end
|
64
64
|
|
65
65
|
def plugin_id
|
66
|
-
if instance_variable_defined?(
|
66
|
+
if instance_variable_defined?(:@id)
|
67
67
|
@id || "object:#{object_id.to_s(16)}"
|
68
68
|
else
|
69
69
|
"object:#{object_id.to_s(16)}"
|
data/lib/fluent/root_agent.rb
CHANGED
@@ -22,6 +22,7 @@ require 'fluent/label'
|
|
22
22
|
require 'fluent/plugin'
|
23
23
|
require 'fluent/system_config'
|
24
24
|
require 'fluent/time'
|
25
|
+
require 'fluent/source_only_buffer_agent'
|
25
26
|
|
26
27
|
module Fluent
|
27
28
|
#
|
@@ -47,28 +48,62 @@ module Fluent
|
|
47
48
|
class RootAgent < Agent
|
48
49
|
ERROR_LABEL = "@ERROR".freeze # @ERROR is built-in error label
|
49
50
|
|
50
|
-
|
51
|
+
class SourceOnlyMode
|
52
|
+
DISABLED = 0
|
53
|
+
NORMAL = 1
|
54
|
+
ONLY_ZERO_DOWNTIME_RESTART_READY = 2
|
55
|
+
|
56
|
+
def initialize(with_source_only, start_in_parallel)
|
57
|
+
if start_in_parallel
|
58
|
+
@mode = ONLY_ZERO_DOWNTIME_RESTART_READY
|
59
|
+
elsif with_source_only
|
60
|
+
@mode = NORMAL
|
61
|
+
else
|
62
|
+
@mode = DISABLED
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def enabled?
|
67
|
+
@mode != DISABLED
|
68
|
+
end
|
69
|
+
|
70
|
+
def only_zero_downtime_restart_ready?
|
71
|
+
@mode == ONLY_ZERO_DOWNTIME_RESTART_READY
|
72
|
+
end
|
73
|
+
|
74
|
+
def disable!
|
75
|
+
@mode = DISABLED
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def initialize(log:, system_config: SystemConfig.new, start_in_parallel: false)
|
51
80
|
super(log: log)
|
52
81
|
|
53
82
|
@labels = {}
|
54
83
|
@inputs = []
|
55
84
|
@suppress_emit_error_log_interval = 0
|
56
85
|
@next_emit_error_log_time = nil
|
57
|
-
@without_source = false
|
58
|
-
@
|
86
|
+
@without_source = system_config.without_source || false
|
87
|
+
@source_only_mode = SourceOnlyMode.new(system_config.with_source_only, start_in_parallel)
|
88
|
+
@source_only_buffer_agent = nil
|
89
|
+
@enable_input_metrics = system_config.enable_input_metrics
|
59
90
|
|
60
91
|
suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
|
61
|
-
@without_source = system_config.without_source unless system_config.without_source.nil?
|
62
|
-
@enable_input_metrics = !!system_config.enable_input_metrics
|
63
92
|
end
|
64
93
|
|
65
94
|
attr_reader :inputs
|
66
95
|
attr_reader :labels
|
67
96
|
|
97
|
+
def source_only_router
|
98
|
+
raise "[BUG] 'RootAgent#source_only_router' should not be called when 'with_source_only' is false" unless @source_only_mode.enabled?
|
99
|
+
@source_only_buffer_agent.event_router
|
100
|
+
end
|
101
|
+
|
68
102
|
def configure(conf)
|
69
103
|
used_worker_ids = []
|
70
104
|
available_worker_ids = (0..Fluent::Engine.system_config.workers - 1).to_a
|
71
105
|
# initialize <worker> elements
|
106
|
+
supported_directives = ['source', 'match', 'filter', 'label']
|
72
107
|
conf.elements(name: 'worker').each do |e|
|
73
108
|
target_worker_id_str = e.arg
|
74
109
|
if target_worker_id_str.empty?
|
@@ -97,7 +132,7 @@ module Fluent
|
|
97
132
|
used_worker_ids << target_worker_id
|
98
133
|
|
99
134
|
e.elements.each do |elem|
|
100
|
-
unless
|
135
|
+
unless supported_directives.include?(elem.name)
|
101
136
|
raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
|
102
137
|
end
|
103
138
|
end
|
@@ -113,7 +148,7 @@ module Fluent
|
|
113
148
|
end
|
114
149
|
|
115
150
|
e.elements.each do |elem|
|
116
|
-
unless
|
151
|
+
unless supported_directives.include?(elem.name)
|
117
152
|
raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
|
118
153
|
end
|
119
154
|
elem.set_target_worker_id(target_worker_id)
|
@@ -148,6 +183,8 @@ module Fluent
|
|
148
183
|
|
149
184
|
super
|
150
185
|
|
186
|
+
setup_source_only_buffer_agent if @source_only_mode.enabled?
|
187
|
+
|
151
188
|
# initialize <source> elements
|
152
189
|
if @without_source
|
153
190
|
log.info :worker0, "'--without-source' is applied. Ignore <source> sections"
|
@@ -169,16 +206,36 @@ module Fluent
|
|
169
206
|
@error_collector = error_label.event_router
|
170
207
|
end
|
171
208
|
|
172
|
-
def
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
209
|
+
def setup_source_only_buffer_agent(flush: false)
|
210
|
+
@source_only_buffer_agent = SourceOnlyBufferAgent.new(log: log, system_config: Fluent::Engine.system_config)
|
211
|
+
@source_only_buffer_agent.configure(flush: flush)
|
212
|
+
end
|
213
|
+
|
214
|
+
def cleanup_source_only_buffer_agent
|
215
|
+
@source_only_buffer_agent&.cleanup
|
216
|
+
end
|
217
|
+
|
218
|
+
def lifecycle(desc: false, kind_callback: nil, kind_or_agent_list: nil)
|
219
|
+
only_zero_downtime_restart_ready = false
|
220
|
+
|
221
|
+
unless kind_or_agent_list
|
222
|
+
if @source_only_mode.enabled?
|
223
|
+
kind_or_agent_list = [:input, @source_only_buffer_agent]
|
224
|
+
only_zero_downtime_restart_ready = @source_only_mode.only_zero_downtime_restart_ready?
|
225
|
+
elsif @source_only_buffer_agent
|
226
|
+
# source_only_buffer_agent can re-reroute events, so the priority is equal to output_with_router.
|
227
|
+
kind_or_agent_list = [:input, :output_with_router, @source_only_buffer_agent, @labels.values, :filter, :output].flatten
|
228
|
+
else
|
229
|
+
kind_or_agent_list = [:input, :output_with_router, @labels.values, :filter, :output].flatten
|
230
|
+
end
|
231
|
+
|
232
|
+
kind_or_agent_list.reverse! if desc
|
233
|
+
end
|
234
|
+
|
235
|
+
kind_or_agent_list.each do |kind|
|
179
236
|
if kind.respond_to?(:lifecycle)
|
180
|
-
|
181
|
-
|
237
|
+
agent = kind
|
238
|
+
agent.lifecycle(desc: desc) do |plugin, display_kind|
|
182
239
|
yield plugin, display_kind
|
183
240
|
end
|
184
241
|
else
|
@@ -189,6 +246,9 @@ module Fluent
|
|
189
246
|
end
|
190
247
|
display_kind = (kind == :output_with_router ? :output : kind)
|
191
248
|
list.each do |instance|
|
249
|
+
if only_zero_downtime_restart_ready
|
250
|
+
next unless instance.respond_to?(:zero_downtime_restart_ready?) and instance.zero_downtime_restart_ready?
|
251
|
+
end
|
192
252
|
yield instance, display_kind
|
193
253
|
end
|
194
254
|
end
|
@@ -198,8 +258,8 @@ module Fluent
|
|
198
258
|
end
|
199
259
|
end
|
200
260
|
|
201
|
-
def start
|
202
|
-
lifecycle(desc: true) do |i| # instance
|
261
|
+
def start(kind_or_agent_list: nil)
|
262
|
+
lifecycle(desc: true, kind_or_agent_list: kind_or_agent_list) do |i| # instance
|
203
263
|
i.start unless i.started?
|
204
264
|
# Input#start sometimes emits lots of events with in_tail/`read_from_head true` case
|
205
265
|
# and it causes deadlock for small buffer/queue output. To avoid such problem,
|
@@ -231,13 +291,46 @@ module Fluent
|
|
231
291
|
flushing_threads.each{|t| t.join }
|
232
292
|
end
|
233
293
|
|
234
|
-
def
|
294
|
+
def cancel_source_only!
|
295
|
+
unless @source_only_mode.enabled?
|
296
|
+
log.info "do nothing for canceling with-source-only because the current mode is not with-source-only."
|
297
|
+
return
|
298
|
+
end
|
299
|
+
|
300
|
+
log.info "cancel with-source-only mode and start the other plugins"
|
301
|
+
all_plugins = [:input, :output_with_router, @labels.values, :filter, :output].flatten.reverse
|
302
|
+
start(kind_or_agent_list: all_plugins)
|
303
|
+
|
304
|
+
lifecycle_control_list[:input].each(&:event_emitter_cancel_source_only)
|
305
|
+
|
306
|
+
# Want to make sure that the source_only_router finishes all process before
|
307
|
+
# shutting down the agent.
|
308
|
+
# Strictly speaking, it would be necessary to have exclusive lock between
|
309
|
+
# EventRouter and the shutting down process of this agent.
|
310
|
+
# However, adding lock to EventRouter would worsen its performance, and
|
311
|
+
# the entire shutting down process does not care about it either.
|
312
|
+
# So, sleep here just in case.
|
313
|
+
sleep 1
|
314
|
+
|
315
|
+
shutdown(kind_or_agent_list: [@source_only_buffer_agent])
|
316
|
+
@source_only_buffer_agent = nil
|
317
|
+
|
318
|
+
# This agent can stop after flushing its all buffer, but it is not implemented for now.
|
319
|
+
log.info "starts the loading agent for with-source-only"
|
320
|
+
setup_source_only_buffer_agent(flush: true)
|
321
|
+
start(kind_or_agent_list: [@source_only_buffer_agent])
|
322
|
+
|
323
|
+
@source_only_mode.disable!
|
324
|
+
end
|
325
|
+
|
326
|
+
def shutdown(kind_or_agent_list: nil)
|
327
|
+
# Fluentd's shutdown sequence is stop, before_shutdown, shutdown, after_shutdown, close, terminate for plugins
|
235
328
|
# These method callers does `rescue Exception` to call methods of shutdown sequence as far as possible
|
236
329
|
# if plugin methods does something like infinite recursive call, `exit`, unregistering signal handlers or others.
|
237
330
|
# Plugins should be separated and be in sandbox to protect data in each plugins/buffers.
|
238
331
|
|
239
332
|
lifecycle_safe_sequence = ->(method, checker) {
|
240
|
-
lifecycle do |instance, kind|
|
333
|
+
lifecycle(kind_or_agent_list: kind_or_agent_list) do |instance, kind|
|
241
334
|
begin
|
242
335
|
log.debug "calling #{method} on #{kind} plugin", type: Plugin.lookup_type_from_class(instance.class), plugin_id: instance.plugin_id
|
243
336
|
instance.__send__(method) unless instance.__send__(checker)
|
@@ -260,7 +353,7 @@ module Fluent
|
|
260
353
|
operation_threads.each{|t| t.join }
|
261
354
|
operation_threads.clear
|
262
355
|
}
|
263
|
-
lifecycle(kind_callback: callback) do |instance, kind|
|
356
|
+
lifecycle(kind_callback: callback, kind_or_agent_list: kind_or_agent_list) do |instance, kind|
|
264
357
|
t = Thread.new do
|
265
358
|
Thread.current.abort_on_exception = true
|
266
359
|
begin
|
@@ -301,6 +394,8 @@ module Fluent
|
|
301
394
|
lifecycle_unsafe_sequence.call(:close, :closed?)
|
302
395
|
|
303
396
|
lifecycle_safe_sequence.call(:terminate, :terminated?)
|
397
|
+
|
398
|
+
cleanup_source_only_buffer_agent unless kind_or_agent_list
|
304
399
|
end
|
305
400
|
|
306
401
|
def suppress_interval(interval_time)
|
@@ -318,6 +413,7 @@ module Fluent
|
|
318
413
|
# See also 'fluentd/plugin/input.rb'
|
319
414
|
input.context_router = @event_router
|
320
415
|
input.configure(conf)
|
416
|
+
input.event_emitter_apply_source_only if @source_only_mode.enabled?
|
321
417
|
if @enable_input_metrics
|
322
418
|
@event_router.add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })
|
323
419
|
end
|