fluentd 1.14.4-x64-mingw32 → 1.15.0-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/config.yml +2 -2
- data/.github/workflows/linux-test.yaml +1 -1
- data/.github/workflows/macos-test.yaml +5 -1
- data/.github/workflows/windows-test.yaml +9 -6
- data/CHANGELOG.md +115 -19
- data/CONTRIBUTING.md +1 -1
- data/MAINTAINERS.md +2 -2
- data/README.md +2 -23
- data/Rakefile +1 -1
- data/fluentd.gemspec +3 -1
- data/lib/fluent/command/ctl.rb +4 -1
- data/lib/fluent/command/fluentd.rb +14 -0
- data/lib/fluent/config/error.rb +12 -0
- data/lib/fluent/config/literal_parser.rb +2 -2
- data/lib/fluent/config/yaml_parser/fluent_value.rb +47 -0
- data/lib/fluent/config/yaml_parser/loader.rb +91 -0
- data/lib/fluent/config/yaml_parser/parser.rb +166 -0
- data/lib/fluent/config/yaml_parser/section_builder.rb +107 -0
- data/lib/fluent/config/yaml_parser.rb +56 -0
- data/lib/fluent/config.rb +14 -1
- data/lib/fluent/event_router.rb +19 -1
- data/lib/fluent/plugin/bare_output.rb +1 -1
- data/lib/fluent/plugin/base.rb +1 -1
- data/lib/fluent/plugin/file_wrapper.rb +52 -107
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +11 -1
- data/lib/fluent/plugin/in_tail/group_watch.rb +204 -0
- data/lib/fluent/plugin/in_tail/position_file.rb +1 -15
- data/lib/fluent/plugin/in_tail.rb +66 -47
- data/lib/fluent/plugin/out_forward/socket_cache.rb +2 -0
- data/lib/fluent/plugin/output.rb +43 -33
- data/lib/fluent/plugin/parser.rb +3 -4
- data/lib/fluent/plugin/parser_syslog.rb +1 -1
- data/lib/fluent/plugin_helper/retry_state.rb +14 -4
- data/lib/fluent/plugin_helper/server.rb +23 -4
- data/lib/fluent/plugin_helper/service_discovery.rb +2 -2
- data/lib/fluent/plugin_helper/socket.rb +13 -2
- data/lib/fluent/registry.rb +2 -1
- data/lib/fluent/rpc.rb +4 -3
- data/lib/fluent/supervisor.rb +114 -27
- data/lib/fluent/system_config.rb +2 -1
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +2 -0
- data/test/command/test_ctl.rb +0 -1
- data/test/command/test_fluentd.rb +33 -0
- data/test/compat/test_parser.rb +1 -1
- data/test/config/test_system_config.rb +3 -1
- data/test/config/test_types.rb +1 -1
- data/test/plugin/in_tail/test_io_handler.rb +14 -4
- data/test/plugin/in_tail/test_position_file.rb +0 -63
- data/test/plugin/out_forward/test_socket_cache.rb +26 -1
- data/test/plugin/test_file_wrapper.rb +0 -68
- data/test/plugin/test_filter_parser.rb +1 -1
- data/test/plugin/test_filter_stdout.rb +2 -2
- data/test/plugin/test_in_forward.rb +0 -2
- data/test/plugin/test_in_http.rb +23 -0
- data/test/plugin/test_in_object_space.rb +9 -3
- data/test/plugin/test_in_syslog.rb +1 -1
- data/test/plugin/test_in_tail.rb +629 -353
- data/test/plugin/test_out_forward.rb +30 -20
- data/test/plugin/test_out_stdout.rb +2 -2
- data/test/plugin/test_output_as_buffered_retries.rb +53 -6
- data/test/plugin/test_output_as_buffered_secondary.rb +1 -1
- data/test/plugin/test_parser_syslog.rb +1 -1
- data/test/plugin_helper/test_cert_option.rb +1 -1
- data/test/plugin_helper/test_child_process.rb +16 -4
- data/test/plugin_helper/test_retry_state.rb +602 -38
- data/test/plugin_helper/test_server.rb +18 -0
- data/test/test_config.rb +135 -4
- data/test/test_event_router.rb +17 -0
- data/test/test_formatter.rb +1 -1
- data/test/test_supervisor.rb +196 -6
- metadata +39 -5
@@ -44,6 +44,8 @@ module Fluent
|
|
44
44
|
|
45
45
|
@timeout = timeout
|
46
46
|
@timeout_at = @start + timeout
|
47
|
+
@has_reached_timeout = false
|
48
|
+
@has_timed_out = false
|
47
49
|
@current = :primary
|
48
50
|
|
49
51
|
if randomize_width < 0 || randomize_width > 0.5
|
@@ -98,7 +100,7 @@ module Fluent
|
|
98
100
|
naive
|
99
101
|
end
|
100
102
|
elsif @current == :secondary
|
101
|
-
naive = naive_next_time(@steps - @secondary_transition_steps
|
103
|
+
naive = naive_next_time(@steps - @secondary_transition_steps)
|
102
104
|
if naive >= @timeout_at
|
103
105
|
@timeout_at
|
104
106
|
else
|
@@ -123,7 +125,15 @@ module Fluent
|
|
123
125
|
@current = :secondary
|
124
126
|
@secondary_transition_steps = @steps
|
125
127
|
end
|
128
|
+
|
126
129
|
@next_time = calc_next_time
|
130
|
+
|
131
|
+
if @has_reached_timeout
|
132
|
+
@has_timed_out = @next_time >= @timeout_at
|
133
|
+
else
|
134
|
+
@has_reached_timeout = @next_time >= @timeout_at
|
135
|
+
end
|
136
|
+
|
127
137
|
nil
|
128
138
|
end
|
129
139
|
|
@@ -135,7 +145,7 @@ module Fluent
|
|
135
145
|
if @forever
|
136
146
|
false
|
137
147
|
else
|
138
|
-
@
|
148
|
+
@has_timed_out || !!(@max_steps && @steps >= @max_steps)
|
139
149
|
end
|
140
150
|
end
|
141
151
|
end
|
@@ -165,7 +175,7 @@ module Fluent
|
|
165
175
|
end
|
166
176
|
|
167
177
|
def calc_interval(num)
|
168
|
-
interval = raw_interval(num
|
178
|
+
interval = raw_interval(num)
|
169
179
|
if @max_interval && interval > @max_interval
|
170
180
|
@max_interval
|
171
181
|
else
|
@@ -175,7 +185,7 @@ module Fluent
|
|
175
185
|
# Calculate previous finite value to avoid inf related errors. If this re-computing is heavy, use cache.
|
176
186
|
until interval.finite?
|
177
187
|
num -= 1
|
178
|
-
interval = raw_interval(num
|
188
|
+
interval = raw_interval(num)
|
179
189
|
end
|
180
190
|
interval
|
181
191
|
end
|
@@ -80,8 +80,8 @@ module Fluent
|
|
80
80
|
raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
|
81
81
|
raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
|
82
82
|
|
83
|
-
if proto == :tcp || proto == :tls
|
84
|
-
socket_options[:linger_timeout] ||= 0
|
83
|
+
if proto == :tcp || proto == :tls
|
84
|
+
socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
|
85
85
|
end
|
86
86
|
|
87
87
|
socket_option_validate!(proto, **socket_options)
|
@@ -132,8 +132,8 @@ module Fluent
|
|
132
132
|
raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
|
133
133
|
raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
|
134
134
|
|
135
|
-
if proto == :tcp || proto == :tls
|
136
|
-
socket_options[:linger_timeout] ||= 0
|
135
|
+
if proto == :tcp || proto == :tls
|
136
|
+
socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
|
137
137
|
end
|
138
138
|
|
139
139
|
unless socket
|
@@ -263,6 +263,25 @@ module Fluent
|
|
263
263
|
include Fluent::Configurable
|
264
264
|
config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
|
265
265
|
config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp
|
266
|
+
|
267
|
+
### Socket Params ###
|
268
|
+
|
269
|
+
# SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src.
|
270
|
+
# Set positive value if needing to send FIN on closing on non-Windows.
|
271
|
+
# (On Windows, Fluentd can send FIN with zero `linger_timeout` since Fluentd doesn't set 0 to SO_LINGER on Windows.
|
272
|
+
# See `socket_option.rb`.)
|
273
|
+
# NOTE:
|
274
|
+
# Socket-options can be specified from each plugin as needed, so most of them is not defined here for now.
|
275
|
+
# This is because there is no positive reason to do so.
|
276
|
+
# `linger_timeout` option in particular needs to be defined here
|
277
|
+
# although it can be specified from each plugin as well.
|
278
|
+
# This is because this helper fixes the default value to `0` for its own reason
|
279
|
+
# and it has a critical effect on the behavior.
|
280
|
+
desc 'The timeout time used to set linger option.'
|
281
|
+
config_param :linger_timeout, :integer, default: 0
|
282
|
+
|
283
|
+
### TLS Params ###
|
284
|
+
|
266
285
|
config_param :version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
|
267
286
|
config_param :min_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
|
268
287
|
config_param :max_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
|
@@ -44,7 +44,7 @@ module Fluent
|
|
44
44
|
|
45
45
|
@discovery_manager.start
|
46
46
|
unless @discovery_manager.static_config?
|
47
|
-
timer_execute(@_plugin_helper_service_discovery_title, @
|
47
|
+
timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_interval) do
|
48
48
|
@discovery_manager.run_once
|
49
49
|
end
|
50
50
|
end
|
@@ -96,7 +96,7 @@ module Fluent
|
|
96
96
|
# @param custom_build_method [Proc]
|
97
97
|
def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)
|
98
98
|
@_plugin_helper_service_discovery_title = title
|
99
|
-
@
|
99
|
+
@_plugin_helper_service_discovery_interval = interval
|
100
100
|
|
101
101
|
@discovery_manager = Fluent::PluginHelper::ServiceDiscovery::Manager.new(
|
102
102
|
log: log,
|
@@ -96,6 +96,7 @@ module Fluent
|
|
96
96
|
enable_system_cert_store: true, allow_self_signed_cert: false, cert_paths: nil,
|
97
97
|
cert_path: nil, private_key_path: nil, private_key_passphrase: nil,
|
98
98
|
cert_thumbprint: nil, cert_logical_store_name: nil, cert_use_enterprise_store: true,
|
99
|
+
connect_timeout: nil,
|
99
100
|
**kwargs, &block)
|
100
101
|
|
101
102
|
host_is_ipaddress = IPAddr.new(host) rescue false
|
@@ -158,13 +159,23 @@ module Fluent
|
|
158
159
|
end
|
159
160
|
Fluent::TLS.set_version_to_context(context, version, min_version, max_version)
|
160
161
|
|
161
|
-
tcpsock = socket_create_tcp(host, port, **kwargs)
|
162
|
+
tcpsock = socket_create_tcp(host, port, connect_timeout: connect_timeout, **kwargs)
|
162
163
|
sock = WrappedSocket::TLS.new(tcpsock, context)
|
163
164
|
sock.sync_close = true
|
164
165
|
sock.hostname = fqdn if verify_fqdn && fqdn && sock.respond_to?(:hostname=)
|
165
166
|
|
166
167
|
log.trace "entering TLS handshake"
|
167
|
-
|
168
|
+
if connect_timeout
|
169
|
+
begin
|
170
|
+
Timeout.timeout(connect_timeout) { sock.connect }
|
171
|
+
rescue Timeout::Error
|
172
|
+
log.warn "timeout while connecting tls session", host: host
|
173
|
+
sock.close rescue nil
|
174
|
+
raise
|
175
|
+
end
|
176
|
+
else
|
177
|
+
sock.connect
|
178
|
+
end
|
168
179
|
|
169
180
|
begin
|
170
181
|
if verify_fqdn
|
data/lib/fluent/registry.rb
CHANGED
@@ -45,7 +45,8 @@ module Fluent
|
|
45
45
|
if value = @map[type]
|
46
46
|
return value
|
47
47
|
end
|
48
|
-
raise
|
48
|
+
raise NotFoundPluginError.new("Unknown #{@kind} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins",
|
49
|
+
kind: @kind, type: type)
|
49
50
|
end
|
50
51
|
|
51
52
|
def reverse_lookup(value)
|
data/lib/fluent/rpc.rb
CHANGED
@@ -20,9 +20,10 @@ module Fluent
|
|
20
20
|
module RPC
|
21
21
|
class Server
|
22
22
|
def initialize(endpoint, log)
|
23
|
-
|
24
|
-
|
25
|
-
@
|
23
|
+
m = endpoint.match(/^\[?(?<host>[0-9a-zA-Z:\-\.]+)\]?:(?<port>[0-9]+)$/)
|
24
|
+
raise Fluent::ConfigError, "Invalid rpc_endpoint: #{endpoint}" unless m
|
25
|
+
@bind = m[:host]
|
26
|
+
@port = m[:port]
|
26
27
|
@log = log
|
27
28
|
|
28
29
|
@server = WEBrick::HTTPServer.new(
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -16,6 +16,7 @@
|
|
16
16
|
|
17
17
|
require 'fileutils'
|
18
18
|
require 'open3'
|
19
|
+
require 'pathname'
|
19
20
|
|
20
21
|
require 'fluent/config'
|
21
22
|
require 'fluent/counter'
|
@@ -215,44 +216,50 @@ module Fluent
|
|
215
216
|
Thread.new do
|
216
217
|
ipc = Win32::Ipc.new(nil)
|
217
218
|
events = [
|
218
|
-
Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
|
219
|
-
Win32::Event.new("#{@pid_signame}"),
|
220
|
-
Win32::Event.new("#{@pid_signame}_HUP"),
|
221
|
-
Win32::Event.new("#{@pid_signame}_USR1"),
|
222
|
-
Win32::Event.new("#{@pid_signame}_USR2"),
|
219
|
+
{win32_event: Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"), action: :stop_event_thread},
|
220
|
+
{win32_event: Win32::Event.new("#{@pid_signame}"), action: :stop},
|
221
|
+
{win32_event: Win32::Event.new("#{@pid_signame}_HUP"), action: :hup},
|
222
|
+
{win32_event: Win32::Event.new("#{@pid_signame}_USR1"), action: :usr1},
|
223
|
+
{win32_event: Win32::Event.new("#{@pid_signame}_USR2"), action: :usr2},
|
224
|
+
{win32_event: Win32::Event.new("#{@pid_signame}_CONT"), action: :cont},
|
223
225
|
]
|
224
226
|
if @signame
|
225
227
|
signame_events = [
|
226
|
-
Win32::Event.new("#{@signame}"),
|
227
|
-
Win32::Event.new("#{@signame}_HUP"),
|
228
|
-
Win32::Event.new("#{@signame}_USR1"),
|
229
|
-
Win32::Event.new("#{@signame}_USR2"),
|
228
|
+
{win32_event: Win32::Event.new("#{@signame}"), action: :stop},
|
229
|
+
{win32_event: Win32::Event.new("#{@signame}_HUP"), action: :hup},
|
230
|
+
{win32_event: Win32::Event.new("#{@signame}_USR1"), action: :usr1},
|
231
|
+
{win32_event: Win32::Event.new("#{@signame}_USR2"), action: :usr2},
|
232
|
+
{win32_event: Win32::Event.new("#{@signame}_CONT"), action: :cont},
|
230
233
|
]
|
231
234
|
events.concat(signame_events)
|
232
235
|
end
|
233
236
|
begin
|
234
237
|
loop do
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
+
ipc_idx = ipc.wait_any(events.map {|e| e[:win32_event]}, Windows::Synchronize::INFINITE)
|
239
|
+
event_idx = ipc_idx - 1
|
240
|
+
|
241
|
+
if event_idx >= 0 && event_idx < events.length
|
242
|
+
$log.debug("Got Win32 event \"#{events[event_idx][:win32_event].name}\"")
|
238
243
|
else
|
239
|
-
$log.warn("Unexpected
|
244
|
+
$log.warn("Unexpected return value of Win32::Ipc#wait_any: #{ipc_idx}")
|
240
245
|
end
|
241
|
-
case
|
242
|
-
when
|
246
|
+
case events[event_idx][:action]
|
247
|
+
when :stop
|
243
248
|
stop(true)
|
244
|
-
when
|
249
|
+
when :hup
|
245
250
|
supervisor_sighup_handler
|
246
|
-
when
|
251
|
+
when :usr1
|
247
252
|
supervisor_sigusr1_handler
|
248
|
-
when
|
253
|
+
when :usr2
|
249
254
|
supervisor_sigusr2_handler
|
250
|
-
when
|
255
|
+
when :cont
|
256
|
+
supervisor_dump_handler_for_windows
|
257
|
+
when :stop_event_thread
|
251
258
|
break
|
252
259
|
end
|
253
260
|
end
|
254
261
|
ensure
|
255
|
-
events.each { |event| event.close }
|
262
|
+
events.each { |event| event[:win32_event].close }
|
256
263
|
end
|
257
264
|
end
|
258
265
|
end
|
@@ -302,6 +309,26 @@ module Fluent
|
|
302
309
|
$log.error "Failed to reload config file: #{e}"
|
303
310
|
end
|
304
311
|
|
312
|
+
def supervisor_dump_handler_for_windows
|
313
|
+
# As for UNIX-like, SIGCONT signal to each process makes the process output its dump-file,
|
314
|
+
# and it is implemented before the implementation of the function for Windows.
|
315
|
+
# It is possible to trap SIGCONT and handle it here also on UNIX-like,
|
316
|
+
# but for backward compatibility, this handler is currently for a Windows-only.
|
317
|
+
raise "[BUG] This function is for Windows ONLY." unless Fluent.windows?
|
318
|
+
|
319
|
+
Thread.new do
|
320
|
+
begin
|
321
|
+
FluentSigdump.dump_windows
|
322
|
+
rescue => e
|
323
|
+
$log.error "failed to dump: #{e}"
|
324
|
+
end
|
325
|
+
end
|
326
|
+
|
327
|
+
send_signal_to_workers(:CONT)
|
328
|
+
rescue => e
|
329
|
+
$log.error "failed to dump: #{e}"
|
330
|
+
end
|
331
|
+
|
305
332
|
def kill_worker
|
306
333
|
if config[:worker_pid]
|
307
334
|
pids = config[:worker_pid].clone
|
@@ -358,6 +385,14 @@ module Fluent
|
|
358
385
|
restart(true)
|
359
386
|
when :USR2
|
360
387
|
reload
|
388
|
+
when :CONT
|
389
|
+
dump_all_windows_workers
|
390
|
+
end
|
391
|
+
end
|
392
|
+
|
393
|
+
def dump_all_windows_workers
|
394
|
+
@monitors.each do |m|
|
395
|
+
m.send_command("DUMP\n")
|
361
396
|
end
|
362
397
|
end
|
363
398
|
end
|
@@ -397,6 +432,7 @@ module Fluent
|
|
397
432
|
log_path = params['log_path']
|
398
433
|
chuser = params['chuser']
|
399
434
|
chgroup = params['chgroup']
|
435
|
+
chumask = params['chumask']
|
400
436
|
log_rotate_age = params['log_rotate_age']
|
401
437
|
log_rotate_size = params['log_rotate_size']
|
402
438
|
|
@@ -436,7 +472,7 @@ module Fluent
|
|
436
472
|
logger_initializer: logger_initializer,
|
437
473
|
chuser: chuser,
|
438
474
|
chgroup: chgroup,
|
439
|
-
chumask:
|
475
|
+
chumask: chumask,
|
440
476
|
suppress_repeated_stacktrace: suppress_repeated_stacktrace,
|
441
477
|
ignore_repeated_log_interval: ignore_repeated_log_interval,
|
442
478
|
ignore_same_log_interval: ignore_same_log_interval,
|
@@ -457,7 +493,8 @@ module Fluent
|
|
457
493
|
config_path: path,
|
458
494
|
main_cmd: params['main_cmd'],
|
459
495
|
signame: params['signame'],
|
460
|
-
disable_shared_socket: params['disable_shared_socket']
|
496
|
+
disable_shared_socket: params['disable_shared_socket'],
|
497
|
+
restart_worker_interval: params['restart_worker_interval'],
|
461
498
|
}
|
462
499
|
if daemonize
|
463
500
|
se_config[:pid_path] = pid_path
|
@@ -565,6 +602,7 @@ module Fluent
|
|
565
602
|
setup_path: nil,
|
566
603
|
chuser: nil,
|
567
604
|
chgroup: nil,
|
605
|
+
chumask: "0",
|
568
606
|
root_dir: nil,
|
569
607
|
suppress_interval: 0,
|
570
608
|
suppress_repeated_stacktrace: true,
|
@@ -578,7 +616,8 @@ module Fluent
|
|
578
616
|
standalone_worker: false,
|
579
617
|
signame: nil,
|
580
618
|
conf_encoding: 'utf-8',
|
581
|
-
disable_shared_socket: nil
|
619
|
+
disable_shared_socket: nil,
|
620
|
+
config_file_type: :guess,
|
582
621
|
}
|
583
622
|
end
|
584
623
|
|
@@ -591,6 +630,7 @@ module Fluent
|
|
591
630
|
end
|
592
631
|
|
593
632
|
def initialize(opt)
|
633
|
+
@config_file_type = opt[:config_file_type]
|
594
634
|
@daemonize = opt[:daemonize]
|
595
635
|
@standalone_worker= opt[:standalone_worker]
|
596
636
|
@config_path = opt[:config_path]
|
@@ -603,6 +643,7 @@ module Fluent
|
|
603
643
|
@plugin_dirs = opt[:plugin_dirs]
|
604
644
|
@chgroup = opt[:chgroup]
|
605
645
|
@chuser = opt[:chuser]
|
646
|
+
@chumask = opt[:chumask]
|
606
647
|
|
607
648
|
@log_rotate_age = opt[:log_rotate_age]
|
608
649
|
@log_rotate_size = opt[:log_rotate_size]
|
@@ -615,7 +656,9 @@ module Fluent
|
|
615
656
|
@conf = Fluent::Config.build(config_path: @config_path,
|
616
657
|
encoding: @conf_encoding ? @conf_encoding : 'utf-8',
|
617
658
|
additional_config: @inline_config ? @inline_config : nil,
|
618
|
-
use_v1_config: !!@use_v1_config
|
659
|
+
use_v1_config: !!@use_v1_config,
|
660
|
+
type: @config_file_type,
|
661
|
+
)
|
619
662
|
@system_config = build_system_config(@conf)
|
620
663
|
if @system_config.log
|
621
664
|
@log_rotate_age ||= @system_config.log.rotate_age
|
@@ -709,7 +752,7 @@ module Fluent
|
|
709
752
|
create_socket_manager if @standalone_worker
|
710
753
|
if @standalone_worker
|
711
754
|
ServerEngine::Privilege.change(@chuser, @chgroup)
|
712
|
-
File.umask(
|
755
|
+
File.umask(@chumask.to_i(8))
|
713
756
|
end
|
714
757
|
MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
|
715
758
|
Fluent::Engine.init(@system_config)
|
@@ -741,7 +784,13 @@ module Fluent
|
|
741
784
|
$log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
|
742
785
|
@inline_config = STDIN.read
|
743
786
|
end
|
744
|
-
@conf = Fluent::Config.build(
|
787
|
+
@conf = Fluent::Config.build(
|
788
|
+
config_path: @config_path,
|
789
|
+
encoding: @conf_encoding,
|
790
|
+
additional_config: @inline_config,
|
791
|
+
use_v1_config: @use_v1_config,
|
792
|
+
type: @config_file_type,
|
793
|
+
)
|
745
794
|
@system_config = build_system_config(@conf)
|
746
795
|
|
747
796
|
@log.level = @system_config.log_level
|
@@ -819,7 +868,8 @@ module Fluent
|
|
819
868
|
'counter_server' => @system_config.counter_server,
|
820
869
|
'log_format' => @system_config.log.format,
|
821
870
|
'log_time_format' => @system_config.log.time_format,
|
822
|
-
'disable_shared_socket' => @system_config.disable_shared_socket
|
871
|
+
'disable_shared_socket' => @system_config.disable_shared_socket,
|
872
|
+
'restart_worker_interval' => @system_config.restart_worker_interval,
|
823
873
|
}
|
824
874
|
|
825
875
|
se = ServerEngine.create(ServerModule, WorkerModule){
|
@@ -893,6 +943,9 @@ module Fluent
|
|
893
943
|
when "RELOAD"
|
894
944
|
$log.debug "fluentd main process get #{cmd} command"
|
895
945
|
reload_config
|
946
|
+
when "DUMP"
|
947
|
+
$log.debug "fluentd main process get #{cmd} command"
|
948
|
+
dump
|
896
949
|
else
|
897
950
|
$log.warn "fluentd main process get unknown command [#{cmd}]"
|
898
951
|
end
|
@@ -926,6 +979,7 @@ module Fluent
|
|
926
979
|
encoding: @conf_encoding,
|
927
980
|
additional_config: @inline_config,
|
928
981
|
use_v1_config: @use_v1_config,
|
982
|
+
type: @config_file_type,
|
929
983
|
)
|
930
984
|
|
931
985
|
Fluent::VariableStore.try_to_reset do
|
@@ -942,6 +996,16 @@ module Fluent
|
|
942
996
|
end
|
943
997
|
end
|
944
998
|
|
999
|
+
def dump
|
1000
|
+
Thread.new do
|
1001
|
+
begin
|
1002
|
+
FluentSigdump.dump_windows
|
1003
|
+
rescue => e
|
1004
|
+
$log.error("failed to dump: #{e}")
|
1005
|
+
end
|
1006
|
+
end
|
1007
|
+
end
|
1008
|
+
|
945
1009
|
def logging_with_console_output
|
946
1010
|
yield $log
|
947
1011
|
unless @log.stdout?
|
@@ -1051,4 +1115,27 @@ module Fluent
|
|
1051
1115
|
fluentd_spawn_cmd
|
1052
1116
|
end
|
1053
1117
|
end
|
1118
|
+
|
1119
|
+
module FluentSigdump
|
1120
|
+
def self.dump_windows
|
1121
|
+
raise "[BUG] WindowsSigdump::dump is for Windows ONLY." unless Fluent.windows?
|
1122
|
+
|
1123
|
+
# Sigdump outputs under `/tmp` dir without `SIGDUMP_PATH` specified,
|
1124
|
+
# but `/tmp` dir may not exist on Windows by default.
|
1125
|
+
# So use the systemroot-temp-dir instead.
|
1126
|
+
dump_filepath = ENV['SIGDUMP_PATH'].nil? || ENV['SIGDUMP_PATH'].empty? \
|
1127
|
+
? "#{ENV['windir']}/Temp/fluentd-sigdump-#{Process.pid}.log"
|
1128
|
+
: get_path_with_pid(ENV['SIGDUMP_PATH'])
|
1129
|
+
|
1130
|
+
require 'sigdump'
|
1131
|
+
Sigdump.dump(dump_filepath)
|
1132
|
+
|
1133
|
+
$log.info "dump to #{dump_filepath}."
|
1134
|
+
end
|
1135
|
+
|
1136
|
+
def self.get_path_with_pid(raw_path)
|
1137
|
+
path = Pathname.new(raw_path)
|
1138
|
+
path.sub_ext("-#{Process.pid}#{path.extname}").to_s
|
1139
|
+
end
|
1140
|
+
end
|
1054
1141
|
end
|
data/lib/fluent/system_config.rb
CHANGED
@@ -22,7 +22,7 @@ module Fluent
|
|
22
22
|
include Configurable
|
23
23
|
|
24
24
|
SYSTEM_CONFIG_PARAMETERS = [
|
25
|
-
:workers, :root_dir, :log_level,
|
25
|
+
:workers, :restart_worker_interval, :root_dir, :log_level,
|
26
26
|
:suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
|
27
27
|
:log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
|
28
28
|
:without_source, :rpc_endpoint, :enable_get_dump, :process_name,
|
@@ -32,6 +32,7 @@ module Fluent
|
|
32
32
|
]
|
33
33
|
|
34
34
|
config_param :workers, :integer, default: 1
|
35
|
+
config_param :restart_worker_interval, :time, default: 0
|
35
36
|
config_param :root_dir, :string, default: nil
|
36
37
|
config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
|
37
38
|
config_param :suppress_repeated_stacktrace, :bool, default: nil
|
data/lib/fluent/version.rb
CHANGED
data/lib/fluent/winsvc.rb
CHANGED
data/test/command/test_ctl.rb
CHANGED
@@ -7,7 +7,6 @@ require 'fluent/command/ctl'
|
|
7
7
|
|
8
8
|
class TestFluentdCtl < ::Test::Unit::TestCase
|
9
9
|
def assert_win32_event(event_name, command, pid_or_svcname)
|
10
|
-
command, event_suffix = data
|
11
10
|
event = Win32::Event.new(event_name)
|
12
11
|
ipc = Win32::Ipc.new(event.handle)
|
13
12
|
ret = Win32::Ipc::TIMEOUT
|
@@ -589,6 +589,39 @@ CONF
|
|
589
589
|
)
|
590
590
|
end
|
591
591
|
|
592
|
+
sub_test_case "YAML config format" do
|
593
|
+
test 'success to start the number of workers specified in configuration' do
|
594
|
+
conf = <<'CONF'
|
595
|
+
system:
|
596
|
+
workers: 2
|
597
|
+
root_dir: "#{@root_path}"
|
598
|
+
config:
|
599
|
+
- source:
|
600
|
+
$type: dummy
|
601
|
+
$id: !fluent/s "dummy.#{worker_id}" # check worker_id works or not with actual command
|
602
|
+
$label: '@dummydata'
|
603
|
+
tag: dummy
|
604
|
+
dummy: !fluent/json {"message": !fluent/s "yay from #{hostname}!"}
|
605
|
+
|
606
|
+
- label:
|
607
|
+
$name: '@dummydata'
|
608
|
+
config:
|
609
|
+
- match:
|
610
|
+
$tag: dummy
|
611
|
+
$type: "null"
|
612
|
+
$id: blackhole
|
613
|
+
CONF
|
614
|
+
conf_path = create_conf_file('workers1.yaml', conf)
|
615
|
+
assert Dir.exist?(@root_path)
|
616
|
+
|
617
|
+
assert_log_matches(
|
618
|
+
create_cmdline(conf_path),
|
619
|
+
"#0 fluentd worker is now running worker=0",
|
620
|
+
"#1 fluentd worker is now running worker=1"
|
621
|
+
)
|
622
|
+
end
|
623
|
+
end
|
624
|
+
|
592
625
|
test 'success to start the number of workers specified by command line option' do
|
593
626
|
conf = <<CONF
|
594
627
|
<system>
|
data/test/compat/test_parser.rb
CHANGED
@@ -22,7 +22,7 @@ class TextParserTest < ::Test::Unit::TestCase
|
|
22
22
|
Fluent::TextParser.register_template('multi_event_test', Proc.new { MultiEventTestParser.new })
|
23
23
|
|
24
24
|
def test_lookup_unknown_format
|
25
|
-
assert_raise Fluent::
|
25
|
+
assert_raise Fluent::NotFoundPluginError do
|
26
26
|
Fluent::Plugin.new_parser('unknown')
|
27
27
|
end
|
28
28
|
end
|
@@ -19,6 +19,7 @@ module Fluent::Config
|
|
19
19
|
@system_config = nil
|
20
20
|
@cl_opt = {
|
21
21
|
wokers: nil,
|
22
|
+
restart_worker_interval: nil,
|
22
23
|
root_dir: nil,
|
23
24
|
log: FakeLoggerInitializer.new,
|
24
25
|
log_level: Fluent::Log::LEVEL_INFO,
|
@@ -72,6 +73,7 @@ module Fluent::Config
|
|
72
73
|
sc = Fluent::SystemConfig.new(conf)
|
73
74
|
sc.overwrite_variables(**s.for_system_config)
|
74
75
|
assert_equal(1, sc.workers)
|
76
|
+
assert_equal(0, sc.restart_worker_interval)
|
75
77
|
assert_nil(sc.root_dir)
|
76
78
|
assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
|
77
79
|
assert_nil(sc.suppress_repeated_stacktrace)
|
@@ -88,6 +90,7 @@ module Fluent::Config
|
|
88
90
|
|
89
91
|
data(
|
90
92
|
'workers' => ['workers', 3],
|
93
|
+
'restart_worker_interval' => ['restart_worker_interval', 60],
|
91
94
|
'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
|
92
95
|
'log_level' => ['log_level', 'error'],
|
93
96
|
'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
|
@@ -174,7 +177,6 @@ module Fluent::Config
|
|
174
177
|
</log>
|
175
178
|
</system>
|
176
179
|
EOS
|
177
|
-
s = FakeSupervisor.new
|
178
180
|
sc = Fluent::SystemConfig.new(conf)
|
179
181
|
assert_equal(3, sc.log.rotate_age)
|
180
182
|
end
|
data/test/config/test_types.rb
CHANGED
@@ -190,7 +190,7 @@ class TestConfigTypes < ::Test::Unit::TestCase
|
|
190
190
|
data("val" => [:val, 'val'],
|
191
191
|
"v" => [:v, 'v'],
|
192
192
|
"value" => [:value, 'value'])
|
193
|
-
test 'enum' do |(expected, val
|
193
|
+
test 'enum' do |(expected, val)|
|
194
194
|
assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]})
|
195
195
|
end
|
196
196
|
|
@@ -21,12 +21,20 @@ class IntailIOHandlerTest < Test::Unit::TestCase
|
|
21
21
|
@file.unlink rescue nil
|
22
22
|
end
|
23
23
|
|
24
|
+
def create_target_info
|
25
|
+
Fluent::Plugin::TailInput::TargetInfo.new(@file.path, Fluent::FileWrapper.stat(@file.path).ino)
|
26
|
+
end
|
27
|
+
|
28
|
+
def create_watcher
|
29
|
+
Fluent::Plugin::TailInput::TailWatcher.new(create_target_info, nil, nil, nil, nil, nil, nil, nil, nil)
|
30
|
+
end
|
31
|
+
|
24
32
|
test '#on_notify load file content and passed it to receive_lines method' do
|
25
33
|
text = "this line is test\ntest line is test\n"
|
26
34
|
@file.write(text)
|
27
35
|
@file.close
|
28
36
|
|
29
|
-
watcher =
|
37
|
+
watcher = create_watcher
|
30
38
|
|
31
39
|
update_pos = 0
|
32
40
|
|
@@ -61,7 +69,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
|
|
61
69
|
|
62
70
|
update_pos = 0
|
63
71
|
|
64
|
-
watcher =
|
72
|
+
watcher = create_watcher
|
65
73
|
stub(watcher).pe do
|
66
74
|
pe = 'position_file'
|
67
75
|
stub(pe).read_pos { 0 }
|
@@ -92,7 +100,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
|
|
92
100
|
|
93
101
|
update_pos = 0
|
94
102
|
|
95
|
-
watcher =
|
103
|
+
watcher = create_watcher
|
96
104
|
stub(watcher).pe do
|
97
105
|
pe = 'position_file'
|
98
106
|
stub(pe).read_pos { 0 }
|
@@ -107,6 +115,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
|
|
107
115
|
end
|
108
116
|
|
109
117
|
r.on_notify
|
118
|
+
assert_equal text.bytesize, update_pos
|
110
119
|
assert_equal 8, returned_lines[0].size
|
111
120
|
end
|
112
121
|
|
@@ -118,7 +127,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
|
|
118
127
|
|
119
128
|
update_pos = 0
|
120
129
|
|
121
|
-
watcher =
|
130
|
+
watcher = create_watcher
|
122
131
|
stub(watcher).pe do
|
123
132
|
pe = 'position_file'
|
124
133
|
stub(pe).read_pos { 0 }
|
@@ -133,6 +142,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
|
|
133
142
|
end
|
134
143
|
|
135
144
|
r.on_notify
|
145
|
+
assert_equal text.bytesize, update_pos
|
136
146
|
assert_equal 5, returned_lines[0].size
|
137
147
|
assert_equal 3, returned_lines[1].size
|
138
148
|
end
|