fluentd 1.11.1-x86-mingw32 → 1.12.0-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.md +1 -1
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/workflows/stale-actions.yml +22 -0
- data/.travis.yml +22 -2
- data/CHANGELOG.md +111 -0
- data/README.md +1 -1
- data/appveyor.yml +3 -0
- data/bin/fluent-cap-ctl +7 -0
- data/bin/fluent-ctl +7 -0
- data/example/copy_roundrobin.conf +3 -3
- data/example/counter.conf +1 -1
- data/example/filter_stdout.conf +2 -2
- data/example/{in_dummy_blocks.conf → in_sample_blocks.conf} +4 -4
- data/example/{in_dummy_with_compression.conf → in_sample_with_compression.conf} +3 -3
- data/example/logevents.conf +5 -5
- data/example/multi_filters.conf +1 -1
- data/example/out_exec_filter.conf +2 -2
- data/example/out_forward.conf +1 -1
- data/example/out_forward_buf_file.conf +1 -1
- data/example/out_forward_client.conf +5 -5
- data/example/out_forward_heartbeat_none.conf +1 -1
- data/example/out_forward_sd.conf +1 -1
- data/example/out_forward_shared_key.conf +2 -2
- data/example/out_forward_tls.conf +1 -1
- data/example/out_forward_users.conf +3 -3
- data/example/out_null.conf +4 -4
- data/example/secondary_file.conf +1 -1
- data/fluentd.gemspec +7 -6
- data/lib/fluent/capability.rb +87 -0
- data/lib/fluent/command/cap_ctl.rb +174 -0
- data/lib/fluent/command/ctl.rb +177 -0
- data/lib/fluent/command/plugin_config_formatter.rb +2 -1
- data/lib/fluent/env.rb +4 -0
- data/lib/fluent/log.rb +33 -3
- data/lib/fluent/match.rb +9 -0
- data/lib/fluent/plugin.rb +5 -0
- data/lib/fluent/plugin/buffer.rb +32 -42
- data/lib/fluent/plugin/buffer/chunk.rb +2 -1
- data/lib/fluent/plugin/formatter.rb +24 -0
- data/lib/fluent/plugin/formatter_csv.rb +1 -1
- data/lib/fluent/plugin/formatter_hash.rb +3 -1
- data/lib/fluent/plugin/formatter_json.rb +3 -1
- data/lib/fluent/plugin/formatter_ltsv.rb +5 -3
- data/lib/fluent/plugin/formatter_out_file.rb +6 -4
- data/lib/fluent/plugin/formatter_single_value.rb +4 -2
- data/lib/fluent/plugin/formatter_tsv.rb +4 -2
- data/lib/fluent/plugin/in_dummy.rb +2 -123
- data/lib/fluent/plugin/in_exec.rb +4 -2
- data/lib/fluent/plugin/in_http.rb +25 -4
- data/lib/fluent/plugin/in_sample.rb +141 -0
- data/lib/fluent/plugin/in_tail.rb +109 -41
- data/lib/fluent/plugin/in_tail/position_file.rb +39 -14
- data/lib/fluent/plugin/in_tcp.rb +1 -0
- data/lib/fluent/plugin/out_http.rb +20 -2
- data/lib/fluent/plugin/output.rb +15 -6
- data/lib/fluent/plugin/parser_json.rb +5 -2
- data/lib/fluent/plugin_helper/cert_option.rb +5 -8
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +1 -1
- data/lib/fluent/plugin_helper/inject.rb +4 -1
- data/lib/fluent/plugin_helper/retry_state.rb +4 -0
- data/lib/fluent/plugin_helper/socket.rb +1 -1
- data/lib/fluent/supervisor.rb +151 -48
- data/lib/fluent/system_config.rb +2 -1
- data/lib/fluent/time.rb +1 -0
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +22 -4
- data/test/command/test_binlog_reader.rb +22 -6
- data/test/command/test_cap_ctl.rb +100 -0
- data/test/command/test_ctl.rb +57 -0
- data/test/command/test_plugin_config_formatter.rb +57 -2
- data/test/plugin/in_tail/test_position_file.rb +45 -25
- data/test/plugin/test_buffer.rb +4 -0
- data/test/plugin/test_filter_stdout.rb +6 -1
- data/test/plugin/test_formatter_hash.rb +6 -3
- data/test/plugin/test_formatter_json.rb +14 -4
- data/test/plugin/test_formatter_ltsv.rb +13 -5
- data/test/plugin/test_formatter_out_file.rb +35 -14
- data/test/plugin/test_formatter_single_value.rb +12 -6
- data/test/plugin/test_formatter_tsv.rb +12 -4
- data/test/plugin/test_in_exec.rb +18 -0
- data/test/plugin/test_in_http.rb +25 -0
- data/test/plugin/{test_in_dummy.rb → test_in_sample.rb} +25 -25
- data/test/plugin/test_in_tail.rb +433 -30
- data/test/plugin/test_out_file.rb +23 -18
- data/test/plugin/test_output.rb +12 -0
- data/test/plugin/test_parser_syslog.rb +2 -2
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/test_cert_option.rb +7 -0
- data/test/plugin_helper/test_compat_parameters.rb +7 -2
- data/test/plugin_helper/test_http_server_helper.rb +5 -0
- data/test/plugin_helper/test_inject.rb +42 -0
- data/test/plugin_helper/test_server.rb +34 -0
- data/test/plugin_helper/test_socket.rb +8 -0
- data/test/test_capability.rb +74 -0
- data/test/test_formatter.rb +34 -10
- data/test/test_log.rb +44 -0
- data/test/test_match.rb +11 -0
- data/test/test_output.rb +6 -1
- data/test/test_static_config_analysis.rb +2 -2
- data/test/test_supervisor.rb +119 -1
- metadata +50 -18
@@ -22,34 +22,40 @@ module Fluent::Plugin
|
|
22
22
|
UNWATCHED_POSITION = 0xffffffffffffffff
|
23
23
|
POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
|
24
24
|
|
25
|
-
def self.load(file, logger:)
|
26
|
-
pf = new(file, logger: logger)
|
25
|
+
def self.load(file, follow_inodes, existing_paths, logger:)
|
26
|
+
pf = new(file, follow_inodes, existing_paths, logger: logger)
|
27
27
|
pf.load
|
28
28
|
pf
|
29
29
|
end
|
30
30
|
|
31
|
-
def initialize(file, logger: nil)
|
31
|
+
def initialize(file, follow_inodes, existing_paths, logger: nil)
|
32
32
|
@file = file
|
33
33
|
@logger = logger
|
34
34
|
@file_mutex = Mutex.new
|
35
35
|
@map = {}
|
36
|
+
@follow_inodes = follow_inodes
|
37
|
+
@existing_paths = existing_paths
|
36
38
|
end
|
37
39
|
|
38
|
-
def [](
|
39
|
-
if m = @map[path]
|
40
|
+
def [](target_info)
|
41
|
+
if m = @map[@follow_inodes ? target_info.ino : target_info.path]
|
40
42
|
return m
|
41
43
|
end
|
42
44
|
|
43
45
|
@file_mutex.synchronize {
|
44
46
|
@file.seek(0, IO::SEEK_END)
|
45
|
-
seek = @file.pos + path.bytesize + 1
|
46
|
-
@file.write "#{path}\t0000000000000000\t0000000000000000\n"
|
47
|
-
|
47
|
+
seek = @file.pos + target_info.path.bytesize + 1
|
48
|
+
@file.write "#{target_info.path}\t0000000000000000\t0000000000000000\n"
|
49
|
+
if @follow_inodes
|
50
|
+
@map[target_info.ino] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
|
51
|
+
else
|
52
|
+
@map[target_info.path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
|
53
|
+
end
|
48
54
|
}
|
49
55
|
end
|
50
56
|
|
51
|
-
def unwatch(
|
52
|
-
if (entry = @map.delete(path))
|
57
|
+
def unwatch(target_info)
|
58
|
+
if (entry = @map.delete(@follow_inodes ? target_info.ino : target_info.path))
|
53
59
|
entry.update_pos(UNWATCHED_POSITION)
|
54
60
|
end
|
55
61
|
end
|
@@ -69,7 +75,11 @@ module Fluent::Plugin
|
|
69
75
|
pos = m[2].to_i(16)
|
70
76
|
ino = m[3].to_i(16)
|
71
77
|
seek = @file.pos - line.bytesize + path.bytesize + 1
|
72
|
-
|
78
|
+
if @follow_inodes
|
79
|
+
map[ino] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
|
80
|
+
else
|
81
|
+
map[path] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
|
82
|
+
end
|
73
83
|
end
|
74
84
|
end
|
75
85
|
|
@@ -94,8 +104,9 @@ module Fluent::Plugin
|
|
94
104
|
@file.truncate(0)
|
95
105
|
@file.write(entries.values.map(&:to_entry_fmt).join)
|
96
106
|
|
97
|
-
|
98
|
-
|
107
|
+
# entry contains path/ino key and value.
|
108
|
+
entries.each do |key, val|
|
109
|
+
if (m = @map[key])
|
99
110
|
m.seek = val.seek
|
100
111
|
end
|
101
112
|
end
|
@@ -139,13 +150,25 @@ module Fluent::Plugin
|
|
139
150
|
@logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
|
140
151
|
end
|
141
152
|
|
142
|
-
|
153
|
+
if @follow_inodes
|
154
|
+
entries[ino] = Entry.new(path, pos, ino, file_pos + path.size + 1)
|
155
|
+
else
|
156
|
+
entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
|
157
|
+
end
|
143
158
|
file_pos += line.size
|
144
159
|
end
|
145
160
|
end
|
146
161
|
|
162
|
+
entries = remove_deleted_files_entries(entries, @existing_paths) if @follow_inodes
|
147
163
|
entries
|
148
164
|
end
|
165
|
+
|
166
|
+
def remove_deleted_files_entries(existent_entries, existing_paths)
|
167
|
+
filtered_entries = existent_entries.select {|file_entry|
|
168
|
+
existing_paths.key?(file_entry)
|
169
|
+
}
|
170
|
+
filtered_entries
|
171
|
+
end
|
149
172
|
end
|
150
173
|
|
151
174
|
Entry = Struct.new(:path, :pos, :ino, :seek) do
|
@@ -224,5 +247,7 @@ module Fluent::Plugin
|
|
224
247
|
@inode
|
225
248
|
end
|
226
249
|
end
|
250
|
+
|
251
|
+
TargetInfo = Struct.new(:path, :ino)
|
227
252
|
end
|
228
253
|
end
|
data/lib/fluent/plugin/in_tcp.rb
CHANGED
@@ -98,6 +98,7 @@ module Fluent::Plugin
|
|
98
98
|
def start
|
99
99
|
super
|
100
100
|
|
101
|
+
log.info "listening tcp socket", bind: @bind, port: @port
|
101
102
|
del_size = @delimiter.length
|
102
103
|
if @_extract_enabled && @_extract_tag_key
|
103
104
|
server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|
|
@@ -21,6 +21,16 @@ require 'fluent/tls'
|
|
21
21
|
require 'fluent/plugin/output'
|
22
22
|
require 'fluent/plugin_helper/socket'
|
23
23
|
|
24
|
+
# patch Net::HTTP to support extra_chain_cert which was added in Ruby feature #9758.
|
25
|
+
# see: https://github.com/ruby/ruby/commit/31af0dafba6d3769d2a39617c0dddedb97883712
|
26
|
+
unless Net::HTTP::SSL_IVNAMES.include?(:@extra_chain_cert)
|
27
|
+
class Net::HTTP
|
28
|
+
SSL_IVNAMES << :@extra_chain_cert
|
29
|
+
SSL_ATTRIBUTES << :extra_chain_cert
|
30
|
+
attr_accessor :extra_chain_cert
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
24
34
|
module Fluent::Plugin
|
25
35
|
class HTTPOutput < Output
|
26
36
|
Fluent::Plugin.register_output('http', self)
|
@@ -171,7 +181,15 @@ module Fluent::Plugin
|
|
171
181
|
end
|
172
182
|
if @tls_client_cert_path
|
173
183
|
raise Fluent::ConfigError, "tls_client_cert_path is wrong: #{@tls_client_cert_path}" unless File.file?(@tls_client_cert_path)
|
174
|
-
|
184
|
+
|
185
|
+
bundle = File.read(@tls_client_cert_path)
|
186
|
+
bundle_certs = bundle.scan(/-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/)
|
187
|
+
opt[:cert] = OpenSSL::X509::Certificate.new(bundle_certs[0])
|
188
|
+
|
189
|
+
intermediate_certs = bundle_certs[1..-1]
|
190
|
+
if intermediate_certs
|
191
|
+
opt[:extra_chain_cert] = intermediate_certs.map { |cert| OpenSSL::X509::Certificate.new(cert) }
|
192
|
+
end
|
175
193
|
end
|
176
194
|
if @tls_private_key_path
|
177
195
|
raise Fluent::ConfigError, "tls_private_key_path is wrong: #{@tls_private_key_path}" unless File.file?(@tls_private_key_path)
|
@@ -215,7 +233,7 @@ module Fluent::Plugin
|
|
215
233
|
req.basic_auth(@auth.username, @auth.password)
|
216
234
|
end
|
217
235
|
set_headers(req)
|
218
|
-
req.body = @json_array ? "[#{chunk.read.chop
|
236
|
+
req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
|
219
237
|
req
|
220
238
|
end
|
221
239
|
|
data/lib/fluent/plugin/output.rb
CHANGED
@@ -340,6 +340,7 @@ module Fluent
|
|
340
340
|
buffer_conf = conf.elements(name: 'buffer').first || Fluent::Config::Element.new('buffer', '', {}, [])
|
341
341
|
@buffer = Plugin.new_buffer(buffer_type, parent: self)
|
342
342
|
@buffer.configure(buffer_conf)
|
343
|
+
@buffer.enable_update_timekeys if @chunk_key_time
|
343
344
|
|
344
345
|
@flush_at_shutdown = @buffer_config.flush_at_shutdown
|
345
346
|
if @flush_at_shutdown.nil?
|
@@ -768,17 +769,19 @@ module Fluent
|
|
768
769
|
end
|
769
770
|
end
|
770
771
|
|
771
|
-
|
772
|
-
log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
|
773
|
-
end
|
774
|
-
|
775
|
-
rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
|
772
|
+
rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
|
776
773
|
if chunk_passed
|
777
774
|
dump_unique_id_hex(chunk.unique_id)
|
778
775
|
else
|
779
776
|
log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
|
780
777
|
end
|
781
778
|
}
|
779
|
+
|
780
|
+
if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
|
781
|
+
log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
|
782
|
+
end
|
783
|
+
|
784
|
+
rvalue
|
782
785
|
end
|
783
786
|
end
|
784
787
|
|
@@ -1249,7 +1252,13 @@ module Fluent
|
|
1249
1252
|
log.debug "buffer queue cleared"
|
1250
1253
|
@retry = nil
|
1251
1254
|
else
|
1252
|
-
@retry.
|
1255
|
+
# Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
|
1256
|
+
# @retry.step is called almost as many times as the number of flush threads in a short time.
|
1257
|
+
if Time.now >= @retry.next_time
|
1258
|
+
@retry.step
|
1259
|
+
else
|
1260
|
+
@retry.recalc_next_time # to prevent all flush threads from retrying at the same time
|
1261
|
+
end
|
1253
1262
|
if error
|
1254
1263
|
if using_secondary
|
1255
1264
|
msg = "failed to flush the buffer with secondary output."
|
@@ -71,8 +71,11 @@ module Fluent
|
|
71
71
|
end
|
72
72
|
|
73
73
|
def parse(text)
|
74
|
-
|
75
|
-
time
|
74
|
+
record = @load_proc.call(text)
|
75
|
+
time = parse_time(record)
|
76
|
+
if @execute_convert_values
|
77
|
+
time, record = convert_values(time, record)
|
78
|
+
end
|
76
79
|
yield time, record
|
77
80
|
rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
|
78
81
|
yield nil, nil
|
@@ -27,13 +27,9 @@ module Fluent
|
|
27
27
|
cert, key, extra = cert_option_server_validate!(conf)
|
28
28
|
|
29
29
|
ctx = OpenSSL::SSL::SSLContext.new
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
ctx.set_params({})
|
34
|
-
|
35
|
-
ctx.ciphers = ciphers
|
36
|
-
end
|
30
|
+
# inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
|
31
|
+
# https://bugs.ruby-lang.org/issues/9424
|
32
|
+
ctx.set_params({}) unless insecure
|
37
33
|
|
38
34
|
if conf.client_cert_auth
|
39
35
|
ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
|
@@ -56,6 +52,7 @@ module Fluent
|
|
56
52
|
end
|
57
53
|
|
58
54
|
Fluent::TLS.set_version_to_context(ctx, version, conf.min_version, conf.max_version)
|
55
|
+
ctx.ciphers = ciphers unless insecure
|
59
56
|
|
60
57
|
ctx
|
61
58
|
end
|
@@ -185,7 +182,7 @@ module Fluent
|
|
185
182
|
list = []
|
186
183
|
data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
|
187
184
|
if list.length == 0
|
188
|
-
|
185
|
+
raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
|
189
186
|
end
|
190
187
|
list
|
191
188
|
end
|
@@ -81,7 +81,7 @@ module Fluent
|
|
81
81
|
|
82
82
|
def build_handler
|
83
83
|
@methods.group_by(&:first).each do |(path, rest)|
|
84
|
-
klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(Hash[rest.map { |e| [e[1], e[2]] }])
|
84
|
+
klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(**Hash[rest.map { |e| [e[1], e[2]] }])
|
85
85
|
@server.mount(path, klass)
|
86
86
|
end
|
87
87
|
end
|
@@ -76,7 +76,7 @@ module Fluent
|
|
76
76
|
config_param :time_key, :string, default: nil
|
77
77
|
|
78
78
|
# To avoid defining :time_type twice
|
79
|
-
config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
|
79
|
+
config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :unixtime_micros, :unixtime_nanos, :string], default: :float
|
80
80
|
|
81
81
|
Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
|
82
82
|
config_param(name, type, **opts)
|
@@ -132,6 +132,9 @@ module Fluent
|
|
132
132
|
if @_inject_time_key
|
133
133
|
@_inject_time_formatter = case @inject_config.time_type
|
134
134
|
when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
|
135
|
+
when :unixtime_millis then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000 + time.nsec / 1_000_000 : (time * 1_000).floor }
|
136
|
+
when :unixtime_micros then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000 + time.nsec / 1_000 : (time * 1_000_000).floor }
|
137
|
+
when :unixtime_nanos then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000_000 + time.nsec : (time * 1_000_000_000).floor }
|
135
138
|
when :unixtime then ->(time){ time.to_i }
|
136
139
|
else
|
137
140
|
localtime = @inject_config.localtime && !@inject_config.utc
|
@@ -199,7 +199,7 @@ module Fluent
|
|
199
199
|
list = []
|
200
200
|
data.scan(pattern) { |match| list << OpenSSL::X509::Certificate.new(match) }
|
201
201
|
if list.length == 0
|
202
|
-
|
202
|
+
raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
|
203
203
|
end
|
204
204
|
list
|
205
205
|
end
|
data/lib/fluent/supervisor.rb
CHANGED
@@ -53,11 +53,11 @@ module Fluent
|
|
53
53
|
@enable_get_dump = config[:enable_get_dump]
|
54
54
|
run_rpc_server
|
55
55
|
end
|
56
|
-
install_supervisor_signal_handlers
|
57
56
|
|
58
|
-
if
|
59
|
-
@signame = config[:signame]
|
57
|
+
if Fluent.windows?
|
60
58
|
install_windows_event_handler
|
59
|
+
else
|
60
|
+
install_supervisor_signal_handlers
|
61
61
|
end
|
62
62
|
|
63
63
|
if counter = config[:counter_server]
|
@@ -70,6 +70,7 @@ module Fluent
|
|
70
70
|
end
|
71
71
|
|
72
72
|
def after_run
|
73
|
+
stop_windows_event_thread if Fluent.windows?
|
73
74
|
stop_rpc_server if @rpc_endpoint
|
74
75
|
stop_counter_server if @counter
|
75
76
|
Fluent::Supervisor.cleanup_resources
|
@@ -92,7 +93,8 @@ module Fluent
|
|
92
93
|
@rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
|
93
94
|
$log.debug "fluentd RPC got /api/processes.flushBuffersAndKillWorkers request"
|
94
95
|
if Fluent.windows?
|
95
|
-
|
96
|
+
supervisor_sigusr1_handler
|
97
|
+
stop(true)
|
96
98
|
else
|
97
99
|
Process.kill :USR1, $$
|
98
100
|
Process.kill :TERM, $$
|
@@ -101,7 +103,9 @@ module Fluent
|
|
101
103
|
}
|
102
104
|
@rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
|
103
105
|
$log.debug "fluentd RPC got /api/plugins.flushBuffers request"
|
104
|
-
|
106
|
+
if Fluent.windows?
|
107
|
+
supervisor_sigusr1_handler
|
108
|
+
else
|
105
109
|
Process.kill :USR1, $$
|
106
110
|
end
|
107
111
|
nil
|
@@ -125,7 +129,9 @@ module Fluent
|
|
125
129
|
|
126
130
|
@rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
|
127
131
|
$log.debug "fluentd RPC got /api/config.gracefulReload request"
|
128
|
-
|
132
|
+
if Fluent.windows?
|
133
|
+
supervisor_sigusr2_handler
|
134
|
+
else
|
129
135
|
Process.kill :USR2, $$
|
130
136
|
end
|
131
137
|
|
@@ -159,38 +165,101 @@ module Fluent
|
|
159
165
|
end
|
160
166
|
|
161
167
|
def install_supervisor_signal_handlers
|
168
|
+
return if Fluent.windows?
|
169
|
+
|
162
170
|
trap :HUP do
|
163
171
|
$log.debug "fluentd supervisor process get SIGHUP"
|
164
172
|
supervisor_sighup_handler
|
165
|
-
end
|
173
|
+
end
|
166
174
|
|
167
175
|
trap :USR1 do
|
168
176
|
$log.debug "fluentd supervisor process get SIGUSR1"
|
169
177
|
supervisor_sigusr1_handler
|
170
|
-
end
|
178
|
+
end
|
171
179
|
|
172
180
|
trap :USR2 do
|
173
181
|
$log.debug 'fluentd supervisor process got SIGUSR2'
|
174
182
|
supervisor_sigusr2_handler
|
175
|
-
end
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
if Fluent.windows?
|
187
|
+
# Override some methods of ServerEngine::MultiSpawnWorker
|
188
|
+
# Since Fluentd's Supervisor doesn't use ServerEngine's HUP, USR1 and USR2
|
189
|
+
# handlers (see install_supervisor_signal_handlers), they should be
|
190
|
+
# disabled also on Windows, just send commands to workers instead.
|
191
|
+
def restart(graceful)
|
192
|
+
@monitors.each do |m|
|
193
|
+
m.send_command(graceful ? "GRACEFUL_RESTART\n" : "IMMEDIATE_RESTART\n")
|
194
|
+
end
|
195
|
+
end
|
196
|
+
|
197
|
+
def reload
|
198
|
+
@monitors.each do |m|
|
199
|
+
m.send_command("RELOAD\n")
|
200
|
+
end
|
201
|
+
end
|
176
202
|
end
|
177
203
|
|
178
204
|
def install_windows_event_handler
|
205
|
+
return unless Fluent.windows?
|
206
|
+
|
207
|
+
@pid_signame = "fluentd_#{$$}"
|
208
|
+
@signame = config[:signame]
|
209
|
+
|
179
210
|
Thread.new do
|
180
|
-
|
211
|
+
ipc = Win32::Ipc.new(nil)
|
212
|
+
events = [
|
213
|
+
Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
|
214
|
+
Win32::Event.new("#{@pid_signame}"),
|
215
|
+
Win32::Event.new("#{@pid_signame}_HUP"),
|
216
|
+
Win32::Event.new("#{@pid_signame}_USR1"),
|
217
|
+
Win32::Event.new("#{@pid_signame}_USR2"),
|
218
|
+
]
|
219
|
+
if @signame
|
220
|
+
signame_events = [
|
221
|
+
Win32::Event.new("#{@signame}"),
|
222
|
+
Win32::Event.new("#{@signame}_HUP"),
|
223
|
+
Win32::Event.new("#{@signame}_USR1"),
|
224
|
+
Win32::Event.new("#{@signame}_USR2"),
|
225
|
+
]
|
226
|
+
events.concat(signame_events)
|
227
|
+
end
|
181
228
|
begin
|
182
|
-
|
183
|
-
|
184
|
-
|
229
|
+
loop do
|
230
|
+
idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
|
231
|
+
if idx > 0 && idx <= events.length
|
232
|
+
$log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
|
233
|
+
else
|
234
|
+
$log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
|
235
|
+
end
|
236
|
+
case idx
|
237
|
+
when 2, 6
|
238
|
+
stop(true)
|
239
|
+
when 3, 7
|
240
|
+
supervisor_sighup_handler
|
241
|
+
when 4, 8
|
242
|
+
supervisor_sigusr1_handler
|
243
|
+
when 5, 9
|
244
|
+
supervisor_sigusr2_handler
|
245
|
+
when 1
|
246
|
+
break
|
247
|
+
end
|
185
248
|
end
|
186
|
-
kill_worker
|
187
|
-
stop(true)
|
188
249
|
ensure
|
189
|
-
|
250
|
+
events.each { |event| event.close }
|
190
251
|
end
|
191
252
|
end
|
192
253
|
end
|
193
254
|
|
255
|
+
def stop_windows_event_thread
|
256
|
+
if Fluent.windows?
|
257
|
+
ev = Win32::Event.open("#{@pid_signame}_STOP_EVENT_THREAD")
|
258
|
+
ev.set
|
259
|
+
ev.close
|
260
|
+
end
|
261
|
+
end
|
262
|
+
|
194
263
|
def supervisor_sighup_handler
|
195
264
|
kill_worker
|
196
265
|
end
|
@@ -265,9 +334,25 @@ module Fluent
|
|
265
334
|
def send_signal_to_workers(signal)
|
266
335
|
return unless config[:worker_pid]
|
267
336
|
|
268
|
-
|
269
|
-
|
270
|
-
|
337
|
+
if Fluent.windows?
|
338
|
+
send_command_to_workers(signal)
|
339
|
+
else
|
340
|
+
config[:worker_pid].each_value do |pid|
|
341
|
+
# don't rescue Errno::ESRCH here (invalid status)
|
342
|
+
Process.kill(signal, pid)
|
343
|
+
end
|
344
|
+
end
|
345
|
+
end
|
346
|
+
|
347
|
+
def send_command_to_workers(signal)
|
348
|
+
# Use SeverEngine's CommandSender on Windows
|
349
|
+
case signal
|
350
|
+
when :HUP
|
351
|
+
restart(false)
|
352
|
+
when :USR1
|
353
|
+
restart(true)
|
354
|
+
when :USR2
|
355
|
+
reload
|
271
356
|
end
|
272
357
|
end
|
273
358
|
end
|
@@ -302,6 +387,7 @@ module Fluent
|
|
302
387
|
log_level = params['log_level']
|
303
388
|
suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
|
304
389
|
ignore_repeated_log_interval = params['ignore_repeated_log_interval']
|
390
|
+
ignore_same_log_interval = params['ignore_same_log_interval']
|
305
391
|
|
306
392
|
log_path = params['log_path']
|
307
393
|
chuser = params['chuser']
|
@@ -309,7 +395,8 @@ module Fluent
|
|
309
395
|
log_rotate_age = params['log_rotate_age']
|
310
396
|
log_rotate_size = params['log_rotate_size']
|
311
397
|
|
312
|
-
log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval
|
398
|
+
log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
|
399
|
+
ignore_same_log_interval: ignore_same_log_interval}
|
313
400
|
logger_initializer = Supervisor::LoggerInitializer.new(
|
314
401
|
log_path, log_level, chuser, chgroup, log_opts,
|
315
402
|
log_rotate_age: log_rotate_age,
|
@@ -347,6 +434,7 @@ module Fluent
|
|
347
434
|
chumask: 0,
|
348
435
|
suppress_repeated_stacktrace: suppress_repeated_stacktrace,
|
349
436
|
ignore_repeated_log_interval: ignore_repeated_log_interval,
|
437
|
+
ignore_same_log_interval: ignore_same_log_interval,
|
350
438
|
daemonize: daemonize,
|
351
439
|
rpc_endpoint: params['rpc_endpoint'],
|
352
440
|
counter_server: params['counter_server'],
|
@@ -441,10 +529,11 @@ module Fluent
|
|
441
529
|
self
|
442
530
|
end
|
443
531
|
|
444
|
-
def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil)
|
532
|
+
def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
|
445
533
|
$log.format = format if format
|
446
534
|
$log.time_format = time_format if time_format
|
447
535
|
$log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
|
536
|
+
$log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
|
448
537
|
|
449
538
|
if @path && log_dir_perm
|
450
539
|
File.chmod(log_dir_perm || 0755, File.dirname(@path))
|
@@ -511,7 +600,8 @@ module Fluent
|
|
511
600
|
@cl_opt = opt
|
512
601
|
@conf = nil
|
513
602
|
|
514
|
-
log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval]
|
603
|
+
log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
|
604
|
+
ignore_same_log_interval: opt[:ignore_same_log_interval]}
|
515
605
|
@log = LoggerInitializer.new(
|
516
606
|
@log_path, opt[:log_level], @chuser, @chgroup, log_opts,
|
517
607
|
log_rotate_age: @log_rotate_age,
|
@@ -635,7 +725,8 @@ module Fluent
|
|
635
725
|
format: @system_config.log.format,
|
636
726
|
time_format: @system_config.log.time_format,
|
637
727
|
log_dir_perm: @system_config.dir_permission,
|
638
|
-
ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval
|
728
|
+
ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
|
729
|
+
ignore_same_log_interval: @system_config.ignore_same_log_interval
|
639
730
|
)
|
640
731
|
|
641
732
|
$log.info :supervisor, 'parsing config file is succeeded', path: @config_path
|
@@ -740,33 +831,45 @@ module Fluent
|
|
740
831
|
end
|
741
832
|
end
|
742
833
|
|
743
|
-
|
744
|
-
|
745
|
-
|
834
|
+
if Fluent.windows?
|
835
|
+
install_main_process_command_handlers
|
836
|
+
else
|
837
|
+
trap :USR1 do
|
838
|
+
flush_buffer
|
839
|
+
end
|
746
840
|
|
747
|
-
|
748
|
-
|
749
|
-
|
841
|
+
trap :USR2 do
|
842
|
+
reload_config
|
843
|
+
end
|
844
|
+
end
|
845
|
+
end
|
750
846
|
|
751
|
-
|
752
|
-
|
753
|
-
|
754
|
-
|
755
|
-
|
847
|
+
def install_main_process_command_handlers
|
848
|
+
command_pipe = $stdin.dup
|
849
|
+
$stdin.reopen(File::NULL, "rb")
|
850
|
+
command_pipe.binmode
|
851
|
+
command_pipe.sync = true
|
756
852
|
|
757
|
-
|
758
|
-
|
759
|
-
|
760
|
-
|
761
|
-
|
762
|
-
|
763
|
-
|
764
|
-
|
765
|
-
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
|
853
|
+
Thread.new do
|
854
|
+
loop do
|
855
|
+
cmd = command_pipe.gets
|
856
|
+
break unless cmd
|
857
|
+
|
858
|
+
case cmd.chomp!
|
859
|
+
when "GRACEFUL_STOP", "IMMEDIATE_STOP"
|
860
|
+
$log.debug "fluentd main process get #{cmd} command"
|
861
|
+
@finished = true
|
862
|
+
$log.debug "getting start to shutdown main process"
|
863
|
+
Fluent::Engine.stop
|
864
|
+
break
|
865
|
+
when "GRACEFUL_RESTART"
|
866
|
+
$log.debug "fluentd main process get #{cmd} command"
|
867
|
+
flush_buffer
|
868
|
+
when "RELOAD"
|
869
|
+
$log.debug "fluentd main process get #{cmd} command"
|
870
|
+
reload_config
|
871
|
+
else
|
872
|
+
$log.warn "fluentd main process get unknown command [#{cmd}]"
|
770
873
|
end
|
771
874
|
end
|
772
875
|
end
|
@@ -804,7 +907,7 @@ module Fluent
|
|
804
907
|
Fluent::Engine.reload_config(conf)
|
805
908
|
end
|
806
909
|
rescue => e
|
807
|
-
# it is
|
910
|
+
# it is guaranteed that config file is valid by supervisor side. but it's not atomic because of using signals to commnicate between worker and super
|
808
911
|
# So need this rescue code
|
809
912
|
$log.error("failed to reload config: #{e}")
|
810
913
|
next
|