fluentd 1.11.1-x64-mingw32 → 1.12.0-x64-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.md +1 -1
- data/.github/ISSUE_TEMPLATE/config.yml +5 -0
- data/.github/workflows/stale-actions.yml +22 -0
- data/.travis.yml +22 -2
- data/CHANGELOG.md +111 -0
- data/README.md +1 -1
- data/appveyor.yml +3 -0
- data/bin/fluent-cap-ctl +7 -0
- data/bin/fluent-ctl +7 -0
- data/example/copy_roundrobin.conf +3 -3
- data/example/counter.conf +1 -1
- data/example/filter_stdout.conf +2 -2
- data/example/{in_dummy_blocks.conf → in_sample_blocks.conf} +4 -4
- data/example/{in_dummy_with_compression.conf → in_sample_with_compression.conf} +3 -3
- data/example/logevents.conf +5 -5
- data/example/multi_filters.conf +1 -1
- data/example/out_exec_filter.conf +2 -2
- data/example/out_forward.conf +1 -1
- data/example/out_forward_buf_file.conf +1 -1
- data/example/out_forward_client.conf +5 -5
- data/example/out_forward_heartbeat_none.conf +1 -1
- data/example/out_forward_sd.conf +1 -1
- data/example/out_forward_shared_key.conf +2 -2
- data/example/out_forward_tls.conf +1 -1
- data/example/out_forward_users.conf +3 -3
- data/example/out_null.conf +4 -4
- data/example/secondary_file.conf +1 -1
- data/fluentd.gemspec +7 -6
- data/lib/fluent/capability.rb +87 -0
- data/lib/fluent/command/cap_ctl.rb +174 -0
- data/lib/fluent/command/ctl.rb +177 -0
- data/lib/fluent/command/plugin_config_formatter.rb +2 -1
- data/lib/fluent/env.rb +4 -0
- data/lib/fluent/log.rb +33 -3
- data/lib/fluent/match.rb +9 -0
- data/lib/fluent/plugin.rb +5 -0
- data/lib/fluent/plugin/buffer.rb +32 -42
- data/lib/fluent/plugin/buffer/chunk.rb +2 -1
- data/lib/fluent/plugin/formatter.rb +24 -0
- data/lib/fluent/plugin/formatter_csv.rb +1 -1
- data/lib/fluent/plugin/formatter_hash.rb +3 -1
- data/lib/fluent/plugin/formatter_json.rb +3 -1
- data/lib/fluent/plugin/formatter_ltsv.rb +5 -3
- data/lib/fluent/plugin/formatter_out_file.rb +6 -4
- data/lib/fluent/plugin/formatter_single_value.rb +4 -2
- data/lib/fluent/plugin/formatter_tsv.rb +4 -2
- data/lib/fluent/plugin/in_dummy.rb +2 -123
- data/lib/fluent/plugin/in_exec.rb +4 -2
- data/lib/fluent/plugin/in_http.rb +25 -4
- data/lib/fluent/plugin/in_sample.rb +141 -0
- data/lib/fluent/plugin/in_tail.rb +109 -41
- data/lib/fluent/plugin/in_tail/position_file.rb +39 -14
- data/lib/fluent/plugin/in_tcp.rb +1 -0
- data/lib/fluent/plugin/out_http.rb +20 -2
- data/lib/fluent/plugin/output.rb +15 -6
- data/lib/fluent/plugin/parser_json.rb +5 -2
- data/lib/fluent/plugin_helper/cert_option.rb +5 -8
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +1 -1
- data/lib/fluent/plugin_helper/inject.rb +4 -1
- data/lib/fluent/plugin_helper/retry_state.rb +4 -0
- data/lib/fluent/plugin_helper/socket.rb +1 -1
- data/lib/fluent/supervisor.rb +151 -48
- data/lib/fluent/system_config.rb +2 -1
- data/lib/fluent/time.rb +1 -0
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +22 -4
- data/test/command/test_binlog_reader.rb +22 -6
- data/test/command/test_cap_ctl.rb +100 -0
- data/test/command/test_ctl.rb +57 -0
- data/test/command/test_plugin_config_formatter.rb +57 -2
- data/test/plugin/in_tail/test_position_file.rb +45 -25
- data/test/plugin/test_buffer.rb +4 -0
- data/test/plugin/test_filter_stdout.rb +6 -1
- data/test/plugin/test_formatter_hash.rb +6 -3
- data/test/plugin/test_formatter_json.rb +14 -4
- data/test/plugin/test_formatter_ltsv.rb +13 -5
- data/test/plugin/test_formatter_out_file.rb +35 -14
- data/test/plugin/test_formatter_single_value.rb +12 -6
- data/test/plugin/test_formatter_tsv.rb +12 -4
- data/test/plugin/test_in_exec.rb +18 -0
- data/test/plugin/test_in_http.rb +25 -0
- data/test/plugin/{test_in_dummy.rb → test_in_sample.rb} +25 -25
- data/test/plugin/test_in_tail.rb +433 -30
- data/test/plugin/test_out_file.rb +23 -18
- data/test/plugin/test_output.rb +12 -0
- data/test/plugin/test_parser_syslog.rb +2 -2
- data/test/plugin_helper/data/cert/empty.pem +0 -0
- data/test/plugin_helper/test_cert_option.rb +7 -0
- data/test/plugin_helper/test_compat_parameters.rb +7 -2
- data/test/plugin_helper/test_http_server_helper.rb +5 -0
- data/test/plugin_helper/test_inject.rb +42 -0
- data/test/plugin_helper/test_server.rb +34 -0
- data/test/plugin_helper/test_socket.rb +8 -0
- data/test/test_capability.rb +74 -0
- data/test/test_formatter.rb +34 -10
- data/test/test_log.rb +44 -0
- data/test/test_match.rb +11 -0
- data/test/test_output.rb +6 -1
- data/test/test_static_config_analysis.rb +2 -2
- data/test/test_supervisor.rb +119 -1
- metadata +50 -18
@@ -14,126 +14,5 @@
|
|
14
14
|
# limitations under the License.
|
15
15
|
#
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
require 'fluent/plugin/input'
|
20
|
-
require 'fluent/config/error'
|
21
|
-
|
22
|
-
module Fluent::Plugin
|
23
|
-
class DummyInput < Input
|
24
|
-
Fluent::Plugin.register_input('dummy', self)
|
25
|
-
|
26
|
-
helpers :thread, :storage
|
27
|
-
|
28
|
-
BIN_NUM = 10
|
29
|
-
DEFAULT_STORAGE_TYPE = 'local'
|
30
|
-
|
31
|
-
desc "The value is the tag assigned to the generated events."
|
32
|
-
config_param :tag, :string
|
33
|
-
desc "The number of events in event stream of each emits."
|
34
|
-
config_param :size, :integer, default: 1
|
35
|
-
desc "It configures how many events to generate per second."
|
36
|
-
config_param :rate, :integer, default: 1
|
37
|
-
desc "If specified, each generated event has an auto-incremented key field."
|
38
|
-
config_param :auto_increment_key, :string, default: nil
|
39
|
-
desc "The boolean to suspend-and-resume incremental value after restart"
|
40
|
-
config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
|
41
|
-
desc "The dummy data to be generated. An array of JSON hashes or a single JSON hash."
|
42
|
-
config_param :dummy, default: [{"message"=>"dummy"}] do |val|
|
43
|
-
begin
|
44
|
-
parsed = JSON.parse(val)
|
45
|
-
rescue JSON::ParserError => ex
|
46
|
-
# Fluent::ConfigParseError, "got incomplete JSON" will be raised
|
47
|
-
# at literal_parser.rb with --use-v1-config, but I had to
|
48
|
-
# take care at here for the case of --use-v0-config.
|
49
|
-
raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
|
50
|
-
end
|
51
|
-
dummy = parsed.is_a?(Array) ? parsed : [parsed]
|
52
|
-
dummy.each_with_index do |e, i|
|
53
|
-
raise Fluent::ConfigError, "#{i}th element of dummy, #{e}, is not a hash" unless e.is_a?(Hash)
|
54
|
-
end
|
55
|
-
dummy
|
56
|
-
end
|
57
|
-
|
58
|
-
def initialize
|
59
|
-
super
|
60
|
-
@storage = nil
|
61
|
-
end
|
62
|
-
|
63
|
-
def configure(conf)
|
64
|
-
super
|
65
|
-
@dummy_index = 0
|
66
|
-
config = conf.elements.select{|e| e.name == 'storage' }.first
|
67
|
-
@storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
|
68
|
-
end
|
69
|
-
|
70
|
-
def multi_workers_ready?
|
71
|
-
true
|
72
|
-
end
|
73
|
-
|
74
|
-
def start
|
75
|
-
super
|
76
|
-
|
77
|
-
@storage.put(:increment_value, 0) unless @storage.get(:increment_value)
|
78
|
-
@storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
|
79
|
-
|
80
|
-
if @auto_increment_key && !@storage.get(:auto_increment_value)
|
81
|
-
@storage.put(:auto_increment_value, -1)
|
82
|
-
end
|
83
|
-
|
84
|
-
thread_create(:dummy_input, &method(:run))
|
85
|
-
end
|
86
|
-
|
87
|
-
def run
|
88
|
-
batch_num = (@rate / BIN_NUM).to_i
|
89
|
-
residual_num = (@rate % BIN_NUM)
|
90
|
-
while thread_current_running?
|
91
|
-
current_time = Time.now.to_i
|
92
|
-
BIN_NUM.times do
|
93
|
-
break unless (thread_current_running? && Time.now.to_i <= current_time)
|
94
|
-
wait(0.1) { emit(batch_num) }
|
95
|
-
end
|
96
|
-
emit(residual_num) if thread_current_running?
|
97
|
-
# wait for next second
|
98
|
-
while thread_current_running? && Time.now.to_i <= current_time
|
99
|
-
sleep 0.01
|
100
|
-
end
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
def emit(num)
|
105
|
-
begin
|
106
|
-
if @size > 1
|
107
|
-
num.times do
|
108
|
-
router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
|
109
|
-
end
|
110
|
-
else
|
111
|
-
num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
|
112
|
-
end
|
113
|
-
rescue => _
|
114
|
-
# ignore all errors not to stop emits by emit errors
|
115
|
-
end
|
116
|
-
end
|
117
|
-
|
118
|
-
def generate
|
119
|
-
d = @dummy[@dummy_index]
|
120
|
-
unless d
|
121
|
-
@dummy_index = 0
|
122
|
-
d = @dummy[@dummy_index]
|
123
|
-
end
|
124
|
-
@dummy_index += 1
|
125
|
-
if @auto_increment_key
|
126
|
-
d = d.dup
|
127
|
-
d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
|
128
|
-
end
|
129
|
-
d
|
130
|
-
end
|
131
|
-
|
132
|
-
def wait(time)
|
133
|
-
start_time = Time.now
|
134
|
-
yield
|
135
|
-
sleep_time = time - (Time.now - start_time)
|
136
|
-
sleep sleep_time if sleep_time > 0
|
137
|
-
end
|
138
|
-
end
|
139
|
-
end
|
17
|
+
# Remove this file in fluentd v2
|
18
|
+
require_relative 'in_sample'
|
@@ -25,6 +25,8 @@ module Fluent::Plugin
|
|
25
25
|
|
26
26
|
desc 'The command (program) to execute.'
|
27
27
|
config_param :command, :string
|
28
|
+
desc 'Specify connect mode to executed process'
|
29
|
+
config_param :connect_mode, :enum, list: [:read, :read_with_stderr], default: :read
|
28
30
|
|
29
31
|
config_section :parse do
|
30
32
|
config_set_default :@type, 'tsv'
|
@@ -72,9 +74,9 @@ module Fluent::Plugin
|
|
72
74
|
super
|
73
75
|
|
74
76
|
if @run_interval
|
75
|
-
child_process_execute(:exec_input, @command, interval: @run_interval, mode: [
|
77
|
+
child_process_execute(:exec_input, @command, interval: @run_interval, mode: [@connect_mode], &method(:run))
|
76
78
|
else
|
77
|
-
child_process_execute(:exec_input, @command, immediate: true, mode: [
|
79
|
+
child_process_execute(:exec_input, @command, immediate: true, mode: [@connect_mode], &method(:run))
|
78
80
|
end
|
79
81
|
end
|
80
82
|
|
@@ -72,7 +72,7 @@ module Fluent::Plugin
|
|
72
72
|
desc 'Add REMOTE_ADDR header to the record.'
|
73
73
|
config_param :add_remote_addr, :bool, default: false
|
74
74
|
config_param :blocking_timeout, :time, default: 0.5
|
75
|
-
desc 'Set a
|
75
|
+
desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'
|
76
76
|
config_param :cors_allow_origins, :array, default: nil
|
77
77
|
desc 'Respond with empty gif image of 1x1 pixel.'
|
78
78
|
config_param :respond_with_empty_img, :bool, default: false
|
@@ -80,6 +80,8 @@ module Fluent::Plugin
|
|
80
80
|
config_param :use_204_response, :bool, default: false
|
81
81
|
desc 'Dump error log or not'
|
82
82
|
config_param :dump_error_log, :bool, default: true
|
83
|
+
desc 'Add QUERY_ prefix query params to record'
|
84
|
+
config_param :add_query_params, :bool, default: false
|
83
85
|
|
84
86
|
config_section :parse do
|
85
87
|
config_set_default :@type, 'in_http'
|
@@ -277,7 +279,7 @@ module Fluent::Plugin
|
|
277
279
|
private
|
278
280
|
|
279
281
|
def on_server_connect(conn)
|
280
|
-
handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins)
|
282
|
+
handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins, @add_query_params)
|
281
283
|
|
282
284
|
conn.on(:data) do |data|
|
283
285
|
handler.on_read(data)
|
@@ -326,6 +328,14 @@ module Fluent::Plugin
|
|
326
328
|
}
|
327
329
|
end
|
328
330
|
|
331
|
+
if @add_query_params
|
332
|
+
params.each_pair { |k, v|
|
333
|
+
if k.start_with?("QUERY_".freeze)
|
334
|
+
record[k] = v
|
335
|
+
end
|
336
|
+
}
|
337
|
+
end
|
338
|
+
|
329
339
|
if @add_remote_addr
|
330
340
|
record['REMOTE_ADDR'] = params['REMOTE_ADDR']
|
331
341
|
end
|
@@ -346,7 +356,7 @@ module Fluent::Plugin
|
|
346
356
|
class Handler
|
347
357
|
attr_reader :content_type
|
348
358
|
|
349
|
-
def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins)
|
359
|
+
def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins, add_query_params)
|
350
360
|
@io = io
|
351
361
|
@km = km
|
352
362
|
@callback = callback
|
@@ -356,6 +366,7 @@ module Fluent::Plugin
|
|
356
366
|
@log = log
|
357
367
|
@cors_allow_origins = cors_allow_origins
|
358
368
|
@idle = 0
|
369
|
+
@add_query_params = add_query_params
|
359
370
|
@km.add(self)
|
360
371
|
|
361
372
|
@remote_port, @remote_addr = io.remote_port, io.remote_addr
|
@@ -490,7 +501,7 @@ module Fluent::Plugin
|
|
490
501
|
# CORS check
|
491
502
|
# ==========
|
492
503
|
# For every incoming request, we check if we have some CORS
|
493
|
-
# restrictions and
|
504
|
+
# restrictions and allow listed origins through @cors_allow_origins.
|
494
505
|
unless @cors_allow_origins.nil?
|
495
506
|
unless @cors_allow_origins.include?('*') or include_cors_allow_origin
|
496
507
|
send_response_and_close(RES_403_STATUS, {'Connection' => 'close'}, "")
|
@@ -533,7 +544,17 @@ module Fluent::Plugin
|
|
533
544
|
end
|
534
545
|
path_info = uri.path
|
535
546
|
|
547
|
+
if (@add_query_params)
|
548
|
+
|
549
|
+
query_params = WEBrick::HTTPUtils.parse_query(uri.query)
|
550
|
+
|
551
|
+
query_params.each_pair {|k,v|
|
552
|
+
params["QUERY_#{k.gsub('-','_').upcase}"] = v
|
553
|
+
}
|
554
|
+
end
|
555
|
+
|
536
556
|
params.merge!(@env)
|
557
|
+
|
537
558
|
@env.clear
|
538
559
|
|
539
560
|
code, header, body = @callback.call(path_info, params)
|
@@ -0,0 +1,141 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'json'
|
18
|
+
|
19
|
+
require 'fluent/plugin/input'
|
20
|
+
require 'fluent/config/error'
|
21
|
+
|
22
|
+
module Fluent::Plugin
|
23
|
+
class SampleInput < Input
|
24
|
+
Fluent::Plugin.register_input('sample', self)
|
25
|
+
Fluent::Plugin.register_input('dummy', self)
|
26
|
+
|
27
|
+
helpers :thread, :storage
|
28
|
+
|
29
|
+
BIN_NUM = 10
|
30
|
+
DEFAULT_STORAGE_TYPE = 'local'
|
31
|
+
|
32
|
+
desc "The value is the tag assigned to the generated events."
|
33
|
+
config_param :tag, :string
|
34
|
+
desc "The number of events in event stream of each emits."
|
35
|
+
config_param :size, :integer, default: 1
|
36
|
+
desc "It configures how many events to generate per second."
|
37
|
+
config_param :rate, :integer, default: 1
|
38
|
+
desc "If specified, each generated event has an auto-incremented key field."
|
39
|
+
config_param :auto_increment_key, :string, default: nil
|
40
|
+
desc "The boolean to suspend-and-resume incremental value after restart"
|
41
|
+
config_param :suspend, :bool, default: false,deprecated: 'This parameters is ignored'
|
42
|
+
desc "The sample data to be generated. An array of JSON hashes or a single JSON hash."
|
43
|
+
config_param :sample, alias: :dummy, default: [{"message" => "sample"}] do |val|
|
44
|
+
begin
|
45
|
+
parsed = JSON.parse(val)
|
46
|
+
rescue JSON::ParserError => ex
|
47
|
+
# Fluent::ConfigParseError, "got incomplete JSON" will be raised
|
48
|
+
# at literal_parser.rb with --use-v1-config, but I had to
|
49
|
+
# take care at here for the case of --use-v0-config.
|
50
|
+
raise Fluent::ConfigError, "#{ex.class}: #{ex.message}"
|
51
|
+
end
|
52
|
+
sample = parsed.is_a?(Array) ? parsed : [parsed]
|
53
|
+
sample.each_with_index do |e, i|
|
54
|
+
raise Fluent::ConfigError, "#{i}th element of sample, #{e}, is not a hash" unless e.is_a?(Hash)
|
55
|
+
end
|
56
|
+
sample
|
57
|
+
end
|
58
|
+
|
59
|
+
def initialize
|
60
|
+
super
|
61
|
+
@storage = nil
|
62
|
+
end
|
63
|
+
|
64
|
+
def configure(conf)
|
65
|
+
super
|
66
|
+
@sample_index = 0
|
67
|
+
config = conf.elements.select{|e| e.name == 'storage' }.first
|
68
|
+
@storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
|
69
|
+
end
|
70
|
+
|
71
|
+
def multi_workers_ready?
|
72
|
+
true
|
73
|
+
end
|
74
|
+
|
75
|
+
def start
|
76
|
+
super
|
77
|
+
|
78
|
+
@storage.put(:increment_value, 0) unless @storage.get(:increment_value)
|
79
|
+
# keep 'dummy' to avoid breaking changes for existing environment. Change it in fluentd v2
|
80
|
+
@storage.put(:dummy_index, 0) unless @storage.get(:dummy_index)
|
81
|
+
|
82
|
+
if @auto_increment_key && !@storage.get(:auto_increment_value)
|
83
|
+
@storage.put(:auto_increment_value, -1)
|
84
|
+
end
|
85
|
+
|
86
|
+
thread_create(:sample_input, &method(:run))
|
87
|
+
end
|
88
|
+
|
89
|
+
def run
|
90
|
+
batch_num = (@rate / BIN_NUM).to_i
|
91
|
+
residual_num = (@rate % BIN_NUM)
|
92
|
+
while thread_current_running?
|
93
|
+
current_time = Time.now.to_i
|
94
|
+
BIN_NUM.times do
|
95
|
+
break unless (thread_current_running? && Time.now.to_i <= current_time)
|
96
|
+
wait(0.1) { emit(batch_num) }
|
97
|
+
end
|
98
|
+
emit(residual_num) if thread_current_running?
|
99
|
+
# wait for next second
|
100
|
+
while thread_current_running? && Time.now.to_i <= current_time
|
101
|
+
sleep 0.01
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
def emit(num)
|
107
|
+
begin
|
108
|
+
if @size > 1
|
109
|
+
num.times do
|
110
|
+
router.emit_array(@tag, Array.new(@size) { [Fluent::EventTime.now, generate] })
|
111
|
+
end
|
112
|
+
else
|
113
|
+
num.times { router.emit(@tag, Fluent::EventTime.now, generate) }
|
114
|
+
end
|
115
|
+
rescue => _
|
116
|
+
# ignore all errors not to stop emits by emit errors
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def generate
|
121
|
+
d = @sample[@sample_index]
|
122
|
+
unless d
|
123
|
+
@sample_index = 0
|
124
|
+
d = @sample[@sample_index]
|
125
|
+
end
|
126
|
+
@sample_index += 1
|
127
|
+
if @auto_increment_key
|
128
|
+
d = d.dup
|
129
|
+
d[@auto_increment_key] = @storage.update(:auto_increment_value){|v| v + 1 }
|
130
|
+
end
|
131
|
+
d
|
132
|
+
end
|
133
|
+
|
134
|
+
def wait(time)
|
135
|
+
start_time = Time.now
|
136
|
+
yield
|
137
|
+
sleep_time = time - (Time.now - start_time)
|
138
|
+
sleep sleep_time if sleep_time > 0
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
@@ -22,6 +22,7 @@ require 'fluent/event'
|
|
22
22
|
require 'fluent/plugin/buffer'
|
23
23
|
require 'fluent/plugin/parser_multiline'
|
24
24
|
require 'fluent/variable_store'
|
25
|
+
require 'fluent/capability'
|
25
26
|
require 'fluent/plugin/in_tail/position_file'
|
26
27
|
|
27
28
|
if Fluent.windows?
|
@@ -104,6 +105,8 @@ module Fluent::Plugin
|
|
104
105
|
config_param :ignore_repeated_permission_error, :bool, default: false
|
105
106
|
desc 'Format path with the specified timezone'
|
106
107
|
config_param :path_timezone, :string, default: nil
|
108
|
+
desc 'Follow inodes instead of following file names. Guarantees more stable delivery and allows to use * in path pattern with rotating files'
|
109
|
+
config_param :follow_inodes, :bool, default: false
|
107
110
|
|
108
111
|
config_section :parse, required: false, multi: true, init: true, param_name: :parser_configs do
|
109
112
|
config_argument :usage, :string, default: 'in_tail_parser'
|
@@ -154,6 +157,9 @@ module Fluent::Plugin
|
|
154
157
|
end
|
155
158
|
@variable_store[@pos_file] = self.plugin_id
|
156
159
|
else
|
160
|
+
if @follow_inodes
|
161
|
+
raise Fluent::ConfigError, "Can't follow inodes without pos_file configuration parameter"
|
162
|
+
end
|
157
163
|
$log.warn "'pos_file PATH' parameter is not set to a 'tail' source."
|
158
164
|
$log.warn "this parameter is highly recommended to save the position to resume tailing."
|
159
165
|
end
|
@@ -171,6 +177,7 @@ module Fluent::Plugin
|
|
171
177
|
@dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
|
172
178
|
# parser is already created by parser helper
|
173
179
|
@parser = parser_create(usage: parser_config['usage'] || @parser_configs.first.usage)
|
180
|
+
@capability = Fluent::Capability.new(:current_process)
|
174
181
|
end
|
175
182
|
|
176
183
|
def configure_tag
|
@@ -214,7 +221,7 @@ module Fluent::Plugin
|
|
214
221
|
FileUtils.mkdir_p(pos_file_dir, mode: @dir_perm) unless Dir.exist?(pos_file_dir)
|
215
222
|
@pf_file = File.open(@pos_file, File::RDWR|File::CREAT|File::BINARY, @file_perm)
|
216
223
|
@pf_file.sync = true
|
217
|
-
@pf = PositionFile.load(@pf_file, logger: log)
|
224
|
+
@pf = PositionFile.load(@pf_file, @follow_inodes, expand_paths, logger: log)
|
218
225
|
|
219
226
|
if @pos_file_compaction_interval
|
220
227
|
timer_execute(:in_tail_refresh_compact_pos_file, @pos_file_compaction_interval) do
|
@@ -238,7 +245,7 @@ module Fluent::Plugin
|
|
238
245
|
|
239
246
|
def shutdown
|
240
247
|
# during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
|
241
|
-
stop_watchers(
|
248
|
+
stop_watchers(existence_path, immediate: true, remove_watcher: false)
|
242
249
|
@pf_file.close if @pf_file
|
243
250
|
|
244
251
|
super
|
@@ -250,6 +257,11 @@ module Fluent::Plugin
|
|
250
257
|
close_watcher_handles
|
251
258
|
end
|
252
259
|
|
260
|
+
def have_read_capability?
|
261
|
+
@capability.have_capability?(:effective, :dac_read_search) ||
|
262
|
+
@capability.have_capability?(:effective, :dac_override)
|
263
|
+
end
|
264
|
+
|
253
265
|
def expand_paths
|
254
266
|
date = Fluent::EventTime.now
|
255
267
|
paths = []
|
@@ -263,7 +275,7 @@ module Fluent::Plugin
|
|
263
275
|
paths += Dir.glob(path).select { |p|
|
264
276
|
begin
|
265
277
|
is_file = !File.directory?(p)
|
266
|
-
if File.readable?(p) && is_file
|
278
|
+
if (File.readable?(p) || have_read_capability?) && is_file
|
267
279
|
if @limit_recently_modified && File.mtime(p) < (date.to_time - @limit_recently_modified)
|
268
280
|
false
|
269
281
|
else
|
@@ -296,7 +308,31 @@ module Fluent::Plugin
|
|
296
308
|
end
|
297
309
|
path.include?('*') ? Dir.glob(path) : path
|
298
310
|
}.flatten.uniq
|
299
|
-
|
311
|
+
# filter out non existing files, so in case pattern is without '*' we don't do unnecessary work
|
312
|
+
hash = {}
|
313
|
+
(paths - excluded).select { |path|
|
314
|
+
FileTest.exist?(path)
|
315
|
+
}.each { |path|
|
316
|
+
target_info = TargetInfo.new(path, Fluent::FileWrapper.stat(path).ino)
|
317
|
+
if @follow_inodes
|
318
|
+
hash[target_info.ino] = target_info
|
319
|
+
else
|
320
|
+
hash[target_info.path] = target_info
|
321
|
+
end
|
322
|
+
}
|
323
|
+
hash
|
324
|
+
end
|
325
|
+
|
326
|
+
def existence_path
|
327
|
+
hash = {}
|
328
|
+
@tails.each_key {|target_info|
|
329
|
+
if @follow_inodes
|
330
|
+
hash[target_info.ino] = target_info
|
331
|
+
else
|
332
|
+
hash[target_info.path] = target_info
|
333
|
+
end
|
334
|
+
}
|
335
|
+
hash
|
300
336
|
end
|
301
337
|
|
302
338
|
# in_tail with '*' path doesn't check rotation file equality at refresh phase.
|
@@ -305,21 +341,21 @@ module Fluent::Plugin
|
|
305
341
|
# In such case, you should separate log directory and specify two paths in path parameter.
|
306
342
|
# e.g. path /path/to/dir/*,/path/to/rotated_logs/target_file
|
307
343
|
def refresh_watchers
|
308
|
-
|
309
|
-
|
344
|
+
target_paths_hash = expand_paths
|
345
|
+
existence_paths_hash = existence_path
|
310
346
|
|
311
347
|
log.debug { "tailing paths: target = #{target_paths.join(",")} | existing = #{existence_paths.join(",")}" }
|
312
348
|
|
313
|
-
|
314
|
-
|
349
|
+
unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
|
350
|
+
added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
|
315
351
|
|
316
|
-
stop_watchers(
|
317
|
-
start_watchers(
|
352
|
+
stop_watchers(unwatched_hash, immediate: false, unwatched: true) unless unwatched_hash.empty?
|
353
|
+
start_watchers(added_hash) unless added_hash.empty?
|
318
354
|
end
|
319
355
|
|
320
|
-
def setup_watcher(
|
356
|
+
def setup_watcher(target_info, pe)
|
321
357
|
line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
|
322
|
-
tw = TailWatcher.new(
|
358
|
+
tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler))
|
323
359
|
|
324
360
|
if @enable_watch_timer
|
325
361
|
tt = TimerTrigger.new(1, log) { tw.on_notify }
|
@@ -350,47 +386,52 @@ module Fluent::Plugin
|
|
350
386
|
raise e
|
351
387
|
end
|
352
388
|
|
353
|
-
def start_watchers(
|
354
|
-
|
389
|
+
def start_watchers(targets_info)
|
390
|
+
targets_info.each_value { |target_info|
|
355
391
|
pe = nil
|
356
392
|
if @pf
|
357
|
-
pe = @pf[
|
393
|
+
pe = @pf[target_info]
|
358
394
|
if @read_from_head && pe.read_inode.zero?
|
359
395
|
begin
|
360
|
-
pe.update(Fluent::FileWrapper.stat(path).ino, 0)
|
396
|
+
pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
|
361
397
|
rescue Errno::ENOENT
|
362
|
-
$log.warn "#{path} not found. Continuing without tailing it."
|
398
|
+
$log.warn "#{target_info.path} not found. Continuing without tailing it."
|
363
399
|
end
|
364
400
|
end
|
365
401
|
end
|
366
402
|
|
367
403
|
begin
|
368
|
-
tw = setup_watcher(
|
404
|
+
tw = setup_watcher(target_info, pe)
|
369
405
|
rescue WatcherSetupError => e
|
370
|
-
log.warn "Skip #{path} because unexpected setup error happens: #{e}"
|
406
|
+
log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
|
371
407
|
next
|
372
408
|
end
|
373
|
-
|
409
|
+
target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
|
410
|
+
@tails[target_info] = tw
|
374
411
|
}
|
375
412
|
end
|
376
413
|
|
377
|
-
def stop_watchers(
|
378
|
-
|
379
|
-
|
414
|
+
def stop_watchers(targets_info, immediate: false, unwatched: false, remove_watcher: true)
|
415
|
+
targets_info.each_value { |target_info|
|
416
|
+
if remove_watcher
|
417
|
+
tw = @tails.delete(target_info)
|
418
|
+
else
|
419
|
+
tw = @tails[target_info]
|
420
|
+
end
|
380
421
|
if tw
|
381
422
|
tw.unwatched = unwatched
|
382
423
|
if immediate
|
383
|
-
detach_watcher(tw, false)
|
424
|
+
detach_watcher(tw, target_info.ino, false)
|
384
425
|
else
|
385
|
-
detach_watcher_after_rotate_wait(tw)
|
426
|
+
detach_watcher_after_rotate_wait(tw, target_info.ino)
|
386
427
|
end
|
387
428
|
end
|
388
429
|
}
|
389
430
|
end
|
390
431
|
|
391
432
|
def close_watcher_handles
|
392
|
-
@tails.keys.each do |
|
393
|
-
tw = @tails.delete(
|
433
|
+
@tails.keys.each do |target_info|
|
434
|
+
tw = @tails.delete(target_info)
|
394
435
|
if tw
|
395
436
|
tw.close
|
396
437
|
end
|
@@ -398,25 +439,39 @@ module Fluent::Plugin
|
|
398
439
|
end
|
399
440
|
|
400
441
|
# refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
|
401
|
-
def update_watcher(
|
402
|
-
log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
|
442
|
+
def update_watcher(target_info, pe)
|
443
|
+
log.info("detected rotation of #{target_info.path}; waiting #{@rotate_wait} seconds")
|
403
444
|
|
404
445
|
if @pf
|
405
|
-
|
446
|
+
pe_inode = pe.read_inode
|
447
|
+
target_info_from_position_entry = TargetInfo.new(target_info.path, pe_inode)
|
448
|
+
unless pe_inode == @pf[target_info_from_position_entry].read_inode
|
406
449
|
log.debug "Skip update_watcher because watcher has been already updated by other inotify event"
|
407
450
|
return
|
408
451
|
end
|
409
452
|
end
|
410
|
-
|
411
|
-
|
412
|
-
|
453
|
+
|
454
|
+
rotated_target_info = TargetInfo.new(target_info.path, pe.read_inode)
|
455
|
+
rotated_tw = @tails[rotated_target_info]
|
456
|
+
new_target_info = target_info.dup
|
457
|
+
|
458
|
+
if @follow_inodes
|
459
|
+
new_position_entry = @pf[target_info]
|
460
|
+
|
461
|
+
if new_position_entry.read_inode == 0
|
462
|
+
@tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
|
463
|
+
end
|
464
|
+
else
|
465
|
+
@tails[new_target_info] = setup_watcher(new_target_info, pe)
|
466
|
+
end
|
467
|
+
detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
|
413
468
|
end
|
414
469
|
|
415
470
|
# TailWatcher#close is called by another thread at shutdown phase.
|
416
471
|
# It causes 'can't modify string; temporarily locked' error in IOHandler
|
417
472
|
# so adding close_io argument to avoid this problem.
|
418
473
|
# At shutdown, IOHandler's io will be released automatically after detached the event loop
|
419
|
-
def detach_watcher(tw, close_io = true)
|
474
|
+
def detach_watcher(tw, ino, close_io = true)
|
420
475
|
tw.watchers.each do |watcher|
|
421
476
|
event_loop_detach(watcher)
|
422
477
|
end
|
@@ -425,15 +480,16 @@ module Fluent::Plugin
|
|
425
480
|
tw.close if close_io
|
426
481
|
|
427
482
|
if tw.unwatched && @pf
|
428
|
-
|
483
|
+
target_info = TargetInfo.new(tw.path, ino)
|
484
|
+
@pf.unwatch(target_info)
|
429
485
|
end
|
430
486
|
end
|
431
487
|
|
432
|
-
def detach_watcher_after_rotate_wait(tw)
|
488
|
+
def detach_watcher_after_rotate_wait(tw, ino)
|
433
489
|
# Call event_loop_attach/event_loop_detach is high-cost for short-live object.
|
434
490
|
# If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
|
435
491
|
timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
|
436
|
-
detach_watcher(tw)
|
492
|
+
detach_watcher(tw, ino)
|
437
493
|
end
|
438
494
|
end
|
439
495
|
|
@@ -601,10 +657,12 @@ module Fluent::Plugin
|
|
601
657
|
end
|
602
658
|
|
603
659
|
class TailWatcher
|
604
|
-
def initialize(
|
605
|
-
@path = path
|
660
|
+
def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build)
|
661
|
+
@path = target_info.path
|
662
|
+
@ino = target_info.ino
|
606
663
|
@pe = pe || MemoryPositionEntry.new
|
607
664
|
@read_from_head = read_from_head
|
665
|
+
@follow_inodes = follow_inodes
|
608
666
|
@update_watcher = update_watcher
|
609
667
|
@log = log
|
610
668
|
@rotate_handler = RotateHandler.new(log, &method(:on_rotate))
|
@@ -614,7 +672,7 @@ module Fluent::Plugin
|
|
614
672
|
@watchers = []
|
615
673
|
end
|
616
674
|
|
617
|
-
attr_reader :path
|
675
|
+
attr_reader :path, :ino
|
618
676
|
attr_reader :pe
|
619
677
|
attr_reader :line_buffer_timer_flusher
|
620
678
|
attr_accessor :unwatched # This is used for removing position entry from PositionFile
|
@@ -709,7 +767,17 @@ module Fluent::Plugin
|
|
709
767
|
end
|
710
768
|
|
711
769
|
if watcher_needs_update
|
712
|
-
|
770
|
+
# No need to update a watcher if stat is nil (file not present), because moving to inodes will create
|
771
|
+
# new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
|
772
|
+
if stat
|
773
|
+
target_info = TargetInfo.new(@path, stat.ino)
|
774
|
+
if @follow_inodes
|
775
|
+
# don't want to swap state because we need latest read offset in pos file even after rotate_wait
|
776
|
+
@update_watcher.call(target_info, @pe)
|
777
|
+
else
|
778
|
+
@update_watcher.call(target_info, swap_state(@pe))
|
779
|
+
end
|
780
|
+
end
|
713
781
|
else
|
714
782
|
@log.info "detected rotation of #{@path}"
|
715
783
|
@io_handler = io_handler
|