fluentd 1.15.3-x86-mingw32 → 1.16.2-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
- data/.github/workflows/linux-test.yaml +2 -2
- data/.github/workflows/macos-test.yaml +2 -2
- data/.github/workflows/stale-actions.yml +24 -0
- data/.github/workflows/windows-test.yaml +2 -2
- data/CHANGELOG.md +151 -0
- data/CONTRIBUTING.md +1 -1
- data/MAINTAINERS.md +5 -3
- data/README.md +0 -1
- data/SECURITY.md +5 -9
- data/fluentd.gemspec +3 -3
- data/lib/fluent/command/ctl.rb +2 -2
- data/lib/fluent/command/fluentd.rb +55 -53
- data/lib/fluent/command/plugin_config_formatter.rb +1 -1
- data/lib/fluent/config/dsl.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +2 -2
- data/lib/fluent/counter/server.rb +1 -1
- data/lib/fluent/counter/validator.rb +3 -3
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/event.rb +8 -4
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +44 -5
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +6 -1
- data/lib/fluent/plugin/base.rb +6 -8
- data/lib/fluent/plugin/buf_file.rb +32 -3
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +21 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +8 -8
- data/lib/fluent/plugin/in_sample.rb +1 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
- data/lib/fluent/plugin/in_tail.rb +58 -24
- data/lib/fluent/plugin/in_tcp.rb +47 -2
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward.rb +2 -2
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +50 -13
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin_helper/event_loop.rb +2 -2
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
- data/lib/fluent/plugin_helper/server.rb +8 -0
- data/lib/fluent/plugin_helper/thread.rb +3 -3
- data/lib/fluent/plugin_id.rb +1 -1
- data/lib/fluent/supervisor.rb +157 -251
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/test/helper.rb.erb +0 -1
- data/test/command/test_ctl.rb +1 -1
- data/test/command/test_fluentd.rb +137 -6
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +5 -5
- data/test/config/test_system_config.rb +0 -8
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/in_tail/test_position_file.rb +31 -1
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/test_base.rb +99 -1
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_buffer_chunk.rb +11 -0
- data/test/plugin/test_in_forward.rb +9 -9
- data/test/plugin/test_in_http.rb +2 -3
- data/test/plugin/test_in_monitor_agent.rb +2 -3
- data/test/plugin/test_in_tail.rb +379 -0
- data/test/plugin/test_in_tcp.rb +87 -2
- data/test/plugin/test_in_udp.rb +28 -0
- data/test/plugin/test_in_unix.rb +2 -2
- data/test/plugin/test_multi_output.rb +1 -1
- data/test/plugin/test_out_exec_filter.rb +2 -2
- data/test/plugin/test_out_file.rb +2 -2
- data/test/plugin/test_out_forward.rb +14 -18
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_output.rb +281 -12
- data/test/plugin/test_output_as_buffered.rb +44 -44
- data/test/plugin/test_output_as_buffered_compress.rb +32 -18
- data/test/plugin/test_output_as_buffered_retries.rb +1 -1
- data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin_helper/test_child_process.rb +2 -2
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_server.rb +60 -6
- data/test/test_config.rb +0 -21
- data/test/test_formatter.rb +23 -20
- data/test/test_log.rb +108 -36
- data/test/test_msgpack_factory.rb +32 -0
- data/test/test_supervisor.rb +287 -279
- metadata +15 -21
- data/.drone.yml +0 -35
- data/.gitlab-ci.yml +0 -103
- data/test/test_logger_initializer.rb +0 -46
data/lib/fluent/config/dsl.rb
CHANGED
@@ -110,7 +110,7 @@ module Fluent
|
|
110
110
|
|
111
111
|
def include(*args)
|
112
112
|
::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
|
113
|
-
if args.first
|
113
|
+
if /\.rb$/.match?(args.first)
|
114
114
|
path = File.expand_path(args.first)
|
115
115
|
data = File.read(path)
|
116
116
|
self.instance_eval(data, path)
|
@@ -150,8 +150,8 @@ module Fluent
|
|
150
150
|
def eval_include(attrs, elems, uri)
|
151
151
|
# replace space(s)(' ') with '+' to prevent invalid uri due to space(s).
|
152
152
|
# See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212
|
153
|
-
u = URI.parse(uri.
|
154
|
-
if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.
|
153
|
+
u = URI.parse(uri.tr(' ', '+'))
|
154
|
+
if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.tr(' ', '+') # file path
|
155
155
|
# When the Windows absolute path then u.scheme.length == 1
|
156
156
|
# e.g. C:
|
157
157
|
path = URI.decode_www_form_component(u.path)
|
@@ -27,7 +27,7 @@ module Fluent
|
|
27
27
|
DEFAULT_PORT = 24321
|
28
28
|
|
29
29
|
def initialize(name, opt = {})
|
30
|
-
raise 'Counter server name is invalid' unless Validator::VALID_NAME
|
30
|
+
raise 'Counter server name is invalid' unless Validator::VALID_NAME.match?(name)
|
31
31
|
@name = name
|
32
32
|
@opt = opt
|
33
33
|
@addr = @opt[:addr] || DEFAULT_ADDR
|
@@ -82,7 +82,7 @@ module Fluent
|
|
82
82
|
raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
|
83
83
|
end
|
84
84
|
|
85
|
-
unless VALID_NAME
|
85
|
+
unless VALID_NAME.match?(name)
|
86
86
|
raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
|
87
87
|
end
|
88
88
|
end
|
@@ -92,7 +92,7 @@ module Fluent
|
|
92
92
|
raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
|
93
93
|
end
|
94
94
|
|
95
|
-
unless VALID_SCOPE_NAME
|
95
|
+
unless VALID_SCOPE_NAME.match?(name)
|
96
96
|
raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
|
97
97
|
end
|
98
98
|
end
|
@@ -109,7 +109,7 @@ module Fluent
|
|
109
109
|
raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
|
110
110
|
end
|
111
111
|
|
112
|
-
unless VALID_NAME
|
112
|
+
unless VALID_NAME.match?(name)
|
113
113
|
raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
|
114
114
|
end
|
115
115
|
end
|
data/lib/fluent/daemon.rb
CHANGED
@@ -9,7 +9,5 @@ require 'fluent/supervisor'
|
|
9
9
|
|
10
10
|
server_module = Fluent.const_get(ARGV[0])
|
11
11
|
worker_module = Fluent.const_get(ARGV[1])
|
12
|
-
|
13
|
-
|
14
|
-
params = JSON.parse(ARGV[3])
|
15
|
-
ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.load_config(config_path, params) }
|
12
|
+
params = JSON.parse(ARGV[2])
|
13
|
+
ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.serverengine_config(params) }
|
data/lib/fluent/engine.rb
CHANGED
data/lib/fluent/event.rb
CHANGED
@@ -49,7 +49,7 @@ module Fluent
|
|
49
49
|
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
50
50
|
end
|
51
51
|
|
52
|
-
def each(
|
52
|
+
def each(unpacker: nil, &block)
|
53
53
|
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
54
54
|
end
|
55
55
|
|
@@ -294,7 +294,7 @@ module Fluent
|
|
294
294
|
super
|
295
295
|
end
|
296
296
|
|
297
|
-
def to_compressed_msgpack_stream(time_int: false)
|
297
|
+
def to_compressed_msgpack_stream(time_int: false, packer: nil)
|
298
298
|
# time_int is always ignored because @data is always packed binary in this class
|
299
299
|
@compressed_data
|
300
300
|
end
|
@@ -308,11 +308,15 @@ module Fluent
|
|
308
308
|
end
|
309
309
|
|
310
310
|
module ChunkMessagePackEventStreamer
|
311
|
-
# chunk.extend(
|
311
|
+
# chunk.extend(ChunkMessagePackEventStreamer)
|
312
312
|
# => chunk.each{|time, record| ... }
|
313
313
|
def each(unpacker: nil, &block)
|
314
|
+
# Note: If need to use `unpacker`, then implement it,
|
315
|
+
# e.g., `unpacker.feed_each(io.read, &block)` (Not tested)
|
316
|
+
raise NotImplementedError, "'unpacker' argument is not implemented." if unpacker
|
317
|
+
|
314
318
|
open do |io|
|
315
|
-
|
319
|
+
Fluent::MessagePackFactory.msgpack_unpacker(io).each(&block)
|
316
320
|
end
|
317
321
|
nil
|
318
322
|
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'console/terminal/logger'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
class Log
|
21
|
+
# Async gem which is used by http_server helper switched logger mechanism to
|
22
|
+
# Console gem which isn't complatible with Ruby's standard Logger (since
|
23
|
+
# v1.17). This class adapts it to Fluentd's logger mechanism.
|
24
|
+
class ConsoleAdapter < Console::Terminal::Logger
|
25
|
+
def self.wrap(logger)
|
26
|
+
_, level = Console::Logger::LEVELS.find { |key, value|
|
27
|
+
if logger.level <= 0
|
28
|
+
key == :debug
|
29
|
+
else
|
30
|
+
value == logger.level - 1
|
31
|
+
end
|
32
|
+
}
|
33
|
+
Console::Logger.new(ConsoleAdapter.new(logger), level: level)
|
34
|
+
end
|
35
|
+
|
36
|
+
def initialize(logger)
|
37
|
+
@logger = logger
|
38
|
+
# When `verbose` is `true`, following items will be added as a prefix or
|
39
|
+
# suffix of the subject:
|
40
|
+
# * Severity
|
41
|
+
# * Object ID
|
42
|
+
# * PID
|
43
|
+
# * Time
|
44
|
+
# Severity and Time are added by Fluentd::Log too so they are redundant.
|
45
|
+
# PID is the worker's PID so it's also redundant.
|
46
|
+
# Object ID will be too verbose in usual cases.
|
47
|
+
# So set it as `false` here to suppress redundant items.
|
48
|
+
super(StringIO.new, verbose: false)
|
49
|
+
end
|
50
|
+
|
51
|
+
def call(subject = nil, *arguments, name: nil, severity: 'info', **options, &block)
|
52
|
+
if LEVEL_TEXT.include?(severity.to_s)
|
53
|
+
level = severity
|
54
|
+
else
|
55
|
+
@logger.warn("Unknown severity: #{severity}")
|
56
|
+
level = 'warn'
|
57
|
+
end
|
58
|
+
|
59
|
+
@io.seek(0)
|
60
|
+
@io.truncate(0)
|
61
|
+
super
|
62
|
+
@logger.send(level, @io.string.chomp)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
data/lib/fluent/log.rb
CHANGED
@@ -51,6 +51,8 @@ module Fluent
|
|
51
51
|
LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
|
52
52
|
LOG_ROTATE_AGE = %w(daily weekly monthly)
|
53
53
|
|
54
|
+
IGNORE_SAME_LOG_MAX_CACHE_SIZE = 1000 # If need, make this an option of system config.
|
55
|
+
|
54
56
|
def self.str_to_level(log_level_str)
|
55
57
|
case log_level_str.downcase
|
56
58
|
when "trace" then LEVEL_TRACE
|
@@ -67,8 +69,29 @@ module Fluent
|
|
67
69
|
LEVEL_TEXT.map{|t| "#{LOG_EVENT_TAG_PREFIX}.#{t}" }
|
68
70
|
end
|
69
71
|
|
72
|
+
# Create a unique path for each process.
|
73
|
+
#
|
74
|
+
# >>> per_process_path("C:/tmp/test.log", :worker, 1)
|
75
|
+
# C:/tmp/test-1.log
|
76
|
+
# >>> per_process_path("C:/tmp/test.log", :supervisor, 0)
|
77
|
+
# C:/tmp/test-supervisor-0.log
|
78
|
+
def self.per_process_path(path, process_type, worker_id)
|
79
|
+
path = Pathname(path)
|
80
|
+
ext = path.extname
|
81
|
+
|
82
|
+
if process_type == :supervisor
|
83
|
+
suffix = "-#{process_type}-0#{ext}" # "-0" for backword compatibility.
|
84
|
+
else
|
85
|
+
suffix = "-#{worker_id}#{ext}"
|
86
|
+
end
|
87
|
+
return path.sub_ext(suffix).to_s
|
88
|
+
end
|
89
|
+
|
70
90
|
def initialize(logger, opts={})
|
71
|
-
#
|
91
|
+
# When ServerEngine changes the logger.level, the Fluentd logger level should also change.
|
92
|
+
# So overwrites logger.level= below.
|
93
|
+
# However, currently Fluentd doesn't use the ServerEngine's reloading feature,
|
94
|
+
# so maybe we don't need this overwriting anymore.
|
72
95
|
orig_logger_level_setter = logger.class.public_instance_method(:level=).bind(logger)
|
73
96
|
me = self
|
74
97
|
# The original ruby logger sets the number as each log level like below.
|
@@ -92,6 +115,7 @@ module Fluent
|
|
92
115
|
# So if serverengine's logger level is changed, fluentd's log level will be changed to that + 1.
|
93
116
|
logger.define_singleton_method(:level=) {|level| orig_logger_level_setter.call(level); me.level = self.level + 1 }
|
94
117
|
|
118
|
+
@path = opts[:path]
|
95
119
|
@logger = logger
|
96
120
|
@out = logger.instance_variable_get(:@logdev)
|
97
121
|
@level = logger.level + 1
|
@@ -102,7 +126,8 @@ module Fluent
|
|
102
126
|
@time_format = nil
|
103
127
|
@formatter = nil
|
104
128
|
|
105
|
-
self.format = :text
|
129
|
+
self.format = opts.fetch(:format, :text)
|
130
|
+
self.time_format = opts[:time_format] if opts.key?(:time_format)
|
106
131
|
enable_color out.tty?
|
107
132
|
# TODO: This variable name is unclear so we should change to better name.
|
108
133
|
@threads_exclude_events = []
|
@@ -154,8 +179,12 @@ module Fluent
|
|
154
179
|
|
155
180
|
attr_reader :format
|
156
181
|
attr_reader :time_format
|
157
|
-
attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval
|
182
|
+
attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval, :suppress_repeated_stacktrace
|
158
183
|
attr_accessor :out
|
184
|
+
# Strictly speaking, we should also change @logger.level when the setter of @level is called.
|
185
|
+
# Currently, we don't need to do it, since Fluentd::Log doesn't use ServerEngine::DaemonLogger.level.
|
186
|
+
# Since We overwrites logger.level= so that @logger.level is applied to @level,
|
187
|
+
# we need to find a good way to do this, otherwise we will end up in an endless loop.
|
159
188
|
attr_accessor :level
|
160
189
|
attr_accessor :optional_header, :optional_attrs
|
161
190
|
|
@@ -202,9 +231,12 @@ module Fluent
|
|
202
231
|
@time_formatter = Strftime.new(@time_format) rescue nil
|
203
232
|
end
|
204
233
|
|
234
|
+
def stdout?
|
235
|
+
@out == $stdout
|
236
|
+
end
|
237
|
+
|
205
238
|
def reopen!
|
206
|
-
|
207
|
-
@logger.reopen! if @logger
|
239
|
+
@out.reopen(@path, "a") if @path && @path != "-"
|
208
240
|
nil
|
209
241
|
end
|
210
242
|
|
@@ -447,6 +479,13 @@ module Fluent
|
|
447
479
|
false
|
448
480
|
end
|
449
481
|
else
|
482
|
+
if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
|
483
|
+
cached_log.reject! do |_, cached_time|
|
484
|
+
(time - cached_time) > @ignore_same_log_interval
|
485
|
+
end
|
486
|
+
end
|
487
|
+
# If the size is still over, we have no choice but to clear it.
|
488
|
+
cached_log.clear if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
|
450
489
|
cached_log[message] = time
|
451
490
|
false
|
452
491
|
end
|
data/lib/fluent/match.rb
CHANGED
@@ -100,7 +100,12 @@ module Fluent
|
|
100
100
|
end
|
101
101
|
|
102
102
|
def self.thread_local_msgpack_unpacker
|
103
|
-
Thread.current[:local_msgpack_unpacker]
|
103
|
+
unpacker = Thread.current[:local_msgpack_unpacker]
|
104
|
+
if unpacker.nil?
|
105
|
+
return Thread.current[:local_msgpack_unpacker] = MessagePackFactory.engine_factory.unpacker
|
106
|
+
end
|
107
|
+
unpacker.reset
|
108
|
+
unpacker
|
104
109
|
end
|
105
110
|
end
|
106
111
|
end
|
data/lib/fluent/plugin/base.rb
CHANGED
@@ -53,14 +53,12 @@ module Fluent
|
|
53
53
|
end
|
54
54
|
|
55
55
|
def configure(conf)
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
1
|
61
|
-
end
|
62
|
-
system_config_override(workers: workers)
|
56
|
+
raise ArgumentError, "BUG: type of conf must be Fluent::Config::Element, but #{conf.class} is passed." unless conf.is_a?(Fluent::Config::Element)
|
57
|
+
|
58
|
+
if conf.for_this_worker? || (Fluent::Engine.supervisor_mode && !conf.for_every_workers?)
|
59
|
+
system_config_override(workers: conf.target_worker_ids.size)
|
63
60
|
end
|
61
|
+
|
64
62
|
super(conf, system_config.strict_config_value)
|
65
63
|
@_state ||= State.new(false, false, false, false, false, false, false, false, false)
|
66
64
|
@_state.configure = true
|
@@ -192,7 +190,7 @@ module Fluent
|
|
192
190
|
# Thread::Backtrace::Location#path returns base filename or absolute path.
|
193
191
|
# #absolute_path returns absolute_path always.
|
194
192
|
# https://bugs.ruby-lang.org/issues/12159
|
195
|
-
if
|
193
|
+
if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
|
196
194
|
return true
|
197
195
|
end
|
198
196
|
end
|
@@ -139,13 +139,20 @@ module Fluent
|
|
139
139
|
def resume
|
140
140
|
stage = {}
|
141
141
|
queue = []
|
142
|
+
exist_broken_file = false
|
142
143
|
|
143
144
|
patterns = [@path]
|
144
145
|
patterns.unshift @additional_resume_path if @additional_resume_path
|
145
146
|
Dir.glob(escaped_patterns(patterns)) do |path|
|
146
147
|
next unless File.file?(path)
|
147
148
|
|
148
|
-
|
149
|
+
if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
|
150
|
+
# When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
|
151
|
+
# since there may be a power failure or similar failure.
|
152
|
+
log.warn { "restoring buffer file: path = #{path}" }
|
153
|
+
else
|
154
|
+
log.debug { "restoring buffer file: path = #{path}" }
|
155
|
+
end
|
149
156
|
|
150
157
|
m = new_metadata() # this metadata will be overwritten by resuming .meta file content
|
151
158
|
# so it should not added into @metadata_list for now
|
@@ -158,6 +165,7 @@ module Fluent
|
|
158
165
|
begin
|
159
166
|
chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
|
160
167
|
rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
|
168
|
+
exist_broken_file = true
|
161
169
|
handle_broken_files(path, mode, e)
|
162
170
|
next
|
163
171
|
end
|
@@ -182,6 +190,15 @@ module Fluent
|
|
182
190
|
|
183
191
|
queue.sort_by!{ |chunk| chunk.modified_at }
|
184
192
|
|
193
|
+
# If one of the files is corrupted, other files may also be corrupted and be undetected.
|
194
|
+
# The time priods of each chunk are helpful to check the data.
|
195
|
+
if exist_broken_file
|
196
|
+
log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
|
197
|
+
(stage.values + queue).each { |chunk|
|
198
|
+
log.info " #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
|
199
|
+
}
|
200
|
+
end
|
201
|
+
|
185
202
|
return stage, queue
|
186
203
|
end
|
187
204
|
|
@@ -195,8 +212,20 @@ module Fluent
|
|
195
212
|
end
|
196
213
|
|
197
214
|
def handle_broken_files(path, mode, e)
|
198
|
-
log.error "found broken chunk file during resume.
|
199
|
-
|
215
|
+
log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
|
216
|
+
unique_id = Fluent::Plugin::Buffer::FileChunk.unique_id_from_path(path)
|
217
|
+
backup(unique_id) { |f|
|
218
|
+
File.open(path, 'rb') { |chunk|
|
219
|
+
chunk.set_encoding(Encoding::ASCII_8BIT)
|
220
|
+
chunk.sync = true
|
221
|
+
chunk.binmode
|
222
|
+
IO.copy_stream(chunk, f)
|
223
|
+
}
|
224
|
+
}
|
225
|
+
rescue => error
|
226
|
+
log.error "backup failed. Delete corresponding files.", :err_msg => error.message
|
227
|
+
ensure
|
228
|
+
log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
|
200
229
|
File.unlink(path, path + '.meta') rescue nil
|
201
230
|
end
|
202
231
|
|
@@ -160,13 +160,20 @@ module Fluent
|
|
160
160
|
def resume
|
161
161
|
stage = {}
|
162
162
|
queue = []
|
163
|
+
exist_broken_file = false
|
163
164
|
|
164
165
|
patterns = [@path]
|
165
166
|
patterns.unshift @additional_resume_path if @additional_resume_path
|
166
167
|
Dir.glob(escaped_patterns(patterns)) do |path|
|
167
168
|
next unless File.file?(path)
|
168
169
|
|
169
|
-
|
170
|
+
if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
|
171
|
+
# When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
|
172
|
+
# since there may be a power failure or similar failure.
|
173
|
+
log.warn { "restoring buffer file: path = #{path}" }
|
174
|
+
else
|
175
|
+
log.debug { "restoring buffer file: path = #{path}" }
|
176
|
+
end
|
170
177
|
|
171
178
|
m = new_metadata() # this metadata will be updated in FileSingleChunk.new
|
172
179
|
mode = Fluent::Plugin::Buffer::FileSingleChunk.assume_chunk_state(path)
|
@@ -179,6 +186,7 @@ module Fluent
|
|
179
186
|
chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress)
|
180
187
|
chunk.restore_size(@chunk_format) if @calc_num_records
|
181
188
|
rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
|
189
|
+
exist_broken_file = true
|
182
190
|
handle_broken_files(path, mode, e)
|
183
191
|
next
|
184
192
|
end
|
@@ -193,6 +201,15 @@ module Fluent
|
|
193
201
|
|
194
202
|
queue.sort_by!(&:modified_at)
|
195
203
|
|
204
|
+
# If one of the files is corrupted, other files may also be corrupted and be undetected.
|
205
|
+
# The time priods of each chunk are helpful to check the data.
|
206
|
+
if exist_broken_file
|
207
|
+
log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
|
208
|
+
(stage.values + queue).each { |chunk|
|
209
|
+
log.info " #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
|
210
|
+
}
|
211
|
+
end
|
212
|
+
|
196
213
|
return stage, queue
|
197
214
|
end
|
198
215
|
|
@@ -207,8 +224,20 @@ module Fluent
|
|
207
224
|
end
|
208
225
|
|
209
226
|
def handle_broken_files(path, mode, e)
|
210
|
-
log.error "found broken chunk file during resume.
|
211
|
-
|
227
|
+
log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
|
228
|
+
unique_id, _ = Fluent::Plugin::Buffer::FileSingleChunk.unique_id_and_key_from_path(path)
|
229
|
+
backup(unique_id) { |f|
|
230
|
+
File.open(path, 'rb') { |chunk|
|
231
|
+
chunk.set_encoding(Encoding::ASCII_8BIT)
|
232
|
+
chunk.sync = true
|
233
|
+
chunk.binmode
|
234
|
+
IO.copy_stream(chunk, f)
|
235
|
+
}
|
236
|
+
}
|
237
|
+
rescue => error
|
238
|
+
log.error "backup failed. Delete corresponding files.", :err_msg => error.message
|
239
|
+
ensure
|
240
|
+
log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
|
212
241
|
File.unlink(path) rescue nil
|
213
242
|
end
|
214
243
|
|
@@ -204,7 +204,7 @@ module Fluent
|
|
204
204
|
end
|
205
205
|
end
|
206
206
|
|
207
|
-
# used only for queued v0.12 buffer path
|
207
|
+
# used only for queued v0.12 buffer path or broken files
|
208
208
|
def self.unique_id_from_path(path)
|
209
209
|
if /\.(b|q)([0-9a-f]+)\.[^\/]*\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern
|
210
210
|
return $2.scan(/../).map{|x| x.to_i(16) }.pack('C*')
|
data/lib/fluent/plugin/buffer.rb
CHANGED
@@ -66,6 +66,9 @@ module Fluent
|
|
66
66
|
desc 'Compress buffered data.'
|
67
67
|
config_param :compress, :enum, list: [:text, :gzip], default: :text
|
68
68
|
|
69
|
+
desc 'If true, chunks are thrown away when unrecoverable error happens'
|
70
|
+
config_param :disable_chunk_backup, :bool, default: false
|
71
|
+
|
69
72
|
Metadata = Struct.new(:timekey, :tag, :variables, :seq) do
|
70
73
|
def initialize(timekey, tag, variables)
|
71
74
|
super(timekey, tag, variables, 0)
|
@@ -903,6 +906,24 @@ module Fluent
|
|
903
906
|
{ 'buffer' => stats }
|
904
907
|
end
|
905
908
|
|
909
|
+
def backup(chunk_unique_id)
|
910
|
+
unique_id = dump_unique_id_hex(chunk_unique_id)
|
911
|
+
|
912
|
+
if @disable_chunk_backup
|
913
|
+
log.warn "disable_chunk_backup is true. #{unique_id} chunk is not backed up."
|
914
|
+
return
|
915
|
+
end
|
916
|
+
|
917
|
+
safe_owner_id = owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
|
918
|
+
backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
|
919
|
+
backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_owner_id, "#{unique_id}.log")
|
920
|
+
backup_dir = File.dirname(backup_file)
|
921
|
+
|
922
|
+
log.warn "bad chunk is moved to #{backup_file}"
|
923
|
+
FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
|
924
|
+
File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f| yield f }
|
925
|
+
end
|
926
|
+
|
906
927
|
private
|
907
928
|
|
908
929
|
def optimistic_queued?(metadata = nil)
|
@@ -316,7 +316,7 @@ module Fluent::Plugin
|
|
316
316
|
end
|
317
317
|
|
318
318
|
(Object.instance_methods).each do |m|
|
319
|
-
undef_method m unless
|
319
|
+
undef_method m unless /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/.match?(m.to_s)
|
320
320
|
end
|
321
321
|
end
|
322
322
|
end
|
@@ -430,7 +430,7 @@ module Fluent::Plugin
|
|
430
430
|
end
|
431
431
|
_ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
|
432
432
|
|
433
|
-
node = @nodes.
|
433
|
+
node = @nodes.find{|n| n[:address].include?(remote_addr) rescue false }
|
434
434
|
if !node && !@security.allow_anonymous_source
|
435
435
|
log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
|
436
436
|
return false, "anonymous source host '#{remote_addr}' denied", nil
|
@@ -428,7 +428,7 @@ module Fluent::Plugin
|
|
428
428
|
@content_type = ""
|
429
429
|
@content_encoding = ""
|
430
430
|
headers.each_pair {|k,v|
|
431
|
-
@env["HTTP_#{k.
|
431
|
+
@env["HTTP_#{k.tr('-','_').upcase}"] = v
|
432
432
|
case k
|
433
433
|
when /\AExpect\z/i
|
434
434
|
expect = v
|
@@ -439,9 +439,9 @@ module Fluent::Plugin
|
|
439
439
|
when /\AContent-Encoding\Z/i
|
440
440
|
@content_encoding = v
|
441
441
|
when /\AConnection\Z/i
|
442
|
-
if
|
442
|
+
if /close/i.match?(v)
|
443
443
|
@keep_alive = false
|
444
|
-
elsif
|
444
|
+
elsif /Keep-alive/i.match?(v)
|
445
445
|
@keep_alive = true
|
446
446
|
end
|
447
447
|
when /\AOrigin\Z/i
|
@@ -566,16 +566,16 @@ module Fluent::Plugin
|
|
566
566
|
|
567
567
|
if @format_name != 'default'
|
568
568
|
params[EVENT_RECORD_PARAMETER] = @body
|
569
|
-
elsif
|
569
|
+
elsif /^application\/x-www-form-urlencoded/.match?(@content_type)
|
570
570
|
params.update WEBrick::HTTPUtils.parse_query(@body)
|
571
571
|
elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
|
572
572
|
boundary = WEBrick::HTTPUtils.dequote($1)
|
573
573
|
params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
|
574
|
-
elsif
|
574
|
+
elsif /^application\/json/.match?(@content_type)
|
575
575
|
params['json'] = @body
|
576
|
-
elsif
|
576
|
+
elsif /^application\/msgpack/.match?(@content_type)
|
577
577
|
params['msgpack'] = @body
|
578
|
-
elsif
|
578
|
+
elsif /^application\/x-ndjson/.match?(@content_type)
|
579
579
|
params['ndjson'] = @body
|
580
580
|
end
|
581
581
|
path_info = uri.path
|
@@ -585,7 +585,7 @@ module Fluent::Plugin
|
|
585
585
|
query_params = WEBrick::HTTPUtils.parse_query(uri.query)
|
586
586
|
|
587
587
|
query_params.each_pair {|k,v|
|
588
|
-
params["QUERY_#{k.
|
588
|
+
params["QUERY_#{k.tr('-','_').upcase}"] = v
|
589
589
|
}
|
590
590
|
end
|
591
591
|
|
@@ -64,7 +64,7 @@ module Fluent::Plugin
|
|
64
64
|
def configure(conf)
|
65
65
|
super
|
66
66
|
@sample_index = 0
|
67
|
-
config = conf.elements.
|
67
|
+
config = conf.elements.find{|e| e.name == 'storage' }
|
68
68
|
@storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
|
69
69
|
end
|
70
70
|
|