fluentd 1.15.3-x86-mingw32 → 1.16.2-x86-mingw32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
- data/.github/workflows/linux-test.yaml +2 -2
- data/.github/workflows/macos-test.yaml +2 -2
- data/.github/workflows/stale-actions.yml +24 -0
- data/.github/workflows/windows-test.yaml +2 -2
- data/CHANGELOG.md +151 -0
- data/CONTRIBUTING.md +1 -1
- data/MAINTAINERS.md +5 -3
- data/README.md +0 -1
- data/SECURITY.md +5 -9
- data/fluentd.gemspec +3 -3
- data/lib/fluent/command/ctl.rb +2 -2
- data/lib/fluent/command/fluentd.rb +55 -53
- data/lib/fluent/command/plugin_config_formatter.rb +1 -1
- data/lib/fluent/config/dsl.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +2 -2
- data/lib/fluent/counter/server.rb +1 -1
- data/lib/fluent/counter/validator.rb +3 -3
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/event.rb +8 -4
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +44 -5
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +6 -1
- data/lib/fluent/plugin/base.rb +6 -8
- data/lib/fluent/plugin/buf_file.rb +32 -3
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +21 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +8 -8
- data/lib/fluent/plugin/in_sample.rb +1 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
- data/lib/fluent/plugin/in_tail.rb +58 -24
- data/lib/fluent/plugin/in_tcp.rb +47 -2
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward.rb +2 -2
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +50 -13
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin_helper/event_loop.rb +2 -2
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
- data/lib/fluent/plugin_helper/server.rb +8 -0
- data/lib/fluent/plugin_helper/thread.rb +3 -3
- data/lib/fluent/plugin_id.rb +1 -1
- data/lib/fluent/supervisor.rb +157 -251
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/test/helper.rb.erb +0 -1
- data/test/command/test_ctl.rb +1 -1
- data/test/command/test_fluentd.rb +137 -6
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +5 -5
- data/test/config/test_system_config.rb +0 -8
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/in_tail/test_position_file.rb +31 -1
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/test_base.rb +99 -1
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_buffer_chunk.rb +11 -0
- data/test/plugin/test_in_forward.rb +9 -9
- data/test/plugin/test_in_http.rb +2 -3
- data/test/plugin/test_in_monitor_agent.rb +2 -3
- data/test/plugin/test_in_tail.rb +379 -0
- data/test/plugin/test_in_tcp.rb +87 -2
- data/test/plugin/test_in_udp.rb +28 -0
- data/test/plugin/test_in_unix.rb +2 -2
- data/test/plugin/test_multi_output.rb +1 -1
- data/test/plugin/test_out_exec_filter.rb +2 -2
- data/test/plugin/test_out_file.rb +2 -2
- data/test/plugin/test_out_forward.rb +14 -18
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_output.rb +281 -12
- data/test/plugin/test_output_as_buffered.rb +44 -44
- data/test/plugin/test_output_as_buffered_compress.rb +32 -18
- data/test/plugin/test_output_as_buffered_retries.rb +1 -1
- data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin_helper/test_child_process.rb +2 -2
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_server.rb +60 -6
- data/test/test_config.rb +0 -21
- data/test/test_formatter.rb +23 -20
- data/test/test_log.rb +108 -36
- data/test/test_msgpack_factory.rb +32 -0
- data/test/test_supervisor.rb +287 -279
- metadata +15 -21
- data/.drone.yml +0 -35
- data/.gitlab-ci.yml +0 -103
- data/test/test_logger_initializer.rb +0 -46
data/lib/fluent/config/dsl.rb
CHANGED
@@ -110,7 +110,7 @@ module Fluent
|
|
110
110
|
|
111
111
|
def include(*args)
|
112
112
|
::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
|
113
|
-
if args.first
|
113
|
+
if /\.rb$/.match?(args.first)
|
114
114
|
path = File.expand_path(args.first)
|
115
115
|
data = File.read(path)
|
116
116
|
self.instance_eval(data, path)
|
@@ -150,8 +150,8 @@ module Fluent
|
|
150
150
|
def eval_include(attrs, elems, uri)
|
151
151
|
# replace space(s)(' ') with '+' to prevent invalid uri due to space(s).
|
152
152
|
# See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212
|
153
|
-
u = URI.parse(uri.
|
154
|
-
if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.
|
153
|
+
u = URI.parse(uri.tr(' ', '+'))
|
154
|
+
if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.tr(' ', '+') # file path
|
155
155
|
# When the Windows absolute path then u.scheme.length == 1
|
156
156
|
# e.g. C:
|
157
157
|
path = URI.decode_www_form_component(u.path)
|
@@ -27,7 +27,7 @@ module Fluent
|
|
27
27
|
DEFAULT_PORT = 24321
|
28
28
|
|
29
29
|
def initialize(name, opt = {})
|
30
|
-
raise 'Counter server name is invalid' unless Validator::VALID_NAME
|
30
|
+
raise 'Counter server name is invalid' unless Validator::VALID_NAME.match?(name)
|
31
31
|
@name = name
|
32
32
|
@opt = opt
|
33
33
|
@addr = @opt[:addr] || DEFAULT_ADDR
|
@@ -82,7 +82,7 @@ module Fluent
|
|
82
82
|
raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
|
83
83
|
end
|
84
84
|
|
85
|
-
unless VALID_NAME
|
85
|
+
unless VALID_NAME.match?(name)
|
86
86
|
raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
|
87
87
|
end
|
88
88
|
end
|
@@ -92,7 +92,7 @@ module Fluent
|
|
92
92
|
raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
|
93
93
|
end
|
94
94
|
|
95
|
-
unless VALID_SCOPE_NAME
|
95
|
+
unless VALID_SCOPE_NAME.match?(name)
|
96
96
|
raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
|
97
97
|
end
|
98
98
|
end
|
@@ -109,7 +109,7 @@ module Fluent
|
|
109
109
|
raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
|
110
110
|
end
|
111
111
|
|
112
|
-
unless VALID_NAME
|
112
|
+
unless VALID_NAME.match?(name)
|
113
113
|
raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
|
114
114
|
end
|
115
115
|
end
|
data/lib/fluent/daemon.rb
CHANGED
@@ -9,7 +9,5 @@ require 'fluent/supervisor'
|
|
9
9
|
|
10
10
|
server_module = Fluent.const_get(ARGV[0])
|
11
11
|
worker_module = Fluent.const_get(ARGV[1])
|
12
|
-
|
13
|
-
|
14
|
-
params = JSON.parse(ARGV[3])
|
15
|
-
ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.load_config(config_path, params) }
|
12
|
+
params = JSON.parse(ARGV[2])
|
13
|
+
ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.serverengine_config(params) }
|
data/lib/fluent/engine.rb
CHANGED
data/lib/fluent/event.rb
CHANGED
@@ -49,7 +49,7 @@ module Fluent
|
|
49
49
|
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
50
50
|
end
|
51
51
|
|
52
|
-
def each(
|
52
|
+
def each(unpacker: nil, &block)
|
53
53
|
raise NotImplementedError, "DO NOT USE THIS CLASS directly."
|
54
54
|
end
|
55
55
|
|
@@ -294,7 +294,7 @@ module Fluent
|
|
294
294
|
super
|
295
295
|
end
|
296
296
|
|
297
|
-
def to_compressed_msgpack_stream(time_int: false)
|
297
|
+
def to_compressed_msgpack_stream(time_int: false, packer: nil)
|
298
298
|
# time_int is always ignored because @data is always packed binary in this class
|
299
299
|
@compressed_data
|
300
300
|
end
|
@@ -308,11 +308,15 @@ module Fluent
|
|
308
308
|
end
|
309
309
|
|
310
310
|
module ChunkMessagePackEventStreamer
|
311
|
-
# chunk.extend(
|
311
|
+
# chunk.extend(ChunkMessagePackEventStreamer)
|
312
312
|
# => chunk.each{|time, record| ... }
|
313
313
|
def each(unpacker: nil, &block)
|
314
|
+
# Note: If need to use `unpacker`, then implement it,
|
315
|
+
# e.g., `unpacker.feed_each(io.read, &block)` (Not tested)
|
316
|
+
raise NotImplementedError, "'unpacker' argument is not implemented." if unpacker
|
317
|
+
|
314
318
|
open do |io|
|
315
|
-
|
319
|
+
Fluent::MessagePackFactory.msgpack_unpacker(io).each(&block)
|
316
320
|
end
|
317
321
|
nil
|
318
322
|
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
#
|
2
|
+
# Fluentd
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
#
|
16
|
+
|
17
|
+
require 'console/terminal/logger'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
class Log
|
21
|
+
# Async gem which is used by http_server helper switched logger mechanism to
|
22
|
+
# Console gem which isn't complatible with Ruby's standard Logger (since
|
23
|
+
# v1.17). This class adapts it to Fluentd's logger mechanism.
|
24
|
+
class ConsoleAdapter < Console::Terminal::Logger
|
25
|
+
def self.wrap(logger)
|
26
|
+
_, level = Console::Logger::LEVELS.find { |key, value|
|
27
|
+
if logger.level <= 0
|
28
|
+
key == :debug
|
29
|
+
else
|
30
|
+
value == logger.level - 1
|
31
|
+
end
|
32
|
+
}
|
33
|
+
Console::Logger.new(ConsoleAdapter.new(logger), level: level)
|
34
|
+
end
|
35
|
+
|
36
|
+
def initialize(logger)
|
37
|
+
@logger = logger
|
38
|
+
# When `verbose` is `true`, following items will be added as a prefix or
|
39
|
+
# suffix of the subject:
|
40
|
+
# * Severity
|
41
|
+
# * Object ID
|
42
|
+
# * PID
|
43
|
+
# * Time
|
44
|
+
# Severity and Time are added by Fluentd::Log too so they are redundant.
|
45
|
+
# PID is the worker's PID so it's also redundant.
|
46
|
+
# Object ID will be too verbose in usual cases.
|
47
|
+
# So set it as `false` here to suppress redundant items.
|
48
|
+
super(StringIO.new, verbose: false)
|
49
|
+
end
|
50
|
+
|
51
|
+
def call(subject = nil, *arguments, name: nil, severity: 'info', **options, &block)
|
52
|
+
if LEVEL_TEXT.include?(severity.to_s)
|
53
|
+
level = severity
|
54
|
+
else
|
55
|
+
@logger.warn("Unknown severity: #{severity}")
|
56
|
+
level = 'warn'
|
57
|
+
end
|
58
|
+
|
59
|
+
@io.seek(0)
|
60
|
+
@io.truncate(0)
|
61
|
+
super
|
62
|
+
@logger.send(level, @io.string.chomp)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
data/lib/fluent/log.rb
CHANGED
@@ -51,6 +51,8 @@ module Fluent
|
|
51
51
|
LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
|
52
52
|
LOG_ROTATE_AGE = %w(daily weekly monthly)
|
53
53
|
|
54
|
+
IGNORE_SAME_LOG_MAX_CACHE_SIZE = 1000 # If need, make this an option of system config.
|
55
|
+
|
54
56
|
def self.str_to_level(log_level_str)
|
55
57
|
case log_level_str.downcase
|
56
58
|
when "trace" then LEVEL_TRACE
|
@@ -67,8 +69,29 @@ module Fluent
|
|
67
69
|
LEVEL_TEXT.map{|t| "#{LOG_EVENT_TAG_PREFIX}.#{t}" }
|
68
70
|
end
|
69
71
|
|
72
|
+
# Create a unique path for each process.
|
73
|
+
#
|
74
|
+
# >>> per_process_path("C:/tmp/test.log", :worker, 1)
|
75
|
+
# C:/tmp/test-1.log
|
76
|
+
# >>> per_process_path("C:/tmp/test.log", :supervisor, 0)
|
77
|
+
# C:/tmp/test-supervisor-0.log
|
78
|
+
def self.per_process_path(path, process_type, worker_id)
|
79
|
+
path = Pathname(path)
|
80
|
+
ext = path.extname
|
81
|
+
|
82
|
+
if process_type == :supervisor
|
83
|
+
suffix = "-#{process_type}-0#{ext}" # "-0" for backword compatibility.
|
84
|
+
else
|
85
|
+
suffix = "-#{worker_id}#{ext}"
|
86
|
+
end
|
87
|
+
return path.sub_ext(suffix).to_s
|
88
|
+
end
|
89
|
+
|
70
90
|
def initialize(logger, opts={})
|
71
|
-
#
|
91
|
+
# When ServerEngine changes the logger.level, the Fluentd logger level should also change.
|
92
|
+
# So overwrites logger.level= below.
|
93
|
+
# However, currently Fluentd doesn't use the ServerEngine's reloading feature,
|
94
|
+
# so maybe we don't need this overwriting anymore.
|
72
95
|
orig_logger_level_setter = logger.class.public_instance_method(:level=).bind(logger)
|
73
96
|
me = self
|
74
97
|
# The original ruby logger sets the number as each log level like below.
|
@@ -92,6 +115,7 @@ module Fluent
|
|
92
115
|
# So if serverengine's logger level is changed, fluentd's log level will be changed to that + 1.
|
93
116
|
logger.define_singleton_method(:level=) {|level| orig_logger_level_setter.call(level); me.level = self.level + 1 }
|
94
117
|
|
118
|
+
@path = opts[:path]
|
95
119
|
@logger = logger
|
96
120
|
@out = logger.instance_variable_get(:@logdev)
|
97
121
|
@level = logger.level + 1
|
@@ -102,7 +126,8 @@ module Fluent
|
|
102
126
|
@time_format = nil
|
103
127
|
@formatter = nil
|
104
128
|
|
105
|
-
self.format = :text
|
129
|
+
self.format = opts.fetch(:format, :text)
|
130
|
+
self.time_format = opts[:time_format] if opts.key?(:time_format)
|
106
131
|
enable_color out.tty?
|
107
132
|
# TODO: This variable name is unclear so we should change to better name.
|
108
133
|
@threads_exclude_events = []
|
@@ -154,8 +179,12 @@ module Fluent
|
|
154
179
|
|
155
180
|
attr_reader :format
|
156
181
|
attr_reader :time_format
|
157
|
-
attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval
|
182
|
+
attr_accessor :log_event_enabled, :ignore_repeated_log_interval, :ignore_same_log_interval, :suppress_repeated_stacktrace
|
158
183
|
attr_accessor :out
|
184
|
+
# Strictly speaking, we should also change @logger.level when the setter of @level is called.
|
185
|
+
# Currently, we don't need to do it, since Fluentd::Log doesn't use ServerEngine::DaemonLogger.level.
|
186
|
+
# Since We overwrites logger.level= so that @logger.level is applied to @level,
|
187
|
+
# we need to find a good way to do this, otherwise we will end up in an endless loop.
|
159
188
|
attr_accessor :level
|
160
189
|
attr_accessor :optional_header, :optional_attrs
|
161
190
|
|
@@ -202,9 +231,12 @@ module Fluent
|
|
202
231
|
@time_formatter = Strftime.new(@time_format) rescue nil
|
203
232
|
end
|
204
233
|
|
234
|
+
def stdout?
|
235
|
+
@out == $stdout
|
236
|
+
end
|
237
|
+
|
205
238
|
def reopen!
|
206
|
-
|
207
|
-
@logger.reopen! if @logger
|
239
|
+
@out.reopen(@path, "a") if @path && @path != "-"
|
208
240
|
nil
|
209
241
|
end
|
210
242
|
|
@@ -447,6 +479,13 @@ module Fluent
|
|
447
479
|
false
|
448
480
|
end
|
449
481
|
else
|
482
|
+
if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
|
483
|
+
cached_log.reject! do |_, cached_time|
|
484
|
+
(time - cached_time) > @ignore_same_log_interval
|
485
|
+
end
|
486
|
+
end
|
487
|
+
# If the size is still over, we have no choice but to clear it.
|
488
|
+
cached_log.clear if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
|
450
489
|
cached_log[message] = time
|
451
490
|
false
|
452
491
|
end
|
data/lib/fluent/match.rb
CHANGED
@@ -100,7 +100,12 @@ module Fluent
|
|
100
100
|
end
|
101
101
|
|
102
102
|
def self.thread_local_msgpack_unpacker
|
103
|
-
Thread.current[:local_msgpack_unpacker]
|
103
|
+
unpacker = Thread.current[:local_msgpack_unpacker]
|
104
|
+
if unpacker.nil?
|
105
|
+
return Thread.current[:local_msgpack_unpacker] = MessagePackFactory.engine_factory.unpacker
|
106
|
+
end
|
107
|
+
unpacker.reset
|
108
|
+
unpacker
|
104
109
|
end
|
105
110
|
end
|
106
111
|
end
|
data/lib/fluent/plugin/base.rb
CHANGED
@@ -53,14 +53,12 @@ module Fluent
|
|
53
53
|
end
|
54
54
|
|
55
55
|
def configure(conf)
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
1
|
61
|
-
end
|
62
|
-
system_config_override(workers: workers)
|
56
|
+
raise ArgumentError, "BUG: type of conf must be Fluent::Config::Element, but #{conf.class} is passed." unless conf.is_a?(Fluent::Config::Element)
|
57
|
+
|
58
|
+
if conf.for_this_worker? || (Fluent::Engine.supervisor_mode && !conf.for_every_workers?)
|
59
|
+
system_config_override(workers: conf.target_worker_ids.size)
|
63
60
|
end
|
61
|
+
|
64
62
|
super(conf, system_config.strict_config_value)
|
65
63
|
@_state ||= State.new(false, false, false, false, false, false, false, false, false)
|
66
64
|
@_state.configure = true
|
@@ -192,7 +190,7 @@ module Fluent
|
|
192
190
|
# Thread::Backtrace::Location#path returns base filename or absolute path.
|
193
191
|
# #absolute_path returns absolute_path always.
|
194
192
|
# https://bugs.ruby-lang.org/issues/12159
|
195
|
-
if
|
193
|
+
if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
|
196
194
|
return true
|
197
195
|
end
|
198
196
|
end
|
@@ -139,13 +139,20 @@ module Fluent
|
|
139
139
|
def resume
|
140
140
|
stage = {}
|
141
141
|
queue = []
|
142
|
+
exist_broken_file = false
|
142
143
|
|
143
144
|
patterns = [@path]
|
144
145
|
patterns.unshift @additional_resume_path if @additional_resume_path
|
145
146
|
Dir.glob(escaped_patterns(patterns)) do |path|
|
146
147
|
next unless File.file?(path)
|
147
148
|
|
148
|
-
|
149
|
+
if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
|
150
|
+
# When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
|
151
|
+
# since there may be a power failure or similar failure.
|
152
|
+
log.warn { "restoring buffer file: path = #{path}" }
|
153
|
+
else
|
154
|
+
log.debug { "restoring buffer file: path = #{path}" }
|
155
|
+
end
|
149
156
|
|
150
157
|
m = new_metadata() # this metadata will be overwritten by resuming .meta file content
|
151
158
|
# so it should not added into @metadata_list for now
|
@@ -158,6 +165,7 @@ module Fluent
|
|
158
165
|
begin
|
159
166
|
chunk = Fluent::Plugin::Buffer::FileChunk.new(m, path, mode, compress: @compress) # file chunk resumes contents of metadata
|
160
167
|
rescue Fluent::Plugin::Buffer::FileChunk::FileChunkError => e
|
168
|
+
exist_broken_file = true
|
161
169
|
handle_broken_files(path, mode, e)
|
162
170
|
next
|
163
171
|
end
|
@@ -182,6 +190,15 @@ module Fluent
|
|
182
190
|
|
183
191
|
queue.sort_by!{ |chunk| chunk.modified_at }
|
184
192
|
|
193
|
+
# If one of the files is corrupted, other files may also be corrupted and be undetected.
|
194
|
+
# The time priods of each chunk are helpful to check the data.
|
195
|
+
if exist_broken_file
|
196
|
+
log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
|
197
|
+
(stage.values + queue).each { |chunk|
|
198
|
+
log.info " #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
|
199
|
+
}
|
200
|
+
end
|
201
|
+
|
185
202
|
return stage, queue
|
186
203
|
end
|
187
204
|
|
@@ -195,8 +212,20 @@ module Fluent
|
|
195
212
|
end
|
196
213
|
|
197
214
|
def handle_broken_files(path, mode, e)
|
198
|
-
log.error "found broken chunk file during resume.
|
199
|
-
|
215
|
+
log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
|
216
|
+
unique_id = Fluent::Plugin::Buffer::FileChunk.unique_id_from_path(path)
|
217
|
+
backup(unique_id) { |f|
|
218
|
+
File.open(path, 'rb') { |chunk|
|
219
|
+
chunk.set_encoding(Encoding::ASCII_8BIT)
|
220
|
+
chunk.sync = true
|
221
|
+
chunk.binmode
|
222
|
+
IO.copy_stream(chunk, f)
|
223
|
+
}
|
224
|
+
}
|
225
|
+
rescue => error
|
226
|
+
log.error "backup failed. Delete corresponding files.", :err_msg => error.message
|
227
|
+
ensure
|
228
|
+
log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
|
200
229
|
File.unlink(path, path + '.meta') rescue nil
|
201
230
|
end
|
202
231
|
|
@@ -160,13 +160,20 @@ module Fluent
|
|
160
160
|
def resume
|
161
161
|
stage = {}
|
162
162
|
queue = []
|
163
|
+
exist_broken_file = false
|
163
164
|
|
164
165
|
patterns = [@path]
|
165
166
|
patterns.unshift @additional_resume_path if @additional_resume_path
|
166
167
|
Dir.glob(escaped_patterns(patterns)) do |path|
|
167
168
|
next unless File.file?(path)
|
168
169
|
|
169
|
-
|
170
|
+
if owner.respond_to?(:buffer_config) && owner.buffer_config&.flush_at_shutdown
|
171
|
+
# When `flush_at_shutdown` is `true`, the remaining chunk files during resuming are possibly broken
|
172
|
+
# since there may be a power failure or similar failure.
|
173
|
+
log.warn { "restoring buffer file: path = #{path}" }
|
174
|
+
else
|
175
|
+
log.debug { "restoring buffer file: path = #{path}" }
|
176
|
+
end
|
170
177
|
|
171
178
|
m = new_metadata() # this metadata will be updated in FileSingleChunk.new
|
172
179
|
mode = Fluent::Plugin::Buffer::FileSingleChunk.assume_chunk_state(path)
|
@@ -179,6 +186,7 @@ module Fluent
|
|
179
186
|
chunk = Fluent::Plugin::Buffer::FileSingleChunk.new(m, path, mode, @key_in_path, compress: @compress)
|
180
187
|
chunk.restore_size(@chunk_format) if @calc_num_records
|
181
188
|
rescue Fluent::Plugin::Buffer::FileSingleChunk::FileChunkError => e
|
189
|
+
exist_broken_file = true
|
182
190
|
handle_broken_files(path, mode, e)
|
183
191
|
next
|
184
192
|
end
|
@@ -193,6 +201,15 @@ module Fluent
|
|
193
201
|
|
194
202
|
queue.sort_by!(&:modified_at)
|
195
203
|
|
204
|
+
# If one of the files is corrupted, other files may also be corrupted and be undetected.
|
205
|
+
# The time priods of each chunk are helpful to check the data.
|
206
|
+
if exist_broken_file
|
207
|
+
log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
|
208
|
+
(stage.values + queue).each { |chunk|
|
209
|
+
log.info " #{chunk.path}:", :created_at => chunk.created_at, :modified_at => chunk.modified_at
|
210
|
+
}
|
211
|
+
end
|
212
|
+
|
196
213
|
return stage, queue
|
197
214
|
end
|
198
215
|
|
@@ -207,8 +224,20 @@ module Fluent
|
|
207
224
|
end
|
208
225
|
|
209
226
|
def handle_broken_files(path, mode, e)
|
210
|
-
log.error "found broken chunk file during resume.
|
211
|
-
|
227
|
+
log.error "found broken chunk file during resume.", :path => path, :mode => mode, :err_msg => e.message
|
228
|
+
unique_id, _ = Fluent::Plugin::Buffer::FileSingleChunk.unique_id_and_key_from_path(path)
|
229
|
+
backup(unique_id) { |f|
|
230
|
+
File.open(path, 'rb') { |chunk|
|
231
|
+
chunk.set_encoding(Encoding::ASCII_8BIT)
|
232
|
+
chunk.sync = true
|
233
|
+
chunk.binmode
|
234
|
+
IO.copy_stream(chunk, f)
|
235
|
+
}
|
236
|
+
}
|
237
|
+
rescue => error
|
238
|
+
log.error "backup failed. Delete corresponding files.", :err_msg => error.message
|
239
|
+
ensure
|
240
|
+
log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(unique_id)} chunk is thrown away." if @disable_chunk_backup
|
212
241
|
File.unlink(path) rescue nil
|
213
242
|
end
|
214
243
|
|
@@ -204,7 +204,7 @@ module Fluent
|
|
204
204
|
end
|
205
205
|
end
|
206
206
|
|
207
|
-
# used only for queued v0.12 buffer path
|
207
|
+
# used only for queued v0.12 buffer path or broken files
|
208
208
|
def self.unique_id_from_path(path)
|
209
209
|
if /\.(b|q)([0-9a-f]+)\.[^\/]*\Z/n =~ path # //n switch means explicit 'ASCII-8BIT' pattern
|
210
210
|
return $2.scan(/../).map{|x| x.to_i(16) }.pack('C*')
|
data/lib/fluent/plugin/buffer.rb
CHANGED
@@ -66,6 +66,9 @@ module Fluent
|
|
66
66
|
desc 'Compress buffered data.'
|
67
67
|
config_param :compress, :enum, list: [:text, :gzip], default: :text
|
68
68
|
|
69
|
+
desc 'If true, chunks are thrown away when unrecoverable error happens'
|
70
|
+
config_param :disable_chunk_backup, :bool, default: false
|
71
|
+
|
69
72
|
Metadata = Struct.new(:timekey, :tag, :variables, :seq) do
|
70
73
|
def initialize(timekey, tag, variables)
|
71
74
|
super(timekey, tag, variables, 0)
|
@@ -903,6 +906,24 @@ module Fluent
|
|
903
906
|
{ 'buffer' => stats }
|
904
907
|
end
|
905
908
|
|
909
|
+
def backup(chunk_unique_id)
|
910
|
+
unique_id = dump_unique_id_hex(chunk_unique_id)
|
911
|
+
|
912
|
+
if @disable_chunk_backup
|
913
|
+
log.warn "disable_chunk_backup is true. #{unique_id} chunk is not backed up."
|
914
|
+
return
|
915
|
+
end
|
916
|
+
|
917
|
+
safe_owner_id = owner.plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
|
918
|
+
backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
|
919
|
+
backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_owner_id, "#{unique_id}.log")
|
920
|
+
backup_dir = File.dirname(backup_file)
|
921
|
+
|
922
|
+
log.warn "bad chunk is moved to #{backup_file}"
|
923
|
+
FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
|
924
|
+
File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f| yield f }
|
925
|
+
end
|
926
|
+
|
906
927
|
private
|
907
928
|
|
908
929
|
def optimistic_queued?(metadata = nil)
|
@@ -316,7 +316,7 @@ module Fluent::Plugin
|
|
316
316
|
end
|
317
317
|
|
318
318
|
(Object.instance_methods).each do |m|
|
319
|
-
undef_method m unless
|
319
|
+
undef_method m unless /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/.match?(m.to_s)
|
320
320
|
end
|
321
321
|
end
|
322
322
|
end
|
@@ -430,7 +430,7 @@ module Fluent::Plugin
|
|
430
430
|
end
|
431
431
|
_ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
|
432
432
|
|
433
|
-
node = @nodes.
|
433
|
+
node = @nodes.find{|n| n[:address].include?(remote_addr) rescue false }
|
434
434
|
if !node && !@security.allow_anonymous_source
|
435
435
|
log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
|
436
436
|
return false, "anonymous source host '#{remote_addr}' denied", nil
|
@@ -428,7 +428,7 @@ module Fluent::Plugin
|
|
428
428
|
@content_type = ""
|
429
429
|
@content_encoding = ""
|
430
430
|
headers.each_pair {|k,v|
|
431
|
-
@env["HTTP_#{k.
|
431
|
+
@env["HTTP_#{k.tr('-','_').upcase}"] = v
|
432
432
|
case k
|
433
433
|
when /\AExpect\z/i
|
434
434
|
expect = v
|
@@ -439,9 +439,9 @@ module Fluent::Plugin
|
|
439
439
|
when /\AContent-Encoding\Z/i
|
440
440
|
@content_encoding = v
|
441
441
|
when /\AConnection\Z/i
|
442
|
-
if
|
442
|
+
if /close/i.match?(v)
|
443
443
|
@keep_alive = false
|
444
|
-
elsif
|
444
|
+
elsif /Keep-alive/i.match?(v)
|
445
445
|
@keep_alive = true
|
446
446
|
end
|
447
447
|
when /\AOrigin\Z/i
|
@@ -566,16 +566,16 @@ module Fluent::Plugin
|
|
566
566
|
|
567
567
|
if @format_name != 'default'
|
568
568
|
params[EVENT_RECORD_PARAMETER] = @body
|
569
|
-
elsif
|
569
|
+
elsif /^application\/x-www-form-urlencoded/.match?(@content_type)
|
570
570
|
params.update WEBrick::HTTPUtils.parse_query(@body)
|
571
571
|
elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
|
572
572
|
boundary = WEBrick::HTTPUtils.dequote($1)
|
573
573
|
params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
|
574
|
-
elsif
|
574
|
+
elsif /^application\/json/.match?(@content_type)
|
575
575
|
params['json'] = @body
|
576
|
-
elsif
|
576
|
+
elsif /^application\/msgpack/.match?(@content_type)
|
577
577
|
params['msgpack'] = @body
|
578
|
-
elsif
|
578
|
+
elsif /^application\/x-ndjson/.match?(@content_type)
|
579
579
|
params['ndjson'] = @body
|
580
580
|
end
|
581
581
|
path_info = uri.path
|
@@ -585,7 +585,7 @@ module Fluent::Plugin
|
|
585
585
|
query_params = WEBrick::HTTPUtils.parse_query(uri.query)
|
586
586
|
|
587
587
|
query_params.each_pair {|k,v|
|
588
|
-
params["QUERY_#{k.
|
588
|
+
params["QUERY_#{k.tr('-','_').upcase}"] = v
|
589
589
|
}
|
590
590
|
end
|
591
591
|
|
@@ -64,7 +64,7 @@ module Fluent::Plugin
|
|
64
64
|
def configure(conf)
|
65
65
|
super
|
66
66
|
@sample_index = 0
|
67
|
-
config = conf.elements.
|
67
|
+
config = conf.elements.find{|e| e.name == 'storage' }
|
68
68
|
@storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
|
69
69
|
end
|
70
70
|
|