fluentd 1.18.0-x64-mingw32 → 1.19.1-x64-mingw32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +116 -0
- data/CHANGELOG.md +270 -12
- data/MAINTAINERS.md +8 -2
- data/README.md +3 -7
- data/Rakefile +2 -0
- data/SECURITY.md +11 -3
- data/lib/fluent/command/cap_ctl.rb +2 -2
- data/lib/fluent/command/fluentd.rb +6 -2
- data/lib/fluent/compat/formatter.rb +6 -0
- data/lib/fluent/compat/socket_util.rb +2 -2
- data/lib/fluent/config/configure_proxy.rb +1 -1
- data/lib/fluent/config/element.rb +2 -2
- data/lib/fluent/config/literal_parser.rb +3 -3
- data/lib/fluent/config/parser.rb +15 -3
- data/lib/fluent/config/section.rb +2 -2
- data/lib/fluent/config/types.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +3 -3
- data/lib/fluent/config/yaml_parser/parser.rb +10 -2
- data/lib/fluent/counter/store.rb +1 -1
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/env.rb +3 -2
- data/lib/fluent/event.rb +7 -6
- data/lib/fluent/log/console_adapter.rb +5 -7
- data/lib/fluent/log.rb +23 -0
- data/lib/fluent/plugin/bare_output.rb +0 -16
- data/lib/fluent/plugin/base.rb +2 -2
- data/lib/fluent/plugin/buf_file.rb +15 -1
- data/lib/fluent/plugin/buf_file_single.rb +15 -1
- data/lib/fluent/plugin/buffer/chunk.rb +74 -10
- data/lib/fluent/plugin/buffer/file_chunk.rb +9 -5
- data/lib/fluent/plugin/buffer/file_single_chunk.rb +3 -3
- data/lib/fluent/plugin/buffer/memory_chunk.rb +2 -2
- data/lib/fluent/plugin/buffer.rb +34 -6
- data/lib/fluent/plugin/compressable.rb +68 -22
- data/lib/fluent/plugin/filter.rb +0 -8
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/formatter_csv.rb +18 -4
- data/lib/fluent/plugin/formatter_json.rb +7 -4
- data/lib/fluent/plugin/formatter_out_file.rb +5 -2
- data/lib/fluent/plugin/in_forward.rb +9 -5
- data/lib/fluent/plugin/in_http.rb +9 -4
- data/lib/fluent/plugin/in_monitor_agent.rb +4 -8
- data/lib/fluent/plugin/in_tail/position_file.rb +1 -1
- data/lib/fluent/plugin/in_tail.rb +80 -57
- data/lib/fluent/plugin/in_tcp.rb +2 -2
- data/lib/fluent/plugin/in_udp.rb +1 -1
- data/lib/fluent/plugin/input.rb +0 -8
- data/lib/fluent/plugin/multi_output.rb +1 -17
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/out_file.rb +37 -30
- data/lib/fluent/plugin/out_forward/connection_manager.rb +2 -2
- data/lib/fluent/plugin/out_forward.rb +23 -13
- data/lib/fluent/plugin/out_http.rb +1 -1
- data/lib/fluent/plugin/out_secondary_file.rb +2 -2
- data/lib/fluent/plugin/out_stdout.rb +10 -3
- data/lib/fluent/plugin/out_stream.rb +3 -3
- data/lib/fluent/plugin/output.rb +24 -35
- data/lib/fluent/plugin/owned_by_mixin.rb +2 -2
- data/lib/fluent/plugin/parser.rb +3 -3
- data/lib/fluent/plugin/parser_json.rb +3 -3
- data/lib/fluent/plugin/sd_file.rb +2 -2
- data/lib/fluent/plugin/storage_local.rb +8 -4
- data/lib/fluent/plugin.rb +1 -1
- data/lib/fluent/plugin_helper/child_process.rb +2 -2
- data/lib/fluent/plugin_helper/http_server/request.rb +13 -2
- data/lib/fluent/plugin_helper/http_server/server.rb +4 -14
- data/lib/fluent/plugin_helper/http_server.rb +1 -8
- data/lib/fluent/plugin_helper/metrics.rb +7 -0
- data/lib/fluent/plugin_helper/server.rb +4 -1
- data/lib/fluent/plugin_helper/service_discovery.rb +1 -1
- data/lib/fluent/plugin_helper/socket_option.rb +2 -2
- data/lib/fluent/plugin_helper/storage.rb +1 -1
- data/lib/fluent/plugin_id.rb +3 -3
- data/lib/fluent/root_agent.rb +4 -3
- data/lib/fluent/static_config_analysis.rb +3 -2
- data/lib/fluent/supervisor.rb +51 -5
- data/lib/fluent/system_config.rb +13 -4
- data/lib/fluent/test/base.rb +1 -1
- data/lib/fluent/test/driver/base.rb +2 -2
- data/lib/fluent/test/filter_test.rb +2 -2
- data/lib/fluent/test/formatter_test.rb +1 -1
- data/lib/fluent/test/helpers.rb +4 -0
- data/lib/fluent/test/input_test.rb +2 -2
- data/lib/fluent/test/output_test.rb +4 -4
- data/lib/fluent/test/parser_test.rb +1 -1
- data/lib/fluent/test.rb +5 -2
- data/lib/fluent/tls.rb +24 -0
- data/lib/fluent/variable_store.rb +1 -1
- data/lib/fluent/version.rb +1 -1
- data/lib/fluent/winsvc.rb +38 -8
- metadata +77 -16
- data/lib/fluent/plugin_helper/http_server/compat/server.rb +0 -92
- data/lib/fluent/plugin_helper/http_server/compat/ssl_context_extractor.rb +0 -52
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +0 -58
|
@@ -31,7 +31,7 @@ module Fluent
|
|
|
31
31
|
@unused = unused || attrs.keys
|
|
32
32
|
@v1_config = false
|
|
33
33
|
@corresponding_proxies = [] # some plugins use flat parameters, e.g. in_http doesn't provide <format> section for parser.
|
|
34
|
-
@unused_in = nil # if this element is not used in plugins,
|
|
34
|
+
@unused_in = nil # if this element is not used in plugins, corresponding plugin name and parent element name is set, e.g. [source, plugin class].
|
|
35
35
|
|
|
36
36
|
# it's global logger, not plugin logger: deprecated message should be global warning, not plugin level.
|
|
37
37
|
@logger = defined?($log) ? $log : nil
|
|
@@ -73,7 +73,7 @@ module Fluent
|
|
|
73
73
|
|
|
74
74
|
def inspect
|
|
75
75
|
attrs = super
|
|
76
|
-
"name:#{@name}, arg:#{@arg},
|
|
76
|
+
"<name:#{@name}, arg:#{@arg}, attrs:#{attrs}, elements:#{@elements.inspect}>"
|
|
77
77
|
end
|
|
78
78
|
|
|
79
79
|
# Used by PP and Pry
|
|
@@ -98,11 +98,11 @@ module Fluent
|
|
|
98
98
|
else
|
|
99
99
|
return string.join
|
|
100
100
|
end
|
|
101
|
-
elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
|
|
102
|
-
if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
|
|
101
|
+
elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
|
|
102
|
+
if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
|
|
103
103
|
string << s
|
|
104
104
|
end
|
|
105
|
-
skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
|
|
105
|
+
skip(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/o)
|
|
106
106
|
elsif s = scan(/\\./)
|
|
107
107
|
string << eval_escape_char(s[1,1])
|
|
108
108
|
elsif skip(/\#\{/)
|
data/lib/fluent/config/parser.rb
CHANGED
|
@@ -92,10 +92,23 @@ module Fluent
|
|
|
92
92
|
else
|
|
93
93
|
basepath = '/'
|
|
94
94
|
fname = path
|
|
95
|
-
|
|
96
|
-
URI.open(uri) {|f|
|
|
95
|
+
parser_proc = ->(f) {
|
|
97
96
|
Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
|
|
98
97
|
}
|
|
98
|
+
|
|
99
|
+
case u.scheme
|
|
100
|
+
when 'http', 'https', 'ftp'
|
|
101
|
+
# URI#open can be able to handle URIs for http, https and ftp.
|
|
102
|
+
require 'open-uri'
|
|
103
|
+
u.open(&parser_proc)
|
|
104
|
+
else
|
|
105
|
+
# TODO: This case should be handled in the previous if condition. Glob is not applied to some Windows path formats.
|
|
106
|
+
# 'c:/path/to/file' will be passed as URI, 'uri' and 'u.path' will be:
|
|
107
|
+
# - uri is 'c:/path/to/file'
|
|
108
|
+
# - u.path is '/path/to/file' and u.scheme is 'c'
|
|
109
|
+
# Therefore, the condition of the if statement above is not met and it is handled here.
|
|
110
|
+
File.open(uri, &parser_proc)
|
|
111
|
+
end
|
|
99
112
|
end
|
|
100
113
|
|
|
101
114
|
rescue SystemCallError => e
|
|
@@ -104,4 +117,3 @@ module Fluent
|
|
|
104
117
|
end
|
|
105
118
|
end
|
|
106
119
|
end
|
|
107
|
-
|
|
@@ -150,7 +150,7 @@ module Fluent
|
|
|
150
150
|
end
|
|
151
151
|
end
|
|
152
152
|
unless section_params.has_key?(proxy.argument.first)
|
|
153
|
-
logger.error "config error in:\n#{conf}" if logger # logger should exist, but
|
|
153
|
+
logger.error "config error in:\n#{conf}" if logger # logger should exist, but sometimes it's nil (e.g, in tests)
|
|
154
154
|
raise ConfigError, "'<#{proxy.name} ARG>' section requires argument" + section_stack
|
|
155
155
|
end
|
|
156
156
|
# argument should NOT be deprecated... (argument always has a value: '')
|
|
@@ -253,7 +253,7 @@ module Fluent
|
|
|
253
253
|
elems = conf.respond_to?(:elements) ? conf.elements : []
|
|
254
254
|
elems.each { |e|
|
|
255
255
|
next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doesn't have proxy section
|
|
256
|
-
next if e.unused_in
|
|
256
|
+
next if e.unused_in&.empty? # the section is used at least once
|
|
257
257
|
|
|
258
258
|
if proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
|
|
259
259
|
e.unused_in = []
|
data/lib/fluent/config/types.rb
CHANGED
|
@@ -71,7 +71,7 @@ module Fluent
|
|
|
71
71
|
else
|
|
72
72
|
# Current parser passes comment without actual values, e.g. "param #foo".
|
|
73
73
|
# parser should pass empty string in this case but changing behaviour may break existing environment so keep parser behaviour. Just ignore comment value in boolean handling for now.
|
|
74
|
-
if str.respond_to?(
|
|
74
|
+
if str.respond_to?(:start_with?) && str.start_with?('#')
|
|
75
75
|
true
|
|
76
76
|
elsif opts[:strict]
|
|
77
77
|
raise Fluent::ConfigError, "#{name}: invalid bool value: #{str}"
|
|
@@ -83,7 +83,7 @@ module Fluent
|
|
|
83
83
|
elsif skip(/\</)
|
|
84
84
|
e_name = scan(ELEMENT_NAME)
|
|
85
85
|
spacing
|
|
86
|
-
e_arg = scan_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
|
|
86
|
+
e_arg = scan_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/o)
|
|
87
87
|
spacing
|
|
88
88
|
unless skip(/\>/)
|
|
89
89
|
parse_error! "expected '>'"
|
|
@@ -98,7 +98,7 @@ module Fluent
|
|
|
98
98
|
new_e.v1_config = true
|
|
99
99
|
elems << new_e
|
|
100
100
|
|
|
101
|
-
elsif root_element && skip(/(\@include|include)#{SPACING}/)
|
|
101
|
+
elsif root_element && skip(/(\@include|include)#{SPACING}/o)
|
|
102
102
|
if !prev_match.start_with?('@')
|
|
103
103
|
@logger.warn "'include' is deprecated. Use '@include' instead" if @logger
|
|
104
104
|
end
|
|
@@ -172,7 +172,7 @@ module Fluent
|
|
|
172
172
|
require 'open-uri'
|
|
173
173
|
basepath = '/'
|
|
174
174
|
fname = path
|
|
175
|
-
data =
|
|
175
|
+
data = u.open { |f| f.read }
|
|
176
176
|
data.force_encoding('UTF-8')
|
|
177
177
|
ss = StringScanner.new(data)
|
|
178
178
|
V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
|
|
@@ -144,8 +144,12 @@ module Fluent
|
|
|
144
144
|
|
|
145
145
|
config.each do |key, val|
|
|
146
146
|
if val.is_a?(Array)
|
|
147
|
-
val.
|
|
148
|
-
|
|
147
|
+
if section?(val.first)
|
|
148
|
+
val.each do |v|
|
|
149
|
+
sb.add_section(section_build(key, v, indent: indent + @base_indent))
|
|
150
|
+
end
|
|
151
|
+
else
|
|
152
|
+
sb.add_line(key, val)
|
|
149
153
|
end
|
|
150
154
|
elsif val.is_a?(Hash)
|
|
151
155
|
harg = val.delete('$arg')
|
|
@@ -164,6 +168,10 @@ module Fluent
|
|
|
164
168
|
|
|
165
169
|
SectionBuilder.new(name, sb, indent, arg)
|
|
166
170
|
end
|
|
171
|
+
|
|
172
|
+
def section?(value)
|
|
173
|
+
value.is_a?(Array) or value.is_a?(Hash)
|
|
174
|
+
end
|
|
167
175
|
end
|
|
168
176
|
end
|
|
169
177
|
end
|
data/lib/fluent/counter/store.rb
CHANGED
|
@@ -156,7 +156,7 @@ module Fluent
|
|
|
156
156
|
}
|
|
157
157
|
end
|
|
158
158
|
|
|
159
|
-
# value is Hash. value requires these
|
|
159
|
+
# value is Hash. value requires these fields.
|
|
160
160
|
# :name, :total, :current, :type, :reset_interval, :last_reset_at, :last_modified_at
|
|
161
161
|
def build_value(data)
|
|
162
162
|
type = data['type'] || 'numeric'
|
data/lib/fluent/engine.rb
CHANGED
|
@@ -177,7 +177,7 @@ module Fluent
|
|
|
177
177
|
|
|
178
178
|
# @param conf [Fluent::Config]
|
|
179
179
|
# @param supervisor [Bool]
|
|
180
|
-
# @
|
|
180
|
+
# @return nil
|
|
181
181
|
def reload_config(conf, supervisor: false)
|
|
182
182
|
@root_agent_mutex.synchronize do
|
|
183
183
|
# configure first to reduce down time while restarting
|
data/lib/fluent/env.rb
CHANGED
|
@@ -21,6 +21,7 @@ require 'fluent/oj_options'
|
|
|
21
21
|
|
|
22
22
|
module Fluent
|
|
23
23
|
DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
|
|
24
|
+
DEFAULT_CONFIG_INCLUDE_DIR = ENV["FLUENT_CONF_INCLUDE_DIR"] || '/etc/fluent/conf.d'
|
|
24
25
|
DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
|
|
25
26
|
DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
|
|
26
27
|
DEFAULT_BACKUP_DIR = ENV['FLUENT_BACKUP_DIR'] || '/tmp/fluent'
|
|
@@ -34,10 +35,10 @@ module Fluent
|
|
|
34
35
|
end
|
|
35
36
|
|
|
36
37
|
def self.linux?
|
|
37
|
-
|
|
38
|
+
RUBY_PLATFORM.include?("linux")
|
|
38
39
|
end
|
|
39
40
|
|
|
40
41
|
def self.macos?
|
|
41
|
-
|
|
42
|
+
RUBY_PLATFORM.include?("darwin")
|
|
42
43
|
end
|
|
43
44
|
end
|
data/lib/fluent/event.rb
CHANGED
|
@@ -62,9 +62,9 @@ module Fluent
|
|
|
62
62
|
out.full_pack
|
|
63
63
|
end
|
|
64
64
|
|
|
65
|
-
def to_compressed_msgpack_stream(time_int: false, packer: nil)
|
|
65
|
+
def to_compressed_msgpack_stream(time_int: false, packer: nil, type: :gzip)
|
|
66
66
|
packed = to_msgpack_stream(time_int: time_int, packer: packer)
|
|
67
|
-
compress(packed)
|
|
67
|
+
compress(packed, type: type)
|
|
68
68
|
end
|
|
69
69
|
|
|
70
70
|
def to_msgpack_stream_forced_integer(packer: nil)
|
|
@@ -247,7 +247,7 @@ module Fluent
|
|
|
247
247
|
end
|
|
248
248
|
|
|
249
249
|
# This method returns MultiEventStream, because there are no reason
|
|
250
|
-
# to
|
|
250
|
+
# to survey binary serialized by msgpack.
|
|
251
251
|
def slice(index, num)
|
|
252
252
|
ensure_unpacked!
|
|
253
253
|
MultiEventStream.new(@unpacked_times.slice(index, num), @unpacked_records.slice(index, num))
|
|
@@ -268,10 +268,11 @@ module Fluent
|
|
|
268
268
|
end
|
|
269
269
|
|
|
270
270
|
class CompressedMessagePackEventStream < MessagePackEventStream
|
|
271
|
-
def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil)
|
|
272
|
-
super
|
|
271
|
+
def initialize(data, cached_unpacker = nil, size = 0, unpacked_times: nil, unpacked_records: nil, compress: :gzip)
|
|
272
|
+
super(data, cached_unpacker, size, unpacked_times: unpacked_times, unpacked_records: unpacked_records)
|
|
273
273
|
@decompressed_data = nil
|
|
274
274
|
@compressed_data = data
|
|
275
|
+
@type = compress
|
|
275
276
|
end
|
|
276
277
|
|
|
277
278
|
def empty?
|
|
@@ -303,7 +304,7 @@ module Fluent
|
|
|
303
304
|
|
|
304
305
|
def ensure_decompressed!
|
|
305
306
|
return if @decompressed_data
|
|
306
|
-
@data = @decompressed_data = decompress(@data)
|
|
307
|
+
@data = @decompressed_data = decompress(@data, type: @type)
|
|
307
308
|
end
|
|
308
309
|
end
|
|
309
310
|
|
|
@@ -19,11 +19,9 @@ require 'console'
|
|
|
19
19
|
module Fluent
|
|
20
20
|
class Log
|
|
21
21
|
# Async gem which is used by http_server helper switched logger mechanism to
|
|
22
|
-
# Console gem which isn't
|
|
22
|
+
# Console gem which isn't compatible with Ruby's standard Logger (since
|
|
23
23
|
# v1.17). This class adapts it to Fluentd's logger mechanism.
|
|
24
|
-
class ConsoleAdapter <
|
|
25
|
-
Console::Output::Terminal : Console::Terminal::Logger
|
|
26
|
-
|
|
24
|
+
class ConsoleAdapter < Console::Output::Terminal
|
|
27
25
|
def self.wrap(logger)
|
|
28
26
|
_, level = Console::Logger::LEVELS.find { |key, value|
|
|
29
27
|
if logger.level <= 0
|
|
@@ -58,10 +56,10 @@ module Fluent
|
|
|
58
56
|
level = 'warn'
|
|
59
57
|
end
|
|
60
58
|
|
|
61
|
-
@
|
|
62
|
-
@
|
|
59
|
+
@stream.seek(0)
|
|
60
|
+
@stream.truncate(0)
|
|
63
61
|
super
|
|
64
|
-
@logger.send(level, @
|
|
62
|
+
@logger.send(level, @stream.string.chomp)
|
|
65
63
|
end
|
|
66
64
|
end
|
|
67
65
|
end
|
data/lib/fluent/log.rb
CHANGED
|
@@ -138,6 +138,7 @@ module Fluent
|
|
|
138
138
|
@optional_attrs = nil
|
|
139
139
|
|
|
140
140
|
@suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
|
|
141
|
+
@forced_stacktrace_level = nil
|
|
141
142
|
@ignore_repeated_log_interval = opts[:ignore_repeated_log_interval]
|
|
142
143
|
@ignore_same_log_interval = opts[:ignore_same_log_interval]
|
|
143
144
|
|
|
@@ -173,6 +174,7 @@ module Fluent
|
|
|
173
174
|
clone.format = @format
|
|
174
175
|
clone.time_format = @time_format
|
|
175
176
|
clone.log_event_enabled = @log_event_enabled
|
|
177
|
+
clone.force_stacktrace_level(@forced_stacktrace_level)
|
|
176
178
|
# optional headers/attrs are not copied, because new PluginLogger should have another one of it
|
|
177
179
|
clone
|
|
178
180
|
end
|
|
@@ -240,6 +242,14 @@ module Fluent
|
|
|
240
242
|
nil
|
|
241
243
|
end
|
|
242
244
|
|
|
245
|
+
def force_stacktrace_level?
|
|
246
|
+
not @forced_stacktrace_level.nil?
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
def force_stacktrace_level(level)
|
|
250
|
+
@forced_stacktrace_level = level
|
|
251
|
+
end
|
|
252
|
+
|
|
243
253
|
def enable_debug(b=true)
|
|
244
254
|
@debug_mode = b
|
|
245
255
|
self
|
|
@@ -500,6 +510,16 @@ module Fluent
|
|
|
500
510
|
def dump_stacktrace(type, backtrace, level)
|
|
501
511
|
return if @level > level
|
|
502
512
|
|
|
513
|
+
dump_stacktrace_internal(
|
|
514
|
+
type,
|
|
515
|
+
backtrace,
|
|
516
|
+
force_stacktrace_level? ? @forced_stacktrace_level : level,
|
|
517
|
+
)
|
|
518
|
+
end
|
|
519
|
+
|
|
520
|
+
def dump_stacktrace_internal(type, backtrace, level)
|
|
521
|
+
return if @level > level
|
|
522
|
+
|
|
503
523
|
time = Time.now
|
|
504
524
|
|
|
505
525
|
if @format == :text
|
|
@@ -633,6 +653,9 @@ module Fluent
|
|
|
633
653
|
if logger.instance_variable_defined?(:@suppress_repeated_stacktrace)
|
|
634
654
|
@suppress_repeated_stacktrace = logger.instance_variable_get(:@suppress_repeated_stacktrace)
|
|
635
655
|
end
|
|
656
|
+
if logger.instance_variable_defined?(:@forced_stacktrace_level)
|
|
657
|
+
@forced_stacktrace_level = logger.instance_variable_get(:@forced_stacktrace_level)
|
|
658
|
+
end
|
|
636
659
|
if logger.instance_variable_defined?(:@ignore_repeated_log_interval)
|
|
637
660
|
@ignore_repeated_log_interval = logger.instance_variable_get(:@ignore_repeated_log_interval)
|
|
638
661
|
end
|
|
@@ -40,22 +40,6 @@ module Fluent
|
|
|
40
40
|
raise NotImplementedError, "BUG: output plugins MUST implement this method"
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
-
def num_errors
|
|
44
|
-
@num_errors_metrics.get
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
def emit_count
|
|
48
|
-
@emit_count_metrics.get
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
def emit_size
|
|
52
|
-
@emit_size_metrics.get
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
def emit_records
|
|
56
|
-
@emit_records_metrics.get
|
|
57
|
-
end
|
|
58
|
-
|
|
59
43
|
def initialize
|
|
60
44
|
super
|
|
61
45
|
@counter_mutex = Mutex.new
|
data/lib/fluent/plugin/base.rb
CHANGED
|
@@ -84,7 +84,7 @@ module Fluent
|
|
|
84
84
|
yield
|
|
85
85
|
end
|
|
86
86
|
# Update access time to prevent tmpwatch from deleting a lock file.
|
|
87
|
-
FileUtils.touch(lock_path)
|
|
87
|
+
FileUtils.touch(lock_path)
|
|
88
88
|
end
|
|
89
89
|
|
|
90
90
|
def string_safe_encoding(str)
|
|
@@ -206,7 +206,7 @@ module Fluent
|
|
|
206
206
|
end
|
|
207
207
|
|
|
208
208
|
def reloadable_plugin?
|
|
209
|
-
# Engine can't capture all class variables. so it's
|
|
209
|
+
# Engine can't capture all class variables. so it's forbidden to use class variables in each plugins if enabling reload.
|
|
210
210
|
self.class.class_variables.empty?
|
|
211
211
|
end
|
|
212
212
|
end
|
|
@@ -191,7 +191,7 @@ module Fluent
|
|
|
191
191
|
queue.sort_by!{ |chunk| chunk.modified_at }
|
|
192
192
|
|
|
193
193
|
# If one of the files is corrupted, other files may also be corrupted and be undetected.
|
|
194
|
-
# The time
|
|
194
|
+
# The time periods of each chunk are helpful to check the data.
|
|
195
195
|
if exist_broken_file
|
|
196
196
|
log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
|
|
197
197
|
(stage.values + queue).each { |chunk|
|
|
@@ -229,6 +229,20 @@ module Fluent
|
|
|
229
229
|
File.unlink(path, path + '.meta') rescue nil
|
|
230
230
|
end
|
|
231
231
|
|
|
232
|
+
def evacuate_chunk(chunk)
|
|
233
|
+
unless chunk.is_a?(Fluent::Plugin::Buffer::FileChunk)
|
|
234
|
+
raise ArgumentError, "The chunk must be FileChunk, but it was #{chunk.class}."
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id)
|
|
238
|
+
FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
|
|
239
|
+
|
|
240
|
+
FileUtils.copy([chunk.path, chunk.meta_path], backup_dir)
|
|
241
|
+
log.warn "chunk files are evacuated to #{backup_dir}.", chunk_id: dump_unique_id_hex(chunk.unique_id)
|
|
242
|
+
rescue => e
|
|
243
|
+
log.error "unexpected error while evacuating chunk files.", error: e
|
|
244
|
+
end
|
|
245
|
+
|
|
232
246
|
private
|
|
233
247
|
|
|
234
248
|
def escaped_patterns(patterns)
|
|
@@ -202,7 +202,7 @@ module Fluent
|
|
|
202
202
|
queue.sort_by!(&:modified_at)
|
|
203
203
|
|
|
204
204
|
# If one of the files is corrupted, other files may also be corrupted and be undetected.
|
|
205
|
-
# The time
|
|
205
|
+
# The time periods of each chunk are helpful to check the data.
|
|
206
206
|
if exist_broken_file
|
|
207
207
|
log.info "Since a broken chunk file was found, it is possible that other files remaining at the time of resuming were also broken. Here is the list of the files."
|
|
208
208
|
(stage.values + queue).each { |chunk|
|
|
@@ -241,6 +241,20 @@ module Fluent
|
|
|
241
241
|
File.unlink(path) rescue nil
|
|
242
242
|
end
|
|
243
243
|
|
|
244
|
+
def evacuate_chunk(chunk)
|
|
245
|
+
unless chunk.is_a?(Fluent::Plugin::Buffer::FileSingleChunk)
|
|
246
|
+
raise ArgumentError, "The chunk must be FileSingleChunk, but it was #{chunk.class}."
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
backup_dir = File.join(backup_base_dir, 'buffer', safe_owner_id)
|
|
250
|
+
FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
|
|
251
|
+
|
|
252
|
+
FileUtils.copy(chunk.path, backup_dir)
|
|
253
|
+
log.warn "chunk files are evacuated to #{backup_dir}.", chunk_id: dump_unique_id_hex(chunk.unique_id)
|
|
254
|
+
rescue => e
|
|
255
|
+
log.error "unexpected error while evacuating chunk files.", error: e
|
|
256
|
+
end
|
|
257
|
+
|
|
244
258
|
private
|
|
245
259
|
|
|
246
260
|
def escaped_patterns(patterns)
|
|
@@ -59,8 +59,11 @@ module Fluent
|
|
|
59
59
|
@size = 0
|
|
60
60
|
@created_at = Fluent::Clock.real_now
|
|
61
61
|
@modified_at = Fluent::Clock.real_now
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
if compress == :gzip
|
|
63
|
+
extend GzipDecompressable
|
|
64
|
+
elsif compress == :zstd
|
|
65
|
+
extend ZstdDecompressable
|
|
66
|
+
end
|
|
64
67
|
end
|
|
65
68
|
|
|
66
69
|
attr_reader :unique_id, :metadata, :state
|
|
@@ -85,10 +88,17 @@ module Fluent
|
|
|
85
88
|
|
|
86
89
|
# data is array of formatted record string
|
|
87
90
|
def append(data, **kwargs)
|
|
88
|
-
raise ArgumentError,
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
91
|
+
raise ArgumentError, "`compress: #{kwargs[:compress]}` can be used for Compressable module" if kwargs[:compress] == :gzip || kwargs[:compress] == :zstd
|
|
92
|
+
begin
|
|
93
|
+
adding = data.join.force_encoding(Encoding::ASCII_8BIT)
|
|
94
|
+
rescue
|
|
95
|
+
# Fallback
|
|
96
|
+
# Array#join throws an exception if data contains strings with a different encoding.
|
|
97
|
+
# Although such cases may be rare, it should be considered as a safety precaution.
|
|
98
|
+
adding = ''.force_encoding(Encoding::ASCII_8BIT)
|
|
99
|
+
data.each do |d|
|
|
100
|
+
adding << d.b
|
|
101
|
+
end
|
|
92
102
|
end
|
|
93
103
|
concat(adding, data.size)
|
|
94
104
|
end
|
|
@@ -165,23 +175,23 @@ module Fluent
|
|
|
165
175
|
end
|
|
166
176
|
|
|
167
177
|
def read(**kwargs)
|
|
168
|
-
raise ArgumentError,
|
|
178
|
+
raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
|
|
169
179
|
raise NotImplementedError, "Implement this method in child class"
|
|
170
180
|
end
|
|
171
181
|
|
|
172
182
|
def open(**kwargs, &block)
|
|
173
|
-
raise ArgumentError,
|
|
183
|
+
raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
|
|
174
184
|
raise NotImplementedError, "Implement this method in child class"
|
|
175
185
|
end
|
|
176
186
|
|
|
177
187
|
def write_to(io, **kwargs)
|
|
178
|
-
raise ArgumentError,
|
|
188
|
+
raise ArgumentError, "`compressed: #{kwargs[:compressed]}` can be used for Compressable module" if kwargs[:compressed] == :gzip || kwargs[:compressed] == :zstd
|
|
179
189
|
open do |i|
|
|
180
190
|
IO.copy_stream(i, io)
|
|
181
191
|
end
|
|
182
192
|
end
|
|
183
193
|
|
|
184
|
-
module
|
|
194
|
+
module GzipDecompressable
|
|
185
195
|
include Fluent::Plugin::Compressable
|
|
186
196
|
|
|
187
197
|
def append(data, **kwargs)
|
|
@@ -234,6 +244,60 @@ module Fluent
|
|
|
234
244
|
end
|
|
235
245
|
end
|
|
236
246
|
end
|
|
247
|
+
|
|
248
|
+
module ZstdDecompressable
|
|
249
|
+
include Fluent::Plugin::Compressable
|
|
250
|
+
|
|
251
|
+
def append(data, **kwargs)
|
|
252
|
+
if kwargs[:compress] == :zstd
|
|
253
|
+
io = StringIO.new
|
|
254
|
+
stream = Zstd::StreamWriter.new(io)
|
|
255
|
+
data.each do |d|
|
|
256
|
+
stream.write(d)
|
|
257
|
+
end
|
|
258
|
+
stream.finish
|
|
259
|
+
concat(io.string, data.size)
|
|
260
|
+
else
|
|
261
|
+
super
|
|
262
|
+
end
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
def open(**kwargs, &block)
|
|
266
|
+
if kwargs[:compressed] == :zstd
|
|
267
|
+
super
|
|
268
|
+
else
|
|
269
|
+
super(**kwargs) do |chunk_io|
|
|
270
|
+
output_io = if chunk_io.is_a?(StringIO)
|
|
271
|
+
StringIO.new
|
|
272
|
+
else
|
|
273
|
+
Tempfile.new('decompressed-data')
|
|
274
|
+
end
|
|
275
|
+
output_io.binmode if output_io.is_a?(Tempfile)
|
|
276
|
+
decompress(input_io: chunk_io, output_io: output_io, type: :zstd)
|
|
277
|
+
output_io.seek(0, IO::SEEK_SET)
|
|
278
|
+
yield output_io
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
def read(**kwargs)
|
|
284
|
+
if kwargs[:compressed] == :zstd
|
|
285
|
+
super
|
|
286
|
+
else
|
|
287
|
+
decompress(super,type: :zstd)
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def write_to(io, **kwargs)
|
|
292
|
+
open(compressed: :zstd) do |chunk_io|
|
|
293
|
+
if kwargs[:compressed] == :zstd
|
|
294
|
+
IO.copy_stream(chunk_io, io)
|
|
295
|
+
else
|
|
296
|
+
decompress(input_io: chunk_io, output_io: io, type: :zstd)
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
end
|
|
300
|
+
end
|
|
237
301
|
end
|
|
238
302
|
end
|
|
239
303
|
end
|
|
@@ -37,7 +37,7 @@ module Fluent
|
|
|
37
37
|
# path_prefix: path prefix string, ended with '.'
|
|
38
38
|
# path_suffix: path suffix string, like '.log' (or any other user specified)
|
|
39
39
|
|
|
40
|
-
attr_reader :path, :permission
|
|
40
|
+
attr_reader :path, :meta_path, :permission
|
|
41
41
|
|
|
42
42
|
def initialize(metadata, path, mode, perm: nil, compress: :text)
|
|
43
43
|
super(metadata, compress: compress)
|
|
@@ -219,13 +219,17 @@ module Fluent
|
|
|
219
219
|
# old type of restore
|
|
220
220
|
data = Fluent::MessagePackFactory.msgpack_unpacker(symbolize_keys: true).feed(bindata).read rescue {}
|
|
221
221
|
end
|
|
222
|
+
raise FileChunkError, "invalid meta data" if data.nil? || !data.is_a?(Hash)
|
|
223
|
+
raise FileChunkError, "invalid unique_id" unless data[:id]
|
|
224
|
+
raise FileChunkError, "invalid created_at" unless data[:c].to_i > 0
|
|
225
|
+
raise FileChunkError, "invalid modified_at" unless data[:m].to_i > 0
|
|
222
226
|
|
|
223
227
|
now = Fluent::Clock.real_now
|
|
224
228
|
|
|
225
|
-
@unique_id = data[:id]
|
|
229
|
+
@unique_id = data[:id]
|
|
226
230
|
@size = data[:s] || 0
|
|
227
|
-
@created_at = data
|
|
228
|
-
@modified_at = data
|
|
231
|
+
@created_at = data[:c]
|
|
232
|
+
@modified_at = data[:m]
|
|
229
233
|
|
|
230
234
|
@metadata.timekey = data[:timekey]
|
|
231
235
|
@metadata.tag = data[:tag]
|
|
@@ -285,7 +289,7 @@ module Fluent
|
|
|
285
289
|
@chunk.binmode
|
|
286
290
|
rescue => e
|
|
287
291
|
# Here assumes "Too many open files" like recoverable error so raising BufferOverflowError.
|
|
288
|
-
# If other cases are possible, we will change
|
|
292
|
+
# If other cases are possible, we will change error handling with proper classes.
|
|
289
293
|
raise BufferOverflowError, "can't create buffer file for #{path}. Stop creating buffer files: error = #{e}"
|
|
290
294
|
end
|
|
291
295
|
begin
|
|
@@ -243,11 +243,11 @@ module Fluent
|
|
|
243
243
|
def encode_key(metadata)
|
|
244
244
|
k = @key ? metadata.variables[@key] : metadata.tag
|
|
245
245
|
k ||= ''
|
|
246
|
-
URI::
|
|
246
|
+
URI::RFC2396_PARSER.escape(k, ESCAPE_REGEXP)
|
|
247
247
|
end
|
|
248
248
|
|
|
249
249
|
def decode_key(key)
|
|
250
|
-
URI::
|
|
250
|
+
URI::RFC2396_PARSER.unescape(key)
|
|
251
251
|
end
|
|
252
252
|
|
|
253
253
|
def create_new_chunk(path, metadata, perm)
|
|
@@ -259,7 +259,7 @@ module Fluent
|
|
|
259
259
|
@chunk.binmode
|
|
260
260
|
rescue => e
|
|
261
261
|
# Here assumes "Too many open files" like recoverable error so raising BufferOverflowError.
|
|
262
|
-
# If other cases are possible, we will change
|
|
262
|
+
# If other cases are possible, we will change error handling with proper classes.
|
|
263
263
|
raise BufferOverflowError, "can't create buffer file for #{path}. Stop creating buffer files: error = #{e}"
|
|
264
264
|
end
|
|
265
265
|
|
|
@@ -68,13 +68,13 @@ module Fluent
|
|
|
68
68
|
|
|
69
69
|
def purge
|
|
70
70
|
super
|
|
71
|
-
@chunk
|
|
71
|
+
@chunk.clear
|
|
72
72
|
@chunk_bytes = @size = @adding_bytes = @adding_size = 0
|
|
73
73
|
true
|
|
74
74
|
end
|
|
75
75
|
|
|
76
76
|
def read(**kwargs)
|
|
77
|
-
@chunk
|
|
77
|
+
@chunk.dup
|
|
78
78
|
end
|
|
79
79
|
|
|
80
80
|
def open(**kwargs, &block)
|