fluentd 1.15.2-x86-mingw32 → 1.16.1-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
- data/.github/workflows/linux-test.yaml +2 -2
- data/.github/workflows/macos-test.yaml +2 -2
- data/.github/workflows/stale-actions.yml +11 -9
- data/.github/workflows/windows-test.yaml +2 -2
- data/CHANGELOG.md +133 -0
- data/CONTRIBUTING.md +1 -1
- data/MAINTAINERS.md +5 -3
- data/README.md +0 -1
- data/SECURITY.md +5 -9
- data/fluentd.gemspec +2 -2
- data/lib/fluent/command/fluentd.rb +55 -64
- data/lib/fluent/config/yaml_parser/loader.rb +18 -1
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/event.rb +2 -2
- data/lib/fluent/file_wrapper.rb +137 -0
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +35 -5
- data/lib/fluent/oj_options.rb +1 -2
- data/lib/fluent/plugin/base.rb +5 -7
- data/lib/fluent/plugin/buf_file.rb +32 -3
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +21 -0
- data/lib/fluent/plugin/in_tail.rb +1 -6
- data/lib/fluent/plugin/in_tcp.rb +47 -2
- data/lib/fluent/plugin/out_file.rb +0 -4
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward.rb +2 -2
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +49 -12
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/plugin_helper/server.rb +8 -0
- data/lib/fluent/supervisor.rb +157 -232
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/test/helper.rb.erb +0 -1
- data/test/command/test_ctl.rb +1 -1
- data/test/command/test_fluentd.rb +168 -22
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +5 -5
- data/test/config/test_system_config.rb +0 -8
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/test_base.rb +98 -0
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_in_http.rb +2 -3
- data/test/plugin/test_in_monitor_agent.rb +2 -3
- data/test/plugin/test_in_tail.rb +105 -103
- data/test/plugin/test_in_tcp.rb +87 -2
- data/test/plugin/test_in_udp.rb +28 -0
- data/test/plugin/test_out_file.rb +3 -2
- data/test/plugin/test_out_forward.rb +14 -18
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_output.rb +269 -0
- data/test/plugin/test_output_as_buffered_compress.rb +32 -18
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_server.rb +59 -5
- data/test/test_config.rb +57 -21
- data/test/{plugin/test_file_wrapper.rb → test_file_wrapper.rb} +2 -2
- data/test/test_formatter.rb +23 -20
- data/test/test_log.rb +85 -40
- data/test/test_supervisor.rb +300 -283
- metadata +15 -23
- data/.drone.yml +0 -35
- data/.github/workflows/issue-auto-closer.yml +0 -12
- data/.gitlab-ci.yml +0 -103
- data/lib/fluent/plugin/file_wrapper.rb +0 -131
- data/test/test_logger_initializer.rb +0 -46
@@ -59,7 +59,13 @@ module Fluent::Plugin
|
|
59
59
|
@ack_waitings = new_list
|
60
60
|
end
|
61
61
|
|
62
|
-
|
62
|
+
begin
|
63
|
+
readable_sockets, _, _ = IO.select(sockets, nil, nil, select_interval)
|
64
|
+
rescue IOError
|
65
|
+
@log.info "connection closed while waiting for readable sockets"
|
66
|
+
readable_sockets = nil
|
67
|
+
end
|
68
|
+
|
63
69
|
if readable_sockets
|
64
70
|
readable_sockets.each do |sock|
|
65
71
|
results << read_ack_from_sock(sock)
|
@@ -109,13 +115,22 @@ module Fluent::Plugin
|
|
109
115
|
raw_data = sock.instance_of?(Fluent::PluginHelper::Socket::WrappedSocket::TLS) ? sock.readpartial(@read_length) : sock.recv(@read_length)
|
110
116
|
rescue Errno::ECONNRESET, EOFError # ECONNRESET for #recv, #EOFError for #readpartial
|
111
117
|
raw_data = ''
|
118
|
+
rescue IOError
|
119
|
+
@log.info "socket closed while receiving ack response"
|
120
|
+
return nil, Result::FAILED
|
112
121
|
end
|
113
122
|
|
114
123
|
info = find(sock)
|
115
124
|
|
116
|
-
|
117
|
-
|
118
|
-
|
125
|
+
if info.nil?
|
126
|
+
# The info can be deleted by another thread during `sock.recv()` and `find()`.
|
127
|
+
# This is OK since another thread has completed to process the ack, so we can skip this.
|
128
|
+
# Note: exclusion mechanism about `collect_response()` may need to be considered.
|
129
|
+
@log.debug "could not find the ack info. this ack may be processed by another thread."
|
130
|
+
return nil, Result::FAILED
|
131
|
+
elsif raw_data.empty?
|
132
|
+
# When connection is closed by remote host, socket is ready to read and #recv returns an empty string that means EOF.
|
133
|
+
# If this happens we assume the data wasn't delivered and retry it.
|
119
134
|
@log.warn 'destination node closed the connection. regard it as unavailable.', host: info.node.host, port: info.node.port
|
120
135
|
# info.node.disable!
|
121
136
|
return info, Result::FAILED
|
@@ -521,8 +521,8 @@ module Fluent::Plugin
|
|
521
521
|
when AckHandler::Result::SUCCESS
|
522
522
|
commit_write(chunk_id)
|
523
523
|
when AckHandler::Result::FAILED
|
524
|
-
node
|
525
|
-
rollback_write(chunk_id, update_retry: false)
|
524
|
+
node&.disable!
|
525
|
+
rollback_write(chunk_id, update_retry: false) if chunk_id
|
526
526
|
when AckHandler::Result::CHUNKID_UNMATCHED
|
527
527
|
rollback_write(chunk_id, update_retry: false)
|
528
528
|
else
|
@@ -64,31 +64,29 @@ module Fluent::Plugin
|
|
64
64
|
end
|
65
65
|
|
66
66
|
def multi_workers_ready?
|
67
|
-
### TODO: add hack to synchronize for multi workers
|
68
67
|
true
|
69
68
|
end
|
70
69
|
|
71
70
|
def write(chunk)
|
72
71
|
path_without_suffix = extract_placeholders(@path_without_suffix, chunk)
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
72
|
+
generate_path(path_without_suffix) do |path|
|
73
|
+
FileUtils.mkdir_p File.dirname(path), mode: @dir_perm
|
74
|
+
|
75
|
+
case @compress
|
76
|
+
when :text
|
77
|
+
File.open(path, "ab", @file_perm) {|f|
|
78
|
+
f.flock(File::LOCK_EX)
|
79
|
+
chunk.write_to(f)
|
80
|
+
}
|
81
|
+
when :gzip
|
82
|
+
File.open(path, "ab", @file_perm) {|f|
|
83
|
+
f.flock(File::LOCK_EX)
|
84
|
+
gz = Zlib::GzipWriter.new(f)
|
85
|
+
chunk.write_to(gz)
|
86
|
+
gz.close
|
87
|
+
}
|
88
|
+
end
|
89
89
|
end
|
90
|
-
|
91
|
-
path
|
92
90
|
end
|
93
91
|
|
94
92
|
private
|
@@ -117,15 +115,34 @@ module Fluent::Plugin
|
|
117
115
|
|
118
116
|
def generate_path(path_without_suffix)
|
119
117
|
if @append
|
120
|
-
"#{path_without_suffix}#{@suffix}"
|
121
|
-
|
118
|
+
path = "#{path_without_suffix}#{@suffix}"
|
119
|
+
synchronize_path(path) do
|
120
|
+
yield path
|
121
|
+
end
|
122
|
+
return path
|
123
|
+
end
|
124
|
+
|
125
|
+
begin
|
122
126
|
i = 0
|
123
127
|
loop do
|
124
128
|
path = "#{path_without_suffix}.#{i}#{@suffix}"
|
125
|
-
|
129
|
+
break unless File.exist?(path)
|
126
130
|
i += 1
|
127
131
|
end
|
132
|
+
synchronize_path(path) do
|
133
|
+
# If multiple processes or threads select the same path and another
|
134
|
+
# one entered this locking block first, the file should already
|
135
|
+
# exist and this one should retry to find new path.
|
136
|
+
raise FileAlreadyExist if File.exist?(path)
|
137
|
+
yield path
|
138
|
+
end
|
139
|
+
rescue FileAlreadyExist
|
140
|
+
retry
|
128
141
|
end
|
142
|
+
path
|
143
|
+
end
|
144
|
+
|
145
|
+
class FileAlreadyExist < StandardError
|
129
146
|
end
|
130
147
|
end
|
131
148
|
end
|
data/lib/fluent/plugin/output.rb
CHANGED
@@ -99,7 +99,6 @@ module Fluent
|
|
99
99
|
config_param :retry_max_interval, :time, default: nil, desc: 'The maximum interval seconds for exponential backoff between retries while failing.'
|
100
100
|
|
101
101
|
config_param :retry_randomize, :bool, default: true, desc: 'If true, output plugin will retry after randomized interval not to do burst retries.'
|
102
|
-
config_param :disable_chunk_backup, :bool, default: false, desc: 'If true, chunks are thrown away when unrecoverable error happens'
|
103
102
|
end
|
104
103
|
|
105
104
|
config_section :secondary, param_name: :secondary_config, required: false, multi: false, final: true do
|
@@ -199,6 +198,7 @@ module Fluent
|
|
199
198
|
def initialize
|
200
199
|
super
|
201
200
|
@counter_mutex = Mutex.new
|
201
|
+
@flush_thread_mutex = Mutex.new
|
202
202
|
@buffering = false
|
203
203
|
@delayed_commit = false
|
204
204
|
@as_secondary = false
|
@@ -378,6 +378,7 @@ module Fluent
|
|
378
378
|
buffer_conf = conf.elements(name: 'buffer').first || Fluent::Config::Element.new('buffer', '', {}, [])
|
379
379
|
@buffer = Plugin.new_buffer(buffer_type, parent: self)
|
380
380
|
@buffer.configure(buffer_conf)
|
381
|
+
keep_buffer_config_compat
|
381
382
|
@buffer.enable_update_timekeys if @chunk_key_time
|
382
383
|
|
383
384
|
@flush_at_shutdown = @buffer_config.flush_at_shutdown
|
@@ -425,7 +426,9 @@ module Fluent
|
|
425
426
|
end
|
426
427
|
@secondary.acts_as_secondary(self)
|
427
428
|
@secondary.configure(secondary_conf)
|
428
|
-
if (
|
429
|
+
if (@secondary.class.to_s != "Fluent::Plugin::SecondaryFileOutput") &&
|
430
|
+
(self.class != @secondary.class) &&
|
431
|
+
(@custom_format || @secondary.implement?(:custom_format))
|
429
432
|
log.warn "Use different plugin for secondary. Check the plugin works with primary like secondary_file", primary: self.class.to_s, secondary: @secondary.class.to_s
|
430
433
|
end
|
431
434
|
else
|
@@ -435,6 +438,12 @@ module Fluent
|
|
435
438
|
self
|
436
439
|
end
|
437
440
|
|
441
|
+
def keep_buffer_config_compat
|
442
|
+
# Need this to call `@buffer_config.disable_chunk_backup` just as before,
|
443
|
+
# since some plugins may use this option in this way.
|
444
|
+
@buffer_config[:disable_chunk_backup] = @buffer.disable_chunk_backup
|
445
|
+
end
|
446
|
+
|
438
447
|
def start
|
439
448
|
super
|
440
449
|
|
@@ -591,6 +600,42 @@ module Fluent
|
|
591
600
|
super
|
592
601
|
end
|
593
602
|
|
603
|
+
def actual_flush_thread_count
|
604
|
+
return 0 unless @buffering
|
605
|
+
return @buffer_config.flush_thread_count unless @as_secondary
|
606
|
+
@primary_instance.buffer_config.flush_thread_count
|
607
|
+
end
|
608
|
+
|
609
|
+
# Ensures `path` (filename or filepath) processable
|
610
|
+
# only by the current thread in the current process.
|
611
|
+
# For multiple workers, the lock is shared if `path` is the same value.
|
612
|
+
# For multiple threads, the lock is shared by all threads in the same process.
|
613
|
+
def synchronize_path(path)
|
614
|
+
synchronize_path_in_workers(path) do
|
615
|
+
synchronize_in_threads do
|
616
|
+
yield
|
617
|
+
end
|
618
|
+
end
|
619
|
+
end
|
620
|
+
|
621
|
+
def synchronize_path_in_workers(path)
|
622
|
+
need_worker_lock = system_config.workers > 1
|
623
|
+
if need_worker_lock
|
624
|
+
acquire_worker_lock(path) { yield }
|
625
|
+
else
|
626
|
+
yield
|
627
|
+
end
|
628
|
+
end
|
629
|
+
|
630
|
+
def synchronize_in_threads
|
631
|
+
need_thread_lock = actual_flush_thread_count > 1
|
632
|
+
if need_thread_lock
|
633
|
+
@flush_thread_mutex.synchronize { yield }
|
634
|
+
else
|
635
|
+
yield
|
636
|
+
end
|
637
|
+
end
|
638
|
+
|
594
639
|
def support_in_v12_style?(feature)
|
595
640
|
# for plugins written in v0.12 styles
|
596
641
|
case feature
|
@@ -1240,18 +1285,10 @@ module Fluent
|
|
1240
1285
|
end
|
1241
1286
|
|
1242
1287
|
def backup_chunk(chunk, using_secondary, delayed_commit)
|
1243
|
-
if @
|
1288
|
+
if @buffer.disable_chunk_backup
|
1244
1289
|
log.warn "disable_chunk_backup is true. #{dump_unique_id_hex(chunk.unique_id)} chunk is thrown away"
|
1245
1290
|
else
|
1246
|
-
|
1247
|
-
safe_plugin_id = plugin_id.gsub(/[ "\/\\:;|*<>?]/, '_')
|
1248
|
-
backup_base_dir = system_config.root_dir || DEFAULT_BACKUP_DIR
|
1249
|
-
backup_file = File.join(backup_base_dir, 'backup', "worker#{fluentd_worker_id}", safe_plugin_id, "#{unique_id}.log")
|
1250
|
-
backup_dir = File.dirname(backup_file)
|
1251
|
-
|
1252
|
-
log.warn "bad chunk is moved to #{backup_file}"
|
1253
|
-
FileUtils.mkdir_p(backup_dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(backup_dir)
|
1254
|
-
File.open(backup_file, 'ab', system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION) { |f|
|
1291
|
+
@buffer.backup(chunk.unique_id) { |f|
|
1255
1292
|
chunk.write_to(f)
|
1256
1293
|
}
|
1257
1294
|
end
|
@@ -21,6 +21,7 @@ require 'async/http/endpoint'
|
|
21
21
|
require 'fluent/plugin_helper/http_server/app'
|
22
22
|
require 'fluent/plugin_helper/http_server/router'
|
23
23
|
require 'fluent/plugin_helper/http_server/methods'
|
24
|
+
require 'fluent/log/console_adapter'
|
24
25
|
|
25
26
|
module Fluent
|
26
27
|
module PluginHelper
|
@@ -38,7 +39,7 @@ module Fluent
|
|
38
39
|
scheme = tls_context ? 'https' : 'http'
|
39
40
|
@uri = URI("#{scheme}://#{@addr}:#{@port}").to_s
|
40
41
|
@router = Router.new(default_app)
|
41
|
-
@reactor = Async::Reactor.new(nil, logger: @logger)
|
42
|
+
@reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(@logger))
|
42
43
|
|
43
44
|
opts = if tls_context
|
44
45
|
{ ssl_context: tls_context }
|
@@ -545,6 +545,10 @@ module Fluent
|
|
545
545
|
data = @sock.recv(@max_bytes, @flags)
|
546
546
|
rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
|
547
547
|
return
|
548
|
+
rescue Errno::EMSGSIZE
|
549
|
+
# Windows ONLY: This happens when the data size is larger than `@max_bytes`.
|
550
|
+
@log.info "A received data was ignored since it was too large."
|
551
|
+
return
|
548
552
|
end
|
549
553
|
@callback.call(data)
|
550
554
|
rescue => e
|
@@ -558,6 +562,10 @@ module Fluent
|
|
558
562
|
data, addr = @sock.recvfrom(@max_bytes)
|
559
563
|
rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
|
560
564
|
return
|
565
|
+
rescue Errno::EMSGSIZE
|
566
|
+
# Windows ONLY: This happens when the data size is larger than `@max_bytes`.
|
567
|
+
@log.info "A received data was ignored since it was too large."
|
568
|
+
return
|
561
569
|
end
|
562
570
|
@callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
|
563
571
|
rescue => e
|