fluentd 1.15.3-x86-mingw32 → 1.16.2-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.yaml +1 -0
- data/.github/ISSUE_TEMPLATE/feature_request.yaml +1 -0
- data/.github/workflows/linux-test.yaml +2 -2
- data/.github/workflows/macos-test.yaml +2 -2
- data/.github/workflows/stale-actions.yml +24 -0
- data/.github/workflows/windows-test.yaml +2 -2
- data/CHANGELOG.md +151 -0
- data/CONTRIBUTING.md +1 -1
- data/MAINTAINERS.md +5 -3
- data/README.md +0 -1
- data/SECURITY.md +5 -9
- data/fluentd.gemspec +3 -3
- data/lib/fluent/command/ctl.rb +2 -2
- data/lib/fluent/command/fluentd.rb +55 -53
- data/lib/fluent/command/plugin_config_formatter.rb +1 -1
- data/lib/fluent/config/dsl.rb +1 -1
- data/lib/fluent/config/v1_parser.rb +2 -2
- data/lib/fluent/counter/server.rb +1 -1
- data/lib/fluent/counter/validator.rb +3 -3
- data/lib/fluent/daemon.rb +2 -4
- data/lib/fluent/engine.rb +1 -1
- data/lib/fluent/event.rb +8 -4
- data/lib/fluent/log/console_adapter.rb +66 -0
- data/lib/fluent/log.rb +44 -5
- data/lib/fluent/match.rb +1 -1
- data/lib/fluent/msgpack_factory.rb +6 -1
- data/lib/fluent/plugin/base.rb +6 -8
- data/lib/fluent/plugin/buf_file.rb +32 -3
- data/lib/fluent/plugin/buf_file_single.rb +32 -3
- data/lib/fluent/plugin/buffer/file_chunk.rb +1 -1
- data/lib/fluent/plugin/buffer.rb +21 -0
- data/lib/fluent/plugin/filter_record_transformer.rb +1 -1
- data/lib/fluent/plugin/in_forward.rb +1 -1
- data/lib/fluent/plugin/in_http.rb +8 -8
- data/lib/fluent/plugin/in_sample.rb +1 -1
- data/lib/fluent/plugin/in_tail/position_file.rb +32 -18
- data/lib/fluent/plugin/in_tail.rb +58 -24
- data/lib/fluent/plugin/in_tcp.rb +47 -2
- data/lib/fluent/plugin/out_exec_filter.rb +2 -2
- data/lib/fluent/plugin/out_forward/ack_handler.rb +19 -4
- data/lib/fluent/plugin/out_forward.rb +2 -2
- data/lib/fluent/plugin/out_secondary_file.rb +39 -22
- data/lib/fluent/plugin/output.rb +50 -13
- data/lib/fluent/plugin/parser_json.rb +1 -1
- data/lib/fluent/plugin_helper/event_loop.rb +2 -2
- data/lib/fluent/plugin_helper/http_server/server.rb +2 -1
- data/lib/fluent/plugin_helper/record_accessor.rb +1 -1
- data/lib/fluent/plugin_helper/server.rb +8 -0
- data/lib/fluent/plugin_helper/thread.rb +3 -3
- data/lib/fluent/plugin_id.rb +1 -1
- data/lib/fluent/supervisor.rb +157 -251
- data/lib/fluent/test/driver/base.rb +11 -5
- data/lib/fluent/test/driver/filter.rb +4 -0
- data/lib/fluent/test/startup_shutdown.rb +6 -8
- data/lib/fluent/version.rb +1 -1
- data/templates/new_gem/test/helper.rb.erb +0 -1
- data/test/command/test_ctl.rb +1 -1
- data/test/command/test_fluentd.rb +137 -6
- data/test/command/test_plugin_config_formatter.rb +0 -1
- data/test/compat/test_parser.rb +5 -5
- data/test/config/test_system_config.rb +0 -8
- data/test/log/test_console_adapter.rb +110 -0
- data/test/plugin/in_tail/test_position_file.rb +31 -1
- data/test/plugin/out_forward/test_ack_handler.rb +39 -0
- data/test/plugin/test_base.rb +99 -1
- data/test/plugin/test_buf_file.rb +62 -23
- data/test/plugin/test_buf_file_single.rb +65 -0
- data/test/plugin/test_buffer_chunk.rb +11 -0
- data/test/plugin/test_in_forward.rb +9 -9
- data/test/plugin/test_in_http.rb +2 -3
- data/test/plugin/test_in_monitor_agent.rb +2 -3
- data/test/plugin/test_in_tail.rb +379 -0
- data/test/plugin/test_in_tcp.rb +87 -2
- data/test/plugin/test_in_udp.rb +28 -0
- data/test/plugin/test_in_unix.rb +2 -2
- data/test/plugin/test_multi_output.rb +1 -1
- data/test/plugin/test_out_exec_filter.rb +2 -2
- data/test/plugin/test_out_file.rb +2 -2
- data/test/plugin/test_out_forward.rb +14 -18
- data/test/plugin/test_out_http.rb +1 -0
- data/test/plugin/test_output.rb +281 -12
- data/test/plugin/test_output_as_buffered.rb +44 -44
- data/test/plugin/test_output_as_buffered_compress.rb +32 -18
- data/test/plugin/test_output_as_buffered_retries.rb +1 -1
- data/test/plugin/test_output_as_buffered_secondary.rb +2 -2
- data/test/plugin/test_parser_regexp.rb +1 -6
- data/test/plugin_helper/test_child_process.rb +2 -2
- data/test/plugin_helper/test_http_server_helper.rb +1 -1
- data/test/plugin_helper/test_server.rb +60 -6
- data/test/test_config.rb +0 -21
- data/test/test_formatter.rb +23 -20
- data/test/test_log.rb +108 -36
- data/test/test_msgpack_factory.rb +32 -0
- data/test/test_supervisor.rb +287 -279
- metadata +15 -21
- data/.drone.yml +0 -35
- data/.gitlab-ci.yml +0 -103
- data/test/test_logger_initializer.rb +0 -46
@@ -35,6 +35,16 @@ module FluentPluginOutputAsBufferedCompressTest
|
|
35
35
|
@format ? @format.call(tag, time, record) : [tag, time, record].to_json
|
36
36
|
end
|
37
37
|
end
|
38
|
+
|
39
|
+
def self.dummy_event_stream
|
40
|
+
Fluent::ArrayEventStream.new(
|
41
|
+
[
|
42
|
+
[event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
|
43
|
+
[event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
|
44
|
+
[event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
|
45
|
+
]
|
46
|
+
)
|
47
|
+
end
|
38
48
|
end
|
39
49
|
|
40
50
|
class BufferedOutputCompressTest < Test::Unit::TestCase
|
@@ -60,16 +70,6 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
60
70
|
end
|
61
71
|
end
|
62
72
|
|
63
|
-
def dummy_event_stream
|
64
|
-
Fluent::ArrayEventStream.new(
|
65
|
-
[
|
66
|
-
[event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
|
67
|
-
[event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
|
68
|
-
[event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
|
69
|
-
]
|
70
|
-
)
|
71
|
-
end
|
72
|
-
|
73
73
|
TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
|
74
74
|
|
75
75
|
setup do
|
@@ -89,20 +89,34 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
89
89
|
end
|
90
90
|
|
91
91
|
data(
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
92
|
+
:buffer_config,
|
93
|
+
[
|
94
|
+
config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
|
95
|
+
config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
|
96
|
+
config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
|
97
|
+
config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
|
98
|
+
],
|
96
99
|
)
|
97
|
-
|
100
|
+
data(
|
101
|
+
:input_es,
|
102
|
+
[
|
103
|
+
FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
|
104
|
+
# If already compressed data is incoming, it must be written as is (i.e. without decompressed).
|
105
|
+
# https://github.com/fluent/fluentd/issues/4146
|
106
|
+
Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
|
107
|
+
],
|
108
|
+
)
|
109
|
+
test 'call a standard format when output plugin adds data to chunk' do |data|
|
110
|
+
buffer_config = data[:buffer_config]
|
111
|
+
es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
|
112
|
+
|
98
113
|
@i = create_output(:async)
|
99
114
|
@i.configure(config_element('ROOT','', {}, [buffer_config]))
|
100
115
|
@i.start
|
101
116
|
@i.after_start
|
102
117
|
|
103
118
|
io = StringIO.new
|
104
|
-
|
105
|
-
expected = es.map { |e| e }
|
119
|
+
expected = es.dup.map { |t, r| [t, r] }
|
106
120
|
compressed_data = ''
|
107
121
|
|
108
122
|
assert_equal :gzip, @i.buffer.compress
|
@@ -138,7 +152,7 @@ class BufferedOutputCompressTest < Test::Unit::TestCase
|
|
138
152
|
@i.after_start
|
139
153
|
|
140
154
|
io = StringIO.new
|
141
|
-
es = dummy_event_stream
|
155
|
+
es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
|
142
156
|
expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
|
143
157
|
compressed_data = ''
|
144
158
|
|
@@ -93,7 +93,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
|
|
93
93
|
end
|
94
94
|
def get_log_time(msg, logs)
|
95
95
|
log_time = nil
|
96
|
-
log = logs.
|
96
|
+
log = logs.find{|l| l.include?(msg) }
|
97
97
|
if log && /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4}) \[error\]/ =~ log
|
98
98
|
log_time = Time.parse($1)
|
99
99
|
end
|
@@ -634,8 +634,8 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
|
|
634
634
|
|
635
635
|
assert @i.retry
|
636
636
|
logs = @i.log.out.logs
|
637
|
-
waiting(4){ sleep 0.1 until logs.
|
638
|
-
assert{ logs.
|
637
|
+
waiting(4){ sleep 0.1 until logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
|
638
|
+
assert{ logs.count{|l| l.include?("[warn]: failed to flush the buffer chunk, timeout to commit.") } == 2 }
|
639
639
|
end
|
640
640
|
|
641
641
|
test 'retry_wait for secondary is same with one for primary' do
|
@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
|
|
28
28
|
if initialize_conf
|
29
29
|
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
|
30
30
|
else
|
31
|
-
|
32
|
-
instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
|
33
|
-
instance.configure(conf)
|
34
|
-
d = Struct.new(:instance).new
|
35
|
-
d.instance = instance
|
36
|
-
d
|
31
|
+
Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
|
37
32
|
end
|
38
33
|
end
|
39
34
|
|
@@ -559,7 +559,7 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
559
559
|
unless Fluent.windows?
|
560
560
|
test 'can specify subprocess name' do
|
561
561
|
io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
|
562
|
-
process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.
|
562
|
+
process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
|
563
563
|
Process.kill(:TERM, io.pid) rescue nil
|
564
564
|
io.close rescue nil
|
565
565
|
|
@@ -584,7 +584,7 @@ class ChildProcessTest < Test::Unit::TestCase
|
|
584
584
|
m.lock
|
585
585
|
pid = pids.first
|
586
586
|
# 16357 sleeeeeeeeeper -e sleep 10; puts "hello"
|
587
|
-
assert{ proc_lines.
|
587
|
+
assert{ proc_lines.find{|line| line =~ /^\s*#{pid}\s/ }.strip.split(/\s+/)[1] == "sleeeeeeeeeper" }
|
588
588
|
@d.stop; @d.shutdown; @d.close; @d.terminate
|
589
589
|
end
|
590
590
|
end
|
@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
|
|
127
127
|
end
|
128
128
|
|
129
129
|
client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
|
130
|
-
reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
|
130
|
+
reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
|
131
131
|
|
132
132
|
resp = nil
|
133
133
|
error = nil
|
@@ -16,14 +16,20 @@ class ServerPluginHelperTest < Test::Unit::TestCase
|
|
16
16
|
|
17
17
|
setup do
|
18
18
|
@port = unused_port
|
19
|
-
|
20
|
-
|
21
|
-
|
19
|
+
if Fluent.windows?
|
20
|
+
@socket_manager_server = ServerEngine::SocketManager::Server.open
|
21
|
+
@socket_manager_path = @socket_manager_server.path
|
22
|
+
else
|
23
|
+
@socket_manager_path = ServerEngine::SocketManager::Server.generate_path
|
24
|
+
if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
|
25
|
+
FileUtils.rm_f @socket_manager_path
|
26
|
+
end
|
27
|
+
@socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
|
22
28
|
end
|
23
|
-
@socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
|
24
29
|
ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
|
25
30
|
|
26
31
|
@d = Dummy.new
|
32
|
+
@d.under_plugin_development = true
|
27
33
|
@d.start
|
28
34
|
@d.after_start
|
29
35
|
end
|
@@ -37,7 +43,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
|
|
37
43
|
(@d.terminated? || @d.terminate) rescue nil
|
38
44
|
|
39
45
|
@socket_manager_server.close
|
40
|
-
if @
|
46
|
+
if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
|
41
47
|
FileUtils.rm_f @socket_manager_path
|
42
48
|
end
|
43
49
|
end
|
@@ -789,6 +795,50 @@ class ServerPluginHelperTest < Test::Unit::TestCase
|
|
789
795
|
end
|
790
796
|
end
|
791
797
|
end
|
798
|
+
|
799
|
+
sub_test_case 'over max_bytes' do
|
800
|
+
data("cut off on Non-Windows", { max_bytes: 32, records: ["a" * 40], expected: ["a" * 32] }, keep: true) unless Fluent.windows?
|
801
|
+
data("drop on Windows", { max_bytes: 32, records: ["a" * 40], expected: [] }, keep: true) if Fluent.windows?
|
802
|
+
test 'with sock' do |data|
|
803
|
+
max_bytes, records, expected = data.values
|
804
|
+
|
805
|
+
actual_records = []
|
806
|
+
@d.server_create_udp(:myserver, @port, max_bytes: max_bytes) do |data, sock|
|
807
|
+
actual_records << data
|
808
|
+
end
|
809
|
+
|
810
|
+
open_client(:udp, "127.0.0.1", @port) do |sock|
|
811
|
+
records.each do |record|
|
812
|
+
sock.send(record, 0)
|
813
|
+
end
|
814
|
+
end
|
815
|
+
|
816
|
+
waiting(10) { sleep 0.1 until actual_records.size >= expected.size }
|
817
|
+
sleep 1 if expected.size == 0 # To confirm no record recieved.
|
818
|
+
|
819
|
+
assert_equal expected, actual_records
|
820
|
+
end
|
821
|
+
|
822
|
+
test 'without sock' do |data|
|
823
|
+
max_bytes, records, expected = data.values
|
824
|
+
|
825
|
+
actual_records = []
|
826
|
+
@d.server_create_udp(:myserver, @port, max_bytes: max_bytes) do |data|
|
827
|
+
actual_records << data
|
828
|
+
end
|
829
|
+
|
830
|
+
open_client(:udp, "127.0.0.1", @port) do |sock|
|
831
|
+
records.each do |record|
|
832
|
+
sock.send(record, 0)
|
833
|
+
end
|
834
|
+
end
|
835
|
+
|
836
|
+
waiting(10) { sleep 0.1 until actual_records.size >= expected.size }
|
837
|
+
sleep 1 if expected.size == 0 # To confirm no record recieved.
|
838
|
+
|
839
|
+
assert_equal expected, actual_records
|
840
|
+
end
|
841
|
+
end
|
792
842
|
end
|
793
843
|
|
794
844
|
module CertUtil
|
@@ -1279,7 +1329,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
|
|
1279
1329
|
# OpenSSL 1.1.1: "TLSv1.2"
|
1280
1330
|
if tls_version == "TLSv1/SSLv3" || tls_version == "TLSv1.2"
|
1281
1331
|
matched = true
|
1282
|
-
unless cipher_name.match(/#{conf.ciphers}/)
|
1332
|
+
unless cipher_name.match?(/#{conf.ciphers}/)
|
1283
1333
|
matched = false
|
1284
1334
|
break
|
1285
1335
|
end
|
@@ -1570,6 +1620,10 @@ class ServerPluginHelperTest < Test::Unit::TestCase
|
|
1570
1620
|
|
1571
1621
|
def open_client(proto, addr, port)
|
1572
1622
|
client = case proto
|
1623
|
+
when :udp
|
1624
|
+
c = UDPSocket.open
|
1625
|
+
c.connect(addr, port)
|
1626
|
+
c
|
1573
1627
|
when :tcp
|
1574
1628
|
TCPSocket.open(addr, port)
|
1575
1629
|
when :tls
|
data/test/test_config.rb
CHANGED
@@ -344,27 +344,6 @@ class ConfigTest < Test::Unit::TestCase
|
|
344
344
|
File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }
|
345
345
|
end
|
346
346
|
|
347
|
-
def test_inline
|
348
|
-
prepare_config
|
349
|
-
opts = {
|
350
|
-
:config_path => "#{TMP_DIR}/config_test_1.conf",
|
351
|
-
:inline_config => "<source>\n type http\n port 2222\n </source>",
|
352
|
-
:use_v1_config => false
|
353
|
-
}
|
354
|
-
assert_nothing_raised do
|
355
|
-
Fluent::Supervisor.new(opts)
|
356
|
-
end
|
357
|
-
create_warn_dummy_logger
|
358
|
-
end
|
359
|
-
|
360
|
-
def create_warn_dummy_logger
|
361
|
-
dl_opts = {}
|
362
|
-
dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
|
363
|
-
logdev = Fluent::Test::DummyLogDevice.new
|
364
|
-
logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
|
365
|
-
$log = Fluent::Log.new(logger)
|
366
|
-
end
|
367
|
-
|
368
347
|
sub_test_case '.build' do
|
369
348
|
test 'read config' do
|
370
349
|
write_config("#{TMP_DIR}/build/config_build.conf", 'key value')
|
data/test/test_formatter.rb
CHANGED
@@ -18,7 +18,7 @@ module FormatterTest
|
|
18
18
|
|
19
19
|
def test_call
|
20
20
|
formatter = Formatter.new
|
21
|
-
formatter.configure(
|
21
|
+
formatter.configure(config_element())
|
22
22
|
assert_raise NotImplementedError do
|
23
23
|
formatter.format('tag', Engine.now, {})
|
24
24
|
end
|
@@ -130,7 +130,7 @@ module FormatterTest
|
|
130
130
|
include FormatterTest
|
131
131
|
|
132
132
|
def setup
|
133
|
-
@formatter = TextFormatter::MessagePackFormatter
|
133
|
+
@formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::MessagePackFormatter)
|
134
134
|
@time = Engine.now
|
135
135
|
end
|
136
136
|
|
@@ -146,7 +146,7 @@ module FormatterTest
|
|
146
146
|
include FormatterTest
|
147
147
|
|
148
148
|
def setup
|
149
|
-
@formatter = TextFormatter::LabeledTSVFormatter
|
149
|
+
@formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::LabeledTSVFormatter)
|
150
150
|
@time = Engine.now
|
151
151
|
@newline = if Fluent.windows?
|
152
152
|
"\r\n"
|
@@ -156,16 +156,16 @@ module FormatterTest
|
|
156
156
|
end
|
157
157
|
|
158
158
|
def test_config_params
|
159
|
-
assert_equal "\t", @formatter.delimiter
|
160
|
-
assert_equal ":", @formatter.label_delimiter
|
159
|
+
assert_equal "\t", @formatter.instance.delimiter
|
160
|
+
assert_equal ":", @formatter.instance.label_delimiter
|
161
161
|
|
162
162
|
@formatter.configure(
|
163
163
|
'delimiter' => ',',
|
164
164
|
'label_delimiter' => '=',
|
165
165
|
)
|
166
166
|
|
167
|
-
assert_equal ",", @formatter.delimiter
|
168
|
-
assert_equal "=", @formatter.label_delimiter
|
167
|
+
assert_equal ",", @formatter.instance.delimiter
|
168
|
+
assert_equal "=", @formatter.instance.label_delimiter
|
169
169
|
end
|
170
170
|
|
171
171
|
def test_format
|
@@ -220,14 +220,14 @@ module FormatterTest
|
|
220
220
|
include FormatterTest
|
221
221
|
|
222
222
|
def setup
|
223
|
-
@formatter = TextFormatter::CsvFormatter
|
223
|
+
@formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::CsvFormatter)
|
224
224
|
@time = Engine.now
|
225
225
|
end
|
226
226
|
|
227
227
|
def test_config_params
|
228
|
-
assert_equal ',', @formatter.delimiter
|
229
|
-
assert_equal true, @formatter.force_quotes
|
230
|
-
assert_nil @formatter.fields
|
228
|
+
assert_equal ',', @formatter.instance.delimiter
|
229
|
+
assert_equal true, @formatter.instance.force_quotes
|
230
|
+
assert_nil @formatter.instance.fields
|
231
231
|
end
|
232
232
|
|
233
233
|
data(
|
@@ -237,8 +237,8 @@ module FormatterTest
|
|
237
237
|
def test_config_params_with_customized_delimiters(data)
|
238
238
|
expected, target = data
|
239
239
|
@formatter.configure('delimiter' => target, 'fields' => 'a,b,c')
|
240
|
-
assert_equal expected, @formatter.delimiter
|
241
|
-
assert_equal ['a', 'b', 'c'], @formatter.fields
|
240
|
+
assert_equal expected, @formatter.instance.delimiter
|
241
|
+
assert_equal ['a', 'b', 'c'], @formatter.instance.fields
|
242
242
|
end
|
243
243
|
|
244
244
|
def test_format
|
@@ -299,7 +299,7 @@ module FormatterTest
|
|
299
299
|
'blank' => 'one,,two,three')
|
300
300
|
def test_config_params_with_fields(data)
|
301
301
|
@formatter.configure('fields' => data)
|
302
|
-
assert_equal %w(one two three), @formatter.fields
|
302
|
+
assert_equal %w(one two three), @formatter.instance.fields
|
303
303
|
end
|
304
304
|
end
|
305
305
|
|
@@ -313,31 +313,34 @@ module FormatterTest
|
|
313
313
|
end
|
314
314
|
end
|
315
315
|
|
316
|
+
def create_driver(klass_or_str)
|
317
|
+
Fluent::Test::FormatterTestDriver.new(klass_or_str)
|
318
|
+
end
|
316
319
|
|
317
320
|
def test_config_params
|
318
|
-
formatter = TextFormatter::SingleValueFormatter
|
319
|
-
assert_equal "message", formatter.message_key
|
321
|
+
formatter = create_driver(TextFormatter::SingleValueFormatter)
|
322
|
+
assert_equal "message", formatter.instance.message_key
|
320
323
|
|
321
324
|
formatter.configure('message_key' => 'foobar')
|
322
|
-
assert_equal "foobar", formatter.message_key
|
325
|
+
assert_equal "foobar", formatter.instance.message_key
|
323
326
|
end
|
324
327
|
|
325
328
|
def test_format
|
326
|
-
formatter =
|
329
|
+
formatter = create_driver('single_value')
|
327
330
|
formatter.configure({})
|
328
331
|
formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
|
329
332
|
assert_equal("awesome#{@newline}", formatted)
|
330
333
|
end
|
331
334
|
|
332
335
|
def test_format_without_newline
|
333
|
-
formatter =
|
336
|
+
formatter = create_driver('single_value')
|
334
337
|
formatter.configure('add_newline' => 'false')
|
335
338
|
formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
|
336
339
|
assert_equal("awesome", formatted)
|
337
340
|
end
|
338
341
|
|
339
342
|
def test_format_with_message_key
|
340
|
-
formatter = TextFormatter::SingleValueFormatter
|
343
|
+
formatter = create_driver(TextFormatter::SingleValueFormatter)
|
341
344
|
formatter.configure('message_key' => 'foobar')
|
342
345
|
formatted = formatter.format('tag', Engine.now, {'foobar' => 'foo'})
|
343
346
|
|
data/test/test_log.rb
CHANGED
@@ -5,6 +5,7 @@ require 'fluent/log'
|
|
5
5
|
require 'timecop'
|
6
6
|
require 'logger'
|
7
7
|
require 'securerandom'
|
8
|
+
require 'pathname'
|
8
9
|
|
9
10
|
class LogTest < Test::Unit::TestCase
|
10
11
|
def tmp_dir
|
@@ -33,6 +34,14 @@ class LogTest < Test::Unit::TestCase
|
|
33
34
|
end
|
34
35
|
end
|
35
36
|
|
37
|
+
def test_per_process_path
|
38
|
+
path = Fluent::Log.per_process_path("C:/tmp/test.log", :supervisor, 0)
|
39
|
+
assert_equal(path, "C:/tmp/test-supervisor-0.log")
|
40
|
+
|
41
|
+
path = Fluent::Log.per_process_path("C:/tmp/test.log", :worker, 1)
|
42
|
+
assert_equal(path, "C:/tmp/test-1.log")
|
43
|
+
end
|
44
|
+
|
36
45
|
sub_test_case "log level" do
|
37
46
|
data(
|
38
47
|
trace: [Fluent::Log::LEVEL_TRACE, 0],
|
@@ -463,6 +472,43 @@ class LogTest < Test::Unit::TestCase
|
|
463
472
|
]
|
464
473
|
assert_equal(expected, log.out.logs)
|
465
474
|
end
|
475
|
+
|
476
|
+
def test_reject_on_max_size
|
477
|
+
ignore_same_log_interval = 10
|
478
|
+
|
479
|
+
logger = Fluent::Log.new(
|
480
|
+
ServerEngine::DaemonLogger.new(@log_device, log_level: ServerEngine::DaemonLogger::INFO),
|
481
|
+
ignore_same_log_interval: ignore_same_log_interval,
|
482
|
+
)
|
483
|
+
|
484
|
+
# Output unique log every second.
|
485
|
+
Fluent::Log::IGNORE_SAME_LOG_MAX_CACHE_SIZE.times do |i|
|
486
|
+
logger.info "Test #{i}"
|
487
|
+
Timecop.freeze(@timestamp + i)
|
488
|
+
end
|
489
|
+
logger.info "Over max size!"
|
490
|
+
|
491
|
+
# The newest cache and the latest caches in `ignore_same_log_interval` should exist.
|
492
|
+
assert { Thread.current[:last_same_log].size == ignore_same_log_interval + 1 }
|
493
|
+
end
|
494
|
+
|
495
|
+
def test_clear_on_max_size
|
496
|
+
ignore_same_log_interval = 10
|
497
|
+
|
498
|
+
logger = Fluent::Log.new(
|
499
|
+
ServerEngine::DaemonLogger.new(@log_device, log_level: ServerEngine::DaemonLogger::INFO),
|
500
|
+
ignore_same_log_interval: ignore_same_log_interval,
|
501
|
+
)
|
502
|
+
|
503
|
+
# Output unique log at the same time.
|
504
|
+
Fluent::Log::IGNORE_SAME_LOG_MAX_CACHE_SIZE.times do |i|
|
505
|
+
logger.info "Test #{i}"
|
506
|
+
end
|
507
|
+
logger.info "Over max size!"
|
508
|
+
|
509
|
+
# Can't reject old logs, so all cache should be cleared and only the newest should exist.
|
510
|
+
assert { Thread.current[:last_same_log].size == 1 }
|
511
|
+
end
|
466
512
|
end
|
467
513
|
|
468
514
|
def test_dup
|
@@ -593,45 +639,71 @@ class LogTest < Test::Unit::TestCase
|
|
593
639
|
|
594
640
|
def test_log_rotates_specified_size_with_logdevio
|
595
641
|
with_timezone('utc') do
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
642
|
+
begin
|
643
|
+
rotate_age = 2
|
644
|
+
rotate_size = 100
|
645
|
+
path = "#{@tmp_dir}/log-dev-io-#{rotate_size}-#{rotate_age}"
|
646
|
+
path0 = path + '.0'
|
647
|
+
path1 = path + '.1'
|
648
|
+
|
649
|
+
logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
|
650
|
+
logger = ServerEngine::DaemonLogger.new(logdev)
|
651
|
+
log = Fluent::Log.new(logger)
|
652
|
+
|
653
|
+
msg = 'a' * 101
|
654
|
+
log.info msg
|
655
|
+
assert_match msg, File.read(path)
|
656
|
+
assert_true File.exist?(path)
|
657
|
+
assert_true !File.exist?(path0)
|
658
|
+
assert_true !File.exist?(path1)
|
659
|
+
|
660
|
+
# create log.0
|
661
|
+
msg2 = 'b' * 101
|
662
|
+
log.info msg2
|
663
|
+
c = File.read(path)
|
664
|
+
c0 = File.read(path0)
|
665
|
+
assert_match msg2, c
|
666
|
+
assert_match msg, c0
|
667
|
+
assert_true File.exist?(path)
|
668
|
+
assert_true File.exist?(path0)
|
669
|
+
assert_true !File.exist?(path1)
|
670
|
+
|
671
|
+
# rotate
|
672
|
+
msg3 = 'c' * 101
|
673
|
+
log.info msg3
|
674
|
+
c = File.read(path)
|
675
|
+
c0 = File.read(path0)
|
676
|
+
assert_match msg3, c
|
677
|
+
assert_match msg2, c0
|
678
|
+
assert_true File.exist?(path)
|
679
|
+
assert_true File.exist?(path0)
|
680
|
+
assert_true !File.exist?(path1)
|
681
|
+
ensure
|
682
|
+
logdev&.close
|
683
|
+
end
|
684
|
+
end
|
685
|
+
end
|
601
686
|
|
602
|
-
|
603
|
-
|
604
|
-
log = Fluent::Log.new(logger)
|
687
|
+
def test_reopen
|
688
|
+
path = Pathname(@tmp_dir) + "fluent.log"
|
605
689
|
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
assert_true File.exist?(path)
|
610
|
-
assert_true !File.exist?(path0)
|
611
|
-
assert_true !File.exist?(path1)
|
690
|
+
logdev = Fluent::LogDeviceIO.new(path.to_s)
|
691
|
+
logger = ServerEngine::DaemonLogger.new(logdev)
|
692
|
+
log = Fluent::Log.new(logger, path: path)
|
612
693
|
|
613
|
-
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
|
619
|
-
|
620
|
-
|
621
|
-
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
log.info msg3
|
627
|
-
c = File.read(path)
|
628
|
-
c0 = File.read(path0)
|
629
|
-
assert_match msg3, c
|
630
|
-
assert_match msg2, c0
|
631
|
-
assert_true File.exist?(path)
|
632
|
-
assert_true File.exist?(path0)
|
633
|
-
assert_true !File.exist?(path1)
|
634
|
-
end
|
694
|
+
message = "This is test message."
|
695
|
+
|
696
|
+
log.info message
|
697
|
+
log.reopen!
|
698
|
+
log.info message
|
699
|
+
|
700
|
+
assert { path.read.lines.count{ |line| line.include?(message) } == 2 }
|
701
|
+
# Assert reopening the same file.
|
702
|
+
# Especially, on Windows, the filepath is fixed for each process with rotate,
|
703
|
+
# so we need to care about this.
|
704
|
+
assert { path.parent.entries.size == 3 } # [".", "..", "fluent.log"]
|
705
|
+
ensure
|
706
|
+
logdev&.close
|
635
707
|
end
|
636
708
|
end
|
637
709
|
|
@@ -15,4 +15,36 @@ class MessagePackFactoryTest < Test::Unit::TestCase
|
|
15
15
|
assert mp.msgpack_factory
|
16
16
|
assert mp.msgpack_factory
|
17
17
|
end
|
18
|
+
|
19
|
+
sub_test_case 'thread_local_msgpack_packer' do
|
20
|
+
test 'packer is cached' do
|
21
|
+
packer1 = Fluent::MessagePackFactory.thread_local_msgpack_packer
|
22
|
+
packer2 = Fluent::MessagePackFactory.thread_local_msgpack_packer
|
23
|
+
assert_equal packer1, packer2
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
sub_test_case 'thread_local_msgpack_unpacker' do
|
28
|
+
test 'unpacker is cached' do
|
29
|
+
unpacker1 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
|
30
|
+
unpacker2 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
|
31
|
+
assert_equal unpacker1, unpacker2
|
32
|
+
end
|
33
|
+
|
34
|
+
# We need to reset the buffer every time so that received incomplete data
|
35
|
+
# must not affect data from other senders.
|
36
|
+
test 'reset the internal buffer of unpacker every time' do
|
37
|
+
unpacker1 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
|
38
|
+
unpacker1.feed_each("\xA6foo") do |result|
|
39
|
+
flunk("This callback must not be called since the data is uncomplete.")
|
40
|
+
end
|
41
|
+
|
42
|
+
records = []
|
43
|
+
unpacker2 = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
|
44
|
+
unpacker2.feed_each("\xA3foo") do |result|
|
45
|
+
records.append(result)
|
46
|
+
end
|
47
|
+
assert_equal ["foo"], records
|
48
|
+
end
|
49
|
+
end
|
18
50
|
end
|