fluentd 1.6.2-x86-mingw32 → 1.6.3-x86-mingw32
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of fluentd might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/lib/fluent/plugin/buf_file.rb +7 -5
- data/lib/fluent/plugin/in_syslog.rb +11 -0
- data/lib/fluent/plugin/in_tail.rb +12 -5
- data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb +3 -3
- data/lib/fluent/version.rb +1 -1
- data/test/plugin/test_buf_file.rb +28 -12
- data/test/plugin/test_in_syslog.rb +27 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: eb2e03cd7a40b2276b0ef281ee57e27af4a16204b8da766d1a97c9ae145a1b02
|
4
|
+
data.tar.gz: 91066de64539ac2a06fa97a730702ce41fb53ecab5818824c26630cacaa756e7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7db0784aeb7136fbc9e6057855ff1c01235acfe4bea6b526d1ca82d01ce40e379a7be9c23534904ee14f036c18c5f3dbeb863bdd141742875b3494057f2a373d
|
7
|
+
data.tar.gz: d1b838408dcc95821fe4cd8487690a1ea7e684157189d743d9e9f94dcfebe2d799b3321ecb3aeb05c08c87d2264a98377fb020af8f3b6d62f78323c5d6f90c03
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,21 @@
|
|
1
1
|
# v1.6
|
2
2
|
|
3
|
+
## Release v1.6.3 - 2019/07/29
|
4
|
+
|
5
|
+
### Enhancement
|
6
|
+
|
7
|
+
* in_syslog: Add `emit_unmatched_lines` parameter
|
8
|
+
https://github.com/fluent/fluentd/pull/2499
|
9
|
+
* buf_file: Add `path_suffix` parameter
|
10
|
+
https://github.com/fluent/fluentd/pull/2524
|
11
|
+
* in_tail: Improve the performance of split lines
|
12
|
+
https://github.com/fluent/fluentd/pull/2527
|
13
|
+
|
14
|
+
### Bug fixes
|
15
|
+
|
16
|
+
* http_server: Fix re-define render_json method
|
17
|
+
https://github.com/fluent/fluentd/pull/2517
|
18
|
+
|
3
19
|
## Release v1.6.2 - 2019/07/11
|
4
20
|
|
5
21
|
### Bug fixes
|
@@ -34,6 +34,8 @@ module Fluent
|
|
34
34
|
|
35
35
|
desc 'The path where buffer chunks are stored.'
|
36
36
|
config_param :path, :string, default: nil
|
37
|
+
desc 'The suffix of buffer chunks'
|
38
|
+
config_param :path_suffix, :string, default: '.log'
|
37
39
|
|
38
40
|
config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
|
39
41
|
config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
|
@@ -78,12 +80,12 @@ module Fluent
|
|
78
80
|
|
79
81
|
if specified_directory_exists || unexisting_path_for_directory # directory
|
80
82
|
if using_plugin_root_dir || !multi_workers_configured
|
81
|
-
@path = File.join(@path,
|
83
|
+
@path = File.join(@path, "buffer.*#{@path_suffix}")
|
82
84
|
else
|
83
|
-
@path = File.join(@path, "worker#{fluentd_worker_id}",
|
85
|
+
@path = File.join(@path, "worker#{fluentd_worker_id}", "buffer.*#{@path_suffix}")
|
84
86
|
if fluentd_worker_id == 0
|
85
87
|
# worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)
|
86
|
-
@additional_resume_path = File.join(File.expand_path("../../", @path),
|
88
|
+
@additional_resume_path = File.join(File.expand_path("../../", @path), "buffer.*#{@path_suffix}")
|
87
89
|
end
|
88
90
|
end
|
89
91
|
@multi_workers_available = true
|
@@ -91,10 +93,10 @@ module Fluent
|
|
91
93
|
if File.basename(@path).include?('.*.')
|
92
94
|
# valid file path
|
93
95
|
elsif File.basename(@path).end_with?('.*')
|
94
|
-
@path = @path +
|
96
|
+
@path = @path + @path_suffix
|
95
97
|
else
|
96
98
|
# existing file will be ignored
|
97
|
-
@path = @path +
|
99
|
+
@path = @path + ".*#{@path_suffix}"
|
98
100
|
end
|
99
101
|
@multi_workers_available = false
|
100
102
|
end
|
@@ -82,6 +82,8 @@ module Fluent::Plugin
|
|
82
82
|
config_param :include_source_host, :bool, default: false, deprecated: 'use "source_hostname_key" or "source_address_key" instead.'
|
83
83
|
desc 'Specify key of source host when include_source_host is true.'
|
84
84
|
config_param :source_host_key, :string, default: 'source_host'.freeze
|
85
|
+
desc 'Enable the option to emit unmatched lines.'
|
86
|
+
config_param :emit_unmatched_lines, :bool, default: false
|
85
87
|
|
86
88
|
desc 'The field name of hostname of sender.'
|
87
89
|
config_param :source_hostname_key, :string, default: nil
|
@@ -203,6 +205,9 @@ module Fluent::Plugin
|
|
203
205
|
unless @parser_parse_priority
|
204
206
|
m = SYSLOG_REGEXP.match(data)
|
205
207
|
unless m
|
208
|
+
if @emit_unmatched_lines
|
209
|
+
emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => data})
|
210
|
+
end
|
206
211
|
log.warn "invalid syslog message: #{data.dump}"
|
207
212
|
return
|
208
213
|
end
|
@@ -212,6 +217,9 @@ module Fluent::Plugin
|
|
212
217
|
|
213
218
|
@parser.parse(text) do |time, record|
|
214
219
|
unless time && record
|
220
|
+
if @emit_unmatched_lines
|
221
|
+
emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => text})
|
222
|
+
end
|
215
223
|
log.warn "failed to parse message", data: data
|
216
224
|
return
|
217
225
|
end
|
@@ -229,6 +237,9 @@ module Fluent::Plugin
|
|
229
237
|
emit(tag, time, record)
|
230
238
|
end
|
231
239
|
rescue => e
|
240
|
+
if @emit_unmatched_lines
|
241
|
+
emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => text})
|
242
|
+
end
|
232
243
|
log.error "invalid input", data: data, error: e
|
233
244
|
log.error_backtrace
|
234
245
|
end
|
@@ -698,9 +698,18 @@ module Fluent::Plugin
|
|
698
698
|
s.encode!(@encoding, @from_encoding, :invalid => :replace, :undef => :replace)
|
699
699
|
end
|
700
700
|
|
701
|
-
def
|
701
|
+
def read_lines(lines)
|
702
702
|
idx = @buffer.index(@eol)
|
703
|
-
|
703
|
+
|
704
|
+
until idx.nil?
|
705
|
+
# Using freeze and slice is faster than slice!
|
706
|
+
# See https://github.com/fluent/fluentd/pull/2527
|
707
|
+
@buffer.freeze
|
708
|
+
rbuf = @buffer.slice(0, idx + 1)
|
709
|
+
@buffer = @buffer.slice(idx + 1, @buffer.size)
|
710
|
+
idx = @buffer.index(@eol)
|
711
|
+
lines << convert(rbuf)
|
712
|
+
end
|
704
713
|
end
|
705
714
|
|
706
715
|
def bytesize
|
@@ -733,9 +742,7 @@ module Fluent::Plugin
|
|
733
742
|
begin
|
734
743
|
while true
|
735
744
|
@fifo << io.readpartial(8192, @iobuf)
|
736
|
-
|
737
|
-
@lines << line
|
738
|
-
end
|
745
|
+
@fifo.read_lines(@lines)
|
739
746
|
if @lines.size >= @watcher.read_lines_limit
|
740
747
|
# not to use too much memory in case the file is very large
|
741
748
|
read_more = true
|
@@ -44,10 +44,10 @@ module Fluent
|
|
44
44
|
headers.each { |k, v| response[k] = v }
|
45
45
|
response.body = body
|
46
46
|
end
|
47
|
+
end
|
47
48
|
|
48
|
-
|
49
|
-
|
50
|
-
end
|
49
|
+
def render_json(code, obj)
|
50
|
+
[code, { 'Content-Type' => 'application/json' }, obj.to_json]
|
51
51
|
end
|
52
52
|
end
|
53
53
|
end
|
data/lib/fluent/version.rb
CHANGED
@@ -57,20 +57,30 @@ class FileBufferTest < Test::Unit::TestCase
|
|
57
57
|
assert_equal File.join(@dir, 'buffer.*.file'), p.path
|
58
58
|
end
|
59
59
|
|
60
|
-
|
60
|
+
data('default' => [nil, 'log'],
|
61
|
+
'conf' => ['.buf', 'buf'])
|
62
|
+
test 'existing directory will be used with additional default file name' do |params|
|
63
|
+
conf, suffix = params
|
61
64
|
d = FluentPluginFileBufferTest::DummyOutputPlugin.new
|
62
65
|
p = Fluent::Plugin::FileBuffer.new
|
63
66
|
p.owner = d
|
64
|
-
|
65
|
-
|
67
|
+
c = {'path' => @dir}
|
68
|
+
c['path_suffix'] = conf if conf
|
69
|
+
p.configure(config_element('buffer', '', c))
|
70
|
+
assert_equal File.join(@dir, "buffer.*.#{suffix}"), p.path
|
66
71
|
end
|
67
72
|
|
68
|
-
|
73
|
+
data('default' => [nil, 'log'],
|
74
|
+
'conf' => ['.buf', 'buf'])
|
75
|
+
test 'unexisting path without * handled as directory' do |params|
|
76
|
+
conf, suffix = params
|
69
77
|
d = FluentPluginFileBufferTest::DummyOutputPlugin.new
|
70
78
|
p = Fluent::Plugin::FileBuffer.new
|
71
79
|
p.owner = d
|
72
|
-
|
73
|
-
|
80
|
+
c = {'path' => File.join(@dir, 'buffer')}
|
81
|
+
c['path_suffix'] = conf if conf
|
82
|
+
p.configure(config_element('buffer', '', c))
|
83
|
+
assert_equal File.join(@dir, 'buffer', "buffer.*.#{suffix}"), p.path
|
74
84
|
end
|
75
85
|
end
|
76
86
|
|
@@ -312,10 +322,6 @@ class FileBufferTest < Test::Unit::TestCase
|
|
312
322
|
@d = FluentPluginFileBufferTest::DummyOutputPlugin.new
|
313
323
|
@p = Fluent::Plugin::FileBuffer.new
|
314
324
|
@p.owner = @d
|
315
|
-
Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
|
316
|
-
@d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
|
317
|
-
@p.configure(config_element('buffer', ''))
|
318
|
-
end
|
319
325
|
end
|
320
326
|
|
321
327
|
teardown do
|
@@ -329,8 +335,18 @@ class FileBufferTest < Test::Unit::TestCase
|
|
329
335
|
end
|
330
336
|
end
|
331
337
|
|
332
|
-
|
333
|
-
|
338
|
+
data('default' => [nil, 'log'],
|
339
|
+
'conf' => ['.buf', 'buf'])
|
340
|
+
test '#start creates directory for buffer chunks' do |params|
|
341
|
+
conf, suffix = params
|
342
|
+
c = {}
|
343
|
+
c['path_suffix'] = conf if conf
|
344
|
+
Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
|
345
|
+
@d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
|
346
|
+
@p.configure(config_element('buffer', '', c))
|
347
|
+
end
|
348
|
+
|
349
|
+
expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', "buffer.*.#{suffix}")
|
334
350
|
expected_buffer_dir = File.dirname(expected_buffer_path)
|
335
351
|
assert_equal expected_buffer_path, @p.path
|
336
352
|
assert_false Dir.exist?(expected_buffer_dir)
|
@@ -361,4 +361,31 @@ EOS
|
|
361
361
|
msgs
|
362
362
|
end
|
363
363
|
end
|
364
|
+
|
365
|
+
def test_emit_unmatched_lines
|
366
|
+
d = create_driver([CONFIG, 'emit_unmatched_lines true'].join("\n"))
|
367
|
+
tests = [
|
368
|
+
# valid message
|
369
|
+
{'msg' => '<6>Sep 10 00:00:00 localhost logger: xxx', 'expected' => {'host'=>'localhost', 'ident'=>'logger', 'message'=>'xxx'}},
|
370
|
+
# missing priority
|
371
|
+
{'msg' => 'hello world', 'expected' => {'unmatched_line' => 'hello world'}},
|
372
|
+
# timestamp parsing failure
|
373
|
+
{'msg' => '<6>ZZZ 99 99:99:99 localhost logger: xxx', 'expected' => {'unmatched_line' => '<6>ZZZ 99 99:99:99 localhost logger: xxx'}},
|
374
|
+
]
|
375
|
+
|
376
|
+
d.run(expect_emits: 3) do
|
377
|
+
u = UDPSocket.new
|
378
|
+
u.do_not_reverse_lookup = false
|
379
|
+
u.connect('127.0.0.1', PORT)
|
380
|
+
tests.each {|test|
|
381
|
+
u.send(test['msg'], 0)
|
382
|
+
}
|
383
|
+
end
|
384
|
+
|
385
|
+
assert_equal tests.size, d.events.size
|
386
|
+
tests.size.times do |i|
|
387
|
+
assert_equal tests[i]['expected'], d.events[i][2]
|
388
|
+
assert_equal 'syslog.unmatched', d.events[i][0] unless i==0
|
389
|
+
end
|
390
|
+
end
|
364
391
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluentd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.6.
|
4
|
+
version: 1.6.3
|
5
5
|
platform: x86-mingw32
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-07-
|
11
|
+
date: 2019-07-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: msgpack
|