fluentd 1.6.2 → 1.6.3

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of fluentd might be problematic. Click here for more details.

checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: faae0c0f8e371e91fb6cf69bbe368546910282f29d2a447d6aa79eb9ed006913
4
- data.tar.gz: 0a28223bc8471c15756a49ad89aac13456dc8ae9b7f31364043db86f95d76213
3
+ metadata.gz: cf4c49c147ccaa5c7ad0c4585a4e2a0e156b525ff5eb0a4a163df1c20c1c8e1d
4
+ data.tar.gz: 813578f6a63b400fadad9fead1b0edd323be8f92e31b1522133a886a28c9978b
5
5
  SHA512:
6
- metadata.gz: d21f537df92d813568d963c8f49775f187013a804a8be59e39e8f2e6cf970de28236bb133d3f2071dba73d73e56f172d233a263b756615f1543d4c1994a2e6f8
7
- data.tar.gz: 042fa8d384f3698ae556f66eefce7f718c132ad63edd16ecab0a9bbc989d3a0497dfd4b9fe7ad5246519cec78a8f9ed51e05967ac8db01b08f7e541eb7d2b2e7
6
+ metadata.gz: fbe1fbbe488245d3bda830a5ec3a0419e347ce7d6a685d5fd5f1d933c71382792cb047a45ffdbe982a763cdb336c1589958e9f84536c67452d1d608f9fea0db3
7
+ data.tar.gz: d9d2370bf3ce08c0500aa99f83c16af9ef3ebb97b15e8992bf88a0b76834b1eaad82a1bab27d3f39b07be90cf20fdf8646d31ee401bdf98a6709b65496417c5d
@@ -1,5 +1,21 @@
1
1
  # v1.6
2
2
 
3
+ ## Release v1.6.3 - 2019/07/29
4
+
5
+ ### Enhancement
6
+
7
+ * in_syslog: Add `emit_unmatched_lines` parameter
8
+ https://github.com/fluent/fluentd/pull/2499
9
+ * buf_file: Add `path_suffix` parameter
10
+ https://github.com/fluent/fluentd/pull/2524
11
+ * in_tail: Improve the performance of split lines
12
+ https://github.com/fluent/fluentd/pull/2527
13
+
14
+ ### Bug fixes
15
+
16
+ * http_server: Fix re-define render_json method
17
+ https://github.com/fluent/fluentd/pull/2517
18
+
3
19
  ## Release v1.6.2 - 2019/07/11
4
20
 
5
21
  ### Bug fixes
@@ -34,6 +34,8 @@ module Fluent
34
34
 
35
35
  desc 'The path where buffer chunks are stored.'
36
36
  config_param :path, :string, default: nil
37
+ desc 'The suffix of buffer chunks'
38
+ config_param :path_suffix, :string, default: '.log'
37
39
 
38
40
  config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
39
41
  config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
@@ -78,12 +80,12 @@ module Fluent
78
80
 
79
81
  if specified_directory_exists || unexisting_path_for_directory # directory
80
82
  if using_plugin_root_dir || !multi_workers_configured
81
- @path = File.join(@path, 'buffer.*.log')
83
+ @path = File.join(@path, "buffer.*#{@path_suffix}")
82
84
  else
83
- @path = File.join(@path, "worker#{fluentd_worker_id}", 'buffer.*.log')
85
+ @path = File.join(@path, "worker#{fluentd_worker_id}", "buffer.*#{@path_suffix}")
84
86
  if fluentd_worker_id == 0
85
87
  # worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)
86
- @additional_resume_path = File.join(File.expand_path("../../", @path), 'buffer.*.log')
88
+ @additional_resume_path = File.join(File.expand_path("../../", @path), "buffer.*#{@path_suffix}")
87
89
  end
88
90
  end
89
91
  @multi_workers_available = true
@@ -91,10 +93,10 @@ module Fluent
91
93
  if File.basename(@path).include?('.*.')
92
94
  # valid file path
93
95
  elsif File.basename(@path).end_with?('.*')
94
- @path = @path + '.log'
96
+ @path = @path + @path_suffix
95
97
  else
96
98
  # existing file will be ignored
97
- @path = @path + '.*.log'
99
+ @path = @path + ".*#{@path_suffix}"
98
100
  end
99
101
  @multi_workers_available = false
100
102
  end
@@ -82,6 +82,8 @@ module Fluent::Plugin
82
82
  config_param :include_source_host, :bool, default: false, deprecated: 'use "source_hostname_key" or "source_address_key" instead.'
83
83
  desc 'Specify key of source host when include_source_host is true.'
84
84
  config_param :source_host_key, :string, default: 'source_host'.freeze
85
+ desc 'Enable the option to emit unmatched lines.'
86
+ config_param :emit_unmatched_lines, :bool, default: false
85
87
 
86
88
  desc 'The field name of hostname of sender.'
87
89
  config_param :source_hostname_key, :string, default: nil
@@ -203,6 +205,9 @@ module Fluent::Plugin
203
205
  unless @parser_parse_priority
204
206
  m = SYSLOG_REGEXP.match(data)
205
207
  unless m
208
+ if @emit_unmatched_lines
209
+ emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => data})
210
+ end
206
211
  log.warn "invalid syslog message: #{data.dump}"
207
212
  return
208
213
  end
@@ -212,6 +217,9 @@ module Fluent::Plugin
212
217
 
213
218
  @parser.parse(text) do |time, record|
214
219
  unless time && record
220
+ if @emit_unmatched_lines
221
+ emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => text})
222
+ end
215
223
  log.warn "failed to parse message", data: data
216
224
  return
217
225
  end
@@ -229,6 +237,9 @@ module Fluent::Plugin
229
237
  emit(tag, time, record)
230
238
  end
231
239
  rescue => e
240
+ if @emit_unmatched_lines
241
+ emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => text})
242
+ end
232
243
  log.error "invalid input", data: data, error: e
233
244
  log.error_backtrace
234
245
  end
@@ -698,9 +698,18 @@ module Fluent::Plugin
698
698
  s.encode!(@encoding, @from_encoding, :invalid => :replace, :undef => :replace)
699
699
  end
700
700
 
701
- def next_line
701
+ def read_lines(lines)
702
702
  idx = @buffer.index(@eol)
703
- convert(@buffer.slice!(0, idx + 1)) unless idx.nil?
703
+
704
+ until idx.nil?
705
+ # Using freeze and slice is faster than slice!
706
+ # See https://github.com/fluent/fluentd/pull/2527
707
+ @buffer.freeze
708
+ rbuf = @buffer.slice(0, idx + 1)
709
+ @buffer = @buffer.slice(idx + 1, @buffer.size)
710
+ idx = @buffer.index(@eol)
711
+ lines << convert(rbuf)
712
+ end
704
713
  end
705
714
 
706
715
  def bytesize
@@ -733,9 +742,7 @@ module Fluent::Plugin
733
742
  begin
734
743
  while true
735
744
  @fifo << io.readpartial(8192, @iobuf)
736
- while (line = @fifo.next_line)
737
- @lines << line
738
- end
745
+ @fifo.read_lines(@lines)
739
746
  if @lines.size >= @watcher.read_lines_limit
740
747
  # not to use too much memory in case the file is very large
741
748
  read_more = true
@@ -44,10 +44,10 @@ module Fluent
44
44
  headers.each { |k, v| response[k] = v }
45
45
  response.body = body
46
46
  end
47
+ end
47
48
 
48
- def render_json(code, obj)
49
- [code, { 'Content-Type' => 'application/json' }, obj.to_json]
50
- end
49
+ def render_json(code, obj)
50
+ [code, { 'Content-Type' => 'application/json' }, obj.to_json]
51
51
  end
52
52
  end
53
53
  end
@@ -16,6 +16,6 @@
16
16
 
17
17
  module Fluent
18
18
 
19
- VERSION = '1.6.2'
19
+ VERSION = '1.6.3'
20
20
 
21
21
  end
@@ -57,20 +57,30 @@ class FileBufferTest < Test::Unit::TestCase
57
57
  assert_equal File.join(@dir, 'buffer.*.file'), p.path
58
58
  end
59
59
 
60
- test 'existing directory will be used with additional default file name' do
60
+ data('default' => [nil, 'log'],
61
+ 'conf' => ['.buf', 'buf'])
62
+ test 'existing directory will be used with additional default file name' do |params|
63
+ conf, suffix = params
61
64
  d = FluentPluginFileBufferTest::DummyOutputPlugin.new
62
65
  p = Fluent::Plugin::FileBuffer.new
63
66
  p.owner = d
64
- p.configure(config_element('buffer', '', {'path' => @dir}))
65
- assert_equal File.join(@dir, 'buffer.*.log'), p.path
67
+ c = {'path' => @dir}
68
+ c['path_suffix'] = conf if conf
69
+ p.configure(config_element('buffer', '', c))
70
+ assert_equal File.join(@dir, "buffer.*.#{suffix}"), p.path
66
71
  end
67
72
 
68
- test 'unexisting path without * handled as directory' do
73
+ data('default' => [nil, 'log'],
74
+ 'conf' => ['.buf', 'buf'])
75
+ test 'unexisting path without * handled as directory' do |params|
76
+ conf, suffix = params
69
77
  d = FluentPluginFileBufferTest::DummyOutputPlugin.new
70
78
  p = Fluent::Plugin::FileBuffer.new
71
79
  p.owner = d
72
- p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer')}))
73
- assert_equal File.join(@dir, 'buffer', 'buffer.*.log'), p.path
80
+ c = {'path' => File.join(@dir, 'buffer')}
81
+ c['path_suffix'] = conf if conf
82
+ p.configure(config_element('buffer', '', c))
83
+ assert_equal File.join(@dir, 'buffer', "buffer.*.#{suffix}"), p.path
74
84
  end
75
85
  end
76
86
 
@@ -312,10 +322,6 @@ class FileBufferTest < Test::Unit::TestCase
312
322
  @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
313
323
  @p = Fluent::Plugin::FileBuffer.new
314
324
  @p.owner = @d
315
- Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
316
- @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
317
- @p.configure(config_element('buffer', ''))
318
- end
319
325
  end
320
326
 
321
327
  teardown do
@@ -329,8 +335,18 @@ class FileBufferTest < Test::Unit::TestCase
329
335
  end
330
336
  end
331
337
 
332
- test '#start creates directory for buffer chunks' do
333
- expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', 'buffer.*.log')
338
+ data('default' => [nil, 'log'],
339
+ 'conf' => ['.buf', 'buf'])
340
+ test '#start creates directory for buffer chunks' do |params|
341
+ conf, suffix = params
342
+ c = {}
343
+ c['path_suffix'] = conf if conf
344
+ Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
345
+ @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
346
+ @p.configure(config_element('buffer', '', c))
347
+ end
348
+
349
+ expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', "buffer.*.#{suffix}")
334
350
  expected_buffer_dir = File.dirname(expected_buffer_path)
335
351
  assert_equal expected_buffer_path, @p.path
336
352
  assert_false Dir.exist?(expected_buffer_dir)
@@ -361,4 +361,31 @@ EOS
361
361
  msgs
362
362
  end
363
363
  end
364
+
365
+ def test_emit_unmatched_lines
366
+ d = create_driver([CONFIG, 'emit_unmatched_lines true'].join("\n"))
367
+ tests = [
368
+ # valid message
369
+ {'msg' => '<6>Sep 10 00:00:00 localhost logger: xxx', 'expected' => {'host'=>'localhost', 'ident'=>'logger', 'message'=>'xxx'}},
370
+ # missing priority
371
+ {'msg' => 'hello world', 'expected' => {'unmatched_line' => 'hello world'}},
372
+ # timestamp parsing failure
373
+ {'msg' => '<6>ZZZ 99 99:99:99 localhost logger: xxx', 'expected' => {'unmatched_line' => '<6>ZZZ 99 99:99:99 localhost logger: xxx'}},
374
+ ]
375
+
376
+ d.run(expect_emits: 3) do
377
+ u = UDPSocket.new
378
+ u.do_not_reverse_lookup = false
379
+ u.connect('127.0.0.1', PORT)
380
+ tests.each {|test|
381
+ u.send(test['msg'], 0)
382
+ }
383
+ end
384
+
385
+ assert_equal tests.size, d.events.size
386
+ tests.size.times do |i|
387
+ assert_equal tests[i]['expected'], d.events[i][2]
388
+ assert_equal 'syslog.unmatched', d.events[i][0] unless i==0
389
+ end
390
+ end
364
391
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluentd
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.6.2
4
+ version: 1.6.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-07-12 00:00:00.000000000 Z
11
+ date: 2019-07-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: msgpack