fluentd 1.6.2 → 1.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: faae0c0f8e371e91fb6cf69bbe368546910282f29d2a447d6aa79eb9ed006913
-  data.tar.gz: 0a28223bc8471c15756a49ad89aac13456dc8ae9b7f31364043db86f95d76213
+  metadata.gz: cf4c49c147ccaa5c7ad0c4585a4e2a0e156b525ff5eb0a4a163df1c20c1c8e1d
+  data.tar.gz: 813578f6a63b400fadad9fead1b0edd323be8f92e31b1522133a886a28c9978b
 SHA512:
-  metadata.gz: d21f537df92d813568d963c8f49775f187013a804a8be59e39e8f2e6cf970de28236bb133d3f2071dba73d73e56f172d233a263b756615f1543d4c1994a2e6f8
-  data.tar.gz: 042fa8d384f3698ae556f66eefce7f718c132ad63edd16ecab0a9bbc989d3a0497dfd4b9fe7ad5246519cec78a8f9ed51e05967ac8db01b08f7e541eb7d2b2e7
+  metadata.gz: fbe1fbbe488245d3bda830a5ec3a0419e347ce7d6a685d5fd5f1d933c71382792cb047a45ffdbe982a763cdb336c1589958e9f84536c67452d1d608f9fea0db3
+  data.tar.gz: d9d2370bf3ce08c0500aa99f83c16af9ef3ebb97b15e8992bf88a0b76834b1eaad82a1bab27d3f39b07be90cf20fdf8646d31ee401bdf98a6709b65496417c5d

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # v1.6
 
+## Release v1.6.3 - 2019/07/29
+
+### Enhancement
+
+* in_syslog: Add `emit_unmatched_lines` parameter
+  https://github.com/fluent/fluentd/pull/2499
+* buf_file: Add `path_suffix` parameter
+  https://github.com/fluent/fluentd/pull/2524
+* in_tail: Improve the performance of split lines
+  https://github.com/fluent/fluentd/pull/2527
+
+### Bug fixes
+
+* http_server: Fix re-define render_json method
+  https://github.com/fluent/fluentd/pull/2517
+
 ## Release v1.6.2 - 2019/07/11
 
 ### Bug fixes

data/lib/fluent/plugin/buf_file.rb

@@ -34,6 +34,8 @@ module Fluent
 
       desc 'The path where buffer chunks are stored.'
       config_param :path, :string, default: nil
+      desc 'The suffix of buffer chunks'
+      config_param :path_suffix, :string, default: '.log'
 
       config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
       config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
@@ -78,12 +80,12 @@ module Fluent
 
         if specified_directory_exists || unexisting_path_for_directory # directory
           if using_plugin_root_dir || !multi_workers_configured
-            @path = File.join(@path, 'buffer.*.log')
+            @path = File.join(@path, "buffer.*#{@path_suffix}")
           else
-            @path = File.join(@path, "worker#{fluentd_worker_id}", 'buffer.*.log')
+            @path = File.join(@path, "worker#{fluentd_worker_id}", "buffer.*#{@path_suffix}")
             if fluentd_worker_id == 0
               # worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)
-              @additional_resume_path = File.join(File.expand_path("../../", @path), 'buffer.*.log')
+              @additional_resume_path = File.join(File.expand_path("../../", @path), "buffer.*#{@path_suffix}")
             end
           end
           @multi_workers_available = true
@@ -91,10 +93,10 @@ module Fluent
           if File.basename(@path).include?('.*.')
             # valid file path
           elsif File.basename(@path).end_with?('.*')
-            @path = @path + '.log'
+            @path = @path + @path_suffix
           else
             # existing file will be ignored
-            @path = @path + '.*.log'
+            @path = @path + ".*#{@path_suffix}"
           end
           @multi_workers_available = false
         end

data/lib/fluent/plugin/in_syslog.rb

@@ -82,6 +82,8 @@ module Fluent::Plugin
     config_param :include_source_host, :bool, default: false, deprecated: 'use "source_hostname_key" or "source_address_key" instead.'
     desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, default: 'source_host'.freeze
+    desc 'Enable the option to emit unmatched lines.'
+    config_param :emit_unmatched_lines, :bool, default: false
 
     desc 'The field name of hostname of sender.'
     config_param :source_hostname_key, :string, default: nil
@@ -203,6 +205,9 @@ module Fluent::Plugin
       unless @parser_parse_priority
         m = SYSLOG_REGEXP.match(data)
         unless m
+          if @emit_unmatched_lines
+            emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => data})
+          end
           log.warn "invalid syslog message: #{data.dump}"
           return
         end
@@ -212,6 +217,9 @@ module Fluent::Plugin
 
       @parser.parse(text) do |time, record|
         unless time && record
+          if @emit_unmatched_lines
+            emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => text})
+          end
           log.warn "failed to parse message", data: data
           return
         end
@@ -229,6 +237,9 @@ module Fluent::Plugin
         emit(tag, time, record)
       end
     rescue => e
+      if @emit_unmatched_lines
+        emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => text})
+      end
       log.error "invalid input", data: data, error: e
       log.error_backtrace
     end

data/lib/fluent/plugin/in_tail.rb

@@ -698,9 +698,18 @@ module Fluent::Plugin
           s.encode!(@encoding, @from_encoding, :invalid => :replace, :undef => :replace)
         end
 
-        def next_line
+        def read_lines(lines)
           idx = @buffer.index(@eol)
-          convert(@buffer.slice!(0, idx + 1)) unless idx.nil?
+
+          until idx.nil?
+            # Using freeze and slice is faster than slice!
+            # See https://github.com/fluent/fluentd/pull/2527
+            @buffer.freeze
+            rbuf = @buffer.slice(0, idx + 1)
+            @buffer = @buffer.slice(idx + 1, @buffer.size)
+            idx = @buffer.index(@eol)
+            lines << convert(rbuf)
+          end
         end
 
         def bytesize
@@ -733,9 +742,7 @@ module Fluent::Plugin
                 begin
                   while true
                     @fifo << io.readpartial(8192, @iobuf)
-                    while (line = @fifo.next_line)
-                      @lines << line
-                    end
+                    @fifo.read_lines(@lines)
                     if @lines.size >= @watcher.read_lines_limit
                       # not to use too much memory in case the file is very large
                       read_more = true

data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb

@@ -44,10 +44,10 @@ module Fluent
                   headers.each { |k, v| response[k] = v }
                   response.body = body
                 end
+              end
 
-                def render_json(code, obj)
-                  [code, { 'Content-Type' => 'application/json' }, obj.to_json]
-                end
+              def render_json(code, obj)
+                [code, { 'Content-Type' => 'application/json' }, obj.to_json]
               end
             end
           end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.6.2'
+  VERSION = '1.6.3'
 
 end

data/test/plugin/test_buf_file.rb

@@ -57,20 +57,30 @@ class FileBufferTest < Test::Unit::TestCase
       assert_equal File.join(@dir, 'buffer.*.file'), p.path
     end
 
-    test 'existing directory will be used with additional default file name' do
+    data('default' => [nil, 'log'],
+         'conf' => ['.buf', 'buf'])
+    test 'existing directory will be used with additional default file name' do |params|
+      conf, suffix = params
       d = FluentPluginFileBufferTest::DummyOutputPlugin.new
       p = Fluent::Plugin::FileBuffer.new
       p.owner = d
-      p.configure(config_element('buffer', '', {'path' => @dir}))
-      assert_equal File.join(@dir, 'buffer.*.log'), p.path
+      c = {'path' => @dir}
+      c['path_suffix'] = conf if conf
+      p.configure(config_element('buffer', '', c))
+      assert_equal File.join(@dir, "buffer.*.#{suffix}"), p.path
     end
 
-    test 'unexisting path without * handled as directory' do
+    data('default' => [nil, 'log'],
+         'conf' => ['.buf', 'buf'])
+    test 'unexisting path without * handled as directory' do |params|
+      conf, suffix = params
       d = FluentPluginFileBufferTest::DummyOutputPlugin.new
       p = Fluent::Plugin::FileBuffer.new
       p.owner = d
-      p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer')}))
-      assert_equal File.join(@dir, 'buffer', 'buffer.*.log'), p.path
+      c = {'path' => File.join(@dir, 'buffer')}
+      c['path_suffix'] = conf if conf
+      p.configure(config_element('buffer', '', c))
+      assert_equal File.join(@dir, 'buffer', "buffer.*.#{suffix}"), p.path
     end
   end
 
@@ -312,10 +322,6 @@ class FileBufferTest < Test::Unit::TestCase
       @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
       @p = Fluent::Plugin::FileBuffer.new
       @p.owner = @d
-      Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
-        @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
-        @p.configure(config_element('buffer', ''))
-      end
     end
 
     teardown do
@@ -329,8 +335,18 @@ class FileBufferTest < Test::Unit::TestCase
       end
     end
 
-    test '#start creates directory for buffer chunks' do
-      expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', 'buffer.*.log')
+    data('default' => [nil, 'log'],
+         'conf' => ['.buf', 'buf'])
+    test '#start creates directory for buffer chunks' do |params|
+      conf, suffix = params
+      c = {}
+      c['path_suffix'] = conf if conf
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
+        @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
+        @p.configure(config_element('buffer', '', c))
+      end
+
+      expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', "buffer.*.#{suffix}")
       expected_buffer_dir = File.dirname(expected_buffer_path)
       assert_equal expected_buffer_path, @p.path
       assert_false Dir.exist?(expected_buffer_dir)

data/test/plugin/test_in_syslog.rb

@@ -361,4 +361,31 @@ EOS
       msgs
     end
   end
+
+  def test_emit_unmatched_lines
+    d = create_driver([CONFIG, 'emit_unmatched_lines true'].join("\n"))
+    tests = [
+      # valid message
+      {'msg' => '<6>Sep 10 00:00:00 localhost logger: xxx', 'expected' => {'host'=>'localhost', 'ident'=>'logger', 'message'=>'xxx'}},
+      # missing priority
+      {'msg' => 'hello world', 'expected' => {'unmatched_line' => 'hello world'}},
+      # timestamp parsing failure
+      {'msg' => '<6>ZZZ 99 99:99:99 localhost logger: xxx', 'expected' => {'unmatched_line' => '<6>ZZZ 99 99:99:99 localhost logger: xxx'}},
+    ]
+
+    d.run(expect_emits: 3) do
+      u = UDPSocket.new
+      u.do_not_reverse_lookup = false
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+    end
+
+    assert_equal tests.size, d.events.size
+    tests.size.times do |i|
+      assert_equal tests[i]['expected'], d.events[i][2]
+      assert_equal 'syslog.unmatched', d.events[i][0] unless i==0
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.6.2
+  version: 1.6.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-12 00:00:00.000000000 Z
+date: 2019-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack