fluentd 1.6.2-x64-mingw32 → 1.6.3-x64-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87303e3a8f368b5f5c7ca75d8cfba43a5fe52e84ff5ff455111bdcc750ea3f8a
-  data.tar.gz: b16f78a81686875a87f401471ebeab06bd2d6deb39baf422c823fd9b3ac8cd30
+  metadata.gz: 45fffc9591fcaf0cf6ebefacd4e33db5351cedc2ab7d24c749f1305a8533786c
+  data.tar.gz: d9d0fbcfe1948ace02115879ae98a313de2a4708acff74eca2a5ca4cdf239dca
 SHA512:
-  metadata.gz: 0a0676a4eef431b603a2c3117fe90f588f544bf53fdc9eece96112fc19ad1179bb6dd4261db783d29b913e48e6f7624e2b524356cd010535bee425664cfbfc5a
-  data.tar.gz: 6abb329408250d0dcce520762e68ed8845921f9b15f22383845879c7e4a7754d39a4d94231a7ba54dba620a99bec55fa85e4699d7555e5c2f44110a535b8047a
+  metadata.gz: 7854f4483d30757c8d5b632328e5c38ea5cfb70bbed50425a4eae74e4cf0dc486f741f4401c6b609253ad1f5acc175f6edf14da0c6ae4f0e1abfc7ca892d4fb5
+  data.tar.gz: 5099c966f5de1e97f3c0af5f8994ab18f9bed746c7de961f96e12b5e651ab1e8087d47289d6f2bd272a0983a7a824bc9fe0a2c9dad2e89990c9f72b6eb58a6f3

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # v1.6
 
+## Release v1.6.3 - 2019/07/29
+
+### Enhancement
+
+* in_syslog: Add `emit_unmatched_lines` parameter
+  https://github.com/fluent/fluentd/pull/2499
+* buf_file: Add `path_suffix` parameter
+  https://github.com/fluent/fluentd/pull/2524
+* in_tail: Improve the performance of split lines
+  https://github.com/fluent/fluentd/pull/2527
+
+### Bug fixes
+
+* http_server: Fix re-define render_json method
+  https://github.com/fluent/fluentd/pull/2517
+
 ## Release v1.6.2 - 2019/07/11
 
 ### Bug fixes

data/lib/fluent/plugin/buf_file.rb

@@ -34,6 +34,8 @@ module Fluent
 
       desc 'The path where buffer chunks are stored.'
       config_param :path, :string, default: nil
+      desc 'The suffix of buffer chunks'
+      config_param :path_suffix, :string, default: '.log'
 
       config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
       config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
@@ -78,12 +80,12 @@ module Fluent
 
         if specified_directory_exists || unexisting_path_for_directory # directory
           if using_plugin_root_dir || !multi_workers_configured
-            @path = File.join(@path, 'buffer.*.log')
+            @path = File.join(@path, "buffer.*#{@path_suffix}")
           else
-            @path = File.join(@path, "worker#{fluentd_worker_id}", 'buffer.*.log')
+            @path = File.join(@path, "worker#{fluentd_worker_id}", "buffer.*#{@path_suffix}")
             if fluentd_worker_id == 0
               # worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)
-              @additional_resume_path = File.join(File.expand_path("../../", @path), 'buffer.*.log')
+              @additional_resume_path = File.join(File.expand_path("../../", @path), "buffer.*#{@path_suffix}")
             end
           end
           @multi_workers_available = true
@@ -91,10 +93,10 @@ module Fluent
           if File.basename(@path).include?('.*.')
             # valid file path
           elsif File.basename(@path).end_with?('.*')
-            @path = @path + '.log'
+            @path = @path + @path_suffix
           else
             # existing file will be ignored
-            @path = @path + '.*.log'
+            @path = @path + ".*#{@path_suffix}"
           end
           @multi_workers_available = false
         end

data/lib/fluent/plugin/in_syslog.rb

@@ -82,6 +82,8 @@ module Fluent::Plugin
     config_param :include_source_host, :bool, default: false, deprecated: 'use "source_hostname_key" or "source_address_key" instead.'
     desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, default: 'source_host'.freeze
+    desc 'Enable the option to emit unmatched lines.'
+    config_param :emit_unmatched_lines, :bool, default: false
 
     desc 'The field name of hostname of sender.'
     config_param :source_hostname_key, :string, default: nil
@@ -203,6 +205,9 @@ module Fluent::Plugin
       unless @parser_parse_priority
         m = SYSLOG_REGEXP.match(data)
         unless m
+          if @emit_unmatched_lines
+            emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => data})
+          end
           log.warn "invalid syslog message: #{data.dump}"
           return
         end
@@ -212,6 +217,9 @@ module Fluent::Plugin
 
       @parser.parse(text) do |time, record|
         unless time && record
+          if @emit_unmatched_lines
+            emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => text})
+          end
           log.warn "failed to parse message", data: data
           return
         end
@@ -229,6 +237,9 @@ module Fluent::Plugin
         emit(tag, time, record)
       end
     rescue => e
+      if @emit_unmatched_lines
+        emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => text})
+      end
       log.error "invalid input", data: data, error: e
       log.error_backtrace
     end

data/lib/fluent/plugin/in_tail.rb

@@ -698,9 +698,18 @@ module Fluent::Plugin
           s.encode!(@encoding, @from_encoding, :invalid => :replace, :undef => :replace)
         end
 
-        def next_line
+        def read_lines(lines)
           idx = @buffer.index(@eol)
-          convert(@buffer.slice!(0, idx + 1)) unless idx.nil?
+
+          until idx.nil?
+            # Using freeze and slice is faster than slice!
+            # See https://github.com/fluent/fluentd/pull/2527
+            @buffer.freeze
+            rbuf = @buffer.slice(0, idx + 1)
+            @buffer = @buffer.slice(idx + 1, @buffer.size)
+            idx = @buffer.index(@eol)
+            lines << convert(rbuf)
+          end
         end
 
         def bytesize
@@ -733,9 +742,7 @@ module Fluent::Plugin
                 begin
                   while true
                     @fifo << io.readpartial(8192, @iobuf)
-                    while (line = @fifo.next_line)
-                      @lines << line
-                    end
+                    @fifo.read_lines(@lines)
                     if @lines.size >= @watcher.read_lines_limit
                       # not to use too much memory in case the file is very large
                       read_more = true

data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb

@@ -44,10 +44,10 @@ module Fluent
                   headers.each { |k, v| response[k] = v }
                   response.body = body
                 end
+              end
 
-                def render_json(code, obj)
-                  [code, { 'Content-Type' => 'application/json' }, obj.to_json]
-                end
+              def render_json(code, obj)
+                [code, { 'Content-Type' => 'application/json' }, obj.to_json]
               end
             end
           end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.6.2'
+  VERSION = '1.6.3'
 
 end

data/test/plugin/test_buf_file.rb

@@ -57,20 +57,30 @@ class FileBufferTest < Test::Unit::TestCase
       assert_equal File.join(@dir, 'buffer.*.file'), p.path
     end
 
-    test 'existing directory will be used with additional default file name' do
+    data('default' => [nil, 'log'],
+         'conf' => ['.buf', 'buf'])
+    test 'existing directory will be used with additional default file name' do |params|
+      conf, suffix = params
       d = FluentPluginFileBufferTest::DummyOutputPlugin.new
       p = Fluent::Plugin::FileBuffer.new
       p.owner = d
-      p.configure(config_element('buffer', '', {'path' => @dir}))
-      assert_equal File.join(@dir, 'buffer.*.log'), p.path
+      c = {'path' => @dir}
+      c['path_suffix'] = conf if conf
+      p.configure(config_element('buffer', '', c))
+      assert_equal File.join(@dir, "buffer.*.#{suffix}"), p.path
     end
 
-    test 'unexisting path without * handled as directory' do
+    data('default' => [nil, 'log'],
+         'conf' => ['.buf', 'buf'])
+    test 'unexisting path without * handled as directory' do |params|
+      conf, suffix = params
       d = FluentPluginFileBufferTest::DummyOutputPlugin.new
       p = Fluent::Plugin::FileBuffer.new
       p.owner = d
-      p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer')}))
-      assert_equal File.join(@dir, 'buffer', 'buffer.*.log'), p.path
+      c = {'path' => File.join(@dir, 'buffer')}
+      c['path_suffix'] = conf if conf
+      p.configure(config_element('buffer', '', c))
+      assert_equal File.join(@dir, 'buffer', "buffer.*.#{suffix}"), p.path
     end
   end
 
@@ -312,10 +322,6 @@ class FileBufferTest < Test::Unit::TestCase
       @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
       @p = Fluent::Plugin::FileBuffer.new
       @p.owner = @d
-      Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
-        @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
-        @p.configure(config_element('buffer', ''))
-      end
     end
 
     teardown do
@@ -329,8 +335,18 @@ class FileBufferTest < Test::Unit::TestCase
       end
     end
 
-    test '#start creates directory for buffer chunks' do
-      expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', 'buffer.*.log')
+    data('default' => [nil, 'log'],
+         'conf' => ['.buf', 'buf'])
+    test '#start creates directory for buffer chunks' do |params|
+      conf, suffix = params
+      c = {}
+      c['path_suffix'] = conf if conf
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
+        @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
+        @p.configure(config_element('buffer', '', c))
+      end
+
+      expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', "buffer.*.#{suffix}")
       expected_buffer_dir = File.dirname(expected_buffer_path)
       assert_equal expected_buffer_path, @p.path
       assert_false Dir.exist?(expected_buffer_dir)

data/test/plugin/test_in_syslog.rb

@@ -361,4 +361,31 @@ EOS
       msgs
     end
   end
+
+  def test_emit_unmatched_lines
+    d = create_driver([CONFIG, 'emit_unmatched_lines true'].join("\n"))
+    tests = [
+      # valid message
+      {'msg' => '<6>Sep 10 00:00:00 localhost logger: xxx', 'expected' => {'host'=>'localhost', 'ident'=>'logger', 'message'=>'xxx'}},
+      # missing priority
+      {'msg' => 'hello world', 'expected' => {'unmatched_line' => 'hello world'}},
+      # timestamp parsing failure
+      {'msg' => '<6>ZZZ 99 99:99:99 localhost logger: xxx', 'expected' => {'unmatched_line' => '<6>ZZZ 99 99:99:99 localhost logger: xxx'}},
+    ]
+
+    d.run(expect_emits: 3) do
+      u = UDPSocket.new
+      u.do_not_reverse_lookup = false
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+    end
+
+    assert_equal tests.size, d.events.size
+    tests.size.times do |i|
+      assert_equal tests[i]['expected'], d.events[i][2]
+      assert_equal 'syslog.unmatched', d.events[i][0] unless i==0
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.6.2
+  version: 1.6.3
 platform: x64-mingw32
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-12 00:00:00.000000000 Z
+date: 2019-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack