fluentd 1.6.2-x86-mingw32 → 1.6.3-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d19e93ee2d9ede08b4ed121d4dce925843febaae5108d4236498803d75f2ce91
-  data.tar.gz: cf69025feeb193aca87f0380f19beba3b5f4761e54398bb2e394dbad251daf8e
+  metadata.gz: eb2e03cd7a40b2276b0ef281ee57e27af4a16204b8da766d1a97c9ae145a1b02
+  data.tar.gz: 91066de64539ac2a06fa97a730702ce41fb53ecab5818824c26630cacaa756e7
 SHA512:
-  metadata.gz: a2da5d490ec08e060c213d360d34f5fc52cb8591b637ea1d7cda0b08f0f8b7596e78552e6ceffea71c294f594816da14d70177fb2e1fb08327fba386ccf2223d
-  data.tar.gz: a8f524a310c5f5d1c72e954a7e8206018e31ca7923a6c62e761acbb6de3028fc223e7c76c32c8a7e46851bbbe7cb2795d729003ec7b5a39b0885d934f4109882
+  metadata.gz: 7db0784aeb7136fbc9e6057855ff1c01235acfe4bea6b526d1ca82d01ce40e379a7be9c23534904ee14f036c18c5f3dbeb863bdd141742875b3494057f2a373d
+  data.tar.gz: d1b838408dcc95821fe4cd8487690a1ea7e684157189d743d9e9f94dcfebe2d799b3321ecb3aeb05c08c87d2264a98377fb020af8f3b6d62f78323c5d6f90c03

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # v1.6
 
+## Release v1.6.3 - 2019/07/29
+
+### Enhancement
+
+* in_syslog: Add `emit_unmatched_lines` parameter
+  https://github.com/fluent/fluentd/pull/2499
+* buf_file: Add `path_suffix` parameter
+  https://github.com/fluent/fluentd/pull/2524
+* in_tail: Improve the performance of split lines
+  https://github.com/fluent/fluentd/pull/2527
+
+### Bug fixes
+
+* http_server: Fix re-define render_json method
+  https://github.com/fluent/fluentd/pull/2517
+
 ## Release v1.6.2 - 2019/07/11
 
 ### Bug fixes

data/lib/fluent/plugin/buf_file.rb

@@ -34,6 +34,8 @@ module Fluent
 
       desc 'The path where buffer chunks are stored.'
       config_param :path, :string, default: nil
+      desc 'The suffix of buffer chunks'
+      config_param :path_suffix, :string, default: '.log'
 
       config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
       config_set_default :total_limit_size, DEFAULT_TOTAL_LIMIT_SIZE
@@ -78,12 +80,12 @@ module Fluent
 
         if specified_directory_exists || unexisting_path_for_directory # directory
           if using_plugin_root_dir || !multi_workers_configured
-            @path = File.join(@path, 'buffer.*.log')
+            @path = File.join(@path, "buffer.*#{@path_suffix}")
           else
-            @path = File.join(@path, "worker#{fluentd_worker_id}", 'buffer.*.log')
+            @path = File.join(@path, "worker#{fluentd_worker_id}", "buffer.*#{@path_suffix}")
             if fluentd_worker_id == 0
               # worker 0 always checks unflushed buffer chunks to be resumed (might be created while non-multi-worker configuration)
-              @additional_resume_path = File.join(File.expand_path("../../", @path), 'buffer.*.log')
+              @additional_resume_path = File.join(File.expand_path("../../", @path), "buffer.*#{@path_suffix}")
             end
           end
           @multi_workers_available = true
@@ -91,10 +93,10 @@ module Fluent
           if File.basename(@path).include?('.*.')
             # valid file path
           elsif File.basename(@path).end_with?('.*')
-            @path = @path + '.log'
+            @path = @path + @path_suffix
           else
             # existing file will be ignored
-            @path = @path + '.*.log'
+            @path = @path + ".*#{@path_suffix}"
           end
           @multi_workers_available = false
         end

data/lib/fluent/plugin/in_syslog.rb

@@ -82,6 +82,8 @@ module Fluent::Plugin
     config_param :include_source_host, :bool, default: false, deprecated: 'use "source_hostname_key" or "source_address_key" instead.'
     desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, default: 'source_host'.freeze
+    desc 'Enable the option to emit unmatched lines.'
+    config_param :emit_unmatched_lines, :bool, default: false
 
     desc 'The field name of hostname of sender.'
     config_param :source_hostname_key, :string, default: nil
@@ -203,6 +205,9 @@ module Fluent::Plugin
       unless @parser_parse_priority
         m = SYSLOG_REGEXP.match(data)
         unless m
+          if @emit_unmatched_lines
+            emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => data})
+          end
           log.warn "invalid syslog message: #{data.dump}"
           return
         end
@@ -212,6 +217,9 @@ module Fluent::Plugin
 
       @parser.parse(text) do |time, record|
         unless time && record
+          if @emit_unmatched_lines
+            emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => text})
+          end
           log.warn "failed to parse message", data: data
           return
         end
@@ -229,6 +237,9 @@ module Fluent::Plugin
         emit(tag, time, record)
       end
     rescue => e
+      if @emit_unmatched_lines
+        emit("#{@tag}.unmatched", Fluent::EventTime.now, {"unmatched_line" => text})
+      end
       log.error "invalid input", data: data, error: e
       log.error_backtrace
     end

data/lib/fluent/plugin/in_tail.rb

@@ -698,9 +698,18 @@ module Fluent::Plugin
           s.encode!(@encoding, @from_encoding, :invalid => :replace, :undef => :replace)
         end
 
-        def next_line
+        def read_lines(lines)
           idx = @buffer.index(@eol)
-          convert(@buffer.slice!(0, idx + 1)) unless idx.nil?
+
+          until idx.nil?
+            # Using freeze and slice is faster than slice!
+            # See https://github.com/fluent/fluentd/pull/2527
+            @buffer.freeze
+            rbuf = @buffer.slice(0, idx + 1)
+            @buffer = @buffer.slice(idx + 1, @buffer.size)
+            idx = @buffer.index(@eol)
+            lines << convert(rbuf)
+          end
         end
 
         def bytesize
@@ -733,9 +742,7 @@ module Fluent::Plugin
                 begin
                   while true
                     @fifo << io.readpartial(8192, @iobuf)
-                    while (line = @fifo.next_line)
-                      @lines << line
-                    end
+                    @fifo.read_lines(@lines)
                     if @lines.size >= @watcher.read_lines_limit
                       # not to use too much memory in case the file is very large
                       read_more = true

data/lib/fluent/plugin_helper/http_server/compat/webrick_handler.rb

@@ -44,10 +44,10 @@ module Fluent
                   headers.each { |k, v| response[k] = v }
                   response.body = body
                 end
+              end
 
-                def render_json(code, obj)
-                  [code, { 'Content-Type' => 'application/json' }, obj.to_json]
-                end
+              def render_json(code, obj)
+                [code, { 'Content-Type' => 'application/json' }, obj.to_json]
               end
             end
           end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.6.2'
+  VERSION = '1.6.3'
 
 end

data/test/plugin/test_buf_file.rb

@@ -57,20 +57,30 @@ class FileBufferTest < Test::Unit::TestCase
       assert_equal File.join(@dir, 'buffer.*.file'), p.path
     end
 
-    test 'existing directory will be used with additional default file name' do
+    data('default' => [nil, 'log'],
+         'conf' => ['.buf', 'buf'])
+    test 'existing directory will be used with additional default file name' do |params|
+      conf, suffix = params
       d = FluentPluginFileBufferTest::DummyOutputPlugin.new
       p = Fluent::Plugin::FileBuffer.new
       p.owner = d
-      p.configure(config_element('buffer', '', {'path' => @dir}))
-      assert_equal File.join(@dir, 'buffer.*.log'), p.path
+      c = {'path' => @dir}
+      c['path_suffix'] = conf if conf
+      p.configure(config_element('buffer', '', c))
+      assert_equal File.join(@dir, "buffer.*.#{suffix}"), p.path
     end
 
-    test 'unexisting path without * handled as directory' do
+    data('default' => [nil, 'log'],
+         'conf' => ['.buf', 'buf'])
+    test 'unexisting path without * handled as directory' do |params|
+      conf, suffix = params
       d = FluentPluginFileBufferTest::DummyOutputPlugin.new
       p = Fluent::Plugin::FileBuffer.new
       p.owner = d
-      p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer')}))
-      assert_equal File.join(@dir, 'buffer', 'buffer.*.log'), p.path
+      c = {'path' => File.join(@dir, 'buffer')}
+      c['path_suffix'] = conf if conf
+      p.configure(config_element('buffer', '', c))
+      assert_equal File.join(@dir, 'buffer', "buffer.*.#{suffix}"), p.path
     end
   end
 
@@ -312,10 +322,6 @@ class FileBufferTest < Test::Unit::TestCase
       @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
       @p = Fluent::Plugin::FileBuffer.new
       @p.owner = @d
-      Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
-        @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
-        @p.configure(config_element('buffer', ''))
-      end
     end
 
     teardown do
@@ -329,8 +335,18 @@ class FileBufferTest < Test::Unit::TestCase
       end
     end
 
-    test '#start creates directory for buffer chunks' do
-      expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', 'buffer.*.log')
+    data('default' => [nil, 'log'],
+         'conf' => ['.buf', 'buf'])
+    test '#start creates directory for buffer chunks' do |params|
+      conf, suffix = params
+      c = {}
+      c['path_suffix'] = conf if conf
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
+        @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
+        @p.configure(config_element('buffer', '', c))
+      end
+
+      expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', "buffer.*.#{suffix}")
       expected_buffer_dir = File.dirname(expected_buffer_path)
       assert_equal expected_buffer_path, @p.path
       assert_false Dir.exist?(expected_buffer_dir)

data/test/plugin/test_in_syslog.rb

@@ -361,4 +361,31 @@ EOS
       msgs
     end
   end
+
+  def test_emit_unmatched_lines
+    d = create_driver([CONFIG, 'emit_unmatched_lines true'].join("\n"))
+    tests = [
+      # valid message
+      {'msg' => '<6>Sep 10 00:00:00 localhost logger: xxx', 'expected' => {'host'=>'localhost', 'ident'=>'logger', 'message'=>'xxx'}},
+      # missing priority
+      {'msg' => 'hello world', 'expected' => {'unmatched_line' => 'hello world'}},
+      # timestamp parsing failure
+      {'msg' => '<6>ZZZ 99 99:99:99 localhost logger: xxx', 'expected' => {'unmatched_line' => '<6>ZZZ 99 99:99:99 localhost logger: xxx'}},
+    ]
+
+    d.run(expect_emits: 3) do
+      u = UDPSocket.new
+      u.do_not_reverse_lookup = false
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+    end
+
+    assert_equal tests.size, d.events.size
+    tests.size.times do |i|
+      assert_equal tests[i]['expected'], d.events[i][2]
+      assert_equal 'syslog.unmatched', d.events[i][0] unless i==0
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.6.2
+  version: 1.6.3
 platform: x86-mingw32
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-12 00:00:00.000000000 Z
+date: 2019-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack