fluentd 1.4.2-x64-mingw32 → 1.5.0-x64-mingw32

data/lib/fluent/plugin/parser_json.rb

@@ -30,6 +30,12 @@ module Fluent
       desc 'Set JSON parser'
       config_param :json_parser, :enum, list: [:oj, :yajl, :json], default: :oj
 
+      # The Yajl library defines a default buffer size of 8KiB when parsing
+      # from IO streams, so maintain this for backwards-compatibility.
+      # https://www.rubydoc.info/github/brianmario/yajl-ruby/Yajl%2FParser:parse
+      desc 'Set the buffer size that Yajl will use when parsing streaming input'
+      config_param :stream_buffer_size, :integer, default: 8192
+
       config_set_default :time_type, :float
 
       def configure(conf)
@@ -81,7 +87,7 @@ module Fluent
         y.on_parse_complete = ->(record){
           block.call(parse_time(record), record)
         }
-        y.parse(io)
+        y.parse(io, @stream_buffer_size)
       end
     end
   end

data/lib/fluent/plugin/storage_local.rb

@@ -87,7 +87,12 @@ module Fluent
           if File.exist?(@path)
             raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
             begin
-              data = Yajl::Parser.parse(open(@path, 'r:utf-8'){ |io| io.read })
+              data = open(@path, 'r:utf-8') { |io| io.read }
+              if data.empty?
+                log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
+                return
+              end
+              data = Yajl::Parser.parse(data)
               raise Fluent::ConfigError, "Invalid contents (not object) in plugin storage file: '#{@path}'" unless data.is_a?(Hash)
             rescue => e
               log.error "failed to read data from plugin storage file", path: @path, error: e
@@ -128,7 +133,7 @@ module Fluent
         tmp_path = @path + '.tmp'
         begin
           json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
-          open(tmp_path, 'w:utf-8', @mode){ |io| io.write json_string }
+          open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
           File.rename(tmp_path, @path)
         rescue => e
           log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e

data/lib/fluent/plugin_helper/server.rb

@@ -751,8 +751,15 @@ module Fluent
                 # Consider write_nonblock with {exception: false} when IO::WaitWritable error happens frequently.
                 written_bytes = @_handler_socket.write_nonblock(@_handler_write_buffer)
                 @_handler_write_buffer.slice!(0, written_bytes)
-                super
               end
+
+              # No need to call `super` in a synchronized context because TLSServer doesn't use the inner buffer(::IO::Buffer) of Coolio::IO.
+              # Instead of using Coolio::IO's inner buffer, TLSServer has own buffer(`@_handler_write_buffer`). See also TLSServer#write.
+              # Actually, the only reason calling `super` here is call Coolio::IO#disable_write_watcher.
+              # If `super` is called in a synchronized context, it could cause a mutex recursive locking since Coolio::IO#on_write_complete
+              # eventually calls TLSServer#close which try to get a lock.
+              super
+
               close if @close_after_write_complete
             rescue IO::WaitWritable, IO::WaitReadable
               return

data/lib/fluent/supervisor.rb

@@ -588,7 +588,7 @@ module Fluent
 
     def show_plugin_config
       name, type = @show_plugin_config.split(":") # input:tail
-      $log.info "Use fluent-plugin-config-format --format=txt #{name} #{type}"
+      $log.info "show_plugin_config option is deprecated. Use fluent-plugin-config-format --format=txt #{name} #{type}"
       exit 0
     end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.4.2'
+  VERSION = '1.5.0'
 
 end

data/test/plugin/test_filter_parser.rb

@@ -680,7 +680,7 @@ class ParserFilterTest < Test::Unit::TestCase
 
     def test_call_emit_error_event_when_pattern_is_mismached
       flexmock(@d.instance.router).should_receive(:emit_error_event).
-        with(String, Integer, Hash, ParserError.new("pattern not match with data '#{INVALID_MESSAGE}'")).once
+        with(String, Integer, Hash, ParserError.new("pattern not matched with data '#{INVALID_MESSAGE}'")).once
       @d.run do
         @d.feed(@tag, Fluent::EventTime.now.to_i, {'message' => INVALID_MESSAGE})
       end

data/test/plugin/test_in_forward.rb

@@ -82,6 +82,14 @@ class ForwardInputTest < Test::Unit::TestCase
       assert_equal 1, d.instance.security.clients.size
     end
 
+    data(tag: "tag",
+         add_tag_prefix: "add_tag_prefix")
+    test 'tag parameters' do |data|
+      assert_raise(Fluent::ConfigError.new("'#{data}' parameter must not be empty")) {
+        create_driver(CONFIG + "#{data} ''")
+      }
+    end
+
     test 'send_keepalive_packet is disabled by default' do
       @d = d = create_driver(CONFIG_AUTH)
       assert_false d.instance.send_keepalive_packet
@@ -269,6 +277,34 @@ class ForwardInputTest < Test::Unit::TestCase
       assert_equal(records, d.events)
     end
 
+    data(tag: {
+           param: "tag new_tag",
+           result: "new_tag"
+         },
+         add_tag_prefix: {
+           param: "add_tag_prefix new_prefix",
+           result: "new_prefix.tag1"
+         })
+    test 'tag parameters' do |data|
+      @d = create_driver(CONFIG + data[:param])
+      time = event_time("2011-01-02 13:14:15 UTC")
+      options = {auth: false}
+
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}],
+      ]
+
+      @d.run(expect_records: records.length, timeout: 20) do
+        entries = []
+        records.each {|tag, _time, record|
+          entries << [_time, record]
+        }
+        send_data packer.write(["tag1", entries]).to_s, **options
+      end
+      assert_equal(data[:result], @d.events[0][0])
+    end
+
     data(tcp: {
            config: CONFIG,
            options: {
@@ -378,6 +414,34 @@ class ForwardInputTest < Test::Unit::TestCase
       assert_equal(records, d.events)
     end
 
+    data(tag: {
+           param: "tag new_tag",
+           result: "new_tag"
+         },
+         add_tag_prefix: {
+           param: "add_tag_prefix new_prefix",
+           result: "new_prefix.tag1"
+         })
+    test 'tag parameters' do |data|
+      @d = create_driver(CONFIG + data[:param])
+      time = event_time("2011-01-02 13:14:15 UTC")
+      options = {auth: false}
+
+      records = [
+        ["tag1", time, {"a"=>1}],
+        ["tag1", time, {"a"=>2}],
+      ]
+
+      @d.run(expect_records: records.length, timeout: 20) do
+        entries = ''
+        records.each {|_tag, _time, record|
+          packer(entries).write([_time, record]).flush
+        }
+        send_data packer.write(["tag1", entries]).to_s, **options
+      end
+      assert_equal(data[:result], @d.events[0][0])
+    end
+
     data(tcp: {
            config: CONFIG,
            options: {

data/test/plugin/test_in_syslog.rb

@@ -55,6 +55,35 @@ EOS
     end
   end
 
+  data('Use protocol_type' => ['protocol_type tcp', :tcp, :udp],
+       'Use transport' => ["<transport tcp>\n </transport>", nil, :tcp],
+       'Use transport and protocol' => ["protocol_type udp\n<transport tcp>\n </transport>", :udp, :tcp])
+  def test_configure_protocol(param)
+    conf, proto_type, transport_proto_type = *param
+    d = create_driver([CONFIG, conf].join("\n"))
+
+    assert_equal(d.instance.protocol_type, proto_type)
+    assert_equal(d.instance.transport_config.protocol, transport_proto_type)
+  end
+
+  # For backward compat
+  def test_respect_protocol_type_than_transport
+    d = create_driver([CONFIG, "<transport tcp> \n</transport>", "protocol_type udp"].join("\n"))
+    tests = create_test_case
+
+    d.run(expect_emits: 2) do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+    end
+
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
+  end
+
+
   data(
     ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
     ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
@@ -119,7 +148,7 @@ EOS
   end
 
   def test_msg_size_with_tcp
-    d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
+    d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
     tests = create_test_case
 
     d.run(expect_emits: 2) do
@@ -135,7 +164,7 @@ EOS
   end
 
   def test_msg_size_with_same_tcp_connection
-    d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
+    d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
     tests = create_test_case
 
     d.run(expect_emits: 2) do
@@ -289,7 +318,7 @@ EOS
 
   sub_test_case 'octet counting frame' do
     def test_msg_size_with_tcp
-      d = create_driver([CONFIG, 'protocol_type tcp', 'frame_type octet_count'].join("\n"))
+      d = create_driver([CONFIG, "<transport tcp> \n</transport>", 'frame_type octet_count'].join("\n"))
       tests = create_test_case
 
       d.run(expect_emits: 2) do
@@ -305,7 +334,7 @@ EOS
     end
 
     def test_msg_size_with_same_tcp_connection
-      d = create_driver([CONFIG, 'protocol_type tcp', 'frame_type octet_count'].join("\n"))
+      d = create_driver([CONFIG, "<transport tcp> \n</transport>", 'frame_type octet_count'].join("\n"))
       tests = create_test_case
 
       d.run(expect_emits: 2) do

data/test/plugin/test_out_exec_filter.rb

@@ -328,6 +328,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     </format>
     <parse>
       @type json
+      stream_buffer_size 1
     </parse>
     <extract>
       tag_key tag
@@ -338,6 +339,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     command cat
     in_keys message
     out_format json
+    out_stream_buffer_size 1
     time_key time
     tag_key tag
   ]
@@ -372,6 +374,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     </format>
     <parse>
       @type json
+      stream_buffer_size 1
     </parse>
     <extract>
       tag_key tag
@@ -382,6 +385,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     command cat
     in_keys message
     out_format json
+    out_stream_buffer_size 1
     time_key time
     tag_key tag
   ]
@@ -414,6 +418,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     </format>
     <parse>
       @type json
+      stream_buffer_size 1
     </parse>
     <extract>
       tag_key tag
@@ -426,6 +431,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     command cat
     in_keys message
     out_format json
+    out_stream_buffer_size 1
     time_key time
     time_format %d/%b/%Y %H:%M:%S.%N %z
     tag_key tag

data/test/plugin/test_out_forward.rb

@@ -920,4 +920,213 @@ EOL
       assert_equal(['test', time, records[1]], events[1])
     end
   end
+
+  test 'Create new connection per send_data' do
+    target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
+    output_conf = CONFIG
+    d = create_driver(output_conf)
+    d.instance_start
+
+    begin
+      chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+      mock.proxy(d.instance).create_transfer_socket(TARGET_HOST, TARGET_PORT, 'test') { |sock| mock(sock).close.once; sock }.twice
+
+      target_input_driver.run(timeout: 15) do
+        d.run(shutdown: false) do
+          node = d.instance.nodes.first
+          2.times do
+            node.send_data('test', chunk) rescue nil
+          end
+        end
+      end
+    ensure
+      d.instance_shutdown
+    end
+  end
+
+  sub_test_case 'keepalive' do
+    test 'Do not create connection per send_data' do
+      target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
+      output_conf = CONFIG + %[
+        keepalive true
+        keepalive_timeout 2
+      ]
+      d = create_driver(output_conf)
+      d.instance_start
+
+      begin
+        chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+        mock.proxy(d.instance).create_transfer_socket(TARGET_HOST, TARGET_PORT, 'test') { |sock| mock(sock).close.once; sock }.once
+
+        target_input_driver.run(timeout: 15) do
+          d.run(shutdown: false) do
+            node = d.instance.nodes.first
+            2.times do
+              node.send_data('test', chunk) rescue nil
+            end
+          end
+        end
+      ensure
+        d.instance_shutdown
+      end
+    end
+
+    sub_test_case 'with require_ack_response' do
+      test 'Do not create connection per send_data' do
+        target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
+        output_conf = CONFIG + %[
+          require_ack_response true
+          keepalive true
+          keepalive_timeout 2
+        ]
+        d = create_driver(output_conf)
+        d.instance_start
+
+        begin
+          chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+          mock.proxy(d.instance).create_transfer_socket(TARGET_HOST, TARGET_PORT, 'test') { |sock| mock(sock).close.once; sock }.once
+
+          target_input_driver.run(timeout: 15) do
+            d.run(shutdown: false) do
+              node = d.instance.nodes.first
+              2.times do
+                node.send_data('test', chunk) rescue nil
+              end
+            end
+          end
+        ensure
+          d.instance_shutdown
+        end
+      end
+    end
+  end
+
+  sub_test_case 'SocketCache' do
+    sub_test_case 'fetch_or' do
+      test 'when gived key does not exist' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        sock = mock!.open { 1 }.subject
+        assert_equal(1, c.fetch_or { sock.open })
+      end
+
+      test 'when given key exists' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        assert_equal(1, c.fetch_or { 1 })
+
+        sock = dont_allow(mock!).open
+        assert_equal(1, c.fetch_or { sock.open })
+      end
+
+      test "when given key's value was expired" do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(0, Logger.new(nil))
+        assert_equal(1, c.fetch_or { 1 })
+
+        sock = mock!.open { 1 }.subject
+        assert_equal(1, c.fetch_or { sock.open })
+      end
+    end
+
+    test 'revoke' do
+      c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+      c.fetch_or { 1 }
+      c.revoke
+
+      sock = mock!.open { 1 }.subject
+      assert_equal(1, c.fetch_or { sock.open })
+    end
+
+    test 'revoke_by_value' do
+      c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+      c.fetch_or { 1 }
+      c.revoke_by_value(1)
+
+      sock = mock!.open { 1 }.subject
+      assert_equal(1, c.fetch_or { sock.open })
+    end
+
+    sub_test_case 'dec_ref' do
+      test 'when value exists in active_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        c.fetch_or { 1 }
+        c.dec_ref
+
+        assert_equal(0, c.instance_variable_get(:@active_socks)[Thread.current.object_id].ref)
+      end
+
+      test 'when value exists in inactive_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        c.fetch_or { 1 }
+        c.revoke
+        c.dec_ref
+        assert_equal(-1, c.instance_variable_get(:@inactive_socks)[Thread.current.object_id].ref)
+      end
+    end
+
+    sub_test_case 'dec_ref_by_value' do
+      test 'when value exists in active_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        c.fetch_or { 1 }
+        c.dec_ref_by_value(1)
+
+        assert_equal(0, c.instance_variable_get(:@active_socks)[Thread.current.object_id].ref)
+      end
+
+      test 'when value exists in inactive_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        c.fetch_or { 1 }
+        c.revoke
+        c.dec_ref_by_value(1)
+        assert_equal(-1, c.instance_variable_get(:@inactive_socks)[Thread.current.object_id].ref)
+      end
+    end
+
+    sub_test_case 'clear' do
+      test 'when value is in active_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        m = mock!.close { 'closed' }.subject
+        c.fetch_or { m }
+        assert_true(!c.instance_variable_get(:@active_socks).empty?)
+
+        c.clear
+        assert_true(c.instance_variable_get(:@active_socks).empty?)
+      end
+
+      test 'when value is in inactive_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        m = mock!.close { 'closed' }.subject
+        c.fetch_or { m }
+        c.revoke
+        assert_true(!c.instance_variable_get(:@inactive_socks).empty?)
+
+        c.clear
+        assert_true(c.instance_variable_get(:@active_socks).empty?)
+      end
+    end
+
+    sub_test_case 'purge_obsolete_socks' do
+      test 'delete key in inactive_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        m = mock!.close { 'closed' }.subject
+        c.fetch_or { m }
+        c.revoke
+        assert_true(!c.instance_variable_get(:@inactive_socks).empty?)
+
+        c.purge_obsolete_socks
+        assert_true(c.instance_variable_get(:@active_socks).empty?)
+      end
+
+      test 'move key from active_socks to inactive_socks' do
+        c = Fluent::Plugin::ForwardOutput::Node::SocketCache.new(10, Logger.new(nil))
+        m = dont_allow(mock!).close
+        stub(m).inspect         # for log
+        c.fetch_or { m }
+        assert_true(!c.instance_variable_get(:@active_socks).empty?)
+        assert_true(c.instance_variable_get(:@inactive_socks).empty?)
+
+        c.purge_obsolete_socks
+        assert_true(!c.instance_variable_get(:@active_socks).empty?)
+        assert_true(c.instance_variable_get(:@inactive_socks).empty?)
+      end
+    end
+  end
 end