fluentd 0.14.1 → 0.14.2

data/test/plugin/test_in_syslog.rb

@@ -1,5 +1,5 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_syslog'
 
 class SyslogInputTest < Test::Unit::TestCase
@@ -14,7 +14,7 @@ class SyslogInputTest < Test::Unit::TestCase
       @server.close
     end
   end
-  
+
   def setup
     Fluent::Test.setup
     require 'fluent/plugin/socket_util'
@@ -34,7 +34,7 @@ class SyslogInputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf=CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::SyslogInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SyslogInput).configure(conf)
   end
 
   def test_configure
@@ -59,19 +59,18 @@ class SyslogInputTest < Test::Unit::TestCase
         {'msg' => '<6>Dec 11 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec 11 00:00:00', '%b %d %H:%M:%S'))},
         {'msg' => '<6>Dec  1 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec  1 00:00:00', '%b  %d %H:%M:%S'))},
       ]
-
-      d.run do
+      d.run(expect_emits: 2) do
         u = Fluent::SocketUtil.create_udp_socket(k)
         u.connect(k, PORT)
         tests.each {|test|
           u.send(test['msg'], 0)
         }
-        sleep 1
       end
 
-      emits = d.emits
-      emits.each_index {|i|
-        assert_equal_event_time(tests[i]['expected'], emits[i][1])
+      events = d.events
+      assert(events.size > 0)
+      events.each_index {|i|
+        assert_equal_event_time(tests[i]['expected'], events[i][1])
       }
     }
   end
@@ -80,48 +79,66 @@ class SyslogInputTest < Test::Unit::TestCase
     d = create_driver
     tests = create_test_case
 
-    d.run do
+    d.run(expect_emits: 2) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
       tests.each {|test|
         u.send(test['msg'], 0)
       }
-      sleep 1
     end
 
-    compare_test_result(d.emits, tests)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
+  end
+
+  def test_msg_size_udp_for_large_msg
+    d = create_driver(CONFIG + %[
+      message_length_limit 5k
+    ])
+    tests = create_test_case(large_message: true)
+
+    d.run(expect_emits: 3) do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+    end
+
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
   end
 
   def test_msg_size_with_tcp
     d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
     tests = create_test_case
 
-    d.run do
+    d.run(expect_emits: 2) do
       tests.each {|test|
         TCPSocket.open('127.0.0.1', PORT) do |s|
           s.send(test['msg'], 0)
         end
       }
-      sleep 1
     end
 
-    compare_test_result(d.emits, tests)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
   end
 
   def test_msg_size_with_same_tcp_connection
     d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
     tests = create_test_case
 
-    d.run do
+    d.run(expect_emits: 2) do
       TCPSocket.open('127.0.0.1', PORT) do |s|
         tests.each {|test|
           s.send(test['msg'], 0)
         }
       end
-      sleep 1
     end
 
-    compare_test_result(d.emits, tests)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
   end
 
   def test_msg_size_with_json_format
@@ -132,49 +149,57 @@ class SyslogInputTest < Test::Unit::TestCase
       {'msg' => '<6>' + event.to_json + "\n", 'expected' => msg}
     }
 
-    d.run do
+    d.run(expect_emits: 2) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
       tests.each {|test|
         u.send(test['msg'], 0)
       }
-      sleep 1
     end
 
-    compare_test_result(d.emits, tests)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
   end
 
   def test_msg_size_with_include_source_host
-    d = create_driver([CONFIG, 'include_source_host'].join("\n"))
+    d = create_driver([CONFIG, 'include_source_host true'].join("\n"))
     tests = create_test_case
 
     host = nil
-    d.run do
+    d.run(expect_emits: 2) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
       host = u.peeraddr[2]
       tests.each {|test|
         u.send(test['msg'], 0)
       }
-      sleep 1
     end
 
-    compare_test_result(d.emits, tests, host)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests, host)
   end
 
-  def create_test_case
+  def create_test_case(large_message: false)
     # actual syslog message has "\n"
-    [
-      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
-      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
-    ]
+    if large_message
+      [
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 4096 + "\n", 'expected' => 'x' * 4096},
+      ]
+    else
+      [
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
+      ]
+    end
   end
 
-  def compare_test_result(emits, tests, host = nil)
-    emits.each_index { |i|
-      assert_equal('syslog.kern.info', emits[0][0]) # <6> means kern.info
-      assert_equal(tests[i]['expected'], emits[i][2]['message'])
-      assert_equal(host, emits[i][2]['source_host']) if host
+  def compare_test_result(events, tests, host = nil)
+    events.each_index { |i|
+      assert_equal('syslog.kern.info', events[i][0]) # <6> means kern.info
+      assert_equal(tests[i]['expected'], events[i][2]['message'])
+      assert_equal(host, events[i][2]['source_host']) if host
     }
   end
 end

data/test/plugin/test_out_buffered_stdout.rb

@@ -16,7 +16,7 @@ class BufferedStdoutOutputTest < Test::Unit::TestCase
 
   test 'default configure' do
     d = create_driver
-    assert_equal 'json', d.instance.output_type
+    assert_equal [], d.instance.formatter_configs
     assert_equal 10 * 1024, d.instance.buffer_config.chunk_limit_size
     assert d.instance.buffer_config.flush_at_shutdown
     assert_equal ['tag'], d.instance.buffer_config.chunk_keys
@@ -27,16 +27,30 @@ class BufferedStdoutOutputTest < Test::Unit::TestCase
 
   test 'configure with output_type' do
     d = create_driver(CONFIG + "\noutput_type json")
-    assert_equal 'json', d.instance.output_type
+    assert_equal 'json', d.instance.formatter_configs.first[:@type]
 
     d = create_driver(CONFIG + "\noutput_type hash")
-    assert_equal 'hash', d.instance.output_type
+    assert_equal 'hash', d.instance.formatter_configs.first[:@type]
 
     assert_raise(Fluent::ConfigError) do
       d = create_driver(CONFIG + "\noutput_type foo")
     end
   end
 
+  sub_test_case "emit with default config" do
+    test '#write(synchronous)' do
+      d = create_driver
+      time = event_time()
+
+      out = capture_log do
+        d.run(default_tag: 'test', flush: true) do
+          d.feed(time, {'test' => 'test'})
+        end
+      end
+      assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test\"}\n", out
+    end
+  end
+
   sub_test_case "emit json" do
     data('oj' => 'oj', 'yajl' => 'yajl')
     test '#write(synchronous)' do |data|

data/test/plugin/test_out_forward.rb

@@ -107,7 +107,6 @@ class ForwardOutputTest < Test::Unit::TestCase
 
   def test_wait_response_timeout_config
     d = create_driver(CONFIG)
-    assert_equal false, d.instance.extend_internal_protocol
     assert_equal false, d.instance.require_ack_response
     assert_equal 190, d.instance.ack_response_timeout
 
@@ -115,11 +114,74 @@ class ForwardOutputTest < Test::Unit::TestCase
       require_ack_response true
       ack_response_timeout 2s
     ])
-    assert d.instance.extend_internal_protocol
     assert d.instance.require_ack_response
     assert_equal 2, d.instance.ack_response_timeout
   end
 
+  def test_sending_contains_with_ack
+    target_input_driver = create_target_input_driver(true)
+
+    d = create_driver(CONFIG + %[
+      ack_response_timeout 1s
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    d.register_run_post_condition do
+      d.instance.responses.length == 1
+    end
+
+    target_input_driver.run do
+      d.run do
+        records.each do |record|
+          d.emit record, time
+        end
+      end
+    end
+
+    emits = target_input_driver.emits
+    assert_equal ['test', time, records[0]], emits[0]
+    assert_equal ['test', time, records[1]], emits[1]
+
+    assert_equal target_input_driver.instance.received_options[0]['size'], 2
+  end
+
+  def test_sending_contains_without_ack
+    target_input_driver = create_target_input_driver(true)
+
+    d = create_driver(CONFIG + %[
+      ack_response_timeout 1s
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    d.register_run_post_condition do
+      d.instance.responses.length == 1
+    end
+
+    target_input_driver.run do
+      d.run do
+        records.each do |record|
+          d.emit record, time
+        end
+      end
+    end
+
+    emits = target_input_driver.emits
+    assert_equal ['test', time, records[0]], emits[0]
+    assert_equal ['test', time, records[1]], emits[1]
+
+    assert_equal target_input_driver.instance.received_options[0]['size'], 2
+  end
+
   def test_send_with_time_as_integer
     target_input_driver = create_target_input_driver
 
@@ -379,12 +441,18 @@ class ForwardOutputTest < Test::Unit::TestCase
     DummyEngineDriver.new(Fluent::ForwardInput) {
       handler_class = Class.new(Fluent::ForwardInput::Handler) { |klass|
         attr_reader :chunk_counter # for checking if received data is successfully deserialized
+        attr_reader :received_options
 
         def initialize(sock, log, on_message)
           @sock = sock
           @log = log
           @chunk_counter = 0
-          @on_message = on_message
+          @received_options = []
+          @on_message = ->(msg, chunk_size, source) {
+            option = on_message.call(msg, chunk_size, source)
+            @received_options << option
+            option
+          }
           @source = nil
         end
 
@@ -409,7 +477,7 @@ class ForwardOutputTest < Test::Unit::TestCase
         end
       }
 
-      define_method(:start) do
+      define_method(:_start) do
         @thread = Thread.new do
           Socket.tcp_server_loop(@bind, @port) do |sock, client_addrinfo|
             begin
@@ -431,12 +499,15 @@ class ForwardOutputTest < Test::Unit::TestCase
                 sock.close_write
                 sock.close
               end
+              @received_options = handler.received_options
             end
           end
         end
       end
 
-      def shutdown
+      attr_reader :received_options
+
+      def _shutdown
         @thread.kill
         @thread.join
       end
@@ -457,9 +528,26 @@ class ForwardOutputTest < Test::Unit::TestCase
     assert_equal node.available, true
   end
 
+  # To suppress calling `ForwardInput#start` in `DummyEngineDriver`,
+  # `DummyEnigneDriver` should avoid calling CallSuperMixin.prepend at `Fluent::Compat::Input#initialize`.
+  module SuppressCallSuperMixin
+    def start
+      _start
+    end
+
+    def before_shutdown
+      # nothing
+    end
+
+    def shutdown
+      _shutdown
+    end
+  end
+
   class DummyEngineDriver < Fluent::Test::TestDriver
     def initialize(klass, &block)
       super(klass, &block)
+      @instance.class.prepend(SuppressCallSuperMixin)
       @engine = DummyEngineClass.new
     end
 

data/test/plugin/test_out_stdout.rb

@@ -16,21 +16,32 @@ class StdoutOutputTest < Test::Unit::TestCase
 
   def test_configure
     d = create_driver
-    assert_equal 'json', d.instance.output_type
+    assert_equal [], d.instance.formatter_configs
   end
 
   def test_configure_output_type
     d = create_driver(CONFIG + "\noutput_type json")
-    assert_equal 'json', d.instance.output_type
+    assert_equal 'json', d.instance.formatter_configs.first[:@type]
 
     d = create_driver(CONFIG + "\noutput_type hash")
-    assert_equal 'hash', d.instance.output_type
+    assert_equal 'hash', d.instance.formatter_configs.first[:@type]
 
     assert_raise(Fluent::ConfigError) do
       d = create_driver(CONFIG + "\noutput_type foo")
     end
   end
 
+  def test_emit_in_default
+    d = create_driver
+    time = event_time()
+    out = capture_log do
+      d.run(default_tag: 'test') do
+        d.feed(time, {'test' => 'test1'})
+      end
+    end
+    assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test1\"}\n", out
+  end
+
   data('oj' => 'oj', 'yajl' => 'yajl')
   def test_emit_json(data)
     d = create_driver(CONFIG + "\noutput_type json\njson_parser #{data}")

data/test/plugin/test_output_as_buffered_retries.rb

@@ -161,6 +161,8 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:write){|chunk| raise "yay, your #write must fail" }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -200,6 +202,8 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:write){|chunk| raise "yay, your #write must fail" }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -252,6 +256,8 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -339,6 +345,8 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -417,6 +425,8 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:write){|chunk| raise "yay, your #write must fail" }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -460,6 +470,8 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -552,6 +564,8 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -647,6 +661,8 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -715,6 +731,8 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:write){|chunk| written_tags << chunk.metadata.tag; raise "yay, your #write must fail" }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )
 
@@ -780,6 +798,8 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.register(:try_write){|chunk| raise "yay, your #write must fail" }
       @i.start
 
+      @i.interrupt_flushes
+
       now = Time.parse('2016-04-13 18:33:30 -0700')
       Timecop.freeze( now )