fluentd 1.9.3 → 1.11.1

data/test/plugin/test_in_unix.rb

@@ -1,126 +1,181 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_unix'
 
-module StreamInputTest
+class UnixInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    @d = nil
   end
 
-  def test_time
-    d = create_driver
+  def teardown
+    @d.instance_shutdown if @d
+  end
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-    Fluent::Engine.now = time
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/in_unix#{ENV['TEST_ENV_NUMBER']}"
+  CONFIG = %[
+    path #{TMP_DIR}/unix
+    backlog 1000
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::UnixInput).configure(conf)
+  end
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+  def packer(*args)
+    Fluent::MessagePackFactory.msgpack_packer(*args)
+  end
 
-    d.run do
-      d.expected_emits.each {|tag,_time,record|
-        send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, 0, record]).to_s
+  def unpacker
+    Fluent::MessagePackFactory.msgpack_unpacker
+  end
+
+  def send_data(data)
+    io = UNIXSocket.new("#{TMP_DIR}/unix")
+    begin
+      io.write data
+    ensure
+      io.close
+    end
+  end
+
+  def test_configure
+    @d = create_driver
+    assert_equal "#{TMP_DIR}/unix", @d.instance.path
+    assert_equal 1000, @d.instance.backlog
+  end
+
+  def test_time
+    @d = create_driver
+
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", 0, {"a" => 1}],
+      ["tag2", nil, {"a" => 2}],
+    ]
+
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
       }
     end
+
+    @d.events.each_with_index { |e, i|
+      orig = records[i]
+      assert_equal(orig[0], e[0])
+      assert_true(time <= e[1])
+      assert_equal(orig[2], e[2])
+    }
   end
 
   def test_message
-    d = create_driver
+    @d = create_driver
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
 
-    d.run do
-      d.expected_emits.each {|tag,_time,record|
-        send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, _time, record]).to_s
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
       }
     end
+
+    assert_equal(records, @d.events)
   end
 
   def test_forward
-    d = create_driver
+    @d = create_driver
 
     time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag1", time, {"a" => 2}]
+    ]
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag1", time, {"a"=>2}
-
-    d.run do
+    @d.run(expect_records: records.length, timeout: 20) do
       entries = []
-      d.expected_emits.each {|tag,_time,record|
+      records.each {|tag, _time, record|
         entries << [_time, record]
       }
-      send_data Fluent::MessagePackFactory.msgpack_packer.write(["tag1", entries]).to_s
+      send_data packer.write(["tag1", entries]).to_s
     end
+    assert_equal(records, @d.events)
   end
 
   def test_packed_forward
-    d = create_driver
+    @d = create_driver
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag1", time, {"a"=>2}
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag1", time, {"a" => 2}],
+    ]
 
-    d.run do
+    @d.run(expect_records: records.length, timeout: 20) do
       entries = ''
-      d.expected_emits.each {|tag,_time,record|
-        Fluent::MessagePackFactory.msgpack_packer(entries).write([_time, record]).flush
+      records.each {|_tag, _time, record|
+        packer(entries).write([_time, record]).flush
       }
-      send_data Fluent::MessagePackFactory.msgpack_packer.write(["tag1", entries]).to_s
+      send_data packer.write(["tag1", entries]).to_s
     end
+    assert_equal(records, @d.events)
   end
 
   def test_message_json
-    d = create_driver
+    @d = create_driver
+
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
+
+    @d.run(expect_records: records.length, timeout: 5) do
+      tag, _time, record = records[0]
+      send_data [tag, _time.to_i, record].to_json
+      tag, _time, record = records[1]
+      send_data [tag, _time.to_f, record].to_json
+    end
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    assert_equal(records, @d.events)
+  end
+
+  def test_message_with_tag
+    @d = create_driver(CONFIG + "tag new_tag")
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
 
-    d.run do
-      d.expected_emits.each {|tag,_time,record|
-        send_data [tag, time, record].to_json
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
       }
     end
-  end
 
-  def create_driver(klass, conf)
-    Fluent::Test::InputTestDriver.new(klass).configure(conf)
+    @d.events.each { |event|
+      assert_equal("new_tag", event[0])
+    }
   end
 
-  def send_data(data)
-    io = connect
-    begin
-      io.write data
-    ensure
-      io.close
+  data('string chunk' => 'broken string',
+       'integer chunk' => 10)
+  def test_broken_message(data)
+    @d = create_driver
+    @d.run(shutdown: false, timeout: 5) do
+      @d.instance.__send__(:on_message, data)
     end
-  end
-end
-
-class UnixInputTest < Test::Unit::TestCase
-  include StreamInputTest
-
-  TMP_DIR = File.dirname(__FILE__) + "/../tmp/in_unix#{ENV['TEST_ENV_NUMBER']}"
-  CONFIG = %[
-    path #{TMP_DIR}/unix
-    backlog 1000
-  ]
 
-  def create_driver(conf=CONFIG)
-    super(Fluent::UnixInput, conf)
-  end
-
-  def test_configure
-    d = create_driver
-    assert_equal "#{TMP_DIR}/unix", d.instance.path
-    assert_equal 1000, d.instance.backlog
-  end
+    assert_equal 0, @d.events.size
 
-  def connect
-    UNIXSocket.new("#{TMP_DIR}/unix")
+    logs = @d.instance.log.logs
+    assert_equal 1, logs.select { |line|
+      line =~ / \[warn\]: incoming data is broken: msg=#{data.inspect}/
+    }.size, "should not accept broken chunk"
   end
 end unless Fluent.windows?

data/test/plugin/test_out_forward.rb

@@ -284,6 +284,29 @@ EOL
     assert_equal 1235, d.instance.discovery_manager.services[1].port
   end
 
+  test 'pass username and password as empty string to HandshakeProtocol' do
+    config_path = File.join(TMP_DIR, "sd_file.conf")
+    File.open(config_path, 'w') do |file|
+      file.write(%[
+- 'host': 127.0.0.1
+  'port': 1234
+  'weight': 1
+])
+    end
+
+    mock(Fluent::Plugin::ForwardOutput::HandshakeProtocol).new(log: anything, hostname: nil, shared_key: anything, password: '', username: '')
+    @d = d = create_driver(%[
+<service_discovery>
+  @type file
+  path #{config_path}
+</service_discovery>
+    ])
+
+    assert_equal 1, d.instance.discovery_manager.services.size
+    assert_equal '127.0.0.1', d.instance.discovery_manager.services[0].host
+    assert_equal 1234, d.instance.discovery_manager.services[0].port
+  end
+
   test 'compress_default_value' do
     @d = d = create_driver
     assert_equal :text, d.instance.compress
@@ -997,7 +1020,11 @@ EOL
       e = assert_raise Fluent::UnrecoverableError do
         d.instance_start
       end
-      assert_match(/Connection refused/, e.message)
+      if Fluent.windows?
+        assert_match(/No connection could be made because the target machine actively refused it/, e.message)
+      else
+        assert_match(/Connection refused/, e.message)
+      end
 
       d.instance_shutdown
     end
@@ -1029,7 +1056,7 @@ EOL
         e = assert_raise Fluent::UnrecoverableError do
           d.instance_start
         end
-        assert_match(/Failed to establish connection/, e.message)
+        assert_match(/failed to establish connection/, e.message)
       end
     end
 
@@ -1065,7 +1092,7 @@ EOL
           d.instance_start
         end
 
-        assert_match(/Failed to establish connection/, e.message)
+        assert_match(/failed to establish connection/, e.message)
       end
     end
 
@@ -1173,6 +1200,15 @@ EOL
       end
     end
 
+    test 'create timer of purging obsolete sockets' do
+      output_conf = CONFIG + %[keepalive true]
+      d = create_driver(output_conf)
+
+      mock(d.instance).timer_execute(:out_forward_heartbeat_request, 1).once
+      mock(d.instance).timer_execute(:out_forward_keep_alived_socket_watcher, 5).once
+      d.instance_start
+    end
+
     sub_test_case 'with require_ack_response' do
       test 'Create connection per send_data' do
         target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)

data/test/plugin/test_out_http.rb

@@ -80,6 +80,8 @@ class HTTPOutputTest < Test::Unit::TestCase
         req.body.each_line { |l|
           data << JSON.parse(l)
         }
+      when 'application/json'
+        data = JSON.parse(req.body)
       when 'text/plain'
         # Use single_value in this test
         req.body.each_line { |line|
@@ -181,6 +183,19 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_not_match(/Status code 503 is going to be removed/, d.instance.log.out.logs.join)
   end
 
+  # Check if an exception is raised on not JSON format use
+  data('not_json' => 'msgpack')
+  def test_configure_with_json_array_err(format_type)
+    assert_raise(Fluent::ConfigError) do
+      create_driver(config + %[
+        json_array true
+        <format>
+          @type #{format_type}
+        </format>
+      ])
+    end
+  end
+
   data('json' => ['json', 'application/x-ndjson'],
        'ltsv' => ['ltsv', 'text/tab-separated-values'],
        'msgpack' => ['msgpack', 'application/x-msgpack'],
@@ -195,6 +210,14 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_equal content_type, d.instance.content_type
   end
 
+  # Check that json_array setting sets content_type = application/json
+  data('json' => 'application/json')
+  def test_configure_content_type_json_array(content_type)
+    d = create_driver(config + "json_array true")
+
+    assert_equal content_type, d.instance.content_type
+  end
+
   data('PUT' => 'put', 'POST' => 'post')
   def test_write_with_method(method)
     d = create_driver(config + "http_method #{method}")
@@ -211,6 +234,21 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_not_empty result.headers
   end
 
+  # Check that JSON at HTTP request body is valid
+  def test_write_with_json_array_setting
+    d = create_driver(config + "json_array true")
+    d.run(default_tag: 'test.http') do
+      test_events.each { |event|
+        d.feed(event)
+      }
+    end
+
+    result = @@result
+    assert_equal 'application/json', result.content_type
+    assert_equal test_events, result.data
+    assert_not_empty result.headers
+  end
+
   def test_write_with_single_value_format
     d = create_driver(config + %[
       <format>

data/test/plugin/test_out_null.rb

@@ -40,7 +40,7 @@ class NullOutputTest < Test::Unit::TestCase
       assert_equal [], d.instance.chunk_keys
     end
 
-    test 'writes standard formattted chunks' do
+    test 'writes standard formatted chunks' do
       d = create_driver(config_element("ROOT", "", {}, [config_element("buffer")]))
       t = event_time("2016-05-23 00:22:13 -0800")
       d.run(default_tag: 'test', flush: true) do

data/test/plugin/test_output_as_buffered_retries.rb

@@ -895,17 +895,25 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
 
       @i.flush_thread_wakeup
       waiting(4){ Thread.pass until @i.write_count > 0 }
+      waiting(4) do
+        state = @i.instance_variable_get(:@output_flush_threads).first
+        state.thread.status == 'sleep'
+      end
 
-      assert{ @i.write_count > 0 }
-      assert{ @i.num_errors > 0 }
+      assert(@i.write_count > 0)
+      assert(@i.num_errors > 0)
 
       now = @i.next_flush_time
       Timecop.freeze( now )
       @i.flush_thread_wakeup
       waiting(4){ Thread.pass until @i.write_count > 1 }
+      waiting(4) do
+        state = @i.instance_variable_get(:@output_flush_threads).first
+        state.thread.status == 'sleep'
+      end
 
-      assert{ @i.write_count > 1 }
-      assert{ @i.num_errors > 1 }
+      assert(@i.write_count > 1)
+      assert(@i.num_errors > 1)
     end
   end
 end

data/test/plugin/test_output_as_buffered_secondary.rb

@@ -617,7 +617,15 @@ class BufferedOutputSecondaryTest < Test::Unit::TestCase
       assert{ !chunks[1].empty? }
 
       30.times do |i| # large enough
-        now = first_failure + 60 * 0.8 + 2 + i
+        # In https://github.com/fluent/fluentd/blob/c90c024576b3d35f356a55fd33d1232947114a9a/lib/fluent/plugin_helper/retry_state.rb
+        # @timeout_at is 2016-04-13 18:34:31, @next_time must be less than 2016-04-13 18:34:30
+        #
+        # first_failure + 60 * 0.8 + 2 # => 2016-04-13 18:34:21
+        # @next_time is not added by 1, but by randomize(@retry_wait) https://github.com/fluent/fluentd/blob/c90c024576b3d35f356a55fd33d1232947114a9a/lib/fluent/plugin_helper/retry_state.rb#L196
+        # current_time(=Time.now) + randomize(@retry_wait) < @timeout_at
+        # (2016-04-13 18:34:21 + 6) + 3 < 2016-04-13 18:34:31
+        # So, current_time must be at most 6
+        now = first_failure + 60 * 0.8 + 2 + [i, 6].min
         Timecop.freeze( now )
         @i.flush_thread_wakeup