fluentd 1.12.0-x64-mingw32 → 1.13.0-x64-mingw32

data/test/plugin/test_out_copy.rb

@@ -2,8 +2,11 @@ require_relative '../helper'
 require 'fluent/test/driver/multi_output'
 require 'fluent/plugin/out_copy'
 require 'fluent/event'
+require 'flexmock/test_unit'
 
 class CopyOutputTest < Test::Unit::TestCase
+  include FlexMock::TestCase
+
   class << self
     def startup
       $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
@@ -54,6 +57,48 @@ class CopyOutputTest < Test::Unit::TestCase
     assert_equal :no_copy, d.instance.copy_mode
   end
 
+  ERRORNEOUS_IGNORE_IF_PREV_SUCCESS_CONFIG = %[
+    <store ignore_if_prev_success ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_if_prev_success ignore_error>
+      @type test
+      name c1
+    </store>
+    <store ignore_if_prev_success>
+      @type test
+      name c2
+    </store>
+  ]
+  def test_configure_with_errorneus_ignore_if_prev_success
+    assert_raise(Fluent::ConfigError) do
+      create_driver(ERRORNEOUS_IGNORE_IF_PREV_SUCCESS_CONFIG)
+    end
+  end
+
+  ALL_IGNORE_ERROR_WITHOUT_IGNORE_IF_PREV_SUCCESS_CONFIG = %[
+    @log_level info
+    <store ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_error>
+      @type test
+      name c1
+    </store>
+    <store ignore_error>
+      @type test
+      name c2
+    </store>
+  ]
+  def test_configure_all_ignore_errors_without_ignore_if_prev_success
+    d = create_driver(ALL_IGNORE_ERROR_WITHOUT_IGNORE_IF_PREV_SUCCESS_CONFIG)
+    expected = /ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified./
+    matches = d.logs.grep(expected)
+    assert_equal(1, matches.length, "Logs do not contain '#{expected}' '#{d.logs}'")
+  end
+
   def test_configure_with_deep_copy_and_use_shallow_copy_mode
     d = create_driver(%[
       deep_copy true
@@ -217,5 +262,47 @@ class CopyOutputTest < Test::Unit::TestCase
       end
     end
   end
+
+  IGNORE_IF_PREV_SUCCESS_CONFIG = %[
+    <store ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_if_prev_success ignore_error>
+      @type test
+      name c1
+    </store>
+    <store ignore_if_prev_success>
+      @type test
+      name c2
+    </store>
+  ]
+
+  def test_ignore_if_prev_success
+    d = create_driver(IGNORE_IF_PREV_SUCCESS_CONFIG)
+
+    # override to raise an error
+    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
+      raise ArgumentError, 'Failed'
+    end
+
+    # check ingore_if_prev_success functionality:
+    # 1. output 2 is succeeded.
+    # 2. output 3 is not called.
+    flexstub(d.instance.outputs[1]) do |output|
+      output.should_receive(:process).once
+    end
+    flexstub(d.instance.outputs[2]) do |output|
+      output.should_receive(:process).never
+    end
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    assert_nothing_raised do
+      d.run(default_tag: 'test') do
+        d.feed(time, {"a"=>1})
+      end
+    end
+  end
+
 end
 

data/test/plugin/test_out_forward.rb

@@ -277,11 +277,16 @@ EOL
 </service_discovery>
     ])
 
-    assert_equal 2, d.instance.discovery_manager.services.size
-    assert_equal '127.0.0.1', d.instance.discovery_manager.services[0].host
-    assert_equal 1234, d.instance.discovery_manager.services[0].port
-    assert_equal '127.0.0.1', d.instance.discovery_manager.services[1].host
-    assert_equal 1235, d.instance.discovery_manager.services[1].port
+
+    assert_equal(
+      [
+        { host: '127.0.0.1', port: 1234 },
+        { host: '127.0.0.1', port: 1235 },
+      ],
+      d.instance.discovery_manager.services.collect do |service|
+        { host: service.host, port: service.port }
+      end
+    )
   end
 
   test 'pass username and password as empty string to HandshakeProtocol' do
@@ -367,10 +372,31 @@ EOL
       require_ack_response true
       ack_response_timeout 2s
     ])
+    d.instance_start
     assert d.instance.require_ack_response
     assert_equal 2, d.instance.ack_response_timeout
   end
 
+  test 'suspend_flush is disable before before_shutdown' do
+    @d = d = create_driver(CONFIG + %[
+      require_ack_response true
+      ack_response_timeout 2s
+    ])
+    d.instance_start
+    assert_false d.instance.instance_variable_get(:@suspend_flush)
+  end
+
+  test 'suspend_flush should be enabled and try_flush returns nil after before_shutdown' do
+    @d = d = create_driver(CONFIG + %[
+      require_ack_response true
+      ack_response_timeout 2s
+    ])
+    d.instance_start
+    d.instance.before_shutdown
+    assert_true d.instance.instance_variable_get(:@suspend_flush)
+    assert_nil d.instance.try_flush
+  end
+
   test 'verify_connection_at_startup is disabled in default' do
     @d = d = create_driver(CONFIG)
     assert_false d.instance.verify_connection_at_startup
@@ -566,31 +592,88 @@ EOL
         flush_mode immediate
         retry_type periodic
         retry_wait 30s
-        flush_at_shutdown false # suppress errors in d.instance_shutdown
+        flush_at_shutdown true
       </buffer>
     ])
 
     time = event_time("2011-01-02 13:14:15 UTC")
 
     acked_chunk_ids = []
+    nacked = false
     mock.proxy(d.instance.ack_handler).read_ack_from_sock(anything) do |info, success|
       if success
         acked_chunk_ids << info.chunk_id
+      else
+        nacked = true
       end
-      [chunk_id, success]
+      [info, success]
     end
 
     records = [
       {"a" => 1},
       {"a" => 2}
     ]
-    target_input_driver.run(expect_records: 2) do
-      d.end_if { acked_chunk_ids.size > 0 }
+    target_input_driver.run(expect_records: 2, timeout: 5) do
+      d.end_if { acked_chunk_ids.size > 0 || nacked }
       d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
         d.feed([[time, records[0]], [time,records[1]]])
       end
     end
 
+    assert(!nacked, d.instance.log.logs.join)
+
+    events = target_input_driver.events
+    assert_equal ['test', time, records[0]], events[0]
+    assert_equal ['test', time, records[1]], events[1]
+
+    assert_equal 1, acked_chunk_ids.size
+    assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first
+  end
+
+  test 'a node supporting responses after stop' do
+    target_input_driver = create_target_input_driver
+
+    @d = d = create_driver(CONFIG + %[
+      require_ack_response true
+      ack_response_timeout 1s
+      <buffer tag>
+        flush_mode immediate
+        retry_type periodic
+        retry_wait 30s
+        flush_at_shutdown true
+      </buffer>
+    ])
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+
+    acked_chunk_ids = []
+    nacked = false
+    mock.proxy(d.instance.ack_handler).read_ack_from_sock(anything) do |info, success|
+      if success
+        acked_chunk_ids << info.chunk_id
+      else
+        nacked = true
+      end
+      [info, success]
+    end
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    target_input_driver.run(expect_records: 2, timeout: 5) do
+      d.end_if { acked_chunk_ids.size > 0 || nacked }
+      d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
+        d.instance.stop
+        d.feed([[time, records[0]], [time,records[1]]])
+        d.instance.before_shutdown
+        d.instance.shutdown
+        d.instance.after_shutdown
+      end
+    end
+
+    assert(!nacked, d.instance.log.logs.join)
+
     events = target_input_driver.events
     assert_equal ['test', time, records[0]], events[0]
     assert_equal ['test', time, records[1]], events[1]
@@ -995,7 +1078,7 @@ EOL
   test 'when out_forward has @id' do
     # cancel https://github.com/fluent/fluentd/blob/077508ac817b7637307434d0c978d7cdc3d1c534/lib/fluent/plugin_id.rb#L43-L53
     # it always return true in test
-    mock.proxy(Fluent::Plugin).new_sd(:static, anything) { |v|
+    mock.proxy(Fluent::Plugin).new_sd('static', parent: anything) { |v|
       stub(v).plugin_id_for_test? { false }
     }.once
 
@@ -1137,6 +1220,8 @@ EOL
   end
 
   test 'Create new connection per send_data' do
+    omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
+
     target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
     output_conf = CONFIG
     d = create_driver(output_conf)
@@ -1175,6 +1260,8 @@ EOL
 
   sub_test_case 'keepalive' do
     test 'Do not create connection per send_data' do
+      omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
+
       target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
       output_conf = CONFIG + %[
         keepalive true
@@ -1222,7 +1309,13 @@ EOL
 
         begin
           chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
-          mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, TARGET_PORT, anything) { |sock| mock(sock).close.once; sock }.twice
+          mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, TARGET_PORT,
+                                                   linger_timeout: anything,
+                                                   send_timeout: anything,
+                                                   recv_timeout: anything,
+                                                   connect_timeout: anything) { |sock|
+            mock(sock).close.once; sock
+          }.twice
 
           target_input_driver.run(timeout: 15) do
             d.run(shutdown: false) do

data/test/plugin/test_out_http.rb

@@ -127,7 +127,7 @@ class HTTPOutputTest < Test::Unit::TestCase
     now = Time.now
     started = false
     until started
-      raise "Server not started" if (now - Time.now > 10.0)
+      raise "Server not started" if (Time.now - now > 10.0)
       begin
         http_client { |c| c.request_get('/') }
         started = true
@@ -280,6 +280,25 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_equal "fluentd!", result.headers['test_header']
   end
 
+  def test_write_with_headers_from_placeholders
+    d = create_driver(config + %[
+      headers_from_placeholders {"x-test":"${$.foo.bar}-test","x-tag":"${tag}"}
+      <buffer tag,$.foo.bar>
+      </buffer>
+    ])
+    d.run(default_tag: 'test.http') do
+      test_events.each { |event|
+        ev = event.dup
+        ev['foo'] = {'bar' => 'abcd'}
+        d.feed(ev)
+      }
+    end
+
+    result = @@result
+    assert_equal "abcd-test", result.headers['x-test']
+    assert_equal "test.http", result.headers['x-tag']
+  end
+
   def test_write_with_retryable_response
     old_report_on_exception = Thread.report_on_exception
     Thread.report_on_exception = false # thread finished as invalid state since RetryableResponse raises.

data/test/plugin/test_output.rb

@@ -390,6 +390,18 @@ class OutputTest < Test::Unit::TestCase
       assert { logs.none? { |log| log.include?("${chunk_id}") } }
     end
 
+    test '#extract_placeholders does not log for ${chunk_id} placeholder (with @chunk_keys)' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1')]))
+      tmpl = "/mypath/${chunk_id}/${key1}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      c = create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+      @i.log.out.logs.clear
+      @i.extract_placeholders(tmpl, c)
+      logs = @i.log.out.logs
+      assert { logs.none? { |log| log.include?("${chunk_id}") } }
+    end
+
     test '#extract_placeholders logs warn message with not replaced key' do
       @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
       tmpl = "/mypath/${key1}/test"
@@ -868,7 +880,7 @@ class OutputTest < Test::Unit::TestCase
       test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
         o = create_output(:buffered)
         mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" })
+                         primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput")
 
         o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
         assert_not_nil o.instance_variable_get(:@secondary)
@@ -877,7 +889,7 @@ class OutputTest < Test::Unit::TestCase
       test "don't warn if primary type is the same as secondary type" do
         o = Fluent::Plugin::TestOutput.new
         mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
+                         primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" ).never
 
         o.configure(config_element('ROOT','',{'name' => "cool2"},
                                    [config_element('secondary','',{'@type'=>'test', 'name' => "cool"}),
@@ -889,7 +901,7 @@ class OutputTest < Test::Unit::TestCase
       test "don't warn if primary type is different from secondary type and both don't have custom_format" do
         o = create_output(:standard)
         mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
+                         primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput").never
 
         o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
         assert_not_nil o.instance_variable_get(:@secondary)

data/test/plugin/test_output_as_buffered_backup.rb

@@ -173,6 +173,8 @@ class BufferedOutputBackupTest < Test::Unit::TestCase
       waiting(5) {
         target_dir = File.join(File.dirname(target_file), "*")
         while Dir.glob(target_dir).size.zero?
+          # Avoid to lose globbed entries on Windows in busy loop
+          sleep 0.1 if Fluent.windows?
         end
       }
     end

data/test/plugin/test_parser_csv.rb

@@ -183,4 +183,18 @@ class CSVParserTest < ::Test::Unit::TestCase
       # And more...
     end
   end
+
+  # "parser_type" config shouldn't hide Fluent::Plugin::Parser#plugin_type
+  # https://github.com/fluent/fluentd/issues/3296
+  data('normal' => :normal, 'fast' => :fast)
+  def test_parser_type_method(engine)
+    d = create_driver('keys' => '["time"]','time_key' => 'time', 'parser_type' => engine.to_s)
+    assert_equal(:text_per_line, d.instance.parser_type)
+  end
+
+  data('normal' => :normal, 'fast' => :fast)
+  def test_parser_engine(engine)
+    d = create_driver('keys' => '["time"]', 'time_key' => 'time', 'parser_engine' => engine.to_s)
+    assert_equal(engine, d.instance.parser_engine)
+  end
 end

data/test/plugin/test_parser_syslog.rb

@@ -633,4 +633,18 @@ class SyslogParserTest < ::Test::Unit::TestCase
       end
     end
   end
+
+  # "parser_type" config shouldn't hide Fluent::Plugin::Parser#plugin_type
+  # https://github.com/fluent/fluentd/issues/3296
+  data('regexp' => :regexp, 'fast' => :string)
+  def test_parser_type_method(engine)
+    @parser.configure({'parser_type' => engine.to_s})
+    assert_equal(:text_per_line, @parser.instance.parser_type)
+  end
+
+  data('regexp' => :regexp, 'string' => :string)
+  def test_parser_engine(engine)
+    d = @parser.configure({'parser_engine' => engine.to_s})
+    assert_equal(engine, @parser.instance.parser_engine)
+  end
 end

data/test/plugin/test_sd_file.rb

@@ -149,7 +149,7 @@ class FileServiceDiscoveryTest < ::Test::Unit::TestCase
       assert_empty queue
     end
 
-    test 'Skip if error is occured' do
+    test 'Skip if error is occurred' do
       @sd_file.extend(TestStatEventHelperWrapper)
 
       create_tmp_config('config.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))

data/test/plugin_helper/service_discovery/test_manager.rb

@@ -71,7 +71,7 @@ class TestServiceDiscoveryManager < ::Test::Unit::TestCase
     test 'if new service added and deleted' do
       sdm = Fluent::PluginHelper::ServiceDiscovery::Manager.new(log: $log)
       t = TestSdPlugin.new
-      mock(Fluent::Plugin).new_sd(:sd_test, anything) { t }
+      mock(Fluent::Plugin).new_sd(:sd_test, parent: anything) { t }
       sdm.configure([{ type: :sd_test, conf: config_element('service_discovery', '', {})}])
       sdm.start
 

data/test/plugin_helper/test_child_process.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 require_relative '../helper'
 require 'fluent/plugin_helper/child_process'
 require 'fluent/plugin/base'
@@ -269,10 +270,12 @@ class ChildProcessTest < Test::Unit::TestCase
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
       ran = false
       @d.child_process_execute(:t4, "ruby -e 'Signal.trap(:TERM, nil); while sleep 0.1; puts 1; STDOUT.flush rescue nil; end'", mode: [:read]) do |io|
-        m.lock
-        ran = true
         begin
           while line = io.readline
+            unless ran
+              m.lock
+              ran = true
+            end
             ary << line
           end
         rescue