fluentd 1.11.3-x86-mingw32 → 1.12.2-x86-mingw32

data/test/plugin/test_out_copy.rb

@@ -2,8 +2,11 @@ require_relative '../helper'
 require 'fluent/test/driver/multi_output'
 require 'fluent/plugin/out_copy'
 require 'fluent/event'
+require 'flexmock/test_unit'
 
 class CopyOutputTest < Test::Unit::TestCase
+  include FlexMock::TestCase
+
   class << self
     def startup
       $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
@@ -54,6 +57,48 @@ class CopyOutputTest < Test::Unit::TestCase
     assert_equal :no_copy, d.instance.copy_mode
   end
 
+  ERRORNEOUS_IGNORE_IF_PREV_SUCCESS_CONFIG = %[
+    <store ignore_if_prev_success ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_if_prev_success ignore_error>
+      @type test
+      name c1
+    </store>
+    <store ignore_if_prev_success>
+      @type test
+      name c2
+    </store>
+  ]
+  def test_configure_with_errorneus_ignore_if_prev_success
+    assert_raise(Fluent::ConfigError) do
+      create_driver(ERRORNEOUS_IGNORE_IF_PREV_SUCCESS_CONFIG)
+    end
+  end
+
+  ALL_IGNORE_ERROR_WITHOUT_IGNORE_IF_PREV_SUCCESS_CONFIG = %[
+    @log_level info
+    <store ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_error>
+      @type test
+      name c1
+    </store>
+    <store ignore_error>
+      @type test
+      name c2
+    </store>
+  ]
+  def test_configure_all_ignore_errors_without_ignore_if_prev_success
+    d = create_driver(ALL_IGNORE_ERROR_WITHOUT_IGNORE_IF_PREV_SUCCESS_CONFIG)
+    expected = /ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified./
+    matches = d.logs.grep(expected)
+    assert_equal(1, matches.length, "Logs do not contain '#{expected}' '#{d.logs}'")
+  end
+
   def test_configure_with_deep_copy_and_use_shallow_copy_mode
     d = create_driver(%[
       deep_copy true
@@ -217,5 +262,47 @@ class CopyOutputTest < Test::Unit::TestCase
       end
     end
   end
+
+  IGNORE_IF_PREV_SUCCESS_CONFIG = %[
+    <store ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_if_prev_success ignore_error>
+      @type test
+      name c1
+    </store>
+    <store ignore_if_prev_success>
+      @type test
+      name c2
+    </store>
+  ]
+
+  def test_ignore_if_prev_success
+    d = create_driver(IGNORE_IF_PREV_SUCCESS_CONFIG)
+
+    # override to raise an error
+    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
+      raise ArgumentError, 'Failed'
+    end
+
+    # check ingore_if_prev_success functionality:
+    # 1. output 2 is succeeded.
+    # 2. output 3 is not called.
+    flexstub(d.instance.outputs[1]) do |output|
+      output.should_receive(:process).once
+    end
+    flexstub(d.instance.outputs[2]) do |output|
+      output.should_receive(:process).never
+    end
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    assert_nothing_raised do
+      d.run(default_tag: 'test') do
+        d.feed(time, {"a"=>1})
+      end
+    end
+  end
+
 end
 

data/test/plugin/test_out_file.rb

@@ -11,6 +11,11 @@ class FileOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
     FileUtils.rm_rf(TMP_DIR)
     FileUtils.mkdir_p(TMP_DIR)
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/out_file#{ENV['TEST_ENV_NUMBER']}")
@@ -91,7 +96,7 @@ class FileOutputTest < Test::Unit::TestCase
         end
       end
       assert_equal 1, d.formatted.size
-      assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}\n], d.formatted[0]
+      assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
     end
 
     test 'no configuration error raised for basic configuration using "*" (v0.12 style)' do
@@ -296,11 +301,11 @@ class FileOutputTest < Test::Unit::TestCase
 
       assert_equal 5, d.formatted.size
 
-      r1 = %!2016-10-03 23:58:09 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:58:09 +0900"}\n!
-      r2 = %!2016-10-03 23:59:33 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:59:33 +0900"}\n!
-      r3 = %!2016-10-03 23:59:57 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 08:59:57 +0900"}\n!
-      r4 = %!2016-10-04 00:00:17 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 09:00:17 +0900"}\n!
-      r5 = %!2016-10-04 00:01:59 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 09:01:59 +0900"}\n!
+      r1 = %!2016-10-03 23:58:09 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:58:09 +0900"}#{@default_newline}!
+      r2 = %!2016-10-03 23:59:33 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 08:59:33 +0900"}#{@default_newline}!
+      r3 = %!2016-10-03 23:59:57 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 08:59:57 +0900"}#{@default_newline}!
+      r4 = %!2016-10-04 00:00:17 +0000,my.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"my.data","time":"2016/10/04 09:00:17 +0900"}#{@default_newline}!
+      r5 = %!2016-10-04 00:01:59 +0000,your.data,{"type":"a","message":"data raw content","hostname":"testing.local","tag":"your.data","time":"2016/10/04 09:01:59 +0900"}#{@default_newline}!
       assert_equal r1, d.formatted[0]
       assert_equal r2, d.formatted[1]
       assert_equal r3, d.formatted[2]
@@ -329,8 +334,8 @@ class FileOutputTest < Test::Unit::TestCase
         d.feed(time, {"a"=>2})
       end
       assert_equal 2, d.formatted.size
-      assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n], d.formatted[0]
-      assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n], d.formatted[1]
+      assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
+      assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}], d.formatted[1]
     end
 
     test 'time formatted with specified timezone, using area name' do
@@ -344,7 +349,7 @@ class FileOutputTest < Test::Unit::TestCase
         d.feed(time, {"a"=>1})
       end
       assert_equal 1, d.formatted.size
-      assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}\n], d.formatted[0]
+      assert_equal %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
     end
 
     test 'time formatted with specified timezone, using offset' do
@@ -358,7 +363,7 @@ class FileOutputTest < Test::Unit::TestCase
         d.feed(time, {"a"=>1})
       end
       assert_equal 1, d.formatted.size
-      assert_equal %[2011-01-02T09:44:15-03:30\ttest\t{"a":1}\n], d.formatted[0]
+      assert_equal %[2011-01-02T09:44:15-03:30\ttest\t{"a":1}#{@default_newline}], d.formatted[0]
     end
 
     test 'configuration error raised for invalid timezone' do
@@ -402,7 +407,7 @@ class FileOutputTest < Test::Unit::TestCase
       end
 
       assert File.exist?("#{TMP_DIR}/out_file_test.20110102_0.log.gz")
-      check_gzipped_result("#{TMP_DIR}/out_file_test.20110102_0.log.gz", %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
+      check_gzipped_result("#{TMP_DIR}/out_file_test.20110102_0.log.gz", %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}])
     end
   end
 
@@ -469,7 +474,7 @@ class FileOutputTest < Test::Unit::TestCase
       end
 
       path = d.instance.last_written_path
-      check_gzipped_result(path, %[#{Yajl.dump({"a" => 1, 'time' => time.to_i})}\n] + %[#{Yajl.dump({"a" => 2, 'time' => time.to_i})}\n])
+      check_gzipped_result(path, %[#{Yajl.dump({"a" => 1, 'time' => time.to_i})}#{@default_newline}] + %[#{Yajl.dump({"a" => 2, 'time' => time.to_i})}#{@default_newline}])
     end
 
     test 'ltsv' do
@@ -482,7 +487,7 @@ class FileOutputTest < Test::Unit::TestCase
       end
 
       path = d.instance.last_written_path
-      check_gzipped_result(path, %[a:1\ttime:2011-01-02T13:14:15Z\n] + %[a:2\ttime:2011-01-02T13:14:15Z\n])
+      check_gzipped_result(path, %[a:1\ttime:2011-01-02T13:14:15Z#{@default_newline}] + %[a:2\ttime:2011-01-02T13:14:15Z#{@default_newline}])
     end
 
     test 'single_value' do
@@ -495,13 +500,13 @@ class FileOutputTest < Test::Unit::TestCase
       end
 
       path = d.instance.last_written_path
-      check_gzipped_result(path, %[1\n] + %[2\n])
+      check_gzipped_result(path, %[1#{@default_newline}] + %[2#{@default_newline}])
     end
   end
 
   test 'path with index number' do
     time = event_time("2011-01-02 13:14:15 UTC")
-    formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+    formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]
 
     write_once = ->(){
       d = create_driver
@@ -532,7 +537,7 @@ class FileOutputTest < Test::Unit::TestCase
 
   test 'append' do
     time = event_time("2011-01-02 13:14:15 UTC")
-    formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+    formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]
 
     write_once = ->(){
       d = create_driver %[
@@ -567,7 +572,7 @@ class FileOutputTest < Test::Unit::TestCase
   test 'append when JST' do
     with_timezone(Fluent.windows? ? "JST-9" : "Asia/Tokyo") do
       time = event_time("2011-01-02 03:14:15+09:00")
-      formatted_lines = %[2011-01-02T03:14:15+09:00\ttest\t{"a":1}\n] + %[2011-01-02T03:14:15+09:00\ttest\t{"a":2}\n]
+      formatted_lines = %[2011-01-02T03:14:15+09:00\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T03:14:15+09:00\ttest\t{"a":2}#{@default_newline}]
 
       write_once = ->(){
         d = create_driver %[
@@ -603,7 +608,7 @@ class FileOutputTest < Test::Unit::TestCase
   test 'append when UTC-02 but timekey_zone is +0900' do
     with_timezone("UTC-02") do # +0200
       time = event_time("2011-01-02 17:14:15+02:00")
-      formatted_lines = %[2011-01-02T17:14:15+02:00\ttest\t{"a":1}\n] + %[2011-01-02T17:14:15+02:00\ttest\t{"a":2}\n]
+      formatted_lines = %[2011-01-02T17:14:15+02:00\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T17:14:15+02:00\ttest\t{"a":2}#{@default_newline}]
 
       write_once = ->(){
         d = create_driver %[

data/test/plugin/test_out_forward.rb

@@ -367,10 +367,31 @@ EOL
       require_ack_response true
       ack_response_timeout 2s
     ])
+    d.instance_start
     assert d.instance.require_ack_response
     assert_equal 2, d.instance.ack_response_timeout
   end
 
+  test 'suspend_flush is disable before before_shutdown' do
+    @d = d = create_driver(CONFIG + %[
+      require_ack_response true
+      ack_response_timeout 2s
+    ])
+    d.instance_start
+    assert_false d.instance.instance_variable_get(:@suspend_flush)
+  end
+
+  test 'suspend_flush should be enabled and try_flush returns nil after before_shutdown' do
+    @d = d = create_driver(CONFIG + %[
+      require_ack_response true
+      ack_response_timeout 2s
+    ])
+    d.instance_start
+    d.instance.before_shutdown
+    assert_true d.instance.instance_variable_get(:@suspend_flush)
+    assert_nil d.instance.try_flush
+  end
+
   test 'verify_connection_at_startup is disabled in default' do
     @d = d = create_driver(CONFIG)
     assert_false d.instance.verify_connection_at_startup
@@ -599,6 +620,53 @@ EOL
     assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first
   end
 
+  test 'a node supporting responses after stop' do
+    target_input_driver = create_target_input_driver
+
+    @d = d = create_driver(CONFIG + %[
+      require_ack_response true
+      ack_response_timeout 1s
+      <buffer tag>
+        flush_mode immediate
+        retry_type periodic
+        retry_wait 30s
+        flush_at_shutdown false # suppress errors in d.instance_shutdown
+      </buffer>
+    ])
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+
+    acked_chunk_ids = []
+    mock.proxy(d.instance.ack_handler).read_ack_from_sock(anything) do |info, success|
+      if success
+        acked_chunk_ids << info.chunk_id
+      end
+      [chunk_id, success]
+    end
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    target_input_driver.run(expect_records: 2) do
+      d.end_if { acked_chunk_ids.size > 0 }
+      d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
+        d.instance.stop
+        d.feed([[time, records[0]], [time,records[1]]])
+        d.instance.before_shutdown
+        d.instance.shutdown
+        d.instance.after_shutdown
+      end
+    end
+
+    events = target_input_driver.events
+    assert_equal ['test', time, records[0]], events[0]
+    assert_equal ['test', time, records[1]], events[1]
+
+    assert_equal 1, acked_chunk_ids.size
+    assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first
+  end
+
   data('ack true' => true,
        'ack false' => false)
   test 'TLS transport and ack parameter combination' do |ack|
@@ -993,6 +1061,8 @@ EOL
   end
 
   test 'when out_forward has @id' do
+    omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
+
     # cancel https://github.com/fluent/fluentd/blob/077508ac817b7637307434d0c978d7cdc3d1c534/lib/fluent/plugin_id.rb#L43-L53
     # it always return true in test
     mock.proxy(Fluent::Plugin).new_sd(:static, anything) { |v|
@@ -1137,6 +1207,8 @@ EOL
   end
 
   test 'Create new connection per send_data' do
+    omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
+
     target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
     output_conf = CONFIG
     d = create_driver(output_conf)
@@ -1175,6 +1247,8 @@ EOL
 
   sub_test_case 'keepalive' do
     test 'Do not create connection per send_data' do
+      omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
+
       target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
       output_conf = CONFIG + %[
         keepalive true

data/test/plugin/test_out_http.rb

@@ -127,7 +127,7 @@ class HTTPOutputTest < Test::Unit::TestCase
     now = Time.now
     started = false
     until started
-      raise "Server not started" if (now - Time.now > 10.0)
+      raise "Server not started" if (Time.now - now > 10.0)
       begin
         http_client { |c| c.request_get('/') }
         started = true
@@ -280,6 +280,25 @@ class HTTPOutputTest < Test::Unit::TestCase
     assert_equal "fluentd!", result.headers['test_header']
   end
 
+  def test_write_with_headers_from_placeholders
+    d = create_driver(config + %[
+      headers_from_placeholders {"x-test":"${$.foo.bar}-test","x-tag":"${tag}"}
+      <buffer tag,$.foo.bar>
+      </buffer>
+    ])
+    d.run(default_tag: 'test.http') do
+      test_events.each { |event|
+        ev = event.dup
+        ev['foo'] = {'bar' => 'abcd'}
+        d.feed(ev)
+      }
+    end
+
+    result = @@result
+    assert_equal "abcd-test", result.headers['x-test']
+    assert_equal "test.http", result.headers['x-tag']
+  end
+
   def test_write_with_retryable_response
     old_report_on_exception = Thread.report_on_exception
     Thread.report_on_exception = false # thread finished as invalid state since RetryableResponse raises.

data/test/plugin/test_output.rb

@@ -378,6 +378,18 @@ class OutputTest < Test::Unit::TestCase
       assert { logs.any? { |log| log.include?("${chunk_id} is not allowed in this plugin") } }
     end
 
+    test '#extract_placeholders does not log for ${chunk_id} placeholder' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+      tmpl = "/mypath/${chunk_id}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      c = create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+      @i.log.out.logs.clear
+      @i.extract_placeholders(tmpl, c)
+      logs = @i.log.out.logs
+      assert { logs.none? { |log| log.include?("${chunk_id}") } }
+    end
+
     test '#extract_placeholders logs warn message with not replaced key' do
       @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
       tmpl = "/mypath/${key1}/test"

data/test/plugin/test_parser_syslog.rb

@@ -38,7 +38,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
   data('regexp' => 'regexp', 'string' => 'string')
   def test_parse_with_time_format2(param)
     @parser.configure('time_format' => '%Y-%m-%dT%H:%M:%SZ', 'parser_type' => param)
-    @parser.instance.parse('2020-03-03T10:14:29Z 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
+    @parser.instance.parse("#{Time.now.year}-03-03T10:14:29Z 192.168.0.1 fluentd[11111]: [error] Syslog test") { |time, record|
       assert_equal(event_time('Mar 03 10:14:29', format: '%b %d %H:%M:%S'), time)
       assert_equal(@expected, record)
     }
@@ -47,7 +47,7 @@ class SyslogParserTest < ::Test::Unit::TestCase
 
   def test_parse_with_time_format_rfc5424
     @parser.configure('time_format' => '%Y-%m-%dT%H:%M:%SZ', 'message_format' => 'rfc5424')
-    @parser.instance.parse('2020-03-03T10:14:29Z 192.168.0.1 fluentd 11111 - - [error] Syslog test') { |time, record|
+    @parser.instance.parse("#{Time.now.year}-03-03T10:14:29Z 192.168.0.1 fluentd 11111 - - [error] Syslog test") { |time, record|
       assert_equal(event_time('Mar 03 10:14:29', format: '%b %d %H:%M:%S'), time)
       assert_equal(@expected.merge('host' => '192.168.0.1', 'msgid' => '-', 'extradata' => '-'), record)
     }

data/test/plugin/test_sd_file.rb

@@ -149,7 +149,7 @@ class FileServiceDiscoveryTest < ::Test::Unit::TestCase
       assert_empty queue
     end
 
-    test 'Skip if error is occured' do
+    test 'Skip if error is occurred' do
       @sd_file.extend(TestStatEventHelperWrapper)
 
       create_tmp_config('config.json', JSON.generate([{ port: 1233, host: '127.0.0.1' }]))

data/test/plugin_helper/test_child_process.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 require_relative '../helper'
 require 'fluent/plugin_helper/child_process'
 require 'fluent/plugin/base'
@@ -269,10 +270,12 @@ class ChildProcessTest < Test::Unit::TestCase
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
       ran = false
       @d.child_process_execute(:t4, "ruby -e 'Signal.trap(:TERM, nil); while sleep 0.1; puts 1; STDOUT.flush rescue nil; end'", mode: [:read]) do |io|
-        m.lock
-        ran = true
         begin
           while line = io.readline
+            unless ran
+              m.lock
+              ran = true
+            end
             ary << line
           end
         rescue