fluentd 1.12.1 → 1.12.2

data/test/plugin/test_out_copy.rb

@@ -2,8 +2,11 @@ require_relative '../helper'
 require 'fluent/test/driver/multi_output'
 require 'fluent/plugin/out_copy'
 require 'fluent/event'
+require 'flexmock/test_unit'
 
 class CopyOutputTest < Test::Unit::TestCase
+  include FlexMock::TestCase
+
   class << self
     def startup
       $LOAD_PATH.unshift File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts'))
@@ -54,6 +57,48 @@ class CopyOutputTest < Test::Unit::TestCase
     assert_equal :no_copy, d.instance.copy_mode
   end
 
+  ERRORNEOUS_IGNORE_IF_PREV_SUCCESS_CONFIG = %[
+    <store ignore_if_prev_success ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_if_prev_success ignore_error>
+      @type test
+      name c1
+    </store>
+    <store ignore_if_prev_success>
+      @type test
+      name c2
+    </store>
+  ]
+  def test_configure_with_errorneus_ignore_if_prev_success
+    assert_raise(Fluent::ConfigError) do
+      create_driver(ERRORNEOUS_IGNORE_IF_PREV_SUCCESS_CONFIG)
+    end
+  end
+
+  ALL_IGNORE_ERROR_WITHOUT_IGNORE_IF_PREV_SUCCESS_CONFIG = %[
+    @log_level info
+    <store ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_error>
+      @type test
+      name c1
+    </store>
+    <store ignore_error>
+      @type test
+      name c2
+    </store>
+  ]
+  def test_configure_all_ignore_errors_without_ignore_if_prev_success
+    d = create_driver(ALL_IGNORE_ERROR_WITHOUT_IGNORE_IF_PREV_SUCCESS_CONFIG)
+    expected = /ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified./
+    matches = d.logs.grep(expected)
+    assert_equal(1, matches.length, "Logs do not contain '#{expected}' '#{d.logs}'")
+  end
+
   def test_configure_with_deep_copy_and_use_shallow_copy_mode
     d = create_driver(%[
       deep_copy true
@@ -217,5 +262,47 @@ class CopyOutputTest < Test::Unit::TestCase
       end
     end
   end
+
+  IGNORE_IF_PREV_SUCCESS_CONFIG = %[
+    <store ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_if_prev_success ignore_error>
+      @type test
+      name c1
+    </store>
+    <store ignore_if_prev_success>
+      @type test
+      name c2
+    </store>
+  ]
+
+  def test_ignore_if_prev_success
+    d = create_driver(IGNORE_IF_PREV_SUCCESS_CONFIG)
+
+    # override to raise an error
+    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
+      raise ArgumentError, 'Failed'
+    end
+
+    # check ingore_if_prev_success functionality:
+    # 1. output 2 is succeeded.
+    # 2. output 3 is not called.
+    flexstub(d.instance.outputs[1]) do |output|
+      output.should_receive(:process).once
+    end
+    flexstub(d.instance.outputs[2]) do |output|
+      output.should_receive(:process).never
+    end
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    assert_nothing_raised do
+      d.run(default_tag: 'test') do
+        d.feed(time, {"a"=>1})
+      end
+    end
+  end
+
 end
 

data/test/plugin/test_out_forward.rb

@@ -367,10 +367,31 @@ EOL
       require_ack_response true
       ack_response_timeout 2s
     ])
+    d.instance_start
     assert d.instance.require_ack_response
     assert_equal 2, d.instance.ack_response_timeout
   end
 
+  test 'suspend_flush is disable before before_shutdown' do
+    @d = d = create_driver(CONFIG + %[
+      require_ack_response true
+      ack_response_timeout 2s
+    ])
+    d.instance_start
+    assert_false d.instance.instance_variable_get(:@suspend_flush)
+  end
+
+  test 'suspend_flush should be enabled and try_flush returns nil after before_shutdown' do
+    @d = d = create_driver(CONFIG + %[
+      require_ack_response true
+      ack_response_timeout 2s
+    ])
+    d.instance_start
+    d.instance.before_shutdown
+    assert_true d.instance.instance_variable_get(:@suspend_flush)
+    assert_nil d.instance.try_flush
+  end
+
   test 'verify_connection_at_startup is disabled in default' do
     @d = d = create_driver(CONFIG)
     assert_false d.instance.verify_connection_at_startup
@@ -599,6 +620,53 @@ EOL
     assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first
   end
 
+  test 'a node supporting responses after stop' do
+    target_input_driver = create_target_input_driver
+
+    @d = d = create_driver(CONFIG + %[
+      require_ack_response true
+      ack_response_timeout 1s
+      <buffer tag>
+        flush_mode immediate
+        retry_type periodic
+        retry_wait 30s
+        flush_at_shutdown false # suppress errors in d.instance_shutdown
+      </buffer>
+    ])
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+
+    acked_chunk_ids = []
+    mock.proxy(d.instance.ack_handler).read_ack_from_sock(anything) do |info, success|
+      if success
+        acked_chunk_ids << info.chunk_id
+      end
+      [chunk_id, success]
+    end
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    target_input_driver.run(expect_records: 2) do
+      d.end_if { acked_chunk_ids.size > 0 }
+      d.run(default_tag: 'test', wait_flush_completion: false, shutdown: false) do
+        d.instance.stop
+        d.feed([[time, records[0]], [time,records[1]]])
+        d.instance.before_shutdown
+        d.instance.shutdown
+        d.instance.after_shutdown
+      end
+    end
+
+    events = target_input_driver.events
+    assert_equal ['test', time, records[0]], events[0]
+    assert_equal ['test', time, records[1]], events[1]
+
+    assert_equal 1, acked_chunk_ids.size
+    assert_equal d.instance.sent_chunk_ids.first, acked_chunk_ids.first
+  end
+
   data('ack true' => true,
        'ack false' => false)
   test 'TLS transport and ack parameter combination' do |ack|
@@ -993,6 +1061,8 @@ EOL
   end
 
   test 'when out_forward has @id' do
+    omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
+
     # cancel https://github.com/fluent/fluentd/blob/077508ac817b7637307434d0c978d7cdc3d1c534/lib/fluent/plugin_id.rb#L43-L53
     # it always return true in test
     mock.proxy(Fluent::Plugin).new_sd(:static, anything) { |v|
@@ -1137,6 +1207,8 @@ EOL
   end
 
   test 'Create new connection per send_data' do
+    omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
+
     target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
     output_conf = CONFIG
     d = create_driver(output_conf)
@@ -1175,6 +1247,8 @@ EOL
 
   sub_test_case 'keepalive' do
     test 'Do not create connection per send_data' do
+      omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
+
       target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)
       output_conf = CONFIG + %[
         keepalive true

data/test/plugin/test_out_http.rb

@@ -127,7 +127,7 @@ class HTTPOutputTest < Test::Unit::TestCase
     now = Time.now
     started = false
     until started
-      raise "Server not started" if (now - Time.now > 10.0)
+      raise "Server not started" if (Time.now - now > 10.0)
       begin
         http_client { |c| c.request_get('/') }
         started = true

data/test/plugin_helper/test_child_process.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 require_relative '../helper'
 require 'fluent/plugin_helper/child_process'
 require 'fluent/plugin/base'
@@ -269,10 +270,12 @@ class ChildProcessTest < Test::Unit::TestCase
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
       ran = false
       @d.child_process_execute(:t4, "ruby -e 'Signal.trap(:TERM, nil); while sleep 0.1; puts 1; STDOUT.flush rescue nil; end'", mode: [:read]) do |io|
-        m.lock
-        ran = true
         begin
           while line = io.readline
+            unless ran
+              m.lock
+              ran = true
+            end
             ary << line
           end
         rescue

data/test/test_event.rb

@@ -401,6 +401,22 @@ module EventTest
         i += 1
       }
     end
+
+    # `any?` represents an Enumerable method which calls `each` internally
+    test 'size_after_any' do
+      @es.any?
+
+      assert_equal 2, @es.size
+    end
+
+    # `any?` represents an Enumerable method which calls `each` internally
+    test 'each_after_any' do
+      @es.any?
+
+      count = 0
+      @es.each { |time, record| count += 1 }
+      assert_equal 2, count
+    end
   end
 
   class CompressedMessagePackEventStreamTest < ::Test::Unit::TestCase

data/test/test_formatter.rb

@@ -184,6 +184,36 @@ module FormatterTest
 
       assert_equal("message=awesome,greeting=hello#{@newline}", formatted)
     end
+
+    def record_with_tab
+      {'message' => "awe\tsome", 'greeting' => "hello\t"}
+    end
+
+    def test_format_suppresses_tab
+      @formatter.configure({})
+      formatted = @formatter.format(tag, @time, record_with_tab)
+
+      assert_equal("message:awe some\tgreeting:hello #{@newline}", formatted)
+    end
+
+    def test_format_suppresses_tab_custom_replacement
+      @formatter.configure(
+        'replacement'      => 'X',
+      )
+      formatted = @formatter.format(tag, @time, record_with_tab)
+
+      assert_equal("message:aweXsome\tgreeting:helloX#{@newline}", formatted)
+    end
+
+    def test_format_suppresses_custom_delimiter
+      @formatter.configure(
+        'delimiter'       => 'w',
+        'label_delimiter' => '=',
+      )
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("message=a esomewgreeting=hello#{@newline}", formatted)
+    end
   end
 
   class CsvFormatterTest < ::Test::Unit::TestCase

data/test/test_time_parser.rb

@@ -228,4 +228,113 @@ class TimeParserTest < ::Test::Unit::TestCase
       assert_equal_event_time(time, parser.parse("#{time.sec}.#{time.nsec}"))
     end
   end
+
+  sub_test_case 'MixedTimeParser fallback' do
+    class DummyForTimeParser
+      include Fluent::Configurable
+      include Fluent::TimeMixin::Parser
+    end
+
+    test 'no time_format_fallbacks failure' do
+      i = DummyForTimeParser.new
+      assert_raise(Fluent::ConfigError.new("time_type is :mixed but time_format and time_format_fallbacks is empty.")) do
+        i.configure(config_element('parse', '', {'time_type' => 'mixed'}))
+      end
+    end
+
+    test 'fallback time format failure' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format_fallbacks' => ['%iso8601']}))
+      parser = i.time_parser_create
+      assert_raise(Fluent::TimeParser::TimeParseError.new("invalid time format: value = INVALID, even though fallbacks: Fluent::TimeParser")) do
+        parser.parse("INVALID")
+      end
+    end
+
+    test 'primary format is unixtime, secondary %iso8601 is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime',
+                                  'time_format_fallbacks' => ['%iso8601']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
+    end
+
+    test 'primary format is %iso8601, secondary unixtime is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601',
+                                  'time_format_fallbacks' => ['unixtime']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("#{time.sec}"))
+    end
+
+    test 'primary format is %iso8601, no secondary is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("2021-01-01T12:00:00+0900"))
+    end
+
+    test 'primary format is unixtime, no secondary is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("#{time.sec}"))
+    end
+
+    test 'primary format is %iso8601, raise error because of no appropriate secondary' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_raise("Fluent::TimeParser::TimeParseError: invalid time format: value = #{time.sec}, even though fallbacks: Fluent::TimeParser") do
+        parser.parse("#{time.sec}")
+      end
+    end
+
+    test 'primary format is unixtime, raise error because of no appropriate secondary' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_raise("Fluent::TimeParser::TimeParseError: invalid time format: value = #{time}, even though fallbacks: Fluent::NumericTimeParser") do
+        parser.parse("2021-01-01T12:00:00+0900")
+      end
+    end
+
+    test 'fallback to unixtime' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '', {'time_type' => 'mixed',
+                                               'time_format_fallbacks' => ['%iso8601', 'unixtime']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(Fluent::EventTime.new(time.to_i), parser.parse("#{time.sec}"))
+    end
+
+    test 'fallback to %iso8601' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '', {'time_type' => 'mixed',
+                                               'time_format_fallbacks' => ['unixtime', '%iso8601']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.12.1
+  version: 1.12.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-18 00:00:00.000000000 Z
+date: 2021-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -186,6 +186,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -350,18 +370,20 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".deepsource.toml"
 - ".drone.yml"
 - ".github/ISSUE_TEMPLATE.md"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/config.yml"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/PULL_REQUEST_TEMPLATE.md"
-- ".github/workflows/build.yaml"
 - ".github/workflows/issue-auto-closer.yml"
+- ".github/workflows/linux-test.yaml"
+- ".github/workflows/macos-test.yaml"
 - ".github/workflows/stale-actions.yml"
+- ".github/workflows/windows-test.yaml"
 - ".gitignore"
 - ".gitlab-ci.yml"
-- ".travis.yml"
 - ADOPTERS.md
 - AUTHORS
 - CHANGELOG.md
@@ -373,7 +395,6 @@ files:
 - MAINTAINERS.md
 - README.md
 - Rakefile
-- appveyor.yml
 - bin/fluent-binlog-reader
 - bin/fluent-ca-generate
 - bin/fluent-cap-ctl
@@ -910,8 +931,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.2.3
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector