fluentd 1.10.3 → 1.11.3

data/lib/fluent/plugin_helper/socket.rb

@@ -199,7 +199,7 @@ module Fluent
         list = []
         data.scan(pattern) { |match| list << OpenSSL::X509::Certificate.new(match) }
         if list.length == 0
-          log.warn "cert_path does not contain a valid certificate"
+          raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
         end
         list
       end

data/lib/fluent/supervisor.rb

@@ -302,6 +302,7 @@ module Fluent
       log_level = params['log_level']
       suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
       ignore_repeated_log_interval = params['ignore_repeated_log_interval']
+      ignore_same_log_interval = params['ignore_same_log_interval']
 
       log_path = params['log_path']
       chuser = params['chuser']
@@ -309,7 +310,8 @@ module Fluent
       log_rotate_age = params['log_rotate_age']
       log_rotate_size = params['log_rotate_size']
 
-      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval}
+      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
+                  ignore_same_log_interval: ignore_same_log_interval}
       logger_initializer = Supervisor::LoggerInitializer.new(
         log_path, log_level, chuser, chgroup, log_opts,
         log_rotate_age: log_rotate_age,
@@ -347,6 +349,7 @@ module Fluent
         chumask: 0,
         suppress_repeated_stacktrace: suppress_repeated_stacktrace,
         ignore_repeated_log_interval: ignore_repeated_log_interval,
+        ignore_same_log_interval: ignore_same_log_interval,
         daemonize: daemonize,
         rpc_endpoint: params['rpc_endpoint'],
         counter_server: params['counter_server'],
@@ -441,10 +444,11 @@ module Fluent
         self
       end
 
-      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil)
+      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
         $log.format = format if format
         $log.time_format = time_format if time_format
         $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
+        $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
 
         if @path && log_dir_perm
           File.chmod(log_dir_perm || 0755, File.dirname(@path))
@@ -511,7 +515,8 @@ module Fluent
       @cl_opt = opt
       @conf = nil
 
-      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval]}
+      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
+                  ignore_same_log_interval: opt[:ignore_same_log_interval]}
       @log = LoggerInitializer.new(
         @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
         log_rotate_age: @log_rotate_age,
@@ -593,7 +598,10 @@ module Fluent
 
       main_process do
         create_socket_manager if @standalone_worker
-        ServerEngine::Privilege.change(@chuser, @chgroup) if @standalone_worker
+        if @standalone_worker
+          ServerEngine::Privilege.change(@chuser, @chgroup)
+          File.umask(0)
+        end
         MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
         Fluent::Engine.init(@system_config)
         Fluent::Engine.run_configure(@conf)
@@ -632,7 +640,8 @@ module Fluent
         format: @system_config.log.format,
         time_format: @system_config.log.time_format,
         log_dir_perm: @system_config.dir_permission,
-        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval
+        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
+        ignore_same_log_interval: @system_config.ignore_same_log_interval
       )
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
@@ -801,7 +810,7 @@ module Fluent
             Fluent::Engine.reload_config(conf)
           end
         rescue => e
-          # it is guranteed that config file is valid by supervisor side. but it's not atomic becuase of using signals to commnicate between worker and super
+          # it is guaranteed that config file is valid by supervisor side. but it's not atomic because of using signals to commnicate between worker and super
           # So need this rescue code
           $log.error("failed to reload config: #{e}")
           next

data/lib/fluent/system_config.rb

@@ -24,7 +24,7 @@ module Fluent
     SYSTEM_CONFIG_PARAMETERS = [
       :workers, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
-      :log_event_verbose, :ignore_repeated_log_interval,
+      :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support
@@ -35,6 +35,7 @@ module Fluent
     config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil
     config_param :ignore_repeated_log_interval, :time, default: nil
+    config_param :ignore_same_log_interval, :time, default: nil
     config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
     config_param :log_event_verbose,    :bool, default: nil

data/lib/fluent/test/filter_test.rb

@@ -30,12 +30,12 @@ module Fluent
       attr_reader :filtered
       attr_accessor :tag
 
-      def emit(record, time = Engine.now)
+      def emit(record, time = EventTime.now)
         emit_with_tag(@tag, record, time)
       end
       alias_method :filter, :emit
 
-      def emit_with_tag(tag, record, time = Engine.now)
+      def emit_with_tag(tag, record, time = EventTime.now)
         @events[tag] ||= MultiEventStream.new
         @events[tag].add(time, record)
       end

data/lib/fluent/test/output_test.rb

@@ -41,7 +41,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         es = OneEventStream.new(time, record)
         @instance.emit_events(@tag, es)
       end
@@ -62,7 +62,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end
@@ -110,7 +110,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.10.3'
+  VERSION = '1.11.3'
 
 end

data/test/plugin/out_forward/test_load_balancer.rb

@@ -49,6 +49,44 @@ class LoadBalancerTest < Test::Unit::TestCase
       end
     end
 
+    test 'call like round robin without weight=0 node' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)
+
+      lb.rebuild_weight_array([n1, n2, n3])
+
+      lb.select_healthy_node do |node|
+        # to handle random choice
+        if node == n1
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+        else
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+        end
+      end
+    end
+
     test 'raise an error if all node are unavialble' do
       lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
       lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
@@ -56,5 +94,13 @@ class LoadBalancerTest < Test::Unit::TestCase
         lb.select_healthy_node
       end
     end
+
+    test 'it regards weight=0 node as unavialble' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)])
+      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
+        lb.select_healthy_node
+      end
+    end
   end
 end

data/test/plugin/test_buffer.rb

@@ -543,6 +543,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
       assert_equal [@dm2,@dm3,m], @p.stage.keys
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 
@@ -675,6 +677,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm2,@dm3,m], @p.stage.keys
       assert_equal 1, @p.stage[m].append_count
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 

data/test/plugin/test_in_exec.rb

@@ -240,4 +240,22 @@ EOC
       assert_equal [tag, time, record], event
     }
   end
+
+  test 'emit error message with read_with_stderr' do
+    d = create_driver %[
+      tag test
+      command ruby #{File.join(File.dirname(SCRIPT_PATH), 'foo_bar_baz_no_existence.rb')}
+      connect_mode read_with_stderr
+      <parse>
+        @type none
+      </parse>
+    ]
+    d.run(expect_records: 1, timeout: 10)
+
+    assert{ d.events.length > 0 }
+    d.events.each do |event|
+      assert_equal 'test', event[0]
+      assert_match /LoadError/, event[2]['message']
+    end
+  end
 end

data/test/plugin/test_in_gc_stat.rb

@@ -22,9 +22,14 @@ class GCStatInputTest < Test::Unit::TestCase
     assert_equal("t1", d.instance.tag)
   end
 
-  def test_emit
+  def setup_gc_stat
     stat = GC.stat
     stub(GC).stat { stat }
+    stat
+  end
+
+  def test_emit
+    stat = setup_gc_stat
 
     d = create_driver
     d.run(expect_emits: 2)
@@ -36,4 +41,22 @@ class GCStatInputTest < Test::Unit::TestCase
       assert(events[i][1].is_a?(Fluent::EventTime))
     }
   end
+
+  def test_emit_with_use_symbol_keys_false
+    stat = setup_gc_stat
+    result = {}
+    stat.each_pair { |k, v|
+      result[k.to_s] = v
+    }
+
+    d = create_driver(CONFIG + "use_symbol_keys false")
+    d.run(expect_emits: 2)
+
+    events = d.events
+    assert(events.length > 0)
+    events.each_index {|i|
+      assert_equal(result, events[i][2])
+      assert(events[i][1].is_a?(Fluent::EventTime))
+    }
+  end
 end

data/test/plugin/test_in_http.rb

@@ -116,6 +116,36 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  data('json' => ['json', :to_json],
+       'msgpack' => ['msgpack', :to_msgpack])
+  def test_default_with_time_format(data)
+    param, method_name = data
+    d = create_driver(CONFIG + %[
+      <parse>
+        keep_time_key
+        time_format %iso8601
+      </parse>
+    ])
+
+    time = event_time("2020-06-10T01:14:27+00:00")
+    events = [
+      ["tag1", time, {"a" => 1, "time" => '2020-06-10T01:14:27+00:00'}],
+      ["tag2", time, {"a" => 2, "time" => '2020-06-10T01:14:27+00:00'}],
+    ]
+    res_codes = []
+
+    d.run(expect_records: 2) do
+      events.each do |tag, t, record|
+        res = post("/#{tag}", {param => record.__send__(method_name)})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_multi_json
     d = create_driver
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -163,6 +193,33 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  data('json' => ['json', :to_json],
+       'msgpack' => ['msgpack', :to_msgpack])
+  def test_default_multi_with_time_format(data)
+    param, method_name = data
+    d = create_driver(CONFIG + %[
+      <parse>
+        keep_time_key
+        time_format %iso8601
+      </parse>
+    ])
+    time = event_time("2020-06-10T01:14:27+00:00")
+    events = [
+      ["tag1", time, {'a' => 1, 'time' => "2020-06-10T01:14:27+00:00"}],
+      ["tag1", time, {'a' => 2, 'time' => "2020-06-10T01:14:27+00:00"}],
+    ]
+    tag = "tag1"
+    res_codes = []
+    d.run(expect_records: 2, timeout: 5) do
+      res = post("/#{tag}", {param => events.map { |e| e[2] }.__send__(method_name)})
+      res_codes << res.code
+    end
+    assert_equal ["200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_multi_json_with_nonexistent_time_key
     d = create_driver(CONFIG + %[
       <parse>

data/test/plugin/{test_in_dummy.rb → test_in_sample.rb}

@@ -1,33 +1,33 @@
 require_relative '../helper'
 require 'fluent/test/driver/input'
-require 'fluent/plugin/in_dummy'
+require 'fluent/plugin/in_sample'
 require 'fileutils'
 
-class DummyTest < Test::Unit::TestCase
+class SampleTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
   def create_driver(conf)
-    Fluent::Test::Driver::Input.new(Fluent::Plugin::DummyInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SampleInput).configure(conf)
   end
 
   sub_test_case 'configure' do
     test 'required parameters' do
       assert_raise_message("'tag' parameter is required") do
-        Fluent::Plugin::DummyInput.new.configure(config_element('ROOT',''))
+        Fluent::Plugin::SampleInput.new.configure(config_element('ROOT',''))
       end
     end
 
     test 'tag' do
       d = create_driver(%[
-        tag dummy
+        tag sample
       ])
-      assert_equal "dummy", d.instance.tag
+      assert_equal "sample", d.instance.tag
     end
 
     config = %[
-      tag dummy
+      tag sample
     ]
 
     test 'auto_increment_key' do
@@ -44,30 +44,30 @@ class DummyTest < Test::Unit::TestCase
       assert_equal 10, d.instance.rate
     end
 
-    test 'dummy' do
+    test 'sample' do
       # hash is okay
-      d = create_driver(config + %[dummy {"foo":"bar"}])
-      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+      d = create_driver(config + %[sample {"foo":"bar"}])
+      assert_equal [{"foo"=>"bar"}], d.instance.sample
 
       # array of hash is okay
-      d = create_driver(config + %[dummy [{"foo":"bar"}]])
-      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+      d = create_driver(config + %[sample [{"foo":"bar"}]])
+      assert_equal [{"foo"=>"bar"}], d.instance.sample
 
       assert_raise_message(/JSON::ParserError|got incomplete JSON/) do
-        create_driver(config + %[dummy "foo"])
+        create_driver(config + %[sample "foo"])
       end
 
       assert_raise_message(/is not a hash/) do
-        create_driver(config + %[dummy ["foo"]])
+        create_driver(config + %[sample ["foo"]])
       end
     end
   end
 
   sub_test_case "emit" do
     config = %[
-      tag dummy
+      tag sample
       rate 10
-      dummy {"foo":"bar"}
+      sample {"foo":"bar"}
     ]
 
     test 'simple' do
@@ -75,7 +75,7 @@ class DummyTest < Test::Unit::TestCase
       d.run(timeout: 0.5)
 
       d.events.each do |tag, time, record|
-        assert_equal("dummy", tag)
+        assert_equal("sample", tag)
         assert_equal({"foo"=>"bar"}, record)
         assert(time.is_a?(Fluent::EventTime))
       end
@@ -86,20 +86,20 @@ class DummyTest < Test::Unit::TestCase
       d.run(timeout: 0.5)
 
       d.events.each_with_index do |(tag, _time, record), i|
-        assert_equal("dummy", tag)
+        assert_equal("sample", tag)
         assert_equal({"foo"=>"bar", "id"=>i}, record)
       end
     end
   end
 
-  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_dummy', 'store' )
+  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_sample', 'store' )
   FileUtils.mkdir_p TEST_PLUGIN_STORAGE_PATH
 
-  sub_test_case 'when dummy plugin has storage which is not specified the path'  do
+  sub_test_case 'when sample plugin has storage which is not specified the path'  do
     config1 = {
-      'tag' => 'dummy',
+      'tag' => 'sample',
       'rate' => '0',
-      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'sample' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
       'auto_increment_key' => 'id',
     }
     conf1 = config_element('ROOT', '', config1, [])
@@ -135,7 +135,7 @@ class DummyTest < Test::Unit::TestCase
     end
   end
 
-  sub_test_case 'when dummy plugin has storage which is specified the path'  do
+  sub_test_case 'when sample plugin has storage which is specified the path'  do
     setup do
       FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
       FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
@@ -144,9 +144,9 @@ class DummyTest < Test::Unit::TestCase
 
     config2 = {
       '@id' => 'test-02',
-      'tag' => 'dummy',
+      'tag' => 'sample',
       'rate' => '0',
-      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'sample' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
       'auto_increment_key' => 'id',
     }
     conf2 = config_element('ROOT', '', config2, [