fluentd 1.13.1 → 1.14.0

data/test/plugin/test_metrics_local.rb

@@ -0,0 +1,96 @@
+require_relative '../helper'
+require 'fluent/plugin/metrics_local'
+require 'fluent/system_config'
+
+class LocalMetricsTest < ::Test::Unit::TestCase
+  sub_test_case 'configure' do
+    test "configured for counter mode" do
+      m = Fluent::Plugin::LocalMetrics.new
+      m.configure(config_element('metrics', '', {"labels" => {test: "test-unit", language: "Ruby"}}))
+
+      assert_false m.use_gauge_metric
+      assert_equal({agent: "Fluentd", hostname: "#{Socket.gethostname}"}, m.default_labels)
+      assert_equal({test: "test-unit", language: "Ruby"}, m.labels)
+      assert_true m.has_methods_for_counter
+      assert_false m.has_methods_for_gauge
+    end
+
+    test "configured for gauge mode" do
+      m = Fluent::Plugin::LocalMetrics.new
+      m.use_gauge_metric = true
+      m.configure(config_element('metrics', '', {"labels" => {test: "test-unit", language: "Ruby"}}))
+
+      assert_true m.use_gauge_metric
+      assert_equal({agent: "Fluentd", hostname: "#{Socket.gethostname}"}, m.default_labels)
+      assert_equal({test: "test-unit", language: "Ruby"}, m.labels)
+      assert_false m.has_methods_for_counter
+      assert_true m.has_methods_for_gauge
+    end
+  end
+
+  sub_test_case 'LocalMetric' do
+    sub_test_case "counter" do
+      setup do
+        @m = Fluent::Plugin::LocalMetrics.new
+        @m.configure(config_element('metrics', '', {}))
+      end
+
+      test '#configure' do
+        assert_true @m.has_methods_for_counter
+        assert_false @m.has_methods_for_gauge
+      end
+
+      test 'all local counter operations work well' do
+        assert_equal 0, @m.get
+        assert_equal 1, @m.inc
+
+        @m.add(20)
+        assert_equal 21, @m.get
+        assert_raise NotImplementedError do
+          @m.dec
+        end
+
+        @m.set(100)
+        assert_equal 100, @m.get
+
+        @m.set(10)
+        assert_equal 100, @m.get # On counter, value should be overwritten bigger than stored one.
+        assert_raise NotImplementedError do
+          @m.sub(11)
+        end
+      end
+    end
+
+    sub_test_case "gauge" do
+      setup do
+        @m = Fluent::Plugin::LocalMetrics.new
+        @m.use_gauge_metric = true
+        @m.configure(config_element('metrics', '', {}))
+      end
+
+      test '#configure' do
+        assert_false @m.has_methods_for_counter
+        assert_true @m.has_methods_for_gauge
+      end
+
+      test 'all local gauge operations work well' do
+        assert_equal 0, @m.get
+        assert_equal 1, @m.inc
+
+        @m.add(20)
+        assert_equal 21, @m.get
+        @m.dec
+        assert_equal 20, @m.get
+
+        @m.set(100)
+        assert_equal 100, @m.get
+
+        @m.sub(11)
+        assert_equal 89, @m.get
+
+        @m.set(10)
+        assert_equal 10, @m.get # On gauge, value always should be overwritten.
+      end
+    end
+  end
+end

data/test/plugin/test_multi_output.rb

@@ -146,8 +146,11 @@ class MultiOutputTest < Test::Unit::TestCase
       @i.configure(conf)
       assert_equal 4, @i.outputs.size
 
+      log_size_for_multi_output_itself = 4
+      log_size_for_metrics_plugin_helper = 4
+      expected_warn_log_size = log_size_for_multi_output_itself + log_size_for_metrics_plugin_helper
       logs = @i.log.out.logs
-      assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == 4 }
+      assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == expected_warn_log_size }
     end
 
     test '#emit_events calls #process always' do
@@ -176,5 +179,26 @@ class MultiOutputTest < Test::Unit::TestCase
 
       assert_equal 2, @i.events.size
     end
+
+    test 'can use metrics plugins and fallback methods' do
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+
+      %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name|
+        assert_true @i.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+      end
+
+      assert_equal 0, @i.num_errors
+      assert_equal 0, @i.emit_count
+      assert_equal 0, @i.emit_size
+      assert_equal 0, @i.emit_records
+    end
   end
 end

data/test/plugin/test_output.rb

@@ -223,6 +223,22 @@ class OutputTest < Test::Unit::TestCase
       assert @i.terminated?
     end
 
+    test 'can use metrics plugins and fallback methods' do
+      @i.configure(config_element())
+
+      %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics
+         write_count_metrics rollback_count_metrics flush_time_count_metrics slow_flush_count_metrics].each do |metric_name|
+        assert_true @i.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+      end
+
+      assert_equal 0, @i.num_errors
+      assert_equal 0, @i.emit_count
+      assert_equal 0, @i.emit_size
+      assert_equal 0, @i.emit_records
+      assert_equal 0, @i.write_count
+      assert_equal 0, @i.rollback_count
+    end
+
     data(:new_api => :chunk,
          :old_api => :metadata)
     test '#extract_placeholders does nothing if chunk key is not specified' do |api|

data/test/plugin_helper/test_event_emitter.rb

@@ -1,6 +1,7 @@
 require_relative '../helper'
 require 'fluent/plugin_helper/event_emitter'
 require 'fluent/plugin/base'
+require 'flexmock/test_unit'
 
 class EventEmitterTest < Test::Unit::TestCase
   setup do
@@ -48,4 +49,32 @@ class EventEmitterTest < Test::Unit::TestCase
 
     d1.terminate
   end
+
+  test 'should not have event_emitter_router' do
+    d0 = Dummy0.new
+    assert !d0.respond_to?(:event_emitter_router)
+  end
+
+  test 'should have event_emitter_router' do
+    d = Dummy.new
+    assert d.respond_to?(:event_emitter_router)
+  end
+
+  test 'get router' do
+    router_mock = flexmock('mytest')
+    label_mock = flexmock('mylabel')
+    label_mock.should_receive(:event_router).twice.and_return(router_mock)
+    Fluent::Engine.root_agent.labels['@mytest'] = label_mock
+
+    d = Dummy.new
+    d.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
+    router = d.event_emitter_router("@mytest")
+    assert_equal router_mock, router
+  end
+
+  test 'get root router' do
+    d = Dummy.new
+    router = d.event_emitter_router("@ROOT")
+    assert_equal Fluent::Engine.root_agent.event_router, router
+  end
 end

data/test/plugin_helper/test_metrics.rb

@@ -0,0 +1,137 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/metrics'
+require 'fluent/plugin/base'
+
+class MetricsTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :metrics
+    def configure(conf)
+      super
+    end
+  end
+
+  setup do
+    @d = nil
+  end
+
+  teardown do
+    if @d
+      @d.stop unless @d.stopped?
+      @d.shutdown unless @d.shutdown?
+      @d.close unless @d.closed?
+      @d.terminate unless @d.terminated?
+    end
+  end
+
+  test 'can be initialized without any metrics at first' do
+    d = Dummy.new
+    assert_equal 0, d._metrics.size
+  end
+
+  test 'can be configured' do
+    d1 = Dummy.new
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+    assert d1.plugin_id
+    assert d1.log
+  end
+
+  test 'creates metrics instances' do
+    d = Dummy.new
+    i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
+    d.configure(config_element())
+    assert do
+      d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
+    end
+    assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
+    assert_true i.has_methods_for_counter
+    assert_false i.has_methods_for_gauge
+
+    d = Dummy.new
+    i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
+    d.configure(config_element())
+    assert do
+      d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
+    end
+    assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
+    assert_false i.has_methods_for_counter
+    assert_true i.has_methods_for_gauge
+  end
+
+  test 'calls lifecycle methods for all plugin instances via owner plugin' do
+    @d = d = Dummy.new
+    i1 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
+    i2 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
+    i3 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics3", help_text: "metrics testing")
+    d.configure(config_element())
+    assert do
+      d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
+    end
+    d.start
+
+    assert i1.started?
+    assert i2.started?
+    assert i3.started?
+
+    assert !i1.stopped?
+    assert !i2.stopped?
+    assert !i3.stopped?
+
+    d.stop
+
+    assert i1.stopped?
+    assert i2.stopped?
+    assert i3.stopped?
+
+    assert !i1.before_shutdown?
+    assert !i2.before_shutdown?
+    assert !i3.before_shutdown?
+
+    d.before_shutdown
+
+    assert i1.before_shutdown?
+    assert i2.before_shutdown?
+    assert i3.before_shutdown?
+
+    assert !i1.shutdown?
+    assert !i2.shutdown?
+    assert !i3.shutdown?
+
+    d.shutdown
+
+    assert i1.shutdown?
+    assert i2.shutdown?
+    assert i3.shutdown?
+
+    assert !i1.after_shutdown?
+    assert !i2.after_shutdown?
+    assert !i3.after_shutdown?
+
+    d.after_shutdown
+
+    assert i1.after_shutdown?
+    assert i2.after_shutdown?
+    assert i3.after_shutdown?
+
+    assert !i1.closed?
+    assert !i2.closed?
+    assert !i3.closed?
+
+    d.close
+
+    assert i1.closed?
+    assert i2.closed?
+    assert i3.closed?
+
+    assert !i1.terminated?
+    assert !i2.terminated?
+    assert !i3.terminated?
+
+    d.terminate
+
+    assert i1.terminated?
+    assert i2.terminated?
+    assert i3.terminated?
+  end
+end

data/test/test_event_time.rb

@@ -64,8 +64,8 @@ class EventTimeTest < Test::Unit::TestCase
 
   test 'Oj.dump' do
     time = Fluent::EventTime.new(100)
-    require 'fluent/env'
-    Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+    require 'fluent/oj_options'
+    Fluent::OjOptions.load_env
     assert_equal('{"time":100}', Oj.dump({'time' => time}))
     assert_equal('["tag",100,{"key":"value"}]', Oj.dump(["tag", time, {"key" => "value"}], mode: :compat))
   end

data/test/test_oj_options.rb

@@ -0,0 +1,55 @@
+require_relative 'helper'
+require 'fluent/test'
+require 'fluent/oj_options'
+
+class OjOptionsTest < ::Test::Unit::TestCase
+  begin
+    require 'oj'
+    @@oj_is_avaibale = true
+  rescue LoadError
+    @@oj_is_avaibale = false
+  end
+
+  setup do
+    @orig_env = {}
+    ENV.each do |key, value|
+      @orig_env[key] = value if key.start_with?("FLUENT_OJ_OPTION_")
+    end
+  end
+
+  teardown do
+    ENV.delete_if { |key| key.start_with?("FLUENT_OJ_OPTION_") }
+    @orig_env.each { |key, value| ENV[key] = value }
+  end
+
+  test "available?" do
+    assert_equal(@@oj_is_avaibale, Fluent::OjOptions.available?)
+  end
+
+  sub_test_case "set by environment variable" do
+    test "when no env vars set, returns default options" do
+      ENV.delete_if { |key| key.start_with?("FLUENT_OJ_OPTION_") }
+      defaults = Fluent::OjOptions::DEFAULTS
+      assert_equal(defaults, Fluent::OjOptions.load_env)
+      assert_equal(defaults, Oj.default_options.slice(*defaults.keys)) if @@oj_is_avaibale
+    end
+
+    test "valid env var passed with valid value, default is overridden" do
+      ENV["FLUENT_OJ_OPTION_BIGDECIMAL_LOAD"] = ":bigdecimal"
+      assert_equal(:bigdecimal, Fluent::OjOptions.load_env[:bigdecimal_load])
+      assert_equal(:bigdecimal, Oj.default_options[:bigdecimal_load]) if @@oj_is_avaibale
+    end
+
+    test "valid env var passed with invalid value, default is not overriden" do
+      ENV["FLUENT_OJ_OPTION_BIGDECIMAL_LOAD"] = ":conor"
+      assert_equal(:float, Fluent::OjOptions.load_env[:bigdecimal_load])
+      assert_equal(:float, Oj.default_options[:bigdecimal_load]) if @@oj_is_avaibale
+    end
+
+    test "invalid env var passed, nothing done with it" do
+      ENV["FLUENT_OJ_OPTION_CONOR"] = ":conor"
+      assert_equal(nil, Fluent::OjOptions.load_env[:conor])
+      assert_equal(nil, Oj.default_options[:conor]) if @@oj_is_avaibale
+    end
+  end
+end

data/test/test_plugin_classes.rb

@@ -5,11 +5,78 @@ require 'fluent/plugin/bare_output'
 require 'fluent/plugin/filter'
 
 module FluentTest
+  class FluentTestCounterMetrics < Fluent::Plugin::Metrics
+    Fluent::Plugin.register_metrics('test_counter', self)
+
+    attr_reader :data
+
+    def initialize
+      super
+      @data = 0
+    end
+    def get
+      @data
+    end
+    def inc
+      @data +=1
+    end
+    def add(value)
+      @data += value
+    end
+    def set(value)
+      @data = value
+    end
+    def close
+      @data = 0
+      super
+    end
+  end
+
+  class FluentTestGaugeMetrics < Fluent::Plugin::Metrics
+    Fluent::Plugin.register_metrics('test_gauge', self)
+
+    attr_reader :data
+
+    def initialize
+      super
+      @data = 0
+    end
+    def get
+      @data
+    end
+    def inc
+      @data += 1
+    end
+    def dec
+      @data -=1
+    end
+    def add(value)
+      @data += value
+    end
+    def sub(value)
+      @data -= value
+    end
+    def set(value)
+      @data = value
+    end
+    def close
+      @data = 0
+      super
+    end
+  end
+
   class FluentTestInput < ::Fluent::Plugin::Input
     ::Fluent::Plugin.register_input('test_in', self)
 
     attr_reader :started
 
+    def initialize
+      super
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def start
       super
       @started = true
@@ -28,6 +95,13 @@ module FluentTest
 
     config_param :num, :integer, default: 10000
 
+    def initialize
+      super
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def start
       super
       @started = true
@@ -49,6 +123,15 @@ module FluentTest
     def initialize
       super
       @events = Hash.new { |h, k| h[k] = [] }
+      # stub metrics instances
+      @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+      @write_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :events
@@ -168,6 +251,19 @@ module FluentTest
   class FluentTestErrorOutput < ::Fluent::Plugin::Output
     ::Fluent::Plugin.register_output('test_out_error', self)
 
+    def initialize
+      super
+      # stub metrics instances
+      @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+      @write_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def format(tag, time, record)
       raise "emit error!"
     end
@@ -184,6 +280,9 @@ module FluentTest
       super()
       @num = 0
       @field = field
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :num
@@ -213,6 +312,9 @@ module FluentTest
       super()
       @num = 0
       @field = field
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :num