fluentd 1.13.2 → 1.14.1

data/test/plugin/test_metrics.rb

@@ -0,0 +1,294 @@
+require_relative '../helper'
+require 'fluent/plugin/metrics'
+require 'fluent/plugin/base'
+require 'fluent/system_config'
+
+class BareMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('bare', self)
+
+  private
+
+  # Just override for tests.
+  def has_methods_for_counter?
+    false
+  end
+end
+
+class BasicCounterMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def add(value)
+    @data += value
+  end
+  def set(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class AliasedCounterMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def configure(conf)
+    super
+    class << self
+      alias_method :set, :set_counter
+    end
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def add(value)
+    @data += value
+  end
+  def set_counter(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class BasicGaugeMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def dec
+    @data -=1
+  end
+  def add(value)
+    @data += value
+  end
+  def sub(value)
+    @data -= value
+  end
+  def set(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class AliasedGaugeMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def configure(conf)
+    super
+    class << self
+      alias_method :dec, :dec_gauge
+      alias_method :set, :set_gauge
+      alias_method :sub, :sub_gauge
+    end
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def dec_gauge
+    @data -=1
+  end
+  def add(value)
+    @data += value
+  end
+  def sub_gauge(value)
+    @data -= value
+  end
+  def set_gauge(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class StorageTest < Test::Unit::TestCase
+  sub_test_case 'BareMetrics' do
+    setup do
+      @m = BareMetrics.new
+      @m.configure(config_element())
+    end
+
+    test 'is configured with plugin information and system config' do
+      m = BareMetrics.new
+      m.configure(config_element('metrics', '', {}))
+
+      assert_false m.use_gauge_metric
+      assert_false m.has_methods_for_counter
+      assert_false m.has_methods_for_gauge
+    end
+
+    test 'all bare operations are not defined yet' do
+      assert_raise NotImplementedError do
+        @m.get
+      end
+      assert_raise NotImplementedError do
+        @m.inc
+      end
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+      assert_raise NotImplementedError do
+        @m.add(10)
+      end
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+      assert_raise NotImplementedError do
+        @m.set(123)
+      end
+    end
+  end
+
+  sub_test_case 'BasicCounterMetric' do
+    setup do
+      @m = BasicCounterMetrics.new
+      @m.configure(config_element('metrics', '', {'@id' => '1'}))
+    end
+
+    test 'all basic counter operations work well' do
+      assert_true @m.has_methods_for_counter
+      assert_false @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+    end
+  end
+
+  sub_test_case 'AliasedCounterMetric' do
+    setup do
+      @m = AliasedCounterMetrics.new
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all aliased counter operations work well' do
+      assert_true @m.has_methods_for_counter
+      assert_false @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+    end
+  end
+
+  sub_test_case 'BasicGaugeMetric' do
+    setup do
+      @m = BasicGaugeMetrics.new
+      @m.use_gauge_metric = true
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all basic gauge operations work well' do
+      assert_false @m.has_methods_for_counter
+      assert_true @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      @m.dec
+      assert_equal 20, @m.get
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      @m.sub(11)
+      assert_equal 89, @m.get
+    end
+  end
+
+  sub_test_case 'AliasedGaugeMetric' do
+    setup do
+      @m = AliasedGaugeMetrics.new
+      @m.use_gauge_metric = true
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all aliased gauge operations work well' do
+      assert_false @m.has_methods_for_counter
+      assert_true @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      @m.dec
+      assert_equal 20, @m.get
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      @m.sub(11)
+      assert_equal 89, @m.get
+    end
+  end
+end

data/test/plugin/test_metrics_local.rb

@@ -0,0 +1,96 @@
+require_relative '../helper'
+require 'fluent/plugin/metrics_local'
+require 'fluent/system_config'
+
+class LocalMetricsTest < ::Test::Unit::TestCase
+  sub_test_case 'configure' do
+    test "configured for counter mode" do
+      m = Fluent::Plugin::LocalMetrics.new
+      m.configure(config_element('metrics', '', {"labels" => {test: "test-unit", language: "Ruby"}}))
+
+      assert_false m.use_gauge_metric
+      assert_equal({agent: "Fluentd", hostname: "#{Socket.gethostname}"}, m.default_labels)
+      assert_equal({test: "test-unit", language: "Ruby"}, m.labels)
+      assert_true m.has_methods_for_counter
+      assert_false m.has_methods_for_gauge
+    end
+
+    test "configured for gauge mode" do
+      m = Fluent::Plugin::LocalMetrics.new
+      m.use_gauge_metric = true
+      m.configure(config_element('metrics', '', {"labels" => {test: "test-unit", language: "Ruby"}}))
+
+      assert_true m.use_gauge_metric
+      assert_equal({agent: "Fluentd", hostname: "#{Socket.gethostname}"}, m.default_labels)
+      assert_equal({test: "test-unit", language: "Ruby"}, m.labels)
+      assert_false m.has_methods_for_counter
+      assert_true m.has_methods_for_gauge
+    end
+  end
+
+  sub_test_case 'LocalMetric' do
+    sub_test_case "counter" do
+      setup do
+        @m = Fluent::Plugin::LocalMetrics.new
+        @m.configure(config_element('metrics', '', {}))
+      end
+
+      test '#configure' do
+        assert_true @m.has_methods_for_counter
+        assert_false @m.has_methods_for_gauge
+      end
+
+      test 'all local counter operations work well' do
+        assert_equal 0, @m.get
+        assert_equal 1, @m.inc
+
+        @m.add(20)
+        assert_equal 21, @m.get
+        assert_raise NotImplementedError do
+          @m.dec
+        end
+
+        @m.set(100)
+        assert_equal 100, @m.get
+
+        @m.set(10)
+        assert_equal 100, @m.get # On counter, value should be overwritten bigger than stored one.
+        assert_raise NotImplementedError do
+          @m.sub(11)
+        end
+      end
+    end
+
+    sub_test_case "gauge" do
+      setup do
+        @m = Fluent::Plugin::LocalMetrics.new
+        @m.use_gauge_metric = true
+        @m.configure(config_element('metrics', '', {}))
+      end
+
+      test '#configure' do
+        assert_false @m.has_methods_for_counter
+        assert_true @m.has_methods_for_gauge
+      end
+
+      test 'all local gauge operations work well' do
+        assert_equal 0, @m.get
+        assert_equal 1, @m.inc
+
+        @m.add(20)
+        assert_equal 21, @m.get
+        @m.dec
+        assert_equal 20, @m.get
+
+        @m.set(100)
+        assert_equal 100, @m.get
+
+        @m.sub(11)
+        assert_equal 89, @m.get
+
+        @m.set(10)
+        assert_equal 10, @m.get # On gauge, value always should be overwritten.
+      end
+    end
+  end
+end

data/test/plugin/test_multi_output.rb

@@ -146,8 +146,11 @@ class MultiOutputTest < Test::Unit::TestCase
       @i.configure(conf)
       assert_equal 4, @i.outputs.size
 
+      log_size_for_multi_output_itself = 4
+      log_size_for_metrics_plugin_helper = 4
+      expected_warn_log_size = log_size_for_multi_output_itself + log_size_for_metrics_plugin_helper
       logs = @i.log.out.logs
-      assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == 4 }
+      assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == expected_warn_log_size }
     end
 
     test '#emit_events calls #process always' do
@@ -176,5 +179,26 @@ class MultiOutputTest < Test::Unit::TestCase
 
       assert_equal 2, @i.events.size
     end
+
+    test 'can use metrics plugins and fallback methods' do
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+
+      %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name|
+        assert_true @i.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+      end
+
+      assert_equal 0, @i.num_errors
+      assert_equal 0, @i.emit_count
+      assert_equal 0, @i.emit_size
+      assert_equal 0, @i.emit_records
+    end
   end
 end

data/test/plugin/test_output.rb

@@ -223,6 +223,22 @@ class OutputTest < Test::Unit::TestCase
       assert @i.terminated?
     end
 
+    test 'can use metrics plugins and fallback methods' do
+      @i.configure(config_element())
+
+      %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics
+         write_count_metrics rollback_count_metrics flush_time_count_metrics slow_flush_count_metrics].each do |metric_name|
+        assert_true @i.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+      end
+
+      assert_equal 0, @i.num_errors
+      assert_equal 0, @i.emit_count
+      assert_equal 0, @i.emit_size
+      assert_equal 0, @i.emit_records
+      assert_equal 0, @i.write_count
+      assert_equal 0, @i.rollback_count
+    end
+
     data(:new_api => :chunk,
          :old_api => :metadata)
     test '#extract_placeholders does nothing if chunk key is not specified' do |api|

data/test/plugin_helper/test_event_emitter.rb

@@ -1,6 +1,7 @@
 require_relative '../helper'
 require 'fluent/plugin_helper/event_emitter'
 require 'fluent/plugin/base'
+require 'flexmock/test_unit'
 
 class EventEmitterTest < Test::Unit::TestCase
   setup do
@@ -48,4 +49,32 @@ class EventEmitterTest < Test::Unit::TestCase
 
     d1.terminate
   end
+
+  test 'should not have event_emitter_router' do
+    d0 = Dummy0.new
+    assert !d0.respond_to?(:event_emitter_router)
+  end
+
+  test 'should have event_emitter_router' do
+    d = Dummy.new
+    assert d.respond_to?(:event_emitter_router)
+  end
+
+  test 'get router' do
+    router_mock = flexmock('mytest')
+    label_mock = flexmock('mylabel')
+    label_mock.should_receive(:event_router).twice.and_return(router_mock)
+    Fluent::Engine.root_agent.labels['@mytest'] = label_mock
+
+    d = Dummy.new
+    d.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
+    router = d.event_emitter_router("@mytest")
+    assert_equal router_mock, router
+  end
+
+  test 'get root router' do
+    d = Dummy.new
+    router = d.event_emitter_router("@ROOT")
+    assert_equal Fluent::Engine.root_agent.event_router, router
+  end
 end

data/test/plugin_helper/test_metrics.rb

@@ -0,0 +1,137 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/metrics'
+require 'fluent/plugin/base'
+
+class MetricsTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :metrics
+    def configure(conf)
+      super
+    end
+  end
+
+  setup do
+    @d = nil
+  end
+
+  teardown do
+    if @d
+      @d.stop unless @d.stopped?
+      @d.shutdown unless @d.shutdown?
+      @d.close unless @d.closed?
+      @d.terminate unless @d.terminated?
+    end
+  end
+
+  test 'can be initialized without any metrics at first' do
+    d = Dummy.new
+    assert_equal 0, d._metrics.size
+  end
+
+  test 'can be configured' do
+    d1 = Dummy.new
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+    assert d1.plugin_id
+    assert d1.log
+  end
+
+  test 'creates metrics instances' do
+    d = Dummy.new
+    i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
+    d.configure(config_element())
+    assert do
+      d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
+    end
+    assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
+    assert_true i.has_methods_for_counter
+    assert_false i.has_methods_for_gauge
+
+    d = Dummy.new
+    i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
+    d.configure(config_element())
+    assert do
+      d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
+    end
+    assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
+    assert_false i.has_methods_for_counter
+    assert_true i.has_methods_for_gauge
+  end
+
+  test 'calls lifecycle methods for all plugin instances via owner plugin' do
+    @d = d = Dummy.new
+    i1 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
+    i2 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
+    i3 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics3", help_text: "metrics testing")
+    d.configure(config_element())
+    assert do
+      d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
+    end
+    d.start
+
+    assert i1.started?
+    assert i2.started?
+    assert i3.started?
+
+    assert !i1.stopped?
+    assert !i2.stopped?
+    assert !i3.stopped?
+
+    d.stop
+
+    assert i1.stopped?
+    assert i2.stopped?
+    assert i3.stopped?
+
+    assert !i1.before_shutdown?
+    assert !i2.before_shutdown?
+    assert !i3.before_shutdown?
+
+    d.before_shutdown
+
+    assert i1.before_shutdown?
+    assert i2.before_shutdown?
+    assert i3.before_shutdown?
+
+    assert !i1.shutdown?
+    assert !i2.shutdown?
+    assert !i3.shutdown?
+
+    d.shutdown
+
+    assert i1.shutdown?
+    assert i2.shutdown?
+    assert i3.shutdown?
+
+    assert !i1.after_shutdown?
+    assert !i2.after_shutdown?
+    assert !i3.after_shutdown?
+
+    d.after_shutdown
+
+    assert i1.after_shutdown?
+    assert i2.after_shutdown?
+    assert i3.after_shutdown?
+
+    assert !i1.closed?
+    assert !i2.closed?
+    assert !i3.closed?
+
+    d.close
+
+    assert i1.closed?
+    assert i2.closed?
+    assert i3.closed?
+
+    assert !i1.terminated?
+    assert !i2.terminated?
+    assert !i3.terminated?
+
+    d.terminate
+
+    assert i1.terminated?
+    assert i2.terminated?
+    assert i3.terminated?
+  end
+end