fluentd 1.13.3 → 1.14.0.rc

data/test/plugin/test_multi_output.rb

@@ -146,8 +146,11 @@ class MultiOutputTest < Test::Unit::TestCase
       @i.configure(conf)
       assert_equal 4, @i.outputs.size
 
+      log_size_for_multi_output_itself = 4
+      log_size_for_metrics_plugin_helper = 4
+      expected_warn_log_size = log_size_for_multi_output_itself + log_size_for_metrics_plugin_helper
       logs = @i.log.out.logs
-      assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == 4 }
+      assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == expected_warn_log_size }
     end
 
     test '#emit_events calls #process always' do
@@ -176,5 +179,26 @@ class MultiOutputTest < Test::Unit::TestCase
 
       assert_equal 2, @i.events.size
     end
+
+    test 'can use metrics plugins and fallback methods' do
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+
+      %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name|
+        assert_true @i.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+      end
+
+      assert_equal 0, @i.num_errors
+      assert_equal 0, @i.emit_count
+      assert_equal 0, @i.emit_size
+      assert_equal 0, @i.emit_records
+    end
   end
 end

data/test/plugin/test_output.rb

@@ -223,6 +223,22 @@ class OutputTest < Test::Unit::TestCase
       assert @i.terminated?
     end
 
+    test 'can use metrics plugins and fallback methods' do
+      @i.configure(config_element())
+
+      %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics
+         write_count_metrics rollback_count_metrics flush_time_count_metrics slow_flush_count_metrics].each do |metric_name|
+        assert_true @i.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+      end
+
+      assert_equal 0, @i.num_errors
+      assert_equal 0, @i.emit_count
+      assert_equal 0, @i.emit_size
+      assert_equal 0, @i.emit_records
+      assert_equal 0, @i.write_count
+      assert_equal 0, @i.rollback_count
+    end
+
     data(:new_api => :chunk,
          :old_api => :metadata)
     test '#extract_placeholders does nothing if chunk key is not specified' do |api|

data/test/plugin_helper/test_event_emitter.rb

@@ -1,6 +1,7 @@
 require_relative '../helper'
 require 'fluent/plugin_helper/event_emitter'
 require 'fluent/plugin/base'
+require 'flexmock/test_unit'
 
 class EventEmitterTest < Test::Unit::TestCase
   setup do
@@ -48,4 +49,32 @@ class EventEmitterTest < Test::Unit::TestCase
 
     d1.terminate
   end
+
+  test 'should not have event_emitter_router' do
+    d0 = Dummy0.new
+    assert !d0.respond_to?(:event_emitter_router)
+  end
+
+  test 'should have event_emitter_router' do
+    d = Dummy.new
+    assert d.respond_to?(:event_emitter_router)
+  end
+
+  test 'get router' do
+    router_mock = flexmock('mytest')
+    label_mock = flexmock('mylabel')
+    label_mock.should_receive(:event_router).twice.and_return(router_mock)
+    Fluent::Engine.root_agent.labels['@mytest'] = label_mock
+
+    d = Dummy.new
+    d.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
+    router = d.event_emitter_router("@mytest")
+    assert_equal router_mock, router
+  end
+
+  test 'get root router' do
+    d = Dummy.new
+    router = d.event_emitter_router("@ROOT")
+    assert_equal Fluent::Engine.root_agent.event_router, router
+  end
 end

data/test/plugin_helper/test_metrics.rb

@@ -0,0 +1,137 @@
+require_relative '../helper'
+require 'fluent/plugin_helper/metrics'
+require 'fluent/plugin/base'
+
+class MetricsTest < Test::Unit::TestCase
+  class Dummy < Fluent::Plugin::TestBase
+    helpers :metrics
+    def configure(conf)
+      super
+    end
+  end
+
+  setup do
+    @d = nil
+  end
+
+  teardown do
+    if @d
+      @d.stop unless @d.stopped?
+      @d.shutdown unless @d.shutdown?
+      @d.close unless @d.closed?
+      @d.terminate unless @d.terminated?
+    end
+  end
+
+  test 'can be initialized without any metrics at first' do
+    d = Dummy.new
+    assert_equal 0, d._metrics.size
+  end
+
+  test 'can be configured' do
+    d1 = Dummy.new
+    assert_nothing_raised do
+      d1.configure(config_element())
+    end
+    assert d1.plugin_id
+    assert d1.log
+  end
+
+  test 'creates metrics instances' do
+    d = Dummy.new
+    i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
+    d.configure(config_element())
+    assert do
+      d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
+    end
+    assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
+    assert_true i.has_methods_for_counter
+    assert_false i.has_methods_for_gauge
+
+    d = Dummy.new
+    i = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
+    d.configure(config_element())
+    assert do
+      d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
+    end
+    assert{ i.is_a?(Fluent::Plugin::LocalMetrics) }
+    assert_false i.has_methods_for_counter
+    assert_true i.has_methods_for_gauge
+  end
+
+  test 'calls lifecycle methods for all plugin instances via owner plugin' do
+    @d = d = Dummy.new
+    i1 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics1", help_text: "metrics testing")
+    i2 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics2", help_text: "metrics testing", prefer_gauge: true)
+    i3 = d.metrics_create(namespace: "fluentd_test", subsystem: "unit-test", name: "metrics3", help_text: "metrics testing")
+    d.configure(config_element())
+    assert do
+      d.instance_variable_get(:@plugin_type_or_id).include?("dummy.object")
+    end
+    d.start
+
+    assert i1.started?
+    assert i2.started?
+    assert i3.started?
+
+    assert !i1.stopped?
+    assert !i2.stopped?
+    assert !i3.stopped?
+
+    d.stop
+
+    assert i1.stopped?
+    assert i2.stopped?
+    assert i3.stopped?
+
+    assert !i1.before_shutdown?
+    assert !i2.before_shutdown?
+    assert !i3.before_shutdown?
+
+    d.before_shutdown
+
+    assert i1.before_shutdown?
+    assert i2.before_shutdown?
+    assert i3.before_shutdown?
+
+    assert !i1.shutdown?
+    assert !i2.shutdown?
+    assert !i3.shutdown?
+
+    d.shutdown
+
+    assert i1.shutdown?
+    assert i2.shutdown?
+    assert i3.shutdown?
+
+    assert !i1.after_shutdown?
+    assert !i2.after_shutdown?
+    assert !i3.after_shutdown?
+
+    d.after_shutdown
+
+    assert i1.after_shutdown?
+    assert i2.after_shutdown?
+    assert i3.after_shutdown?
+
+    assert !i1.closed?
+    assert !i2.closed?
+    assert !i3.closed?
+
+    d.close
+
+    assert i1.closed?
+    assert i2.closed?
+    assert i3.closed?
+
+    assert !i1.terminated?
+    assert !i2.terminated?
+    assert !i3.terminated?
+
+    d.terminate
+
+    assert i1.terminated?
+    assert i2.terminated?
+    assert i3.terminated?
+  end
+end

data/test/test_plugin_classes.rb

@@ -5,11 +5,78 @@ require 'fluent/plugin/bare_output'
 require 'fluent/plugin/filter'
 
 module FluentTest
+  class FluentTestCounterMetrics < Fluent::Plugin::Metrics
+    Fluent::Plugin.register_metrics('test_counter', self)
+
+    attr_reader :data
+
+    def initialize
+      super
+      @data = 0
+    end
+    def get
+      @data
+    end
+    def inc
+      @data +=1
+    end
+    def add(value)
+      @data += value
+    end
+    def set(value)
+      @data = value
+    end
+    def close
+      @data = 0
+      super
+    end
+  end
+
+  class FluentTestGaugeMetrics < Fluent::Plugin::Metrics
+    Fluent::Plugin.register_metrics('test_gauge', self)
+
+    attr_reader :data
+
+    def initialize
+      super
+      @data = 0
+    end
+    def get
+      @data
+    end
+    def inc
+      @data += 1
+    end
+    def dec
+      @data -=1
+    end
+    def add(value)
+      @data += value
+    end
+    def sub(value)
+      @data -= value
+    end
+    def set(value)
+      @data = value
+    end
+    def close
+      @data = 0
+      super
+    end
+  end
+
   class FluentTestInput < ::Fluent::Plugin::Input
     ::Fluent::Plugin.register_input('test_in', self)
 
     attr_reader :started
 
+    def initialize
+      super
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def start
       super
       @started = true
@@ -28,6 +95,13 @@ module FluentTest
 
     config_param :num, :integer, default: 10000
 
+    def initialize
+      super
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def start
       super
       @started = true
@@ -49,6 +123,15 @@ module FluentTest
     def initialize
       super
       @events = Hash.new { |h, k| h[k] = [] }
+      # stub metrics instances
+      @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+      @write_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :events
@@ -168,6 +251,19 @@ module FluentTest
   class FluentTestErrorOutput < ::Fluent::Plugin::Output
     ::Fluent::Plugin.register_output('test_out_error', self)
 
+    def initialize
+      super
+      # stub metrics instances
+      @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+      @write_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def format(tag, time, record)
       raise "emit error!"
     end
@@ -184,6 +280,9 @@ module FluentTest
       super()
       @num = 0
       @field = field
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :num
@@ -213,6 +312,9 @@ module FluentTest
       super()
       @num = 0
       @field = field
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :num

data/test/test_root_agent.rb

@@ -16,7 +16,8 @@ class RootAgentTest < ::Test::Unit::TestCase
 
   data(
     'suppress interval' => [{'emit_error_log_interval' => 30}, {:@suppress_emit_error_log_interval => 30}],
-    'without source' => [{'without_source' => true}, {:@without_source => true}]
+    'without source' => [{'without_source' => true}, {:@without_source => true}],
+    'enable input metrics' => [{'enable_input_metrics' => true}, {:@enable_input_metrics => true}],
     )
   def test_initialize_with_opt(data)
     opt, expected = data
@@ -109,6 +110,34 @@ EOC
       end
     end
 
+    test 'raises configuration error for label without name' do
+      conf = <<-EOC
+<label>
+  @type test_out
+</label>
+EOC
+      errmsg = "Missing symbol argument on <label> directive"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error for <label @ROOT>' do
+      conf = <<-EOC
+<source>
+  @type test_in
+  @label @ROOT
+</source>
+<label @ROOT>
+  @type test_out
+</label>
+EOC
+      errmsg = "@ROOT for <label> is not permitted, reserved for getting root router"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        configure_ra(conf)
+      end
+    end
+
     test 'raises configuration error if there are not match sections in label section' do
       conf = <<-EOC
 <source>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.13.3
+  version: 1.14.0.rc
 platform: ruby
 authors:
 - Sadayuki Furuhashi
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-27 00:00:00.000000000 Z
+date: 2021-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -569,6 +569,8 @@ files:
 - lib/fluent/plugin/in_udp.rb
 - lib/fluent/plugin/in_unix.rb
 - lib/fluent/plugin/input.rb
+- lib/fluent/plugin/metrics.rb
+- lib/fluent/plugin/metrics_local.rb
 - lib/fluent/plugin/multi_output.rb
 - lib/fluent/plugin/out_copy.rb
 - lib/fluent/plugin/out_exec.rb
@@ -633,6 +635,7 @@ files:
 - lib/fluent/plugin_helper/http_server/server.rb
 - lib/fluent/plugin_helper/http_server/ssl_context_builder.rb
 - lib/fluent/plugin_helper/inject.rb
+- lib/fluent/plugin_helper/metrics.rb
 - lib/fluent/plugin_helper/parser.rb
 - lib/fluent/plugin_helper/record_accessor.rb
 - lib/fluent/plugin_helper/retry_state.rb
@@ -794,6 +797,8 @@ files:
 - test/plugin/test_in_unix.rb
 - test/plugin/test_input.rb
 - test/plugin/test_metadata.rb
+- test/plugin/test_metrics.rb
+- test/plugin/test_metrics_local.rb
 - test/plugin/test_multi_output.rb
 - test/plugin/test_out_copy.rb
 - test/plugin/test_out_exec.rb
@@ -869,6 +874,7 @@ files:
 - test/plugin_helper/test_formatter.rb
 - test/plugin_helper/test_http_server_helper.rb
 - test/plugin_helper/test_inject.rb
+- test/plugin_helper/test_metrics.rb
 - test/plugin_helper/test_parser.rb
 - test/plugin_helper/test_record_accessor.rb
 - test/plugin_helper/test_retry_state.rb
@@ -923,7 +929,7 @@ homepage: https://www.fluentd.org/
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -934,12 +940,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Fluentd event collector
 test_files:
@@ -1034,6 +1040,8 @@ test_files:
 - test/plugin/test_in_unix.rb
 - test/plugin/test_input.rb
 - test/plugin/test_metadata.rb
+- test/plugin/test_metrics.rb
+- test/plugin/test_metrics_local.rb
 - test/plugin/test_multi_output.rb
 - test/plugin/test_out_copy.rb
 - test/plugin/test_out_exec.rb
@@ -1109,6 +1117,7 @@ test_files:
 - test/plugin_helper/test_formatter.rb
 - test/plugin_helper/test_http_server_helper.rb
 - test/plugin_helper/test_inject.rb
+- test/plugin_helper/test_metrics.rb
 - test/plugin_helper/test_parser.rb
 - test/plugin_helper/test_record_accessor.rb
 - test/plugin_helper/test_retry_state.rb