fluentd 1.13.3 → 1.14.0.rc

data/lib/fluent/supervisor.rb

@@ -570,6 +570,8 @@ module Fluent
         suppress_repeated_stacktrace: true,
         ignore_repeated_log_interval: nil,
         without_source: nil,
+        enable_input_metrics: nil,
+        enable_size_metrics: nil,
         use_v1_config: true,
         strict_config_value: nil,
         supervise: true,

data/lib/fluent/system_config.rb

@@ -27,7 +27,8 @@ module Fluent
       :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
-      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket
+      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
+      :metrics, :enable_input_metrics, :enable_size_metrics
     ]
 
     config_param :workers,   :integer, default: 1
@@ -46,6 +47,8 @@ module Fluent
     config_param :strict_config_value, :bool, default: nil
     config_param :enable_msgpack_time_support, :bool, default: nil
     config_param :disable_shared_socket, :bool, default: nil
+    config_param :enable_input_metrics, :bool, default: nil
+    config_param :enable_size_metrics, :bool, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end
@@ -93,6 +96,11 @@ module Fluent
       config_param :timeout, :time, default: nil
     end
 
+    config_section :metrics, multi: false do
+      config_param :@type, :string, default: "local"
+      config_param :labels, :hash, default: {}
+    end
+
     def self.create(conf, strict_config_value=false)
       systems = conf.elements(name: 'system')
       return SystemConfig.new if systems.empty?

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.13.3'
+  VERSION = '1.14.0.rc'
 
 end

data/test/config/test_system_config.rb

@@ -28,6 +28,8 @@ module Fluent::Config
         log_event_label: nil,
         log_event_verbose: nil,
         without_source: nil,
+        enable_input_metrics: nil,
+        enable_size_metrics: nil,
         emit_error_log_interval: nil,
         file_permission: nil,
         dir_permission: nil,
@@ -77,6 +79,8 @@ module Fluent::Config
       assert_nil(sc.emit_error_log_interval)
       assert_nil(sc.suppress_config_dump)
       assert_nil(sc.without_source)
+      assert_nil(sc.enable_input_metrics)
+      assert_nil(sc.enable_size_metrics)
       assert_nil(sc.enable_msgpack_time_support)
       assert_equal(:text, sc.log.format)
       assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
@@ -93,6 +97,8 @@ module Fluent::Config
       'without_source' => ['without_source', true],
       'strict_config_value' => ['strict_config_value', true],
       'enable_msgpack_time_support' => ['enable_msgpack_time_support', true],
+      'enable_input_metrics' => ['enable_input_metrics', true],
+      'enable_size_metrics' => ['enable_size_metrics', true],
     )
     test "accepts parameters" do |(k, v)|
       conf = parse_text(<<-EOS)

data/test/plugin/in_tail/test_position_file.rb

@@ -22,6 +22,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
     invalidpath100000000000000000000000000000000
     unwatched\t#{UNWATCHED_STR}\t0000000000000000
   EOF
+  TEST_CONTENT_PATHS = {
+    "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
+    "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
+  }
 
   def write_data(f, content)
     f.write(content)
@@ -36,7 +40,11 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
   test '.load' do
     write_data(@file, TEST_CONTENT)
-    Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+    paths = {
+      "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
+      "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 2),
+    }
+    Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
     @file.seek(0)
     lines = @file.readlines
@@ -118,7 +126,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'should ignore initial existing files on follow_inode' do
       write_data(@file, TEST_CONTENT)
-      pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, {}, **{logger: $log})
+      pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_PATHS, **{logger: $log})
       @file.seek(0)
       assert_equal([], @file.readlines)
 
@@ -138,7 +146,12 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#load' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+      invalid_path = "invalidpath100000000000000000000000000000000"
+      paths = TEST_CONTENT_PATHS.merge({
+        invalid_path => Fluent::Plugin::TailInput::TargetInfo.new(invalid_path, 0),
+        "unwatched" => Fluent::Plugin::TailInput::TargetInfo.new("unwatched", 0),
+      })
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
       @file.seek(0)
       lines = @file.readlines
@@ -147,6 +160,15 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
     end
 
+    test 'compact deleted paths' do
+      write_data(@file, TEST_CONTENT)
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal [], lines
+    end
+
     test 'compact data if duplicated line' do
       write_data(@file, <<~EOF)
         valid_path\t0000000000000002\t0000000000000001
@@ -163,7 +185,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#[]' do
     test 'return entry' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
       valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)
       f = pf[valid_target_info]

data/test/plugin/test_bare_output.rb

@@ -95,6 +95,19 @@ class BareOutputTest < Test::Unit::TestCase
     end
   end
 
+  test 'can use metrics plugins and fallback methods' do
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name|
+      assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+    end
+
+    assert_equal 0, @p.num_errors
+    assert_equal 0, @p.emit_count
+    assert_equal 0, @p.emit_size
+    assert_equal 0, @p.emit_records
+  end
+
   test 'can get input event stream to write' do
     @p.configure(config_element('ROOT'))
     @p.start

data/test/plugin/test_buffer.rb

@@ -238,8 +238,14 @@ class BufferTest < Test::Unit::TestCase
       assert_nil @p.queue
       assert_nil @p.dequeued
       assert_nil @p.queued_num
-      assert_equal 0, @p.stage_size
-      assert_equal 0, @p.queue_size
+      assert_nil @p.stage_length_metrics
+      assert_nil @p.stage_size_metrics
+      assert_nil @p.queue_length_metrics
+      assert_nil @p.queue_size_metrics
+      assert_nil @p.available_buffer_space_ratios_metrics
+      assert_nil @p.total_queued_size_metrics
+      assert_nil @p.newest_timekey_metrics
+      assert_nil @p.oldest_timekey_metrics
       assert_equal [], @p.timekeys
     end
 

data/test/plugin/test_filter.rb

@@ -165,6 +165,17 @@ class FilterPluginTest < Test::Unit::TestCase
       end
     end
 
+    test 'can use metrics plugins and fallback methods' do
+      @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+      %w[emit_size_metrics emit_records_metrics].each do |metric_name|
+        assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+      end
+
+      assert_equal 0, @p.emit_size
+      assert_equal 0, @p.emit_records
+    end
+
     test 'are available with multi worker configuration in default' do
       assert @p.multi_workers_ready?
     end

data/test/plugin/test_in_monitor_agent.rb

@@ -37,6 +37,194 @@ class MonitorAgentInputTest < Test::Unit::TestCase
     assert_true d.instance.include_config
   end
 
+  sub_test_case "collect in_monitor_agent plugin statistics" do
+    # Input Test Driver does not register metric callbacks.
+    # We should stub them here.
+    class TestEventMetricRouter < Fluent::Test::Driver::TestEventRouter
+      def initialize(driver)
+        super
+
+        raise ArgumentError, "plugin does not respond metric_callback method" unless @driver.instance.respond_to?(:metric_callback)
+      end
+
+      def emit(tag, time, record)
+        super
+        @driver.instance.metric_callback(OneEventStream.new(time, record))
+      end
+
+      def emit_array(tag, array)
+        super
+        @driver.instance.metric_callback(ArrayEventStream.new(array))
+      end
+
+      def emit_stream(tag, es)
+        super
+        @driver.instance.metric_callback(es)
+      end
+    end
+
+    class MetricInputDriver < Fluent::Test::Driver::Input
+      def configure(conf, syntax: :v1)
+        if conf.is_a?(Fluent::Config::Element)
+          @config = conf
+        else
+          @config = Fluent::Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
+        end
+
+        if @instance.respond_to?(:router=)
+          @event_streams = []
+          @error_events = []
+
+          driver = self
+          mojule =  Module.new do
+            define_method(:event_emitter_router) do |label_name|
+              TestEventMetricRouter.new(driver)
+            end
+          end
+          @instance.singleton_class.prepend mojule
+        end
+
+        @instance.configure(@config)
+        self
+      end
+    end
+
+    setup do
+      # check @type and type in one configuration
+      conf = <<-EOC
+<source>
+  @type test_in_gen
+  @id test_in_gen
+  num 10
+</source>
+<filter>
+  @type test_filter
+  @id test_filter
+</filter>
+<match **>
+  @type relabel
+  @id test_relabel
+  @label @test
+</match>
+<label @test>
+  <match **>
+    @type test_out
+    @id test_out
+  </match>
+</label>
+<label @copy>
+  <match **>
+    @type copy
+    <store>
+      @type test_out
+      @id copy_out_1
+    </store>
+    <store>
+      @type test_out
+      @id copy_out_2
+    </store>
+  </match>
+</label>
+<label @ERROR>
+  <match>
+    @type null
+    @id null
+  </match>
+</label>
+EOC
+      @ra = Fluent::RootAgent.new(log: $log)
+      stub(Fluent::Engine).root_agent { @ra }
+      @ra = configure_ra(@ra, conf)
+    end
+
+    data(:with_config_yes => true,
+         :with_config_no => false)
+    def test_enable_input_metrics(with_config)
+      monitor_agent_conf = <<-CONF
+        tag test.monitor
+        emit_interval 1
+CONF
+      @ra.inputs.first.context_router.emit("test.event", Fluent::Engine.now, {"message":"ok"})
+      d = MetricInputDriver.new(Fluent::Plugin::MonitorAgentInput).configure(monitor_agent_conf)
+      d.run(expect_emits: 1, timeout: 3)
+
+      test_label = @ra.labels['@test']
+      error_label = @ra.labels['@ERROR']
+      input_info = {
+        "output_plugin"  => false,
+        "plugin_category"=> "input",
+        "plugin_id"      => "test_in_gen",
+        "retry_count"    => nil,
+        "type"           => "test_in_gen",
+        "emit_records"   => 0, # This field is not updated due to not to be assigned metric callback.
+        "emit_size"      => 0, # Ditto.
+      }
+      input_info.merge!("config" => {"@id" => "test_in_gen", "@type" => "test_in_gen", "num" => "10"}) if with_config
+      filter_info = {
+        "output_plugin"   => false,
+        "plugin_category" => "filter",
+        "plugin_id"       => "test_filter",
+        "retry_count"     => nil,
+        "type"            => "test_filter",
+        "emit_records"   => Integer,
+        "emit_size"      => Integer,
+      }
+      filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
+      output_info = {
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "test_out",
+        "retry_count"     => 0,
+        "type"            => "test_out",
+        "emit_count"      => Integer,
+        "emit_records"    => Integer,
+        "emit_size"       => Integer,
+        "write_count"     => Integer,
+        "rollback_count"  => Integer,
+        "slow_flush_count" => Integer,
+        "flush_time_count" => Integer,
+      }
+      output_info.merge!("config" => {"@id" => "test_out", "@type" => "test_out"}) if with_config
+      error_label_info = {
+        "buffer_queue_length" => 0,
+        "buffer_timekeys" => [],
+        "buffer_total_queued_size" => 0,
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "null",
+        "retry_count"     => 0,
+        "type"            => "null",
+        "buffer_available_buffer_space_ratios" => Float,
+        "buffer_queue_byte_size" => Integer,
+        "buffer_stage_byte_size" => Integer,
+        "buffer_stage_length" => Integer,
+        "emit_count"      => Integer,
+        "emit_records"    => Integer,
+        "emit_size"       => Integer,
+        "write_count"     => Integer,
+        "rollback_count"  => Integer,
+        "slow_flush_count" => Integer,
+        "flush_time_count" => Integer,
+      }
+      error_label_info.merge!("config" => {"@id"=>"null", "@type" => "null"}) if with_config
+      opts = {with_config: with_config}
+      assert_equal(input_info, d.instance.get_monitor_info(@ra.inputs.first, opts))
+      assert_fuzzy_equal(filter_info, d.instance.get_monitor_info(@ra.filters.first, opts))
+      assert_fuzzy_equal(output_info, d.instance.get_monitor_info(test_label.outputs.first, opts))
+      assert_fuzzy_equal(error_label_info, d.instance.get_monitor_info(error_label.outputs.first, opts))
+      monitor_agent_emit_info = {
+        "emit_records" => Integer,
+        "emit_size" => Integer,
+      }
+      filter_statistics_info = {
+        "emit_records" => Integer,
+        "emit_size" => Integer,
+      }
+      assert_fuzzy_equal(monitor_agent_emit_info, d.instance.statistics["input"])
+      assert_fuzzy_equal(filter_statistics_info, @ra.filters.first.statistics["filter"])
+    end
+  end
+
   sub_test_case "collect plugin information" do
     setup do
       # check @type and type in one configuration
@@ -106,7 +294,9 @@ EOC
         "plugin_category"=> "input",
         "plugin_id"      => "test_in",
         "retry_count"    => nil,
-        "type"           => "test_in"
+        "type"           => "test_in",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       input_info.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       filter_info = {
@@ -114,7 +304,9 @@ EOC
         "plugin_category" => "filter",
         "plugin_id"       => "test_filter",
         "retry_count"     => nil,
-        "type"            => "test_filter"
+        "type"            => "test_filter",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
       output_info = {
@@ -125,6 +317,7 @@ EOC
         "type"            => "test_out",
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -146,6 +339,7 @@ EOC
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -220,6 +414,7 @@ EOC
         "retry_count"     => 0,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -233,6 +428,7 @@ EOC
         "retry_count"     => 0,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -322,9 +518,9 @@ EOC
 ")
       d.instance.start
       expected_test_in_response = "\
-plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:"
+plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
       expected_test_filter_response = "\
-plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:"
+plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
 
       response = get("http://127.0.0.1:#{@port}/api/plugins").body
       test_in = response.split("\n")[0]
@@ -350,7 +546,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_category" => "input",
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
-        "type"            => "test_in"
+        "type"            => "test_in",
+        "emit_records"    => 0,
+        "emit_size"       => 0,
       }
       expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       expected_null_response = {
@@ -368,6 +566,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -412,7 +611,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_category" => "input",
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
-        "type"            => "test_in"
+        "type"            => "test_in",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       expected_null_response = {
@@ -430,6 +631,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -458,7 +660,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
         "type"            => "test_in",
-        "instance_variables" => {"id" => "test_in"}
+        "instance_variables" => {"id" => "test_in"},
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       expected_null_response = {
         "buffer_queue_length" => 0,
@@ -469,13 +673,14 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_id"       => "null",
         "retry_count"     => 0,
         "type"            => "null",
-        "instance_variables" => {"id" => "null", "num_errors" => 0},
+        "instance_variables" => {"id" => "null"},
         "buffer_available_buffer_space_ratios" => Float,
         "buffer_queue_byte_size" => Integer,
         "buffer_stage_byte_size" => Integer,
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -606,6 +811,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
           "buffer_stage_length" => Integer,
           "emit_count" => Integer,
           "emit_records" => Integer,
+          "emit_size" => Integer,
           "write_count" => Integer,
           "rollback_count" => Integer,
           'slow_flush_count' => Integer,