fluentd 1.13.2-x86-mingw32 → 1.14.2-x86-mingw32

data/test/plugin/test_in_monitor_agent.rb

@@ -37,6 +37,194 @@ class MonitorAgentInputTest < Test::Unit::TestCase
     assert_true d.instance.include_config
   end
 
+  sub_test_case "collect in_monitor_agent plugin statistics" do
+    # Input Test Driver does not register metric callbacks.
+    # We should stub them here.
+    class TestEventMetricRouter < Fluent::Test::Driver::TestEventRouter
+      def initialize(driver)
+        super
+
+        raise ArgumentError, "plugin does not respond metric_callback method" unless @driver.instance.respond_to?(:metric_callback)
+      end
+
+      def emit(tag, time, record)
+        super
+        @driver.instance.metric_callback(OneEventStream.new(time, record))
+      end
+
+      def emit_array(tag, array)
+        super
+        @driver.instance.metric_callback(ArrayEventStream.new(array))
+      end
+
+      def emit_stream(tag, es)
+        super
+        @driver.instance.metric_callback(es)
+      end
+    end
+
+    class MetricInputDriver < Fluent::Test::Driver::Input
+      def configure(conf, syntax: :v1)
+        if conf.is_a?(Fluent::Config::Element)
+          @config = conf
+        else
+          @config = Fluent::Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
+        end
+
+        if @instance.respond_to?(:router=)
+          @event_streams = []
+          @error_events = []
+
+          driver = self
+          mojule =  Module.new do
+            define_method(:event_emitter_router) do |label_name|
+              TestEventMetricRouter.new(driver)
+            end
+          end
+          @instance.singleton_class.prepend mojule
+        end
+
+        @instance.configure(@config)
+        self
+      end
+    end
+
+    setup do
+      # check @type and type in one configuration
+      conf = <<-EOC
+<source>
+  @type test_in_gen
+  @id test_in_gen
+  num 10
+</source>
+<filter>
+  @type test_filter
+  @id test_filter
+</filter>
+<match **>
+  @type relabel
+  @id test_relabel
+  @label @test
+</match>
+<label @test>
+  <match **>
+    @type test_out
+    @id test_out
+  </match>
+</label>
+<label @copy>
+  <match **>
+    @type copy
+    <store>
+      @type test_out
+      @id copy_out_1
+    </store>
+    <store>
+      @type test_out
+      @id copy_out_2
+    </store>
+  </match>
+</label>
+<label @ERROR>
+  <match>
+    @type null
+    @id null
+  </match>
+</label>
+EOC
+      @ra = Fluent::RootAgent.new(log: $log)
+      stub(Fluent::Engine).root_agent { @ra }
+      @ra = configure_ra(@ra, conf)
+    end
+
+    data(:with_config_yes => true,
+         :with_config_no => false)
+    def test_enable_input_metrics(with_config)
+      monitor_agent_conf = <<-CONF
+        tag test.monitor
+        emit_interval 1
+CONF
+      @ra.inputs.first.context_router.emit("test.event", Fluent::Engine.now, {"message":"ok"})
+      d = MetricInputDriver.new(Fluent::Plugin::MonitorAgentInput).configure(monitor_agent_conf)
+      d.run(expect_emits: 1, timeout: 3)
+
+      test_label = @ra.labels['@test']
+      error_label = @ra.labels['@ERROR']
+      input_info = {
+        "output_plugin"  => false,
+        "plugin_category"=> "input",
+        "plugin_id"      => "test_in_gen",
+        "retry_count"    => nil,
+        "type"           => "test_in_gen",
+        "emit_records"   => 0, # This field is not updated due to not to be assigned metric callback.
+        "emit_size"      => 0, # Ditto.
+      }
+      input_info.merge!("config" => {"@id" => "test_in_gen", "@type" => "test_in_gen", "num" => "10"}) if with_config
+      filter_info = {
+        "output_plugin"   => false,
+        "plugin_category" => "filter",
+        "plugin_id"       => "test_filter",
+        "retry_count"     => nil,
+        "type"            => "test_filter",
+        "emit_records"   => Integer,
+        "emit_size"      => Integer,
+      }
+      filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
+      output_info = {
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "test_out",
+        "retry_count"     => 0,
+        "type"            => "test_out",
+        "emit_count"      => Integer,
+        "emit_records"    => Integer,
+        "emit_size"       => Integer,
+        "write_count"     => Integer,
+        "rollback_count"  => Integer,
+        "slow_flush_count" => Integer,
+        "flush_time_count" => Integer,
+      }
+      output_info.merge!("config" => {"@id" => "test_out", "@type" => "test_out"}) if with_config
+      error_label_info = {
+        "buffer_queue_length" => 0,
+        "buffer_timekeys" => [],
+        "buffer_total_queued_size" => 0,
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "null",
+        "retry_count"     => 0,
+        "type"            => "null",
+        "buffer_available_buffer_space_ratios" => Float,
+        "buffer_queue_byte_size" => Integer,
+        "buffer_stage_byte_size" => Integer,
+        "buffer_stage_length" => Integer,
+        "emit_count"      => Integer,
+        "emit_records"    => Integer,
+        "emit_size"       => Integer,
+        "write_count"     => Integer,
+        "rollback_count"  => Integer,
+        "slow_flush_count" => Integer,
+        "flush_time_count" => Integer,
+      }
+      error_label_info.merge!("config" => {"@id"=>"null", "@type" => "null"}) if with_config
+      opts = {with_config: with_config}
+      assert_equal(input_info, d.instance.get_monitor_info(@ra.inputs.first, opts))
+      assert_fuzzy_equal(filter_info, d.instance.get_monitor_info(@ra.filters.first, opts))
+      assert_fuzzy_equal(output_info, d.instance.get_monitor_info(test_label.outputs.first, opts))
+      assert_fuzzy_equal(error_label_info, d.instance.get_monitor_info(error_label.outputs.first, opts))
+      monitor_agent_emit_info = {
+        "emit_records" => Integer,
+        "emit_size" => Integer,
+      }
+      filter_statistics_info = {
+        "emit_records" => Integer,
+        "emit_size" => Integer,
+      }
+      assert_fuzzy_equal(monitor_agent_emit_info, d.instance.statistics["input"])
+      assert_fuzzy_equal(filter_statistics_info, @ra.filters.first.statistics["filter"])
+    end
+  end
+
   sub_test_case "collect plugin information" do
     setup do
       # check @type and type in one configuration
@@ -106,7 +294,9 @@ EOC
         "plugin_category"=> "input",
         "plugin_id"      => "test_in",
         "retry_count"    => nil,
-        "type"           => "test_in"
+        "type"           => "test_in",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       input_info.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       filter_info = {
@@ -114,7 +304,9 @@ EOC
         "plugin_category" => "filter",
         "plugin_id"       => "test_filter",
         "retry_count"     => nil,
-        "type"            => "test_filter"
+        "type"            => "test_filter",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
       output_info = {
@@ -125,6 +317,7 @@ EOC
         "type"            => "test_out",
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -146,6 +339,7 @@ EOC
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -220,6 +414,7 @@ EOC
         "retry_count"     => 0,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -233,6 +428,7 @@ EOC
         "retry_count"     => 0,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -322,9 +518,9 @@ EOC
 ")
       d.instance.start
       expected_test_in_response = "\
-plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:"
+plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
       expected_test_filter_response = "\
-plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:"
+plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
 
       response = get("http://127.0.0.1:#{@port}/api/plugins").body
       test_in = response.split("\n")[0]
@@ -350,7 +546,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_category" => "input",
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
-        "type"            => "test_in"
+        "type"            => "test_in",
+        "emit_records"    => 0,
+        "emit_size"       => 0,
       }
       expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       expected_null_response = {
@@ -368,6 +566,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -412,7 +611,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_category" => "input",
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
-        "type"            => "test_in"
+        "type"            => "test_in",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       expected_null_response = {
@@ -430,6 +631,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -458,7 +660,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
         "type"            => "test_in",
-        "instance_variables" => {"id" => "test_in"}
+        "instance_variables" => {"id" => "test_in"},
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       expected_null_response = {
         "buffer_queue_length" => 0,
@@ -469,13 +673,14 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_id"       => "null",
         "retry_count"     => 0,
         "type"            => "null",
-        "instance_variables" => {"id" => "null", "num_errors" => 0},
+        "instance_variables" => {"id" => "null"},
         "buffer_available_buffer_space_ratios" => Float,
         "buffer_queue_byte_size" => Integer,
         "buffer_stage_byte_size" => Integer,
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -606,6 +811,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
           "buffer_stage_length" => Integer,
           "emit_count" => Integer,
           "emit_records" => Integer,
+          "emit_size" => Integer,
           "write_count" => Integer,
           "rollback_count" => Integer,
           'slow_flush_count' => Integer,

data/test/plugin/test_in_syslog.rb

@@ -467,4 +467,39 @@ EOS
     assert_equal tests.size, d.events.size
     compare_unmatched_lines_test_result(d.events, tests, {address: address})
   end
+
+  def test_send_keepalive_packet_is_disabled_by_default
+    d = create_driver(ipv4_config + %[
+      <transport tcp>
+      </transport>
+      protocol tcp
+    ])
+    assert_false d.instance.send_keepalive_packet
+  end
+
+  def test_send_keepalive_packet_can_be_enabled
+    addr = "127.0.0.1"
+    d = create_driver(ipv4_config + %[
+      <transport tcp>
+      </transport>
+      send_keepalive_packet true
+    ])
+    assert_true d.instance.send_keepalive_packet
+    mock.proxy(d.instance).server_create_connection(
+      :in_syslog_tcp_server, @port,
+      bind: addr,
+      resolve_name: nil,
+      send_keepalive_packet: true)
+    d.run do
+      TCPSocket.open(addr, @port)
+    end
+  end
+
+  def test_send_keepalive_packet_can_not_be_enabled_for_udp
+    assert_raise(Fluent::ConfigError) do
+      d = create_driver(ipv4_config + %[
+        send_keepalive_packet true
+      ])
+    end
+  end
 end

data/test/plugin/test_in_tail.rb

@@ -109,6 +109,7 @@ class TailInputTest < Test::Unit::TestCase
                                                 "refresh_interval" => "1s",
                                                 "read_from_head" => "true",
                                                 "format" => "none",
+                                                "rotate_wait" => "1s",
                                                 "follow_inodes" => "true"
                                               })
   SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
@@ -1523,11 +1524,18 @@ class TailInputTest < Test::Unit::TestCase
     plugin.instance_eval do
       @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
       @loop = Coolio::Loop.new
+      opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+      opened_file_metrics.configure(config_element('metrics', '', {}))
+      closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+      closed_file_metrics.configure(config_element('metrics', '', {}))
+      rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+      rotated_file_metrics.configure(config_element('metrics', '', {}))
+      @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
     end
 
     Timecop.freeze(2010, 1, 2, 3, 4, 5) do
       ex_paths.each do |target_info|
-        mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
+        mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
       end
 
       plugin.refresh_watchers
@@ -1541,7 +1549,7 @@ class TailInputTest < Test::Unit::TestCase
       path = "test/plugin/data/2010/01/20100102-030406.log"
       inode = Fluent::FileWrapper.stat(path).ino
       target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
-      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
+      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
       plugin.refresh_watchers
 
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
@@ -1738,7 +1746,7 @@ class TailInputTest < Test::Unit::TestCase
       d.instance_shutdown
     end
 
-    def test_should_keep_and_update_existing_file_pos_entry_for_deleted_file_when_new_file_with_same_name_created
+    def test_should_remove_deleted_file
       config = config_element("", "", {"format" => "none"})
 
       path = "#{TMP_DIR}/tail.txt"
@@ -1749,30 +1757,11 @@ class TailInputTest < Test::Unit::TestCase
       }
 
       d = create_driver(config)
-      d.run(shutdown: false)
-
-      pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
-      pos_file.pos = 0
-
-      path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
-      assert_equal(path, path_pos_ino[1])
-      assert_equal(pos, path_pos_ino[2].to_i(16))
-      assert_equal(ino, path_pos_ino[3].to_i(16))
-
-      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
-        f.puts "test1"
-        f.puts "test2"
-      }
-      Timecop.travel(Time.now + 10) do
-        sleep 5
+      d.run do
+        pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
         pos_file.pos = 0
-        tuple = create_target_info("#{TMP_DIR}/tail.txt")
-        path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
-        assert_equal(tuple.path, path_pos_ino[1])
-        assert_equal(12, path_pos_ino[2].to_i(16))
-        assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
+        assert_equal([], pos_file.readlines)
       end
-      d.instance_shutdown
     end
 
     def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
@@ -1903,13 +1892,22 @@ class TailInputTest < Test::Unit::TestCase
       config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
 
       d = create_driver(config, false)
+      d.instance.instance_eval do
+        opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+        opened_file_metrics.configure(config_element('metrics', '', {}))
+        closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+        closed_file_metrics.configure(config_element('metrics', '', {}))
+        rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+        rotated_file_metrics.configure(config_element('metrics', '', {}))
+        @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
+      end
 
       File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
         f.puts "test1"
         f.puts "test2"
       }
       target_info = create_target_info("#{TMP_DIR}/tail.txt")
-      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
+      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once
       d.run(shutdown: false)
       assert d.instance.instance_variable_get(:@tails)[target_info]
 
@@ -1984,6 +1982,49 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal({"message" => "test4"}, events[3][2])
       d.instance_shutdown
     end
+
+    # issue #3464
+    def test_should_replace_target_info
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1\n"
+      }
+      target_info = create_target_info("#{TMP_DIR}/tail.txt")
+      inodes = []
+
+      config = config_element("ROOT", "", {
+                                "path" => "#{TMP_DIR}/tail.txt*",
+                                "pos_file" => "#{TMP_DIR}/tail.pos",
+                                "tag" => "t1",
+                                "refresh_interval" => "60s",
+                                "read_from_head" => "true",
+                                "format" => "none",
+                                "rotate_wait" => "1s",
+                                "follow_inodes" => "true"
+                              })
+      d = create_driver(config, false)
+      d.run(timeout: 5) do
+        while d.events.size < 1 do
+          sleep 0.1
+        end
+        inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
+          key.ino
+        end
+        assert_equal([target_info.ino], inodes)
+
+        cleanup_file("#{TMP_DIR}/tail.txt")
+        File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test2\n"}
+
+        while d.events.size < 2 do
+          sleep 0.1
+        end
+        inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
+          key.ino
+        end
+        new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
+        assert_not_equal(target_info.ino, new_target_info.ino)
+        assert_equal([new_target_info.ino], inodes)
+      end
+    end
   end
 
   sub_test_case "tail_path" do
@@ -2154,8 +2195,9 @@ class TailInputTest < Test::Unit::TestCase
     assert_nothing_raised do
       d.run(shutdown: false) {}
     end
-    d.instance_shutdown
     assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
+  ensure
+    d.instance_shutdown if d && d.instance
   end
 
   def test_EACCES_error_after_setup_watcher
@@ -2178,10 +2220,10 @@ class TailInputTest < Test::Unit::TestCase
       assert_nothing_raised do
         d.run(shutdown: false) {}
       end
-      d.instance_shutdown
       assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
     end
   ensure
+    d.instance_shutdown if d && d.instance
     if File.exist?("#{TMP_DIR}/noaccess")
       FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
       FileUtils.rm_rf("#{TMP_DIR}/noaccess")
@@ -2201,8 +2243,9 @@ class TailInputTest < Test::Unit::TestCase
     assert_nothing_raised do
       d.run(shutdown: false) {}
     end
-    d.instance_shutdown
     assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
+  ensure
+    d.instance_shutdown if d && d.instance
   end
 
   def test_shutdown_timeout

data/test/plugin/test_input.rb

@@ -85,6 +85,17 @@ class InputTest < Test::Unit::TestCase
     end
   end
 
+  test 'can use metrics plugins and fallback methods' do
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    %w[emit_size_metrics emit_records_metrics].each do |metric_name|
+      assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+    end
+
+    assert_equal 0, @p.emit_size
+    assert_equal 0, @p.emit_records
+  end
+
   test 'are not available with multi workers configuration in default' do
     assert_false @p.multi_workers_ready?
   end