fluentd 1.13.0 → 1.14.0.rc

data/test/plugin/test_in_http.rb

@@ -19,29 +19,32 @@ class HttpInputTest < Test::Unit::TestCase
 
   def setup
     Fluent::Test.setup
+    @port = unused_port
   end
 
   def teardown
     Timecop.return
+    @port = nil
   end
 
-  PORT = unused_port
-  CONFIG = %[
-    port #{PORT}
-    bind "127.0.0.1"
-    body_size_limit 10m
-    keepalive_timeout 5
-    respond_with_empty_img true
-    use_204_response false
-  ]
-
-  def create_driver(conf=CONFIG)
+  def config
+    %[
+      port #{@port}
+      bind "127.0.0.1"
+      body_size_limit 10m
+      keepalive_timeout 5
+      respond_with_empty_img true
+     use_204_response false
+    ]
+  end
+
+  def create_driver(conf=config)
     Fluent::Test::Driver::Input.new(Fluent::Plugin::HttpInput).configure(conf)
   end
 
   def test_configure
     d = create_driver
-    assert_equal PORT, d.instance.port
+    assert_equal @port, d.instance.port
     assert_equal '127.0.0.1', d.instance.bind
     assert_equal 10*1024*1024, d.instance.body_size_limit
     assert_equal 5, d.instance.keepalive_timeout
@@ -121,7 +124,7 @@ class HttpInputTest < Test::Unit::TestCase
        'msgpack' => ['msgpack', :to_msgpack])
   def test_default_with_time_format(data)
     param, method_name = data
-    d = create_driver(CONFIG + %[
+    d = create_driver(config + %[
       <parse>
         keep_time_key
         time_format %iso8601
@@ -198,7 +201,7 @@ class HttpInputTest < Test::Unit::TestCase
        'msgpack' => ['msgpack', :to_msgpack])
   def test_default_multi_with_time_format(data)
     param, method_name = data
-    d = create_driver(CONFIG + %[
+    d = create_driver(config + %[
       <parse>
         keep_time_key
         time_format %iso8601
@@ -222,7 +225,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_multi_json_with_nonexistent_time_key
-    d = create_driver(CONFIG + %[
+    d = create_driver(config + %[
       <parse>
         time_key missing
       </parse>
@@ -245,7 +248,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_json_with_add_remote_addr
-    d = create_driver(CONFIG + "add_remote_addr true")
+    d = create_driver(config + "add_remote_addr true")
     time = event_time("2011-01-02 13:14:15 UTC")
     time_i = time.to_i
 
@@ -267,7 +270,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_exact_match_for_expect
-    d = create_driver(CONFIG)
+    d = create_driver(config)
     records = [{ "a" => 1}, { "a" => 2 }]
     tag = "tag1"
     res_codes = []
@@ -280,7 +283,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_exact_match_for_expect_with_other_header
-    d = create_driver(CONFIG)
+    d = create_driver(config)
 
     records = [{ "a" => 1}, { "a" => 2 }]
     tag = "tag1"
@@ -299,7 +302,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_multi_json_with_add_remote_addr
-    d = create_driver(CONFIG + "add_remote_addr true")
+    d = create_driver(config + "add_remote_addr true")
     time = event_time("2011-01-02 13:14:15 UTC")
     time_i = time.to_i
 
@@ -324,7 +327,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_json_with_add_remote_addr_given_x_forwarded_for
-    d = create_driver(CONFIG + "add_remote_addr true")
+    d = create_driver(config + "add_remote_addr true")
     time = event_time("2011-01-02 13:14:15 UTC")
     time_i = time.to_i
 
@@ -352,7 +355,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_multi_json_with_add_remote_addr_given_x_forwarded_for
-    d = create_driver(CONFIG + "add_remote_addr true")
+    d = create_driver(config + "add_remote_addr true")
 
     tag = "tag1"
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -375,7 +378,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_add_remote_addr_given_multi_x_forwarded_for
-    d = create_driver(CONFIG + "add_remote_addr true")
+    d = create_driver(config + "add_remote_addr true")
 
     tag = "tag1"
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -406,7 +409,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_multi_json_with_add_http_headers
-    d = create_driver(CONFIG + "add_http_headers true")
+    d = create_driver(config + "add_http_headers true")
     time = event_time("2011-01-02 13:14:15 UTC")
     time_i = time.to_i
     records = [{"a"=>1},{"a"=>2}]
@@ -432,7 +435,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_json_with_add_http_headers
-    d = create_driver(CONFIG + "add_http_headers true")
+    d = create_driver(config + "add_http_headers true")
     time = event_time("2011-01-02 13:14:15 UTC")
     time_i = time.to_i
     events = [
@@ -462,7 +465,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_multi_json_with_custom_parser
-    d = create_driver(CONFIG + %[
+    d = create_driver(config + %[
         <parse>
           @type json
           keep_time_key true
@@ -584,7 +587,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_with_regexp
-    d = create_driver(CONFIG + %[
+    d = create_driver(config + %[
       format /^(?<field_1>\\d+):(?<field_2>\\w+)$/
       types field_1:integer
     ])
@@ -615,7 +618,7 @@ class HttpInputTest < Test::Unit::TestCase
   def test_with_csv
     require 'csv'
 
-    d = create_driver(CONFIG + %[
+    d = create_driver(config + %[
       format csv
       keys foo,bar
     ])
@@ -641,7 +644,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_response_with_empty_img
-    d = create_driver(CONFIG)
+    d = create_driver(config)
     assert_equal true, d.instance.respond_with_empty_img
 
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -669,7 +672,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_response_without_empty_img
-    d = create_driver(CONFIG + "respond_with_empty_img false")
+    d = create_driver(config + "respond_with_empty_img false")
     assert_equal false, d.instance.respond_with_empty_img
 
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -695,7 +698,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_response_use_204_response
-    d = create_driver(CONFIG + %[
+    d = create_driver(config + %[
           respond_with_empty_img false
           use_204_response true
         ])
@@ -724,7 +727,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_cors_allowed
-    d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
+    d = create_driver(config + "cors_allow_origins [\"http://foo.com\"]")
     assert_equal ["http://foo.com"], d.instance.cors_allow_origins
 
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -751,7 +754,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_cors_allowed_wildcard
-    d = create_driver(CONFIG + 'cors_allow_origins ["*"]')
+    d = create_driver(config + 'cors_allow_origins ["*"]')
 
     time = event_time("2011-01-02 13:14:15 UTC")
     events = [
@@ -771,7 +774,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_get_request
-    d = create_driver(CONFIG)
+    d = create_driver(config)
 
     d.run do
       res = get("/cors.test", {}, {})
@@ -780,7 +783,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_cors_preflight
-    d = create_driver(CONFIG + 'cors_allow_origins ["*"]')
+    d = create_driver(config + 'cors_allow_origins ["*"]')
 
     d.run do
       header = {
@@ -797,7 +800,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_cors_allowed_wildcard_for_subdomain
-    d = create_driver(CONFIG + 'cors_allow_origins ["http://*.foo.com"]')
+    d = create_driver(config + 'cors_allow_origins ["http://*.foo.com"]')
 
     time = event_time("2011-01-02 13:14:15 UTC")
     events = [
@@ -817,7 +820,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_cors_allowed_exclude_empty_string
-    d = create_driver(CONFIG + 'cors_allow_origins ["", "http://*.foo.com"]')
+    d = create_driver(config + 'cors_allow_origins ["", "http://*.foo.com"]')
 
     time = event_time("2011-01-02 13:14:15 UTC")
     events = [
@@ -837,7 +840,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_cors_allowed_wildcard_preflight_for_subdomain
-    d = create_driver(CONFIG + 'cors_allow_origins ["http://*.foo.com"]')
+    d = create_driver(config + 'cors_allow_origins ["http://*.foo.com"]')
 
     d.run do
       header = {
@@ -900,7 +903,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_cors_disallowed
-    d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
+    d = create_driver(config + "cors_allow_origins [\"http://foo.com\"]")
     assert_equal ["http://foo.com"], d.instance.cors_allow_origins
 
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -918,7 +921,7 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def test_add_query_params 
-    d = create_driver(CONFIG + "add_query_params true")
+    d = create_driver(config + "add_query_params true")
     assert_equal true, d.instance.add_query_params
 
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -978,7 +981,7 @@ class HttpInputTest < Test::Unit::TestCase
       $test_in_http_content_types_flag = true
       d.run do
         # Send two requests the second one has no Content-Type in Keep-Alive
-        Net::HTTP.start("127.0.0.1", PORT) do |http|
+        Net::HTTP.start("127.0.0.1", @port) do |http|
           req = Net::HTTP::Post.new("/foodb/bartbl", {"connection" => "keepalive", "Content-Type" => "application/json"})
           http.request(req)
           req = Net::HTTP::Get.new("/foodb/bartbl", {"connection" => "keepalive"})
@@ -995,19 +998,19 @@ class HttpInputTest < Test::Unit::TestCase
   end
 
   def get(path, params, header = {})
-    http = Net::HTTP.new("127.0.0.1", PORT)
+    http = Net::HTTP.new("127.0.0.1", @port)
     req = Net::HTTP::Get.new(path, header)
     http.request(req)
   end
 
   def options(path, params, header = {})
-    http = Net::HTTP.new("127.0.0.1", PORT)
+    http = Net::HTTP.new("127.0.0.1", @port)
     req = Net::HTTP::Options.new(path, header)
     http.request(req)
   end
 
   def post(path, params, header = {}, &block)
-    http = Net::HTTP.new("127.0.0.1", PORT)
+    http = Net::HTTP.new("127.0.0.1", @port)
     req = Net::HTTP::Post.new(path, header)
     block.call(http, req) if block
     if params.is_a?(String)

data/test/plugin/test_in_monitor_agent.rb

@@ -37,6 +37,194 @@ class MonitorAgentInputTest < Test::Unit::TestCase
     assert_true d.instance.include_config
   end
 
+  sub_test_case "collect in_monitor_agent plugin statistics" do
+    # Input Test Driver does not register metric callbacks.
+    # We should stub them here.
+    class TestEventMetricRouter < Fluent::Test::Driver::TestEventRouter
+      def initialize(driver)
+        super
+
+        raise ArgumentError, "plugin does not respond metric_callback method" unless @driver.instance.respond_to?(:metric_callback)
+      end
+
+      def emit(tag, time, record)
+        super
+        @driver.instance.metric_callback(OneEventStream.new(time, record))
+      end
+
+      def emit_array(tag, array)
+        super
+        @driver.instance.metric_callback(ArrayEventStream.new(array))
+      end
+
+      def emit_stream(tag, es)
+        super
+        @driver.instance.metric_callback(es)
+      end
+    end
+
+    class MetricInputDriver < Fluent::Test::Driver::Input
+      def configure(conf, syntax: :v1)
+        if conf.is_a?(Fluent::Config::Element)
+          @config = conf
+        else
+          @config = Fluent::Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
+        end
+
+        if @instance.respond_to?(:router=)
+          @event_streams = []
+          @error_events = []
+
+          driver = self
+          mojule =  Module.new do
+            define_method(:event_emitter_router) do |label_name|
+              TestEventMetricRouter.new(driver)
+            end
+          end
+          @instance.singleton_class.prepend mojule
+        end
+
+        @instance.configure(@config)
+        self
+      end
+    end
+
+    setup do
+      # check @type and type in one configuration
+      conf = <<-EOC
+<source>
+  @type test_in_gen
+  @id test_in_gen
+  num 10
+</source>
+<filter>
+  @type test_filter
+  @id test_filter
+</filter>
+<match **>
+  @type relabel
+  @id test_relabel
+  @label @test
+</match>
+<label @test>
+  <match **>
+    @type test_out
+    @id test_out
+  </match>
+</label>
+<label @copy>
+  <match **>
+    @type copy
+    <store>
+      @type test_out
+      @id copy_out_1
+    </store>
+    <store>
+      @type test_out
+      @id copy_out_2
+    </store>
+  </match>
+</label>
+<label @ERROR>
+  <match>
+    @type null
+    @id null
+  </match>
+</label>
+EOC
+      @ra = Fluent::RootAgent.new(log: $log)
+      stub(Fluent::Engine).root_agent { @ra }
+      @ra = configure_ra(@ra, conf)
+    end
+
+    data(:with_config_yes => true,
+         :with_config_no => false)
+    def test_enable_input_metrics(with_config)
+      monitor_agent_conf = <<-CONF
+        tag test.monitor
+        emit_interval 1
+CONF
+      @ra.inputs.first.context_router.emit("test.event", Fluent::Engine.now, {"message":"ok"})
+      d = MetricInputDriver.new(Fluent::Plugin::MonitorAgentInput).configure(monitor_agent_conf)
+      d.run(expect_emits: 1, timeout: 3)
+
+      test_label = @ra.labels['@test']
+      error_label = @ra.labels['@ERROR']
+      input_info = {
+        "output_plugin"  => false,
+        "plugin_category"=> "input",
+        "plugin_id"      => "test_in_gen",
+        "retry_count"    => nil,
+        "type"           => "test_in_gen",
+        "emit_records"   => 0, # This field is not updated due to not to be assigned metric callback.
+        "emit_size"      => 0, # Ditto.
+      }
+      input_info.merge!("config" => {"@id" => "test_in_gen", "@type" => "test_in_gen", "num" => "10"}) if with_config
+      filter_info = {
+        "output_plugin"   => false,
+        "plugin_category" => "filter",
+        "plugin_id"       => "test_filter",
+        "retry_count"     => nil,
+        "type"            => "test_filter",
+        "emit_records"   => Integer,
+        "emit_size"      => Integer,
+      }
+      filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
+      output_info = {
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "test_out",
+        "retry_count"     => 0,
+        "type"            => "test_out",
+        "emit_count"      => Integer,
+        "emit_records"    => Integer,
+        "emit_size"       => Integer,
+        "write_count"     => Integer,
+        "rollback_count"  => Integer,
+        "slow_flush_count" => Integer,
+        "flush_time_count" => Integer,
+      }
+      output_info.merge!("config" => {"@id" => "test_out", "@type" => "test_out"}) if with_config
+      error_label_info = {
+        "buffer_queue_length" => 0,
+        "buffer_timekeys" => [],
+        "buffer_total_queued_size" => 0,
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "null",
+        "retry_count"     => 0,
+        "type"            => "null",
+        "buffer_available_buffer_space_ratios" => Float,
+        "buffer_queue_byte_size" => Integer,
+        "buffer_stage_byte_size" => Integer,
+        "buffer_stage_length" => Integer,
+        "emit_count"      => Integer,
+        "emit_records"    => Integer,
+        "emit_size"       => Integer,
+        "write_count"     => Integer,
+        "rollback_count"  => Integer,
+        "slow_flush_count" => Integer,
+        "flush_time_count" => Integer,
+      }
+      error_label_info.merge!("config" => {"@id"=>"null", "@type" => "null"}) if with_config
+      opts = {with_config: with_config}
+      assert_equal(input_info, d.instance.get_monitor_info(@ra.inputs.first, opts))
+      assert_fuzzy_equal(filter_info, d.instance.get_monitor_info(@ra.filters.first, opts))
+      assert_fuzzy_equal(output_info, d.instance.get_monitor_info(test_label.outputs.first, opts))
+      assert_fuzzy_equal(error_label_info, d.instance.get_monitor_info(error_label.outputs.first, opts))
+      monitor_agent_emit_info = {
+        "emit_records" => Integer,
+        "emit_size" => Integer,
+      }
+      filter_statistics_info = {
+        "emit_records" => Integer,
+        "emit_size" => Integer,
+      }
+      assert_fuzzy_equal(monitor_agent_emit_info, d.instance.statistics["input"])
+      assert_fuzzy_equal(filter_statistics_info, @ra.filters.first.statistics["filter"])
+    end
+  end
+
   sub_test_case "collect plugin information" do
     setup do
       # check @type and type in one configuration
@@ -106,7 +294,9 @@ EOC
         "plugin_category"=> "input",
         "plugin_id"      => "test_in",
         "retry_count"    => nil,
-        "type"           => "test_in"
+        "type"           => "test_in",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       input_info.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       filter_info = {
@@ -114,7 +304,9 @@ EOC
         "plugin_category" => "filter",
         "plugin_id"       => "test_filter",
         "retry_count"     => nil,
-        "type"            => "test_filter"
+        "type"            => "test_filter",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
       output_info = {
@@ -125,6 +317,7 @@ EOC
         "type"            => "test_out",
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -146,6 +339,7 @@ EOC
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -220,6 +414,7 @@ EOC
         "retry_count"     => 0,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -233,6 +428,7 @@ EOC
         "retry_count"     => 0,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -322,9 +518,9 @@ EOC
 ")
       d.instance.start
       expected_test_in_response = "\
-plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:"
+plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
       expected_test_filter_response = "\
-plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:"
+plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
 
       response = get("http://127.0.0.1:#{@port}/api/plugins").body
       test_in = response.split("\n")[0]
@@ -350,7 +546,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_category" => "input",
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
-        "type"            => "test_in"
+        "type"            => "test_in",
+        "emit_records"    => 0,
+        "emit_size"       => 0,
       }
       expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       expected_null_response = {
@@ -368,6 +566,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -412,7 +611,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_category" => "input",
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
-        "type"            => "test_in"
+        "type"            => "test_in",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       expected_null_response = {
@@ -430,6 +631,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -458,7 +660,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
         "type"            => "test_in",
-        "instance_variables" => {"id" => "test_in"}
+        "instance_variables" => {"id" => "test_in"},
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       expected_null_response = {
         "buffer_queue_length" => 0,
@@ -469,13 +673,14 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_id"       => "null",
         "retry_count"     => 0,
         "type"            => "null",
-        "instance_variables" => {"id" => "null", "num_errors" => 0},
+        "instance_variables" => {"id" => "null"},
         "buffer_available_buffer_space_ratios" => Float,
         "buffer_queue_byte_size" => Integer,
         "buffer_stage_byte_size" => Integer,
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -606,6 +811,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
           "buffer_stage_length" => Integer,
           "emit_count" => Integer,
           "emit_records" => Integer,
+          "emit_size" => Integer,
           "write_count" => Integer,
           "rollback_count" => Integer,
           'slow_flush_count' => Integer,