fluentd 1.13.1 → 1.14.0

data/test/config/test_types.rb

@@ -180,6 +180,13 @@ class TestConfigTypes < ::Test::Unit::TestCase
       assert_equal Encoding::UTF_8, actual.encoding
     end
 
+    data('starts_with_semicolon' => [:conor, ':conor'],
+         'simple_string' => [:conor, 'conor'],
+         'empty_string' => [nil, ''])
+    test 'symbol' do |(expected, val)|
+      assert_equal Config::SYMBOL_TYPE.call(val, {}), expected
+    end
+
     data("val" => [:val, 'val'],
          "v" => [:v, 'v'],
          "value" => [:value, 'value'])

data/test/plugin/in_tail/test_position_file.rb

@@ -22,6 +22,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
     invalidpath100000000000000000000000000000000
     unwatched\t#{UNWATCHED_STR}\t0000000000000000
   EOF
+  TEST_CONTENT_PATHS = {
+    "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
+    "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
+  }
 
   def write_data(f, content)
     f.write(content)
@@ -36,7 +40,11 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
   test '.load' do
     write_data(@file, TEST_CONTENT)
-    Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+    paths = {
+      "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
+      "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 2),
+    }
+    Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
     @file.seek(0)
     lines = @file.readlines
@@ -48,7 +56,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#try_compact' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -62,7 +70,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).try_compact
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).try_compact
 
       @file.seek(0)
       lines = @file.readlines
@@ -71,7 +79,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'does not change when the file is changed' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log})
 
       mock.proxy(pf).fetch_compacted_entries do |r|
         @file.write("unwatched\t#{UNWATCHED_STR}\t0000000000000000\n")
@@ -86,7 +94,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
     end
 
     test 'update seek position of remained position entry' do
-      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log})
+      pf = Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log})
       target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('path1', -1)
       target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('path2', -1)
       target_info3 = Fluent::Plugin::TailInput::TargetInfo.new('path3', -1)
@@ -115,12 +123,35 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal "path3\t#{UNWATCHED_STR}\t0000000000000000\n", lines[1]
       assert_equal 2, lines.size
     end
+
+    test 'should ignore initial existing files on follow_inode' do
+      write_data(@file, TEST_CONTENT)
+      pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_PATHS, **{logger: $log})
+      @file.seek(0)
+      assert_equal([], @file.readlines)
+
+      @file.seek(0)
+      write_data(@file, TEST_CONTENT)
+      pos_file.try_compact
+
+      @file.seek(0)
+      assert_equal([
+                     "valid_path\t0000000000000002\t0000000000000001\n",
+                     "inode23bit\t0000000000000000\t0000000000000000\n",
+                   ],
+                   @file.readlines)
+    end
   end
 
   sub_test_case '#load' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+      invalid_path = "invalidpath100000000000000000000000000000000"
+      paths = TEST_CONTENT_PATHS.merge({
+        invalid_path => Fluent::Plugin::TailInput::TargetInfo.new(invalid_path, 0),
+        "unwatched" => Fluent::Plugin::TailInput::TargetInfo.new("unwatched", 0),
+      })
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
       @file.seek(0)
       lines = @file.readlines
@@ -129,12 +160,21 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
     end
 
+    test 'compact deleted paths' do
+      write_data(@file, TEST_CONTENT)
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal [], lines
+    end
+
     test 'compact data if duplicated line' do
       write_data(@file, <<~EOF)
         valid_path\t0000000000000002\t0000000000000001
         valid_path\t0000000000000003\t0000000000000004
       EOF
-      Fluent::Plugin::TailInput::PositionFile.new(@file, false, {}, **{logger: $log}).load
+      Fluent::Plugin::TailInput::PositionFile.new(@file, false, **{logger: $log}).load
 
       @file.seek(0)
       lines = @file.readlines
@@ -145,7 +185,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#[]' do
     test 'return entry' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
       valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)
       f = pf[valid_target_info]

data/test/plugin/test_bare_output.rb

@@ -95,6 +95,19 @@ class BareOutputTest < Test::Unit::TestCase
     end
   end
 
+  test 'can use metrics plugins and fallback methods' do
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name|
+      assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+    end
+
+    assert_equal 0, @p.num_errors
+    assert_equal 0, @p.emit_count
+    assert_equal 0, @p.emit_size
+    assert_equal 0, @p.emit_records
+  end
+
   test 'can get input event stream to write' do
     @p.configure(config_element('ROOT'))
     @p.start

data/test/plugin/test_buffer.rb

@@ -238,8 +238,14 @@ class BufferTest < Test::Unit::TestCase
       assert_nil @p.queue
       assert_nil @p.dequeued
       assert_nil @p.queued_num
-      assert_equal 0, @p.stage_size
-      assert_equal 0, @p.queue_size
+      assert_nil @p.stage_length_metrics
+      assert_nil @p.stage_size_metrics
+      assert_nil @p.queue_length_metrics
+      assert_nil @p.queue_size_metrics
+      assert_nil @p.available_buffer_space_ratios_metrics
+      assert_nil @p.total_queued_size_metrics
+      assert_nil @p.newest_timekey_metrics
+      assert_nil @p.oldest_timekey_metrics
       assert_equal [], @p.timekeys
     end
 

data/test/plugin/test_file_wrapper.rb

@@ -96,6 +96,17 @@ class FileWrapperTest < Test::Unit::TestCase
       end
     end
 
+    test 'Errno::ENOENT raised on DeletePending' do
+      path = "#{TMP_DIR}/deletepending.txt"
+      file = Fluent::WindowsFile.new(path, mode='w')
+      File.delete(path)
+      assert_raise(Errno::ENOENT) do
+        file.stat
+      ensure
+        file.close if file
+      end
+    end
+
     test 'ERROR_SHARING_VIOLATION raised' do
       begin
         path = "#{TMP_DIR}/test_windows_file.txt"

data/test/plugin/test_filter.rb

@@ -165,6 +165,17 @@ class FilterPluginTest < Test::Unit::TestCase
       end
     end
 
+    test 'can use metrics plugins and fallback methods' do
+      @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+      %w[emit_size_metrics emit_records_metrics].each do |metric_name|
+        assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+      end
+
+      assert_equal 0, @p.emit_size
+      assert_equal 0, @p.emit_records
+    end
+
     test 'are available with multi worker configuration in default' do
       assert @p.multi_workers_ready?
     end

data/test/plugin/test_in_http.rb

@@ -856,6 +856,46 @@ class HttpInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_cors_allow_credentials
+    d = create_driver(config + %[
+          cors_allow_origins ["http://foo.com"]
+          cors_allow_credentials
+        ])
+    assert_equal true, d.instance.cors_allow_credentials
+
+    time = event_time("2011-01-02 13:14:15 UTC")
+    event = ["tag1", time, {"a"=>1}]
+    res_code = nil
+    res_header = nil
+
+    d.run do
+      res = post("/#{event[0]}", {"json"=>event[2].to_json, "time"=>time.to_i.to_s}, {"Origin"=>"http://foo.com"})
+      res_code = res.code
+      res_header = res["Access-Control-Allow-Credentials"]
+    end
+    assert_equal(
+      {
+        response_code: "200",
+        allow_credentials_header: "true",
+        events: [event]
+      },
+      {
+        response_code: res_code,
+        allow_credentials_header: res_header,
+        events: d.events
+      }
+    )
+  end
+
+  def test_cors_allow_credentials_for_wildcard_origins
+    assert_raise(Fluent::ConfigError) do
+      create_driver(config + %[
+        cors_allow_origins ["*"]
+        cors_allow_credentials
+      ])
+    end
+  end
+
   def test_content_encoding_gzip
     d = create_driver
 

data/test/plugin/test_in_monitor_agent.rb

@@ -37,6 +37,194 @@ class MonitorAgentInputTest < Test::Unit::TestCase
     assert_true d.instance.include_config
   end
 
+  sub_test_case "collect in_monitor_agent plugin statistics" do
+    # Input Test Driver does not register metric callbacks.
+    # We should stub them here.
+    class TestEventMetricRouter < Fluent::Test::Driver::TestEventRouter
+      def initialize(driver)
+        super
+
+        raise ArgumentError, "plugin does not respond metric_callback method" unless @driver.instance.respond_to?(:metric_callback)
+      end
+
+      def emit(tag, time, record)
+        super
+        @driver.instance.metric_callback(OneEventStream.new(time, record))
+      end
+
+      def emit_array(tag, array)
+        super
+        @driver.instance.metric_callback(ArrayEventStream.new(array))
+      end
+
+      def emit_stream(tag, es)
+        super
+        @driver.instance.metric_callback(es)
+      end
+    end
+
+    class MetricInputDriver < Fluent::Test::Driver::Input
+      def configure(conf, syntax: :v1)
+        if conf.is_a?(Fluent::Config::Element)
+          @config = conf
+        else
+          @config = Fluent::Config.parse(conf, "(test)", "(test_dir)", syntax: syntax)
+        end
+
+        if @instance.respond_to?(:router=)
+          @event_streams = []
+          @error_events = []
+
+          driver = self
+          mojule =  Module.new do
+            define_method(:event_emitter_router) do |label_name|
+              TestEventMetricRouter.new(driver)
+            end
+          end
+          @instance.singleton_class.prepend mojule
+        end
+
+        @instance.configure(@config)
+        self
+      end
+    end
+
+    setup do
+      # check @type and type in one configuration
+      conf = <<-EOC
+<source>
+  @type test_in_gen
+  @id test_in_gen
+  num 10
+</source>
+<filter>
+  @type test_filter
+  @id test_filter
+</filter>
+<match **>
+  @type relabel
+  @id test_relabel
+  @label @test
+</match>
+<label @test>
+  <match **>
+    @type test_out
+    @id test_out
+  </match>
+</label>
+<label @copy>
+  <match **>
+    @type copy
+    <store>
+      @type test_out
+      @id copy_out_1
+    </store>
+    <store>
+      @type test_out
+      @id copy_out_2
+    </store>
+  </match>
+</label>
+<label @ERROR>
+  <match>
+    @type null
+    @id null
+  </match>
+</label>
+EOC
+      @ra = Fluent::RootAgent.new(log: $log)
+      stub(Fluent::Engine).root_agent { @ra }
+      @ra = configure_ra(@ra, conf)
+    end
+
+    data(:with_config_yes => true,
+         :with_config_no => false)
+    def test_enable_input_metrics(with_config)
+      monitor_agent_conf = <<-CONF
+        tag test.monitor
+        emit_interval 1
+CONF
+      @ra.inputs.first.context_router.emit("test.event", Fluent::Engine.now, {"message":"ok"})
+      d = MetricInputDriver.new(Fluent::Plugin::MonitorAgentInput).configure(monitor_agent_conf)
+      d.run(expect_emits: 1, timeout: 3)
+
+      test_label = @ra.labels['@test']
+      error_label = @ra.labels['@ERROR']
+      input_info = {
+        "output_plugin"  => false,
+        "plugin_category"=> "input",
+        "plugin_id"      => "test_in_gen",
+        "retry_count"    => nil,
+        "type"           => "test_in_gen",
+        "emit_records"   => 0, # This field is not updated due to not to be assigned metric callback.
+        "emit_size"      => 0, # Ditto.
+      }
+      input_info.merge!("config" => {"@id" => "test_in_gen", "@type" => "test_in_gen", "num" => "10"}) if with_config
+      filter_info = {
+        "output_plugin"   => false,
+        "plugin_category" => "filter",
+        "plugin_id"       => "test_filter",
+        "retry_count"     => nil,
+        "type"            => "test_filter",
+        "emit_records"   => Integer,
+        "emit_size"      => Integer,
+      }
+      filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
+      output_info = {
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "test_out",
+        "retry_count"     => 0,
+        "type"            => "test_out",
+        "emit_count"      => Integer,
+        "emit_records"    => Integer,
+        "emit_size"       => Integer,
+        "write_count"     => Integer,
+        "rollback_count"  => Integer,
+        "slow_flush_count" => Integer,
+        "flush_time_count" => Integer,
+      }
+      output_info.merge!("config" => {"@id" => "test_out", "@type" => "test_out"}) if with_config
+      error_label_info = {
+        "buffer_queue_length" => 0,
+        "buffer_timekeys" => [],
+        "buffer_total_queued_size" => 0,
+        "output_plugin"   => true,
+        "plugin_category" => "output",
+        "plugin_id"       => "null",
+        "retry_count"     => 0,
+        "type"            => "null",
+        "buffer_available_buffer_space_ratios" => Float,
+        "buffer_queue_byte_size" => Integer,
+        "buffer_stage_byte_size" => Integer,
+        "buffer_stage_length" => Integer,
+        "emit_count"      => Integer,
+        "emit_records"    => Integer,
+        "emit_size"       => Integer,
+        "write_count"     => Integer,
+        "rollback_count"  => Integer,
+        "slow_flush_count" => Integer,
+        "flush_time_count" => Integer,
+      }
+      error_label_info.merge!("config" => {"@id"=>"null", "@type" => "null"}) if with_config
+      opts = {with_config: with_config}
+      assert_equal(input_info, d.instance.get_monitor_info(@ra.inputs.first, opts))
+      assert_fuzzy_equal(filter_info, d.instance.get_monitor_info(@ra.filters.first, opts))
+      assert_fuzzy_equal(output_info, d.instance.get_monitor_info(test_label.outputs.first, opts))
+      assert_fuzzy_equal(error_label_info, d.instance.get_monitor_info(error_label.outputs.first, opts))
+      monitor_agent_emit_info = {
+        "emit_records" => Integer,
+        "emit_size" => Integer,
+      }
+      filter_statistics_info = {
+        "emit_records" => Integer,
+        "emit_size" => Integer,
+      }
+      assert_fuzzy_equal(monitor_agent_emit_info, d.instance.statistics["input"])
+      assert_fuzzy_equal(filter_statistics_info, @ra.filters.first.statistics["filter"])
+    end
+  end
+
   sub_test_case "collect plugin information" do
     setup do
       # check @type and type in one configuration
@@ -106,7 +294,9 @@ EOC
         "plugin_category"=> "input",
         "plugin_id"      => "test_in",
         "retry_count"    => nil,
-        "type"           => "test_in"
+        "type"           => "test_in",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       input_info.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       filter_info = {
@@ -114,7 +304,9 @@ EOC
         "plugin_category" => "filter",
         "plugin_id"       => "test_filter",
         "retry_count"     => nil,
-        "type"            => "test_filter"
+        "type"            => "test_filter",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       filter_info.merge!("config" => {"@id" => "test_filter", "@type" => "test_filter"}) if with_config
       output_info = {
@@ -125,6 +317,7 @@ EOC
         "type"            => "test_out",
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -146,6 +339,7 @@ EOC
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -220,6 +414,7 @@ EOC
         "retry_count"     => 0,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -233,6 +428,7 @@ EOC
         "retry_count"     => 0,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -322,9 +518,9 @@ EOC
 ")
       d.instance.start
       expected_test_in_response = "\
-plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:"
+plugin_id:test_in\tplugin_category:input\ttype:test_in\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
       expected_test_filter_response = "\
-plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:"
+plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:false\tretry_count:\temit_records:0\temit_size:0"
 
       response = get("http://127.0.0.1:#{@port}/api/plugins").body
       test_in = response.split("\n")[0]
@@ -350,7 +546,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_category" => "input",
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
-        "type"            => "test_in"
+        "type"            => "test_in",
+        "emit_records"    => 0,
+        "emit_size"       => 0,
       }
       expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       expected_null_response = {
@@ -368,6 +566,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -412,7 +611,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_category" => "input",
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
-        "type"            => "test_in"
+        "type"            => "test_in",
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       expected_test_in_response.merge!("config" => {"@id" => "test_in", "@type" => "test_in"}) if with_config
       expected_null_response = {
@@ -430,6 +631,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -458,7 +660,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_id"       => "test_in",
         "retry_count"     => nil,
         "type"            => "test_in",
-        "instance_variables" => {"id" => "test_in"}
+        "instance_variables" => {"id" => "test_in"},
+        "emit_records"   => 0,
+        "emit_size"      => 0,
       }
       expected_null_response = {
         "buffer_queue_length" => 0,
@@ -469,13 +673,14 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "plugin_id"       => "null",
         "retry_count"     => 0,
         "type"            => "null",
-        "instance_variables" => {"id" => "null", "num_errors" => 0},
+        "instance_variables" => {"id" => "null"},
         "buffer_available_buffer_space_ratios" => Float,
         "buffer_queue_byte_size" => Integer,
         "buffer_stage_byte_size" => Integer,
         "buffer_stage_length" => Integer,
         "emit_count"      => Integer,
         "emit_records"    => Integer,
+        "emit_size"       => Integer,
         "write_count"     => Integer,
         "rollback_count"  => Integer,
         "slow_flush_count" => Integer,
@@ -606,6 +811,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
           "buffer_stage_length" => Integer,
           "emit_count" => Integer,
           "emit_records" => Integer,
+          "emit_size" => Integer,
           "write_count" => Integer,
           "rollback_count" => Integer,
           'slow_flush_count' => Integer,