fluentd 1.13.3 → 1.16.5

data/test/plugin/test_in_tcp.rb

@@ -156,6 +156,19 @@ class TcpInputTest < Test::Unit::TestCase
     assert_equal hostname, event[2]['host']
   end
 
+  test "send_keepalive_packet_can_be_enabled" do
+    d = create_driver(base_config + %!
+      format none
+      send_keepalive_packet true
+    !)
+    assert_true d.instance.send_keepalive_packet
+
+    d = create_driver(base_config + %!
+      format none
+    !)
+    assert_false d.instance.send_keepalive_packet
+  end
+
   test 'source_address_key' do
     d = create_driver(base_config + %!
       format none
@@ -205,13 +218,13 @@ class TcpInputTest < Test::Unit::TestCase
           </client>
         </security>
       !)
-      d.run(shutdown: false, expect_records: 1, timeout: 2) do
+      d.run(expect_records: 1, timeout: 2) do
         create_tcp_socket('127.0.0.1', @port) do |sock|
           sock.send("hello\n", 0)
         end
       end
 
-      assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
+      assert_equal 1, d.logs.count { |l| l =~ /anonymous client/ }
       assert_equal 0, d.events.size
     end
   end
@@ -240,4 +253,76 @@ class TcpInputTest < Test::Unit::TestCase
       assert_equal 'hello', event[2]['msg']
     end
   end
+
+  sub_test_case "message_length_limit" do
+    data("batch_emit", { extract: "" }, keep: true)
+    data("single_emit", { extract: "<extract>\ntag_key tag\n</extract>\n" }, keep: true)
+    test "drop records exceeding limit" do |data|
+      message_length_limit = 10
+      d = create_driver(base_config + %!
+        message_length_limit #{message_length_limit}
+        <parse>
+          @type none
+        </parse>
+        #{data[:extract]}
+      !)
+      d.run(expect_records: 2, timeout: 10) do
+        create_tcp_socket('127.0.0.1', @port) do |sock|
+          sock.send("a" * message_length_limit + "\n", 0)
+          sock.send("b" * (message_length_limit + 1) + "\n", 0)
+          sock.send("c" * (message_length_limit - 1) + "\n", 0)
+        end
+      end
+
+      expected_records = [
+        "a" * message_length_limit,
+        "c" * (message_length_limit - 1)
+      ]
+      actual_records = d.events.collect do |event|
+        event[2]["message"]
+      end
+
+      assert_equal expected_records, actual_records
+    end
+
+    test "clear buffer and discard the subsequent data until the next delimiter" do |data|
+      message_length_limit = 12
+      d = create_driver(base_config + %!
+        message_length_limit #{message_length_limit}
+        delimiter ";"
+        <parse>
+          @type json
+        </parse>
+        #{data[:extract]}
+      !)
+      d.run(expect_records: 1, timeout: 10) do
+        create_tcp_socket('127.0.0.1', @port) do |sock|
+          sock.send('{"message":', 0)
+          sock.send('"hello', 0)
+          sleep 1 # To make the server read data and clear the buffer here.
+          sock.send('world!"};', 0) # This subsequent data must be discarded so that a parsing failure doesn't occur.
+          sock.send('{"k":"v"};', 0) # This will succeed to parse.
+        end
+      end
+
+      logs = d.logs.collect do |log|
+        log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
+      end
+      actual_records = d.events.collect do |event|
+        event[2]
+      end
+
+      assert_equal(
+        {
+          # Asserting that '[warn]: pattern not matched message="world!\"}"' warning does not occur.
+          logs: ['[info]: The buffer size exceeds \'message_length_limit\', cleared: limit=12 size=17 head="{\"message\":\"hello"' + "\n"],
+          records: [{"k" => "v"}],
+        },
+        {
+          logs: logs[1..],
+          records: actual_records,
+        }
+      )
+    end
+  end
 end

data/test/plugin/test_in_udp.rb

@@ -265,4 +265,32 @@ class UdpInputTest < Test::Unit::TestCase
       end
     end
   end
+
+  test 'message_length_limit' do
+    message_length_limit = 32
+    d = create_driver(base_config + %!
+      format none
+      message_length_limit #{message_length_limit}
+    !)
+    d.run(expect_records: 3) do
+      create_udp_socket('127.0.0.1', @port) do |u|
+        3.times do |i|
+          u.send("#{i}" * 40 + "\n", 0)
+        end
+      end
+    end
+
+    if Fluent.windows?
+      expected_records = []
+    else
+      expected_records = 3.times.collect do |i|
+        "#{i}" * message_length_limit
+      end
+    end
+    actual_records = d.events.collect do |event|
+      event[2]["message"]
+    end
+
+    assert_equal expected_records, actual_records
+  end
 end

data/test/plugin/test_in_unix.rb

@@ -174,8 +174,8 @@ class UnixInputTest < Test::Unit::TestCase
     assert_equal 0, @d.events.size
 
     logs = @d.instance.log.logs
-    assert_equal 1, logs.select { |line|
+    assert_equal 1, logs.count { |line|
       line =~ / \[warn\]: incoming data is broken: msg=#{data.inspect}/
-    }.size, "should not accept broken chunk"
+    }, "should not accept broken chunk"
   end
 end unless Fluent.windows?

data/test/plugin/test_input.rb

@@ -73,7 +73,7 @@ class InputTest < Test::Unit::TestCase
 
     @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
 
-    assert{ @p.log.object_id != original_logger.object_id }
+    assert(@p.log.object_id != original_logger.object_id)
     assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
   end
 
@@ -85,6 +85,17 @@ class InputTest < Test::Unit::TestCase
     end
   end
 
+  test 'can use metrics plugins and fallback methods' do
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    %w[emit_size_metrics emit_records_metrics].each do |metric_name|
+      assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+    end
+
+    assert_equal 0, @p.emit_size
+    assert_equal 0, @p.emit_records
+  end
+
   test 'are not available with multi workers configuration in default' do
     assert_false @p.multi_workers_ready?
   end

data/test/plugin/test_metrics.rb

@@ -0,0 +1,294 @@
+require_relative '../helper'
+require 'fluent/plugin/metrics'
+require 'fluent/plugin/base'
+require 'fluent/system_config'
+
+class BareMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('bare', self)
+
+  private
+
+  # Just override for tests.
+  def has_methods_for_counter?
+    false
+  end
+end
+
+class BasicCounterMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def add(value)
+    @data += value
+  end
+  def set(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class AliasedCounterMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def configure(conf)
+    super
+    class << self
+      alias_method :set, :set_counter
+    end
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def add(value)
+    @data += value
+  end
+  def set_counter(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class BasicGaugeMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def dec
+    @data -=1
+  end
+  def add(value)
+    @data += value
+  end
+  def sub(value)
+    @data -= value
+  end
+  def set(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class AliasedGaugeMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def configure(conf)
+    super
+    class << self
+      alias_method :dec, :dec_gauge
+      alias_method :set, :set_gauge
+      alias_method :sub, :sub_gauge
+    end
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def dec_gauge
+    @data -=1
+  end
+  def add(value)
+    @data += value
+  end
+  def sub_gauge(value)
+    @data -= value
+  end
+  def set_gauge(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class StorageTest < Test::Unit::TestCase
+  sub_test_case 'BareMetrics' do
+    setup do
+      @m = BareMetrics.new
+      @m.configure(config_element())
+    end
+
+    test 'is configured with plugin information and system config' do
+      m = BareMetrics.new
+      m.configure(config_element('metrics', '', {}))
+
+      assert_false m.use_gauge_metric
+      assert_false m.has_methods_for_counter
+      assert_false m.has_methods_for_gauge
+    end
+
+    test 'all bare operations are not defined yet' do
+      assert_raise NotImplementedError do
+        @m.get
+      end
+      assert_raise NotImplementedError do
+        @m.inc
+      end
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+      assert_raise NotImplementedError do
+        @m.add(10)
+      end
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+      assert_raise NotImplementedError do
+        @m.set(123)
+      end
+    end
+  end
+
+  sub_test_case 'BasicCounterMetric' do
+    setup do
+      @m = BasicCounterMetrics.new
+      @m.configure(config_element('metrics', '', {'@id' => '1'}))
+    end
+
+    test 'all basic counter operations work well' do
+      assert_true @m.has_methods_for_counter
+      assert_false @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+    end
+  end
+
+  sub_test_case 'AliasedCounterMetric' do
+    setup do
+      @m = AliasedCounterMetrics.new
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all aliased counter operations work well' do
+      assert_true @m.has_methods_for_counter
+      assert_false @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+    end
+  end
+
+  sub_test_case 'BasicGaugeMetric' do
+    setup do
+      @m = BasicGaugeMetrics.new
+      @m.use_gauge_metric = true
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all basic gauge operations work well' do
+      assert_false @m.has_methods_for_counter
+      assert_true @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      @m.dec
+      assert_equal 20, @m.get
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      @m.sub(11)
+      assert_equal 89, @m.get
+    end
+  end
+
+  sub_test_case 'AliasedGaugeMetric' do
+    setup do
+      @m = AliasedGaugeMetrics.new
+      @m.use_gauge_metric = true
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all aliased gauge operations work well' do
+      assert_false @m.has_methods_for_counter
+      assert_true @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      @m.dec
+      assert_equal 20, @m.get
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      @m.sub(11)
+      assert_equal 89, @m.get
+    end
+  end
+end

data/test/plugin/test_metrics_local.rb

@@ -0,0 +1,96 @@
+require_relative '../helper'
+require 'fluent/plugin/metrics_local'
+require 'fluent/system_config'
+
+class LocalMetricsTest < ::Test::Unit::TestCase
+  sub_test_case 'configure' do
+    test "configured for counter mode" do
+      m = Fluent::Plugin::LocalMetrics.new
+      m.configure(config_element('metrics', '', {"labels" => {test: "test-unit", language: "Ruby"}}))
+
+      assert_false m.use_gauge_metric
+      assert_equal({agent: "Fluentd", hostname: "#{Socket.gethostname}"}, m.default_labels)
+      assert_equal({test: "test-unit", language: "Ruby"}, m.labels)
+      assert_true m.has_methods_for_counter
+      assert_false m.has_methods_for_gauge
+    end
+
+    test "configured for gauge mode" do
+      m = Fluent::Plugin::LocalMetrics.new
+      m.use_gauge_metric = true
+      m.configure(config_element('metrics', '', {"labels" => {test: "test-unit", language: "Ruby"}}))
+
+      assert_true m.use_gauge_metric
+      assert_equal({agent: "Fluentd", hostname: "#{Socket.gethostname}"}, m.default_labels)
+      assert_equal({test: "test-unit", language: "Ruby"}, m.labels)
+      assert_false m.has_methods_for_counter
+      assert_true m.has_methods_for_gauge
+    end
+  end
+
+  sub_test_case 'LocalMetric' do
+    sub_test_case "counter" do
+      setup do
+        @m = Fluent::Plugin::LocalMetrics.new
+        @m.configure(config_element('metrics', '', {}))
+      end
+
+      test '#configure' do
+        assert_true @m.has_methods_for_counter
+        assert_false @m.has_methods_for_gauge
+      end
+
+      test 'all local counter operations work well' do
+        assert_equal 0, @m.get
+        assert_equal 1, @m.inc
+
+        @m.add(20)
+        assert_equal 21, @m.get
+        assert_raise NotImplementedError do
+          @m.dec
+        end
+
+        @m.set(100)
+        assert_equal 100, @m.get
+
+        @m.set(10)
+        assert_equal 100, @m.get # On counter, value should be overwritten bigger than stored one.
+        assert_raise NotImplementedError do
+          @m.sub(11)
+        end
+      end
+    end
+
+    sub_test_case "gauge" do
+      setup do
+        @m = Fluent::Plugin::LocalMetrics.new
+        @m.use_gauge_metric = true
+        @m.configure(config_element('metrics', '', {}))
+      end
+
+      test '#configure' do
+        assert_false @m.has_methods_for_counter
+        assert_true @m.has_methods_for_gauge
+      end
+
+      test 'all local gauge operations work well' do
+        assert_equal 0, @m.get
+        assert_equal 1, @m.inc
+
+        @m.add(20)
+        assert_equal 21, @m.get
+        @m.dec
+        assert_equal 20, @m.get
+
+        @m.set(100)
+        assert_equal 100, @m.get
+
+        @m.sub(11)
+        assert_equal 89, @m.get
+
+        @m.set(10)
+        assert_equal 10, @m.get # On gauge, value always should be overwritten.
+      end
+    end
+  end
+end

data/test/plugin/test_multi_output.rb

@@ -146,8 +146,11 @@ class MultiOutputTest < Test::Unit::TestCase
       @i.configure(conf)
       assert_equal 4, @i.outputs.size
 
+      log_size_for_multi_output_itself = 4
+      log_size_for_metrics_plugin_helper = 4
+      expected_warn_log_size = log_size_for_multi_output_itself + log_size_for_metrics_plugin_helper
       logs = @i.log.out.logs
-      assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == 4 }
+      assert{ logs.count{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") } == expected_warn_log_size }
     end
 
     test '#emit_events calls #process always' do
@@ -176,5 +179,26 @@ class MultiOutputTest < Test::Unit::TestCase
 
       assert_equal 2, @i.events.size
     end
+
+    test 'can use metrics plugins and fallback methods' do
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+
+      %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name|
+        assert_true @i.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+      end
+
+      assert_equal 0, @i.num_errors
+      assert_equal 0, @i.emit_count
+      assert_equal 0, @i.emit_size
+      assert_equal 0, @i.emit_records
+    end
   end
 end

data/test/plugin/test_out_exec.rb

@@ -243,7 +243,7 @@ class ExecOutputTest < Test::Unit::TestCase
   sub_test_case 'when executed process dies unexpectedly' do
     setup do
       @gen_config = ->(num){ <<EOC
-    command ruby -e "ARGV.first.to_i == 0 ? open(ARGV[1]){|f| STDOUT.write f.read} : (sleep 1 ; exit ARGV.first.to_i)" #{num} >#{TMP_DIR}/fail_out
+    command ruby -e "ARGV.first.to_i == 0 ? open(ARGV[1]){|f| STDOUT.write(f.read); STDOUT.flush} : (sleep 1 ; exit ARGV.first.to_i)" #{num} >#{TMP_DIR}/fail_out
     <inject>
       tag_key tag
       time_key time
@@ -265,7 +265,7 @@ EOC
       expect_path = "#{TMP_DIR}/fail_out"
 
       d.end_if{ File.exist?(expect_path) }
-      d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
+      d.run(default_tag: 'test', flush: true, wait_flush_completion: true, shutdown: false) do
         d.feed(time, records[0])
         d.feed(time, records[1])
       end
@@ -281,7 +281,8 @@ EOC
       assert{ d.instance.buffer.queue.empty? }
       assert{ d.instance.dequeued_chunks.empty? }
 
-      d.instance_shutdown
+    ensure
+      d.instance_shutdown if d && d.instance
     end
 
     test 'flushed chunk will be taken back after child process unexpectedly exits' do
@@ -304,7 +305,8 @@ EOC
 
       assert{ File.exist?(expect_path) && File.size(expect_path) == 0 }
 
-      d.instance_shutdown
+    ensure
+      d.instance_shutdown if d && d.instance
     end
   end
 end

data/test/plugin/test_out_exec_filter.rb

@@ -524,6 +524,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     assert_equal pid_list[1], events[1][2]['child_pid']
     assert_equal pid_list[0], events[2][2]['child_pid']
     assert_equal pid_list[1], events[3][2]['child_pid']
+
+  ensure
+    d.run(start: false, shutdown: true)
   end
 
   # child process exits per 3 lines
@@ -594,9 +597,10 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     # the number of pids should be same with number of child processes
     assert_equal 2, pid_list.size
     logs = d.instance.log.out.logs
-    assert_equal 2, logs.select { |l| l.include?('child process exits with error code') }.size
-    assert_equal 2, logs.select { |l| l.include?('respawning child process') }.size
+    assert_equal 2, logs.count { |l| l.include?('child process exits with error code') }
+    assert_equal 2, logs.count { |l| l.include?('respawning child process') }
 
+  ensure
     d.run(start: false, shutdown: true)
   end
 end