fluentd 1.13.1 → 1.14.0

data/test/plugin/test_in_syslog.rb

@@ -467,4 +467,39 @@ EOS
     assert_equal tests.size, d.events.size
     compare_unmatched_lines_test_result(d.events, tests, {address: address})
   end
+
+  def test_send_keepalive_packet_is_disabled_by_default
+    d = create_driver(ipv4_config + %[
+      <transport tcp>
+      </transport>
+      protocol tcp
+    ])
+    assert_false d.instance.send_keepalive_packet
+  end
+
+  def test_send_keepalive_packet_can_be_enabled
+    addr = "127.0.0.1"
+    d = create_driver(ipv4_config + %[
+      <transport tcp>
+      </transport>
+      send_keepalive_packet true
+    ])
+    assert_true d.instance.send_keepalive_packet
+    mock.proxy(d.instance).server_create_connection(
+      :in_syslog_tcp_server, @port,
+      bind: addr,
+      resolve_name: nil,
+      send_keepalive_packet: true)
+    d.run do
+      TCPSocket.open(addr, @port)
+    end
+  end
+
+  def test_send_keepalive_packet_can_not_be_enabled_for_udp
+    assert_raise(Fluent::ConfigError) do
+      d = create_driver(ipv4_config + %[
+        send_keepalive_packet true
+      ])
+    end
+  end
 end

data/test/plugin/test_in_tail.rb

@@ -109,6 +109,7 @@ class TailInputTest < Test::Unit::TestCase
                                                 "refresh_interval" => "1s",
                                                 "read_from_head" => "true",
                                                 "format" => "none",
+                                                "rotate_wait" => "1s",
                                                 "follow_inodes" => "true"
                                               })
   SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
@@ -456,6 +457,91 @@ class TailInputTest < Test::Unit::TestCase
           assert_equal([], d.events)
         end
       end
+
+      sub_test_case "EOF with reads_bytes_per_second" do
+        def test_longer_than_rotate_wait
+          limit_bytes = 8192
+          num_lines = 1024 * 3
+          msg = "08bytes"
+
+          File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
+            f.write("#{msg}\n" * num_lines)
+          end
+
+          config = CONFIG_READ_FROM_HEAD +
+                   SINGLE_LINE_CONFIG +
+                   config_element("", "", {
+                                    "read_bytes_limit_per_second" => limit_bytes,
+                                    "rotate_wait" => 0.1,
+                                    "refresh_interval" => 0.5,
+                                  })
+
+          rotated = false
+          d = create_driver(config)
+          d.run(timeout: 10) do
+            while d.events.size < num_lines do
+              if d.events.size > 0 && !rotated
+                cleanup_file("#{TMP_DIR}/tail.txt")
+                FileUtils.touch("#{TMP_DIR}/tail.txt")
+                rotated = true
+              end
+              sleep 0.3
+            end
+          end
+
+          assert_equal(num_lines,
+                       d.events.count do |event|
+                         event[2]["message"] == msg
+                       end)
+        end
+
+        def test_shorter_than_rotate_wait
+          limit_bytes = 8192
+          num_lines = 1024 * 2
+          msg = "08bytes"
+
+          File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
+            f.write("#{msg}\n" * num_lines)
+          end
+
+          config = CONFIG_READ_FROM_HEAD +
+                   SINGLE_LINE_CONFIG +
+                   config_element("", "", {
+                                    "read_bytes_limit_per_second" => limit_bytes,
+                                    "rotate_wait" => 2,
+                                    "refresh_interval" => 0.5,
+                                  })
+
+          start_time = Fluent::Clock.now
+          rotated = false
+          detached = false
+          d = create_driver(config)
+          mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
+            mock.proxy(tw).detach(anything) do |v|
+              detached = true
+              v
+            end
+            tw
+          end.twice
+
+          d.run(timeout: 10) do
+            until detached do
+              if d.events.size > 0 && !rotated
+                cleanup_file("#{TMP_DIR}/tail.txt")
+                FileUtils.touch("#{TMP_DIR}/tail.txt")
+                rotated = true
+              end
+              sleep 0.3
+            end
+          end
+
+          assert_true(Fluent::Clock.now - start_time > 2)
+          assert_equal(num_lines,
+                       d.events.count do |event|
+                         event[2]["message"] == msg
+                       end)
+        end
+      end
     end
 
     data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
@@ -1653,7 +1739,7 @@ class TailInputTest < Test::Unit::TestCase
       d.instance_shutdown
     end
 
-    def test_should_keep_and_update_existing_file_pos_entry_for_deleted_file_when_new_file_with_same_name_created
+    def test_should_remove_deleted_file
       config = config_element("", "", {"format" => "none"})
 
       path = "#{TMP_DIR}/tail.txt"
@@ -1664,30 +1750,11 @@ class TailInputTest < Test::Unit::TestCase
       }
 
       d = create_driver(config)
-      d.run(shutdown: false)
-
-      pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
-      pos_file.pos = 0
-
-      path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
-      assert_equal(path, path_pos_ino[1])
-      assert_equal(pos, path_pos_ino[2].to_i(16))
-      assert_equal(ino, path_pos_ino[3].to_i(16))
-
-      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
-        f.puts "test1"
-        f.puts "test2"
-      }
-      Timecop.travel(Time.now + 10) do
-        sleep 5
+      d.run do
+        pos_file = File.open("#{TMP_DIR}/tail.pos", "r")
         pos_file.pos = 0
-        tuple = create_target_info("#{TMP_DIR}/tail.txt")
-        path_pos_ino = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.match(pos_file.readline)
-        assert_equal(tuple.path, path_pos_ino[1])
-        assert_equal(12, path_pos_ino[2].to_i(16))
-        assert_equal(tuple.ino, path_pos_ino[3].to_i(16))
+        assert_equal([], pos_file.readlines)
       end
-      d.instance_shutdown
     end
 
     def test_should_mark_file_unwatched_after_limit_recently_modified_and_rotate_wait
@@ -1899,6 +1966,49 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal({"message" => "test4"}, events[3][2])
       d.instance_shutdown
     end
+
+    # issue #3464
+    def test_should_replace_target_info
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1\n"
+      }
+      target_info = create_target_info("#{TMP_DIR}/tail.txt")
+      inodes = []
+
+      config = config_element("ROOT", "", {
+                                "path" => "#{TMP_DIR}/tail.txt*",
+                                "pos_file" => "#{TMP_DIR}/tail.pos",
+                                "tag" => "t1",
+                                "refresh_interval" => "60s",
+                                "read_from_head" => "true",
+                                "format" => "none",
+                                "rotate_wait" => "1s",
+                                "follow_inodes" => "true"
+                              })
+      d = create_driver(config, false)
+      d.run(timeout: 5) do
+        while d.events.size < 1 do
+          sleep 0.1
+        end
+        inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
+          key.ino
+        end
+        assert_equal([target_info.ino], inodes)
+
+        cleanup_file("#{TMP_DIR}/tail.txt")
+        File.open("#{TMP_DIR}/tail.txt", "wb") {|f| f.puts "test2\n"}
+
+        while d.events.size < 2 do
+          sleep 0.1
+        end
+        inodes = d.instance.instance_variable_get(:@tails).keys.collect do |key|
+          key.ino
+        end
+        new_target_info = create_target_info("#{TMP_DIR}/tail.txt")
+        assert_not_equal(target_info.ino, new_target_info.ino)
+        assert_equal([new_target_info.ino], inodes)
+      end
+    end
   end
 
   sub_test_case "tail_path" do
@@ -2069,8 +2179,9 @@ class TailInputTest < Test::Unit::TestCase
     assert_nothing_raised do
       d.run(shutdown: false) {}
     end
-    d.instance_shutdown
     assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
+  ensure
+    d.instance_shutdown if d && d.instance
   end
 
   def test_EACCES_error_after_setup_watcher
@@ -2093,10 +2204,10 @@ class TailInputTest < Test::Unit::TestCase
       assert_nothing_raised do
         d.run(shutdown: false) {}
       end
-      d.instance_shutdown
       assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
     end
   ensure
+    d.instance_shutdown if d && d.instance
     if File.exist?("#{TMP_DIR}/noaccess")
       FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
       FileUtils.rm_rf("#{TMP_DIR}/noaccess")
@@ -2116,8 +2227,9 @@ class TailInputTest < Test::Unit::TestCase
     assert_nothing_raised do
       d.run(shutdown: false) {}
     end
-    d.instance_shutdown
     assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
+  ensure
+    d.instance_shutdown if d && d.instance
   end
 
   def test_shutdown_timeout

data/test/plugin/test_input.rb

@@ -85,6 +85,17 @@ class InputTest < Test::Unit::TestCase
     end
   end
 
+  test 'can use metrics plugins and fallback methods' do
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    %w[emit_size_metrics emit_records_metrics].each do |metric_name|
+      assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+    end
+
+    assert_equal 0, @p.emit_size
+    assert_equal 0, @p.emit_records
+  end
+
   test 'are not available with multi workers configuration in default' do
     assert_false @p.multi_workers_ready?
   end

data/test/plugin/test_metrics.rb

@@ -0,0 +1,294 @@
+require_relative '../helper'
+require 'fluent/plugin/metrics'
+require 'fluent/plugin/base'
+require 'fluent/system_config'
+
+class BareMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('bare', self)
+
+  private
+
+  # Just override for tests.
+  def has_methods_for_counter?
+    false
+  end
+end
+
+class BasicCounterMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def add(value)
+    @data += value
+  end
+  def set(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class AliasedCounterMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def configure(conf)
+    super
+    class << self
+      alias_method :set, :set_counter
+    end
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def add(value)
+    @data += value
+  end
+  def set_counter(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class BasicGaugeMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def dec
+    @data -=1
+  end
+  def add(value)
+    @data += value
+  end
+  def sub(value)
+    @data -= value
+  end
+  def set(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class AliasedGaugeMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def configure(conf)
+    super
+    class << self
+      alias_method :dec, :dec_gauge
+      alias_method :set, :set_gauge
+      alias_method :sub, :sub_gauge
+    end
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def dec_gauge
+    @data -=1
+  end
+  def add(value)
+    @data += value
+  end
+  def sub_gauge(value)
+    @data -= value
+  end
+  def set_gauge(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class StorageTest < Test::Unit::TestCase
+  sub_test_case 'BareMetrics' do
+    setup do
+      @m = BareMetrics.new
+      @m.configure(config_element())
+    end
+
+    test 'is configured with plugin information and system config' do
+      m = BareMetrics.new
+      m.configure(config_element('metrics', '', {}))
+
+      assert_false m.use_gauge_metric
+      assert_false m.has_methods_for_counter
+      assert_false m.has_methods_for_gauge
+    end
+
+    test 'all bare operations are not defined yet' do
+      assert_raise NotImplementedError do
+        @m.get
+      end
+      assert_raise NotImplementedError do
+        @m.inc
+      end
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+      assert_raise NotImplementedError do
+        @m.add(10)
+      end
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+      assert_raise NotImplementedError do
+        @m.set(123)
+      end
+    end
+  end
+
+  sub_test_case 'BasicCounterMetric' do
+    setup do
+      @m = BasicCounterMetrics.new
+      @m.configure(config_element('metrics', '', {'@id' => '1'}))
+    end
+
+    test 'all basic counter operations work well' do
+      assert_true @m.has_methods_for_counter
+      assert_false @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+    end
+  end
+
+  sub_test_case 'AliasedCounterMetric' do
+    setup do
+      @m = AliasedCounterMetrics.new
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all aliased counter operations work well' do
+      assert_true @m.has_methods_for_counter
+      assert_false @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+    end
+  end
+
+  sub_test_case 'BasicGaugeMetric' do
+    setup do
+      @m = BasicGaugeMetrics.new
+      @m.use_gauge_metric = true
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all basic gauge operations work well' do
+      assert_false @m.has_methods_for_counter
+      assert_true @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      @m.dec
+      assert_equal 20, @m.get
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      @m.sub(11)
+      assert_equal 89, @m.get
+    end
+  end
+
+  sub_test_case 'AliasedGaugeMetric' do
+    setup do
+      @m = AliasedGaugeMetrics.new
+      @m.use_gauge_metric = true
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all aliased gauge operations work well' do
+      assert_false @m.has_methods_for_counter
+      assert_true @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      @m.dec
+      assert_equal 20, @m.get
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      @m.sub(11)
+      assert_equal 89, @m.get
+    end
+  end
+end