fluentd 1.13.3 → 1.14.2

data/lib/fluent/plugin.rb

@@ -36,8 +36,9 @@ module Fluent
     FORMATTER_REGISTRY = Registry.new(:formatter, 'fluent/plugin/formatter_',  dir_search_prefix: 'formatter_')
     STORAGE_REGISTRY   = Registry.new(:storage,   'fluent/plugin/storage_',    dir_search_prefix: 'storage_')
     SD_REGISTRY        = Registry.new(:sd,        'fluent/plugin/sd_',         dir_search_prefix: 'sd_')
+    METRICS_REGISTRY   = Registry.new(:metrics,   'fluent/plugin/metrics_',    dir_search_prefix: 'metrics_')
 
-    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY]
+    REGISTRIES = [INPUT_REGISTRY, OUTPUT_REGISTRY, FILTER_REGISTRY, BUFFER_REGISTRY, PARSER_REGISTRY, FORMATTER_REGISTRY, STORAGE_REGISTRY, SD_REGISTRY, METRICS_REGISTRY]
 
     def self.register_input(type, klass)
       register_impl('input', INPUT_REGISTRY, type, klass)
@@ -59,6 +60,10 @@ module Fluent
       register_impl('sd', SD_REGISTRY, type, klass)
     end
 
+    def self.register_metrics(type, klass)
+      register_impl('metrics', METRICS_REGISTRY, type, klass)
+    end
+
     def self.register_parser(type, klass_or_proc)
       if klass_or_proc.is_a?(Regexp)
         # This usage is not recommended for new API
@@ -121,6 +126,10 @@ module Fluent
       new_impl('sd', SD_REGISTRY, type, parent)
     end
 
+    def self.new_metrics(type, parent: nil)
+      new_impl('metrics', METRICS_REGISTRY, type, parent)
+    end
+
     class << self
       # This should be defined for fluent-plugin-config-formatter type arguments.
       alias_method :new_service_discovery, :new_sd

data/lib/fluent/plugin_helper/event_emitter.rb

@@ -29,6 +29,9 @@ module Fluent
         if @_event_emitter_lazy_init
           @router = @primary_instance.router
         end
+        if @router.respond_to?(:caller_plugin_id=)
+          @router.caller_plugin_id = self.plugin_id
+        end
         @router
       end
 
@@ -47,7 +50,11 @@ module Fluent
 
       def event_emitter_router(label_name)
         if label_name
-          Engine.root_agent.find_label(label_name).event_router
+          if label_name == "@ROOT"
+            Engine.root_agent.event_router
+          else
+            Engine.root_agent.find_label(label_name).event_router
+          end
         elsif self.respond_to?(:as_secondary) && self.as_secondary
           if @primary_instance.has_router?
             @_event_emitter_lazy_init = true

data/lib/fluent/plugin_helper/metrics.rb

@@ -0,0 +1,129 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'forwardable'
+
+require 'fluent/plugin'
+require 'fluent/plugin/metrics'
+require 'fluent/plugin_helper/timer'
+require 'fluent/config/element'
+require 'fluent/configurable'
+require 'fluent/system_config'
+
+module Fluent
+  module PluginHelper
+    module Metrics
+      include Fluent::SystemConfig::Mixin
+
+      attr_reader :_metrics # For tests.
+
+      def initialize
+        super
+        @_metrics_started = false
+        @_metrics = {} # usage => metrics_state
+      end
+
+      def configure(conf)
+        super
+
+        @plugin_type_or_id = if self.plugin_id_configured?
+                               self.plugin_id
+                             else
+                               if type = (conf["@type"] || conf["type"])
+                                 "#{type}.#{self.plugin_id}"
+                               else
+                                 "#{self.class.to_s.split("::").last.downcase}.#{self.plugin_id}"
+                               end
+                             end
+      end
+
+      def metrics_create(namespace: "fluentd", subsystem: "metrics", name:, help_text:, labels: {}, prefer_gauge: false)
+        metrics = if system_config.metrics
+                    Fluent::Plugin.new_metrics(system_config.metrics[:@type], parent: self)
+                  else
+                    Fluent::Plugin.new_metrics(Fluent::Plugin::Metrics::DEFAULT_TYPE, parent: self)
+                  end
+        config = if system_config.metrics
+                   system_config.metrics.corresponding_config_element
+                 else
+                   Fluent::Config::Element.new('metrics', '', {'@type' => Fluent::Plugin::Metrics::DEFAULT_TYPE}, [])
+                 end
+        metrics.use_gauge_metric = prefer_gauge
+        metrics.configure(config)
+        # For multi workers environment, cmetrics should be distinguish with static labels.
+        if Fluent::Engine.system_config.workers > 1
+          labels.merge!(worker_id: fluentd_worker_id.to_s)
+        end
+        labels.merge!(plugin: @plugin_type_or_id)
+        metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
+
+        @_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics
+
+        metrics
+      end
+
+      def metrics_operate(method_name, &block)
+        @_metrics.each_pair do |key, m|
+          begin
+            block.call(s) if block_given?
+            m.__send__(method_name)
+          rescue => e
+            log.error "unexpected error while #{method_name}", key: key, metrics: m, error: e
+          end
+        end
+      end
+
+      def start
+        super
+
+        metrics_operate(:start)
+        @_metrics_started = true
+      end
+
+      def stop
+        super
+        # timer stops automatically in super
+        metrics_operate(:stop)
+      end
+
+      def before_shutdown
+        metrics_operate(:before_shutdown)
+        super
+      end
+
+      def shutdown
+        metrics_operate(:shutdown)
+        super
+      end
+
+      def after_shutdown
+        metrics_operate(:after_shutdown)
+        super
+      end
+
+      def close
+        metrics_operate(:close)
+        super
+      end
+
+      def terminate
+        metrics_operate(:terminate)
+        @_metrics = {}
+        super
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/server.rb

@@ -709,13 +709,15 @@ module Fluent
                 return true
               end
             rescue Errno::EPIPE, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH => e
+              peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
               @log.trace "unexpected error before accepting TLS connection",
-                         host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
+                         addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
               close rescue nil
             rescue OpenSSL::SSL::SSLError => e
+              peeraddr = (@_handler_socket.peeraddr rescue PEERADDR_FAILED)
               # Use same log level as on_readable
               @log.warn "unexpected error before accepting TLS connection by OpenSSL",
-                        host: @_handler_socket.peeraddr[3], port: @_handler_socket.peeraddr[1], error: e
+                        addr: peeraddr[3], host: peeraddr[2], port: peeraddr[1], error: e
               close rescue nil
             end
 

data/lib/fluent/plugin_helper.rb

@@ -32,6 +32,7 @@ require 'fluent/plugin_helper/retry_state'
 require 'fluent/plugin_helper/record_accessor'
 require 'fluent/plugin_helper/compat_parameters'
 require 'fluent/plugin_helper/service_discovery'
+require 'fluent/plugin_helper/metrics'
 
 module Fluent
   module PluginHelper

data/lib/fluent/plugin_id.rb

@@ -15,6 +15,7 @@
 #
 
 require 'set'
+require 'fluent/env'
 require 'fluent/variable_store'
 
 module Fluent
@@ -76,7 +77,7 @@ module Fluent
 
       # Fluent::Plugin::Base#fluentd_worker_id
       dir = File.join(system_config.root_dir, "worker#{fluentd_worker_id}", plugin_id)
-      FileUtils.mkdir_p(dir, mode: system_config.dir_permission || 0755) unless Dir.exist?(dir)
+      FileUtils.mkdir_p(dir, mode: system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION) unless Dir.exist?(dir)
       @_plugin_root_dir = dir.freeze
       dir
     end

data/lib/fluent/root_agent.rb

@@ -55,9 +55,11 @@ module Fluent
       @suppress_emit_error_log_interval = 0
       @next_emit_error_log_time = nil
       @without_source = false
+      @enable_input_metrics = false
 
       suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
       @without_source = system_config.without_source unless system_config.without_source.nil?
+      @enable_input_metrics = !!system_config.enable_input_metrics
     end
 
     attr_reader :inputs
@@ -131,6 +133,7 @@ module Fluent
         end
         name = e.arg
         raise ConfigError, "Missing symbol argument on <label> directive" if name.empty?
+        raise ConfigError, "@ROOT for <label> is not permitted, reserved for getting root router" if name == '@ROOT'
 
         if name == ERROR_LABEL
           error_label_config = e
@@ -315,6 +318,9 @@ module Fluent
       # See also 'fluentd/plugin/input.rb'
       input.context_router = @event_router
       input.configure(conf)
+      if @enable_input_metrics
+        @event_router.add_metric_callbacks(input.plugin_id, Proc.new {|es| input.metric_callback(es) })
+      end
       @inputs << input
 
       input

data/lib/fluent/supervisor.rb

@@ -544,7 +544,7 @@ module Fluent
         $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
 
         if @path && log_dir_perm
-          File.chmod(log_dir_perm || 0755, File.dirname(@path))
+          File.chmod(log_dir_perm || Fluent::DEFAULT_DIR_PERMISSION, File.dirname(@path))
         end
       end
 
@@ -570,6 +570,8 @@ module Fluent
         suppress_repeated_stacktrace: true,
         ignore_repeated_log_interval: nil,
         without_source: nil,
+        enable_input_metrics: nil,
+        enable_size_metrics: nil,
         use_v1_config: true,
         strict_config_value: nil,
         supervise: true,
@@ -649,7 +651,7 @@ module Fluent
           end
         else
           begin
-            FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || 0755)
+            FileUtils.mkdir_p(root_dir, mode: @system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION)
           rescue => e
             raise Fluent::InvalidRootDirectory, "failed to create root directory:#{root_dir}, #{e.inspect}"
           end

data/lib/fluent/system_config.rb

@@ -27,7 +27,8 @@ module Fluent
       :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
-      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket
+      :strict_config_value, :enable_msgpack_time_support, :disable_shared_socket,
+      :metrics, :enable_input_metrics, :enable_size_metrics
     ]
 
     config_param :workers,   :integer, default: 1
@@ -46,6 +47,8 @@ module Fluent
     config_param :strict_config_value, :bool, default: nil
     config_param :enable_msgpack_time_support, :bool, default: nil
     config_param :disable_shared_socket, :bool, default: nil
+    config_param :enable_input_metrics, :bool, default: nil
+    config_param :enable_size_metrics, :bool, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end
@@ -93,6 +96,11 @@ module Fluent
       config_param :timeout, :time, default: nil
     end
 
+    config_section :metrics, multi: false do
+      config_param :@type, :string, default: "local"
+      config_param :labels, :hash, default: {}
+    end
+
     def self.create(conf, strict_config_value=false)
       systems = conf.elements(name: 'system')
       return SystemConfig.new if systems.empty?

data/lib/fluent/time.rb

@@ -226,19 +226,16 @@ module Fluent
 
       format_with_timezone = format && (format.include?("%z") || format.include?("%Z"))
 
-      # unixtime_in_expected_tz = unixtime_in_localtime + offset_diff
-      offset_diff = case
-                    when format_with_timezone then nil
-                    when timezone  then
-                      offset = Fluent::Timezone.utc_offset(timezone)
-                      if offset.respond_to?(:call)
-                        ->(t) { Time.now.localtime.utc_offset - offset.call(t) }
-                      else
-                        Time.now.localtime.utc_offset - offset
-                      end
-                    when localtime then 0
-                    else Time.now.localtime.utc_offset # utc
-                    end
+      utc_offset = case
+                   when format_with_timezone then
+                     nil
+                   when timezone then
+                     Fluent::Timezone.utc_offset(timezone)
+                   when localtime then
+                     nil
+                   else
+                     0 # utc
+                   end
 
       strptime = format && (Strptime.new(format) rescue nil)
 
@@ -247,16 +244,20 @@ module Fluent
                when format_with_timezone             then ->(v){ Fluent::EventTime.from_time(Time.strptime(v, format)) }
                when format == '%iso8601'             then ->(v){ Fluent::EventTime.from_time(Time.iso8601(v)) }
                when strptime then
-                 if offset_diff.respond_to?(:call)
-                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
+                 if utc_offset.nil?
+                   ->(v){ t = strptime.exec(v); Fluent::EventTime.new(t.to_i, t.nsec) }
+                 elsif utc_offset.respond_to?(:call)
+                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset.call(t), t.nsec) }
                  else
-                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+                   ->(v) { t = strptime.exec(v); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset, t.nsec) }
                  end
-               when format   then
-                 if offset_diff.respond_to?(:call)
-                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff.call(t), t.nsec) }
+               when format then
+                 if utc_offset.nil?
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i, t.nsec) }
+                 elsif utc_offset.respond_to?(:call)
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset.call(t), t.nsec) }
                  else
-                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + offset_diff, t.nsec) }
+                   ->(v){ t = Time.strptime(v, format); Fluent::EventTime.new(t.to_i + t.utc_offset - utc_offset, t.nsec) }
                  end
                else ->(v){ Fluent::EventTime.parse(v) }
                end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.13.3'
+  VERSION = '1.14.2'
 
 end

data/test/command/test_cat.rb

@@ -69,7 +69,7 @@ class TestFluentCat < ::Test::Unit::TestCase
     def test_cat_json
       d = create_driver
       d.run(expect_records: 1) do
-        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{@port} json") do |stdin|
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} json") do |stdin|
           stdin.puts('{"key":"value"}')
           stdin.close
         end
@@ -86,7 +86,7 @@ class TestFluentCat < ::Test::Unit::TestCase
       path = d.instance.write(@chunk)
       d = create_driver
       d.run(expect_records: 1) do
-        Open3.pipeline_w("ruby #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
           stdin.write(File.read(path))
           stdin.close
         end
@@ -96,4 +96,33 @@ class TestFluentCat < ::Test::Unit::TestCase
                    [d.events.size, event.first, event.last])
     end
   end
+
+  sub_test_case "send specific event time" do
+    def test_without_event_time
+      event_time = Fluent::EventTime.now
+      d = create_driver
+      d.run(expect_records: 1) do
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} tag") do |stdin|
+          stdin.puts('{"key":"value"}')
+          stdin.close
+        end
+      end
+      event = d.events.first
+      assert_in_delta(event_time.to_f, event[1].to_f, 3.0) # expect command to be finished in 3 seconds
+      assert_equal([1, "tag", true, @record],
+                   [d.events.size, event.first, event_time.to_f < event[1].to_f, event.last])
+    end
+
+    def test_with_event_time
+      event_time = "2021-01-02 13:14:15.0+00:00"
+      d = create_driver
+      d.run(expect_records: 1) do
+        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --event-time '#{event_time}' tag") do |stdin|
+          stdin.puts('{"key":"value"}')
+          stdin.close
+        end
+      end
+      assert_equal([["tag", Fluent::EventTime.parse(event_time), @record]], d.events)
+    end
+  end
 end

data/test/config/test_system_config.rb

@@ -28,6 +28,8 @@ module Fluent::Config
         log_event_label: nil,
         log_event_verbose: nil,
         without_source: nil,
+        enable_input_metrics: nil,
+        enable_size_metrics: nil,
         emit_error_log_interval: nil,
         file_permission: nil,
         dir_permission: nil,
@@ -77,6 +79,8 @@ module Fluent::Config
       assert_nil(sc.emit_error_log_interval)
       assert_nil(sc.suppress_config_dump)
       assert_nil(sc.without_source)
+      assert_nil(sc.enable_input_metrics)
+      assert_nil(sc.enable_size_metrics)
       assert_nil(sc.enable_msgpack_time_support)
       assert_equal(:text, sc.log.format)
       assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
@@ -93,6 +97,8 @@ module Fluent::Config
       'without_source' => ['without_source', true],
       'strict_config_value' => ['strict_config_value', true],
       'enable_msgpack_time_support' => ['enable_msgpack_time_support', true],
+      'enable_input_metrics' => ['enable_input_metrics', true],
+      'enable_size_metrics' => ['enable_size_metrics', true],
     )
     test "accepts parameters" do |(k, v)|
       conf = parse_text(<<-EOS)

data/test/plugin/in_tail/test_io_handler.rb

@@ -1,11 +1,19 @@
 require_relative '../../helper'
 
 require 'fluent/plugin/in_tail'
+require 'fluent/plugin/metrics_local'
 require 'tempfile'
 
 class IntailIOHandlerTest < Test::Unit::TestCase
   setup do
     @file = Tempfile.new('intail_io_handler').binmode
+    opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+    opened_file_metrics.configure(config_element('metrics', '', {}))
+    closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+    closed_file_metrics.configure(config_element('metrics', '', {}))
+    rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+    rotated_file_metrics.configure(config_element('metrics', '', {}))
+    @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
   end
 
   teardown do
@@ -30,7 +38,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
     end
 
     returned_lines = ''
-    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
+    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
       returned_lines << lines.join
       true
     end
@@ -62,7 +70,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = ''
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true, metrics: @metrics) do |lines, _watcher|
         returned_lines << lines.join
         true
       end
@@ -93,7 +101,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = []
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
         returned_lines << lines.dup
         true
       end
@@ -119,7 +127,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = []
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
         returned_lines << lines.dup
         true
       end

data/test/plugin/in_tail/test_position_file.rb

@@ -22,6 +22,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
     invalidpath100000000000000000000000000000000
     unwatched\t#{UNWATCHED_STR}\t0000000000000000
   EOF
+  TEST_CONTENT_PATHS = {
+    "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
+    "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
+  }
 
   def write_data(f, content)
     f.write(content)
@@ -36,7 +40,11 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
   test '.load' do
     write_data(@file, TEST_CONTENT)
-    Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+    paths = {
+      "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
+      "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 2),
+    }
+    Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
     @file.seek(0)
     lines = @file.readlines
@@ -118,7 +126,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
     test 'should ignore initial existing files on follow_inode' do
       write_data(@file, TEST_CONTENT)
-      pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, {}, **{logger: $log})
+      pos_file = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_PATHS, **{logger: $log})
       @file.seek(0)
       assert_equal([], @file.readlines)
 
@@ -138,7 +146,12 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#load' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+      invalid_path = "invalidpath100000000000000000000000000000000"
+      paths = TEST_CONTENT_PATHS.merge({
+        invalid_path => Fluent::Plugin::TailInput::TargetInfo.new(invalid_path, 0),
+        "unwatched" => Fluent::Plugin::TailInput::TargetInfo.new("unwatched", 0),
+      })
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
       @file.seek(0)
       lines = @file.readlines
@@ -147,6 +160,15 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal "inode23bit\t0000000000000000\t0000000000000000\n", lines[1]
     end
 
+    test 'compact deleted paths' do
+      write_data(@file, TEST_CONTENT)
+      Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+
+      @file.seek(0)
+      lines = @file.readlines
+      assert_equal [], lines
+    end
+
     test 'compact data if duplicated line' do
       write_data(@file, <<~EOF)
         valid_path\t0000000000000002\t0000000000000001
@@ -163,7 +185,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#[]' do
     test 'return entry' do
       write_data(@file, TEST_CONTENT)
-      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
       valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)
       f = pf[valid_target_info]

data/test/plugin/test_bare_output.rb

@@ -95,6 +95,19 @@ class BareOutputTest < Test::Unit::TestCase
     end
   end
 
+  test 'can use metrics plugins and fallback methods' do
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    %w[num_errors_metrics emit_count_metrics emit_size_metrics emit_records_metrics].each do |metric_name|
+      assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+    end
+
+    assert_equal 0, @p.num_errors
+    assert_equal 0, @p.emit_count
+    assert_equal 0, @p.emit_size
+    assert_equal 0, @p.emit_records
+  end
+
   test 'can get input event stream to write' do
     @p.configure(config_element('ROOT'))
     @p.start

data/test/plugin/test_buffer.rb

@@ -238,8 +238,14 @@ class BufferTest < Test::Unit::TestCase
       assert_nil @p.queue
       assert_nil @p.dequeued
       assert_nil @p.queued_num
-      assert_equal 0, @p.stage_size
-      assert_equal 0, @p.queue_size
+      assert_nil @p.stage_length_metrics
+      assert_nil @p.stage_size_metrics
+      assert_nil @p.queue_length_metrics
+      assert_nil @p.queue_size_metrics
+      assert_nil @p.available_buffer_space_ratios_metrics
+      assert_nil @p.total_queued_size_metrics
+      assert_nil @p.newest_timekey_metrics
+      assert_nil @p.oldest_timekey_metrics
       assert_equal [], @p.timekeys
     end
 

data/test/plugin/test_filter.rb

@@ -165,6 +165,17 @@ class FilterPluginTest < Test::Unit::TestCase
       end
     end
 
+    test 'can use metrics plugins and fallback methods' do
+      @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+      %w[emit_size_metrics emit_records_metrics].each do |metric_name|
+        assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+      end
+
+      assert_equal 0, @p.emit_size
+      assert_equal 0, @p.emit_records
+    end
+
     test 'are available with multi worker configuration in default' do
       assert @p.multi_workers_ready?
     end