fluentd 1.13.1-x64-mingw32 → 1.14.1-x64-mingw32

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -22,19 +22,18 @@ module Fluent::Plugin
       UNWATCHED_POSITION = 0xffffffffffffffff
       POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
 
-      def self.load(file, follow_inodes, existing_paths, logger:)
-        pf = new(file, follow_inodes, existing_paths, logger: logger)
-        pf.load
+      def self.load(file, follow_inodes, existing_targets, logger:)
+        pf = new(file, follow_inodes, logger: logger)
+        pf.load(existing_targets)
         pf
       end
 
-      def initialize(file, follow_inodes, existing_paths, logger: nil)
+      def initialize(file, follow_inodes, logger: nil)
         @file = file
         @logger = logger
         @file_mutex = Mutex.new
         @map = {}
         @follow_inodes = follow_inodes
-        @existing_paths = existing_paths
       end
 
       def [](target_info)
@@ -60,8 +59,8 @@ module Fluent::Plugin
         end
       end
 
-      def load
-        compact
+      def load(existing_targets = nil)
+        compact(existing_targets)
 
         map = {}
         @file_mutex.synchronize do
@@ -118,9 +117,9 @@ module Fluent::Plugin
 
       private
 
-      def compact
+      def compact(existing_targets = nil)
         @file_mutex.synchronize do
-          entries = fetch_compacted_entries.values.map(&:to_entry_fmt)
+          entries = fetch_compacted_entries(existing_targets).values.map(&:to_entry_fmt)
 
           @file.pos = 0
           @file.truncate(0)
@@ -128,7 +127,7 @@ module Fluent::Plugin
         end
       end
 
-      def fetch_compacted_entries
+      def fetch_compacted_entries(existing_targets = nil)
         entries = {}
 
         @file.pos = 0
@@ -151,23 +150,26 @@ module Fluent::Plugin
             end
 
             if @follow_inodes
-              entries[ino] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+              entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             else
-              entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+              entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             end
             file_pos += line.size
           end
         end
 
-        entries = remove_deleted_files_entries(entries, @existing_paths) if @follow_inodes
+        entries = remove_deleted_files_entries(entries, existing_targets)
         entries
       end
 
-      def remove_deleted_files_entries(existent_entries, existing_paths)
-        filtered_entries = existent_entries.select {|file_entry|
-          existing_paths.key?(file_entry)
-        }
-        filtered_entries
+      def remove_deleted_files_entries(existent_entries, existing_targets)
+        if existing_targets
+          existent_entries.select { |path_or_ino|
+            existing_targets.key?(path_or_ino)
+          }
+        else
+          existent_entries
+        end
       end
     end
 

data/lib/fluent/plugin/in_tail.rb

@@ -38,6 +38,7 @@ module Fluent::Plugin
     helpers :timer, :event_loop, :parser, :compat_parameters
 
     RESERVED_CHARS = ['/', '*', '%'].freeze
+    MetricsInfo = Struct.new(:opened, :closed, :rotated)
 
     class WatcherSetupError < StandardError
       def initialize(msg)
@@ -57,6 +58,7 @@ module Fluent::Plugin
       @pf = nil
       @ignore_list = []
       @shutdown_start_time = nil
+      @metrics = nil
     end
 
     desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -191,6 +193,10 @@ module Fluent::Plugin
           @read_bytes_limit_per_second = min_bytes
         end
       end
+      opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
+      closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
+      rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
+      @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
     end
 
     def configure_tag
@@ -375,7 +381,7 @@ module Fluent::Plugin
 
     def setup_watcher(target_info, pe)
       line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler))
+      tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler), @metrics)
 
       if @enable_watch_timer
         tt = TimerTrigger.new(1, log) { tw.on_notify }
@@ -426,6 +432,7 @@ module Fluent::Plugin
 
       begin
         target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
+        @tails.delete(target_info)
         @tails[target_info] = tw
         tw.on_notify
       rescue Errno::ENOENT, Errno::EACCES => e
@@ -491,10 +498,17 @@ module Fluent::Plugin
         new_position_entry = @pf[target_info]
 
         if new_position_entry.read_inode == 0
+          # When follow_inodes is true, it's not cleaned up by refresh_watcher.
+          # So it should be unwatched here explicitly.
+          rotated_tw.unwatched = true
+          # Make sure to delete old key, it has a different ino while the hash key is same.
+          @tails.delete(rotated_target_info)
           @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
           @tails[new_target_info].on_notify
         end
       else
+        # Make sure to delete old key, it has a different ino while the hash key is same.
+        @tails.delete(rotated_target_info)
         @tails[new_target_info] = setup_watcher(new_target_info, pe)
         @tails[new_target_info].on_notify
       end
@@ -522,8 +536,25 @@ module Fluent::Plugin
     def detach_watcher_after_rotate_wait(tw, ino)
       # Call event_loop_attach/event_loop_detach is high-cost for short-live object.
       # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
-      timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+      if @open_on_every_update
+        # Detach now because it's already closed, waiting it doesn't make sense.
         detach_watcher(tw, ino)
+      elsif @read_bytes_limit_per_second < 0
+        # throttling isn't enabled, just wait @rotate_wait
+        timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+          detach_watcher(tw, ino)
+        end
+      else
+        # When the throttling feature is enabled, it might not reach EOF yet.
+        # Should ensure to read all contents before closing it, with keeping throttling.
+        start_time_to_wait = Fluent::Clock.now
+        timer = timer_execute(:in_tail_close_watcher, 1, repeat: true) do
+          elapsed = Fluent::Clock.now - start_time_to_wait
+          if tw.eof? && elapsed >= @rotate_wait
+            timer.detach
+            detach_watcher(tw, ino)
+          end
+        end
       end
     end
 
@@ -645,6 +676,19 @@ module Fluent::Plugin
       es
     end
 
+    def statistics
+      stats = super
+
+      stats = {
+        'input' => stats["input"].merge({
+          'opened_file_count' => @metrics.opened.get,
+          'closed_file_count' => @metrics.closed.get,
+          'rotated_file_count' => @metrics.rotated.get,
+        })
+      }
+      stats
+    end
+
     private
 
     def io_handler(watcher, path)
@@ -657,6 +701,7 @@ module Fluent::Plugin
         open_on_every_update: @open_on_every_update,
         from_encoding: @from_encoding,
         encoding: @encoding,
+        metrics: @metrics,
         &method(:receive_lines)
       )
     end
@@ -692,7 +737,7 @@ module Fluent::Plugin
     end
 
     class TailWatcher
-      def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build)
+      def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build, metrics)
         @path = target_info.path
         @ino = target_info.ino
         @pe = pe || MemoryPositionEntry.new
@@ -704,6 +749,7 @@ module Fluent::Plugin
         @line_buffer_timer_flusher = line_buffer_timer_flusher
         @io_handler = nil
         @io_handler_build = io_handler_build
+        @metrics = metrics
         @watchers = []
       end
 
@@ -736,6 +782,10 @@ module Fluent::Plugin
         end
       end
 
+      def eof?
+        @io_handler.nil? || @io_handler.eof?
+      end
+
       def on_notify
         begin
           stat = Fluent::FileWrapper.stat(@path)
@@ -810,7 +860,7 @@ module Fluent::Plugin
               # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
               # don't want to swap state because we need latest read offset in pos file even after rotate_wait
               if stat
-                target_info = TargetInfo.new(@path, stat)
+                target_info = TargetInfo.new(@path, stat.ino)
                 @update_watcher.call(target_info, @pe)
               end
             else
@@ -826,6 +876,7 @@ module Fluent::Plugin
             @log.info "detected rotation of #{@path}"
             @io_handler = io_handler
           end
+          @metrics.rotated.inc
         end
       end
 
@@ -905,7 +956,7 @@ module Fluent::Plugin
 
         attr_accessor :shutdown_timeout
 
-        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
+        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
           @watcher = watcher
           @path = path
           @read_lines_limit = read_lines_limit
@@ -923,6 +974,8 @@ module Fluent::Plugin
           @shutdown_start_time = nil
           @shutdown_timeout = SHUTDOWN_TIMEOUT
           @shutdown_mutex = Mutex.new
+          @eof = false
+          @metrics = metrics
 
           @log.info "following tail of #{@path}"
         end
@@ -942,6 +995,7 @@ module Fluent::Plugin
           if @io && !@io.closed?
             @io.close
             @io = nil
+            @metrics.closed.inc
           end
         end
 
@@ -949,6 +1003,10 @@ module Fluent::Plugin
           !!@io
         end
 
+        def eof?
+          @eof
+        end
+
         private
 
         def limit_bytes_per_second_reached?
@@ -989,6 +1047,7 @@ module Fluent::Plugin
                   while true
                     @start_reading_time ||= Fluent::Clock.now
                     data = io.readpartial(BYTES_TO_READ, @iobuf)
+                    @eof = false
                     @number_bytes_read += data.bytesize
                     @fifo << data
                     @fifo.read_lines(@lines)
@@ -1005,6 +1064,7 @@ module Fluent::Plugin
                     end
                   end
                 rescue EOFError
+                  @eof = true
                 end
               end
 
@@ -1023,11 +1083,15 @@ module Fluent::Plugin
         def open
           io = Fluent::FileWrapper.open(@path)
           io.seek(@watcher.pe.read_pos + @fifo.bytesize)
+          @metrics.opened.inc
           io
         rescue RangeError
           io.close if io
           raise WatcherSetupError, "seek error with #{@path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
-        rescue Errno::ENOENT, Errno::EACCES
+        rescue Errno::EACCES => e
+          @log.warn "#{e}"
+          nil
+        rescue Errno::ENOENT
           nil
         end
 
@@ -1042,14 +1106,17 @@ module Fluent::Plugin
           else
             @io ||= open
             yield @io
+            @eof = true if @io.nil?
           end
         rescue WatcherSetupError => e
           close
+          @eof = true
           raise e
         rescue
           @log.error $!.to_s
           @log.error_backtrace
           close
+          @eof = true
         end
       end
 
@@ -1069,6 +1136,10 @@ module Fluent::Plugin
         def opened?
           false
         end
+
+        def eof?
+          true
+        end
       end
 
       class RotateHandler

data/lib/fluent/plugin/input.rb

@@ -27,7 +27,45 @@ module Fluent
       include PluginLoggerMixin
       include PluginHelper::Mixin
 
-      helpers_internal :event_emitter
+      helpers_internal :event_emitter, :metrics
+
+      def initialize
+        super
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+        @counter_mutex = Mutex.new
+        @enable_size_metrics = false
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def configure(conf)
+        super
+
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_records", help_text: "Number of count emit records")
+        @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+      end
+
+      def statistics
+        stats = {
+          'emit_records' => @emit_records_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'input' => stats }
+      end
+
+      def metric_callback(es)
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
+      end
 
       def multi_workers_ready?
         false

data/lib/fluent/plugin/metrics.rb

@@ -0,0 +1,119 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'socket'
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/unique_id'
+require 'fluent/plugin_id'
+
+module Fluent
+  module Plugin
+    class Metrics < Base
+      include PluginId
+      include PluginLoggerMixin
+      include UniqueId::Mixin
+
+      DEFAULT_TYPE = 'local'
+
+      configured_in :metrics
+
+      config_param :default_labels, :hash, default: {agent: "Fluentd", hostname: "#{Socket.gethostname}"}
+      config_param :labels, :hash, default: {}
+
+      attr_reader :use_gauge_metric
+      attr_reader :has_methods_for_gauge, :has_methods_for_counter
+
+      def initialize
+        super
+
+        @has_methods_for_counter = false
+        @has_methods_for_gauge = false
+        @use_gauge_metric = false
+      end
+
+      def configure(conf)
+        super
+
+        if use_gauge_metric
+          @has_methods_for_gauge = has_methods_for_gauge?
+        else
+          @has_methods_for_counter = has_methods_for_counter?
+        end
+      end
+
+      # Some metrics should be counted by gauge.
+      # ref: https://prometheus.io/docs/concepts/metric_types/#gauge
+      def use_gauge_metric=(use_gauge_metric=false)
+        @use_gauge_metric = use_gauge_metric
+      end
+
+      def create(namespace:, subsystem:,name:,help_text:,labels: {})
+        # This API is for cmetrics type.
+      end
+
+      def get
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def inc
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def dec
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def add(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def sub(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def set(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      private
+
+      def has_methods_for_counter?
+        implemented_methods = self.class.instance_methods(false)
+
+        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
+           [:set].all?{|e| self.class.method_defined?(e)}
+          true
+        else
+          raise "BUG: metrics plugin on counter mode MUST implement `get`, `inc`, `add` methods. And aliased `set` methods should be aliased from another method"
+        end
+      end
+
+      def has_methods_for_gauge?
+        implemented_methods = self.class.instance_methods(false)
+
+        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
+           [:set, :dec, :sub].all?{|e| self.class.method_defined?(e)}
+          true
+        else
+          raise "BUG: metrics plugin on gauge mode MUST implement `get`, `inc`, and `add` methods. And `dec`, `sub`, and `set` methods should be aliased from other methods"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/metrics_local.rb

@@ -0,0 +1,96 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/metrics'
+
+module Fluent
+  module Plugin
+    class LocalMetrics < Metrics
+      Fluent::Plugin.register_metrics('local', self)
+
+      def initialize
+        super
+        @store = 0
+        @monitor = Monitor.new
+      end
+
+      def configure(conf)
+        super
+
+        if use_gauge_metric
+          class << self
+            alias_method :dec, :dec_gauge
+            alias_method :set, :set_gauge
+            alias_method :sub, :sub_gauge
+          end
+        else
+          class << self
+            alias_method :set, :set_counter
+          end
+        end
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def get
+        @monitor.synchronize do
+          @store
+        end
+      end
+
+      def inc
+        @monitor.synchronize do
+          @store += 1
+        end
+      end
+
+      def dec_gauge
+        @monitor.synchronize do
+          @store -= 1
+        end
+      end
+
+      def add(value)
+        @monitor.synchronize do
+          @store += value
+        end
+      end
+
+      def sub_gauge(value)
+        @monitor.synchronize do
+          @store -= value
+        end
+      end
+
+      def set_counter(value)
+        return if @store > value
+
+        @monitor.synchronize do
+          @store = value
+        end
+      end
+
+      def set_gauge(value)
+        @monitor.synchronize do
+          @store = value
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/multi_output.rb

@@ -27,6 +27,7 @@ module Fluent
       include PluginHelper::Mixin # for event_emitter
 
       helpers :event_emitter # to get router from agent, which will be supplied to child plugins
+      helpers_internal :metrics
 
       config_section :store, param_name: :stores, multi: true, required: true do
         config_argument :arg, :string, default: ''
@@ -46,11 +47,40 @@ module Fluent
 
         @counter_mutex = Mutex.new
         # TODO: well organized counters
-        @num_errors = 0
-        @emit_count = 0
-        @emit_records = 0
+        @num_errors_metrics = nil
+        @emit_count_metrics = nil
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
         # @write_count = 0
         # @rollback_count = 0
+        @enable_size_metrics = false
+      end
+
+      def num_errors
+        @num_errors_metrics.get
+      end
+
+      def emit_count
+        @emit_count_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def statistics
+        stats = {
+          'num_errors' => @num_errors_metrics.get,
+          'emit_records' => @emit_records_metrics.get,
+          'emit_count' => @emit_count_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'multi_output' => stats }
       end
 
       def multi_output?
@@ -60,6 +90,12 @@ module Fluent
       def configure(conf)
         super
 
+        @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "num_errors", help_text: "Number of count num errors")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of count emits")
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_records", help_text: "Number of emit records")
+        @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "multi_output", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+
         @stores.each do |store|
           store_conf = store.corresponding_config_element
           type = store_conf['@type']
@@ -143,12 +179,13 @@ module Fluent
       end
 
       def emit_sync(tag, es)
-        @counter_mutex.synchronize{ @emit_count += 1 }
+        @emit_count_metrics.inc
         begin
           process(tag, es)
-          @counter_mutex.synchronize{ @emit_records += es.size }
+          @emit_records_metrics.add(es.size)
+          @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
         rescue
-          @counter_mutex.synchronize{ @num_errors += 1 }
+          @num_errors_metrics.inc
           raise
         end
       end

data/lib/fluent/plugin/out_copy.rb

@@ -46,7 +46,7 @@ module Fluent::Plugin
         @ignore_errors << (store.arg.include?('ignore_error'))
         @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
       }
-      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
+      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)
         log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
       end
     end