fluentd 1.14.0 → 1.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d57a00fbe9b7fdbbac7ce75dfd9a4cf2a963b2cc4e5288dd37caf967fc5572f
-  data.tar.gz: b447f9876816029af978af3a27efd579a51794b991a79b2a57dfea3a65b0d234
+  metadata.gz: 344c4d346fb6121087e70870be6d762fdf2ed9a73dad8258c32d35cf1dec73e8
+  data.tar.gz: f2178164a14485d7714bf381ced22c2f8e7fddd64c772a07fdb0e35ce44ce8cf
 SHA512:
-  metadata.gz: ae67626874b6294ec88fc20ebd61fd34e6b809199227624eb703e9a7010d4269b87a4d10d8cd02b689e82b6be175200ff7b7d1649bfd9b1df7e864834a6a16fd
-  data.tar.gz: cc3694e775abe4aba0223592f602f68be04367c5a0eb50e86129457d604c0bf8f557b5936e44fb550fafe849556687f84e75eba5dc44f2e806652c5694eb5804
+  metadata.gz: 5eea9fb4546ce7ad032122b1f3b53750e21f7c336ffe116444dc4d0d1355edf8c0fa97a2bcd0ca48cc9e15e2ff839ee145672ec45ab569ebb403298836bc1b05
+  data.tar.gz: f58977d2cdd407531a103242e8e2223ac84dc1bfbc662cb0acb31c57a5f8df6af6e1a5d96718a5ffc31b973b895d300d7d71c6f8475b64c793fe415c70c1c832

data/CHANGELOG.md

@@ -1,3 +1,34 @@
+# v1.14.1
+
+## Release v1.14.1 - 2021/09/29
+
+### Enhancement
+
+* in_tail: Added file related metrics.
+  These metrics should be collected same as fluent-bit's in_tail.
+  https://github.com/fluent/fluentd/pull/3504
+* out_forward: Changed to use metrics mechanism for node statistics
+  https://github.com/fluent/fluentd/pull/3506
+
+### Bug fix
+
+* in_tail: Fixed a crash bug that it raise undefined method of eof? error.
+  This error may happen only when `read_bytes_limit_per_second` was specified.
+  https://github.com/fluent/fluentd/pull/3500
+* out_forward: Fixed a bug that node statistics information is not included correctly.
+  https://github.com/fluent/fluentd/pull/3503
+  https://github.com/fluent/fluentd/pull/3507
+* Fixed a error when using `@include` directive
+  It was occurred when http/https scheme URI is used in `@include` directive with Ruby 3.
+  https://github.com/fluent/fluentd/pull/3517
+* out_copy: Fixed to suppress a wrong warning for `ignore_if_prev_success`
+  It didn't work even if a user set it.
+  https://github.com/fluent/fluentd/pull/3515
+* Fixed not to output nanoseconds field of next retry time in warning log
+  Then, inappropriate labels in log are also fixed. (retry_time -> retry_times,
+  next_retry_seconds -> next_retry_time)
+  https://github.com/fluent/fluentd/pull/3518
+
 # v1.14.0
 
 ## Release v1.14.0 - 2021/08/30

data/lib/fluent/config/parser.rb

@@ -93,7 +93,7 @@ module Fluent
           basepath = '/'
           fname = path
           require 'open-uri'
-          open(uri) {|f|
+          URI.open(uri) {|f|
             Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
           }
         end

data/lib/fluent/config/v1_parser.rb

@@ -172,7 +172,7 @@ module Fluent
           require 'open-uri'
           basepath = '/'
           fname = path
-          data = open(uri) { |f| f.read }
+          data = URI.open(uri) { |f| f.read }
           data.force_encoding('UTF-8')
           ss = StringScanner.new(data)
           V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)

data/lib/fluent/plugin/in_tail.rb

@@ -38,6 +38,7 @@ module Fluent::Plugin
     helpers :timer, :event_loop, :parser, :compat_parameters
 
     RESERVED_CHARS = ['/', '*', '%'].freeze
+    MetricsInfo = Struct.new(:opened, :closed, :rotated)
 
     class WatcherSetupError < StandardError
       def initialize(msg)
@@ -57,6 +58,7 @@ module Fluent::Plugin
       @pf = nil
       @ignore_list = []
       @shutdown_start_time = nil
+      @metrics = nil
     end
 
     desc 'The paths to read. Multiple paths can be specified, separated by comma.'
@@ -191,6 +193,10 @@ module Fluent::Plugin
           @read_bytes_limit_per_second = min_bytes
         end
       end
+      opened_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_opened_total", help_text: "Total number of opened files")
+      closed_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_closed_total", help_text: "Total number of closed files")
+      rotated_file_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "files_rotated_total", help_text: "Total number of rotated files")
+      @metrics = MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
     end
 
     def configure_tag
@@ -375,7 +381,7 @@ module Fluent::Plugin
 
     def setup_watcher(target_info, pe)
       line_buffer_timer_flusher = @multiline_mode ? TailWatcher::LineBufferTimerFlusher.new(log, @multiline_flush_interval, &method(:flush_buffer)) : nil
-      tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler))
+      tw = TailWatcher.new(target_info, pe, log, @read_from_head, @follow_inodes, method(:update_watcher), line_buffer_timer_flusher, method(:io_handler), @metrics)
 
       if @enable_watch_timer
         tt = TimerTrigger.new(1, log) { tw.on_notify }
@@ -670,6 +676,19 @@ module Fluent::Plugin
       es
     end
 
+    def statistics
+      stats = super
+
+      stats = {
+        'input' => stats["input"].merge({
+          'opened_file_count' => @metrics.opened.get,
+          'closed_file_count' => @metrics.closed.get,
+          'rotated_file_count' => @metrics.rotated.get,
+        })
+      }
+      stats
+    end
+
     private
 
     def io_handler(watcher, path)
@@ -682,6 +701,7 @@ module Fluent::Plugin
         open_on_every_update: @open_on_every_update,
         from_encoding: @from_encoding,
         encoding: @encoding,
+        metrics: @metrics,
         &method(:receive_lines)
       )
     end
@@ -717,7 +737,7 @@ module Fluent::Plugin
     end
 
     class TailWatcher
-      def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build)
+      def initialize(target_info, pe, log, read_from_head, follow_inodes, update_watcher, line_buffer_timer_flusher, io_handler_build, metrics)
         @path = target_info.path
         @ino = target_info.ino
         @pe = pe || MemoryPositionEntry.new
@@ -729,6 +749,7 @@ module Fluent::Plugin
         @line_buffer_timer_flusher = line_buffer_timer_flusher
         @io_handler = nil
         @io_handler_build = io_handler_build
+        @metrics = metrics
         @watchers = []
       end
 
@@ -762,7 +783,7 @@ module Fluent::Plugin
       end
 
       def eof?
-        @io_handler.eof?
+        @io_handler.nil? || @io_handler.eof?
       end
 
       def on_notify
@@ -855,6 +876,7 @@ module Fluent::Plugin
             @log.info "detected rotation of #{@path}"
             @io_handler = io_handler
           end
+          @metrics.rotated.inc
         end
       end
 
@@ -934,7 +956,7 @@ module Fluent::Plugin
 
         attr_accessor :shutdown_timeout
 
-        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, &receive_lines)
+        def initialize(watcher, path:, read_lines_limit:, read_bytes_limit_per_second:, log:, open_on_every_update:, from_encoding: nil, encoding: nil, metrics:, &receive_lines)
           @watcher = watcher
           @path = path
           @read_lines_limit = read_lines_limit
@@ -953,6 +975,7 @@ module Fluent::Plugin
           @shutdown_timeout = SHUTDOWN_TIMEOUT
           @shutdown_mutex = Mutex.new
           @eof = false
+          @metrics = metrics
 
           @log.info "following tail of #{@path}"
         end
@@ -972,6 +995,7 @@ module Fluent::Plugin
           if @io && !@io.closed?
             @io.close
             @io = nil
+            @metrics.closed.inc
           end
         end
 
@@ -1059,6 +1083,7 @@ module Fluent::Plugin
         def open
           io = Fluent::FileWrapper.open(@path)
           io.seek(@watcher.pe.read_pos + @fifo.bytesize)
+          @metrics.opened.inc
           io
         rescue RangeError
           io.close if io
@@ -1111,6 +1136,10 @@ module Fluent::Plugin
         def opened?
           false
         end
+
+        def eof?
+          true
+        end
       end
 
       class RotateHandler

data/lib/fluent/plugin/out_copy.rb

@@ -46,7 +46,7 @@ module Fluent::Plugin
         @ignore_errors << (store.arg.include?('ignore_error'))
         @ignore_if_prev_successes << (store.arg.include?('ignore_if_prev_success'))
       }
-      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && @ignore_if_prev_successes.include?(false)
+      if @ignore_errors.uniq.size == 1 && @ignore_errors.include?(true) && !@ignore_if_prev_successes.include?(true)
         log.warn "ignore_errors are specified in all <store>, but ignore_if_prev_success is not specified. Is this intended?"
       end
     end

data/lib/fluent/plugin/out_forward.rb

@@ -167,6 +167,8 @@ module Fluent::Plugin
       @usock = nil
       @keep_alive_watcher_interval = 5 # TODO
       @suspend_flush = false
+      @healthy_nodes_count_metrics = nil
+      @registered_nodes_count_metrics = nil
     end
 
     def configure(conf)
@@ -265,6 +267,9 @@ module Fluent::Plugin
       end
 
       raise Fluent::ConfigError, "ack_response_timeout must be a positive integer" if @ack_response_timeout < 1
+      @healthy_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "healthy_nodes_count", help_text: "Number of count healthy nodes", prefer_gauge: true)
+      @registered_nodes_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "registered_nodes_count", help_text: "Number of count registered nodes", prefer_gauge: true)
+
     end
 
     def multi_workers_ready?
@@ -418,18 +423,21 @@ module Fluent::Plugin
     def statistics
       stats = super
       services = service_discovery_services
-      healthy_nodes_count = 0
-      registed_nodes_count = services.size
+      @healthy_nodes_count_metrics.set(0)
+      @registered_nodes_count_metrics.set(services.size)
       services.each do |s|
         if s.available?
-          healthy_nodes_count += 1
+          @healthy_nodes_count_metrics.inc
         end
       end
 
-      stats.merge(
-        'healthy_nodes_count' => healthy_nodes_count,
-        'registered_nodes_count' => registed_nodes_count,
-      )
+      stats = {
+        'output' => stats["output"].merge({
+          'healthy_nodes_count' => @healthy_nodes_count_metrics.get,
+          'registered_nodes_count' => @registered_nodes_count_metrics.get,
+        })
+      }
+      stats
     end
 
     # MessagePack FixArray length is 3

data/lib/fluent/plugin/output.rb

@@ -1275,7 +1275,7 @@ module Fluent
           unless @retry
             @retry = retry_state(@buffer_config.retry_randomize)
             if error
-              log.warn "failed to flush the buffer.", retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+              log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
               log.warn_backtrace error.backtrace
             end
             return
@@ -1304,11 +1304,11 @@ module Fluent
             if error
               if using_secondary
                 msg = "failed to flush the buffer with secondary output."
-                log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+                log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
                 log.warn_backtrace error.backtrace
               else
                 msg = "failed to flush the buffer."
-                log.warn msg, retry_time: @retry.steps, next_retry_seconds: @retry.next_time, chunk: chunk_id_hex, error: error
+                log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
                 log.warn_backtrace error.backtrace
               end
             end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.14.0'
+  VERSION = '1.14.1'
 
 end

data/test/plugin/in_tail/test_io_handler.rb

@@ -1,11 +1,19 @@
 require_relative '../../helper'
 
 require 'fluent/plugin/in_tail'
+require 'fluent/plugin/metrics_local'
 require 'tempfile'
 
 class IntailIOHandlerTest < Test::Unit::TestCase
   setup do
     @file = Tempfile.new('intail_io_handler').binmode
+    opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+    opened_file_metrics.configure(config_element('metrics', '', {}))
+    closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+    closed_file_metrics.configure(config_element('metrics', '', {}))
+    rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+    rotated_file_metrics.configure(config_element('metrics', '', {}))
+    @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
   end
 
   teardown do
@@ -30,7 +38,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
     end
 
     returned_lines = ''
-    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
+    r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
       returned_lines << lines.join
       true
     end
@@ -62,7 +70,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = ''
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 100, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: true, metrics: @metrics) do |lines, _watcher|
         returned_lines << lines.join
         true
       end
@@ -93,7 +101,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = []
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
         returned_lines << lines.dup
         true
       end
@@ -119,7 +127,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       returned_lines = []
-      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false) do |lines, _watcher|
+      r = Fluent::Plugin::TailInput::TailWatcher::IOHandler.new(watcher, path: @file.path, read_lines_limit: 5, read_bytes_limit_per_second: -1, log: $log, open_on_every_update: false, metrics: @metrics) do |lines, _watcher|
         returned_lines << lines.dup
         true
       end

data/test/plugin/test_in_tail.rb

@@ -1524,11 +1524,18 @@ class TailInputTest < Test::Unit::TestCase
     plugin.instance_eval do
       @pf = Fluent::Plugin::TailInput::PositionFile.load(sio, EX_FOLLOW_INODES, {}, logger: $log)
       @loop = Coolio::Loop.new
+      opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+      opened_file_metrics.configure(config_element('metrics', '', {}))
+      closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+      closed_file_metrics.configure(config_element('metrics', '', {}))
+      rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+      rotated_file_metrics.configure(config_element('metrics', '', {}))
+      @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
     end
 
     Timecop.freeze(2010, 1, 2, 3, 4, 5) do
       ex_paths.each do |target_info|
-        mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
+        mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
       end
 
       plugin.refresh_watchers
@@ -1542,7 +1549,7 @@ class TailInputTest < Test::Unit::TestCase
       path = "test/plugin/data/2010/01/20100102-030406.log"
       inode = Fluent::FileWrapper.stat(path).ino
       target_info = Fluent::Plugin::TailInput::TargetInfo.new(path, inode)
-      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything).once
+      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, false, anything, nil, anything, anything).once
       plugin.refresh_watchers
 
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
@@ -1885,13 +1892,22 @@ class TailInputTest < Test::Unit::TestCase
       config = COMMON_FOLLOW_INODE_CONFIG + config_element('', '', {"rotate_wait" => "1s", "limit_recently_modified" => "1s"})
 
       d = create_driver(config, false)
+      d.instance.instance_eval do
+        opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+        opened_file_metrics.configure(config_element('metrics', '', {}))
+        closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+        closed_file_metrics.configure(config_element('metrics', '', {}))
+        rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+        rotated_file_metrics.configure(config_element('metrics', '', {}))
+        @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
+      end
 
       File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
         f.puts "test1"
         f.puts "test2"
       }
       target_info = create_target_info("#{TMP_DIR}/tail.txt")
-      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything).once
+      mock.proxy(Fluent::Plugin::TailInput::TailWatcher).new(target_info, anything, anything, true, true, anything, nil, anything, anything).once
       d.run(shutdown: false)
       assert d.instance.instance_variable_get(:@tails)[target_info]
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.14.0
+  version: 1.14.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-30 00:00:00.000000000 Z
+date: 2021-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler