fluentd 1.16.1 → 1.16.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e88bbbd20e75035745c46d2a6d86395113244ebdd773071c10dd1093b20216db
-  data.tar.gz: cbf603733cd993bfe412328c75ffb82fd068a1779eb2a2750f0caca84f7da845
+  metadata.gz: f8a64e2943ee3f6abc8c90018a85205b0bd54a8fc5f367da3d5e9ce53f03db4f
+  data.tar.gz: 0d06e0bec37b228eea16a92353a70b0198a9f6b3fef88627898f6ef16ae2322b
 SHA512:
-  metadata.gz: a9b58aefd54f71282d1bbe40d3ed6def9cd45c4b82e4b5c069004af89530cde348fa5e8af63767a13813aea0e6bac7627d9bf86df4e910117f4d17d4e565b767
-  data.tar.gz: 64e0de5f1ffa604a0b7e1087145f62fa8de0dc766bcedf370f037531a2d0f817516bb91d7c3eb45aaa9b02471cf1617f7868754b559faa7eac02f0f641f16925
+  metadata.gz: 90e357ccd0f4c02013739c538585d3a1b79f3f3d423efc4bd668496e7b318b87358e3bc0618393b85ce07885445734dadbdc4112a7cd082be4aa89d738d94066
+  data.tar.gz: bfa4f0817153e79538b24af9af0a840cd6ecd6a8ecc2a0a1536a5268b7d29aae7671371181050dac4ba670538d9eb40efbbd9634936217caf7b3ecec090b67e1

data/.github/workflows/linux-test.yaml

@@ -2,9 +2,9 @@ name: Testing on Ubuntu
 
 on:
   push:
-    branches: [master]
+    branches: [master, v1.16]
   pull_request:
-    branches: [master]
+    branches: [master, v1.16]
 
 jobs:
   test:

data/.github/workflows/macos-test.yaml

@@ -2,9 +2,9 @@ name: Testing on macOS
 
 on:
   push:
-    branches: [master]
+    branches: [master, v1.16]
   pull_request:
-    branches: [master]
+    branches: [master, v1.16]
 
 jobs:
   test:

data/.github/workflows/windows-test.yaml

@@ -2,9 +2,9 @@ name: Testing on Windows
 
 on:
   push:
-    branches: [master]
+    branches: [master, v1.16]
   pull_request:
-    branches: [master]
+    branches: [master, v1.16]
 
 jobs:
   test:

data/CHANGELOG.md

@@ -1,5 +1,55 @@
 # v1.16
 
+## Release v1.16.3 - 2023/11/14
+
+### Bug Fix
+
+* in_tail: Fix a stall bug on !follow_inode case
+  https://github.com/fluent/fluentd/pull/4327
+* in_tail: add warning for silent stop on !follow_inodes case
+  https://github.com/fluent/fluentd/pull/4339
+* Buffer: Fix NoMethodError with empty unstaged chunk arrays
+  https://github.com/fluent/fluentd/pull/4303
+* Fix for rotate_age where Fluentd passes as Symbol
+  https://github.com/fluent/fluentd/pull/4311
+
+## Release v1.16.2 - 2023/07/14
+
+### Bug Fix
+
+* in_tail: Fix new watcher is wrongly detached on rotation when `follow_inodes`,
+  which causes stopping tailing the file
+  https://github.com/fluent/fluentd/pull/4208
+* in_tail: Prevent wrongly unwatching when `follow_inodes`, which causes log
+  duplication
+  https://github.com/fluent/fluentd/pull/4237
+* in_tail: Fix warning log about overwriting entry when `follow_inodes`
+  https://github.com/fluent/fluentd/pull/4214
+* in_tail: Ensure to discard TailWatcher with missing target when `follow_inodes`
+  https://github.com/fluent/fluentd/pull/4239
+* MessagePackFactory: Make sure to reset local unpacker to prevent received
+  broken data from affecting other receiving data
+  https://github.com/fluent/fluentd/pull/4178
+* Fix failure to launch Fluentd on Windows when the log path isn't specified in
+  the command line
+  https://github.com/fluent/fluentd/pull/4188
+* logger: Prevent growing cache size of `ignore_same_log_interval` unlimitedly
+  https://github.com/fluent/fluentd/pull/4229
+* Update sigdump to 0.2.5 to fix wrong value of object counts
+  https://github.com/fluent/fluentd/pull/4225
+
+### Misc
+
+* in_tail: Check detaching inode when `follow_inodes`
+  https://github.com/fluent/fluentd/pull/4191
+* in_tail: Add debug log for pos file compaction
+  https://github.com/fluent/fluentd/pull/4228
+* Code improvements detected by RuboCop Performance
+  https://github.com/fluent/fluentd/pull/4201
+  https://github.com/fluent/fluentd/pull/4210
+* Add notice for unused argument `unpacker` of `ChunkMessagePackEventStreamer.each`
+  https://github.com/fluent/fluentd/pull/4159
+
 ## Release v1.16.1 - 2023/04/17
 
 ### Enhancement

data/fluentd.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("cool.io", [">= 1.4.5", "< 2.0.0"])
   gem.add_runtime_dependency("serverengine", [">= 2.3.2", "< 3.0.0"])
   gem.add_runtime_dependency("http_parser.rb", [">= 0.5.1", "< 0.9.0"])
-  gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
+  gem.add_runtime_dependency("sigdump", ["~> 0.2.5"])
   gem.add_runtime_dependency("tzinfo", [">= 1.0", "< 3.0"])
   gem.add_runtime_dependency("tzinfo-data", ["~> 1.0"])
   gem.add_runtime_dependency("strptime", [">= 0.2.4", "< 1.0.0"])

data/lib/fluent/command/ctl.rb

@@ -92,7 +92,7 @@ module Fluent
 
     def call
       if Fluent.windows?
-        if @pid_or_svcname =~ /^[0-9]+$/
+        if /^[0-9]+$/.match?(@pid_or_svcname)
           # Use as PID
           return call_windows_event(@command, "fluentd_#{@pid_or_svcname}")
         end
@@ -172,7 +172,7 @@ module Fluent
         usage("PID or SVCNAME isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
       else
         usage("PID isn't specified!") if @pid_or_svcname.nil? || @pid_or_svcname.empty?
-        usage("Invalid PID: #{pid}") unless @pid_or_svcname =~ /^[0-9]+$/
+        usage("Invalid PID: #{pid}") unless /^[0-9]+$/.match?(@pid_or_svcname)
       end
     end
   end

data/lib/fluent/command/plugin_config_formatter.rb

@@ -61,7 +61,7 @@ class FluentPluginConfigFormatter
     @plugin.class.ancestors.reverse_each do |plugin_class|
       next unless plugin_class.respond_to?(:dump_config_definition)
       unless @verbose
-        next if plugin_class.name =~ /::PluginHelper::/
+        next if /::PluginHelper::/.match?(plugin_class.name)
       end
       dumped_config_definition = plugin_class.dump_config_definition
       dumped_config[plugin_class.name] = dumped_config_definition unless dumped_config_definition.empty?

data/lib/fluent/config/dsl.rb

@@ -110,7 +110,7 @@ module Fluent
 
         def include(*args)
           ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
-          if args.first =~ /\.rb$/
+          if /\.rb$/.match?(args.first)
             path = File.expand_path(args.first)
             data = File.read(path)
             self.instance_eval(data, path)

data/lib/fluent/config/v1_parser.rb

@@ -150,8 +150,8 @@ module Fluent
       def eval_include(attrs, elems, uri)
         # replace space(s)(' ') with '+' to prevent invalid uri due to space(s).
         # See: https://github.com/fluent/fluentd/pull/2780#issuecomment-576081212
-        u = URI.parse(uri.gsub(/ /, '+'))
-        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.gsub(/ /, '+') # file path
+        u = URI.parse(uri.tr(' ', '+'))
+        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri.tr(' ', '+') # file path
           # When the Windows absolute path then u.scheme.length == 1
           # e.g. C:
           path = URI.decode_www_form_component(u.path)

data/lib/fluent/counter/server.rb

@@ -27,7 +27,7 @@ module Fluent
       DEFAULT_PORT = 24321
 
       def initialize(name, opt = {})
-        raise 'Counter server name is invalid' unless Validator::VALID_NAME =~ name
+        raise 'Counter server name is invalid' unless Validator::VALID_NAME.match?(name)
         @name = name
         @opt = opt
         @addr = @opt[:addr] || DEFAULT_ADDR

data/lib/fluent/counter/validator.rb

@@ -82,7 +82,7 @@ module Fluent
           raise Fluent::Counter::InvalidParams.new('The type of `key` should be String')
         end
 
-        unless VALID_NAME =~ name
+        unless VALID_NAME.match?(name)
           raise Fluent::Counter::InvalidParams.new('`key` is the invalid format')
         end
       end
@@ -92,7 +92,7 @@ module Fluent
           raise Fluent::Counter::InvalidParams.new('The type of `scope` should be String')
         end
 
-        unless VALID_SCOPE_NAME =~ name
+        unless VALID_SCOPE_NAME.match?(name)
           raise Fluent::Counter::InvalidParams.new('`scope` is the invalid format')
         end
       end
@@ -109,7 +109,7 @@ module Fluent
           raise Fluent::Counter::InvalidParams.new('The type of `name` should be String')
         end
 
-        unless VALID_NAME =~ name
+        unless VALID_NAME.match?(name)
           raise Fluent::Counter::InvalidParams.new("`name` is the invalid format")
         end
       end

data/lib/fluent/engine.rb

@@ -68,7 +68,7 @@ module Fluent
     end
 
     def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
-      if fname =~ /\.rb$/
+      if /\.rb$/.match?(fname)
         require 'fluent/config/dsl'
         Config::DSL::Parser.parse(io, File.join(basepath, fname))
       else

data/lib/fluent/event.rb

@@ -308,11 +308,15 @@ module Fluent
   end
 
   module ChunkMessagePackEventStreamer
-    # chunk.extend(ChunkEventStreamer)
+    # chunk.extend(ChunkMessagePackEventStreamer)
     #  => chunk.each{|time, record| ... }
     def each(unpacker: nil, &block)
+      # Note: If need to use `unpacker`, then implement it,
+      # e.g., `unpacker.feed_each(io.read, &block)` (Not tested)
+      raise NotImplementedError, "'unpacker' argument is not implemented." if unpacker
+
       open do |io|
-        (unpacker || Fluent::MessagePackFactory.msgpack_unpacker(io)).each(&block)
+        Fluent::MessagePackFactory.msgpack_unpacker(io).each(&block)
       end
       nil
     end

data/lib/fluent/log.rb

@@ -51,6 +51,8 @@ module Fluent
     LOG_TYPES = [LOG_TYPE_SUPERVISOR, LOG_TYPE_WORKER0, LOG_TYPE_DEFAULT].freeze
     LOG_ROTATE_AGE = %w(daily weekly monthly)
 
+    IGNORE_SAME_LOG_MAX_CACHE_SIZE = 1000 # If need, make this an option of system config.
+
     def self.str_to_level(log_level_str)
       case log_level_str.downcase
       when "trace" then LEVEL_TRACE
@@ -477,6 +479,13 @@ module Fluent
           false
         end
       else
+        if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
+          cached_log.reject! do |_, cached_time|
+            (time - cached_time) > @ignore_same_log_interval
+          end
+        end
+        # If the size is still over, we have no choice but to clear it.
+        cached_log.clear if cached_log.size >= IGNORE_SAME_LOG_MAX_CACHE_SIZE
         cached_log[message] = time
         false
       end

data/lib/fluent/match.rb

@@ -115,7 +115,7 @@ module Fluent
           stack.last << regex.pop
           regex.push ''
 
-        elsif c =~ /[a-zA-Z0-9_]/
+        elsif /[a-zA-Z0-9_]/.match?(c)
           regex.last << c
 
         else

data/lib/fluent/msgpack_factory.rb

@@ -100,7 +100,12 @@ module Fluent
     end
 
     def self.thread_local_msgpack_unpacker
-      Thread.current[:local_msgpack_unpacker] ||= MessagePackFactory.engine_factory.unpacker
+      unpacker = Thread.current[:local_msgpack_unpacker]
+      if unpacker.nil?
+        return Thread.current[:local_msgpack_unpacker] = MessagePackFactory.engine_factory.unpacker
+      end
+      unpacker.reset
+      unpacker
     end
   end
 end

data/lib/fluent/plugin/base.rb

@@ -190,7 +190,7 @@ module Fluent
           # Thread::Backtrace::Location#path returns base filename or absolute path.
           # #absolute_path returns absolute_path always.
           # https://bugs.ruby-lang.org/issues/12159
-          if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+          if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
             return true
           end
         end

data/lib/fluent/plugin/buffer.rb

@@ -417,7 +417,7 @@ module Fluent
             if c.staged? && (enqueue || chunk_size_full?(c))
               m = c.metadata
               enqueue_chunk(m)
-              if unstaged_chunks[m]
+              if unstaged_chunks[m] && !unstaged_chunks[m].empty?
                 u = unstaged_chunks[m].pop
                 u.synchronize do
                   if u.unstaged? && !chunk_size_full?(u)

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -316,7 +316,7 @@ module Fluent::Plugin
         end
 
         (Object.instance_methods).each do |m|
-          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/
+          undef_method m unless /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/.match?(m.to_s)
         end
       end
     end

data/lib/fluent/plugin/in_forward.rb

@@ -430,7 +430,7 @@ module Fluent::Plugin
       end
       _ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
 
-      node = @nodes.select{|n| n[:address].include?(remote_addr) rescue false }.first
+      node = @nodes.find{|n| n[:address].include?(remote_addr) rescue false }
       if !node && !@security.allow_anonymous_source
         log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
         return false, "anonymous source host '#{remote_addr}' denied", nil

data/lib/fluent/plugin/in_http.rb

@@ -428,7 +428,7 @@ module Fluent::Plugin
         @content_type = ""
         @content_encoding = ""
         headers.each_pair {|k,v|
-          @env["HTTP_#{k.gsub('-','_').upcase}"] = v
+          @env["HTTP_#{k.tr('-','_').upcase}"] = v
           case k
           when /\AExpect\z/i
             expect = v
@@ -439,9 +439,9 @@ module Fluent::Plugin
           when /\AContent-Encoding\Z/i
             @content_encoding = v
           when /\AConnection\Z/i
-            if v =~ /close/i
+            if /close/i.match?(v)
               @keep_alive = false
-            elsif v =~ /Keep-alive/i
+            elsif /Keep-alive/i.match?(v)
               @keep_alive = true
             end
           when /\AOrigin\Z/i
@@ -566,16 +566,16 @@ module Fluent::Plugin
 
         if @format_name != 'default'
           params[EVENT_RECORD_PARAMETER] = @body
-        elsif @content_type =~ /^application\/x-www-form-urlencoded/
+        elsif /^application\/x-www-form-urlencoded/.match?(@content_type)
           params.update WEBrick::HTTPUtils.parse_query(@body)
         elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
           boundary = WEBrick::HTTPUtils.dequote($1)
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
-        elsif @content_type =~ /^application\/json/
+        elsif /^application\/json/.match?(@content_type)
           params['json'] = @body
-        elsif @content_type =~ /^application\/msgpack/
+        elsif /^application\/msgpack/.match?(@content_type)
           params['msgpack'] = @body
-        elsif @content_type =~ /^application\/x-ndjson/
+        elsif /^application\/x-ndjson/.match?(@content_type)
           params['ndjson'] = @body
         end
         path_info = uri.path
@@ -585,7 +585,7 @@ module Fluent::Plugin
           query_params = WEBrick::HTTPUtils.parse_query(uri.query)
 
           query_params.each_pair {|k,v|
-            params["QUERY_#{k.gsub('-','_').upcase}"] = v
+            params["QUERY_#{k.tr('-','_').upcase}"] = v
           }
         end
 

data/lib/fluent/plugin/in_sample.rb

@@ -64,7 +64,7 @@ module Fluent::Plugin
     def configure(conf)
       super
       @sample_index = 0
-      config = conf.elements.select{|e| e.name == 'storage' }.first
+      config = conf.elements.find{|e| e.name == 'storage' }
       @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
     end
 

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -53,10 +53,16 @@ module Fluent::Plugin
         }
       end
 
+      def unwatch_removed_targets(existing_targets)
+        @map.reject { |key, entry|
+          existing_targets.key?(key)
+        }.each_key { |key|
+          unwatch_key(key)
+        }
+      end
+
       def unwatch(target_info)
-        if (entry = @map.delete(@follow_inodes ? target_info.ino : target_info.path))
-          entry.update_pos(UNWATCHED_POSITION)
-        end
+        unwatch_key(@follow_inodes ? target_info.ino : target_info.path)
       end
 
       def load(existing_targets = nil)
@@ -96,6 +102,7 @@ module Fluent::Plugin
         end
 
         entries = fetch_compacted_entries
+        @logger&.debug "Compacted entries: ", entries.keys
 
         @file_mutex.synchronize do
           if last_modified == @file.mtime && size == @file.size
@@ -117,17 +124,31 @@ module Fluent::Plugin
 
       private
 
+      def unwatch_key(key)
+        if (entry = @map.delete(key))
+          entry.update_pos(UNWATCHED_POSITION)
+        end
+      end
+
       def compact(existing_targets = nil)
         @file_mutex.synchronize do
-          entries = fetch_compacted_entries(existing_targets).values.map(&:to_entry_fmt)
+          entries = fetch_compacted_entries
+          @logger&.debug "Compacted entries: ", entries.keys
+
+          if existing_targets
+            entries = remove_deleted_files_entries(entries, existing_targets)
+            @logger&.debug "Remove missing entries.",
+                           existing_targets: existing_targets.keys,
+                           entries_after_removing: entries.keys
+          end
 
           @file.pos = 0
           @file.truncate(0)
-          @file.write(entries.join)
+          @file.write(entries.values.map(&:to_entry_fmt).join)
         end
       end
 
-      def fetch_compacted_entries(existing_targets = nil)
+      def fetch_compacted_entries
         entries = {}
 
         @file.pos = 0
@@ -145,31 +166,24 @@ module Fluent::Plugin
           if pos == UNWATCHED_POSITION
             @logger.debug "Remove unwatched line from pos_file: #{line}" if @logger
           else
-            if entries.include?(path)
-              @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
-            end
-
             if @follow_inodes
+              @logger&.warn("#{path} (inode: #{ino}) already exists. use latest one: deleted #{entries[ino]}") if entries.include?(ino)
               entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             else
+              @logger&.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if entries.include?(path)
               entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             end
             file_pos += line.size
           end
         end
 
-        entries = remove_deleted_files_entries(entries, existing_targets)
         entries
       end
 
       def remove_deleted_files_entries(existent_entries, existing_targets)
-        if existing_targets
-          existent_entries.select { |path_or_ino|
-            existing_targets.key?(path_or_ino)
-          }
-        else
-          existent_entries
-        end
+        existent_entries.select { |path_or_ino|
+          existing_targets.key?(path_or_ino)
+        }
       end
     end