fluentd 1.15.0-x64-mingw-ucrt → 1.15.1-x64-mingw-ucrt

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5a6e43bfa63b90a8a0955bf136969601b7b82a5670d15e5a11c5754e7fb9fc0f
-  data.tar.gz: 8f0d2eb534deabed428d241abc3ff465354667cda27fb14b0aefd4473dc1fe64
+  metadata.gz: a8a4ebaf830b3781a6abcdbaf11e7f09a90b16cab77077a752b1bf9410bf4cfa
+  data.tar.gz: e5e23fbaf4d812a195b9d298118f9f826bc1715436006007434df2f1497d18a6
 SHA512:
-  metadata.gz: 2d8cd91e569687321b67ece222365241d81eccd69076e5bfaa1d65082d32741d83bfa3ee2837be73cae8f0786fdd0eefc6cb797e6aa85de8237f7dee5471dd10
-  data.tar.gz: a11bcbf8bfefa286a6f82dc679ba54959b482f55c5f71f18b76a2db4838e4b0aa2b6ecd9c0391ed6e8f9e7b16066e6bf1e0a01ecfba66e52e48e3093d8eb6069
+  metadata.gz: b11d570ebef680b7028293220b70c265fc81d932fb46e315b4ce1b6ccce6c683cddca89223500ab53c1435ce46b2dfdd37a29d01403bf5fea1001e44a00c30ca
+  data.tar.gz: e839fe3659ae89e5f0aa59cf938b9f4e8d8075148b8d526dd214e6ab5a423e9c497aad7e9b152dbb03f84b19ed26353682f842846b6a4fb770932eaf8b6c00cb

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # v1.15
 
+## Release v1.15.1 - 2022/07/27
+
+### Bug Fix
+
+* Add support for concurrent append in out_file
+  https://github.com/fluent/fluentd/pull/3808
+
+### Misc
+
+* in_tail: Show more information on skipping update_watcher
+  https://github.com/fluent/fluentd/pull/3829
+
 ## Release v1.15.0 - 2022/06/29
 
 ### Enhancement

data/lib/fluent/error.rb

@@ -28,6 +28,9 @@ module Fluent
   class InvalidRootDirectory < UnrecoverableError
   end
 
+  class InvalidLockDirectory < UnrecoverableError
+  end
+
   # For internal use
   class UncatchableError < Exception
   end

data/lib/fluent/plugin/base.rb

@@ -31,6 +31,7 @@ module Fluent
       def initialize
         @log = nil
         super
+        @fluentd_lock_dir = ENV['FLUENTD_LOCK_DIR']
         @_state = State.new(false, false, false, false, false, false, false, false, false)
         @_context_router = nil
         @_fluentd_worker_id = nil
@@ -70,6 +71,24 @@ module Fluent
         true
       end
 
+      def get_lock_path(name)
+        name = name.gsub(/[^a-zA-Z0-9]/, "_")
+        File.join(@fluentd_lock_dir, "fluentd-#{name}.lock")
+      end
+
+      def acquire_worker_lock(name)
+        if @fluentd_lock_dir.nil?
+          raise InvalidLockDirectory, "can't acquire lock because FLUENTD_LOCK_DIR isn't set"
+        end
+        lock_path = get_lock_path(name)
+        File.open(lock_path, "w") do |f|
+          f.flock(File::LOCK_EX)
+          yield
+        end
+        # Update access time to prevent tmpwatch from deleting a lock file.
+        FileUtils.touch(lock_path);
+      end
+
       def string_safe_encoding(str)
         unless str.valid_encoding?
           str = str.scrub('?')

data/lib/fluent/plugin/in_tail.rb

@@ -498,7 +498,8 @@ module Fluent::Plugin
         pe_inode = pe.read_inode
         target_info_from_position_entry = TargetInfo.new(path, pe_inode)
         unless pe_inode == @pf[target_info_from_position_entry].read_inode
-          log.debug "Skip update_watcher because watcher has been already updated by other inotify event"
+          log.warn "Skip update_watcher because watcher has been already updated by other inotify event",
+                   path: path, inode: pe.read_inode, inode_in_pos_file: @pf[target_info_from_position_entry].read_inode
           return
         end
       end

data/lib/fluent/plugin/out_file.rb

@@ -188,6 +188,10 @@ module Fluent::Plugin
         condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
         @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
       end
+
+      if @need_lock && @append && @fluentd_lock_dir.nil?
+        raise Fluent::InvalidLockDirectory, "must set FLUENTD_LOCK_DIR on multi-worker append mode"
+      end
     end
 
     def multi_workers_ready?
@@ -217,7 +221,13 @@ module Fluent::Plugin
                end
 
       if @append
-        writer.call(path, chunk)
+        if @need_lock
+          acquire_worker_lock(path) do
+            writer.call(path, chunk)
+          end
+        else
+          writer.call(path, chunk)
+        end
       else
         find_filepath_available(path, with_lock: @need_lock) do |actual_path|
           writer.call(actual_path, chunk)

data/lib/fluent/supervisor.rb

@@ -875,7 +875,11 @@ module Fluent
       se = ServerEngine.create(ServerModule, WorkerModule){
         Fluent::Supervisor.load_config(@config_path, params)
       }
-      se.run
+
+      Dir.mktmpdir("fluentd-lock-") do |lock_dir|
+        ENV['FLUENTD_LOCK_DIR'] = lock_dir
+        se.run
+      end
     end
 
     def install_main_process_signal_handlers

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.15.0'
+  VERSION = '1.15.1'
 
 end

data/test/plugin/test_base.rb

@@ -1,4 +1,5 @@
 require_relative '../helper'
+require 'tmpdir'
 require 'fluent/plugin/base'
 
 module FluentPluginBaseTest
@@ -112,4 +113,37 @@ class BaseTest < Test::Unit::TestCase
     assert_equal 1, logger.logs.size
     assert{ logger.logs.first.include?("invalid byte sequence is replaced in ") }
   end
+
+  test 'generates worker lock path safely' do
+    Dir.mktmpdir("test-fluentd-lock-") do |lock_dir|
+      ENV['FLUENTD_LOCK_DIR'] = lock_dir
+      p = FluentPluginBaseTest::DummyPlugin.new
+      path = p.get_lock_path("Aa\\|=~/_123");
+
+      assert_equal lock_dir, File.dirname(path)
+      assert_equal "fluentd-Aa______123.lock", File.basename(path)
+    end
+  end
+
+  test 'can acquire inter-worker locking' do
+    Dir.mktmpdir("test-fluentd-lock-") do |lock_dir|
+      ENV['FLUENTD_LOCK_DIR'] = lock_dir
+      p = FluentPluginBaseTest::DummyPlugin.new
+      lock_path = p.get_lock_path("test_base")
+
+      p.acquire_worker_lock("test_base") do
+        # With LOCK_NB set, flock() returns `false` when the
+        # file is already locked.
+        File.open(lock_path, "w") do |f|
+          assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)
+        end
+      end
+
+      # Lock should be release by now. In that case, flock
+      # must return 0.
+      File.open(lock_path, "w") do |f|
+        assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.15.0
+  version: 1.15.1
 platform: x64-mingw-ucrt
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-29 00:00:00.000000000 Z
+date: 2022-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler