fluentd 1.15.0-x86-mingw32 → 1.15.1-x86-mingw32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 941e114cec61513919b4fbd711bfc2bb58254a75d8f5f76028027f432cc89116
-  data.tar.gz: 8ec3c41476caf98e3a82ef624786d1fb9900361480326476141d8a0eb58c19d4
+  metadata.gz: 14de5d3497bb33697ce746637f710eb09e7504142a90fcdf80b61dd6de9997ed
+  data.tar.gz: 4273c9f6178dc0bb04ac79433c12fc60a0b2ebe5a5876f7fe39b1c4278cae2ad
 SHA512:
-  metadata.gz: 923a7cc3e4449082c7e9de20e6fb39cd9d2e42f481bece900e779a8a711d961b81c5fe38ff34f907473ef8a35e9ddff1d33d6e9b42d63377c147ddd88c9ba368
-  data.tar.gz: 2f4e5a26e4ab6d82784aad47784c3371c0b59920222301d3ef5ec105807e6bcd58c6153d2af0316842b1f62ca9f68aaa7c7f811f0b8b8d54e1d7bbef101ed1a4
+  metadata.gz: 689134be721ca4af6bf05f278dea10687f24eef985978032be425fdef1a43056f903d04efb8aaf81efc4418ece9127da88e39546643dbb872c962410528dc3f3
+  data.tar.gz: bd4ef3aacb731e0f8eeb013b52160e40225f57d15f698fbb8b034e79fbfff2ae3567c76bb5f7d9302b01fff1dca28537886d254a94e5a3af4d913c580636a35d

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # v1.15
 
+## Release v1.15.1 - 2022/07/27
+
+### Bug Fix
+
+* Add support for concurrent append in out_file
+  https://github.com/fluent/fluentd/pull/3808
+
+### Misc
+
+* in_tail: Show more information on skipping update_watcher
+  https://github.com/fluent/fluentd/pull/3829
+
 ## Release v1.15.0 - 2022/06/29
 
 ### Enhancement

data/lib/fluent/error.rb

@@ -28,6 +28,9 @@ module Fluent
   class InvalidRootDirectory < UnrecoverableError
   end
 
+  class InvalidLockDirectory < UnrecoverableError
+  end
+
   # For internal use
   class UncatchableError < Exception
   end

data/lib/fluent/plugin/base.rb

@@ -31,6 +31,7 @@ module Fluent
       def initialize
         @log = nil
         super
+        @fluentd_lock_dir = ENV['FLUENTD_LOCK_DIR']
         @_state = State.new(false, false, false, false, false, false, false, false, false)
         @_context_router = nil
         @_fluentd_worker_id = nil
@@ -70,6 +71,24 @@ module Fluent
         true
       end
 
+      def get_lock_path(name)
+        name = name.gsub(/[^a-zA-Z0-9]/, "_")
+        File.join(@fluentd_lock_dir, "fluentd-#{name}.lock")
+      end
+
+      def acquire_worker_lock(name)
+        if @fluentd_lock_dir.nil?
+          raise InvalidLockDirectory, "can't acquire lock because FLUENTD_LOCK_DIR isn't set"
+        end
+        lock_path = get_lock_path(name)
+        File.open(lock_path, "w") do |f|
+          f.flock(File::LOCK_EX)
+          yield
+        end
+        # Update access time to prevent tmpwatch from deleting a lock file.
+        FileUtils.touch(lock_path);
+      end
+
       def string_safe_encoding(str)
         unless str.valid_encoding?
           str = str.scrub('?')

data/lib/fluent/plugin/in_tail.rb

@@ -498,7 +498,8 @@ module Fluent::Plugin
         pe_inode = pe.read_inode
         target_info_from_position_entry = TargetInfo.new(path, pe_inode)
         unless pe_inode == @pf[target_info_from_position_entry].read_inode
-          log.debug "Skip update_watcher because watcher has been already updated by other inotify event"
+          log.warn "Skip update_watcher because watcher has been already updated by other inotify event",
+                   path: path, inode: pe.read_inode, inode_in_pos_file: @pf[target_info_from_position_entry].read_inode
           return
         end
       end

data/lib/fluent/plugin/out_file.rb

@@ -188,6 +188,10 @@ module Fluent::Plugin
         condition = Gem::Dependency.new('', [">= 2.7.0", "< 3.1.0"])
         @need_ruby_on_macos_workaround = true if condition.match?('', RUBY_VERSION)
       end
+
+      if @need_lock && @append && @fluentd_lock_dir.nil?
+        raise Fluent::InvalidLockDirectory, "must set FLUENTD_LOCK_DIR on multi-worker append mode"
+      end
     end
 
     def multi_workers_ready?
@@ -217,7 +221,13 @@ module Fluent::Plugin
                end
 
       if @append
-        writer.call(path, chunk)
+        if @need_lock
+          acquire_worker_lock(path) do
+            writer.call(path, chunk)
+          end
+        else
+          writer.call(path, chunk)
+        end
       else
         find_filepath_available(path, with_lock: @need_lock) do |actual_path|
           writer.call(actual_path, chunk)

data/lib/fluent/supervisor.rb

@@ -875,7 +875,11 @@ module Fluent
       se = ServerEngine.create(ServerModule, WorkerModule){
         Fluent::Supervisor.load_config(@config_path, params)
       }
-      se.run
+
+      Dir.mktmpdir("fluentd-lock-") do |lock_dir|
+        ENV['FLUENTD_LOCK_DIR'] = lock_dir
+        se.run
+      end
     end
 
     def install_main_process_signal_handlers

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.15.0'
+  VERSION = '1.15.1'
 
 end

data/test/plugin/test_base.rb

@@ -1,4 +1,5 @@
 require_relative '../helper'
+require 'tmpdir'
 require 'fluent/plugin/base'
 
 module FluentPluginBaseTest
@@ -112,4 +113,37 @@ class BaseTest < Test::Unit::TestCase
     assert_equal 1, logger.logs.size
     assert{ logger.logs.first.include?("invalid byte sequence is replaced in ") }
   end
+
+  test 'generates worker lock path safely' do
+    Dir.mktmpdir("test-fluentd-lock-") do |lock_dir|
+      ENV['FLUENTD_LOCK_DIR'] = lock_dir
+      p = FluentPluginBaseTest::DummyPlugin.new
+      path = p.get_lock_path("Aa\\|=~/_123");
+
+      assert_equal lock_dir, File.dirname(path)
+      assert_equal "fluentd-Aa______123.lock", File.basename(path)
+    end
+  end
+
+  test 'can acquire inter-worker locking' do
+    Dir.mktmpdir("test-fluentd-lock-") do |lock_dir|
+      ENV['FLUENTD_LOCK_DIR'] = lock_dir
+      p = FluentPluginBaseTest::DummyPlugin.new
+      lock_path = p.get_lock_path("test_base")
+
+      p.acquire_worker_lock("test_base") do
+        # With LOCK_NB set, flock() returns `false` when the
+        # file is already locked.
+        File.open(lock_path, "w") do |f|
+          assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)
+        end
+      end
+
+      # Lock should be release by now. In that case, flock
+      # must return 0.
+      File.open(lock_path, "w") do |f|
+        assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.15.0
+  version: 1.15.1
 platform: x86-mingw32
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-29 00:00:00.000000000 Z
+date: 2022-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler