logstash-output-s3 4.4.0 → 4.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7fe328033b222f10871103a51430bfe6f6a269f15460f70e72e423b6b135927e
-  data.tar.gz: e845c48187f640a948624f7da5bf3b6cec3eee3b25c81f0024014494a71363e5
+  metadata.gz: 1915e4499e1950b269e287e9bbec3d88efca2f274a390f99949c27af5e2da105
+  data.tar.gz: 6a464adee35655b6f06f5fdf8dea9461af550b51912c4edbe46fb0d7c656774c
 SHA512:
-  metadata.gz: 9c5a89d3551c5d199b0b289c17b0a36138c4efbb4c4191610167039aab4692588a0c9fc17a158af788ca704d9fbd454a9f2de501b2aa34d17f5aee5820345829
-  data.tar.gz: 989eef2c121767e315199177f0d672bea2b2ff18fcb6a5a9acf21e8b813adadbcd35ab0ce221eff4a633bdaf735e9267b7943a3fbefb788dee42bdc4d1df293a
+  metadata.gz: 31ff8fa96f60dffc824f4881ec162c2db15a707ddc43dee4ff81a043ad23dd9296dd5bafaedcef26a27e053f6db8156c918da4c68ec95b4dd496783218595cbe
+  data.tar.gz: 67e830c249b6a5042b442b89b0b0e0b5b7d88dd6064119d0b9816a680aff1d1ae975addcc02b6f5cbf8a36fc5faf2d5e9b6af164e7a471f08f8f8ecd010188a8

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.4.1
+  - Fixes several closely-related race conditions that could cause plugin crashes or data-loss [#252](https://github.com/logstash-plugins/logstash-output-s3/pull/252)
+    - race condition in initializing a prefix could cause one or more local temp files to be abandoned and only recovered after next pipeline start
+    - race condition in stale watcher could cause the plugin to crash when working with a stale (empty) file that had been deleted
+    - race condition in stale watcher could cause a non-empty file to be deleted if bytes were written to it after it was detected as stale
+
 ## 4.4.0
   - Logstash recovers corrupted gzip and uploads to S3 [#249](https://github.com/logstash-plugins/logstash-output-s3/pull/249)
 

data/lib/logstash/outputs/s3/file_repository.rb

@@ -15,8 +15,9 @@ module LogStash
         class PrefixedValue
           def initialize(file_factory, stale_time)
             @file_factory = file_factory
-            @lock = Mutex.new
+            @lock = Monitor.new
             @stale_time = stale_time
+            @is_deleted = false
           end
 
           def with_lock
@@ -34,7 +35,14 @@ module LogStash
           end
 
           def delete!
-            with_lock{ |factory| factory.current.delete! }
+            with_lock do |factory|
+              factory.current.delete!
+              @is_deleted = true
+            end
+          end
+
+          def deleted?
+            with_lock { |_| @is_deleted }
           end
         end
 
@@ -72,17 +80,52 @@ module LogStash
         end
 
         def each_files
-          @prefixed_factories.values.each do |prefixed_file|
-            prefixed_file.with_lock { |factory| yield factory.current }
+          each_factory(keys) do |factory|
+            yield factory.current
           end
         end
 
-        # Return the file factory
+        ##
+        # Yields the file factory while the current thread has exclusive access to it, creating a new
+        # one if one does not exist or if the current one is being reaped by the stale watcher.
+        # @param prefix_key [String]: the prefix key
+        # @yieldparam factory [TemporaryFileFactory]: a temporary file factory that this thread has exclusive access to
+        # @yieldreturn [Object]: a value to return; should NOT be the factory, which should be contained by the exclusive access scope.
+        # @return [Object]: the value returned by the provided block
         def get_factory(prefix_key)
-          prefix_val = @prefixed_factories.fetch_or_store(prefix_key) { @factory_initializer.create_value(prefix_key) }
+
+          # fast-path: if factory exists and is not deleted, yield it with exclusive access and return
+          prefix_val = @prefixed_factories.get(prefix_key)
+          prefix_val&.with_lock do |factory|
+            # intentional local-jump to ensure deletion detection
+            # is done inside the exclusive access.
+            return yield(factory) unless prefix_val.deleted?
+          end
+
+          # slow-path:
+          # the Concurrent::Map#get operation is lock-free, but may have returned an entry that was being deleted by
+          # another thread (such as via stale detection). If we failed to retrieve a value, or retrieved one that had
+          # been marked deleted, use the atomic Concurrent::Map#compute to retrieve a non-deleted entry.
+          prefix_val = @prefixed_factories.compute(prefix_key) do |existing|
+            existing && !existing.deleted? ? existing : @factory_initializer.create_value(prefix_key)
+          end
           prefix_val.with_lock { |factory| yield factory }
         end
 
+        ##
+        # Yields each non-deleted file factory while the current thread has exclusive access to it.
+        # @param prefixes [Array<String>]: the prefix keys
+        # @yieldparam factory [TemporaryFileFactory]
+        # @return [void]
+        def each_factory(prefixes)
+          prefixes.each do |prefix_key|
+            prefix_val = @prefixed_factories.get(prefix_key)
+            prefix_val&.with_lock do |factory|
+              yield factory unless prefix_val.deleted?
+            end
+          end
+        end
+
         def get_file(prefix_key)
           get_factory(prefix_key) { |factory| yield factory.current }
         end
@@ -95,10 +138,21 @@ module LogStash
           @prefixed_factories.size
         end
 
-        def remove_stale(k, v)
-          if v.stale?
-            @prefixed_factories.delete_pair(k, v)
-            v.delete!
+        def remove_if_stale(prefix_key)
+          # we use the ATOMIC `Concurrent::Map#compute_if_present` to atomically
+          # detect the staleness, mark a stale prefixed factory as deleted, and delete from the map.
+          @prefixed_factories.compute_if_present(prefix_key) do |prefixed_factory|
+            # once we have retrieved an instance, we acquire exclusive access to it
+            # for stale detection, marking it as deleted before releasing the lock
+            # and causing it to become deleted from the map.
+            prefixed_factory.with_lock do |_|
+              if prefixed_factory.stale?
+                prefixed_factory.delete! # mark deleted to prevent reuse
+                nil # cause deletion
+              else
+                prefixed_factory # keep existing
+              end
+            end
           end
         end
 
@@ -106,7 +160,9 @@ module LogStash
           @stale_sweeper = Concurrent::TimerTask.new(:execution_interval => @sweeper_interval) do
             LogStash::Util.set_thread_name("S3, Stale factory sweeper")
 
-            @prefixed_factories.each { |k, v| remove_stale(k,v) }
+            @prefixed_factories.keys.each do |prefix|
+              remove_if_stale(prefix)
+            end
           end
 
           @stale_sweeper.execute

data/lib/logstash/outputs/s3.rb

@@ -266,6 +266,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
     @logger.debug("Uploading current workspace")
 
+    @file_repository.shutdown # stop stale sweeps
+
     # The plugin has stopped receiving new events, but we still have
     # data on disk, lets make sure it get to S3.
     # If Logstash get interrupted, the `restore_from_crash` (when set to true) method will pickup
@@ -275,8 +277,6 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       upload_file(file)
     end
 
-    @file_repository.shutdown
-
     @uploader.stop # wait until all the current upload are complete
     @crash_uploader.stop if @restore # we might have still work to do for recovery so wait until we are done
   end
@@ -344,22 +344,22 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def rotate_if_needed(prefixes)
-    prefixes.each do |prefix|
-      # Each file access is thread safe,
-      # until the rotation is done then only
-      # one thread has access to the resource.
-      @file_repository.get_factory(prefix) do |factory|
-        temp_file = factory.current
-
-        if @rotation.rotate?(temp_file)
-          @logger.debug? && @logger.debug("Rotate file",
-                                          :key => temp_file.key,
-                                          :path => temp_file.path,
-                                          :strategy => @rotation.class.name)
-
-          upload_file(temp_file)
-          factory.rotate!
-        end
+    # Each file access is thread safe,
+    # until the rotation is done then only
+    # one thread has access to the resource.
+    @file_repository.each_factory(prefixes) do |factory|
+      # we have exclusive access to the one-and-only
+      # prefix WRAPPER for this factory.
+      temp_file = factory.current
+
+      if @rotation.rotate?(temp_file)
+        @logger.debug? && @logger.debug("Rotate file",
+                                        :key => temp_file.key,
+                                        :path => temp_file.path,
+                                        :strategy => @rotation.class.name)
+
+        upload_file(temp_file) # may be async or blocking
+        factory.rotate!
       end
     end
   end

data/logstash-output-s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-s3'
-  s.version         = '4.4.0'
+  s.version         = '4.4.1'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Sends Logstash events to the Amazon Simple Storage Service"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 4.4.0
+  version: 4.4.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-19 00:00:00.000000000 Z
+date: 2022-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement