logstash-integration-aws 0.1.0.pre → 7.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9aed616c6b3abd22f2196b4939ffbedbf1d2c8785a5aef119fc9fe8242847921
-  data.tar.gz: e80bfc1603c49e9271111c8ff4767e42ddcfaa8dedbca2100e3b9e76ae10d735
+  metadata.gz: 17a23ba90d8888e550c2116d81a5dde97b5e57f762c02ceea44a2afecb3e5579
+  data.tar.gz: 41d6e0d5e4e5d9bed6170baa2b44fdfc4bce7d89488e0f15bd80ff9e7a44a3c0
 SHA512:
-  metadata.gz: 5a9c24ffdc56ec13be5b32e27016fc55d4a4a256b16f6d8f28fa38128b1cb1433abd19e63b245ab6d693539af612ba29ad0f345c5a28e96c64a52d66f867c778
-  data.tar.gz: b14a38aea9eafb67c3b0d516463c43540f807f5f3b63fe459841023becd7003ebdb188f202dc7b3233c40b995c3a13ca41a1bcdf31e55abc124eac0954ff0da9
+  metadata.gz: 14cef0ec2b9d04466fcf9ee6f612cd2c8e1055bc5853b147c9086c4914d52c695aab3ab270ef81fe9ae3c9add45391a380228d77befdfe54910decfc27ad60fd
+  data.tar.gz: 5da492ac21e100028d58a6549efae8d110e764ff28a79d21f835946c41afba88e11d240fedde9ac2842357b87e9610f31e098c8ffbd7946ffc36ca5555cd170a

data/CHANGELOG.md

@@ -1,15 +1,27 @@
-## 1.0.0
+## 7.0.1
+  - resolves two closely-related race conditions in the S3 Output plugin's handling of stale temporary files that could cause plugin crashes or data-loss [#19](https://github.com/logstash-plugins/logstash-integration-aws/pull/19)
+    - prevents a `No such file or directory` crash that could occur when a temporary file is accessed after it has been detected as stale (empty+old) and deleted.
+    - prevents a possible deletion of a non-empty temporary file that could occur if bytes were written to it _after_ it was detected as stale (empty+old) and _before_ the deletion completed.
+
+## 7.0.0
+  - bump integration to upper bound of all underlying plugins versions (biggest is sqs output 6.x)
+  - this is necessary to facilitate versioning continuity between older standalone plugins and plugins within the integration
+
+## 0.1.1
+  - remove mention of mixin in gemspec to facilitate docs publishing
+
+## 0.1.0
 
 * Added the initial set of Logstash AWS plugins that ship with Logstash.
   You can find the merged changelog of the individual plugins in CHANGELOG.PRE.MERGE.md.
   These are (along with the version that was used for import):
-  - logstash-codec-cloudfront (3.0.3)
-  - logstash-codec-cloudtrail (3.0.5)
-  - logstash-input-cloudwatch (2.2.4)
-  - logstash-input-s3 (3.8.4)
-  - logstash-input-sqs (3.3.2)
-  - logstash-mixin-aws (5.1.0)
-  - logstash-output-cloudwatch (3.0.10)
-  - logstash-output-sns (4.0.8)
-  - logstash-output-sqs (6.0.1)
-  - logstash-output-s3 (4.3.5)
+  - logstash-codec-cloudfront (3.0.3) [[link]](CHANGELOG.PRE.MERGE.md#changelog---logstash-input-cloudwatch)
+  - logstash-codec-cloudtrail (3.0.5) [[link]](CHANGELOG.PRE.MERGE.md#changelog---logstash-codec-cloudtrail)
+  - logstash-input-cloudwatch (2.2.4) [[link]](CHANGELOG.PRE.MERGE.md#changelog---logstash-input-cloudwatch)
+  - logstash-input-s3 (3.8.4) [[link]](CHANGELOG.PRE.MERGE.md#changelog---logstash-input-s3)
+  - logstash-input-sqs (3.3.2) [[link]](CHANGELOG.PRE.MERGE.md#changelog---logstash-input-sqs)
+  - logstash-mixin-aws (5.1.0) [[link]](CHANGELOG.PRE.MERGE.md#changelog---logstash-mixin-aws)
+  - logstash-output-cloudwatch (3.0.10) [[link]](CHANGELOG.PRE.MERGE.md#changelog---logstash-output-cloudwatch)
+  - logstash-output-sns (4.0.8) [[link]](CHANGELOG.PRE.MERGE.md#changelog---logstash-output-sns)
+  - logstash-output-sqs (6.0.1) [[link]](CHANGELOG.PRE.MERGE.md#changelog---logstash-output-sqs)
+  - logstash-output-s3 (4.3.5) [[link]](CHANGELOG.PRE.MERGE.md#changelog---logstash-output-s3)

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-codec-cloudfront.svg)](https://travis-ci.com/logstash-plugins/logstash-codec-cloudfront)
+    [![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-integration-aws.svg)](https://travis-ci.com/logstash-plugins/logstash-integration-aws)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/codec-cloudfront.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: cloudfront
 :type: codec
 
@@ -17,7 +17,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Cloudfront codec plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/codec-cloudtrail.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: cloudtrail
 :type: codec
 
@@ -17,7 +17,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Cloudtrail codec plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/index.asciidoc

@@ -17,22 +17,20 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === AWS Integration Plugin
 
-// include::{include_path}/plugin_header.asciidoc[]
+include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
 The AWS Integration Plugin provides integrated plugins for working with Amazon Web Services:
 
-////
- - {logstash-ref}/plugins-codec-cloudfront.html[Cloudfront Codec Plugin]
- - {logstash-ref}/plugins-codec-cloudtrail.html[Cloudtrail Codec Plugin]
- - {logstash-ref}/plugins-input-cloudwatch.html[Cloudwatch Input Plugin]
- - {logstash-ref}/plugins-input-s3.html[S3 Input Plugin]
- - {logstash-ref}/plugins-input-sqs.html[Sqs Input Plugin]
- - {logstash-ref}/plugins-output-cloudwatch.html[Cloudwatch Output Plugin]
- - {logstash-ref}/plugins-output-s3.html[S3 Output Plugin]
- - {logstash-ref}/plugins-output-sns.html[Sns Output Plugin]
- - {logstash-ref}/plugins-output-sqs.html[Sqs Output Plugin]
-////
+ - {logstash-ref}/plugins-codecs-cloudfront.html[Cloudfront Codec Plugin]
+ - {logstash-ref}/plugins-codecs-cloudtrail.html[Cloudtrail Codec Plugin]
+ - {logstash-ref}/plugins-inputs-cloudwatch.html[Cloudwatch Input Plugin]
+ - {logstash-ref}/plugins-inputs-s3.html[S3 Input Plugin]
+ - {logstash-ref}/plugins-inputs-sqs.html[Sqs Input Plugin]
+ - {logstash-ref}/plugins-outputs-cloudwatch.html[Cloudwatch Output Plugin]
+ - {logstash-ref}/plugins-outputs-s3.html[S3 Output Plugin]
+ - {logstash-ref}/plugins-outputs-sns.html[Sns Output Plugin]
+ - {logstash-ref}/plugins-outputs-sqs.html[Sqs Output Plugin]
 
 :no_codec!:

data/docs/input-cloudwatch.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: cloudwatch
 :type: input
 :default_codec: plain
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Cloudwatch input plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/input-s3.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: s3
 :type: input
 :default_codec: plain
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === S3 input plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/input-sqs.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: sqs
 :type: input
 :default_codec: json
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Sqs input plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/output-cloudwatch.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: cloudwatch
 :type: output
 :default_codec: plain
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Cloudwatch output plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/output-s3.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: s3
 :type: output
 :default_codec: line
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === S3 output plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/output-sns.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: sns
 :type: output
 :default_codec: plain
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Sns output plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/output-sqs.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: sqs
 :type: output
 :default_codec: json
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Sqs output plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/lib/logstash/outputs/s3/file_repository.rb

@@ -17,8 +17,9 @@ module LogStash
         class PrefixedValue
           def initialize(file_factory, stale_time)
             @file_factory = file_factory
-            @lock = Mutex.new
+            @lock = Monitor.new # reentrant Mutex
             @stale_time = stale_time
+            @is_deleted = false
           end
 
           def with_lock
@@ -36,7 +37,14 @@ module LogStash
           end
 
           def delete!
-            with_lock{ |factory| factory.current.delete! }
+            with_lock do |factory|
+              factory.current.delete!
+              @is_deleted = true
+            end
+          end
+
+          def deleted?
+            with_lock { |_| @is_deleted }
           end
         end
 
@@ -72,19 +80,70 @@ module LogStash
           @prefixed_factories.keySet
         end
 
+        ##
+        # Yields the current file of each non-deleted file factory while the current thread has exclusive access to it.
+        # @yieldparam file [TemporaryFile]
+        # @return [void]
         def each_files
-          @prefixed_factories.elements.each do |prefixed_file|
-            prefixed_file.with_lock { |factory| yield factory.current }
+          each_factory(keys) do |factory|
+            yield factory.current
           end
+          nil # void return avoid leaking unsynchronized access
         end
 
-        # Return the file factory
+        ##
+        # Yields the file factory while the current thread has exclusive access to it, creating a new
+        # one if one does not exist or if the current one is being reaped by the stale watcher.
+        # @param prefix_key [String]: the prefix key
+        # @yieldparam factory [TemporaryFileFactory]: a temporary file factory that this thread has exclusive access to
+        # @return [void]
         def get_factory(prefix_key)
-          @prefixed_factories.computeIfAbsent(prefix_key, @factory_initializer).with_lock { |factory| yield factory }
+          # fast-path: if factory exists and is not deleted, yield it with exclusive access and return
+          prefix_val = @prefixed_factories.get(prefix_key)
+          prefix_val&.with_lock do |factory|
+            # intentional local-jump to ensure deletion detection
+            # is done inside the exclusive access.
+            unless prefix_val.deleted?
+              yield(factory)
+              return nil # void return avoid leaking unsynchronized access
+            end
+          end
+
+          # slow-path:
+          # the ConcurrentHashMap#get operation is lock-free, but may have returned an entry that was being deleted by
+          # another thread (such as via stale detection). If we failed to retrieve a value, or retrieved one that had
+          # been marked deleted, use the atomic ConcurrentHashMap#compute to retrieve a non-deleted entry.
+          prefix_val = @prefixed_factories.compute(prefix_key) do |_, existing|
+            existing && !existing.deleted? ? existing : @factory_initializer.apply(prefix_key)
+          end
+          prefix_val.with_lock { |factory| yield factory }
+          nil # void return avoid leaking unsynchronized access
+        end
+
+        ##
+        # Yields each non-deleted file factory while the current thread has exclusive access to it.
+        # @param prefixes [Array<String>]: the prefix keys
+        # @yieldparam factory [TemporaryFileFactory]
+        # @return [void]
+        def each_factory(prefixes)
+          prefixes.each do |prefix_key|
+            prefix_val = @prefixed_factories.get(prefix_key)
+            prefix_val&.with_lock do |factory|
+              yield factory unless prefix_val.deleted?
+            end
+          end
+          nil # void return avoid leaking unsynchronized access
         end
 
+        ##
+        # Ensures that a non-deleted factory exists for the provided prefix and yields its current file
+        # while the current thread has exclusive access to it.
+        # @param prefix_key [String]
+        # @yieldparam file [TemporaryFile]
+        # @return [void]
         def get_file(prefix_key)
           get_factory(prefix_key) { |factory| yield factory.current }
+          nil # void return avoid leaking unsynchronized access
         end
 
         def shutdown
@@ -95,10 +154,21 @@ module LogStash
           @prefixed_factories.size
         end
 
-        def remove_stale(k, v)
-          if v.stale?
-            @prefixed_factories.remove(k, v)
-            v.delete!
+        def remove_if_stale(prefix_key)
+          # we use the ATOMIC `ConcurrentHashMap#computeIfPresent` to atomically
+          # detect the staleness, mark a stale prefixed factory as deleted, and delete from the map.
+          @prefixed_factories.computeIfPresent(prefix_key) do |_, prefixed_factory|
+            # once we have retrieved an instance, we acquire exclusive access to it
+            # for stale detection, marking it as deleted before releasing the lock
+            # and causing it to become deleted from the map.
+            prefixed_factory.with_lock do |_|
+              if prefixed_factory.stale?
+                prefixed_factory.delete!  # mark deleted to prevent reuse
+                nil # cause deletion
+              else
+                prefixed_factory # keep existing
+              end
+            end
           end
         end
 
@@ -106,7 +176,9 @@ module LogStash
           @stale_sweeper = Concurrent::TimerTask.new(:execution_interval => @sweeper_interval) do
             LogStash::Util.set_thread_name("S3, Stale factory sweeper")
 
-            @prefixed_factories.forEach{|k,v| remove_stale(k,v)}
+            @prefixed_factories.keys.each do |prefix|
+              remove_if_stale(prefix)
+            end
           end
 
           @stale_sweeper.execute

data/lib/logstash/outputs/s3.rb

@@ -258,6 +258,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
     @logger.debug("Uploading current workspace")
 
+    @file_repository.shutdown # stop stale sweeps
+
     # The plugin has stopped receiving new events, but we still have
     # data on disk, lets make sure it get to S3.
     # If Logstash get interrupted, the `restore_from_crash` (when set to true) method will pickup
@@ -267,8 +269,6 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       upload_file(file)
     end
 
-    @file_repository.shutdown
-
     @uploader.stop # wait until all the current upload are complete
     @crash_uploader.stop if @restore # we might have still work to do for recovery so wait until we are done
   end
@@ -336,22 +336,21 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def rotate_if_needed(prefixes)
-    prefixes.each do |prefix|
-      # Each file access is thread safe,
-      # until the rotation is done then only
-      # one thread has access to the resource.
-      @file_repository.get_factory(prefix) do |factory|
-        temp_file = factory.current
-
-        if @rotation.rotate?(temp_file)
-          @logger.debug("Rotate file",
-                        :strategy => @rotation.class.name,
-                        :key => temp_file.key,
-                        :path => temp_file.path)
-
-          upload_file(temp_file)
-          factory.rotate!
-        end
+
+    # Each file access is thread safe,
+    # until the rotation is done then only
+    # one thread has access to the resource.
+    @file_repository.each_factory(prefixes) do |factory|
+      temp_file = factory.current
+
+      if @rotation.rotate?(temp_file)
+        @logger.debug("Rotate file",
+                      :strategy => @rotation.class.name,
+                      :key => temp_file.key,
+                      :path => temp_file.path)
+
+        upload_file(temp_file)
+        factory.rotate!
       end
     end
   end

data/logstash-integration-aws.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = "logstash-integration-aws"
-  s.version         = "0.1.0.pre"
+  s.version         = "7.0.1"
   s.licenses        = ["Apache-2.0"]
   s.summary         = "Collection of Logstash plugins that integrate with AWS"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -16,7 +16,6 @@ Gem::Specification.new do |s|
       logstash-input-cloudwatch
       logstash-input-s3
       logstash-input-sqs
-      logstash-mixin-aws
       logstash-output-cloudwatch
       logstash-output-s3
       logstash-output-sns

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-aws
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre
+  version: 7.0.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-13 00:00:00.000000000 Z
+date: 2022-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -397,7 +397,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: integration
-  integration_plugins: logstash-codec-cloudfront,logstash-codec-cloudtrail,logstash-input-cloudwatch,logstash-input-s3,logstash-input-sqs,logstash-mixin-aws,logstash-output-cloudwatch,logstash-output-s3,logstash-output-sns,logstash-output-sqs
+  integration_plugins: logstash-codec-cloudfront,logstash-codec-cloudtrail,logstash-input-cloudwatch,logstash-input-s3,logstash-input-sqs,logstash-output-cloudwatch,logstash-output-s3,logstash-output-sns,logstash-output-sqs
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -409,9 +409,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.1.6
 signing_key: