logstash-integration-aws 7.0.0 → 7.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4cea87525319253265e10b11632067a0470dd461bcd6febad35e13bb94d5e2b2
-  data.tar.gz: ae9b187795bc416f1c1e7e16b4207b8fc476b54c4cbc55325104f91930a65ae1
+  metadata.gz: cee9f1de988917319a825f2297f67e3c436cf887eaea605b0ae0a863e7b9a2b8
+  data.tar.gz: 357432106dd5710a0c8a4addda66922c5d7f66c4c6fbf82bba638ad71f595ab2
 SHA512:
-  metadata.gz: e62194ffbe42ac97834a189eb9821feed9eeebd28bb72ffed26de4b087d76b75ca30006c630d2d30f1892f7dc41825e176bf25fd97487e3554aec5fc7e847b36
-  data.tar.gz: 30e9270e7933fd448ff1e9ae41f768edbe6fd956719c79e9f828743e583d06ddef1fa2ab55462df6c7334990b01e14fb766290c1a8f9e405cf1e8673a99f4d83
+  metadata.gz: e3b500c0b652a50498c85f204bee216965b04b8468cb2a1b798d82e08bd1d1b6d73876521770c6bacc956786bb749342316d06bd7254af140b11f3660922a2ec
+  data.tar.gz: ff157154dc10816d86a035cac58b85a8cf13841e9948026113386a66c941bf6b9256f882d1811cd18824fda7348ae9c9dd66e9ab390ef6438f4021845521e849

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 7.1.0
+  - Plugin restores and uploads corrupted GZIP files (caused by abnormal termination) to AWS S3 [#20](https://github.com/logstash-plugins/logstash-integration-aws/pull/20)
+
+## 7.0.1
+  - resolves two closely-related race conditions in the S3 Output plugin's handling of stale temporary files that could cause plugin crashes or data-loss [#19](https://github.com/logstash-plugins/logstash-integration-aws/pull/19)
+    - prevents a `No such file or directory` crash that could occur when a temporary file is accessed after it has been detected as stale (empty+old) and deleted.
+    - prevents a possible deletion of a non-empty temporary file that could occur if bytes were written to it _after_ it was detected as stale (empty+old) and _before_ the deletion completed.
+
 ## 7.0.0
   - bump integration to upper bound of all underlying plugins versions (biggest is sqs output 6.x)
   - this is necessary to facilitate versioning continuity between older standalone plugins and plugins within the integration

data/README.md

@@ -19,7 +19,7 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
 
 ## Developing
 
-### 1. Plugin Developement and Testing
+### 1. Plugin Development and Testing
 
 #### Code
 - To get started, you'll need JRuby with the Bundler gem installed.

data/VERSION

@@ -0,0 +1 @@
+7.1.0

data/docs/codec-cloudfront.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: cloudfront
 :type: codec
 
@@ -17,7 +17,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Cloudfront codec plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/codec-cloudtrail.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: cloudtrail
 :type: codec
 
@@ -17,7 +17,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Cloudtrail codec plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/index.asciidoc

@@ -17,13 +17,12 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === AWS Integration Plugin
 
-// include::{include_path}/plugin_header.asciidoc[]
+include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
 The AWS Integration Plugin provides integrated plugins for working with Amazon Web Services:
 
-////
  - {logstash-ref}/plugins-codecs-cloudfront.html[Cloudfront Codec Plugin]
  - {logstash-ref}/plugins-codecs-cloudtrail.html[Cloudtrail Codec Plugin]
  - {logstash-ref}/plugins-inputs-cloudwatch.html[Cloudwatch Input Plugin]
@@ -33,6 +32,5 @@ The AWS Integration Plugin provides integrated plugins for working with Amazon W
  - {logstash-ref}/plugins-outputs-s3.html[S3 Output Plugin]
  - {logstash-ref}/plugins-outputs-sns.html[Sns Output Plugin]
  - {logstash-ref}/plugins-outputs-sqs.html[Sqs Output Plugin]
-////
 
 :no_codec!:

data/docs/input-cloudwatch.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: cloudwatch
 :type: input
 :default_codec: plain
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Cloudwatch input plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/input-s3.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: s3
 :type: input
 :default_codec: plain
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === S3 input plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/input-sqs.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: sqs
 :type: input
 :default_codec: json
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Sqs input plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/output-cloudwatch.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: cloudwatch
 :type: output
 :default_codec: plain
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Cloudwatch output plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/output-s3.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: s3
 :type: output
 :default_codec: line
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === S3 output plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/output-sns.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: sns
 :type: output
 :default_codec: plain
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Sns output plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/docs/output-sqs.asciidoc

@@ -1,4 +1,4 @@
-// :integration: aws
+:integration: aws
 :plugin: sqs
 :type: output
 :default_codec: json
@@ -18,7 +18,7 @@ END - GENERATED VARIABLES, DO NOT EDIT!
 
 === Sqs output plugin
 
-// include::{include_path}/plugin_header-integration.asciidoc[]
+include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 

data/lib/logstash/outputs/s3/file_repository.rb

@@ -17,8 +17,9 @@ module LogStash
         class PrefixedValue
           def initialize(file_factory, stale_time)
             @file_factory = file_factory
-            @lock = Mutex.new
+            @lock = Monitor.new # reentrant Mutex
             @stale_time = stale_time
+            @is_deleted = false
           end
 
           def with_lock
@@ -36,7 +37,14 @@ module LogStash
           end
 
           def delete!
-            with_lock{ |factory| factory.current.delete! }
+            with_lock do |factory|
+              factory.current.delete!
+              @is_deleted = true
+            end
+          end
+
+          def deleted?
+            with_lock { |_| @is_deleted }
           end
         end
 
@@ -72,19 +80,70 @@ module LogStash
           @prefixed_factories.keySet
         end
 
+        ##
+        # Yields the current file of each non-deleted file factory while the current thread has exclusive access to it.
+        # @yieldparam file [TemporaryFile]
+        # @return [void]
         def each_files
-          @prefixed_factories.elements.each do |prefixed_file|
-            prefixed_file.with_lock { |factory| yield factory.current }
+          each_factory(keys) do |factory|
+            yield factory.current
           end
+          nil # void return avoid leaking unsynchronized access
         end
 
-        # Return the file factory
+        ##
+        # Yields the file factory while the current thread has exclusive access to it, creating a new
+        # one if one does not exist or if the current one is being reaped by the stale watcher.
+        # @param prefix_key [String]: the prefix key
+        # @yieldparam factory [TemporaryFileFactory]: a temporary file factory that this thread has exclusive access to
+        # @return [void]
         def get_factory(prefix_key)
-          @prefixed_factories.computeIfAbsent(prefix_key, @factory_initializer).with_lock { |factory| yield factory }
+          # fast-path: if factory exists and is not deleted, yield it with exclusive access and return
+          prefix_val = @prefixed_factories.get(prefix_key)
+          prefix_val&.with_lock do |factory|
+            # intentional local-jump to ensure deletion detection
+            # is done inside the exclusive access.
+            unless prefix_val.deleted?
+              yield(factory)
+              return nil # void return avoid leaking unsynchronized access
+            end
+          end
+
+          # slow-path:
+          # the ConcurrentHashMap#get operation is lock-free, but may have returned an entry that was being deleted by
+          # another thread (such as via stale detection). If we failed to retrieve a value, or retrieved one that had
+          # been marked deleted, use the atomic ConcurrentHashMap#compute to retrieve a non-deleted entry.
+          prefix_val = @prefixed_factories.compute(prefix_key) do |_, existing|
+            existing && !existing.deleted? ? existing : @factory_initializer.apply(prefix_key)
+          end
+          prefix_val.with_lock { |factory| yield factory }
+          nil # void return avoid leaking unsynchronized access
+        end
+
+        ##
+        # Yields each non-deleted file factory while the current thread has exclusive access to it.
+        # @param prefixes [Array<String>]: the prefix keys
+        # @yieldparam factory [TemporaryFileFactory]
+        # @return [void]
+        def each_factory(prefixes)
+          prefixes.each do |prefix_key|
+            prefix_val = @prefixed_factories.get(prefix_key)
+            prefix_val&.with_lock do |factory|
+              yield factory unless prefix_val.deleted?
+            end
+          end
+          nil # void return avoid leaking unsynchronized access
         end
 
+        ##
+        # Ensures that a non-deleted factory exists for the provided prefix and yields its current file
+        # while the current thread has exclusive access to it.
+        # @param prefix_key [String]
+        # @yieldparam file [TemporaryFile]
+        # @return [void]
         def get_file(prefix_key)
           get_factory(prefix_key) { |factory| yield factory.current }
+          nil # void return avoid leaking unsynchronized access
         end
 
         def shutdown
@@ -95,10 +154,21 @@ module LogStash
           @prefixed_factories.size
         end
 
-        def remove_stale(k, v)
-          if v.stale?
-            @prefixed_factories.remove(k, v)
-            v.delete!
+        def remove_if_stale(prefix_key)
+          # we use the ATOMIC `ConcurrentHashMap#computeIfPresent` to atomically
+          # detect the staleness, mark a stale prefixed factory as deleted, and delete from the map.
+          @prefixed_factories.computeIfPresent(prefix_key) do |_, prefixed_factory|
+            # once we have retrieved an instance, we acquire exclusive access to it
+            # for stale detection, marking it as deleted before releasing the lock
+            # and causing it to become deleted from the map.
+            prefixed_factory.with_lock do |_|
+              if prefixed_factory.stale?
+                prefixed_factory.delete!  # mark deleted to prevent reuse
+                nil # cause deletion
+              else
+                prefixed_factory # keep existing
+              end
+            end
           end
         end
 
@@ -106,7 +176,9 @@ module LogStash
           @stale_sweeper = Concurrent::TimerTask.new(:execution_interval => @sweeper_interval) do
             LogStash::Util.set_thread_name("S3, Stale factory sweeper")
 
-            @prefixed_factories.forEach{|k,v| remove_stale(k,v)}
+            @prefixed_factories.keys.each do |prefix|
+              remove_if_stale(prefix)
+            end
           end
 
           @stale_sweeper.execute

data/lib/logstash/outputs/s3/size_rotation_policy.rb

@@ -7,7 +7,7 @@ module LogStash
 
         def initialize(size_file)
           if size_file <= 0
-            raise LogStash::ConfigurationError, "`size_file` need to be greather than 0"
+            raise LogStash::ConfigurationError, "`size_file` need to be greater than 0"
           end
 
           @size_file = size_file

data/lib/logstash/outputs/s3/temporary_file.rb

@@ -2,15 +2,23 @@
 require "thread"
 require "forwardable"
 require "fileutils"
+require "logstash-integration-aws_jars"
 
 module LogStash
   module Outputs
     class S3
-      # Wrap the actual file descriptor into an utility classe
-      # It make it more OOP and easier to reason with the paths.
+
+      java_import 'org.logstash.plugins.integration.outputs.s3.GzipUtil'
+
+      # Wrap the actual file descriptor into an utility class
+      # Make it more OOP and easier to reason with the paths.
       class TemporaryFile
         extend Forwardable
 
+        GZIP_EXTENSION = "txt.gz"
+        TXT_EXTENSION = "txt"
+        RECOVERED_FILE_NAME_TAG = "-recovered"
+
         def_delegators :@fd, :path, :write, :close, :fsync
 
         attr_reader :fd
@@ -33,8 +41,10 @@ module LogStash
         def size
           # Use the fd size to get the accurate result,
           # so we dont have to deal with fsync
-          # if the file is close we will use the File::size
+          # if the file is close, fd.size raises an IO exception so we use the File::size
           begin
+            # fd is nil when LS tries to recover gzip file but fails
+            return 0 unless @fd != nil
             @fd.size
           rescue IOError
             ::File.size(path)
@@ -45,7 +55,7 @@ module LogStash
           @key.gsub(/^\//, "")
         end
 
-        # Each temporary file is made inside a directory named with an UUID,
+        # Each temporary file is created inside a directory named with an UUID,
         # instead of deleting the file directly and having the risk of deleting other files
         # we delete the root of the UUID, using a UUID also remove the risk of deleting unwanted file, it acts as
         # a sandbox.
@@ -58,12 +68,45 @@ module LogStash
           size == 0
         end
 
+        # only to cover the case where LS cannot restore corrupted file, file is not exist
+        def recoverable?
+          !@fd.nil?
+        end
+
         def self.create_from_existing_file(file_path, temporary_folder)
           key_parts = Pathname.new(file_path).relative_path_from(temporary_folder).to_s.split(::File::SEPARATOR)
 
+          # recover gzip file and compress back before uploading to S3
+          if file_path.end_with?("." + GZIP_EXTENSION)
+            file_path = self.recover(file_path)
+          end
           TemporaryFile.new(key_parts.slice(1, key_parts.size).join("/"),
-                         ::File.open(file_path, "r"),
-                         ::File.join(temporary_folder, key_parts.slice(0, 1)))
+                            ::File.exist?(file_path) ? ::File.open(file_path, "r") : nil, # for the nil case, file size will be 0 and upload will be ignored.
+                            ::File.join(temporary_folder, key_parts.slice(0, 1)))
+        end
+
+        def self.gzip_extension
+          GZIP_EXTENSION
+        end
+
+        def self.text_extension
+          TXT_EXTENSION
+        end
+
+        def self.recovery_file_name_tag
+          RECOVERED_FILE_NAME_TAG
+        end
+
+        private
+        def self.recover(file_path)
+          full_gzip_extension = "." + GZIP_EXTENSION
+          recovered_txt_file_path = file_path.gsub(full_gzip_extension, RECOVERED_FILE_NAME_TAG + "." + TXT_EXTENSION)
+          recovered_gzip_file_path = file_path.gsub(full_gzip_extension, RECOVERED_FILE_NAME_TAG + full_gzip_extension)
+          GzipUtil.recover(file_path, recovered_txt_file_path)
+          if ::File.exist?(recovered_txt_file_path) && !::File.zero?(recovered_txt_file_path)
+            GzipUtil.compress(recovered_txt_file_path, recovered_gzip_file_path)
+          end
+          recovered_gzip_file_path
         end
       end
     end

data/lib/logstash/outputs/s3/temporary_file_factory.rb

@@ -19,9 +19,6 @@ module LogStash
       # I do not have to mess around to check if the other directory have file in it before destroying them.
       class TemporaryFileFactory
         FILE_MODE = "a"
-        GZIP_ENCODING = "gzip"
-        GZIP_EXTENSION = "txt.gz"
-        TXT_EXTENSION = "txt"
         STRFTIME = "%Y-%m-%dT%H.%M"
 
         attr_accessor :counter, :tags, :prefix, :encoding, :temporary_directory, :current
@@ -48,7 +45,7 @@ module LogStash
 
         private
         def extension
-          gzip? ? GZIP_EXTENSION : TXT_EXTENSION
+          gzip? ? TemporaryFile.gzip_extension : TemporaryFile.text_extension
         end
 
         def gzip?

data/lib/logstash/outputs/s3/time_rotation_policy.rb

@@ -7,7 +7,7 @@ module LogStash
 
         def initialize(time_file)
           if time_file <= 0
-            raise LogStash::ConfigurationError, "`time_file` need to be greather than 0"
+            raise LogStash::ConfigurationError, "`time_file` need to be greater than 0"
           end
 
           @time_file = time_file * 60

data/lib/logstash/outputs/s3/uploader.rb

@@ -31,6 +31,7 @@ module LogStash
           end
         end
 
+        # uploads a TemporaryFile to S3
         def upload(file, options = {})
           upload_options = options.fetch(:upload_options, {})
 
@@ -68,6 +69,7 @@ module LogStash
           @workers_pool.shutdown
           @workers_pool.wait_for_termination(nil) # block until its done
         end
+
       end
     end
   end

data/lib/logstash/outputs/s3.rb

@@ -94,6 +94,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
                                                                    :fallback_policy => :caller_runs
                                                                  })
 
+  GZIP_ENCODING = "gzip"
 
   config_name "s3"
   default :codec, "line"
@@ -107,7 +108,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   # Set the size of file in bytes, this means that files on bucket when have dimension > file_size, they are stored in two or more file.
   # If you have tags then it will generate a specific size file for every tags
-  ##NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.
+  #
+  # NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.
   config :size_file, :validate => :number, :default => 1024 * 1024 * 5
 
   # Set the time, in MINUTES, to close the current sub_time_section of bucket.
@@ -115,10 +117,10 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   # If it's valued 0 and rotation_strategy is 'time' or 'size_and_time' then the plugin reaise a configuration error.
   config :time_file, :validate => :number, :default => 15
 
-  ## IMPORTANT: if you use multiple instance of s3, you should specify on one of them the "restore=> true" and on the others "restore => false".
-  ## This is hack for not destroy the new files after restoring the initial files.
-  ## If you do not specify "restore => true" when logstash crashes or is restarted, the files are not sent into the bucket,
-  ## for example if you have single Instance.
+  # If `restore => false` is specified and Logstash crashes, the unprocessed files are not sent into the bucket.
+  #
+  # NOTE: that the `recovery => true` default assumes multiple S3 outputs would set a unique `temporary_directory => ...`
+  # if they do not than only a single S3 output is safe to recover (since let-over files are processed and deleted).
   config :restore, :validate => :boolean, :default => true
 
   # The S3 canned ACL to use when putting the file. Defaults to "private".
@@ -144,6 +146,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   # Set the directory where logstash will store the tmp files before sending it to S3
   # default to the current OS temporary directory in linux /tmp/logstash
+  #
+  # NOTE: the reason we do not have a unique (isolated) temporary directory as a default, to support multiple plugin instances,
+  # is that we would have to rely on something static that does not change between restarts (e.g. a user set id => ...).
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
 
   # Specify a prefix to the uploaded filename, this can simulate directories on S3.  Prefix does not require leading slash.
@@ -174,7 +179,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   config :tags, :validate => :array, :default => []
 
   # Specify the content encoding. Supports ("gzip"). Defaults to "none"
-  config :encoding, :validate => ["none", "gzip"], :default => "none"
+  config :encoding, :validate => ["none", GZIP_ENCODING], :default => "none"
 
   # Define the strategy to use to decide when we need to rotate the file and push it to S3,
   # The default strategy is to check for both size and time, the first one to match will rotate the file.
@@ -258,6 +263,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
     @logger.debug("Uploading current workspace")
 
+    @file_repository.shutdown # stop stale sweeps
+
     # The plugin has stopped receiving new events, but we still have
     # data on disk, lets make sure it get to S3.
     # If Logstash get interrupted, the `restore_from_crash` (when set to true) method will pickup
@@ -267,8 +274,6 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       upload_file(file)
     end
 
-    @file_repository.shutdown
-
     @uploader.stop # wait until all the current upload are complete
     @crash_uploader.stop if @restore # we might have still work to do for recovery so wait until we are done
   end
@@ -308,7 +313,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       :server_side_encryption => @server_side_encryption ? @server_side_encryption_algorithm : nil,
       :ssekms_key_id => @server_side_encryption_algorithm == "aws:kms" ? @ssekms_key_id : nil,
       :storage_class => @storage_class,
-      :content_encoding => @encoding == "gzip" ? "gzip" : nil,
+      :content_encoding => @encoding == GZIP_ENCODING ? GZIP_ENCODING : nil,
       :multipart_threshold => @upload_multipart_threshold
     }
   end
@@ -336,28 +341,28 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def rotate_if_needed(prefixes)
-    prefixes.each do |prefix|
-      # Each file access is thread safe,
-      # until the rotation is done then only
-      # one thread has access to the resource.
-      @file_repository.get_factory(prefix) do |factory|
-        temp_file = factory.current
-
-        if @rotation.rotate?(temp_file)
-          @logger.debug("Rotate file",
-                        :strategy => @rotation.class.name,
-                        :key => temp_file.key,
-                        :path => temp_file.path)
-
-          upload_file(temp_file)
-          factory.rotate!
-        end
+    # Each file access is thread safe,
+    # until the rotation is done then only
+    # one thread has access to the resource.
+    @file_repository.each_factory(prefixes) do |factory|
+      # we have exclusive access to the one-and-only
+      # prefix WRAPPER for this factory.
+      temp_file = factory.current
+
+      if @rotation.rotate?(temp_file)
+        @logger.debug? && @logger.debug("Rotate file",
+                                        :key => temp_file.key,
+                                        :path => temp_file.path,
+                                        :strategy => @rotation.class.name)
+
+        upload_file(temp_file) # may be async or blocking
+        factory.rotate!
       end
     end
   end
 
   def upload_file(temp_file)
-    @logger.debug("Queue for upload", :path => temp_file.path)
+    @logger.debug? && @logger.debug("Queue for upload", :path => temp_file.path)
 
     # if the queue is full the calling thread will be used to upload
     temp_file.close # make sure the content is on disk
@@ -380,7 +385,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def clean_temporary_file(file)
-    @logger.debug("Removing temporary file", :file => file.path)
+    @logger.debug? && @logger.debug("Removing temporary file", :path => file.path)
     file.delete!
   end
 
@@ -390,16 +395,48 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
     @crash_uploader = Uploader.new(bucket_resource, @logger, CRASH_RECOVERY_THREADPOOL)
 
     temp_folder_path = Pathname.new(@temporary_directory)
-    Dir.glob(::File.join(@temporary_directory, "**/*"))
-      .select { |file| ::File.file?(file) }
-      .each do |file|
-      temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
-      if temp_file.size > 0
-        @logger.debug("Recovering from crash and uploading", :file => temp_file.path)
-        @crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file), :upload_options => upload_options)
+    files = Dir.glob(::File.join(@temporary_directory, "**/*"))
+               .select { |file_path| ::File.file?(file_path) }
+    under_recovery_files = get_under_recovery_files(files)
+
+    files.each do |file_path|
+      # when encoding is GZIP, if file is already recovering or recovered and uploading to S3, log and skip
+      if under_recovery_files.include?(file_path)
+        unless file_path.include?(TemporaryFile.gzip_extension)
+          @logger.warn("The #{file_path} file either under recover process or failed to recover before.")
+        end
       else
-        clean_temporary_file(temp_file)
+        temp_file = TemporaryFile.create_from_existing_file(file_path, temp_folder_path)
+        # do not remove or upload if Logstash tries to recover file but fails
+        if temp_file.recoverable?
+          if temp_file.size > 0
+            @logger.debug? && @logger.debug("Recovering from crash and uploading", :path => temp_file.path)
+            @crash_uploader.upload_async(temp_file,
+                                         :on_complete => method(:clean_temporary_file),
+                                         :upload_options => upload_options)
+          else
+            clean_temporary_file(temp_file)
+          end
+        end
+      end
+    end
+  end
+
+  # figures out the recovering files and
+  # creates a skip list to ignore for the rest of processes
+  def get_under_recovery_files(files)
+    skip_files = Set.new
+    return skip_files unless @encoding == GZIP_ENCODING
+
+    files.each do |file_path|
+      if file_path.include?(TemporaryFile.recovery_file_name_tag)
+        skip_files << file_path
+        if file_path.include?(TemporaryFile.gzip_extension)
+          # also include the original corrupted gzip file
+          skip_files << file_path.gsub(TemporaryFile.recovery_file_name_tag, "")
+        end
       end
     end
+    skip_files
   end
 end

data/lib/logstash-integration-aws_jars.rb

@@ -0,0 +1,4 @@
+# AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
+
+require 'jar_dependencies'
+require_jar('org.logstash.plugins.integration.aws', 'logstash-integration-aws', '7.1.0')

data/lib/tasks/build.rake

@@ -0,0 +1,15 @@
+# encoding: utf-8
+require "jars/installer"
+require "fileutils"
+
+task :vendor do
+  exit(1) unless system './gradlew vendor'
+  version = File.read("VERSION").strip
+end
+
+desc "clean"
+task :clean do
+  ["build", "vendor/jar-dependencies", "Gemfile.lock"].each do |p|
+    FileUtils.rm_rf(p)
+  end
+end

data/logstash-integration-aws.gemspec

@@ -1,6 +1,8 @@
+INTEGRATION_AWS_VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(INTEGRATION_AWS_VERSION)
+
 Gem::Specification.new do |s|
   s.name            = "logstash-integration-aws"
-  s.version         = "7.0.0"
+  s.version         = INTEGRATION_AWS_VERSION
   s.licenses        = ["Apache-2.0"]
   s.summary         = "Collection of Logstash plugins that integrate with AWS"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -23,7 +25,7 @@ Gem::Specification.new do |s|
   }
 
 
-  s.require_paths   = ["lib"]
+  s.require_paths   = ["lib", "vendor/jar-dependencies"]
   s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "VERSION", "docs/**/*"]
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 

data/spec/integration/restore_from_crash_spec.rb

@@ -7,18 +7,17 @@ require "stud/temporary"
 describe "Restore from crash", :integration => true do
   include_context "setup plugin"
 
-  let(:options) { main_options.merge({ "restore" => true, "canned_acl" => "public-read-write" }) }
-
   let(:number_of_files) { 5 }
   let(:dummy_content) { "foobar\n" * 100 }
-  let(:factory) { LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "none", temporary_directory)}
 
   before do
     clean_remote_files(prefix)
   end
 
-
   context 'with a non-empty tempfile' do
+    let(:options) { main_options.merge({ "restore" => true, "canned_acl" => "public-read-write" }) }
+    let(:factory) { LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "none", temporary_directory)}
+
     before do
       # Creating a factory always create a file
       factory.current.write(dummy_content)
@@ -30,6 +29,7 @@ describe "Restore from crash", :integration => true do
         factory.current.fsync
       end
     end
+
     it "uploads the file to the bucket" do
       subject.register
       try(20) do
@@ -41,6 +41,9 @@ describe "Restore from crash", :integration => true do
   end
 
   context 'with an empty tempfile' do
+    let(:options) { main_options.merge({ "restore" => true, "canned_acl" => "public-read-write" }) }
+    let(:factory) { LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "none", temporary_directory)}
+
     before do
       factory.current
       factory.rotate!
@@ -63,5 +66,68 @@ describe "Restore from crash", :integration => true do
       expect(bucket_resource.objects(:prefix => prefix).count).to eq(0)
     end
   end
+
+  context "#gzip encoding" do
+    let(:options) { main_options.merge({ "restore" => true, "canned_acl" => "public-read-write", "encoding" => "gzip" }) }
+    let(:factory) { LogStash::Outputs::S3::TemporaryFileFactory.new(prefix, tags, "gzip", temporary_directory)}
+    describe "with empty recovered file" do
+      before do
+        # Creating a factory always create a file
+        factory.current.write('')
+        factory.current.fsync
+        factory.current.close
+      end
+
+      it 'should not upload and not remove temp file' do
+        subject.register
+        try(20) do
+          expect(bucket_resource.objects(:prefix => prefix).count).to eq(0)
+          expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(1)
+        end
+      end
+    end
+
+    describe "with healthy recovered, size is greater than zero file" do
+      before do
+        # Creating a factory always create a file
+        factory.current.write(dummy_content)
+        factory.current.fsync
+        factory.current.close
+
+        (number_of_files - 1).times do
+          factory.rotate!
+          factory.current.write(dummy_content)
+          factory.current.fsync
+          factory.current.close
+        end
+      end
+
+      it 'should recover, upload to S3 and remove temp file' do
+        subject.register
+        try(20) do
+          expect(bucket_resource.objects(:prefix => prefix).count).to eq(number_of_files)
+          expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(0)
+          expect(bucket_resource.objects(:prefix => prefix).first.acl.grants.collect(&:permission)).to include("READ", "WRITE")
+        end
+      end
+    end
+
+    describe "with failure when recovering" do
+      before do
+        # Creating a factory always create a file
+        factory.current.write(dummy_content)
+        factory.current.fsync
+      end
+
+      it 'should not upload to S3 and not remove temp file' do
+        subject.register
+        try(20) do
+          expect(bucket_resource.objects(:prefix => prefix).count).to eq(0)
+          expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(1)
+        end
+      end
+    end
+  end
+
 end
 

data/spec/outputs/s3/size_rotation_policy_spec.rb

@@ -25,11 +25,11 @@ describe LogStash::Outputs::S3::SizeRotationPolicy do
   end
 
   it "raises an exception if the `size_file` is 0" do
-    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /need to be greather than 0/)
+    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /need to be greater than 0/)
   end
 
   it "raises an exception if the `size_file` is < 0" do
-    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /need to be greather than 0/)
+    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /need to be greater than 0/)
   end
 
   context "#needs_periodic?" do

data/spec/outputs/s3/time_rotation_policy_spec.rb

@@ -14,11 +14,11 @@ describe LogStash::Outputs::S3::TimeRotationPolicy do
   let(:file) { LogStash::Outputs::S3::TemporaryFile.new(name, temporary_file, temporary_directory) }
 
   it "raises an exception if the `file_time` is set to 0" do
-    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greather than 0/)
+    expect { described_class.new(0) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greater than 0/)
   end
 
   it "raises an exception if the `file_time` is < 0" do
-    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greather than 0/)
+    expect { described_class.new(-100) }.to raise_error(LogStash::ConfigurationError, /`time_file` need to be greater than 0/)
   end
 
   context "when the size of the file is superior to 0" do

data/spec/support/helpers.rb

@@ -67,6 +67,7 @@ shared_context "setup plugin" do
   let(:bucket) { ENV["AWS_LOGSTASH_TEST_BUCKET"] }
   let(:access_key_id) {  ENV["AWS_ACCESS_KEY_ID"] }
   let(:secret_access_key) { ENV["AWS_SECRET_ACCESS_KEY"] }
+  let(:session_token) { ENV["AWS_SESSION_TOKEN"] }
   let(:size_file) { 100 }
   let(:time_file) { 100 }
   let(:tags) { [] }
@@ -80,6 +81,7 @@ shared_context "setup plugin" do
       "temporary_directory" => temporary_directory,
       "access_key_id" => access_key_id,
       "secret_access_key" => secret_access_key,
+      "session_token" => session_token,
       "size_file" => size_file,
       "time_file" => time_file,
       "region" => region,
@@ -87,7 +89,7 @@ shared_context "setup plugin" do
     }
   end
 
-  let(:client_credentials) { Aws::Credentials.new(access_key_id, secret_access_key) }
+  let(:client_credentials) { Aws::Credentials.new(access_key_id, secret_access_key, session_token) }
   let(:bucket_resource) { Aws::S3::Bucket.new(bucket, { :credentials => client_credentials, :region => region }) }
 
   subject { LogStash::Outputs::S3.new(options) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-aws
 version: !ruby/object:Gem::Version
-  version: 7.0.0
+  version: 7.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-25 00:00:00.000000000 Z
+date: 2023-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -311,6 +311,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- VERSION
 - docs/codec-cloudfront.asciidoc
 - docs/codec-cloudtrail.asciidoc
 - docs/index.asciidoc
@@ -321,6 +322,7 @@ files:
 - docs/output-s3.asciidoc
 - docs/output-sns.asciidoc
 - docs/output-sqs.asciidoc
+- lib/logstash-integration-aws_jars.rb
 - lib/logstash/codecs/cloudfront.rb
 - lib/logstash/codecs/cloudtrail.rb
 - lib/logstash/inputs/cloudwatch.rb
@@ -343,6 +345,7 @@ files:
 - lib/logstash/plugin_mixins/aws_config.rb
 - lib/logstash/plugin_mixins/aws_config/generic.rb
 - lib/logstash/plugin_mixins/aws_config/v2.rb
+- lib/tasks/build.rake
 - logstash-integration-aws.gemspec
 - spec/codecs/cloudfront_spec.rb
 - spec/codecs/cloudtrail_spec.rb
@@ -402,6 +405,7 @@ post_install_message:
 rdoc_options: []
 require_paths:
 - lib
+- vendor/jar-dependencies
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="