logstash-output-s3 4.3.5 → 4.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc8cc704a468f0576b34c9fad38dd2506ea303c4b8671e8f575a48c7e1f4e602
-  data.tar.gz: d39aaebe320c7a3dd109a5b35b3e5eaf9b06c9f382b88ca934e51c4ae990491d
+  metadata.gz: 4671860be7ad0735c12ef8f886c157dce2990975a30022a07b9f95cfe5187e3b
+  data.tar.gz: e17d8f575abcd12968b5a4065be4786f904536aa0e77a2c48854e20ac259ddf3
 SHA512:
-  metadata.gz: 67c47cbdcf47983beed7edd157541f5d614c300c6c28f2eccd9ed039a1cd1b29eeab4f512c7b45e3a8f2936a128e95ca028f4b4b94797c8217bd9144bad6e6ab
-  data.tar.gz: faeeb2f29ac962ebc50dc2a874617f85f3d7dbb882ab754b7a6fc571c721d416ddb25e63ce01ab951b18b9585e3b287faadea612b99790dee624ec82c61303a4
+  metadata.gz: 3c20213ce22c85afc5512adb8641aa72c365f0644b42939acd40fb189c846118153523b8de749bb8b19e866faed7c80e870494d61c4f99b1691417f0618fcc8e
+  data.tar.gz: 046f0d0f0ceaf6cc69c3494348d0e6b867c919be4d1a053b3c1fcbe0fe5591e47a97fd7e71b2606fa0949a72fc807bf1877a73cb4cbdd30bf091d91317c7a261

data/CHANGELOG.md

@@ -1,20 +1,24 @@
+## 4.3.6
+  - Docs: more documentation on restore + temp dir [#236](https://github.com/logstash-plugins/logstash-output-s3/pull/236)
+    * minor logging improvements - use the same path: naming convention
+
 ## 4.3.5
-  -  Feat: cast true/false values for additional_settings [#241](https://github.com/logstash-plugins/logstash-output-s3/pull/241)
+  - Feat: cast true/false values for additional_settings [#241](https://github.com/logstash-plugins/logstash-output-s3/pull/241)
 
 ## 4.3.4
-  -  [DOC] Added note about performance implications of interpolated strings in prefixes [#233](https://github.com/logstash-plugins/logstash-output-s3/pull/233)
+  - [DOC] Added note about performance implications of interpolated strings in prefixes [#233](https://github.com/logstash-plugins/logstash-output-s3/pull/233)
 
 ## 4.3.3
-  -  [DOC] Updated links to use shared attributes [#230](https://github.com/logstash-plugins/logstash-output-s3/pull/230)
+  - [DOC] Updated links to use shared attributes [#230](https://github.com/logstash-plugins/logstash-output-s3/pull/230)
 
 ## 4.3.2
-  -  [DOC] Added note that only AWS S3 is supported. No other S3 compatible storage solutions are supported. [#223](https://github.com/logstash-plugins/logstash-output-s3/pull/223)
+  - [DOC] Added note that only AWS S3 is supported. No other S3 compatible storage solutions are supported. [#223](https://github.com/logstash-plugins/logstash-output-s3/pull/223)
 
 ## 4.3.1
-  -  [DOC] Updated setting descriptions for clarity [#219](https://github.com/logstash-plugins/logstash-output-s3/pull/219) and [#220](https://github.com/logstash-plugins/logstash-output-s3/pull/220)
+  - [DOC] Updated setting descriptions for clarity [#219](https://github.com/logstash-plugins/logstash-output-s3/pull/219) and [#220](https://github.com/logstash-plugins/logstash-output-s3/pull/220)
 
 ## 4.3.0
-  -  Feat: Added retry_count and retry_delay config [#218](https://github.com/logstash-plugins/logstash-output-s3/pull/218)
+  - Feat: Added retry_count and retry_delay config [#218](https://github.com/logstash-plugins/logstash-output-s3/pull/218)
 
 ## 4.2.0
   - Added ability to specify [ONEZONE_IA](https://aws.amazon.com/s3/storage-classes/#__) as storage_class

data/docs/index.asciidoc

@@ -30,8 +30,9 @@ Other S3 compatible storage solutions are not supported.
 S3 outputs create temporary files into the OS' temporary directory. 
 You can specify where to save them using the `temporary_directory` option.
 
-IMPORTANT: For configurations containing multiple s3 outputs with the restore
-option enabled, each output should define its own 'temporary_directory'.
+IMPORTANT: For configurations containing multiple s3 outputs with the `restore`
+option enabled, each output should define its own `temporary_directory`.
+Shared or nested directories can cause data loss upon recovery.
 
 ===== Requirements
 
@@ -255,6 +256,10 @@ The AWS Region
 Used to enable recovery after crash/abnormal termination.
 Temporary log files will be recovered and uploaded.
 
+NOTE: If you're using multiple S3 outputs, always set
+<<plugins-{type}s-{plugin}-temporary_directory>> to a
+unique directory. Otherwise the recovery mechanism won't work correctly.
+
 [id="plugins-{type}s-{plugin}-retry_count"]
 ===== `retry_count`
 
@@ -388,7 +393,12 @@ Defaults to STANDARD.
   * Default value is `"/tmp/logstash"`
 
 Set the directory where logstash will store the tmp files before sending it to S3
-default to the current OS temporary directory in linux /tmp/logstash
+default to the current OS temporary directory in linux `/tmp/logstash`.
+
+WARNING: Using multiple S3 outputs with `restore => true` requires unique directories
+per output. All of the directory's contents are processed and deleted upon recovery, and shared or nested directories can cause data loss.
+For example, an output using `/tmp/s3` and a second configured with `/tmp/s3/sub` would
+cause issues. Having temporary directories `/tmp/s3/sub1` and `/tmp/s3/sub2` is fine.
 
 [id="plugins-{type}s-{plugin}-time_file"]
 ===== `time_file` 

data/lib/logstash/outputs/s3/file_repository.rb

@@ -4,8 +4,6 @@ require "concurrent"
 require "concurrent/timer_task"
 require "logstash/util"
 
-ConcurrentHashMap = java.util.concurrent.ConcurrentHashMap
-
 module LogStash
   module Outputs
     class S3
@@ -59,7 +57,7 @@ module LogStash
                        sweeper_interval = DEFAULT_STATE_SWEEPER_INTERVAL_SECS)
           # The path need to contains the prefix so when we start
           # logtash after a crash we keep the remote structure
-          @prefixed_factories =  ConcurrentHashMap.new
+          @prefixed_factories = java.util.concurrent.ConcurrentHashMap.new
 
           @sweeper_interval = sweeper_interval
 

data/lib/logstash/outputs/s3.rb

@@ -110,7 +110,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   # Set the size of file in bytes, this means that files on bucket when have dimension > file_size, they are stored in two or more file.
   # If you have tags then it will generate a specific size file for every tags
-  ##NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.
+  #
+  # NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.
   config :size_file, :validate => :number, :default => 1024 * 1024 * 5
 
   # Set the time, in MINUTES, to close the current sub_time_section of bucket.
@@ -118,10 +119,10 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   # If it's valued 0 and rotation_strategy is 'time' or 'size_and_time' then the plugin reaise a configuration error.
   config :time_file, :validate => :number, :default => 15
 
-  ## IMPORTANT: if you use multiple instance of s3, you should specify on one of them the "restore=> true" and on the others "restore => false".
-  ## This is hack for not destroy the new files after restoring the initial files.
-  ## If you do not specify "restore => true" when logstash crashes or is restarted, the files are not sent into the bucket,
-  ## for example if you have single Instance.
+  # If `restore => false` is specified and Logstash crashes, the unprocessed files are not sent into the bucket.
+  #
+  # NOTE: that the `recovery => true` default assumes multiple S3 outputs would set a unique `temporary_directory => ...`
+  # if they do not than only a single S3 output is safe to recover (since let-over files are processed and deleted).
   config :restore, :validate => :boolean, :default => true
 
   # The S3 canned ACL to use when putting the file. Defaults to "private".
@@ -147,6 +148,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   # Set the directory where logstash will store the tmp files before sending it to S3
   # default to the current OS temporary directory in linux /tmp/logstash
+  #
+  # NOTE: the reason we do not have a unique (isolated) temporary directory as a default, to support multiple plugin instances,
+  # is that we would have to rely on something static that does not change between restarts (e.g. a user set id => ...).
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
 
   # Specify a prefix to the uploaded filename, this can simulate directories on S3.  Prefix does not require leading slash.
@@ -347,10 +351,10 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
         temp_file = factory.current
 
         if @rotation.rotate?(temp_file)
-          @logger.debug("Rotate file",
-                        :strategy => @rotation.class.name,
-                        :key => temp_file.key,
-                        :path => temp_file.path)
+          @logger.debug? && @logger.debug("Rotate file",
+                                          :key => temp_file.key,
+                                          :path => temp_file.path,
+                                          :strategy => @rotation.class.name)
 
           upload_file(temp_file)
           factory.rotate!
@@ -360,7 +364,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def upload_file(temp_file)
-    @logger.debug("Queue for upload", :path => temp_file.path)
+    @logger.debug? && @logger.debug("Queue for upload", :path => temp_file.path)
 
     # if the queue is full the calling thread will be used to upload
     temp_file.close # make sure the content is on disk
@@ -383,7 +387,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def clean_temporary_file(file)
-    @logger.debug("Removing temporary file", :file => file.path)
+    @logger.debug? && @logger.debug("Removing temporary file", :path => file.path)
     file.delete!
   end
 
@@ -398,7 +402,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       .each do |file|
       temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
       if temp_file.size > 0
-        @logger.debug("Recovering from crash and uploading", :file => temp_file.path)
+        @logger.debug? && @logger.debug("Recovering from crash and uploading", :path => temp_file.path)
         @crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file), :upload_options => upload_options)
       else
         clean_temporary_file(temp_file)

data/logstash-output-s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-s3'
-  s.version         = '4.3.5'
+  s.version         = '4.3.6'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Sends Logstash events to the Amazon Simple Storage Service"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/s3/file_repository_spec.rb

@@ -88,8 +88,8 @@ describe LogStash::Outputs::S3::FileRepository do
 
   it "returns all available keys" do
     subject.get_file(prefix_key) { |file| file.write("something") }
-    expect(subject.keys.toArray).to include(prefix_key)
-    expect(subject.keys.toArray.size).to eq(1)
+    expect(subject.keys).to include(prefix_key)
+    expect(subject.keys.to_a.size).to eq(1)
   end
 
   it "clean stale factories" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 4.3.5
+  version: 4.3.6
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-06 00:00:00.000000000 Z
+date: 2022-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement