logstash-output-s3 4.3.3 → 4.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ddee0a9caa6ff45882472bb1ef9d198e5263d1da62feca4cda463ecf32a85192
-  data.tar.gz: da07f987033ceecb078fd32a658f85502492b1364622aa77fc384c5eb13c702c
+  metadata.gz: 4671860be7ad0735c12ef8f886c157dce2990975a30022a07b9f95cfe5187e3b
+  data.tar.gz: e17d8f575abcd12968b5a4065be4786f904536aa0e77a2c48854e20ac259ddf3
 SHA512:
-  metadata.gz: 2da28864bab1b28e28830055d7a21f63216ab00e9a4385638afb4f7f1034f61c208ed50487976f6c832761c1f4456ad39fda621c3d24af8a242ff38e00ac6268
-  data.tar.gz: 8d5e29f0d672856594a1ca87a844ace560d2f054461141670fb18a1acfc6eeccb797c8e8492b37795859f69fb85f12abc42ade74b5348fc6bc1c08b7b3c4054a
+  metadata.gz: 3c20213ce22c85afc5512adb8641aa72c365f0644b42939acd40fb189c846118153523b8de749bb8b19e866faed7c80e870494d61c4f99b1691417f0618fcc8e
+  data.tar.gz: 046f0d0f0ceaf6cc69c3494348d0e6b867c919be4d1a053b3c1fcbe0fe5591e47a97fd7e71b2606fa0949a72fc807bf1877a73cb4cbdd30bf091d91317c7a261

data/CHANGELOG.md

@@ -1,14 +1,24 @@
+## 4.3.6
+  - Docs: more documentation on restore + temp dir [#236](https://github.com/logstash-plugins/logstash-output-s3/pull/236)
+    * minor logging improvements - use the same path: naming convention
+
+## 4.3.5
+  - Feat: cast true/false values for additional_settings [#241](https://github.com/logstash-plugins/logstash-output-s3/pull/241)
+
+## 4.3.4
+  - [DOC] Added note about performance implications of interpolated strings in prefixes [#233](https://github.com/logstash-plugins/logstash-output-s3/pull/233)
+
 ## 4.3.3
-  -  [DOC] Update links to use shared attributes [#230](https://github.com/logstash-plugins/logstash-output-s3/pull/230)
+  - [DOC] Updated links to use shared attributes [#230](https://github.com/logstash-plugins/logstash-output-s3/pull/230)
 
 ## 4.3.2
-  -  [DOC] Added note that only AWS S3 is supported. No other S3 compatible storage solutions are supported. [#223](https://github.com/logstash-plugins/logstash-output-s3/pull/223)
+  - [DOC] Added note that only AWS S3 is supported. No other S3 compatible storage solutions are supported. [#223](https://github.com/logstash-plugins/logstash-output-s3/pull/223)
 
 ## 4.3.1
-  -  [DOC] Updated setting descriptions for clarity [#219](https://github.com/logstash-plugins/logstash-output-s3/pull/219) and [#220](https://github.com/logstash-plugins/logstash-output-s3/pull/220)
+  - [DOC] Updated setting descriptions for clarity [#219](https://github.com/logstash-plugins/logstash-output-s3/pull/219) and [#220](https://github.com/logstash-plugins/logstash-output-s3/pull/220)
 
 ## 4.3.0
-  -  Feat: Added retry_count and retry_delay config [#218](https://github.com/logstash-plugins/logstash-output-s3/pull/218)
+  - Feat: Added retry_count and retry_delay config [#218](https://github.com/logstash-plugins/logstash-output-s3/pull/218)
 
 ## 4.2.0
   - Added ability to specify [ONEZONE_IA](https://aws.amazon.com/s3/storage-classes/#__) as storage_class

data/docs/index.asciidoc

@@ -30,8 +30,9 @@ Other S3 compatible storage solutions are not supported.
 S3 outputs create temporary files into the OS' temporary directory. 
 You can specify where to save them using the `temporary_directory` option.
 
-IMPORTANT: For configurations containing multiple s3 outputs with the restore
-option enabled, each output should define its own 'temporary_directory'.
+IMPORTANT: For configurations containing multiple s3 outputs with the `restore`
+option enabled, each output should define its own `temporary_directory`.
+Shared or nested directories can cause data loss upon recovery.
 
 ===== Requirements
 
@@ -217,13 +218,18 @@ The endpoint should be an HTTP or HTTPS URL, e.g. https://example.com
   * Value type is <<string,string>>
   * Default value is `""`
 
-Specify a prefix to the uploaded filename, this can simulate directories on S3.
+Specify a prefix to the uploaded filename to simulate directories on S3.
 Prefix does not require leading slash.
 This option supports
 {logstash-ref}/event-dependent-configuration.html#sprintf[Logstash
-interpolation]; for example, files can be prefixed with the event date using
+interpolation]. For example, files can be prefixed with the event date using
 `prefix = "%{+YYYY}/%{+MM}/%{+dd}"`.
-Be warned this can create a lot of temporary local files.
+
+IMPORTANT: Take care when you are using interpolated strings in prefixes. This
+has the potential to create large numbers of unique prefixes, causing large
+numbers of in-progress uploads. This scenario may result in performance and
+stability issues, which can be further exacerbated when you use a
+rotation_strategy that delays uploads.
 
 [id="plugins-{type}s-{plugin}-proxy_uri"]
 ===== `proxy_uri` 
@@ -250,6 +256,10 @@ The AWS Region
 Used to enable recovery after crash/abnormal termination.
 Temporary log files will be recovered and uploaded.
 
+NOTE: If you're using multiple S3 outputs, always set
+<<plugins-{type}s-{plugin}-temporary_directory>> to a
+unique directory. Otherwise the recovery mechanism won't work correctly.
+
 [id="plugins-{type}s-{plugin}-retry_count"]
 ===== `retry_count`
 
@@ -383,7 +393,12 @@ Defaults to STANDARD.
   * Default value is `"/tmp/logstash"`
 
 Set the directory where logstash will store the tmp files before sending it to S3
-default to the current OS temporary directory in linux /tmp/logstash
+default to the current OS temporary directory in linux `/tmp/logstash`.
+
+WARNING: Using multiple S3 outputs with `restore => true` requires unique directories
+per output. All of the directory's contents are processed and deleted upon recovery, and shared or nested directories can cause data loss.
+For example, an output using `/tmp/s3` and a second configured with `/tmp/s3/sub` would
+cause issues. Having temporary directories `/tmp/s3/sub1` and `/tmp/s3/sub2` is fine.
 
 [id="plugins-{type}s-{plugin}-time_file"]
 ===== `time_file` 

data/lib/logstash/outputs/s3/file_repository.rb

@@ -4,8 +4,6 @@ require "concurrent"
 require "concurrent/timer_task"
 require "logstash/util"
 
-ConcurrentHashMap = java.util.concurrent.ConcurrentHashMap
-
 module LogStash
   module Outputs
     class S3
@@ -59,7 +57,7 @@ module LogStash
                        sweeper_interval = DEFAULT_STATE_SWEEPER_INTERVAL_SECS)
           # The path need to contains the prefix so when we start
           # logtash after a crash we keep the remote structure
-          @prefixed_factories =  ConcurrentHashMap.new
+          @prefixed_factories = java.util.concurrent.ConcurrentHashMap.new
 
           @sweeper_interval = sweeper_interval
 

data/lib/logstash/outputs/s3.rb

@@ -110,7 +110,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   # Set the size of file in bytes, this means that files on bucket when have dimension > file_size, they are stored in two or more file.
   # If you have tags then it will generate a specific size file for every tags
-  ##NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.
+  #
+  # NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.
   config :size_file, :validate => :number, :default => 1024 * 1024 * 5
 
   # Set the time, in MINUTES, to close the current sub_time_section of bucket.
@@ -118,10 +119,10 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   # If it's valued 0 and rotation_strategy is 'time' or 'size_and_time' then the plugin reaise a configuration error.
   config :time_file, :validate => :number, :default => 15
 
-  ## IMPORTANT: if you use multiple instance of s3, you should specify on one of them the "restore=> true" and on the others "restore => false".
-  ## This is hack for not destroy the new files after restoring the initial files.
-  ## If you do not specify "restore => true" when logstash crashes or is restarted, the files are not sent into the bucket,
-  ## for example if you have single Instance.
+  # If `restore => false` is specified and Logstash crashes, the unprocessed files are not sent into the bucket.
+  #
+  # NOTE: that the `recovery => true` default assumes multiple S3 outputs would set a unique `temporary_directory => ...`
+  # if they do not than only a single S3 output is safe to recover (since let-over files are processed and deleted).
   config :restore, :validate => :boolean, :default => true
 
   # The S3 canned ACL to use when putting the file. Defaults to "private".
@@ -147,6 +148,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   # Set the directory where logstash will store the tmp files before sending it to S3
   # default to the current OS temporary directory in linux /tmp/logstash
+  #
+  # NOTE: the reason we do not have a unique (isolated) temporary directory as a default, to support multiple plugin instances,
+  # is that we would have to rely on something static that does not change between restarts (e.g. a user set id => ...).
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
 
   # Specify a prefix to the uploaded filename, this can simulate directories on S3.  Prefix does not require leading slash.
@@ -283,17 +287,24 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def symbolized_settings
-    @symbolized_settings ||= symbolize_keys(@additional_settings)
+    @symbolized_settings ||= symbolize_keys_and_cast_true_false(@additional_settings)
   end
 
-  def symbolize_keys(hash)
-    return hash unless hash.is_a?(Hash)
-    symbolized = {}
-    hash.each { |key, value| symbolized[key.to_sym] = symbolize_keys(value) }
-    symbolized
+  def symbolize_keys_and_cast_true_false(hash)
+    case hash
+    when Hash
+      symbolized = {}
+      hash.each { |key, value| symbolized[key.to_sym] = symbolize_keys_and_cast_true_false(value) }
+      symbolized
+    when 'true'
+      true
+    when 'false'
+      false
+    else
+      hash
+    end
   end
 
-
   def normalize_key(prefix_key)
     prefix_key.gsub(PathValidator.matches_re, PREFIX_KEY_NORMALIZE_CHARACTER)
   end
@@ -340,10 +351,10 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
         temp_file = factory.current
 
         if @rotation.rotate?(temp_file)
-          @logger.debug("Rotate file",
-                        :strategy => @rotation.class.name,
-                        :key => temp_file.key,
-                        :path => temp_file.path)
+          @logger.debug? && @logger.debug("Rotate file",
+                                          :key => temp_file.key,
+                                          :path => temp_file.path,
+                                          :strategy => @rotation.class.name)
 
           upload_file(temp_file)
           factory.rotate!
@@ -353,7 +364,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def upload_file(temp_file)
-    @logger.debug("Queue for upload", :path => temp_file.path)
+    @logger.debug? && @logger.debug("Queue for upload", :path => temp_file.path)
 
     # if the queue is full the calling thread will be used to upload
     temp_file.close # make sure the content is on disk
@@ -376,7 +387,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def clean_temporary_file(file)
-    @logger.debug("Removing temporary file", :file => file.path)
+    @logger.debug? && @logger.debug("Removing temporary file", :path => file.path)
     file.delete!
   end
 
@@ -391,7 +402,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       .each do |file|
       temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
       if temp_file.size > 0
-        @logger.debug("Recovering from crash and uploading", :file => temp_file.path)
+        @logger.debug? && @logger.debug("Recovering from crash and uploading", :path => temp_file.path)
         @crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file), :upload_options => upload_options)
       else
         clean_temporary_file(temp_file)

data/logstash-output-s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-s3'
-  s.version         = '4.3.3'
+  s.version         = '4.3.6'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Sends Logstash events to the Amazon Simple Storage Service"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/s3/file_repository_spec.rb

@@ -88,8 +88,8 @@ describe LogStash::Outputs::S3::FileRepository do
 
   it "returns all available keys" do
     subject.get_file(prefix_key) { |file| file.write("something") }
-    expect(subject.keys.toArray).to include(prefix_key)
-    expect(subject.keys.toArray.size).to eq(1)
+    expect(subject.keys).to include(prefix_key)
+    expect(subject.keys.to_a.size).to eq(1)
   end
 
   it "clean stale factories" do

data/spec/outputs/s3_spec.rb

@@ -140,7 +140,7 @@ describe LogStash::Outputs::S3 do
 
     describe "temporary directory" do
       let(:temporary_directory) { Stud::Temporary.pathname }
-      let(:options) { super.merge({ "temporary_directory" => temporary_directory }) }
+      let(:options) { super().merge({ "temporary_directory" => temporary_directory }) }
 
       it "creates the directory when it doesn't exist" do
         expect(Dir.exist?(temporary_directory)).to be_falsey
@@ -160,13 +160,15 @@ describe LogStash::Outputs::S3 do
     end
 
     describe "additional_settings" do
-      context "when enabling force_path_style" do
+      context "supported settings" do
         let(:additional_settings) do
-          { "additional_settings" => { "force_path_style" => true } }
+          { "additional_settings" => { "force_path_style" => 'true', "ssl_verify_peer" => 'false', "profile" => 'logstash' } }
         end
 
         it "validates the prefix" do
-          expect(Aws::S3::Bucket).to receive(:new).twice.with(anything, hash_including(:force_path_style => true)).and_call_original
+          expect(Aws::S3::Bucket).to receive(:new).twice.
+              with(anything, hash_including(:force_path_style => true, :ssl_verify_peer => false, :profile => 'logstash')).
+              and_call_original
           described_class.new(options.merge(additional_settings)).register
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 4.3.3
+  version: 4.3.6
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-19 00:00:00.000000000 Z
+date: 2022-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -185,8 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Sends Logstash events to the Amazon Simple Storage Service