logstash-output-s3 4.3.4 → 4.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 841014c4e389a87dc3e63adb53303b5f6ea231221c45bc17b0eec5efb89a4bc6
-  data.tar.gz: a6f52f77844ce60a32a0fe05915e2865838a05505beee6ff095d402f2201d502
+  metadata.gz: 238e7be91fe40e4fcb80736f3e8d62e76b6f8108be35e5a0c054cb19bb239428
+  data.tar.gz: eb0c70181aa21d20b794cd05d2e458b322b10f6e343ae41d96a1e6f49cf85858
 SHA512:
-  metadata.gz: 211e4bb8bef9cfa55e8af453d456fa4dbe4670559a747a625b0bf91344c8c475d0a8f8d4e00eba070c7d32f81b39cf7af2ab9a8f248a18042678722ea7393165
-  data.tar.gz: bd1f5c0e3f3337b20cdc1c4d9e835b24dda0882b0b8d2ad7d47a55fe9658c91e6137ca5869869e7dee90d0a506601e7d1557287c7c7aa833c44678bd2ebaae01
+  metadata.gz: 1194c4ee3defe1104fcc1c68d914c20e2ca2548ba6405960c75d4b398a7cd9949287c3410ead913ff82b5183164c4bd5bf858fe836f94f63654c582efab78ab5
+  data.tar.gz: cb135c3b28297db0f5ad90a4f7ab4ba6b8c112574815e38bdccdec685f5caa28ed7636f68569586f71205d651118dad7cc7dd0194786efb59bfb8113fa7d0afb

data/CHANGELOG.md

@@ -1,17 +1,27 @@
+## 4.3.7
+  - Refactor: avoid usage of CHM (JRuby 9.3.4 work-around) [#248](https://github.com/logstash-plugins/logstash-output-s3/pull/248)
+
+## 4.3.6
+  - Docs: more documentation on restore + temp dir [#236](https://github.com/logstash-plugins/logstash-output-s3/pull/236)
+    * minor logging improvements - use the same path: naming convention
+
+## 4.3.5
+  - Feat: cast true/false values for additional_settings [#241](https://github.com/logstash-plugins/logstash-output-s3/pull/241)
+
 ## 4.3.4
-  -  [DOC] Added note about performance implications of interpolated strings in prefixes [#233](https://github.com/logstash-plugins/logstash-output-s3/pull/233)
+  - [DOC] Added note about performance implications of interpolated strings in prefixes [#233](https://github.com/logstash-plugins/logstash-output-s3/pull/233)
 
 ## 4.3.3
-  -  [DOC] Updated links to use shared attributes [#230](https://github.com/logstash-plugins/logstash-output-s3/pull/230)
+  - [DOC] Updated links to use shared attributes [#230](https://github.com/logstash-plugins/logstash-output-s3/pull/230)
 
 ## 4.3.2
-  -  [DOC] Added note that only AWS S3 is supported. No other S3 compatible storage solutions are supported. [#223](https://github.com/logstash-plugins/logstash-output-s3/pull/223)
+  - [DOC] Added note that only AWS S3 is supported. No other S3 compatible storage solutions are supported. [#223](https://github.com/logstash-plugins/logstash-output-s3/pull/223)
 
 ## 4.3.1
-  -  [DOC] Updated setting descriptions for clarity [#219](https://github.com/logstash-plugins/logstash-output-s3/pull/219) and [#220](https://github.com/logstash-plugins/logstash-output-s3/pull/220)
+  - [DOC] Updated setting descriptions for clarity [#219](https://github.com/logstash-plugins/logstash-output-s3/pull/219) and [#220](https://github.com/logstash-plugins/logstash-output-s3/pull/220)
 
 ## 4.3.0
-  -  Feat: Added retry_count and retry_delay config [#218](https://github.com/logstash-plugins/logstash-output-s3/pull/218)
+  - Feat: Added retry_count and retry_delay config [#218](https://github.com/logstash-plugins/logstash-output-s3/pull/218)
 
 ## 4.2.0
   - Added ability to specify [ONEZONE_IA](https://aws.amazon.com/s3/storage-classes/#__) as storage_class

data/docs/index.asciidoc

@@ -30,8 +30,9 @@ Other S3 compatible storage solutions are not supported.
 S3 outputs create temporary files into the OS' temporary directory. 
 You can specify where to save them using the `temporary_directory` option.
 
-IMPORTANT: For configurations containing multiple s3 outputs with the restore
-option enabled, each output should define its own 'temporary_directory'.
+IMPORTANT: For configurations containing multiple s3 outputs with the `restore`
+option enabled, each output should define its own `temporary_directory`.
+Shared or nested directories can cause data loss upon recovery.
 
 ===== Requirements
 
@@ -255,6 +256,10 @@ The AWS Region
 Used to enable recovery after crash/abnormal termination.
 Temporary log files will be recovered and uploaded.
 
+NOTE: If you're using multiple S3 outputs, always set
+<<plugins-{type}s-{plugin}-temporary_directory>> to a
+unique directory. Otherwise the recovery mechanism won't work correctly.
+
 [id="plugins-{type}s-{plugin}-retry_count"]
 ===== `retry_count`
 
@@ -388,7 +393,12 @@ Defaults to STANDARD.
   * Default value is `"/tmp/logstash"`
 
 Set the directory where logstash will store the tmp files before sending it to S3
-default to the current OS temporary directory in linux /tmp/logstash
+default to the current OS temporary directory in linux `/tmp/logstash`.
+
+WARNING: Using multiple S3 outputs with `restore => true` requires unique directories
+per output. All of the directory's contents are processed and deleted upon recovery, and shared or nested directories can cause data loss.
+For example, an output using `/tmp/s3` and a second configured with `/tmp/s3/sub` would
+cause issues. Having temporary directories `/tmp/s3/sub1` and `/tmp/s3/sub2` is fine.
 
 [id="plugins-{type}s-{plugin}-time_file"]
 ===== `time_file` 

data/lib/logstash/outputs/s3/file_repository.rb

@@ -1,11 +1,9 @@
 # encoding: utf-8
 require "java"
-require "concurrent"
+require "concurrent/map"
 require "concurrent/timer_task"
 require "logstash/util"
 
-ConcurrentHashMap = java.util.concurrent.ConcurrentHashMap
-
 module LogStash
   module Outputs
     class S3
@@ -41,7 +39,7 @@ module LogStash
         end
 
         class FactoryInitializer
-          include java.util.function.Function
+
           def initialize(tags, encoding, temporary_directory, stale_time)
             @tags = tags
             @encoding = encoding
@@ -49,9 +47,10 @@ module LogStash
             @stale_time = stale_time
           end
 
-          def apply(prefix_key)
+          def create_value(prefix_key)
             PrefixedValue.new(TemporaryFileFactory.new(prefix_key, @tags, @encoding, @temporary_directory), @stale_time)
           end
+
         end
 
         def initialize(tags, encoding, temporary_directory,
@@ -59,7 +58,7 @@ module LogStash
                        sweeper_interval = DEFAULT_STATE_SWEEPER_INTERVAL_SECS)
           # The path need to contains the prefix so when we start
           # logtash after a crash we keep the remote structure
-          @prefixed_factories =  ConcurrentHashMap.new
+          @prefixed_factories = Concurrent::Map.new
 
           @sweeper_interval = sweeper_interval
 
@@ -69,18 +68,19 @@ module LogStash
         end
 
         def keys
-          @prefixed_factories.keySet
+          @prefixed_factories.keys
         end
 
         def each_files
-          @prefixed_factories.elements.each do |prefixed_file|
+          @prefixed_factories.values.each do |prefixed_file|
             prefixed_file.with_lock { |factory| yield factory.current }
           end
         end
 
         # Return the file factory
         def get_factory(prefix_key)
-          @prefixed_factories.computeIfAbsent(prefix_key, @factory_initializer).with_lock { |factory| yield factory }
+          prefix_val = @prefixed_factories.fetch_or_store(prefix_key) { @factory_initializer.create_value(prefix_key) }
+          prefix_val.with_lock { |factory| yield factory }
         end
 
         def get_file(prefix_key)
@@ -97,7 +97,7 @@ module LogStash
 
         def remove_stale(k, v)
           if v.stale?
-            @prefixed_factories.remove(k, v)
+            @prefixed_factories.delete_pair(k, v)
             v.delete!
           end
         end
@@ -106,7 +106,7 @@ module LogStash
           @stale_sweeper = Concurrent::TimerTask.new(:execution_interval => @sweeper_interval) do
             LogStash::Util.set_thread_name("S3, Stale factory sweeper")
 
-            @prefixed_factories.forEach{|k,v| remove_stale(k,v)}
+            @prefixed_factories.each { |k, v| remove_stale(k,v) }
           end
 
           @stale_sweeper.execute

data/lib/logstash/outputs/s3.rb

@@ -110,7 +110,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   # Set the size of file in bytes, this means that files on bucket when have dimension > file_size, they are stored in two or more file.
   # If you have tags then it will generate a specific size file for every tags
-  ##NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.
+  #
+  # NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.
   config :size_file, :validate => :number, :default => 1024 * 1024 * 5
 
   # Set the time, in MINUTES, to close the current sub_time_section of bucket.
@@ -118,10 +119,10 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   # If it's valued 0 and rotation_strategy is 'time' or 'size_and_time' then the plugin reaise a configuration error.
   config :time_file, :validate => :number, :default => 15
 
-  ## IMPORTANT: if you use multiple instance of s3, you should specify on one of them the "restore=> true" and on the others "restore => false".
-  ## This is hack for not destroy the new files after restoring the initial files.
-  ## If you do not specify "restore => true" when logstash crashes or is restarted, the files are not sent into the bucket,
-  ## for example if you have single Instance.
+  # If `restore => false` is specified and Logstash crashes, the unprocessed files are not sent into the bucket.
+  #
+  # NOTE: that the `recovery => true` default assumes multiple S3 outputs would set a unique `temporary_directory => ...`
+  # if they do not than only a single S3 output is safe to recover (since let-over files are processed and deleted).
   config :restore, :validate => :boolean, :default => true
 
   # The S3 canned ACL to use when putting the file. Defaults to "private".
@@ -147,6 +148,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
   # Set the directory where logstash will store the tmp files before sending it to S3
   # default to the current OS temporary directory in linux /tmp/logstash
+  #
+  # NOTE: the reason we do not have a unique (isolated) temporary directory as a default, to support multiple plugin instances,
+  # is that we would have to rely on something static that does not change between restarts (e.g. a user set id => ...).
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
 
   # Specify a prefix to the uploaded filename, this can simulate directories on S3.  Prefix does not require leading slash.
@@ -283,17 +287,24 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def symbolized_settings
-    @symbolized_settings ||= symbolize_keys(@additional_settings)
+    @symbolized_settings ||= symbolize_keys_and_cast_true_false(@additional_settings)
   end
 
-  def symbolize_keys(hash)
-    return hash unless hash.is_a?(Hash)
-    symbolized = {}
-    hash.each { |key, value| symbolized[key.to_sym] = symbolize_keys(value) }
-    symbolized
+  def symbolize_keys_and_cast_true_false(hash)
+    case hash
+    when Hash
+      symbolized = {}
+      hash.each { |key, value| symbolized[key.to_sym] = symbolize_keys_and_cast_true_false(value) }
+      symbolized
+    when 'true'
+      true
+    when 'false'
+      false
+    else
+      hash
+    end
   end
 
-
   def normalize_key(prefix_key)
     prefix_key.gsub(PathValidator.matches_re, PREFIX_KEY_NORMALIZE_CHARACTER)
   end
@@ -340,10 +351,10 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
         temp_file = factory.current
 
         if @rotation.rotate?(temp_file)
-          @logger.debug("Rotate file",
-                        :strategy => @rotation.class.name,
-                        :key => temp_file.key,
-                        :path => temp_file.path)
+          @logger.debug? && @logger.debug("Rotate file",
+                                          :key => temp_file.key,
+                                          :path => temp_file.path,
+                                          :strategy => @rotation.class.name)
 
           upload_file(temp_file)
           factory.rotate!
@@ -353,7 +364,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def upload_file(temp_file)
-    @logger.debug("Queue for upload", :path => temp_file.path)
+    @logger.debug? && @logger.debug("Queue for upload", :path => temp_file.path)
 
     # if the queue is full the calling thread will be used to upload
     temp_file.close # make sure the content is on disk
@@ -376,7 +387,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def clean_temporary_file(file)
-    @logger.debug("Removing temporary file", :file => file.path)
+    @logger.debug? && @logger.debug("Removing temporary file", :path => file.path)
     file.delete!
   end
 
@@ -391,7 +402,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       .each do |file|
       temp_file = TemporaryFile.create_from_existing_file(file, temp_folder_path)
       if temp_file.size > 0
-        @logger.debug("Recovering from crash and uploading", :file => temp_file.path)
+        @logger.debug? && @logger.debug("Recovering from crash and uploading", :path => temp_file.path)
         @crash_uploader.upload_async(temp_file, :on_complete => method(:clean_temporary_file), :upload_options => upload_options)
       else
         clean_temporary_file(temp_file)

data/logstash-output-s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-s3'
-  s.version         = '4.3.4'
+  s.version         = '4.3.7'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Sends Logstash events to the Amazon Simple Storage Service"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/s3/file_repository_spec.rb

@@ -88,8 +88,8 @@ describe LogStash::Outputs::S3::FileRepository do
 
   it "returns all available keys" do
     subject.get_file(prefix_key) { |file| file.write("something") }
-    expect(subject.keys.toArray).to include(prefix_key)
-    expect(subject.keys.toArray.size).to eq(1)
+    expect(subject.keys).to include(prefix_key)
+    expect(subject.keys.size).to eq(1)
   end
 
   it "clean stale factories" do
@@ -105,9 +105,14 @@ describe LogStash::Outputs::S3::FileRepository do
 
     @file_repository.get_file("another-prefix") { |file| file.write("hello") }
     expect(@file_repository.size).to eq(2)
+    sleep 1.2 # allow sweeper to kick in
     try(10) { expect(@file_repository.size).to eq(1) }
     expect(File.directory?(path)).to be_falsey
+
+    sleep 1.5 # allow sweeper to kick in, again
+    expect(@file_repository.size).to eq(1)
   end
+
 end
 
 

data/spec/outputs/s3_spec.rb

@@ -140,7 +140,7 @@ describe LogStash::Outputs::S3 do
 
     describe "temporary directory" do
       let(:temporary_directory) { Stud::Temporary.pathname }
-      let(:options) { super.merge({ "temporary_directory" => temporary_directory }) }
+      let(:options) { super().merge({ "temporary_directory" => temporary_directory }) }
 
       it "creates the directory when it doesn't exist" do
         expect(Dir.exist?(temporary_directory)).to be_falsey
@@ -160,13 +160,15 @@ describe LogStash::Outputs::S3 do
     end
 
     describe "additional_settings" do
-      context "when enabling force_path_style" do
+      context "supported settings" do
         let(:additional_settings) do
-          { "additional_settings" => { "force_path_style" => true } }
+          { "additional_settings" => { "force_path_style" => 'true', "ssl_verify_peer" => 'false', "profile" => 'logstash' } }
         end
 
         it "validates the prefix" do
-          expect(Aws::S3::Bucket).to receive(:new).twice.with(anything, hash_including(:force_path_style => true)).and_call_original
+          expect(Aws::S3::Bucket).to receive(:new).twice.
+              with(anything, hash_including(:force_path_style => true, :ssl_verify_peer => false, :profile => 'logstash')).
+              and_call_original
           described_class.new(options.merge(additional_settings)).register
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 4.3.4
+  version: 4.3.7
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-02 00:00:00.000000000 Z
+date: 2022-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -185,8 +185,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Sends Logstash events to the Amazon Simple Storage Service