logstash-output-s3 4.2.0 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 890ff03d14bda56d56fe801c3432bd7a4da11643fd160c52a2b7caf238a4fb8d
-  data.tar.gz: ea0fb6181b73b91d8d1c18a89ed92329abae4c23d5f55e85d06a8789f2d05667
+  metadata.gz: 1d9a7a272abc73ba56936d68d273ec4042421d0082fc3e92e5e27dda3e93af21
+  data.tar.gz: 2a583ab3d5d69aef1ce03c98ec24e5f621d64e56517ed552fdc0495adecee24b
 SHA512:
-  metadata.gz: 1134e3aa8531cbb29716f4db08513ed15fa6a0013efda15cda54e8dd3b2d5fa885148988662494119d680fd1ff597c232f48f67816bde953e35b3e93e5b0ee4e
-  data.tar.gz: 21f03d765f73b4bbb52779a00a6879dd3bb99714b67dfefc042e3d683692ddefeb299a7c182d297a218a9dbe7c93813728205677a3006aa87c9f12b55644800b
+  metadata.gz: dcae487d4c4a129aa7d7c53883f680807b2bbcf472566099fbca39316a685e9ae965aecb1f880f60227815097b13f1a773f0a1bd6d1f89bd236c809502141a6f
+  data.tar.gz: 9bac6b7e4bad12b8d3766dbf82ab6f8f1cbd3a366dee87b93ab12df21f380b458d233f4dbc87dae3cd787f5cb968468eea004389eee36ffb0d8806ab70fe36c5

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.3.0
+  -  Feat: Added retry_count and retry_delay config [#218](https://github.com/logstash-plugins/logstash-output-s3/pull/218)
+
 ## 4.2.0
   - Added ability to specify [ONEZONE_IA](https://aws.amazon.com/s3/storage-classes/#__) as storage_class
 

data/docs/index.asciidoc

@@ -90,6 +90,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-region>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-restore>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-retry_count>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-retry_delay>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-role_arn>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-role_session_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-rotation_strategy>> |<<string,string>>, one of `["size_and_time", "size", "time"]`|No
@@ -241,6 +243,22 @@ The AWS Region
 Used to enable recovery after crash/abnormal termination.
 Temporary log files will be recovered and uploaded.
 
+[id="plugins-{type}s-{plugin}-retry_count"]
+===== `retry_count`
+
+* Value type is <<number,number>>
+* Default value is `Infinity`
+
+Allows to limit number of retries when S3 uploading fails.
+
+[id="plugins-{type}s-{plugin}-retry_delay"]
+===== `retry_delay`
+
+* Value type is <<number,number>>
+* Default value is `1`
+
+Delay (in seconds) to wait between consecutive retries on upload failures.
+
 [id="plugins-{type}s-{plugin}-role_arn"]
 ===== `role_arn`
 
@@ -315,7 +333,7 @@ specified here
   * Value type is <<number,number>>
   * Default value is `5242880`
 
-Set the size of file in bytes, this means that files on bucket when have dimension > file_size, they are stored in two or more file.
+Set the size of file in bytes, this means that files on bucket when dimension > file_size, are stored in two or more files.
 If you have tags then it will generate a specific size file for every tags
 
 [id="plugins-{type}s-{plugin}-ssekms_key_id"]
@@ -390,7 +408,7 @@ Specify how many workers to use to upload the files to S3
   * Default value is `true`
 
 The common use case is to define permission on the root bucket and give Logstash full access to write its logs.
-In some circumstances you need finer grained permission on subfolder, this allow you to disable the check at startup.
+In some circumstances you need finer grained permission on subfolder, this allows you to disable the check at startup.
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/outputs/s3.rb

@@ -188,6 +188,12 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   # In some circonstances you need finer grained permission on subfolder, this allow you to disable the check at startup.
   config :validate_credentials_on_root_bucket, :validate => :boolean, :default => true
 
+  # The number of times to retry a failed S3 upload.
+  config :retry_count, :validate => :number, :default => Float::INFINITY
+
+  # The amount of time to wait in seconds before attempting to retry a failed upload.
+  config :retry_delay, :validate => :number, :default => 1
+
   def register
     # I've move the validation of the items into custom classes
     # to prepare for the new config validation that will be part of the core so the core can
@@ -219,7 +225,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
                                                     :max_queue => @upload_queue_size,
                                                     :fallback_policy => :caller_runs })
 
-    @uploader = Uploader.new(bucket_resource, @logger, executor)
+    @uploader = Uploader.new(bucket_resource, @logger, executor, retry_count: @retry_count, retry_delay: @retry_delay)
 
     # Restoring from crash will use a new threadpool to slowly recover
     # New events should have more priority.
@@ -326,10 +332,6 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
     Aws::S3::Bucket.new(@bucket, full_options)
   end
 
-  def aws_service_endpoint(region)
-    { :s3_endpoint => region == 'us-east-1' ? 's3.amazonaws.com' : "s3-#{region}.amazonaws.com"}
-  end
-
   def rotate_if_needed(prefixes)
     prefixes.each do |prefix|
       # Each file access is thread safe,

data/lib/logstash/outputs/s3/uploader.rb

@@ -6,7 +6,7 @@ module LogStash
   module Outputs
     class S3
       class Uploader
-        TIME_BEFORE_RETRYING_SECONDS = 1
+
         DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new({
                                                                   :min_threads => 1,
                                                                   :max_threads => 8,
@@ -14,13 +14,14 @@ module LogStash
                                                                   :fallback_policy => :caller_runs
                                                                 })
 
-
         attr_reader :bucket, :upload_options, :logger
 
-        def initialize(bucket, logger, threadpool = DEFAULT_THREADPOOL)
+        def initialize(bucket, logger, threadpool = DEFAULT_THREADPOOL, retry_count: Float::INFINITY, retry_delay: 1)
           @bucket = bucket
           @workers_pool = threadpool
           @logger = logger
+          @retry_count = retry_count
+          @retry_delay = retry_delay
         end
 
         def upload_async(file, options = {})
@@ -33,6 +34,7 @@ module LogStash
         def upload(file, options = {})
           upload_options = options.fetch(:upload_options, {})
 
+          tries = 0
           begin
             obj = bucket.object(file.key)
             obj.upload_file(file.path, upload_options)
@@ -44,15 +46,22 @@ module LogStash
             #
             # Thread might be stuck here, but I think its better than losing anything
             # its either a transient errors or something bad really happened.
-            logger.error("Uploading failed, retrying.", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
-            sleep TIME_BEFORE_RETRYING_SECONDS
-            retry
+            if tries < @retry_count
+              tries += 1
+              logger.warn("Uploading failed, retrying (##{tries} of #{@retry_count})", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+              sleep @retry_delay
+              retry
+            else
+              logger.error("Failed to upload file (retried #{@retry_count} times).", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+            end
           end
 
-          options[:on_complete].call(file) unless options[:on_complete].nil?
-        rescue => e
-          logger.error("An error occured in the `on_complete` uploader", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
-          raise e # reraise it since we don't deal with it now
+          begin
+            options[:on_complete].call(file) unless options[:on_complete].nil?
+          rescue => e
+            logger.error("An error occurred in the `on_complete` uploader", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
+            raise e # reraise it since we don't deal with it now
+          end
         end
 
         def stop

data/logstash-output-s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-s3'
-  s.version         = '4.2.0'
+  s.version         = '4.3.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Sends Logstash events to the Amazon Simple Storage Service"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/s3/uploader_spec.rb

@@ -6,7 +6,7 @@ require "aws-sdk"
 require "stud/temporary"
 
 describe LogStash::Outputs::S3::Uploader do
-  let(:logger) { spy(:logger ) }
+  let(:logger) { spy(:logger) }
   let(:max_upload_workers) { 1 }
   let(:bucket_name) { "foobar-bucket" }
   let(:client) { Aws::S3::Client.new(stub_responses: true) }
@@ -14,7 +14,6 @@ describe LogStash::Outputs::S3::Uploader do
   let(:temporary_directory) { Stud::Temporary.pathname }
   let(:temporary_file) { Stud::Temporary.file }
   let(:key) { "foobar" }
-  let(:upload_options) { {} }
   let(:threadpool) do
     Concurrent::ThreadPoolExecutor.new({
                                          :min_threads => 1,
@@ -31,27 +30,41 @@ describe LogStash::Outputs::S3::Uploader do
     f
   end
 
-  subject { described_class.new(bucket, logger, threadpool) }
+  subject { described_class.new(bucket, logger, threadpool, retry_delay: 0.01) }
 
   it "upload file to the s3 bucket" do
     expect { subject.upload(file) }.not_to raise_error
   end
 
   it "execute a callback when the upload is complete" do
-    callback = proc { |f| }
+    callback = proc { |_| }
 
     expect(callback).to receive(:call).with(file)
-    subject.upload(file, { :on_complete => callback })
+    subject.upload(file, :on_complete => callback)
   end
 
-  it "retries errors indefinitively" do
+  it "retries errors indefinitely" do
     s3 = double("s3").as_null_object
 
-    expect(logger).to receive(:error).with(any_args).once
-    expect(bucket).to receive(:object).with(file.key).and_return(s3).twice
-    expect(s3).to receive(:upload_file).with(any_args).and_raise(StandardError)
+    allow(bucket).to receive(:object).with(file.key).and_return(s3)
+
+    expect(logger).to receive(:warn).with(any_args)
+    expect(s3).to receive(:upload_file).with(any_args).and_raise(RuntimeError.new('UPLOAD FAILED')).exactly(5).times
     expect(s3).to receive(:upload_file).with(any_args).and_return(true)
 
     subject.upload(file)
   end
+
+  it "retries errors specified times" do
+    subject = described_class.new(bucket, logger, threadpool, retry_count: 3, retry_delay: 0.01)
+    s3 = double("s3").as_null_object
+
+    allow(bucket).to receive(:object).with(file.key).and_return(s3)
+
+    expect(logger).to receive(:warn).with(any_args).exactly(3).times
+    expect(logger).to receive(:error).with(any_args).once
+    expect(s3).to receive(:upload_file).with(file.path, {}).and_raise(RuntimeError).at_least(1).times
+
+    subject.upload(file)
+  end
 end

data/spec/outputs/s3_spec.rb

@@ -204,4 +204,27 @@ describe LogStash::Outputs::S3 do
       subject.multi_receive_encoded(events_and_encoded)
     end
   end
+
+  describe "aws service" do
+    context 'us-east-1' do
+      let(:region) { 'us-east-1' }
+      it "sets endpoint" do
+        expect( subject.send(:bucket_resource).client.config.endpoint.to_s ).to eql 'https://s3.us-east-1.amazonaws.com'
+      end
+    end
+
+    context 'ap-east-1' do
+      let(:region) { 'ap-east-1' }
+      it "sets endpoint" do
+        expect( subject.send(:bucket_resource).client.config.endpoint.to_s ).to eql 'https://s3.ap-east-1.amazonaws.com'
+      end
+    end
+
+    context 'cn-northwest-1' do
+      let(:region) { 'cn-northwest-1' }
+      it "sets endpoint" do
+        expect( subject.send(:bucket_resource).client.config.endpoint.to_s ).to eql 'https://s3.cn-northwest-1.amazonaws.com.cn'
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 4.2.0
+  version: 4.3.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-18 00:00:00.000000000 Z
+date: 2020-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement