logstash-output-s3 4.1.7 → 4.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d0273662b68bd9801f7320aa01736ad40539ca29a4d5f895a6245f4cf7b9542
-  data.tar.gz: 676f7a10a587bc64fe4e2229f3ab79e60f807636ce9bcf2c80ecb2b91ff97ded
+  metadata.gz: a4cf264de35b436280daac5ccab48529f9f3e8037a37c40f634e479c49dd0b3e
+  data.tar.gz: 6c5fe67b82ea4dc36a471ad06199fb241a171bb4142ef754da2436211d99d4b6
 SHA512:
-  metadata.gz: 010f9bd2d079f32077df912cfc0d333208d8a2f76ba4ee0ac341c0f3b84c94aabe3faa9ca76eca3f9998a50e1c772fbfead9ce9b453caa511901817ca89332e0
-  data.tar.gz: 8cd6b94465ba40e5b771ffaf33abf052fcac3da1ba82614e73cd8fb74292c07b53497b2b607d849fbbf7590225f36f43972e5ad98c9a8d93217c0480d7bc7db2
+  metadata.gz: f1afe8b78490ae972d08216adf887447ada0fef5adb3c67badb8243b84b907d35da6bebc0dbd354dae10f3adb0f1840a996192eac43482725a58eb70fa55eb0e
+  data.tar.gz: 611ef2103780b7ef705dace7e1bd95e5c931c4f4276cf3db22c648acf555667f5eb88c43856014d6e7a093f9ea22c50731955ade23a779d85a0d9088085e9e33

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.1.8
+  - Add support for setting mutipart upload threshold [#202](https://github.com/logstash-plugins/logstash-output-s3/pull/202)
+
 ## 4.1.7
   - Fixed issue where on restart, 0 byte files could erroneously be uploaded to s3 [#195](https://github.com/logstash-plugins/logstash-output-s3/issues/195)
 

data/docs/index.asciidoc

@@ -101,6 +101,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-storage_class>> |<<string,string>>, one of `["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA"]`|No
 | <<plugins-{type}s-{plugin}-temporary_directory>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-time_file>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-upload_multipart_threshold>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-upload_queue_size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-upload_workers_count>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-validate_credentials_on_root_bucket>> |<<boolean,boolean>>|No
@@ -352,6 +353,14 @@ If you define file_size you have a number of files in consideration of the secti
 0 stay all time on listerner, beware if you specific 0 and size_file 0, because you will not put the file on bucket,
 for now the only thing this plugin can do is to put the file when logstash restart.
 
+[id="plugins-{type}s-{plugin}-upload_multipart_threshold"]
+===== `upload_multipart_threshold`
+
+  * Value type is <<number,number>>
+  * Default value is `15728640`
+
+Files larger than this number are uploaded using the S3 multipart APIs
+
 [id="plugins-{type}s-{plugin}-upload_queue_size"]
 ===== `upload_queue_size` 
 

data/lib/logstash/outputs/s3.rb

@@ -160,6 +160,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   # Number of items we can keep in the local queue before uploading them
   config :upload_queue_size, :validate => :number, :default => 2 * (Concurrent.processor_count * 0.25).ceil
 
+  # Files larger than this number are uploaded using the S3 multipart APIs. Default threshold is 15MB.
+  config :upload_multipart_threshold, :validate => :number, :default => 15 * 1024 * 1024
+
   # The version of the S3 signature hash to use. Normally uses the internal client default, can be explicitly
   # specified here
   config :signature_version, :validate => ['v2', 'v4']
@@ -296,7 +299,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       :server_side_encryption => @server_side_encryption ? @server_side_encryption_algorithm : nil,
       :ssekms_key_id => @server_side_encryption_algorithm == "aws:kms" ? @ssekms_key_id : nil,
       :storage_class => @storage_class,
-      :content_encoding => @encoding == "gzip" ? "gzip" : nil
+      :content_encoding => @encoding == "gzip" ? "gzip" : nil,
+      :multipart_threshold => @upload_multipart_threshold
     }
   end
 

data/logstash-output-s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-s3'
-  s.version         = '4.1.7'
+  s.version         = '4.1.8'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Sends Logstash events to the Amazon Simple Storage Service"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/outputs/s3_spec.rb

@@ -60,6 +60,23 @@ describe LogStash::Outputs::S3 do
       end
     end
 
+    describe "Multipart upload threshold" do
+      context "when configured" do
+        it "should use the configured threshold" do
+          threshold = 1 * 1024 * 1024
+          s3 = described_class.new(options.merge({ "upload_multipart_threshold" => threshold }))
+          expect(s3.upload_options).to include(:multipart_threshold => threshold)
+        end
+      end
+
+      context "when not configured" do
+        it "should use 15MB as the default" do
+          s3 = described_class.new(options)
+          expect(s3.upload_options).to include(:multipart_threshold => 15 * 1024 * 1024)
+        end
+      end
+    end
+
     describe "Service Side Encryption" do
 
       context "when configured" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 4.1.7
+  version: 4.1.8
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-25 00:00:00.000000000 Z
+date: 2019-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement