logstash-output-s3 3.1.2 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35c6cb316aa0ec036121f892c821dbdb11de3c8d
-  data.tar.gz: faa1102b573616d9b02496a32b16eca96d151ba6
+  metadata.gz: e681a1dd7ceea2cedb08ddd6dfc6eae4310a48cb
+  data.tar.gz: c86a97c06c234211329e24184370b1a0fdf19fc7
 SHA512:
-  metadata.gz: e56c266affad3d06f0e04d324a9cd8ccd1827a92c0eb6a70965b70ff40d60f5761314f0d4689275bcfc4510ce3ee26fa8de5f8f54d18230bcb04872320055d99
-  data.tar.gz: 8483fed9096096e0d0842a5c720642b7382ec4a4b311d57f160ebc0f5b53a1ddc8af369adc0a67fa69511717302858fdf329551cb99a2993835a59985b74c395
+  metadata.gz: 139c3e3749e83a6490ecd982294ab65e6c8628aeb246d9b04f30cdd7f05640c6afd0865133e6c0f017e307d81f2438a675b4024d16a0ece34266ce74d7b46c68
+  data.tar.gz: 1d6b05b4aa255c1db7887d93e8eed9f6781fd923312a38095e7bea6eaff56db49641c2b78263c757b46f522df6131cebe6f28e3af331d0ebb7bdb0c3ac0cbd82

data/CHANGELOG.md

@@ -1,6 +1,13 @@
+## 3.2.0
+  - Move to the new concurrency model `:single`
+  - use correct license identifier #99
+  - add support for `bucket_owner_full_control` in the canned ACL #87
+  - delete the test file but ignore any errors, because we actually only need to be able to write to S3. #97
+
 ## 3.1.2
   - Fix improper shutdown of output worker threads
   - improve exception handling
+
 ## 3.0.1
  - Republish all the gems under jruby.
 

data/lib/logstash/outputs/s3.rb

@@ -13,8 +13,8 @@ require "fileutils"
 # INFORMATION:
 #
 # This plugin batches and uploads logstash events into Amazon Simple Storage Service (Amazon S3).
-# 
-# Requirements: 
+#
+# Requirements:
 # * Amazon S3 Bucket and S3 Access Permissions (Typically access_key_id and secret_access_key)
 # * S3 PutObject permission
 # * Run logstash as superuser to establish connection
@@ -42,7 +42,7 @@ require "fileutils"
 # Both time_file and size_file settings can trigger a log "file rotation"
 # A log rotation pushes the current log "part" to s3 and deleted from local temporary storage.
 #
-## If you specify BOTH size_file and time_file then it will create file for each tag (if specified). 
+## If you specify BOTH size_file and time_file then it will create file for each tag (if specified).
 ## When EITHER time_file minutes have elapsed OR log file size > size_file, a log rotation is triggered.
 ##
 ## If you ONLY specify time_file but NOT file_size, one file for each tag (if specified) will be created..
@@ -67,7 +67,7 @@ require "fileutils"
 #      size_file => 2048                        (optional) - Bytes
 #      time_file => 5                           (optional) - Minutes
 #      format => "plain"                        (optional)
-#      canned_acl => "private"                  (optional. Options are "private", "public_read", "public_read_write", "authenticated_read". Defaults to "private" )
+#      canned_acl => "private"                  (optional. Options are "private", "public_read", "public_read_write", "authenticated_read", "bucket_owner_full_control". Defaults to "private" )
 #    }
 #
 class LogStash::Outputs::S3 < LogStash::Outputs::Base
@@ -79,6 +79,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   config_name "s3"
   default :codec, 'line'
 
+  concurrency :single
+
   # S3 bucket
   config :bucket, :validate => :string
 
@@ -100,7 +102,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   config :restore, :validate => :boolean, :default => false
 
   # The S3 canned ACL to use when putting the file. Defaults to "private".
-  config :canned_acl, :validate => ["private", "public_read", "public_read_write", "authenticated_read"],
+  config :canned_acl, :validate => ["private", "public_read", "public_read_write", "authenticated_read", "bucket_owner_full_control"],
          :default => "private"
 
   # Specifies wether or not to use S3's AES256 server side encryption. Defaults to false.
@@ -214,8 +216,6 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
     # http://ruby.awsblog.com/post/Tx16QY1CI5GVBFT/Threading-with-the-AWS-SDK-for-Ruby
     AWS.eager_autoload!(AWS::S3)
 
-    workers_not_supported
-
     @s3 = aws_s3_config
     @upload_queue = Queue.new
     @file_rotation_lock = Mutex.new
@@ -258,7 +258,15 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
 
     begin
       write_on_bucket(test_filename)
-      delete_on_bucket(test_filename)
+
+      begin
+        remote_filename = "#{@prefix}#{File.basename(test_filename)}"
+        bucket = @s3.buckets[@bucket]
+        bucket.objects[remote_filename].delete
+      rescue StandardError => e
+        # we actually only need `put_object`, but if we dont delete them
+        # we can have a lot of tests files
+      end
     ensure
       File.delete(test_filename)
     end
@@ -466,24 +474,6 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
     @page_counter = 0
   end
 
-  private
-  def delete_on_bucket(filename)
-    bucket = @s3.buckets[@bucket]
-
-    remote_filename = "#{@prefix}#{File.basename(filename)}"
-
-    @logger.debug("S3: delete file from bucket", :remote_filename => remote_filename, :bucket => @bucket)
-
-    begin
-      # prepare for write the file
-      object = bucket.objects[remote_filename]
-      object.delete
-    rescue AWS::Errors::Base => e
-      @logger.error("S3: AWS error", :error => e)
-      raise LogStash::ConfigurationError, "AWS Configuration Error"
-    end
-  end
-
   private
   def move_file_to_bucket_async(file)
     @logger.debug("S3: Sending the file to the upload queue.", :filename => File.basename(file))

data/logstash-output-s3.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-s3'
-  s.version         = '3.1.2'
-  s.licenses        = ['Apache License (2.0)']
+  s.version         = '3.2.0'
+  s.licenses        = ['Apache-2.0']
   s.summary         = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 3.1.2
+  version: 3.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-08 00:00:00.000000000 Z
+date: 2016-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -120,7 +120,7 @@ files:
 - spec/supports/helpers.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
-- Apache License (2.0)
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: output