logstash-output-s3 3.0.1 → 3.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d3dd0c588a48279affa6584336d870da42ab6e67
-  data.tar.gz: 2c7ea06c3e76410fcc0cb7feb1eab05d03914448
+  metadata.gz: 9008635509ce94ad1d6e2242f256c67e0e2ed269
+  data.tar.gz: 5f05f5f226976a70c3df9342dd6b4db0e3760274
 SHA512:
-  metadata.gz: 071873b7b74319613c27086fa39e921d9833ea499e8273acf02cd3b1d03026990470ddcd0188b57d4d51a15d9f9917e128e098d5c0f4bec1b142392e167efa7e
-  data.tar.gz: 58adf92d4bda4c6d982f61dd188630e305c1461912498a6a36e2c424439fa510632cf45a45ca3382c7a0016033b1ef2abbf26bcf7a96d981429e93297525e3eb
+  metadata.gz: f6a4a44f0b6c9da9f8ec46c230b9aa7d01c2508d01fbb68632debceb23223bc252c9d80e7eb2a241764718e67eee3ba289f7100677cad2ffa73db0e4b4a5892e
+  data.tar.gz: ad0adb0d2a1125a427e013fdcc1d1c2884c6d215523df590081e3b670440656c1d92cebb5765dfe7c898220596d4f6553e74ba336929247ee1f606440dbf654f

data/CHANGELOG.md

@@ -1,25 +1,39 @@
+## 3.1.1
+  - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
+
+## 3.1.0
+ - breaking,config: Remove deprecated config `endpoint_region`. Please use `region` instead.
+
 ## 3.0.1
-  - Republish all the gems under jruby.
+ - Republish all the gems under jruby.
+
 ## 3.0.0
-  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
-# 2.0.7
-  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
-# 2.0.6
-  - New dependency requirements for logstash-core for the 5.0 release
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+
+## 2.0.7
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+
+## 2.0.6
+ - New dependency requirements for logstash-core for the 5.0 release
+
 ## 2.0.5
  - Support signature_version option for v4 S3 keys
+
 ## 2.0.4
  - Remove the `Time.now` stub in the spec, it was conflicting with other test when running inside the default plugins test #63
  - Make the spec run faster by adjusting the values of time rotation test.
+
 ## 2.0.3
  - Update deps for logstash 2.0
+
 ## 2.0.2
  - Fixes an issue when tags were defined #39
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 
-# 1.0.1
+## 1.0.1
 - Fix a synchronization issue when doing file rotation and checking the size of the current file
 - Fix an issue with synchronization when shutting down the plugin and closing the current temp file

data/lib/logstash/outputs/s3.rb

@@ -62,7 +62,7 @@ require "fileutils"
 #    s3{
 #      access_key_id => "crazy_key"             (required)
 #      secret_access_key => "monkey_access_key" (required)
-#      endpoint_region => "eu-west-1"           (required) - Deprecated
+#      region => "eu-west-1"                    (optional, default = "us-east-1")
 #      bucket => "boss_please_open_your_bucket" (required)
 #      size_file => 2048                        (optional) - Bytes
 #      time_file => 5                           (optional) - Minutes
@@ -82,11 +82,6 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   # S3 bucket
   config :bucket, :validate => :string
 
-  # AWS endpoint_region
-  config :endpoint_region, :validate => ["us-east-1", "us-west-1", "us-west-2",
-                                         "eu-west-1", "ap-southeast-1", "ap-southeast-2",
-                                        "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :deprecated => 'Deprecated, use region instead.'
-
   # Set the size of file in bytes, this means that files on bucket when have dimension > file_size, they are stored in two or more file.
   # If you have tags then it will generate a specific size file for every tags
   ##NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.
@@ -135,6 +130,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   #
   config :tags, :validate => :array, :default => []
 
+  # Specify the content encoding. Supports ("gzip"). Defaults to "none"
+  config :encoding, :validate => ["none", "gzip"], :default => "none"
+
   # Exposed attributes for testing purpose.
   attr_accessor :tempfile
   attr_reader :page_counter
@@ -158,17 +156,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   end
 
   def aws_service_endpoint(region)
-    # Make the deprecated endpoint_region work
-    # TODO: (ph) Remove this after deprecation.
-
-    if @endpoint_region
-      region_to_use = @endpoint_region
-    else
-      region_to_use = @region
-    end
-
     return {
-      :s3_endpoint => region_to_use == 'us-east-1' ? 's3.amazonaws.com' : "s3-#{region_to_use}.amazonaws.com"
+      :s3_endpoint => region == 'us-east-1' ? 's3.amazonaws.com' : "s3-#{region}.amazonaws.com"
     }
   end
 
@@ -185,7 +174,10 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       begin
         # prepare for write the file
         object = bucket.objects[remote_filename]
-        object.write(fileIO, :acl => @canned_acl, :server_side_encryption => @server_side_encryption ? :aes256 : nil)
+        object.write(fileIO,
+                     :acl => @canned_acl,
+                     :server_side_encryption => @server_side_encryption ? :aes256 : nil,
+                     :content_encoding => @encoding == "gzip" ? "gzip" : nil)
       rescue AWS::Errors::Base => error
         @logger.error("S3: AWS error", :error => error)
         raise LogStash::Error, "AWS Configuration Error, #{error}"
@@ -207,7 +199,11 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
         @tempfile.close
       end
 
-      @tempfile = File.open(filename, "a")
+      if @encoding == "gzip"
+        @tempfile = Zlib::GzipWriter.open(filename)
+      else
+        @tempfile = File.open(filename, "a")
+      end
     end
   end
 
@@ -272,7 +268,7 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   def restore_from_crashes
     @logger.debug("S3: Checking for temp files from a previoius crash...")
 
-    Dir[File.join(@temporary_directory, "*.#{TEMPFILE_EXTENSION}")].each do |file|
+    Dir[File.join(@temporary_directory, "*.#{get_tempfile_extension}")].each do |file|
       name_file = File.basename(file)
       @logger.warn("S3: Found temporary file from crash.  Uploading file to S3.", :filename => name_file)
       move_file_to_bucket_async(file)
@@ -301,15 +297,20 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
     @time_file * 60
   end
 
+  private
+  def get_tempfile_extension
+    @encoding == "gzip" ? "#{TEMPFILE_EXTENSION}.gz" : "#{TEMPFILE_EXTENSION}"
+  end
+
   public
   def get_temporary_filename(page_counter = 0)
     current_time = Time.now
     filename = "ls.s3.#{Socket.gethostname}.#{current_time.strftime("%Y-%m-%dT%H.%M")}"
 
     if @tags.size > 0
-      return "#{filename}.tag_#{@tags.join('.')}.part#{page_counter}.#{TEMPFILE_EXTENSION}"
+      return "#{filename}.tag_#{@tags.join('.')}.part#{page_counter}.#{get_tempfile_extension}"
     else
-      return "#{filename}.part#{page_counter}.#{TEMPFILE_EXTENSION}"
+      return "#{filename}.part#{page_counter}.#{get_tempfile_extension}"
     end
   end
 
@@ -322,7 +323,18 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   public
   def rotate_events_log?
     @file_rotation_lock.synchronize do
-      @tempfile.size > @size_file
+      tempfile_size > @size_file
+    end
+  end
+
+  private
+  def tempfile_size
+    if @tempfile.instance_of? File
+      @tempfile.size
+    elsif @tempfile.instance_of? Zlib::GzipWriter
+      @tempfile.tell
+    else
+      raise LogStash::Error, "Unable to get size of temp file of type #{@tempfile.class}"
     end
   end
 
@@ -334,10 +346,10 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   public
   def write_to_tempfile(event)
     begin
-      @logger.debug("S3: put event into tempfile ", :tempfile => File.basename(@tempfile))
+      @logger.debug("S3: put event into tempfile ", :tempfile => File.basename(@tempfile.path))
 
       @file_rotation_lock.synchronize do
-        @tempfile.syswrite(event)
+        @tempfile.write(event)
       end
     rescue Errno::ENOSPC
       @logger.error("S3: No space left in temporary directory", :temporary_directory => @temporary_directory)
@@ -365,13 +377,17 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
   def handle_event(encoded_event)
     if write_events_to_multiple_files?
       if rotate_events_log?
-        @logger.debug("S3: tempfile is too large, let's bucket it and create new file", :tempfile => File.basename(@tempfile))
+        @logger.debug("S3: tempfile is too large, let's bucket it and create new file", :tempfile => File.basename(@tempfile.path))
 
-        move_file_to_bucket_async(@tempfile.path)
+        tempfile_path = @tempfile.path
+        # close and start next file before sending the previous one
         next_page
         create_temporary_file
+
+        # send to s3
+        move_file_to_bucket_async(tempfile_path)
       else
-        @logger.debug("S3: tempfile file size report.", :tempfile_size => @tempfile.size, :size_file => @size_file)
+        @logger.debug("S3: tempfile file size report.", :tempfile_size => tempfile_size, :size_file => @size_file)
       end
     end
 
@@ -386,9 +402,13 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base
       Stud.interval(periodic_interval, :sleep_then_run => true) do
         @logger.debug("S3: time_file triggered, bucketing the file", :filename => @tempfile.path)
 
-        move_file_to_bucket_async(@tempfile.path)
+        tempfile_path = @tempfile.path
+        # close and start next file before sending the previous one
         next_page
         create_temporary_file
+
+        # send to s3
+        move_file_to_bucket_async(tempfile_path)
       end
     end
   end

data/logstash-output-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-s3'
-  s.version         = '3.0.1'
+  s.version         = '3.1.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-aws'
   s.add_runtime_dependency 'stud', '~> 0.0.22'
   s.add_development_dependency 'logstash-devutils'

data/spec/integration/s3_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/s3"
 require 'socket'

data/spec/outputs/s3_spec.rb

@@ -20,17 +20,7 @@ describe LogStash::Outputs::S3 do
                                "bucket" => "my-bucket" } }
 
   describe "configuration" do
-    let!(:config) { { "endpoint_region" => "sa-east-1" } }
-
-    it "should support the deprecated endpoint_region as a configuration option" do
-      s3 = LogStash::Outputs::S3.new(config)
-      expect(s3.aws_options_hash[:s3_endpoint]).to eq("s3-sa-east-1.amazonaws.com")
-    end
-
-    it "should fallback to region if endpoint_region isnt defined" do
-      s3 = LogStash::Outputs::S3.new(config.merge({ "region" => 'sa-east-1' }))
-      expect(s3.aws_options_hash).to include(:s3_endpoint => "s3-sa-east-1.amazonaws.com")
-    end
+    let!(:config) { { "region" => "sa-east-1" } }
 
     describe "signature version" do
       it "should set the signature version if specified" do

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-s3
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.1.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-09 00:00:00.000000000 Z
+date: 2016-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.99'
   name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -134,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.3
 signing_key:
 specification_version: 4
 summary: This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)