logstash-input-s3 3.0.1 → 3.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 42608fd6049e4607221fbef88af403d4348c9f1a
-  data.tar.gz: 34c0c022ade6eed9dbba36d61b18f9b3ba2aa71c
+  metadata.gz: b975c1700f9a6aab421e175f94d1bce38b9d9e01
+  data.tar.gz: 1ebe414e3ba68ae1d4e9d3477d9ec9663f13dc10
 SHA512:
-  metadata.gz: 5cdf1baa6105f3e57e49788c36f3225be77bacd7e30fb0a99d0c3ee5571100432b5ce9ade2335f76ef121f4b305e35d5bbad3d6a4203e22c9b713dcb0b4910f1
-  data.tar.gz: 886d4f1368c282504e05d610387dcb3570a9a2404850fc253e479123bb976a3f04826e1145fea0f5ce78b50a44d024838315bcf2a573b4e85676ce13517e20a2
+  metadata.gz: 5477fd22a31ab077044f93c089a9b466ac692b9c3cfdf6156fcfc7b70135d7881b34ea314c54aa7fc1566b46e045f1bc0f714acd554e6971318db4a2e468fda1
+  data.tar.gz: 44adae47471d9ad5a495b63bb155110cb86df5d0167fc94562e5b6e35c8f20d069727c166ac8bee438e4f22573410d25e247e010ff9398cff8554c3c15ec52cc

data/CHANGELOG.md

@@ -1,15 +1,27 @@
+## 3.1.1
+  - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
+
+## 3.1.0
+ - breaking,config: Remove deprecated config `credentials` and `region_endpoint`. Please use AWS mixin.
+
 ## 3.0.1
-  - Republish all the gems under jruby.
+ - Republish all the gems under jruby.
+
 ## 3.0.0
-  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
-# 2.0.6
-  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
-# 2.0.5
-  - New dependency requirements for logstash-core for the 5.0 release
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+
+## 2.0.6
+ - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+
+## 2.0.5
+ - New dependency requirements for logstash-core for the 5.0 release
+
 ## 2.0.4
  - Fix for Error: No Such Key problem when deleting
+
 ## 2.0.3
  - Do not raise an exception if the sincedb file is empty, instead return the current time #66
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/lib/logstash/inputs/s3.rb

@@ -12,27 +12,15 @@ require "stud/temporary"
 # Each line from each file generates an event.
 # Files ending in `.gz` are handled as gzip'ed files.
 class LogStash::Inputs::S3 < LogStash::Inputs::Base
-  include LogStash::PluginMixins::AwsConfig
+  include LogStash::PluginMixins::AwsConfig::V2
 
   config_name "s3"
 
   default :codec, "plain"
 
-  # DEPRECATED: The credentials of the AWS account used to access the bucket.
-  # Credentials can be specified:
-  # - As an ["id","secret"] array
-  # - As a path to a file containing AWS_ACCESS_KEY_ID=... and AWS_SECRET_ACCESS_KEY=...
-  # - In the environment, if not set (using variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
-  config :credentials, :validate => :array, :default => [], :deprecated => "This only exists to be backwards compatible. This plugin now uses the AwsConfig from PluginMixins"
-
   # The name of the S3 bucket.
   config :bucket, :validate => :string, :required => true
 
-  # The AWS region for your bucket.
-  config :region_endpoint, :validate => ["us-east-1", "us-west-1", "us-west-2",
-                                "eu-west-1", "ap-southeast-1", "ap-southeast-2",
-                                "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :deprecated => "This only exists to be backwards compatible. This plugin now uses the AwsConfig from PluginMixins"
-
   # If specified, the prefix of filenames in the bucket must match (not a regexp)
   config :prefix, :validate => :string, :default => nil
 
@@ -71,20 +59,20 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   def register
     require "fileutils"
     require "digest/md5"
-    require "aws-sdk"
-
-    @region = get_region
+    require "aws-sdk-resources"
 
     @logger.info("Registering s3 input", :bucket => @bucket, :region => @region)
 
     s3 = get_s3object
 
-    @s3bucket = s3.buckets[@bucket]
+    @s3bucket = s3.bucket(@bucket)
 
     unless @backup_to_bucket.nil?
-      @backup_bucket = s3.buckets[@backup_to_bucket]
-      unless @backup_bucket.exists?
-        s3.buckets.create(@backup_to_bucket)
+      @backup_bucket = s3.bucket(@backup_to_bucket)
+      begin
+        s3.client.head_bucket({ :bucket => @backup_to_bucket})
+      rescue Aws::S3::Errors::NoSuchBucket
+        s3.create_bucket({ :bucket => @backup_to_bucket})
       end
     end
 
@@ -93,7 +81,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     end
 
     FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory)
-  end # def register
+  end
 
   public
   def run(queue)
@@ -107,13 +95,14 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   def list_new_files
     objects = {}
 
-    @s3bucket.objects.with_prefix(@prefix).each do |log|
+    @s3bucket.objects(:prefix => @prefix).each do |log|
       @logger.debug("S3 input: Found key", :key => log.key)
 
       unless ignore_filename?(log.key)
         if sincedb.newer?(log.last_modified)
           objects[log.key] = log.last_modified
           @logger.debug("S3 input: Adding to objects[]", :key => log.key)
+          @logger.debug("objects[] length is: ", :length => objects.length)
         end
       end
     end
@@ -121,13 +110,12 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   end # def fetch_new_files
 
   public
-  def backup_to_bucket(object, key)
+  def backup_to_bucket(object)
     unless @backup_to_bucket.nil?
-      backup_key = "#{@backup_add_prefix}#{key}"
+      backup_key = "#{@backup_add_prefix}#{object.key}"
+      @backup_bucket.object(backup_key).copy_from(:copy_source => "#{object.bucket_name}/#{object.key}")
       if @delete
-        object.move_to(backup_key, :bucket => @backup_bucket)
-      else
-        object.copy_to(backup_key, :bucket => @backup_bucket)
+        object.delete()
       end
     end
   end
@@ -155,22 +143,12 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
   public
   def stop
-    # @current_thread is initialized in the `#run` method, 
-    # this variable is needed because the `#stop` is a called in another thread 
+    # @current_thread is initialized in the `#run` method,
+    # this variable is needed because the `#stop` is a called in another thread
     # than the `#run` method and requiring us to call stop! with a explicit thread.
     Stud.stop!(@current_thread)
   end
 
-  public
-  def aws_service_endpoint(region)
-    region_to_use = get_region
-
-    return {
-      :s3_endpoint => region_to_use == 'us-east-1' ?
-        's3.amazonaws.com' : "s3-#{region_to_use}.amazonaws.com"
-    }
-  end
-
   private
 
   # Read the content of the local file
@@ -180,7 +158,6 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   # @return [Boolean] True if the file was completely read, false otherwise.
   def process_local_log(queue, filename)
     @logger.debug('Processing file', :filename => filename)
-
     metadata = {}
     # Currently codecs operates on bytes instead of stream.
     # So all IO stuff: decompression, reading need to be done in the actual
@@ -301,6 +278,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   def ignore_filename?(filename)
     if @prefix == filename
       return true
+    elsif filename.end_with?("/")
+      return true
     elsif (@backup_add_prefix && @backup_to_bucket == @bucket && filename =~ /^#{backup_add_prefix}/)
       return true
     elsif @exclude_pattern.nil?
@@ -314,14 +293,13 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
   private
   def process_log(queue, key)
-    object = @s3bucket.objects[key]
+    object = @s3bucket.object(key)
 
     filename = File.join(temporary_directory, File.basename(key))
-    
     if download_remote_file(object, filename)
       if process_local_log(queue, filename)
         lastmod = object.last_modified
-        backup_to_bucket(object, key)
+        backup_to_bucket(object)
         backup_to_dir(filename)
         delete_file_from_bucket(object)
         FileUtils.remove_entry_secure(filename, true)
@@ -340,13 +318,10 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   # @return [Boolean] True if the file was completely downloaded
   def download_remote_file(remote_object, local_filename)
     completed = false
-
     @logger.debug("S3 input: Download remote file", :remote_key => remote_object.key, :local_filename => local_filename)
     File.open(local_filename, 'wb') do |s3file|
-      remote_object.read do |chunk|
-        return completed if stop?
-        s3file.write(chunk)
-      end
+      return completed if stop?
+      remote_object.get(:response_target => s3file)
     end
     completed = true
 
@@ -360,41 +335,9 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     end
   end
 
-  private
-  def get_region
-    # TODO: (ph) Deprecated, it will be removed
-    if @region_endpoint
-      @region_endpoint
-    else
-      @region
-    end
-  end
-
   private
   def get_s3object
-    # TODO: (ph) Deprecated, it will be removed
-    if @credentials.length == 1
-      File.open(@credentials[0]) { |f| f.each do |line|
-        unless (/^\#/.match(line))
-          if(/\s*=\s*/.match(line))
-            param, value = line.split('=', 2)
-            param = param.chomp().strip()
-            value = value.chomp().strip()
-            if param.eql?('AWS_ACCESS_KEY_ID')
-              @access_key_id = value
-            elsif param.eql?('AWS_SECRET_ACCESS_KEY')
-              @secret_access_key = value
-            end
-          end
-        end
-      end
-      }
-    elsif @credentials.length == 2
-      @access_key_id = @credentials[0]
-      @secret_access_key = @credentials[1]
-    end
-
-    s3 = AWS::S3.new(aws_options_hash)
+    s3 = Aws::S3::Resource.new(aws_options_hash)
   end
 
   private

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.0.1'
+  s.version         = '3.1.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files from a S3 bucket."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,9 +20,10 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-aws'
   s.add_runtime_dependency 'stud', '~> 0.0.18'
+ # s.add_runtime_dependency 'aws-sdk-resources', '>= 2.0.33'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'simplecov'

data/spec/inputs/s3_spec.rb

@@ -2,6 +2,7 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/s3"
 require "logstash/errors"
+require "aws-sdk-resources"
 require_relative "../support/helpers"
 require "stud/temporary"
 require "aws-sdk"
@@ -11,6 +12,7 @@ describe LogStash::Inputs::S3 do
   let(:temporary_directory) { Stud::Temporary.pathname }
   let(:sincedb_path) { Stud::Temporary.pathname }
   let(:day) { 3600 * 24 }
+  let(:creds) { Aws::Credentials.new('1234', 'secret') }
   let(:config) {
     {
       "access_key_id" => "1234",
@@ -21,9 +23,10 @@ describe LogStash::Inputs::S3 do
     }
   }
 
+
   before do
     FileUtils.mkdir_p(sincedb_path)
-    AWS.stub!
+    Aws.config[:stub_responses] = true
     Thread.abort_on_exception = true
   end
 
@@ -52,27 +55,6 @@ describe LogStash::Inputs::S3 do
   describe '#get_s3object' do
     subject { LogStash::Inputs::S3.new(settings) }
 
-    context 'with deprecated credentials option' do
-      let(:settings) {
-        {
-          "credentials" => ["1234", "secret"],
-          "proxy_uri" => "http://example.com",
-          "bucket" => "logstash-test",
-        }
-      }
-
-      it 'should instantiate AWS::S3 clients with a proxy set' do
-        expect(AWS::S3).to receive(:new).with({
-          :access_key_id => "1234",
-          :secret_access_key => "secret",
-          :proxy_uri => 'http://example.com',
-          :use_ssl => subject.use_ssl,
-        }.merge(subject.aws_service_endpoint(subject.region)))
-
-        subject.send(:get_s3object)
-      end
-    end
-
     context 'with modern access key options' do
       let(:settings) {
         {
@@ -84,13 +66,11 @@ describe LogStash::Inputs::S3 do
       }
 
       it 'should instantiate AWS::S3 clients with a proxy set' do
-        expect(AWS::S3).to receive(:new).with({
-          :access_key_id => "1234",
-          :secret_access_key => "secret",
-          :proxy_uri => 'http://example.com',
-          :use_ssl => subject.use_ssl,
-        }.merge(subject.aws_service_endpoint(subject.region)))
-        
+        expect(Aws::S3::Resource).to receive(:new).with({
+          :credentials => kind_of(Aws::Credentials),
+          :http_proxy => 'http://example.com',
+          :region => subject.region
+        })
 
         subject.send(:get_s3object)
       end
@@ -98,7 +78,7 @@ describe LogStash::Inputs::S3 do
   end
 
   describe "#list_new_files" do
-    before { allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(nil) { objects_list } }
+    before { allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects_list } }
 
     let!(:present_object) { double(:key => 'this-should-be-present', :last_modified => Time.now) }
     let(:objects_list) {
@@ -128,7 +108,7 @@ describe LogStash::Inputs::S3 do
           present_object
         ]
 
-        allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(nil) { objects_list }
+        allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects_list }
 
         plugin = LogStash::Inputs::S3.new(config.merge({ 'backup_add_prefix' => 'mybackup',
                                                          'backup_to_bucket' => config['bucket']}))
@@ -154,7 +134,7 @@ describe LogStash::Inputs::S3 do
           present_object
         ]
 
-        allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(prefix) { objects_list }
+        allow_any_instance_of(Aws::S3::Bucket).to receive(:objects).with(:prefix => prefix) { objects_list }
 
         plugin = LogStash::Inputs::S3.new(config.merge({ 'prefix' => prefix }))
         plugin.register
@@ -168,7 +148,7 @@ describe LogStash::Inputs::S3 do
         double(:key => 'TWO_DAYS_AGO', :last_modified => Time.now - 2 * day)
       ]
 
-      allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(nil) { objects }
+      allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects }
 
 
       plugin = LogStash::Inputs::S3.new(config)
@@ -181,20 +161,22 @@ describe LogStash::Inputs::S3 do
         plugin = LogStash::Inputs::S3.new(config.merge({ "backup_to_bucket" => "mybackup"}))
         plugin.register
 
-        s3object = double()
-        expect(s3object).to receive(:copy_to).with('test-file', :bucket => an_instance_of(AWS::S3::Bucket))
+        s3object = Aws::S3::Object.new('mybucket', 'testkey')
+        expect_any_instance_of(Aws::S3::Object).to receive(:copy_from).with(:copy_source => "mybucket/testkey")
+        expect(s3object).to_not receive(:delete)
 
-        plugin.backup_to_bucket(s3object, 'test-file')
+        plugin.backup_to_bucket(s3object)
       end
 
-      it 'should move to another s3 bucket when deleting the original file' do
+      it 'should copy to another s3 bucket when deleting the original file' do
         plugin = LogStash::Inputs::S3.new(config.merge({ "backup_to_bucket" => "mybackup", "delete" => true }))
         plugin.register
 
-        s3object = double()
-        expect(s3object).to receive(:move_to).with('test-file', :bucket => an_instance_of(AWS::S3::Bucket))
+        s3object = Aws::S3::Object.new('mybucket', 'testkey')
+        expect_any_instance_of(Aws::S3::Object).to receive(:copy_from).with(:copy_source => "mybucket/testkey")
+        expect(s3object).to receive(:delete)
 
-        plugin.backup_to_bucket(s3object, 'test-file')
+        plugin.backup_to_bucket(s3object)
       end
 
       it 'should add the specified prefix to the backup file' do
@@ -202,10 +184,11 @@ describe LogStash::Inputs::S3 do
                                                            "backup_add_prefix" => 'backup-' }))
         plugin.register
 
-        s3object = double()
-        expect(s3object).to receive(:copy_to).with('backup-test-file', :bucket => an_instance_of(AWS::S3::Bucket))
+        s3object = Aws::S3::Object.new('mybucket', 'testkey')
+        expect_any_instance_of(Aws::S3::Object).to receive(:copy_from).with(:copy_source => "mybucket/testkey")
+        expect(s3object).to_not receive(:delete)
 
-        plugin.backup_to_bucket(s3object, 'test-file')
+        plugin.backup_to_bucket(s3object)
       end
     end
 
@@ -222,19 +205,6 @@ describe LogStash::Inputs::S3 do
         end
       end
     end
-
-    it 'should accepts a list of credentials for the aws-sdk, this is deprecated' do
-      Stud::Temporary.directory do |tmp_directory|
-        old_credentials_config = {
-          "credentials" => ['1234', 'secret'],
-          "backup_to_dir" => tmp_directory,
-          "bucket" => "logstash-test"
-        }
-
-        plugin = LogStash::Inputs::S3.new(old_credentials_config)
-        expect{ plugin.register }.not_to raise_error
-      end
-    end
   end
 
   shared_examples "generated events"  do
@@ -252,11 +222,19 @@ describe LogStash::Inputs::S3 do
   context 'when working with logs' do
     let(:objects) { [log] }
     let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day) }
+    let(:data) { File.read(log_file) }
 
     before do
-      allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(nil) { objects }
-      allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:[]).with(log.key) { log }
-      expect(log).to receive(:read)  { |&block| block.call(File.read(log_file)) }
+      Aws.config[:s3] = {
+          stub_responses: {
+              get_object: { body: data }
+          }
+      }
+      allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects }
+      allow_any_instance_of(Aws::S3::Bucket).to receive(:object).with(log.key) { log }
+      expect(log).to receive(:get).with(instance_of(Hash)) do |arg|
+        File.open(arg[:response_target], 'wb') { |s3file| s3file.write(data) }
+      end
     end
 
     context "when event doesn't have a `message` field" do

data/spec/support/helpers.rb

@@ -8,28 +8,28 @@ end
 
 # delete_files(prefix)
 def upload_file(local_file, remote_name)
-  bucket = s3object.buckets[ENV['AWS_LOGSTASH_TEST_BUCKET']]
+  bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
   file = File.expand_path(File.join(File.dirname(__FILE__), local_file))
-  bucket.objects[remote_name].write(:file => file)
+  bucket.object(remote_name).upload_file(file)
 end
 
 def delete_remote_files(prefix)
-  bucket = s3object.buckets[ENV['AWS_LOGSTASH_TEST_BUCKET']]
-  bucket.objects.with_prefix(prefix).each { |object| object.delete }
+  bucket = s3object.bucket(ENV['AWS_LOGSTASH_TEST_BUCKET'])
+  bucket.objects(:prefix => prefix).each { |object| object.delete }
 end
 
 def list_remote_files(prefix, target_bucket = ENV['AWS_LOGSTASH_TEST_BUCKET'])
-  bucket = s3object.buckets[target_bucket]
-  bucket.objects.with_prefix(prefix).collect(&:key)
+  bucket = s3object.bucket(target_bucket)
+  bucket.objects(:prefix => prefix).collect(&:key)
 end
 
 def delete_bucket(name)
-  s3object.buckets[name].objects.map(&:delete)
-  s3object.buckets[name].delete
+  s3object.bucket(name).objects.map(&:delete)
+  s3object.bucket(name).delete
 end
 
 def s3object
-  AWS::S3.new
+  Aws::S3::Resource.new
 end
 
 class TestInfiniteS3Object

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.1.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-09 00:00:00.000000000 Z
+date: 2016-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.99'
   name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -153,7 +159,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.3
 signing_key:
 specification_version: 4
 summary: Stream events from files from a S3 bucket.