logstash-input-s3 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 97fc564bb487d52387e701d636f9a3dc9e680664
-  data.tar.gz: f913210e99472100a84fb118c22aa1dbd8322e41
+  metadata.gz: 091d07a422c428d7e3a286487f0105fa74061871
+  data.tar.gz: 13f4efb4b72e68feab947f88a6188d4a253847c7
 SHA512:
-  metadata.gz: 35cd954ada60910705cb28bea4f15a99a361cc7c1c4a366ce695989410496214a7aba99c906ad7d7ab70cebac82406ff3cb7d801f4c32cdb7566af321976252a
-  data.tar.gz: eae0811f559d08279afaa248a54b6aa2e7578a4fd5a1f24430f5b2e3054d779fef41ba7b6dd229a865526dcd9577d926680345a3b10efe262dc20e8c6e67dc52
+  metadata.gz: 4031253ceb6d2df1a99dbc55017012efe0b599953ab47717c9a2335b55c8b486ed3de00ade5b005541f6e45c9c16daacc5aee776e7e19c5d123e33d84d4b1fd8
+  data.tar.gz: 32261a2c04084c6ac9674277070c6e965bf4c78d5fb2099855909f8e847ee3ae9ded5a904fabe65fa33382331daa3cfbadd8df6f540a87b94b3f3ab3904e285b

data/README.md

@@ -1,15 +1,33 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
+## Required S3 Permissions
+
+This plugin reads from your S3 bucket, and would require the following
+permissions applied to the AWS IAM Policy being used:
+
+* `s3:ListBucket` to check if the S3 bucket exists and list objects in it.
+* `s3:GetObject` to check object metadata and download objects from S3 buckets.
+
+You might also need `s3:DeleteObject` when setting S3 input to delete on read.
+And the `s3:CreateBucket` permission to create a backup bucket unless already
+exists.
+
+For buckets that have versioning enabled, you might need to add additional
+permissions.
+
+More information about S3 permissions can be found at -
+  http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -83,4 +101,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/inputs/s3.rb

@@ -97,6 +97,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
   public
   def run(queue)
+    @current_thread = Thread.current
     Stud.interval(@interval) do
       process_files(queue)
     end
@@ -143,18 +144,40 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     objects = list_new_files
 
     objects.each do |key|
-      @logger.debug("S3 input processing", :bucket => @bucket, :key => key)
-
-      lastmod = @s3bucket.objects[key].last_modified
-
-      process_log(queue, key)
-
-      sincedb.write(lastmod)
+      if stop?
+        break
+      else
+        @logger.debug("S3 input processing", :bucket => @bucket, :key => key)
+        process_log(queue, key)
+      end
     end
   end # def process_files
 
+  public
+  def stop
+    # @current_thread is initialized in the `#run` method, 
+    # this variable is needed because the `#stop` is a called in another thread 
+    # than the `#run` method and requiring us to call stop! with a explicit thread.
+    Stud.stop!(@current_thread)
+  end
+
+  public
+  def aws_service_endpoint(region)
+    region_to_use = get_region
+
+    return {
+      :s3_endpoint => region_to_use == 'us-east-1' ?
+        's3.amazonaws.com' : "s3-#{region_to_use}.amazonaws.com"
+    }
+  end
 
   private
+
+  # Read the content of the local file
+  #
+  # @param [Queue] Where to push the event
+  # @param [String] Which file to read from
+  # @return [Boolean] True if the file was completely read, false otherwise.
   def process_local_log(queue, filename)
     @logger.debug('Processing file', :filename => filename)
 
@@ -163,6 +186,11 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     # So all IO stuff: decompression, reading need to be done in the actual
     # input and send as bytes to the codecs.
     read_file(filename) do |line|
+      if stop?
+        @logger.warn("Logstash S3 input, stop reading in the middle of the file, we will read it again when logstash is started")
+        return false
+      end
+
       @codec.decode(line) do |event|
         # We are making an assumption concerning cloudfront
         # log format, the user will use the plain or the line codec
@@ -186,6 +214,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
         end
       end
     end
+
+    return true
   end # def process_local_log
 
   private
@@ -287,26 +317,40 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     object = @s3bucket.objects[key]
 
     filename = File.join(temporary_directory, File.basename(key))
-
-    download_remote_file(object, filename)
-
-    process_local_log(queue, filename)
-
-    backup_to_bucket(object, key)
-    backup_to_dir(filename)
-
-    delete_file_from_bucket(object)
-    FileUtils.remove_entry_secure(filename, true)
+    
+    if download_remote_file(object, filename)
+      if process_local_log(queue, filename)
+        backup_to_bucket(object, key)
+        backup_to_dir(filename)
+        delete_file_from_bucket(object)
+        FileUtils.remove_entry_secure(filename, true)
+        lastmod = object.last_modified
+        sincedb.write(lastmod)
+      end
+    else
+      FileUtils.remove_entry_secure(filename, true)
+    end
   end
 
   private
+  # Stream the remove file to the local disk
+  #
+  # @param [S3Object] Reference to the remove S3 objec to download
+  # @param [String] The Temporary filename to stream to.
+  # @return [Boolean] True if the file was completely downloaded
   def download_remote_file(remote_object, local_filename)
+    completed = false
+
     @logger.debug("S3 input: Download remote file", :remote_key => remote_object.key, :local_filename => local_filename)
     File.open(local_filename, 'wb') do |s3file|
       remote_object.read do |chunk|
+        return completed if stop?
         s3file.write(chunk)
       end
     end
+    completed = true
+
+    return completed
   end
 
   private
@@ -350,22 +394,10 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
       @secret_access_key = @credentials[1]
     end
 
-    if @credentials
-      s3 = AWS::S3.new(
-        :access_key_id => @access_key_id,
-        :secret_access_key => @secret_access_key,
-        :region => @region
-      )
-    else
-      s3 = AWS::S3.new(aws_options_hash)
-    end
+    s3 = AWS::S3.new(aws_options_hash)
   end
 
   private
-  def aws_service_endpoint(region)
-    return { :s3_endpoint => region }
-  end
-
   module SinceDB
     class File
       def initialize(file)

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '1.0.0'
+  s.version         = '2.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files from a S3 bucket."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "logstash-core", "~> 2.0.0.snapshot"
   s.add_runtime_dependency 'logstash-mixin-aws'
   s.add_runtime_dependency 'stud', '~> 0.0.18'
 

data/spec/inputs/s3_spec.rb

@@ -9,38 +9,90 @@ require "fileutils"
 
 describe LogStash::Inputs::S3 do
   let(:temporary_directory) { Stud::Temporary.pathname }
+  let(:sincedb_path) { Stud::Temporary.pathname }
   let(:day) { 3600 * 24 }
-  let(:settings) {
+  let(:config) {
     {
       "access_key_id" => "1234",
       "secret_access_key" => "secret",
       "bucket" => "logstash-test",
-      "temporary_directory" => temporary_directory
+      "temporary_directory" => temporary_directory,
+      "sincedb_path" => File.join(sincedb_path, ".sincedb")
     }
   }
 
   before do
+    FileUtils.mkdir_p(sincedb_path)
     AWS.stub!
     Thread.abort_on_exception = true
   end
 
+  context "when interrupting the plugin" do
+    let(:config) { super.merge({ "interval" => 5 }) }
+
+    before do
+      expect_any_instance_of(LogStash::Inputs::S3).to receive(:list_new_files).and_return(TestInfiniteS3Object.new)
+    end
+
+    it_behaves_like "an interruptible input plugin"
+  end
+
   describe "#register" do
-    subject { LogStash::Inputs::S3.new(settings) }
+    subject { LogStash::Inputs::S3.new(config) }
 
     context "with temporary directory" do
+      let(:temporary_directory) { Stud::Temporary.pathname }
+
+      it "creates the direct when it doesn't exist" do
+        expect { subject.register }.to change { Dir.exist?(temporary_directory) }.from(false).to(true)
+      end
+    end
+  end
+
+  describe '#get_s3object' do
+    subject { LogStash::Inputs::S3.new(settings) }
+
+    context 'with deprecated credentials option' do
+      let(:settings) {
+        {
+          "credentials" => ["1234", "secret"],
+          "proxy_uri" => "http://example.com",
+          "bucket" => "logstash-test",
+        }
+      }
+
+      it 'should instantiate AWS::S3 clients with a proxy set' do
+        expect(AWS::S3).to receive(:new).with({
+          :access_key_id => "1234",
+          :secret_access_key => "secret",
+          :proxy_uri => 'http://example.com',
+          :use_ssl => subject.use_ssl,
+        }.merge(subject.aws_service_endpoint(subject.region)))
+
+        subject.send(:get_s3object)
+      end
+    end
+
+    context 'with modern access key options' do
       let(:settings) {
         {
           "access_key_id" => "1234",
           "secret_access_key" => "secret",
+          "proxy_uri" => "http://example.com",
           "bucket" => "logstash-test",
-          "temporary_directory" => temporary_directory
         }
       }
 
-      let(:temporary_directory) { Stud::Temporary.pathname }
+      it 'should instantiate AWS::S3 clients with a proxy set' do
+        expect(AWS::S3).to receive(:new).with({
+          :access_key_id => "1234",
+          :secret_access_key => "secret",
+          :proxy_uri => 'http://example.com',
+          :use_ssl => subject.use_ssl,
+        }.merge(subject.aws_service_endpoint(subject.region)))
+        
 
-      it "creates the direct when it doesn't exist" do
-        expect { subject.register }.to change { Dir.exist?(temporary_directory) }.from(false).to(true)
+        subject.send(:get_s3object)
       end
     end
   end
@@ -58,15 +110,15 @@ describe LogStash::Inputs::S3 do
     }
 
     it 'should allow user to exclude files from the s3 bucket' do
-      config = LogStash::Inputs::S3.new(settings.merge({ "exclude_pattern" => "^exclude" }))
-      config.register
-      expect(config.list_new_files).to eq([present_object.key])
+      plugin = LogStash::Inputs::S3.new(config.merge({ "exclude_pattern" => "^exclude" }))
+      plugin.register
+      expect(plugin.list_new_files).to eq([present_object.key])
     end
 
     it 'should support not providing a exclude pattern' do
-      config = LogStash::Inputs::S3.new(settings)
-      config.register
-      expect(config.list_new_files).to eq(objects_list.map(&:key))
+      plugin = LogStash::Inputs::S3.new(config)
+      plugin.register
+      expect(plugin.list_new_files).to eq(objects_list.map(&:key))
     end
 
     context "If the bucket is the same as the backup bucket" do
@@ -78,20 +130,20 @@ describe LogStash::Inputs::S3 do
 
         allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(nil) { objects_list }
 
-        config = LogStash::Inputs::S3.new(settings.merge({ 'backup_add_prefix' => 'mybackup',
-                                                           'backup_to_bucket' => settings['bucket']}))
-        config.register
-        expect(config.list_new_files).to eq([present_object.key])
+        plugin = LogStash::Inputs::S3.new(config.merge({ 'backup_add_prefix' => 'mybackup',
+                                                         'backup_to_bucket' => config['bucket']}))
+        plugin.register
+        expect(plugin.list_new_files).to eq([present_object.key])
       end
     end
 
     it 'should ignore files older than X' do
-      config = LogStash::Inputs::S3.new(settings.merge({ 'backup_add_prefix' => 'exclude-this-file'}))
+      plugin = LogStash::Inputs::S3.new(config.merge({ 'backup_add_prefix' => 'exclude-this-file'}))
 
       expect_any_instance_of(LogStash::Inputs::S3::SinceDB::File).to receive(:read).exactly(objects_list.size) { Time.now - day }
-      config.register
+      plugin.register
 
-      expect(config.list_new_files).to eq([present_object.key])
+      expect(plugin.list_new_files).to eq([present_object.key])
     end
 
     it 'should ignore file if the file match the prefix' do
@@ -104,9 +156,9 @@ describe LogStash::Inputs::S3 do
 
         allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(prefix) { objects_list }
 
-        config = LogStash::Inputs::S3.new(settings.merge({ 'prefix' => prefix }))
-        config.register
-        expect(config.list_new_files).to eq([present_object.key])
+        plugin = LogStash::Inputs::S3.new(config.merge({ 'prefix' => prefix }))
+        plugin.register
+        expect(plugin.list_new_files).to eq([present_object.key])
     end
 
     it 'should sort return object sorted by last_modification date with older first' do
@@ -119,41 +171,41 @@ describe LogStash::Inputs::S3 do
       allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(nil) { objects }
 
 
-      config = LogStash::Inputs::S3.new(settings)
-      config.register
-      expect(config.list_new_files).to eq(['TWO_DAYS_AGO', 'YESTERDAY', 'TODAY'])
+      plugin = LogStash::Inputs::S3.new(config)
+      plugin.register
+      expect(plugin.list_new_files).to eq(['TWO_DAYS_AGO', 'YESTERDAY', 'TODAY'])
     end
 
     describe "when doing backup on the s3" do
       it 'should copy to another s3 bucket when keeping the original file' do
-        config = LogStash::Inputs::S3.new(settings.merge({ "backup_to_bucket" => "mybackup"}))
-        config.register
+        plugin = LogStash::Inputs::S3.new(config.merge({ "backup_to_bucket" => "mybackup"}))
+        plugin.register
 
         s3object = double()
         expect(s3object).to receive(:copy_to).with('test-file', :bucket => an_instance_of(AWS::S3::Bucket))
 
-        config.backup_to_bucket(s3object, 'test-file')
+        plugin.backup_to_bucket(s3object, 'test-file')
       end
 
       it 'should move to another s3 bucket when deleting the original file' do
-        config = LogStash::Inputs::S3.new(settings.merge({ "backup_to_bucket" => "mybackup", "delete" => true }))
-        config.register
+        plugin = LogStash::Inputs::S3.new(config.merge({ "backup_to_bucket" => "mybackup", "delete" => true }))
+        plugin.register
 
         s3object = double()
         expect(s3object).to receive(:move_to).with('test-file', :bucket => an_instance_of(AWS::S3::Bucket))
 
-        config.backup_to_bucket(s3object, 'test-file')
+        plugin.backup_to_bucket(s3object, 'test-file')
       end
 
       it 'should add the specified prefix to the backup file' do
-        config = LogStash::Inputs::S3.new(settings.merge({ "backup_to_bucket" => "mybackup",
+        plugin = LogStash::Inputs::S3.new(config.merge({ "backup_to_bucket" => "mybackup",
                                                            "backup_add_prefix" => 'backup-' }))
-        config.register
+        plugin.register
 
         s3object = double()
         expect(s3object).to receive(:copy_to).with('backup-test-file', :bucket => an_instance_of(AWS::S3::Bucket))
 
-        config.backup_to_bucket(s3object, 'test-file')
+        plugin.backup_to_bucket(s3object, 'test-file')
       end
     end
 
@@ -162,9 +214,9 @@ describe LogStash::Inputs::S3 do
         Stud::Temporary.file do |source_file|
           backup_file = File.join(backup_dir.to_s, Pathname.new(source_file.path).basename.to_s)
 
-          config = LogStash::Inputs::S3.new(settings.merge({ "backup_to_dir" => backup_dir }))
+          plugin = LogStash::Inputs::S3.new(config.merge({ "backup_to_dir" => backup_dir }))
 
-          config.backup_to_dir(source_file)
+          plugin.backup_to_dir(source_file)
 
           expect(File.exists?(backup_file)).to eq(true)
         end
@@ -173,26 +225,26 @@ describe LogStash::Inputs::S3 do
 
     it 'should accepts a list of credentials for the aws-sdk, this is deprecated' do
       Stud::Temporary.directory do |tmp_directory|
-        old_credentials_settings = {
+        old_credentials_config = {
           "credentials" => ['1234', 'secret'],
           "backup_to_dir" => tmp_directory,
           "bucket" => "logstash-test"
         }
 
-        config = LogStash::Inputs::S3.new(old_credentials_settings)
-        expect{ config.register }.not_to raise_error
+        plugin = LogStash::Inputs::S3.new(old_credentials_config)
+        expect{ plugin.register }.not_to raise_error
       end
     end
   end
 
   shared_examples "generated events"  do
     it 'should process events' do
-      events = fetch_events(settings)
+      events = fetch_events(config)
       expect(events.size).to eq(2)
     end
 
     it "deletes the temporary file" do
-      events = fetch_events(settings)
+      events = fetch_events(config)
       expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(0)
     end
   end
@@ -209,7 +261,7 @@ describe LogStash::Inputs::S3 do
 
     context "when event doesn't have a `message` field" do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'json.log') }
-      let(:settings) {
+      let(:config) {
         {
           "access_key_id" => "1234",
           "secret_access_key" => "secret",
@@ -225,7 +277,6 @@ describe LogStash::Inputs::S3 do
       let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day) }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gz') }
 
-
       include_examples "generated events"
     end
 
@@ -245,7 +296,7 @@ describe LogStash::Inputs::S3 do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'cloudfront.log') }
 
       it 'should extract metadata from cloudfront log' do
-        events = fetch_events(settings)
+        events = fetch_events(config)
 
         events.each do |event|
           expect(event['cloudfront_fields']).to eq('date time x-edge-location c-ip x-event sc-bytes x-cf-status x-cf-client-id cs-uri-stem cs-uri-query c-referrer x-page-url​  c-user-agent x-sname x-sname-query x-file-ext x-sid')

data/spec/support/helpers.rb

@@ -3,7 +3,6 @@ def fetch_events(settings)
   s3 = LogStash::Inputs::S3.new(settings)
   s3.register
   s3.process_files(queue)
-  s3.teardown
   queue
 end
 
@@ -32,3 +31,15 @@ end
 def s3object
   AWS::S3.new
 end
+
+class TestInfiniteS3Object
+  def each
+    counter = 1
+
+    loop do
+      yield "awesome-#{counter}"
+      counter +=1
+    end
+  end
+end
+

metadata

@@ -1,133 +1,125 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
-      - !ruby/object:Gem::Version
-        version: 2.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - <
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: 2.0.0.snapshot
+  name: logstash-core
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-mixin-aws
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.0.snapshot
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-mixin-aws
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.18
+        version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.18
+  name: stud
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.0.18
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: simplecov
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: simplecov
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: coveralls
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: coveralls
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-json
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
 - CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
 - README.md
-- Rakefile
 - lib/logstash/inputs/s3.rb
 - logstash-input-s3.gemspec
 - spec/fixtures/cloudfront.log
@@ -160,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Stream events from files from a S3 bucket.

data/.gitignore

@@ -1,5 +0,0 @@
-*.gem
-Gemfile.lock
-.bundle
-vendor
-coverage/

data/Rakefile

@@ -1,7 +0,0 @@
-@files=[]
-
-task :default do
-  system("rake -T")
-end
-
-require "logstash/devutils/rake"