logstash-input-s3 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23e08a044fa6f77d449aabcb4a738dbc60ccced5
-  data.tar.gz: d11d3dbff1bd06b7d51dca355f59c91ac2fdae29
+  metadata.gz: 817a07b535fd53bce46a4015d694c66bbacb62db
+  data.tar.gz: 998c24e3ea4faa4d76c825d5f128a2024b9429be
 SHA512:
-  metadata.gz: f587cca3d88b16db9941c92f4826d28c693f91f640471b887a74729afd9097ffcbc3637e7808c6daad630d56fec4bb95253aa21fefa46acabf581759bb2387cc
-  data.tar.gz: d132d27dea448bf1e894a6164e57e5d171f865f3f689131b355d156879dfe12944e428c4e4e9ca5a741a62e825c0d35ab8b4e8ab23070d3d6bdd1aa14a53b5eb
+  metadata.gz: b6c8086b030bf048fb96e85409a106ac222500311c1531e9836304d44725d51ec5e4cc7457bc68480aa6c2c632e1fe2e62296890a8c9ae7f77def69e9fedbc14
+  data.tar.gz: eeee1c7e0a47bbe091ccf3ce6b8e49c70c40a100f10f1311b3fe0d6d966d3ebfb286802c82d408eac27ff73416ad877ad02ee6ebad9d095491832457188aac6f

data/lib/logstash/inputs/s3.rb

@@ -2,7 +2,6 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/plugin_mixins/aws_config"
-
 require "time"
 require "tmpdir"
 require "stud/interval"
@@ -70,6 +69,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
   public
   def register
+    require "fileutils"
     require "digest/md5"
     require "aws-sdk"
 
@@ -91,6 +91,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     unless @backup_to_dir.nil?
       Dir.mkdir(@backup_to_dir, 0700) unless File.exists?(@backup_to_dir)
     end
+
+    FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory)
   end # def register
 
   public
@@ -188,7 +190,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
   private
   def event_is_metadata?(event)
-    line = event['message']
+    return false if event["message"].nil?
+    line = event["message"]
     version_metadata?(line) || fields_metadata?(line)
   end
 
@@ -293,11 +296,12 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     backup_to_dir(filename)
 
     delete_file_from_bucket(object)
+    FileUtils.remove_entry_secure(filename, true)
   end
 
   private
   def download_remote_file(remote_object, local_filename)
-    @logger.debug("S3 input: Download remove file", :remote_key => remote_object.key, :local_filename => local_filename)
+    @logger.debug("S3 input: Download remote file", :remote_key => remote_object.key, :local_filename => local_filename)
     File.open(local_filename, 'wb') do |s3file|
       remote_object.read do |chunk|
         s3file.write(chunk)

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '0.1.9'
+  s.version         = '0.1.10'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files from a S3 bucket."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -27,5 +27,6 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'simplecov'
   s.add_development_dependency 'coveralls'
+  s.add_development_dependency "logstash-codec-json"
 end
 

data/spec/fixtures/json.log

@@ -0,0 +1,2 @@
+{ "hello": "world" }
+{ "hello": "awesome world" }

data/spec/inputs/s3_spec.rb

@@ -2,24 +2,49 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/s3"
 require "logstash/errors"
-require "aws-sdk"
-require "stud/temporary"
 require_relative "../support/helpers"
+require "stud/temporary"
+require "aws-sdk"
+require "fileutils"
 
 describe LogStash::Inputs::S3 do
-  before do
-    AWS.stub!
-    Thread.abort_on_exception = true
-  end
+  let(:temporary_directory) { Stud::Temporary.pathname }
   let(:day) { 3600 * 24 }
   let(:settings) {
     {
       "access_key_id" => "1234",
       "secret_access_key" => "secret",
-      "bucket" => "logstash-test"
+      "bucket" => "logstash-test",
+      "temporary_directory" => temporary_directory
     }
   }
 
+  before do
+    AWS.stub!
+    Thread.abort_on_exception = true
+  end
+
+  describe "#register" do
+    subject { LogStash::Inputs::S3.new(settings) }
+
+    context "with temporary directory" do
+      let(:settings) {
+        {
+          "access_key_id" => "1234",
+          "secret_access_key" => "secret",
+          "bucket" => "logstash-test",
+          "temporary_directory" => temporary_directory
+        }
+      }
+
+      let(:temporary_directory) { Stud::Temporary.pathname }
+
+      it "creates the direct when it doesn't exist" do
+        expect { subject.register }.to change { Dir.exist?(temporary_directory) }.from(false).to(true)
+      end
+    end
+  end
+
   describe "#list_new_files" do
     before { allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(nil) { objects_list } }
 
@@ -160,6 +185,18 @@ describe LogStash::Inputs::S3 do
     end
   end
 
+  shared_examples "generated events"  do
+    it 'should process events' do
+      events = fetch_events(settings)
+      expect(events.size).to eq(2)
+    end
+
+    it "deletes the temporary file" do
+      events = fetch_events(settings)
+      expect(Dir.glob(File.join(temporary_directory, "*")).size).to eq(0)
+    end
+  end
+
   context 'when working with logs' do
     let(:objects) { [log] }
     let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day) }
@@ -170,32 +207,38 @@ describe LogStash::Inputs::S3 do
       expect(log).to receive(:read)  { |&block| block.call(File.read(log_file)) }
     end
 
+    context "when event doesn't have a `message` field" do
+      let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'json.log') }
+      let(:settings) {
+        {
+          "access_key_id" => "1234",
+          "secret_access_key" => "secret",
+          "bucket" => "logstash-test",
+          "codec" => "json",
+        }
+      }
+
+      include_examples "generated events"
+    end
+
     context 'compressed' do
       let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day) }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gz') }
 
-      it 'should process events' do
-        events = fetch_events(settings)
-        expect(events.size).to eq(2)
-      end
+
+      include_examples "generated events"
     end
 
     context 'plain text' do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'uncompressed.log') }
 
-      it 'should process events' do
-        events = fetch_events(settings)
-        expect(events.size).to eq(2)
-      end
+      include_examples "generated events"
     end
 
     context 'encoded' do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'invalid_utf8.log') }
 
-      it 'should work with invalid utf-8 log event' do
-        events = fetch_events(settings)
-        expect(events.size).to eq(2)
-      end
+      include_examples "generated events"
     end
 
     context 'cloudfront' do
@@ -204,13 +247,13 @@ describe LogStash::Inputs::S3 do
       it 'should extract metadata from cloudfront log' do
         events = fetch_events(settings)
 
-        expect(events.size).to eq(2)
-
         events.each do |event|
           expect(event['cloudfront_fields']).to eq('date time x-edge-location c-ip x-event sc-bytes x-cf-status x-cf-client-id cs-uri-stem cs-uri-query c-referrer x-page-url​  c-user-agent x-sname x-sname-query x-file-ext x-sid')
           expect(event['cloudfront_version']).to eq('1.0')
         end
       end
+
+      include_examples "generated events"
     end
   end
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-20 00:00:00.000000000 Z
+date: 2015-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,76 +28,92 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-aws
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-mixin-aws
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 0.0.18
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.18
-  name: stud
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.18
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: simplecov
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: simplecov
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: coveralls
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: coveralls
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -117,6 +131,7 @@ files:
 - spec/fixtures/cloudfront.log
 - spec/fixtures/compressed.log.gz
 - spec/fixtures/invalid_utf8.log
+- spec/fixtures/json.log
 - spec/fixtures/uncompressed.log
 - spec/inputs/s3_spec.rb
 - spec/integration/s3_spec.rb
@@ -151,6 +166,7 @@ test_files:
 - spec/fixtures/cloudfront.log
 - spec/fixtures/compressed.log.gz
 - spec/fixtures/invalid_utf8.log
+- spec/fixtures/json.log
 - spec/fixtures/uncompressed.log
 - spec/inputs/s3_spec.rb
 - spec/integration/s3_spec.rb