logstash-input-s3 3.1.2 → 3.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44b70069c72227302a7d1c75314f1c3bcc40c67d
-  data.tar.gz: 3a42b5a34ae5df3f77732dddf7929a91d7494685
+  metadata.gz: edc29d7a88714576ad9dfb0dd152e5026825ba2f
+  data.tar.gz: faddf8e9a48bf01ea30fb1c4c77fe28bb80990e8
 SHA512:
-  metadata.gz: be6f6e9873262bd8236ed76797c05c547c718e30a283bd5ada48b7566a1eb9da314a3e92b1859c7bd128f3f761fcb1337a6a0af6386066fb89677f4b3eec2b39
-  data.tar.gz: 6ddb29cde0e89777bbf924620f49658ca0400c6c852322a54ec38753a627e40d7252da5d55419b1d7cf9118c2813d12cf9d4b5c11f767dbbad9e63d4189c6fe7
+  metadata.gz: 57d6c5a51e9208297edb48168fb83d091368ac9658f4befb0c1a60af1d7f5d2065ef71ff707292a072bac9d7487b8aabdc4639fda517f6d29a5bc5fcafc6db0f
+  data.tar.gz: a7e0dbc17be73c37f24b5595a133b25f9f731ec8c13b882f01af2b55abf2f62e5bf6e30f7a49b4b75ebb94990cb57ba230ce83551da0cbaa29e4f3ff13b16f36

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.1.3
+ - The plugin will now include the s3 key in the metadata #105
+
 ## 3.1.2
  - Fix an issue when the remote file contains multiple blob of gz in the same file #101
  - Make the integration suite run

data/lib/logstash/inputs/s3.rb

@@ -159,7 +159,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   # @param [Queue] Where to push the event
   # @param [String] Which file to read from
   # @return [Boolean] True if the file was completely read, false otherwise.
-  def process_local_log(queue, filename)
+  def process_local_log(queue, filename, key)
     @logger.debug('Processing file', :filename => filename)
     metadata = {}
     # Currently codecs operates on bytes instead of stream.
@@ -190,6 +190,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
           event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
           event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
 
+          event.set("[@metadata][s3]", { "key" => key })
+
           queue << event
         end
       end
@@ -306,7 +308,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
     filename = File.join(temporary_directory, File.basename(key))
     if download_remote_file(object, filename)
-      if process_local_log(queue, filename)
+      if process_local_log(queue, filename, key)
         lastmod = object.last_modified
         backup_to_bucket(object)
         backup_to_dir(filename)

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.1.2'
+  s.version         = '3.1.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files from a S3 bucket."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/s3_spec.rb

@@ -213,6 +213,7 @@ describe LogStash::Inputs::S3 do
     it 'should process events' do
       events = fetch_events(config)
       expect(events.size).to eq(events_to_process)
+      insist { events[0].get("[@metadata][s3]") } == {"key" => log.key }
     end
 
     it "deletes the temporary file" do
@@ -261,7 +262,7 @@ describe LogStash::Inputs::S3 do
         let(:events_to_process) { 16 }
       end
     end
-
+      
     context 'compressed' do
       let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day) }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gz') }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.1.2
+  version: 3.1.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-24 00:00:00.000000000 Z
+date: 2017-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement