logstash-input-s3 3.3.6 → 3.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33d02e0410e180a35672fc42215f5758ab34592b0acbac20caeedc3b2c555f8c
-  data.tar.gz: 5010dba8d504458b455a4941db2de4accd039b8ff47eeeb31da4059d0580847f
+  metadata.gz: b6b2e69a2cc95f3fc7bdcc0a5b828e08a3795d88f5468795fafc4a518b7e4128
+  data.tar.gz: 5a1d0103482e624fe8131eed9c28e636c71bb38b70980e463e1548bcb7efaff1
 SHA512:
-  metadata.gz: 57ad952479ee8ac3a8582898f630e1db633101a595693566dd7697f4f70b7fc74568f1ab6da95e9498bae49f73dc6a4d76e96b28d677eab492965fc70e948f99
-  data.tar.gz: 34d23aebf95e88011d9bb6c2d4e6a14a2e7fc8110003672c9c703613bcbba095f5311043906022fcec443424b865c87b48d49682a13d5667d7fd5184deadb949
+  metadata.gz: 7cb50c30b2acdd5f8da2602346b0fc9b3ef82f4dd5d1649e7cff8e7d746c28ef0e997cb8db8c1821639e3e8b8001ef19e2c8a0557d5b8a6f37ea9b5a27e451f8
+  data.tar.gz: 41be7059efac9cd05b2379a5035cc11104bd7c9716939ac9a2c63aa94ee7bbf79832c7219492ce5576987d36bae038c530af64c11e021c89dee47aa53ea4bf51

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.3.7
+  - Added ability to optionally include S3 object properties inside @metadata [#155](https://github.com/logstash-plugins/logstash-input-s3/pull/155)
+
 ## 3.3.6
   - Fixed error in documentation by removing illegal commas [#154](https://github.com/logstash-plugins/logstash-input-s3/pull/154)
 

data/docs/index.asciidoc

@@ -44,6 +44,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-delete>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-exclude_pattern>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-include_object_properties>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-interval>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-proxy_uri>> |<<string,string>>|No
@@ -176,6 +177,15 @@ the connection to s3. See full list in https://docs.aws.amazon.com/sdkforruby/ap
       }
     }
 
+[id="plugins-{type}s-{plugin}-include_object_properties"]
+===== `include_object_properties`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Whether or not to include the S3 object's properties (last_modified, content_type, metadata) into each Event at
+`[@metadata][s3]`. Regardless of this setting, `[@metdata][s3][key]` will always be present.
+
 [id="plugins-{type}s-{plugin}-interval"]
 ===== `interval` 
 

data/lib/logstash/inputs/s3.rb

@@ -70,6 +70,11 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   # default to the current OS temporary directory in linux /tmp/logstash
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
 
+  # Whether or not to include the S3 object's properties (last_modified, content_type, metadata)
+  # into each Event at [@metadata][s3]. Regardless of this setting, [@metdata][s3][key] will always
+  # be present.
+  config :include_object_properties, :validate => :boolean, :default => false
+
   public
   def register
     require "fileutils"
@@ -177,8 +182,9 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   #
   # @param [Queue] Where to push the event
   # @param [String] Which file to read from
+  # @param [S3Object] Source s3 object
   # @return [Boolean] True if the file was completely read, false otherwise.
-  def process_local_log(queue, filename, key)
+  def process_local_log(queue, filename, object)
     @logger.debug('Processing file', :filename => filename)
     metadata = {}
     # Currently codecs operates on bytes instead of stream.
@@ -209,7 +215,13 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
           event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
           event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
 
-          event.set("[@metadata][s3]", { "key" => key })
+          if @include_object_properties
+            event.set("[@metadata][s3]", object.data.to_h)
+          else
+            event.set("[@metadata][s3]", {})
+          end
+
+          event.set("[@metadata][s3][key]", object.key)
 
           queue << event
         end
@@ -365,7 +377,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
     filename = File.join(temporary_directory, File.basename(key))
     if download_remote_file(object, filename)
-      if process_local_log(queue, filename, key)
+      if process_local_log(queue, filename, object)
         lastmod = object.last_modified
         backup_to_bucket(object)
         backup_to_dir(filename)

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.3.6'
+  s.version         = '3.3.7'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files in a S3 bucket"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/s3_spec.rb

@@ -277,7 +277,7 @@ describe LogStash::Inputs::S3 do
     it 'should process events' do
       events = fetch_events(config)
       expect(events.size).to eq(events_to_process)
-      insist { events[0].get("[@metadata][s3]") } == {"key" => log.key }
+      insist { events[0].get("[@metadata][s3][key]") } == log.key
     end
 
     it "deletes the temporary file" do
@@ -344,7 +344,7 @@ describe LogStash::Inputs::S3 do
 
   context 'when working with logs' do
     let(:objects) { [log] }
-    let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5) }
+    let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5, :data => { "etag" => 'c2c966251da0bc3229d12c2642ba50a4' }) }
     let(:data) { File.read(log_file) }
 
     before do
@@ -451,5 +451,34 @@ describe LogStash::Inputs::S3 do
 
       include_examples "generated events"
     end
+
+    context 'when include_object_properties is set to true' do
+      let(:config) { super.merge({ "include_object_properties" => true }) }
+      let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'uncompressed.log') }
+
+      it 'should extract object properties onto [@metadata][s3]' do
+        events = fetch_events(config)
+        events.each do |event|
+          expect(event.get('[@metadata][s3]')).to include(log.data)
+        end
+      end
+
+      include_examples "generated events"
+    end
+
+    context 'when include_object_properties is set to false' do
+      let(:config) { super.merge({ "include_object_properties" => false }) }
+      let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'uncompressed.log') }
+
+      it 'should NOT extract object properties onto [@metadata][s3]' do
+        events = fetch_events(config)
+        events.each do |event|
+          expect(event.get('[@metadata][s3]')).to_not include(log.data)
+        end
+      end
+
+      include_examples "generated events"
+    end
+
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.3.6
+  version: 3.3.7
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-07-19 00:00:00.000000000 Z
+date: 2018-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement