logstash-input-s3 3.7.0 → 3.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0b5ad2fe42e0003da67ebb3e1aad774f5bab3cb2afc07e4699fbfa5e84c0d472
-  data.tar.gz: fb8691a6276181e17a2c80319247d04f11c8d547affff1b109f5447547eca4be
+  metadata.gz: 584ac32b0d4a7d0c3d478b90ad92d3170d5926d6d802d46b31a5075fc7bd27e9
+  data.tar.gz: 2a24ecd946ce52b6aa29acf3352314a26750c9b1350f33ecd685f0e206cd5d4a
 SHA512:
-  metadata.gz: 3782d59c63fe4e53cb01dff9c7a431773fc69126dfc17c90ea45b8018205897dfabd8d942a732c37cb1740a4c924608110e81955205734523a9f5d1a928bed5d
-  data.tar.gz: 3ff5255afa6d8a978a2231bc69a3c11dd9f354a90895e779be569971aaa75c34c456d35b979effbb0304801160cb15ce5f81b5967e121133d79e0a48f59638b7
+  metadata.gz: c306fb6d68a0b58cbb109508590d07cbd9cca9b67534e039c2fad2cc033e8756bdddd2ecde0f9177e8faf2ec65e4fb84f8b2e6576d65742d32f0867da0ab10b0
+  data.tar.gz: 293e978d3301e7d545927daa5a087bc433bf948d18662bf97cfb164dd59928c7e58719b6dcc7ede1fbed6c5d9b519c416ff44a043a5fd130123a1bf64d86231e

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 3.8.3
+ - Fix missing `metadata` and `type` of the last event [#223](https://github.com/logstash-plugins/logstash-input-s3/pull/223)
+
+## 3.8.2
+ - Refactor: read sincedb time once per bucket listing [#233](https://github.com/logstash-plugins/logstash-input-s3/pull/233)
+
+## 3.8.1
+ - Feat: cast true/false values for additional_settings [#232](https://github.com/logstash-plugins/logstash-input-s3/pull/232)
+
+## 3.8.0
+ - Add ECS v8 support.
+
 ## 3.7.0
  - Add ECS support. [#228](https://github.com/logstash-plugins/logstash-input-s3/pull/228)
  - Fix missing file in cutoff time change. [#224](https://github.com/logstash-plugins/logstash-input-s3/pull/224)

data/docs/index.asciidoc

@@ -189,7 +189,7 @@ Whether to delete processed files from the original bucket.
   * Value type is <<string,string>>
   * Supported values are:
   ** `disabled`: does not use ECS-compatible field names
-  ** `v1`: uses metadata fields that are compatible with Elastic Common Schema
+  ** `v1`,`v8`: uses metadata fields that are compatible with Elastic Common Schema
 
 Controls this plugin's compatibility with the
 {ecs-ref}[Elastic Common Schema (ECS)].

data/lib/logstash/inputs/s3.rb

@@ -28,7 +28,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   java_import java.util.zip.ZipException
 
   include LogStash::PluginMixins::AwsConfig::V2
-  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1)
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
 
   config_name "s3"
 
@@ -139,6 +139,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     objects = []
     found = false
     current_time = Time.now
+    sincedb_time = sincedb.read
     begin
       @s3bucket.objects(:prefix => @prefix).each do |log|
         found = true
@@ -147,7 +148,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
           @logger.debug('Ignoring', :key => log.key)
         elsif log.content_length <= 0
           @logger.debug('Object Zero Length', :key => log.key)
-        elsif !sincedb.newer?(log.last_modified)
+        elsif log.last_modified <= sincedb_time
           @logger.debug('Object Not Modified', :key => log.key)
         elsif log.last_modified > (current_time - CUTOFF_SECOND).utc # file modified within last two seconds will be processed in next cycle
           @logger.debug('Object Modified After Cutoff Time', :key => log.key)
@@ -234,30 +235,34 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
           @logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
           update_metadata(metadata, event)
         else
-          decorate(event)
-
-          if @include_object_properties
-            event.set("[@metadata][s3]", object.data.to_h)
-          else
-            event.set("[@metadata][s3]", {})
-          end
-
-          event.set("[@metadata][s3][key]", object.key)
-          event.set(@cloudfront_version_key, metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
-          event.set(@cloudfront_fields_key, metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
-
-          queue << event
+          push_decoded_event(queue, metadata, object, event)
         end
       end
     end
     # #ensure any stateful codecs (such as multi-line ) are flushed to the queue
     @codec.flush do |event|
-      queue << event
+      push_decoded_event(queue, metadata, object, event)
     end
 
     return true
   end # def process_local_log
 
+  def push_decoded_event(queue, metadata, object, event)
+    decorate(event)
+
+    if @include_object_properties
+      event.set("[@metadata][s3]", object.data.to_h)
+    else
+      event.set("[@metadata][s3]", {})
+    end
+
+    event.set("[@metadata][s3][key]", object.key)
+    event.set(@cloudfront_version_key, metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
+    event.set(@cloudfront_fields_key, metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
+
+    queue << event
+  end
+
   def event_is_metadata?(event)
     return false unless event.get("message").class == String
     line = event.get("message")
@@ -353,14 +358,22 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   end
 
   def symbolized_settings
-    @symbolized_settings ||= symbolize(@additional_settings)
+    @symbolized_settings ||= symbolize_keys_and_cast_true_false(@additional_settings)
   end
 
-  def symbolize(hash)
-    return hash unless hash.is_a?(Hash)
-    symbolized = {}
-    hash.each { |key, value| symbolized[key.to_sym] = symbolize(value) }
-    symbolized
+  def symbolize_keys_and_cast_true_false(hash)
+    case hash
+    when Hash
+      symbolized = {}
+      hash.each { |key, value| symbolized[key.to_sym] = symbolize_keys_and_cast_true_false(value) }
+      symbolized
+    when 'true'
+      true
+    when 'false'
+      false
+    else
+      hash
+    end
   end
 
   def ignore_filename?(filename)
@@ -456,10 +469,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
         @sincedb_path = file
       end
 
-      def newer?(date)
-        date > read
-      end
-
+      # @return [Time]
       def read
         if ::File.exists?(@sincedb_path)
           content = ::File.read(@sincedb_path).chomp.strip
@@ -471,7 +481,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
       end
 
       def write(since = nil)
-        since = Time.now() if since.nil?
+        since = Time.now if since.nil?
         ::File.open(@sincedb_path, 'w') { |file| file.write(since.to_s) }
       end
     end

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.7.0'
+  s.version         = '3.8.3'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files in a S3 bucket"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -27,5 +27,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency "logstash-codec-json"
   s.add_development_dependency "logstash-codec-multiline"
-  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1'
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.2'
 end

data/spec/inputs/s3_spec.rb

@@ -42,7 +42,9 @@ describe LogStash::Inputs::S3 do
       expect_any_instance_of(LogStash::Inputs::S3).to receive(:list_new_files).and_return(TestInfiniteS3Object.new(s3_obj))
     end
 
-    it_behaves_like "an interruptible input plugin"
+    it_behaves_like "an interruptible input plugin" do
+      let(:allowed_lag) { 16 } if LOGSTASH_VERSION.split('.').first.to_i <= 6
+    end
   end
 
   describe "#register" do
@@ -82,10 +84,10 @@ describe LogStash::Inputs::S3 do
     end
 
     describe "additional_settings" do
-      context 'when force_path_style is set' do
+      context "supported settings" do
         let(:settings) {
           {
-            "additional_settings" => { "force_path_style" => true },
+            "additional_settings" => { "force_path_style" => 'true', "ssl_verify_peer" => 'false', "profile" => 'logstash' },
             "bucket" => "logstash-test",
           }
         }
@@ -93,7 +95,7 @@ describe LogStash::Inputs::S3 do
         it 'should instantiate AWS::S3 clients with force_path_style set' do
           expect(Aws::S3::Resource).to receive(:new).with({
             :region => subject.region,
-            :force_path_style => true
+            :force_path_style => true, :ssl_verify_peer => false, :profile => 'logstash'
           }).and_call_original
 
           subject.send(:get_s3object)
@@ -193,6 +195,7 @@ describe LogStash::Inputs::S3 do
       it 'should log that no files were found in the bucket' do
         plugin = LogStash::Inputs::S3.new(config)
         plugin.register
+        allow(plugin.logger).to receive(:info).with(/Using the provided sincedb_path/, anything)
         expect(plugin.logger).to receive(:info).with(/No files found/, anything)
         expect(plugin.list_new_files).to be_empty
       end
@@ -326,6 +329,7 @@ describe LogStash::Inputs::S3 do
       events = fetch_events(config)
       expect(events.size).to eq(events_to_process)
       expect(events[0].get("[@metadata][s3][key]")).to eql log.key
+      expect(events[1].get("[@metadata][s3][key]")).to eql log.key
     end
 
     it "deletes the temporary file" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.7.0
+  version: 3.8.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-24 00:00:00.000000000 Z
+date: 2022-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -105,7 +105,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.2'
   name: logstash-mixin-ecs_compatibility_support
   prerelease: false
   type: :runtime
@@ -113,7 +113,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.2'
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program
@@ -167,8 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Streams events from files in a S3 bucket