logstash-input-s3 3.6.0 → 3.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4e1cc2ba334eb9e35fc68cc6a773e13b9b7de5bc6c3b4ee40cf98903d140323
-  data.tar.gz: a43fe645c1016095639e092fa4d3e8e8b77384dd17190a712a0eaa5163e68854
+  metadata.gz: 1efe9981371c5b8306ef18f09c13fff78558b66ae2ee667bd2c576b8b88218e9
+  data.tar.gz: 843c506610bd7ca9c48759cb7339f80e65211c17df9c3a83c8a5ca6fe850f39f
 SHA512:
-  metadata.gz: e10a6e21aa62270ce5f78c269fb3abf0a28b0755f78b0cb7adcc76183be460472327bdabbc94fab51c35774ae930d25846084ad3b9b0e95bd9e021c995c08bbd
-  data.tar.gz: 23b6882049688b1cd57ccba4b8aa247103bf5b9f91011f1aa426e81280c3b4ca701bf88fd96e278968c7e38fd58a01980a830b4ce2bb9ad8e034cfe07113de79
+  metadata.gz: 0cd8360c8eafce836259fada827ab42fc1972a42cec30bafe4ed7d61bea222bcc3db0c4a1ee19471a3727dffd7e9170ad48c0fe57baff5408e76e5ea60de3d44
+  data.tar.gz: 2cdeb2452504940dbd999d838d1abd905108ae748c9efa7ab9788a67b9e79c139f247c360c33393ce742189e27bb3a8763bde1725c95c76145884b0aa7a54039

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 3.8.2
+ - Refactor: read sincedb time once per bucket listing [#233](https://github.com/logstash-plugins/logstash-input-s3/pull/233)
+
+## 3.8.1
+ - Feat: cast true/false values for additional_settings [#232](https://github.com/logstash-plugins/logstash-input-s3/pull/232)
+
+## 3.8.0
+ - Add ECS v8 support.
+
+## 3.7.0
+ - Add ECS support. [#228](https://github.com/logstash-plugins/logstash-input-s3/pull/228)
+ - Fix missing file in cutoff time change. [#224](https://github.com/logstash-plugins/logstash-input-s3/pull/224)
+
 ## 3.6.0
  - Fixed unprocessed file with the same `last_modified` in ingestion. [#220](https://github.com/logstash-plugins/logstash-input-s3/pull/220)
 

data/docs/index.asciidoc

@@ -31,6 +31,21 @@ Files ending in `.gz` are handled as gzip'ed files.
 
 Files that are archived to AWS Glacier will be skipped.
 
+[id="plugins-{type}s-{plugin}-ecs_metadata"]
+==== Event Metadata and the Elastic Common Schema (ECS)
+This plugin adds cloudfront metadata to event.
+When ECS compatibility is disabled, the value is stored in the root level.
+When ECS is enabled, the value is stored in the `@metadata` where it can be used by other plugins in your pipeline.
+
+Here’s how ECS compatibility mode affects output.
+[cols="<l,<l,e,<e"]
+|=======================================================================
+| ECS disabled | ECS v1 | Availability | Description
+
+| cloudfront_fields | [@metadata][s3][cloudfront][fields] | available when the file is a CloudFront log | column names of log
+| cloudfront_version | [@metadata][s3][cloudfront][version] | available when the file is a CloudFront log | version of log
+|=======================================================================
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== S3 Input Configuration Options
 
@@ -47,6 +62,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-backup_to_dir>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-delete>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-exclude_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-gzip_pattern>> |<<string,string>>|No
@@ -96,12 +112,12 @@ the connection to s3. See full list in https://docs.aws.amazon.com/sdkforruby/ap
 [source,ruby]
     input {
       s3 {
-        "access_key_id" => "1234"
-        "secret_access_key" => "secret"
-        "bucket" => "logstash-test"
-        "additional_settings" => {
-          "force_path_style" => true
-          "follow_redirects" => false
+        access_key_id => "1234"
+        secret_access_key => "secret"
+        bucket => "logstash-test"
+        additional_settings => {
+          force_path_style => true
+          follow_redirects => false
         }
       }
     }
@@ -167,6 +183,18 @@ The name of the S3 bucket.
 
 Whether to delete processed files from the original bucket.
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+  ** `disabled`: does not use ECS-compatible field names
+  ** `v1`,`v8`: uses metadata fields that are compatible with Elastic Common Schema
+
+Controls this plugin's compatibility with the
+{ecs-ref}[Elastic Common Schema (ECS)].
+See <<plugins-{type}s-{plugin}-ecs_metadata>> for detailed information.
+
 [id="plugins-{type}s-{plugin}-endpoint"]
 ===== `endpoint`
 

data/lib/logstash/inputs/s3.rb

@@ -9,6 +9,7 @@ require "stud/interval"
 require "stud/temporary"
 require "aws-sdk"
 require "logstash/inputs/s3/patch"
+require "logstash/plugin_mixins/ecs_compatibility_support"
 
 require 'java'
 
@@ -27,6 +28,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   java_import java.util.zip.ZipException
 
   include LogStash::PluginMixins::AwsConfig::V2
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
 
   config_name "s3"
 
@@ -88,6 +90,12 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
   CUTOFF_SECOND = 3
 
+  def initialize(*params)
+    super
+    @cloudfront_fields_key = ecs_select[disabled: 'cloudfront_fields', v1: '[@metadata][s3][cloudfront][fields]']
+    @cloudfront_version_key = ecs_select[disabled: 'cloudfront_version', v1: '[@metadata][s3][cloudfront][version]']
+  end
+
   def register
     require "fileutils"
     require "digest/md5"
@@ -130,6 +138,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   def list_new_files
     objects = []
     found = false
+    current_time = Time.now
+    sincedb_time = sincedb.read
     begin
       @s3bucket.objects(:prefix => @prefix).each do |log|
         found = true
@@ -138,9 +148,9 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
           @logger.debug('Ignoring', :key => log.key)
         elsif log.content_length <= 0
           @logger.debug('Object Zero Length', :key => log.key)
-        elsif !sincedb.newer?(log.last_modified)
+        elsif log.last_modified <= sincedb_time
           @logger.debug('Object Not Modified', :key => log.key)
-        elsif log.last_modified > (Time.now - CUTOFF_SECOND).utc # file modified within last two seconds will be processed in next cycle
+        elsif log.last_modified > (current_time - CUTOFF_SECOND).utc # file modified within last two seconds will be processed in next cycle
           @logger.debug('Object Modified After Cutoff Time', :key => log.key)
         elsif (log.storage_class == 'GLACIER' || log.storage_class == 'DEEP_ARCHIVE') && !file_restored?(log.object)
           @logger.debug('Object Archived to Glacier', :key => log.key)
@@ -227,9 +237,6 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
         else
           decorate(event)
 
-          event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
-          event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
-
           if @include_object_properties
             event.set("[@metadata][s3]", object.data.to_h)
           else
@@ -237,6 +244,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
           end
 
           event.set("[@metadata][s3][key]", object.key)
+          event.set(@cloudfront_version_key, metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
+          event.set(@cloudfront_fields_key, metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
 
           queue << event
         end
@@ -345,14 +354,22 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   end
 
   def symbolized_settings
-    @symbolized_settings ||= symbolize(@additional_settings)
+    @symbolized_settings ||= symbolize_keys_and_cast_true_false(@additional_settings)
   end
 
-  def symbolize(hash)
-    return hash unless hash.is_a?(Hash)
-    symbolized = {}
-    hash.each { |key, value| symbolized[key.to_sym] = symbolize(value) }
-    symbolized
+  def symbolize_keys_and_cast_true_false(hash)
+    case hash
+    when Hash
+      symbolized = {}
+      hash.each { |key, value| symbolized[key.to_sym] = symbolize_keys_and_cast_true_false(value) }
+      symbolized
+    when 'true'
+      true
+    when 'false'
+      false
+    else
+      hash
+    end
   end
 
   def ignore_filename?(filename)
@@ -448,10 +465,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
         @sincedb_path = file
       end
 
-      def newer?(date)
-        date > read
-      end
-
+      # @return [Time]
       def read
         if ::File.exists?(@sincedb_path)
           content = ::File.read(@sincedb_path).chomp.strip
@@ -463,7 +477,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
       end
 
       def write(since = nil)
-        since = Time.now() if since.nil?
+        since = Time.now if since.nil?
         ::File.open(@sincedb_path, 'w') { |file| file.write(since.to_s) }
       end
     end

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.6.0'
+  s.version         = '3.8.2'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files in a S3 bucket"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -27,4 +27,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency "logstash-codec-json"
   s.add_development_dependency "logstash-codec-multiline"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.2'
 end

data/spec/inputs/s3_spec.rb

@@ -9,6 +9,7 @@ require_relative "../support/helpers"
 require "stud/temporary"
 require "aws-sdk"
 require "fileutils"
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
 describe LogStash::Inputs::S3 do
   let(:temporary_directory) { Stud::Temporary.pathname }
@@ -41,7 +42,9 @@ describe LogStash::Inputs::S3 do
       expect_any_instance_of(LogStash::Inputs::S3).to receive(:list_new_files).and_return(TestInfiniteS3Object.new(s3_obj))
     end
 
-    it_behaves_like "an interruptible input plugin"
+    it_behaves_like "an interruptible input plugin" do
+      let(:allowed_lag) { 16 } if LOGSTASH_VERSION.split('.').first.to_i <= 6
+    end
   end
 
   describe "#register" do
@@ -81,10 +84,10 @@ describe LogStash::Inputs::S3 do
     end
 
     describe "additional_settings" do
-      context 'when force_path_style is set' do
+      context "supported settings" do
         let(:settings) {
           {
-            "additional_settings" => { "force_path_style" => true },
+            "additional_settings" => { "force_path_style" => 'true', "ssl_verify_peer" => 'false', "profile" => 'logstash' },
             "bucket" => "logstash-test",
           }
         }
@@ -92,7 +95,7 @@ describe LogStash::Inputs::S3 do
         it 'should instantiate AWS::S3 clients with force_path_style set' do
           expect(Aws::S3::Resource).to receive(:new).with({
             :region => subject.region,
-            :force_path_style => true
+            :force_path_style => true, :ssl_verify_peer => false, :profile => 'logstash'
           }).and_call_original
 
           subject.send(:get_s3object)
@@ -192,6 +195,7 @@ describe LogStash::Inputs::S3 do
       it 'should log that no files were found in the bucket' do
         plugin = LogStash::Inputs::S3.new(config)
         plugin.register
+        allow(plugin.logger).to receive(:info).with(/Using the provided sincedb_path/, anything)
         expect(plugin.logger).to receive(:info).with(/No files found/, anything)
         expect(plugin.list_new_files).to be_empty
       end
@@ -494,12 +498,20 @@ describe LogStash::Inputs::S3 do
     context 'cloudfront' do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'cloudfront.log') }
 
-      it 'should extract metadata from cloudfront log' do
-        events = fetch_events(config)
+      describe "metadata", :ecs_compatibility_support, :aggregate_failures do
+        ecs_compatibility_matrix(:disabled, :v1) do |ecs_select|
+          before(:each) do
+            allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+          end
 
-        events.each do |event|
-          expect(event.get('cloudfront_fields')).to eq('date time x-edge-location c-ip x-event sc-bytes x-cf-status x-cf-client-id cs-uri-stem cs-uri-query c-referrer x-page-url​  c-user-agent x-sname x-sname-query x-file-ext x-sid')
-          expect(event.get('cloudfront_version')).to eq('1.0')
+          it 'should extract metadata from cloudfront log' do
+            events = fetch_events(config)
+
+            events.each do |event|
+              expect(event.get ecs_select[disabled: "cloudfront_fields", v1: "[@metadata][s3][cloudfront][fields]"] ).to eq('date time x-edge-location c-ip x-event sc-bytes x-cf-status x-cf-client-id cs-uri-stem cs-uri-query c-referrer x-page-url​  c-user-agent x-sname x-sname-query x-file-ext x-sid')
+              expect(event.get ecs_select[disabled: "cloudfront_version", v1: "[@metadata][s3][cloudfront][version]"] ).to eq('1.0')
+            end
+          end
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.6.0
+  version: 3.8.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-11 00:00:00.000000000 Z
+date: 2021-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -100,6 +100,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program
@@ -153,8 +167,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Streams events from files in a S3 bucket