logstash-input-s3 3.6.0 → 3.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4e1cc2ba334eb9e35fc68cc6a773e13b9b7de5bc6c3b4ee40cf98903d140323
-  data.tar.gz: a43fe645c1016095639e092fa4d3e8e8b77384dd17190a712a0eaa5163e68854
+  metadata.gz: 0b5ad2fe42e0003da67ebb3e1aad774f5bab3cb2afc07e4699fbfa5e84c0d472
+  data.tar.gz: fb8691a6276181e17a2c80319247d04f11c8d547affff1b109f5447547eca4be
 SHA512:
-  metadata.gz: e10a6e21aa62270ce5f78c269fb3abf0a28b0755f78b0cb7adcc76183be460472327bdabbc94fab51c35774ae930d25846084ad3b9b0e95bd9e021c995c08bbd
-  data.tar.gz: 23b6882049688b1cd57ccba4b8aa247103bf5b9f91011f1aa426e81280c3b4ca701bf88fd96e278968c7e38fd58a01980a830b4ce2bb9ad8e034cfe07113de79
+  metadata.gz: 3782d59c63fe4e53cb01dff9c7a431773fc69126dfc17c90ea45b8018205897dfabd8d942a732c37cb1740a4c924608110e81955205734523a9f5d1a928bed5d
+  data.tar.gz: 3ff5255afa6d8a978a2231bc69a3c11dd9f354a90895e779be569971aaa75c34c456d35b979effbb0304801160cb15ce5f81b5967e121133d79e0a48f59638b7

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 3.7.0
+ - Add ECS support. [#228](https://github.com/logstash-plugins/logstash-input-s3/pull/228)
+ - Fix missing file in cutoff time change. [#224](https://github.com/logstash-plugins/logstash-input-s3/pull/224)
+
 ## 3.6.0
  - Fixed unprocessed file with the same `last_modified` in ingestion. [#220](https://github.com/logstash-plugins/logstash-input-s3/pull/220)
 

data/docs/index.asciidoc

@@ -31,6 +31,21 @@ Files ending in `.gz` are handled as gzip'ed files.
 
 Files that are archived to AWS Glacier will be skipped.
 
+[id="plugins-{type}s-{plugin}-ecs_metadata"]
+==== Event Metadata and the Elastic Common Schema (ECS)
+This plugin adds cloudfront metadata to event.
+When ECS compatibility is disabled, the value is stored in the root level.
+When ECS is enabled, the value is stored in the `@metadata` where it can be used by other plugins in your pipeline.
+
+Here’s how ECS compatibility mode affects output.
+[cols="<l,<l,e,<e"]
+|=======================================================================
+| ECS disabled | ECS v1 | Availability | Description
+
+| cloudfront_fields | [@metadata][s3][cloudfront][fields] | available when the file is a CloudFront log | column names of log
+| cloudfront_version | [@metadata][s3][cloudfront][version] | available when the file is a CloudFront log | version of log
+|=======================================================================
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== S3 Input Configuration Options
 
@@ -47,6 +62,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-backup_to_dir>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-bucket>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-delete>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-endpoint>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-exclude_pattern>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-gzip_pattern>> |<<string,string>>|No
@@ -96,12 +112,12 @@ the connection to s3. See full list in https://docs.aws.amazon.com/sdkforruby/ap
 [source,ruby]
     input {
       s3 {
-        "access_key_id" => "1234"
-        "secret_access_key" => "secret"
-        "bucket" => "logstash-test"
-        "additional_settings" => {
-          "force_path_style" => true
-          "follow_redirects" => false
+        access_key_id => "1234"
+        secret_access_key => "secret"
+        bucket => "logstash-test"
+        additional_settings => {
+          force_path_style => true
+          follow_redirects => false
         }
       }
     }
@@ -167,6 +183,18 @@ The name of the S3 bucket.
 
 Whether to delete processed files from the original bucket.
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+  ** `disabled`: does not use ECS-compatible field names
+  ** `v1`: uses metadata fields that are compatible with Elastic Common Schema
+
+Controls this plugin's compatibility with the
+{ecs-ref}[Elastic Common Schema (ECS)].
+See <<plugins-{type}s-{plugin}-ecs_metadata>> for detailed information.
+
 [id="plugins-{type}s-{plugin}-endpoint"]
 ===== `endpoint`
 

data/lib/logstash/inputs/s3.rb

@@ -9,6 +9,7 @@ require "stud/interval"
 require "stud/temporary"
 require "aws-sdk"
 require "logstash/inputs/s3/patch"
+require "logstash/plugin_mixins/ecs_compatibility_support"
 
 require 'java'
 
@@ -27,6 +28,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   java_import java.util.zip.ZipException
 
   include LogStash::PluginMixins::AwsConfig::V2
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1)
 
   config_name "s3"
 
@@ -88,6 +90,12 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
   CUTOFF_SECOND = 3
 
+  def initialize(*params)
+    super
+    @cloudfront_fields_key = ecs_select[disabled: 'cloudfront_fields', v1: '[@metadata][s3][cloudfront][fields]']
+    @cloudfront_version_key = ecs_select[disabled: 'cloudfront_version', v1: '[@metadata][s3][cloudfront][version]']
+  end
+
   def register
     require "fileutils"
     require "digest/md5"
@@ -130,6 +138,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   def list_new_files
     objects = []
     found = false
+    current_time = Time.now
     begin
       @s3bucket.objects(:prefix => @prefix).each do |log|
         found = true
@@ -140,7 +149,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
           @logger.debug('Object Zero Length', :key => log.key)
         elsif !sincedb.newer?(log.last_modified)
           @logger.debug('Object Not Modified', :key => log.key)
-        elsif log.last_modified > (Time.now - CUTOFF_SECOND).utc # file modified within last two seconds will be processed in next cycle
+        elsif log.last_modified > (current_time - CUTOFF_SECOND).utc # file modified within last two seconds will be processed in next cycle
           @logger.debug('Object Modified After Cutoff Time', :key => log.key)
         elsif (log.storage_class == 'GLACIER' || log.storage_class == 'DEEP_ARCHIVE') && !file_restored?(log.object)
           @logger.debug('Object Archived to Glacier', :key => log.key)
@@ -227,9 +236,6 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
         else
           decorate(event)
 
-          event.set("cloudfront_version", metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
-          event.set("cloudfront_fields", metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
-
           if @include_object_properties
             event.set("[@metadata][s3]", object.data.to_h)
           else
@@ -237,6 +243,8 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
           end
 
           event.set("[@metadata][s3][key]", object.key)
+          event.set(@cloudfront_version_key, metadata[:cloudfront_version]) unless metadata[:cloudfront_version].nil?
+          event.set(@cloudfront_fields_key, metadata[:cloudfront_fields]) unless metadata[:cloudfront_fields].nil?
 
           queue << event
         end

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.6.0'
+  s.version         = '3.7.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files in a S3 bucket"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -27,4 +27,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency "logstash-codec-json"
   s.add_development_dependency "logstash-codec-multiline"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1'
 end

data/spec/inputs/s3_spec.rb

@@ -9,6 +9,7 @@ require_relative "../support/helpers"
 require "stud/temporary"
 require "aws-sdk"
 require "fileutils"
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
 describe LogStash::Inputs::S3 do
   let(:temporary_directory) { Stud::Temporary.pathname }
@@ -494,12 +495,20 @@ describe LogStash::Inputs::S3 do
     context 'cloudfront' do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'cloudfront.log') }
 
-      it 'should extract metadata from cloudfront log' do
-        events = fetch_events(config)
+      describe "metadata", :ecs_compatibility_support, :aggregate_failures do
+        ecs_compatibility_matrix(:disabled, :v1) do |ecs_select|
+          before(:each) do
+            allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+          end
 
-        events.each do |event|
-          expect(event.get('cloudfront_fields')).to eq('date time x-edge-location c-ip x-event sc-bytes x-cf-status x-cf-client-id cs-uri-stem cs-uri-query c-referrer x-page-url​  c-user-agent x-sname x-sname-query x-file-ext x-sid')
-          expect(event.get('cloudfront_version')).to eq('1.0')
+          it 'should extract metadata from cloudfront log' do
+            events = fetch_events(config)
+
+            events.each do |event|
+              expect(event.get ecs_select[disabled: "cloudfront_fields", v1: "[@metadata][s3][cloudfront][fields]"] ).to eq('date time x-edge-location c-ip x-event sc-bytes x-cf-status x-cf-client-id cs-uri-stem cs-uri-query c-referrer x-page-url​  c-user-agent x-sname x-sname-query x-file-ext x-sid')
+              expect(event.get ecs_select[disabled: "cloudfront_version", v1: "[@metadata][s3][cloudfront][version]"] ).to eq('1.0')
+            end
+          end
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.6.0
+  version: 3.7.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-11 00:00:00.000000000 Z
+date: 2021-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -100,6 +100,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program