logstash-input-s3 3.3.7 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b6b2e69a2cc95f3fc7bdcc0a5b828e08a3795d88f5468795fafc4a518b7e4128
-  data.tar.gz: 5a1d0103482e624fe8131eed9c28e636c71bb38b70980e463e1548bcb7efaff1
+  metadata.gz: e291d3321f85c506ace8f9e2d4d7e72bd7455ed4d7caf3319e021f5b9295ea7b
+  data.tar.gz: 2907311101de6a87b984864deecb9ac33b490f4949375f3f3d3482939b2a7ae6
 SHA512:
-  metadata.gz: 7cb50c30b2acdd5f8da2602346b0fc9b3ef82f4dd5d1649e7cff8e7d746c28ef0e997cb8db8c1821639e3e8b8001ef19e2c8a0557d5b8a6f37ea9b5a27e451f8
-  data.tar.gz: 41be7059efac9cd05b2379a5035cc11104bd7c9716939ac9a2c63aa94ee7bbf79832c7219492ce5576987d36bae038c530af64c11e021c89dee47aa53ea4bf51
+  metadata.gz: 48bf1a17935b31018118eb00c84bae6e977eb4aca39f57cce09c769b054f4486874b5762a90cc778c235fcda7b731625fcbcf70b571d7534ac0c8550a78c9ffc
+  data.tar.gz: 4841a05b01e106b1ff5f3012ae443c091d6cea3a6036b9990729f4aaa95e05272477a6ddf7213ca246803146b34b83f5ff45dedd5382bff7b3f42a6ca59355df

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 3.4.0
+ - Skips objects that are archived to AWS Glacier with a helpful log message (previously they would log as matched, but then fail to load events) [#160](https://github.com/logstash-plugins/logstash-input-s3/pull/160)
+ - Added `watch_for_new_files` option, enabling single-batch imports [#159](https://github.com/logstash-plugins/logstash-input-s3/pull/159)
+
 ## 3.3.7
   - Added ability to optionally include S3 object properties inside @metadata [#155](https://github.com/logstash-plugins/logstash-input-s3/pull/155)
 

data/docs/index.asciidoc

@@ -26,6 +26,8 @@ Stream events from files from a S3 bucket.
 Each line from each file generates an event.
 Files ending in `.gz` are handled as gzip'ed files.
 
+Files that are archived to AWS Glacier will be skipped.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== S3 Input Configuration Options
 
@@ -55,6 +57,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-session_token>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-sincedb_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-temporary_directory>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-watch_for_new_files>> | <boolean,boolean>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -273,7 +276,14 @@ If specified, this setting must be a filename path and not just a directory.
 
 Set the directory where logstash will store the tmp files before processing them.
 
+[id="plugins-{type}s-{plugin}-watch_for_new_files"]
+===== `watch_for_new_files`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
 
+Whether or not to watch for new files.
+Disabling this option causes the input to close itself after processing the files from a single listing.
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/s3.rb

@@ -63,6 +63,10 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   # Value is in seconds.
   config :interval, :validate => :number, :default => 60
 
+  # Whether to watch for new files with the interval. 
+  # If false, overrides any interval and only lists the s3 bucket once.
+  config :watch_for_new_files, :validate => :boolean, :default => true
+
   # Ruby style regexp of keys to exclude from the bucket
   config :exclude_pattern, :validate => :string, :default => nil
 
@@ -101,6 +105,10 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     end
 
     FileUtils.mkdir_p(@temporary_directory) unless Dir.exist?(@temporary_directory)
+
+    if !@watch_for_new_files && original_params.include?('interval')
+      logger.warn("`watch_for_new_files` has been disabled; `interval` directive will be ignored.")
+    end
   end
 
   public
@@ -108,6 +116,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     @current_thread = Thread.current
     Stud.interval(@interval) do
       process_files(queue)
+      stop unless @watch_for_new_files
     end
   end # def run
 
@@ -119,14 +128,18 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
       @s3bucket.objects(:prefix => @prefix).each do |log|
         found = true
         @logger.debug("S3 input: Found key", :key => log.key)
-        if !ignore_filename?(log.key)
-          if sincedb.newer?(log.last_modified) && log.content_length > 0
-            objects[log.key] = log.last_modified
-            @logger.debug("S3 input: Adding to objects[]", :key => log.key)
-            @logger.debug("objects[] length is: ", :length => objects.length)
-          end
-        else
+        if ignore_filename?(log.key)
           @logger.debug('S3 input: Ignoring', :key => log.key)
+        elsif log.content_length <= 0
+          @logger.debug('S3 Input: Object Zero Length', :key => log.key)
+        elsif !sincedb.newer?(log.last_modified)
+          @logger.debug('S3 Input: Object Not Modified', :key => log.key)
+        elsif log.storage_class.start_with?('GLACIER')
+          @logger.debug('S3 Input: Object Archived to Glacier', :key => log.key)
+        else
+          objects[log.key] = log.last_modified
+          @logger.debug("S3 input: Adding to objects[]", :key => log.key)
+          @logger.debug("objects[] length is: ", :length => objects.length)
         end
       end
       @logger.info('S3 input: No files found in bucket', :prefix => prefix) unless found

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.3.7'
+  s.version         = '3.4.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Streams events from files in a S3 bucket"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/s3_spec.rb

@@ -114,11 +114,13 @@ describe LogStash::Inputs::S3 do
   describe "#list_new_files" do
     before { allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects_list } }
 
-    let!(:present_object) { double(:key => 'this-should-be-present', :last_modified => Time.now, :content_length => 10) }
+    let!(:present_object) { double(:key => 'this-should-be-present', :last_modified => Time.now, :content_length => 10, :storage_class => 'STANDARD') }
+    let!(:archived_object) {double(:key => 'this-should-be-archived', :last_modified => Time.now, :content_length => 10, :storage_class => 'GLACIER') }
     let(:objects_list) {
       [
-        double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day, :content_length => 100),
-        double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day, :content_length => 50),
+        double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day, :content_length => 100, :storage_class => 'STANDARD'),
+        double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day, :content_length => 50, :storage_class => 'STANDARD'),
+        archived_object,
         present_object
       ]
     }
@@ -126,20 +128,32 @@ describe LogStash::Inputs::S3 do
     it 'should allow user to exclude files from the s3 bucket' do
       plugin = LogStash::Inputs::S3.new(config.merge({ "exclude_pattern" => "^exclude" }))
       plugin.register
-      expect(plugin.list_new_files).to eq([present_object.key])
+
+      files = plugin.list_new_files
+      expect(files).to include(present_object.key)
+      expect(files).to_not include('exclude-this-file-1') # matches exclude pattern
+      expect(files).to_not include('exclude/logstash')    # matches exclude pattern
+      expect(files).to_not include(archived_object.key)   # archived
+      expect(files.size).to eq(1)
     end
 
     it 'should support not providing a exclude pattern' do
       plugin = LogStash::Inputs::S3.new(config)
       plugin.register
-      expect(plugin.list_new_files).to eq(objects_list.map(&:key))
+
+      files = plugin.list_new_files
+      expect(files).to include(present_object.key)
+      expect(files).to include('exclude-this-file-1')   # no exclude pattern given
+      expect(files).to include('exclude/logstash')      # no exclude pattern given
+      expect(files).to_not include(archived_object.key) # archived
+      expect(files.size).to eq(3)
     end
 
     context 'when all files are excluded from a bucket' do
       let(:objects_list) {
         [
-            double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day, :content_length => 100),
-            double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day, :content_length => 50),
+            double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day, :content_length => 100, :storage_class => 'STANDARD'),
+            double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day, :content_length => 50, :storage_class => 'STANDARD'),
         ]
       }
 
@@ -168,7 +182,7 @@ describe LogStash::Inputs::S3 do
     context "If the bucket is the same as the backup bucket" do
       it 'should ignore files from the bucket if they match the backup prefix' do
         objects_list = [
-          double(:key => 'mybackup-log-1', :last_modified => Time.now, :content_length => 5),
+          double(:key => 'mybackup-log-1', :last_modified => Time.now, :content_length => 5, :storage_class => 'STANDARD'),
           present_object
         ]
 
@@ -177,24 +191,34 @@ describe LogStash::Inputs::S3 do
         plugin = LogStash::Inputs::S3.new(config.merge({ 'backup_add_prefix' => 'mybackup',
                                                          'backup_to_bucket' => config['bucket']}))
         plugin.register
-        expect(plugin.list_new_files).to eq([present_object.key])
+
+        files = plugin.list_new_files
+        expect(files).to include(present_object.key)
+        expect(files).to_not include('mybackup-log-1') # matches backup prefix
+        expect(files.size).to eq(1)
       end
     end
 
     it 'should ignore files older than X' do
       plugin = LogStash::Inputs::S3.new(config.merge({ 'backup_add_prefix' => 'exclude-this-file'}))
 
-      expect_any_instance_of(LogStash::Inputs::S3::SinceDB::File).to receive(:read).exactly(objects_list.size) { Time.now - day }
+
+      allow_any_instance_of(LogStash::Inputs::S3::SinceDB::File).to receive(:read).and_return(Time.now - day)
       plugin.register
 
-      expect(plugin.list_new_files).to eq([present_object.key])
+      files = plugin.list_new_files
+      expect(files).to include(present_object.key)
+      expect(files).to_not include('exclude-this-file-1') # too old
+      expect(files).to_not include('exclude/logstash')    # too old
+      expect(files).to_not include(archived_object.key)   # archived
+      expect(files.size).to eq(1)
     end
 
     it 'should ignore file if the file match the prefix' do
         prefix = 'mysource/'
 
         objects_list = [
-          double(:key => prefix, :last_modified => Time.now, :content_length => 5),
+          double(:key => prefix, :last_modified => Time.now, :content_length => 5, :storage_class => 'STANDARD'),
           present_object
         ]
 
@@ -207,9 +231,9 @@ describe LogStash::Inputs::S3 do
 
     it 'should sort return object sorted by last_modification date with older first' do
       objects = [
-        double(:key => 'YESTERDAY', :last_modified => Time.now - day, :content_length => 5),
-        double(:key => 'TODAY', :last_modified => Time.now, :content_length => 5),
-        double(:key => 'TWO_DAYS_AGO', :last_modified => Time.now - 2 * day, :content_length => 5)
+        double(:key => 'YESTERDAY', :last_modified => Time.now - day, :content_length => 5, :storage_class => 'STANDARD'),
+        double(:key => 'TODAY', :last_modified => Time.now, :content_length => 5, :storage_class => 'STANDARD'),
+        double(:key => 'TWO_DAYS_AGO', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD')
       ]
 
       allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects }
@@ -315,7 +339,7 @@ describe LogStash::Inputs::S3 do
     %w(AccessDenied NoSuchKey).each do |error|
       context "when retrieving an object, #{error} is returned" do
         let(:objects) { [log] }
-        let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5) }
+        let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD') }
 
         let(:config) {
           {
@@ -344,7 +368,7 @@ describe LogStash::Inputs::S3 do
 
   context 'when working with logs' do
     let(:objects) { [log] }
-    let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5, :data => { "etag" => 'c2c966251da0bc3229d12c2642ba50a4' }) }
+    let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5, :data => { "etag" => 'c2c966251da0bc3229d12c2642ba50a4' }, :storage_class => 'STANDARD') }
     let(:data) { File.read(log_file) }
 
     before do
@@ -389,7 +413,7 @@ describe LogStash::Inputs::S3 do
     end
 
     context "multiple compressed streams" do
-      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5) }
+      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD') }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'multiple_compressed_streams.gz') }
 
       include_examples "generated events" do
@@ -398,14 +422,14 @@ describe LogStash::Inputs::S3 do
     end
       
     context 'compressed' do
-      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5) }
+      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD') }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gz') }
 
       include_examples "generated events"
     end
 
     context 'compressed with gzip extension' do
-      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5) }
+      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5, :storage_class => 'STANDARD') }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gzip') }
 
       include_examples "generated events"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.3.7
+  version: 3.4.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-07-20 00:00:00.000000000 Z
+date: 2018-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement