logstash-input-s3 3.3.0 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7b3da5d60d7cc955dc088b8373efb14fa0d4b8318ba689b102726f92b8f947c
-  data.tar.gz: 3ed4bb7fc75bb4ef6f173a90eb9a6ff2788ef517c72c88b814dd70ae72879ab3
+  metadata.gz: 40d44b6d4f759ae776a74c6d2e316ec74f62dba65dc2d75702749913471feda8
+  data.tar.gz: cc518e42d35b2ef3c39045f9700a7f970bccb0e3cf145d7cebb0dd51860bcd13
 SHA512:
-  metadata.gz: b18302b15ea951230da913422c462a8ac70d61e17030c8f8273145d618ad41317b8b4c87f12d986dc979ea02fed8780594511bbe3669c4d8de0982b8f722fc76
-  data.tar.gz: e82da29add08a72dc8ba4ed2d0a32c551d85b57fadb8fbb7e929a5e8028b8f821ec5a358535a89882d33cedd16c308cf9a9137938c1cf11bbe9e176a92e34823
+  metadata.gz: 450923e8e0d886fe972ea82365a7b38eda37cef594f79fa3839615ab64c3fbcdad6fa12d724a80e05c57136126168c16ada1d5587e0e3152f04540846a145bdb
+  data.tar.gz: 70c2526ec6b8acf37d3e82af2a30bed30864e41aae8182cc3f336d68623604c3a1b3380118f4ec6cfecc87e317304fb961d198b9383d70d5f74b5d3c7513e894

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.3.1
+ - Improve error handling when listing/downloading from S3 #144
+
 ## 3.3.0
   - Add documentation for endpoint, role_arn and role_session_name #142
   - Add support for additional_settings option #141

data/lib/logstash/inputs/s3.rb

@@ -110,18 +110,21 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   def list_new_files
     objects = {}
 
-    @s3bucket.objects(:prefix => @prefix).each do |log|
-      @logger.debug("S3 input: Found key", :key => log.key)
-
-      unless ignore_filename?(log.key)
-        if sincedb.newer?(log.last_modified) && log.content_length > 0
-          objects[log.key] = log.last_modified
-          @logger.debug("S3 input: Adding to objects[]", :key => log.key)
-          @logger.debug("objects[] length is: ", :length => objects.length)
+    begin
+      @s3bucket.objects(:prefix => @prefix).each do |log|
+        @logger.debug("S3 input: Found key", :key => log.key)
+        unless ignore_filename?(log.key)
+          if sincedb.newer?(log.last_modified) && log.content_length > 0
+            objects[log.key] = log.last_modified
+            @logger.debug("S3 input: Adding to objects[]", :key => log.key)
+            @logger.debug("objects[] length is: ", :length => objects.length)
+          end
         end
       end
+    rescue Aws::Errors::ServiceError => e
+      @logger.error("S3 input: Unable to list objects in bucket", :prefix => prefix, :message => e.message)
     end
-    return objects.keys.sort {|a,b| objects[a] <=> objects[b]}
+    objects.keys.sort {|a,b| objects[a] <=> objects[b]}
   end # def fetch_new_files
 
   public
@@ -372,11 +375,14 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     @logger.debug("S3 input: Download remote file", :remote_key => remote_object.key, :local_filename => local_filename)
     File.open(local_filename, 'wb') do |s3file|
       return completed if stop?
-      remote_object.get(:response_target => s3file)
+      begin
+        remote_object.get(:response_target => s3file)
+        completed = true
+      rescue Aws::Errors::ServiceError => e
+        @logger.warn("S3 input: Unable to download remote file", :remote_key => remote_object.key, :message => e.message)
+      end
     end
-    completed = true
-
-    return completed
+    completed
   end
 
   private

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.3.0'
+  s.version         = '3.3.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Streams events from files in a S3 bucket"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/s3_spec.rb

@@ -256,6 +256,62 @@ describe LogStash::Inputs::S3 do
     end
   end
 
+  context 'while communicating with s3' do
+    let(:config) {
+      {
+          "access_key_id" => "1234",
+          "secret_access_key" => "secret",
+          "bucket" => "logstash-test",
+          "codec" => "json",
+      }
+    }
+    %w(AccessDenied NotFound).each do |error|
+      context "while listing bucket contents, #{error} is returned" do
+        before do
+          Aws.config[:s3] = {
+              stub_responses: {
+                list_objects: error
+              }
+          }
+        end
+
+        it 'should not crash the plugin' do
+          events = fetch_events(config)
+          expect(events.size).to eq(0)
+        end
+      end
+    end
+
+    %w(AccessDenied NoSuchKey).each do |error|
+      context "when retrieving an object, #{error} is returned" do
+        let(:objects) { [log] }
+        let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5) }
+
+        let(:config) {
+          {
+              "access_key_id" => "1234",
+              "secret_access_key" => "secret",
+              "bucket" => "logstash-test",
+              "codec" => "json",
+          }
+        }
+        before do
+          Aws.config[:s3] = {
+              stub_responses: {
+              get_object: error
+              }
+          }
+          allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects }
+        end
+
+        it 'should not crash the plugin' do
+          events = fetch_events(config)
+          expect(events.size).to eq(0)
+        end
+      end
+    end
+  end
+
   context 'when working with logs' do
     let(:objects) { [log] }
     let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5) }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.3.1
 platform: ruby
 authors:
 - Elastic