logstash-input-s3-sns-sqs 1.1.6 → 1.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 75d610de1c48fd3e5cc00b1c867cc21eaea6e3c1
-  data.tar.gz: 0fdf5bb05918562318431308ea15ace790b0108c
+  metadata.gz: cbba025cb4e274be8fd5af60c33fdc445486831c
+  data.tar.gz: 1aea14626b1290ca3837fea2cd8b35dee986b841
 SHA512:
-  metadata.gz: 2d6ffeca392ea7f5ac597942df3faabb85b9fd41ec0c5c5341e2ae115c13ae7345315fd31ca2bd7491707042e3be931fbeca2365827d2c9e017e04f512d0ffae
-  data.tar.gz: d5fc3995e1ac8bcf43e251e63ae68619e8b8ade5b35713449a2d296ad3be73677a6828991e848cb0603bb12b99544e8245a14a4c11835b60e174f874eb7fa490
+  metadata.gz: a7ad292c17c49d1b9a0fa3c8ab9f454a026b38defdd73007dde13f6349f358a927da92bba46f8c082d269a1464431ba8d114d844c44a4383c8b43b1940026c0a
+  data.tar.gz: 5c969909a581f9aeb3d1704310a0556a27ed2936a73e3c0a5b2a9590801a63bbdf6fd16d5797ff79af3f87855b541614fc155acf3ae46da58ef19991073e6e5a

data/CHANGELOG.md

@@ -1,3 +1,5 @@
+## 1.1.6
+- Fix a nil exception in message parsing
 ## 1.1.5
 - Fix loglevel for some debug messages
 ## 1.1.4

data/lib/logstash/inputs/s3snssqs.rb

@@ -131,74 +131,76 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     @logger.debug("handle_message", :hash => hash, :message => message)
     # there may be test events sent from the s3 bucket which won't contain a Records array,
     # we will skip those events and remove them from queue
-    message = JSON.parse(hash['Message'])
-    if message['Records'] then
-	    # typically there will be only 1 record per event, but since it is an array we will
-      # treat it as if there could be more records
-      message['Records'].each do |record|
-        @logger.debug("We found a record", :record => record)
-	# in case there are any events with Records that aren't s3 object-created events and can't therefore be
-        # processed by this plugin, we will skip them and remove them from queue
-        if record['eventSource'] == EVENT_SOURCE and record['eventName'].start_with?(EVENT_TYPE) then
-          @logger.debug("It is a valid record")
-	  bucket = CGI.unescape(record['s3']['bucket']['name'])
-	  key    = CGI.unescape(record['s3']['object']['key'])
-
-          # try download and :skip_delete if it fails
-          begin
-            response = @s3.get_object(
-              bucket: bucket,
-              key: key,
-            )
-          rescue => e
-            @logger.warn("issuing :skip_delete on failed download", :bucket => bucket, :object => key, :error => e)
-            throw :skip_delete
-          end
+    if hash['Message'].present?
+      message = JSON.parse(hash['Message'])
+      if message['Records'] then
+	      # typically there will be only 1 record per event, but since it is an array we will
+        # treat it as if there could be more records
+        message['Records'].each do |record|
+          @logger.debug("We found a record", :record => record)
+	        # in case there are any events with Records that aren't s3 object-created events and can't therefore be
+          # processed by this plugin, we will skip them and remove them from queue
+          if record['eventSource'] == EVENT_SOURCE and record['eventName'].start_with?(EVENT_TYPE) then
+            @logger.debug("It is a valid record")
+	          bucket = CGI.unescape(record['s3']['bucket']['name'])
+	          key    = CGI.unescape(record['s3']['object']['key'])
+
+            # try download and :skip_delete if it fails
+            begin
+              response = @s3.get_object(
+                bucket: bucket,
+                key: key,
+              )
+            rescue => e
+              @logger.warn("issuing :skip_delete on failed download", :bucket => bucket, :object => key, :error => e)
+              throw :skip_delete
+            end
 
-          # verify downloaded content size
-          if response.content_length == record['s3']['object']['size'] then
-            body = response.body
-            # if necessary unzip
-            if response.content_encoding == "gzip" or record['s3']['object']['key'].end_with?(".gz") then
-              @logger.debug("Ohhh i´ll try to unzip")
-	      begin
-		            temp = Zlib::GzipReader.new(body)
-              rescue => e
-                @logger.warn("content is marked to be gzipped but can't unzip it, assuming plain text", :bucket => bucket, :object => key, :error => e)
-                temp = body
+            # verify downloaded content size
+            if response.content_length == record['s3']['object']['size'] then
+              body = response.body
+              # if necessary unzip
+              if response.content_encoding == "gzip" or record['s3']['object']['key'].end_with?(".gz") then
+                @logger.debug("Ohhh i´ll try to unzip")
+	              begin
+		              temp = Zlib::GzipReader.new(body)
+                  rescue => e
+                  @logger.warn("content is marked to be gzipped but can't unzip it, assuming plain text", :bucket => bucket, :object => key, :error => e)
+                  temp = body
+                end
+                body = temp
               end
-              body = temp
-            end
-            # process the plain text content
-            begin
-              lines = body.read.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370").split(/\n/)
-              lines.each do |line|
-                @logger.debug("Decorating the event")
-		  @codec.decode(line) do |event|
+              # process the plain text content
+              begin
+                lines = body.read.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370").split(/\n/)
+                lines.each do |line|
+                  @logger.debug("Decorating the event")
+		              @codec.decode(line) do |event|
                   decorate(event)
 
                   event.set('[@metadata][s3_bucket_name]', record['s3']['bucket']['name'])
                   event.set('[@metadata][s3_object_key]', record['s3']['object']['key'])
 
                   queue << event
+                  end
                 end
+              rescue => e
+                @logger.warn("issuing :skip_delete on failed plain text processing", :bucket => bucket, :object => key, :error => e)
+                throw :skip_delete
               end
-            rescue => e
-              @logger.warn("issuing :skip_delete on failed plain text processing", :bucket => bucket, :object => key, :error => e)
-              throw :skip_delete
-            end
 
-            # Delete the files from S3
-            begin
-              @s3.delete_object(bucket: bucket, key: key) if @delete_on_success
-            rescue => e
-              @logger.warn("Failed to delete S3 object", :bucket => bucket, :object => key, :error => e)
+              # Delete the files from S3
+              begin
+                @s3.delete_object(bucket: bucket, key: key) if @delete_on_success
+              rescue => e
+                @logger.warn("Failed to delete S3 object", :bucket => bucket, :object => key, :error => e)
+              end
+            # otherwise try again later
+            else
+              @logger.warn("issuing :skip_delete on wrong download content size", :bucket => bucket, :object => key,
+                :download_size => response.content_length, :expected => record['s3']['object']['size'])
+              throw :skip_delete
             end
-          # otherwise try again later
-          else
-            @logger.warn("issuing :skip_delete on wrong download content size", :bucket => bucket, :object => key,
-              :download_size => response.content_length, :expected => record['s3']['object']['size'])
-            throw :skip_delete
           end
         end
       end

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.1.6'
+  s.version         = '1.1.7'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.1.6
+  version: 1.1.7
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-19 00:00:00.000000000 Z
+date: 2018-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement