logstash-input-s3sqs 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dc2abca41f6c8e2391970c664e12e20a41ec3c86
-  data.tar.gz: 31fd27a7048dafe8f5ee6b9ef57aa2674f53336d
+  metadata.gz: 0bc3f016a0e987f5299a013d717f89f92b36cc66
+  data.tar.gz: 41ff3344f0114535e7bf699ba99f2770889e9e4d
 SHA512:
-  metadata.gz: e174ca0a264e7788135d224157a5bd06ef886dae4640472d12707417352c2afc002134ffe03e7d481e9b4b579a138c5f7d295ea993b6644fe5a9f7a7f6388563
-  data.tar.gz: 69c44afedd3714bafb341ce72b3c188e6e6d5682ce91bd19cb1079fcb46e66a71cdfad9c7a72b33899069fa80ec57c094a779e3a3e7fad8de8880fd64b48e4a4
+  metadata.gz: cd6ceebadbfe378d7e83c55c1d8912ee5dbf294e9ea1dce5f16be8455797a6aed5d35e1cf78ea8cc2371ce16d3392e5ee2bdacdb05db0600878fe87204ee2acf
+  data.tar.gz: 97ca5b6b89bca7631e74b1377c9cf52526c3bff12aaddd827eda3cf773257dcbccaade6f60fabc394d88c9265bf0a63a7b1ad6da0540cbed5d172086460da9b6

data/CHANGELOG.md

@@ -1,6 +1,16 @@
 # 1.0.0
+
 - Initial Release
+
 # 1.0.1
+
 - same (because of screwed up rubygems.org release)
+
 # 1.0.2
+
 - fix for broken UTF-8 (so we won't lose a whole s3 log file because of a single invalid line, ruby's split will die on those)
+
+# 1.0.3
+
+- added some metadata to the event (bucket and object name as commited by joshuaspence)
+- also try to unzip files ending with ".gz" (ALB logs are zipped but not marked with proper Content-Encoding)

data/CONTRIBUTORS

@@ -2,7 +2,8 @@ The following is a list of people who have contributed ideas, code, bug
 reports, or in general have helped logstash along its way.
 
 Contributors:
-* Heiko Finzel (Heiko-san)
+* joshuaspence (event metadata)
+* Heiko-san (initial contributor)
 * logstash-input-sqs plugin as code base
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to

data/lib/logstash/inputs/s3sqs.rb

@@ -26,7 +26,7 @@ require "logstash/errors"
 # This plugin is meant for high availability setups, in contrast to logstash-input-s3 you can safely
 # use multiple logstash nodes, since the usage of sqs will ensure that each logfile is processed
 # only once and no file will get lost on node failure or downscaling for auto-scaling groups.
-# (You should use a "Message Retention Period" >= 4 days for your sqs to ensure you can survive 
+# (You should use a "Message Retention Period" >= 4 days for your sqs to ensure you can survive
 # a weekend of faulty log file processing)
 # The plugin will not delete objects from s3 buckets, so make sure to have a reasonable "Lifecycle"
 # configured for your buckets, which should keep the files at least "Message Retention Period" days.
@@ -37,7 +37,7 @@ require "logstash/errors"
 # (The plugin supports gzipped content if it is marked with "contend-encoding: gzip" as it is the
 # case for cloudtrail logs)
 #
-# The logstash node therefore must have sqs permissions + the permissions to download objects 
+# The logstash node therefore must have sqs permissions + the permissions to download objects
 # from the s3 buckets that send events to the queue.
 # (If logstash nodes are running on EC2 you should use a ServerRole to provide permissions)
 # [source,json]
@@ -107,20 +107,20 @@ class LogStash::Inputs::S3SQS < LogStash::Inputs::Threadable
   end
 
   def polling_options
-    { 
+    {
       # we will query 1 message at a time, so we can ensure correct error handling if we can't download a single file correctly
       # (we will throw :skip_delete if download size isn't correct to process the event again later
       # -> set a reasonable "Default Visibility Timeout" for your queue, so that there's enough time to process the log files)
-      :max_number_of_messages => 1, 
+      :max_number_of_messages => 1,
       # we will use the queue's setting, a good value is 10 seconds
       # (to ensure fast logstash shutdown on the one hand and few api calls on the other hand)
-      :wait_time_seconds => nil, 
+      :wait_time_seconds => nil,
     }
   end
 
   def handle_message(message, queue)
     hash = JSON.parse message.body
-    # there may be test events sent from the s3 bucket which won't contain a Records array, 
+    # there may be test events sent from the s3 bucket which won't contain a Records array,
     # we will skip those events and remove them from queue
     if hash['Records'] then
       # typically there will be only 1 record per event, but since it is an array we will
@@ -143,7 +143,7 @@ class LogStash::Inputs::S3SQS < LogStash::Inputs::Threadable
           if response.content_length == record['s3']['object']['size'] then
             body = response.body
             # if necessary unzip
-            if response.content_encoding == "gzip" then
+            if response.content_encoding == "gzip" or record['s3']['object']['key'].end_with?(".gz") then
               begin
 		            temp = Zlib::GzipReader.new(body)
               rescue => e
@@ -158,6 +158,10 @@ class LogStash::Inputs::S3SQS < LogStash::Inputs::Threadable
               lines.each do |line|
                 @codec.decode(line) do |event|
                   decorate(event)
+
+                  event['[@metadata][s3_bucket_name]'] = record['s3']['bucket']['name']
+                  event['[@metadata][s3_object_key]']  = record['s3']['object']['key']
+
                   queue << event
                 end
               end
@@ -208,7 +212,7 @@ class LogStash::Inputs::S3SQS < LogStash::Inputs::Threadable
     rescue Aws::SQS::Errors::ServiceError => e
       @logger.warn("Aws::SQS::Errors::ServiceError ... retrying SQS request with exponential backoff", :queue => @queue, :sleep_time => sleep_time, :error => e)
       sleep(next_sleep)
-      next_sleep =  next_sleep > max_time ? sleep_time : sleep_time * BACKOFF_FACTOR 
+      next_sleep =  next_sleep > max_time ? sleep_time : sleep_time * BACKOFF_FACTOR
       retry
     end
   end

data/logstash-input-s3sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3sqs'
-  s.version         = '1.0.2'
+  s.version         = '1.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3sqs
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Heiko Finzel
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-03 00:00:00.000000000 Z
+date: 2016-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement