logstash-input-sqs_s3 1.1.15 → 1.1.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7810b377cc95a8256dc0b9f331cf0b99cb0567fbc87e82dba5f586e9454d87a8
-  data.tar.gz: 7e8533137aa8aac6d3f47b0ba58835f95dc1a34dc088e498a767bdf7358d49d3
+  metadata.gz: acadffc6b1aa893595029bf29b3e7ff63f24d3b88705436bb33d0f0cfc40791e
+  data.tar.gz: f0dde7708dc206d108d991227c4c98a51d2d2b36f685f51f147c7e222f7d24e2
 SHA512:
-  metadata.gz: 6fb3e428faa96009cbe150a0317c9c584113170a92ad908b37399f665ea2c7797db5bad48948a5c8fec49b21d47acf7fb5fd8b5dc5d0b84906c0f04badb380b1
-  data.tar.gz: 4cc96ead08ac6c7e59c0ab5c8df323dabd99dddfe94be7d899145c91c93f4e8c8d3c48cd5c964bd85bb341185fd99271d83bb65baa5ebf403234704a64c31a04
+  metadata.gz: fc81387bc885f05ae1f4f75c01994d6c715df9468d423dc6e61faa058dc5ac927add038b1eba3db619e5fe57123397dca8067bd910b43d2b591dc981d4892f59
+  data.tar.gz: cecae554a9af3dad5de7652d4ba6ee13bade61443e381f142e0493e1383b7abee2cfb185eaa43ca5d23613db6c87fbd703e62d61e3ad825c6763d918f80491e2

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.1.16
+- Fixed not being able to read multiple gzipped file streams in a single file issue
+
 ## 1.1.15
 - Fixed throttle error
 

data/Gemfile

@@ -1,2 +1,3 @@
 source 'https://rubygems.org'
-gemspec
+gemspec
+gem 'multiple_files_gzip_reader', :git => 'https://github.com/exAspArk/multiple_files_gzip_reader.git'

data/lib/logstash/inputs/sqs_s3.rb

@@ -6,6 +6,7 @@ require "logstash/timestamp"
 require "logstash/plugin_mixins/aws_config"
 require "logstash/errors"
 require "logstash/inputs/sqs_s3/patch"
+require "multiple_files_gzip_reader"
 
 # Forcibly load all modules marked to be lazily loaded.
 #
@@ -178,10 +179,10 @@ class LogStash::Inputs::SQSS3 < LogStash::Inputs::Threadable
           # verify downloaded content size
           if response.content_length == record['s3']['object']['size'] then
             body = response.body
-            # if necessary unzip
-            if response.content_encoding == "gzip" or record['s3']['object']['key'].end_with?(".gz") then
+            # if necessary unzip. Note: Firehose is automatically gzipped but does NOT include the content encoding or the extension.
+            if response.content_encoding == "gzip" or record['s3']['object']['key'].end_with?(".gz") or record['s3']['object']['key'].include?("/firehose/") then
               begin
-		            temp = Zlib::GzipReader.new(body)
+		            temp = MultipleFilesGzipReader.new(body)
               rescue => e
                 @logger.warn("content is marked to be gzipped but can't unzip it, assuming plain text", :bucket => record['s3']['bucket']['name'], :object => record['s3']['object']['key'], :error => e)
                 temp = body
@@ -196,7 +197,8 @@ class LogStash::Inputs::SQSS3 < LogStash::Inputs::Threadable
 	      lines = body.read.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: "\u2370").split(/\n/)
 
 	      # Set the codec to json if required, otherwise the default is plain text
-              if response.content_type == "application/json" then
+              if response.content_type == "application/json" or record['s3']['object']['key'].include?("/firehose/") then
+		p "its firehose!!!"
                 @codec = @jsonCodec
 		
 		if response.content_encoding != "gzip" then

data/logstash-input-sqs_s3.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-sqs_s3'
-  s.version         = '1.1.15'
+  s.version         = '1.1.16'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program. Full credit goes to Heiko Finzel. Republishing this gem to support Logstash 5."
@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
+  s.add_dependency "multiple_files_gzip_reader"
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency "logstash-mixin-aws", ">= 1.0.0"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-sqs_s3
 version: !ruby/object:Gem::Version
-  version: 1.1.15
+  version: 1.1.16
 platform: ruby
 authors:
 - Heiko Finzel
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-16 00:00:00.000000000 Z
+date: 2018-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,20 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: multiple_files_gzip_reader
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: