logstash-input-s3 3.1.1 → 3.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b975c1700f9a6aab421e175f94d1bce38b9d9e01
-  data.tar.gz: 1ebe414e3ba68ae1d4e9d3477d9ec9663f13dc10
+  metadata.gz: 44b70069c72227302a7d1c75314f1c3bcc40c67d
+  data.tar.gz: 3a42b5a34ae5df3f77732dddf7929a91d7494685
 SHA512:
-  metadata.gz: 5477fd22a31ab077044f93c089a9b466ac692b9c3cfdf6156fcfc7b70135d7881b34ea314c54aa7fc1566b46e045f1bc0f714acd554e6971318db4a2e468fda1
-  data.tar.gz: 44adae47471d9ad5a495b63bb155110cb86df5d0167fc94562e5b6e35c8f20d069727c166ac8bee438e4f22573410d25e247e010ff9398cff8554c3c15ec52cc
+  metadata.gz: be6f6e9873262bd8236ed76797c05c547c718e30a283bd5ada48b7566a1eb9da314a3e92b1859c7bd128f3f761fcb1337a6a0af6386066fb89677f4b3eec2b39
+  data.tar.gz: 6ddb29cde0e89777bbf924620f49658ca0400c6c852322a54ec38753a627e40d7252da5d55419b1d7cf9118c2813d12cf9d4b5c11f767dbbad9e63d4189c6fe7

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 3.1.2
+ - Fix an issue when the remote file contains multiple blob of gz in the same file #101
+ - Make the integration suite run
+ - Remove uneeded development dependency
+
 ## 3.1.1
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 

data/README.md

@@ -17,6 +17,7 @@ permissions applied to the AWS IAM Policy being used:
 You might also need `s3:DeleteObject` when setting S3 input to delete on read.
 And the `s3:CreateBucket` permission to create a backup bucket unless already
 exists.
+In addition, when `backup_to_bucket` is used, the `s3:PutObject` action is also required.
 
 For buckets that have versioning enabled, you might need to add additional
 permissions.
@@ -113,4 +114,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/inputs/s3.rb

@@ -6,7 +6,10 @@ require "time"
 require "tmpdir"
 require "stud/interval"
 require "stud/temporary"
+require "aws-sdk"
+require "logstash/inputs/s3/patch"
 
+Aws.eager_autoload!
 # Stream events from files from a S3 bucket.
 #
 # Each line from each file generates an event.
@@ -242,14 +245,20 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
   private
   def read_gzip_file(filename, block)
-    begin
-      Zlib::GzipReader.open(filename) do |decoder|
-        decoder.each_line { |line| block.call(line) }
+    # Details about multiple streams and the usage of unused from: http://code.activestate.com/lists/ruby-talk/11168/
+    File.open(filename) do |zio|
+      while true do
+        io = Zlib::GzipReader.new(zio)
+        io.each_line { |line| block.call(line) }
+        unused = io.unused
+        io.finish
+        break if unused.nil?
+        zio.pos -= unused.length # reset the position to the other block in the stream
       end
-    rescue Zlib::Error, Zlib::GzipFile::Error => e
-      @logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename)
-      raise e
     end
+  rescue Zlib::Error, Zlib::GzipFile::Error => e
+    @logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename)
+    raise e
   end
 
   private

data/lib/logstash/inputs/s3/patch.rb

@@ -0,0 +1,20 @@
+# This is patch related to the autoloading and ruby
+#
+# The fix exist in jruby 9k but not in the current jruby, not sure when or it will be backported
+# https://github.com/jruby/jruby/issues/3645
+#
+# AWS is doing tricky name discovery in the module to generate the correct error class and
+# this strategy is bogus in jruby and `eager_autoload` don't fix this issue.
+#
+# This will be a short lived patch since AWS is removing the need.
+# see: https://github.com/aws/aws-sdk-ruby/issues/1301#issuecomment-261115960
+old_stderr = $stderr
+
+$stderr = StringIO.new
+begin
+  module Aws
+    const_set(:S3, Aws::S3)
+  end
+ensure
+  $stderr = old_stderr
+end

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.1.1'
+  s.version         = '3.1.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files from a S3 bucket."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -24,10 +24,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-mixin-aws'
   s.add_runtime_dependency 'stud', '~> 0.0.18'
  # s.add_runtime_dependency 'aws-sdk-resources', '>= 2.0.33'
-
   s.add_development_dependency 'logstash-devutils'
-  s.add_development_dependency 'simplecov'
-  s.add_development_dependency 'coveralls'
   s.add_development_dependency "logstash-codec-json"
 end
 

data/spec/inputs/s3_spec.rb

@@ -208,9 +208,11 @@ describe LogStash::Inputs::S3 do
   end
 
   shared_examples "generated events"  do
+    let(:events_to_process) { 2 }
+
     it 'should process events' do
       events = fetch_events(config)
-      expect(events.size).to eq(2)
+      expect(events.size).to eq(events_to_process)
     end
 
     it "deletes the temporary file" do
@@ -251,6 +253,15 @@ describe LogStash::Inputs::S3 do
       include_examples "generated events"
     end
 
+    context "multiple compressed streams" do
+      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day) }
+      let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'multiple_compressed_streams.gz') }
+
+      include_examples "generated events" do
+        let(:events_to_process) { 16 }
+      end
+    end
+
     context 'compressed' do
       let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day) }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gz') }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.1.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2017-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -72,34 +72,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: simplecov
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: coveralls
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -127,11 +99,13 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/inputs/s3.rb
+- lib/logstash/inputs/s3/patch.rb
 - logstash-input-s3.gemspec
 - spec/fixtures/cloudfront.log
 - spec/fixtures/compressed.log.gz
 - spec/fixtures/invalid_utf8.log
 - spec/fixtures/json.log
+- spec/fixtures/multiple_compressed_streams.gz
 - spec/fixtures/uncompressed.log
 - spec/inputs/s3_spec.rb
 - spec/inputs/sincedb_spec.rb
@@ -159,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Stream events from files from a S3 bucket.
@@ -168,6 +142,7 @@ test_files:
 - spec/fixtures/compressed.log.gz
 - spec/fixtures/invalid_utf8.log
 - spec/fixtures/json.log
+- spec/fixtures/multiple_compressed_streams.gz
 - spec/fixtures/uncompressed.log
 - spec/inputs/s3_spec.rb
 - spec/inputs/sincedb_spec.rb