logstash-input-s3 3.1.9 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 60a19b3b0dff622aedd2d5dbbc373af2e7d778d0dadc50cbbccfb374ad81bca5
-  data.tar.gz: dcf91569049bb072699ec4134429bef539ccdd593a4096c99a944c2ea1b903b9
+  metadata.gz: e1328e9b8c79b50ccbd386fa6f304bafffffbb4a2dfe6921311d9cae022d47c1
+  data.tar.gz: 261262cb55fed216fbcc1799925ccbaf1d75799289b9fa3b298cdb21fd8364a8
 SHA512:
-  metadata.gz: 8d4f33328b99e69f45d69740b434d30c8eea1e3ffe9d07faf021c350447520b3a5d227c99fdfbe7f52ddbd8812ae1c4d44605a08e44f3c1dbf20cca9ff756143
-  data.tar.gz: 6eaa8fa809d8b8ecb9390cf148b964edadca7aee861f94d0f4cb62c694f9d1a6174a19025c4286dcb1f694d3395bbd3013a432486009749c49d5d2961f284734
+  metadata.gz: 5a68a70f262ec1d005e122ba4498119a6649aefb430b8d970ba48c3b358691cd76d301f52004e326455edfcc0e2503419264c87b48b2fcdb8e25601370f8fadc
+  data.tar.gz: d9ade73b5c471199fbe37dcbbf92bfff6f86839294d7641bc07666822c994c16c51229bebed7e6221f45d9bb45e33eae6dde580c990bce3054e841959516002f

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 3.2.0
+ - Add support for auto-detecting gzip files with `.gzip` extension, in addition to existing support for `*.gz`
+ - Improve performance of gzip decoding by 10x by using Java's Zlib
+
 ## 3.1.9
   - Change default sincedb path to live in `{path.data}/plugins/inputs/s3` instead of $HOME.
     Prior Logstash installations (using $HOME default) are automatically migrated.

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/lib/logstash/inputs/s3.rb

@@ -9,6 +9,14 @@ require "stud/temporary"
 require "aws-sdk"
 require "logstash/inputs/s3/patch"
 
+require 'java'
+java_import java.io.InputStream
+java_import java.io.InputStreamReader
+java_import java.io.FileInputStream
+java_import java.io.BufferedReader
+java_import java.util.zip.GZIPInputStream
+java_import java.util.zip.ZipException
+
 Aws.eager_autoload!
 # Stream events from files from a S3 bucket.
 #
@@ -253,25 +261,27 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
   private
   def read_gzip_file(filename, block)
-    # Details about multiple streams and the usage of unused from: http://code.activestate.com/lists/ruby-talk/11168/
-    File.open(filename) do |zio|
-      while true do
-        io = Zlib::GzipReader.new(zio)
-        io.each_line { |line| block.call(line) }
-        unused = io.unused
-        io.finish
-        break if unused.nil?
-        zio.pos -= unused.length # reset the position to the other block in the stream
-      end
+    file_stream = FileInputStream.new(filename)
+    gzip_stream = GZIPInputStream.new(file_stream)
+    decoder = InputStreamReader.new(gzip_stream, "UTF-8")
+    buffered = BufferedReader.new(decoder)
+
+    while (line = buffered.readLine())
+      block.call(line)
     end
-  rescue Zlib::Error, Zlib::GzipFile::Error => e
+  rescue ZipException => e
     @logger.error("Gzip codec: We cannot uncompress the gzip file", :filename => filename)
     raise e
+  ensure
+    buffered.close unless buffered.nil?
+    decoder.close unless decoder.nil?
+    gzip_stream.close unless gzip_stream.nil?
+    file_stream.close unless file_stream.nil?
   end
 
   private
   def gzip?(filename)
-    filename.end_with?('.gz')
+    filename.end_with?('.gz','.gzip')
   end
   
   private

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.1.9'
+  s.version         = '3.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Streams events from files in a S3 bucket"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/fixtures/{invalid_utf8.log → invalid_utf8.gbk.log}

@@ -1,2 +1,2 @@
-2015-01-01T02:52:45.866722Z no "GET http://www.logstash.com:80/utfmadness/≈4od HTTP/1.1"
+2015-01-01T02:52:45.866722Z no "GET http://www.logstash.com:80/utfmadness/��4od HTTP/1.1"
 

data/spec/inputs/s3_spec.rb

@@ -285,6 +285,13 @@ describe LogStash::Inputs::S3 do
       include_examples "generated events"
     end
 
+    context 'compressed with gzip extension' do
+      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5) }
+      let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gzip') }
+
+      include_examples "generated events"
+    end
+
     context 'plain text' do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'uncompressed.log') }
 
@@ -306,7 +313,7 @@ describe LogStash::Inputs::S3 do
     end
 
     context 'encoded' do
-      let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'invalid_utf8.log') }
+      let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'invalid_utf8.gbk.log') }
 
       include_examples "generated events"
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.1.9
+  version: 3.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-19 00:00:00.000000000 Z
+date: 2018-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -120,7 +120,8 @@ files:
 - logstash-input-s3.gemspec
 - spec/fixtures/cloudfront.log
 - spec/fixtures/compressed.log.gz
-- spec/fixtures/invalid_utf8.log
+- spec/fixtures/compressed.log.gzip
+- spec/fixtures/invalid_utf8.gbk.log
 - spec/fixtures/json.log
 - spec/fixtures/json_with_message.log
 - spec/fixtures/multiline.log
@@ -159,7 +160,8 @@ summary: Streams events from files in a S3 bucket
 test_files:
 - spec/fixtures/cloudfront.log
 - spec/fixtures/compressed.log.gz
-- spec/fixtures/invalid_utf8.log
+- spec/fixtures/compressed.log.gzip
+- spec/fixtures/invalid_utf8.gbk.log
 - spec/fixtures/json.log
 - spec/fixtures/json_with_message.log
 - spec/fixtures/multiline.log