fluent-plugin-s3 1.1.2 → 1.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 29ae6edc5711fd7c6409cdcf3f592db33f50cf2c
-  data.tar.gz: 00b0445936e92e75346e4e9fd2f3b1943031349c
+  metadata.gz: 16e7bfdfa2c8a9ac640bc5aab2f4f340ff764874
+  data.tar.gz: a4711544ff30e4312474b85a730aa00fb8cd3c1b
 SHA512:
-  metadata.gz: 9c4d1b1d4d344cc1360db807f46512a39a203843b0a5735086814bea80b0a3940c2f50ac98f15f1c30016537d0c53e6d5589211b310b3fe1a6d9bff498445bff
-  data.tar.gz: a4f94d231b9c0d9aef6eaee2f03aa7a75f95a88dced554731bd0b5cfabf5229063220d601317eb9d12cc18c9541a12a4b50cce53e4d902205b93d0b9ebe26e88
+  metadata.gz: d084d89f1fc8ba64405c417e2f71d5a2316d47de454be606d28b54ad9a52d567e09e401a5f1bf72268c638d37e5b17812e4c3d85be6221081c6921fe2160adc1
+  data.tar.gz: fdd479ee6b6bbae6a0d4bd2e25817d70b6f82fe2d8ab87ca835b4b803cd86f7b3432266bdd4ed099edcda8501d11d790818ed4336740f79126cc87d621c4031a

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 1.1.3 - 2018/04/15
+
+  * in_s3: Fixed extracting gzip's multiple stream.
+
 Release 1.1.2 - 2018/04/10
 
   * out_s3: Fix memory leak when s3_object_key_format has time related placeholder

data/VERSION

@@ -1 +1 @@
-1.1.2
+1.1.3

data/lib/fluent/plugin/in_s3.rb

@@ -284,10 +284,23 @@ module Fluent::Plugin
         'application/x-gzip'.freeze
       end
 
+      # https://bugs.ruby-lang.org/issues/9790
+      # https://bugs.ruby-lang.org/issues/11180
+      # https://github.com/exAspArk/multiple_files_gzip_reader
       def extract(io)
-        Zlib::GzipReader.wrap(io) do |gz|
-          gz.read
+        parts = []
+        loop do
+          unused = nil
+          Zlib::GzipReader.wrap(io) do |gz|
+            parts << gz.read
+            unused = gz.unused
+            gz.finish
+          end
+          io.pos -= unused ? unused.length : 0
+          break if io.eof?
         end
+        io.close
+        parts.join
       end
     end
 

data/test/test_in_s3.rb

@@ -89,6 +89,7 @@ class S3InputTest < Test::Unit::TestCase
 
     data("json" => ["json", "json", "application/json"],
          "text" => ["text", "txt", "text/plain"],
+         "gzip" => ["gzip", "gz", "application/x-gzip"],
          "gzip_command" => ["gzip_command", "gz", "application/x-gzip"],
          "lzo" => ["lzo", "lzo", "application/x-lzop"],
          "lzma2" => ["lzma2", "xz", "application/x-xz"])
@@ -262,4 +263,104 @@ class S3InputTest < Test::Unit::TestCase
     ]
     assert_equal(expected_records, events.map {|_tag, _time, record| record })
   end
+
+  def test_gzip_single_stream
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as gzip\nformat none\n")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body {
+      io = StringIO.new
+      Zlib::GzipWriter.wrap(io) do |gz|
+        gz.write "aaa\nbbb\n"
+        gz.finish
+      end
+      io.rewind
+      io
+    }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    expected_records = [
+      { "message" => "aaa\n" },
+      { "message" => "bbb\n" }
+    ]
+    assert_equal(expected_records, events.map {|_tag, _time, record| record })
+  end
+
+  def test_gzip_multiple_steams
+    setup_mocks
+    d = create_driver(CONFIG + "\ncheck_apikey_on_start false\nstore_as gzip\nformat none\n")
+
+    s3_object = stub(Object.new)
+    s3_response = stub(Object.new)
+    s3_response.body {
+      io = StringIO.new
+      Zlib::GzipWriter.wrap(io) do |gz|
+        gz.write "aaa\nbbb\n"
+        gz.finish
+      end
+      Zlib::GzipWriter.wrap(io) do |gz|
+        gz.write "ccc\nddd\n"
+        gz.finish
+      end
+      io.rewind
+      io
+    }
+    s3_object.get { s3_response }
+    @s3_bucket.object(anything).at_least(1) { s3_object }
+
+    body = {
+      "Records" => [
+        {
+          "s3" => {
+            "object" => {
+              "key" => "test_key"
+            }
+          }
+        }
+      ]
+    }
+    message = Struct::StubMessage.new(1, 1, Yajl.dump(body))
+    @sqs_poller.get_messages(anything, anything) do |config, stats|
+      config.before_request.call(stats) if config.before_request
+      stats.request_count += 1
+      if stats.request_count >= 1
+        d.instance.instance_variable_set(:@running, false)
+      end
+      [message]
+    end
+    d.run(expect_emits: 1)
+    events = d.events
+    expected_records = [
+      { "message" => "aaa\n" },
+      { "message" => "bbb\n" },
+      { "message" => "ccc\n" },
+      { "message" => "ddd\n" }
+    ]
+    assert_equal(expected_records, events.map {|_tag, _time, record| record })
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-10 00:00:00.000000000 Z
+date: 2018-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -163,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.6.14.1
 signing_key: 
 specification_version: 4
 summary: Amazon S3 output plugin for Fluentd event collector