fluent-plugin-kinesis 3.4.1 → 3.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90a9718baa3e6849ce6ad7be32104672dcdf8993940c71d2d9fb666ce64b35e2
-  data.tar.gz: 064de8d0bbe66a8b8a8f4102bb686487fc0d7294904399dd745ce641b593c9bf
+  metadata.gz: 2a80a4e62aad6b32241c8db4ce69903db51da13b06391d9762737ef9b39a3987
+  data.tar.gz: 6dd4150b29f65e50980dd8977227de8675a6125b3d3977fe20089650f76bae90
 SHA512:
-  metadata.gz: 248f0fd2de87171ea58d878de9450edf95eb1dee58e31bfaf1aa716080e3921f68aa8ebdac2cbf052759a7e4262d5e598c5380c0c75edef503c54c8e760f1d3e
-  data.tar.gz: 79128aa4b4d07372ecc8d53d2f7196fa442d2c7c1694e215daca6cac7aae4d097af3b0512ddeae74708c053d6fb92fdce3c87811f4c0a2f5060b2c76649597e4
+  metadata.gz: 26ef149f10a4e953573bacdc3bf19185edeff7d13fc72058bc02d93d453d9573455eb9a9f5d75e0016ece72b1f0440f2431a0351ce9d988784de6a273efc3a9c
+  data.tar.gz: 1e591b860e4f1d4d2152d1b1327488da16d0efc2273d5ceff214180fb58f38bfc78d25859166b0ab67e4f883e140a2d5242a5a3dd95161cc8f2084f58cc9e7ac

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # CHANGELOG
 
+## 3.4.2
+- Feature - Add option to support gzip compression : [#215](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/215)
+
 ## 3.4.1
 
 - Enhancement - Add stream name to debug logs to identify : [#214](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/214)

data/README.md

@@ -353,7 +353,7 @@ Also, there are some format related options below:
 If your record contains a field whose string should be sent to Amazon Kinesis directly (without formatter), use this parameter to specify the field. In that case, other fields than **data_key** are thrown away and never sent to Amazon Kinesis. Default `nil`, which means whole record will be formatted and sent.
 
 ### compression
-Specifing compression way for data of each record. Current accepted options are `zlib`. Otherwise, no compression will be preformed.
+Specifying compression way for data of each record. Current accepted options are `zlib` and `gzip`. Otherwise, no compression will be preformed.
 
 ### log_truncate_max_size
 Integer, default 1024. When emitting the log entry, the message will be truncated by this size to avoid infinite loop when the log is also sent to Kinesis. The value 0 means no truncation.

data/lib/fluent/plugin/kinesis.rb

@@ -17,6 +17,7 @@ require 'fluent/msgpack_factory'
 require 'fluent/plugin/output'
 require 'fluent/plugin/kinesis_helper/client'
 require 'fluent/plugin/kinesis_helper/api'
+require 'fluent/plugin/kinesis_helper/compression'
 require 'zlib'
 
 module Fluent
@@ -121,6 +122,8 @@ module Fluent
         case @compression
         when "zlib"
           ->(data) { Zlib::Deflate.deflate(data) }
+        when "gzip"
+          ->(data) { Gzip.compress(data) }
         else
           ->(data) { data }
         end

data/lib/fluent/plugin/kinesis_helper/compression.rb

@@ -0,0 +1,27 @@
+require "stringio"
+require "zlib"
+
+class Stream < StringIO
+  def initialize(*)
+    super
+    set_encoding "BINARY"
+  end
+
+  def close
+    rewind;
+  end
+end
+
+class Gzip
+  def self.compress(string, level = Zlib::DEFAULT_COMPRESSION, strategy = Zlib::DEFAULT_STRATEGY)
+    output = Stream.new
+    gz = Zlib::GzipWriter.new(output, level, strategy)
+    gz.write(string)
+    gz.close
+    output.string
+  end
+
+  def self.decompress(string)
+    Zlib::GzipReader.wrap(StringIO.new(string), &:read)
+  end
+end

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '3.4.1'
+  VERSION = '3.4.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 3.4.1
+  version: 3.4.2
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-09 00:00:00.000000000 Z
+date: 2021-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -308,6 +308,7 @@ files:
 - lib/fluent/plugin/kinesis_helper/aggregator.rb
 - lib/fluent/plugin/kinesis_helper/api.rb
 - lib/fluent/plugin/kinesis_helper/client.rb
+- lib/fluent/plugin/kinesis_helper/compression.rb
 - lib/fluent/plugin/out_kinesis_firehose.rb
 - lib/fluent/plugin/out_kinesis_streams.rb
 - lib/fluent/plugin/out_kinesis_streams_aggregated.rb