fluent-plugin-kinesis 3.2.3 → 3.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d1bd84d96b64e32b8aac649672adeb5333c7931cb7a7b5a4ec13cd6a8031f1be
-  data.tar.gz: 657ed567a713283fbc40c6ea7ed1fe7cc940f7898547e584aa4ed447fbfabdd6
+  metadata.gz: 2a80a4e62aad6b32241c8db4ce69903db51da13b06391d9762737ef9b39a3987
+  data.tar.gz: 6dd4150b29f65e50980dd8977227de8675a6125b3d3977fe20089650f76bae90
 SHA512:
-  metadata.gz: 6da3c960496b64b446718ff79cd95523c3058372893e313a5049ca3fad51895a613bc06b5ce5e8dc6a71966b5c0dbe00fcfd9b0b5ff7438813f0be581efc1122
-  data.tar.gz: e5fff3586fd0f1519a9f686ccaca4212a45b30c9f7199fb82460bbd04d4a98c080fe1c93bf1dc33099b432dab9da4118a943c619e31069b082ce98ac366b80e2
+  metadata.gz: 26ef149f10a4e953573bacdc3bf19185edeff7d13fc72058bc02d93d453d9573455eb9a9f5d75e0016ece72b1f0440f2431a0351ce9d988784de6a273efc3a9c
+  data.tar.gz: 1e591b860e4f1d4d2152d1b1327488da16d0efc2273d5ceff214180fb58f38bfc78d25859166b0ab67e4f883e140a2d5242a5a3dd95161cc8f2084f58cc9e7ac

data/.gitignore

@@ -4,6 +4,7 @@
 /gemfiles/Gemfile*.lock
 /_yardoc/
 /coverage/
+/vendor/
 /doc/
 /pkg/
 /spec/reports/

data/.travis.yml

@@ -2,35 +2,55 @@ language: ruby
 os: linux
 sudo: false
 
-gemfile:
-  - Gemfile
-  - gemfiles/Gemfile.td-agent-3.8.0
-
 # Test with supported td-agent versions
-# https://tddocs.atlassian.net/wiki/spaces/PD/pages/6291635/The+td-agent+ChangeLog
+# https://docs.fluentd.org/quickstart/support#change-logs
+# https://docs.treasuredata.com/display/public/PD/The+td-agent+Change+Log
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
 matrix:
   include:
-    - rvm: 2.6.6 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads/
-      # Waiting protobuf support for Ruby 2.7 / https://github.com/protocolbuffers/protobuf/issues/7070
-      gemfile: gemfiles/Gemfile.fluentd-1.11.1 # Latest Fluentd / https://www.fluentd.org/download
-    - rvm: 2.3.0 # Oldest supported Ruby
-      gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
-    - rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
-      gemfile: gemfiles/Gemfile.td-agent-3.1.1
-    - rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.1/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.2.1
-    - rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.3.0
-    - rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.4.1
-    - rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.1/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.5.1
-    - rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.6.0
-    - rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.7.1
-    - rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.8.0
+    - gemfile: gemfiles/Gemfile.fluentd-1.13.3 # Latest Fluentd / https://www.fluentd.org/download
+      rvm: 3.0.2 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads
+      dist: focal
+    - gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
+      rvm: 2.3.0 # Oldest supported Ruby
+    - gemfile: gemfiles/Gemfile.td-agent-3.1.0
+      rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L20
+    # - gemfile: gemfiles/Gemfile.td-agent-3.1.1
+    #   rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
+    # - gemfile: gemfiles/Gemfile.td-agent-3.2.0
+    #   rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.2.1
+      rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.1/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.3.0
+      rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/config/projects/td-agent3.rb#L22
+    # - gemfile: gemfiles/Gemfile.td-agent-3.4.0
+    #   rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.4.1
+      rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L22
+    # - gemfile: gemfiles/Gemfile.td-agent-3.5.0
+    #   rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.5.1
+      rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.1/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.6.0
+      rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/config/projects/td-agent3.rb#L22
+    # - gemfile: gemfiles/Gemfile.td-agent-3.7.0
+    #   rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/commit/da5cb39ec5396f652cd30bad5bb8ef4ec19071a9#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R22
+    - gemfile: gemfiles/Gemfile.td-agent-3.7.1
+      rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.8.0 # Latest official release of td-agent v3
+      rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.8.1 # Maintenance release of td-agent v3
+      rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R22
+    # - gemfile: gemfiles/Gemfile.td-agent-4.0.0
+    #   rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L16
+    - gemfile: gemfiles/Gemfile.td-agent-4.0.1
+      rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L16
+    # - gemfile: gemfiles/Gemfile.td-agent-4.1.0
+    #   rvm: 2.7.2 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L19
+    - gemfile: gemfiles/Gemfile.td-agent-4.1.1
+      rvm: 2.7.3 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L17
+    - gemfile: gemfiles/Gemfile.td-agent-4.2.0
+      rvm: 2.7.4 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L21
   fast_finish: true
 
 script: bundle exec rake test

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # CHANGELOG
 
+## 3.4.2
+- Feature - Add option to support gzip compression : [#215](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/215)
+
+## 3.4.1
+
+- Enhancement - Add stream name to debug logs to identify : [#214](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/214)
+- Enhancement - Add td-agent 4.1.1, 4.2.0 and Fluentd 1.13.3 with Ruby 3.0.2 to test cases
+
+## 3.4.0
+
+- Enhancement - Enable to monitor batch request failure and retries : [#150](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/150) [#211](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/211)
+- Enhancement - Make sleep reliable by measuring actual slept time : [#162](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/162)
+- Enhancement - Add td-agent 4.1.0 and Fluentd 1.12.3 to test cases
+
+## 3.3.0
+
+- Feature - Add web_identity_credentials configuration for IRSA : [#208](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/208) [#209](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/209)
+- Enhancement - Remove strict gem pinning of google-protobuf to support Ruby 2.7 : [#199](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/199) [#206](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/206)
+- Enhancement - Add td-agent v4 (4.0.1) and Fluentd 1.11.2 to test cases
+
 ## 3.2.3
 
 - Enhancement - Add placeholder support for delivery_stream_name of kinesis_firehose : [#203](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/203) [#204](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/204)

data/CONTRIBUTORS.txt

@@ -4,3 +4,5 @@ to the Fluent Plugin for Amazon Kinesis Streams and Amazon Kinesis Firehose. Giv
 Genki Sugawara
 sgwr_dts@yahoo.co.jp
 https://github.com/winebarrel
+https://github.com/adammw
+https://github.com/cosmo0920

data/README.md

@@ -1,7 +1,6 @@
 # Fluent plugin for Amazon Kinesis
 
-[![Gitter](https://badges.gitter.im/awslabs/aws-fluent-plugin-kinesis.svg)](https://gitter.im/awslabs/aws-fluent-plugin-kinesis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
-[![Build Status](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis)
+[![Build Status](https://api.travis-ci.com/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://app.travis-ci.com/github/awslabs/aws-fluent-plugin-kinesis)
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 [![Gem Downloads](https://img.shields.io/gem/dt/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 
@@ -186,6 +185,50 @@ It should be added to assume_role_credentials configuration stanza in the next f
 
 STS API endpoint url. This can be used to override the default global STS API endpoint of sts.amazonaws.com. Using regional endpoints may be preferred to reduce latency, and are required if utilizing a PrivateLink VPC Endpoint for STS API calls.
 
+
+### web_identity_credentials
+
+Similar to the assume_role_credentials, but for usage in EKS.
+
+    <match *>
+      @type kinesis_streams
+
+      <web_identity_credentials>
+        role_arn          ROLE_ARN
+        role_session_name ROLE_SESSION_NAME
+        web_identity_token_file AWS_WEB_IDENTITY_TOKEN_FILE
+      </web_identity_credentials>
+    </match>
+
+See also:
+
+*   [Using IAM Roles - AWS Identity and Access Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
+*   [IAM Roles For Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html)
+*   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
+*   [Aws::AssumeRoleWebIdentityCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleWebIdentityCredentials.html)
+
+**role_arn (required)**
+
+The Amazon Resource Name (ARN) of the role to assume.
+
+**role_session_name (required)**
+
+An identifier for the assumed role session.
+
+**web_identity_token_file (required)**
+
+The absolute path to the file on disk containing the OIDC token
+
+**policy**
+
+An IAM policy in JSON format.
+
+**duration_seconds**
+
+The duration, in seconds, of the role session. The value can range from
+900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
+is set to 3600 seconds (1 hour).
+
 ### instance_profile_credentials
 
 Retrieve temporary security credentials via HTTP request. This is useful on EC2 instance.
@@ -310,7 +353,7 @@ Also, there are some format related options below:
 If your record contains a field whose string should be sent to Amazon Kinesis directly (without formatter), use this parameter to specify the field. In that case, other fields than **data_key** are thrown away and never sent to Amazon Kinesis. Default `nil`, which means whole record will be formatted and sent.
 
 ### compression
-Specifing compression way for data of each record. Current accepted options are `zlib`. Otherwise, no compression will be preformed.
+Specifying compression way for data of each record. Current accepted options are `zlib` and `gzip`. Otherwise, no compression will be preformed.
 
 ### log_truncate_max_size
 Integer, default 1024. When emitting the log entry, the message will be truncated by this size to avoid infinite loop when the log is also sent to Kinesis. The value 0 means no truncation.
@@ -365,6 +408,16 @@ Default:
 - `kinesis_firehose`: 4 MB
 - `kinesis_streams_aggregated`: 1 MB
 
+### drop_failed_records_after_batch_request_retries
+Boolean, default `true`.
+
+If *drop_failed_records_after_batch_request_retries* is enabled (default), the plugin will drop failed records when batch request fails after retrying max times configured as *retries_on_batch_request*. This dropping can be monitored from [monitor_agent](https://docs.fluentd.org/input/monitor_agent) or [fluent-plugin-prometheus](https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus) as *retry_count* or *num_errors* metrics.
+
+If *drop_failed_records_after_batch_request_retries* is disabled, the plugin will raise error and return chunk to Fluentd buffer when batch request fails after retrying max times. Fluentd will retry to send chunk records according to retry config in [Buffer Section](https://docs.fluentd.org/configuration/buffer-section). Note that this retryng may create duplicate records since [PutRecords API](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html) of Kinesis Data Streams and [PutRecordBatch API](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecordBatch.html) of Kinesis Data Firehose may return a partially successful response.
+
+### monitor_num_of_batch_request_retries
+Boolean, default `false`. If enabled, the plugin will increment *retry_count* monitoring metrics after internal retrying to send batch request. This configuration enables you to monitor [ProvisionedThroughputExceededException](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html) from [monitor_agent](https://docs.fluentd.org/input/monitor_agent) or [fluent-plugin-prometheus](https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus). Note that *retry_count* metrics will be counted by the plugin in addition to original Fluentd buffering mechanism if *monitor_num_of_batch_request_retries* is enabled.
+
 ## Configuration: kinesis_streams
 Here are `kinesis_streams` specific configurations.
 

data/fluent-plugin-kinesis.gemspec

@@ -55,7 +55,7 @@ Gem::Specification.new do |spec|
   #   https://github.com/aws/aws-sdk-ruby/commit/e26577d2a426a4be79cd2d9edc1a4a4176e388ba#diff-10f50e27b30c3dc522b3c25db5782e2e
   spec.add_dependency "aws-sdk-firehose", "~> 1", "!= 1.5", "!= 1.9", "!= 1.15"
 
-  spec.add_dependency "google-protobuf", "~> 3", "< 3.12"
+  spec.add_dependency "google-protobuf", "~> 3"
 
   spec.add_development_dependency "bundler", ">= 1.10"
   spec.add_development_dependency "rake", ">= 10.0"

data/gemfiles/Gemfile.fluentd-0.14.22

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.fluentd-1.13.3

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+gem "fluentd", "1.13.3"

data/gemfiles/Gemfile.td-agent-3.1.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.1.1

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.2.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.2.1

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.3.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.4.0

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.4.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.4.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.149.0"
+gem "aws-sigv4", "1.1.0"
+gem "aws-sdk-core", "3.48.3"
+gem "aws-sdk-kms", "1.16.0"
+gem "aws-sdk-sqs", "1.13.0"
+gem "aws-sdk-s3", "1.36.0"
+gem "fluent-plugin-s3", "1.1.9"

data/gemfiles/Gemfile.td-agent-3.4.1

@@ -1,29 +1,12 @@
-#
-# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
 gemspec path: ".."
 
 # Specify related gems for td-agent v3.4.1
-# td-agent v3.4.0 has the same dependencies
 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L27
-# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/config/projects/td-agent3.rb#L27
 gem "fluentd", "1.4.2"
 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/plugin_gems.rb#L16-L23
-# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/plugin_gems.rb#L16-L23
 gem "jmespath", "1.4.0"
 gem "aws-partitions", "1.149.0"
 gem "aws-sigv4", "1.1.0"

data/gemfiles/Gemfile.td-agent-3.5.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.5.1

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.6.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.7.0

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.7.0
+# https://docs.treasuredata.com/display/public/PD/The+td-agent+Change+Log
+gem "fluentd", "1.10.0"
+# https://github.com/treasure-data/omnibus-td-agent/commit/74920cb2a55e40195961a58c6b6de4bd5b3f1f87#diff-8d45eaa8758f7591f884c9e9917ea96105331a1daf5a891b4645b89e120f2d69R16-R23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.288.0"
+gem "aws-sigv4", "1.1.1"
+gem "aws-sdk-core", "3.92.0"
+gem "aws-sdk-kms", "1.30.0"
+gem "aws-sdk-sqs", "1.24.0"
+gem "aws-sdk-s3", "1.61.1"
+gem "fluent-plugin-s3", "1.3.0"

data/gemfiles/Gemfile.td-agent-3.7.1

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.8.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.8.1

@@ -0,0 +1,18 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Maintenance release of td-agent v3
+# Specify related gems for td-agent v3.8.1
+# https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R27
+gem "fluentd", "1.11.5"
+# https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-8d45eaa8758f7591f884c9e9917ea96105331a1daf5a891b4645b89e120f2d69R19-R25
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.399.0"
+gem "aws-sigv4", "1.2.2"
+gem "aws-sdk-core", "3.109.3"
+gem "aws-sdk-kms", "1.39.0"
+gem "aws-sdk-sqs", "1.34.0"
+gem "aws-sdk-s3", "1.85.0"
+gem "fluent-plugin-s3", "1.4.0"

data/gemfiles/Gemfile.td-agent-4.0.0

@@ -0,0 +1,25 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.0.0
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L4
+gem "fluentd", "1.11.1"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L31
+gem "prometheus-client", "0.9.0"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L19-L20
+gem "fluent-plugin-prometheus", "1.8.0"
+gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L28
+gem "jmespath", "1.4.0"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L8-L13
+gem "aws-partitions", "1.337.0"
+gem "aws-sdk-core", "3.102.1"
+gem "aws-sdk-kms", "1.35.0"
+gem "aws-sdk-s3", "1.72.0"
+gem "aws-sdk-sqs", "1.29.0"
+gem "aws-sigv4", "1.2.1"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L23
+gem "fluent-plugin-s3", "1.3.3"

data/gemfiles/Gemfile.td-agent-4.0.1

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.0.1
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L4
+gem "fluentd", "1.11.2"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/gemfiles/linux/Gemfile#L35-L42
+gem "prometheus-client", "0.9.0"
+gem "fluent-plugin-prometheus", "1.8.2"
+gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.352.0"
+gem "aws-sdk-core", "3.104.3"
+gem "aws-sdk-kms", "1.36.0"
+gem "aws-sdk-sqs", "1.30.0"
+gem "aws-sigv4", "1.2.1"
+gem "aws-sdk-s3", "1.75.0"
+gem "fluent-plugin-s3", "1.4.0"

data/gemfiles/Gemfile.td-agent-4.1.0

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.1.0
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L4
+gem "fluentd", "1.12.1"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/Gemfile#L32-L42
+gem "prometheus-client", "0.9.0"
+gem "fluent-plugin-prometheus", "1.8.5"
+gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.427.0"
+gem "aws-sdk-core", "3.112.0"
+gem "aws-sdk-kms", "1.42.0"
+gem "aws-sdk-sqs", "1.36.0"
+gem "aws-sigv4", "1.2.2"
+gem "aws-sdk-s3", "1.88.1"
+gem "fluent-plugin-s3", "1.5.1"

data/gemfiles/Gemfile.td-agent-4.1.1

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.1.1
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L4
+gem "fluentd", "1.12.3"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/Gemfile#L32-L42
+gem "prometheus-client", "0.9.0"
+gem "fluent-plugin-prometheus", "1.8.5"
+gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.446.0"
+gem "aws-sdk-core", "3.114.0"
+gem "aws-sdk-kms", "1.43.0"
+gem "aws-sdk-sqs", "1.38.0"
+gem "aws-sigv4", "1.2.3"
+gem "aws-sdk-s3", "1.93.1"
+gem "fluent-plugin-s3", "1.6.0"

data/gemfiles/Gemfile.td-agent-4.2.0

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.2.0
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L4
+gem "fluentd", "1.13.3"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/Gemfile#L42-L52
+gem "prometheus-client", "2.1.0"
+gem "fluent-plugin-prometheus", "2.0.1"
+gem "fluent-plugin-prometheus_pushgateway", "0.1.0"
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.478.0"
+gem "aws-sdk-core", "3.117.0"
+gem "aws-sdk-kms", "1.44.0"
+gem "aws-sdk-sqs", "1.40.0"
+gem "aws-sigv4", "1.2.4"
+gem "aws-sdk-s3", "1.96.1"
+gem "fluent-plugin-s3", "1.6.0"

data/lib/fluent/plugin/kinesis.rb

@@ -17,6 +17,7 @@ require 'fluent/msgpack_factory'
 require 'fluent/plugin/output'
 require 'fluent/plugin/kinesis_helper/client'
 require 'fluent/plugin/kinesis_helper/api'
+require 'fluent/plugin/kinesis_helper/compression'
 require 'zlib'
 
 module Fluent
@@ -84,6 +85,10 @@ module Fluent
         true
       end
 
+      def formatted_to_msgpack_binary?
+        true
+      end
+
       private
 
       def data_formatter_create(conf)
@@ -117,6 +122,8 @@ module Fluent
         case @compression
         when "zlib"
           ->(data) { Zlib::Deflate.deflate(data) }
+        when "gzip"
+          ->(data) { Gzip.compress(data) }
         else
           ->(data) { data }
         end
@@ -142,15 +149,13 @@ module Fluent
         include Fluent::MessagePackFactory::Mixin
       end
 
-      def write_records_batch(chunk, &block)
+      def write_records_batch(chunk, stream_name, &block)
         unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
-        chunk.open do |io|
-          records = msgpack_unpacker(io).to_enum
-          split_to_batches(records) do |batch, size|
-            log.debug(sprintf "Write chunk %s / %3d records / %4d KB", unique_id, batch.size, size/1024)
-            batch_request_with_retry(batch, &block)
-            log.debug("Finish writing chunk")
-          end
+        records = chunk.to_enum(:msgpack_each)
+        split_to_batches(records) do |batch, size|
+          log.debug(sprintf "%s: Write chunk %s / %3d records / %4d KB", stream_name, unique_id, batch.size, size/1024)
+          batch_request_with_retry(batch, &block)
+          log.debug(sprintf "%s: Finish writing chunk", stream_name)
         end
       end
 

data/lib/fluent/plugin/kinesis_helper/api.rb

@@ -14,6 +14,7 @@
 
 require 'fluent_plugin_kinesis/version'
 require 'fluent/configurable'
+require 'benchmark'
 
 module Fluent
   module Plugin
@@ -40,10 +41,12 @@ module Fluent
         module BatchRequest
           module BatchRequestParams
             include Fluent::Configurable
-            config_param :retries_on_batch_request, :integer, default: 8
-            config_param :reset_backoff_if_success, :bool,    default: true
-            config_param :batch_request_max_count,  :integer, default: nil
-            config_param :batch_request_max_size,   :integer, default: nil
+            config_param :retries_on_batch_request,                        :integer, default: 8
+            config_param :reset_backoff_if_success,                        :bool,    default: true
+            config_param :batch_request_max_count,                         :integer, default: nil
+            config_param :batch_request_max_size,                          :integer, default: nil
+            config_param :drop_failed_records_after_batch_request_retries, :bool,    default: true
+            config_param :monitor_num_of_batch_request_retries,            :bool,    default: false
           end
 
           def self.included(mod)
@@ -96,11 +99,9 @@ module Fluent
                 wait_second = backoff.next
                 msg = 'Retrying to request batch. Retry count: %3d, Retry records: %3d, Wait seconds %3.2f' % [retry_count+1, failed_records.size, wait_second]
                 log.warn(truncate msg)
-                # TODO: sleep() doesn't wait the given seconds sometime.
-                # The root cause is unknown so far, so I'd like to add debug print only. It should be fixed in the future.
-                log.debug("#{Thread.current.object_id} sleep start")
-                sleep(wait_second)
-                log.debug("#{Thread.current.object_id} sleep finish")
+                # Increment num_errors to monitor batch request retries from "monitor_agent" or "fluent-plugin-prometheus"
+                increment_num_errors if @monitor_num_of_batch_request_retries
+                reliable_sleep(wait_second)
                 batch_request_with_retry(retry_records(failed_records), retry_count+1, backoff: backoff, &block)
               else
                 give_up_retries(failed_records)
@@ -108,6 +109,18 @@ module Fluent
             end
           end
 
+          # Sleep seems to not sleep as long as we ask it, our guess is that something wakes up the thread,
+          # so we keep on going to sleep if that happens.
+          # TODO: find out who is causing the sleep to be too short and try to make them stop it instead
+          def reliable_sleep(wait_second)
+            loop do
+              actual = Benchmark.realtime { sleep(wait_second) }
+              break if actual >= wait_second
+              log.error("#{Thread.current.object_id} sleep failed expected #{wait_second} but slept #{actual}")
+              wait_second -= actual
+            end
+          end
+
           def any_records_shipped?(res)
             results(res).size > failed_count(res)
           end
@@ -164,6 +177,49 @@ module Fluent
                 record[:original]
               ])
             }
+
+            if @drop_failed_records_after_batch_request_retries
+              # Increment num_errors to monitor batch request failure from "monitor_agent" or "fluent-plugin-prometheus"
+              increment_num_errors
+            else
+              # Raise error and return chunk to Fluentd for retrying
+              case request_type
+              # @see https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Kinesis/Client.html#put_records-instance_method
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Kinesis/Errors.html
+              when :streams, :streams_aggregated
+                provisioned_throughput_exceeded_records = failed_records.select { |record| record[:error_code] == 'ProvisionedThroughputExceededException' }
+                target_failed_record = provisioned_throughput_exceeded_records.first || failed_records.first
+                target_error = provisioned_throughput_exceeded_records.empty? ?
+                  Aws::Kinesis::Errors::ServiceError :
+                  Aws::Kinesis::Errors::ProvisionedThroughputExceededException
+              # @see https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Firehose/Client.html#put_record_batch-instance_method
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Firehose/Errors.html
+              when :firehose
+                service_unavailable_exception_records = failed_records.select { |record| record[:error_code] == 'ServiceUnavailableException' }
+                target_failed_record = service_unavailable_exception_records.first || failed_records.first
+                target_error = service_unavailable_exception_records.empty? ?
+                  Aws::Firehose::Errors::ServiceError :
+                  Aws::Firehose::Errors::ServiceUnavailableException
+              end
+              log.error("Raise #{target_failed_record[:error_code]} and return chunk to Fluentd buffer for retrying")
+              raise target_error.new(Seahorse::Client::RequestContext.new, target_failed_record[:error_message])
+            end
+          end
+
+          def increment_num_errors
+            # Prepare Fluent::Plugin::Output instance variables to count errors in this method.
+            # These instance variables are initialized here for possible future breaking changes of Fluentd.
+            @num_errors ||= 0
+            # @see https://github.com/fluent/fluentd/commit/d245454658d16170431d276fcd5849fb0d88ab2b
+            if Gem::Version.new(Fluent::VERSION) >= Gem::Version.new('1.7.0')
+              @counter_mutex ||= Mutex.new
+              @counter_mutex.synchronize{ @num_errors += 1 }
+            else
+              @counters_monitor ||= Monitor.new
+              @counters_monitor.synchronize{ @num_errors += 1 }
+            end
           end
 
           class Backoff

data/lib/fluent/plugin/kinesis_helper/client.rb

@@ -46,6 +46,20 @@ module Fluent
             desc "A URL for a regional STS API endpoint, the default is global"
             config_param :sts_endpoint_url, :string, default: nil
           end
+          # Refer to the following link for additional parameters that could be added:
+          # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
+          config_section :web_identity_credentials, multi: false do
+            desc "The Amazon Resource Name (ARN) of the role to assume"
+            config_param :role_arn, :string
+            desc "An identifier for the assumed role session"
+            config_param :role_session_name, :string
+            desc "The absolute path to the file on disk containing the OIDC token"
+            config_param :web_identity_token_file, :string, default: nil #required
+            desc "An IAM policy in JSON format"
+            config_param :policy, :string, default: nil
+            desc "The duration, in seconds, of the role session (900-43200)"
+            config_param :duration_seconds, :time, default: nil
+          end
           config_section :instance_profile_credentials, multi: false do
             desc "Number of times to retry when retrieving credentials"
             config_param :retries, :integer, default: nil
@@ -149,6 +163,17 @@ module Fluent
                 credentials_options[:client] = Aws::STS::Client.new(region: @region)
             end
             options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+          when @web_identity_credentials
+            c = @web_identity_credentials
+            credentials_options[:role_arn] = c.role_arn
+            credentials_options[:role_session_name] = c.role_session_name
+            credentials_options[:web_identity_token_file] = c.web_identity_token_file
+            credentials_options[:policy] = c.policy if c.policy
+            credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
+            if @region
+              credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+            end
+            options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
           when @instance_profile_credentials
             c = @instance_profile_credentials
             credentials_options[:retries] = c.retries if c.retries

data/lib/fluent/plugin/kinesis_helper/compression.rb

@@ -0,0 +1,27 @@
+require "stringio"
+require "zlib"
+
+class Stream < StringIO
+  def initialize(*)
+    super
+    set_encoding "BINARY"
+  end
+
+  def close
+    rewind;
+  end
+end
+
+class Gzip
+  def self.compress(string, level = Zlib::DEFAULT_COMPRESSION, strategy = Zlib::DEFAULT_STRATEGY)
+    output = Stream.new
+    gz = Zlib::GzipWriter.new(output, level, strategy)
+    gz.write(string)
+    gz.close
+    output.string
+  end
+
+  def self.decompress(string)
+    Zlib::GzipReader.wrap(StringIO.new(string), &:read)
+  end
+end

data/lib/fluent/plugin/out_kinesis_firehose.rb

@@ -45,7 +45,7 @@ module Fluent
 
       def write(chunk)
         delivery_stream_name = extract_placeholders(@delivery_stream_name, chunk)
-        write_records_batch(chunk) do |batch|
+        write_records_batch(chunk, delivery_stream_name) do |batch|
           records = batch.map{|(data)|
             { data: data }
           }

data/lib/fluent/plugin/out_kinesis_streams.rb

@@ -42,7 +42,7 @@ module Fluent
 
       def write(chunk)
         stream_name = extract_placeholders(@stream_name, chunk)
-        write_records_batch(chunk) do |batch|
+        write_records_batch(chunk, stream_name) do |batch|
           records = batch.map{|(data, partition_key)|
             { data: data, partition_key: partition_key }
           }

data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb

@@ -44,7 +44,7 @@ module Fluent
 
       def write(chunk)
         stream_name = extract_placeholders(@stream_name, chunk)
-        write_records_batch(chunk) do |batch|
+        write_records_batch(chunk, stream_name) do |batch|
           key = @partition_key_generator.call
           records = batch.map{|(data)|data}
           client.put_records(

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '3.2.3'
+  VERSION = '3.4.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 3.2.3
+  version: 3.4.2
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-15 00:00:00.000000000 Z
+date: 2021-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -101,9 +101,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -111,9 +108,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.12'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -290,22 +284,31 @@ files:
 - benchmark/task.rake
 - fluent-plugin-kinesis.gemspec
 - gemfiles/Gemfile.fluentd-0.14.22
-- gemfiles/Gemfile.fluentd-1.11.1
+- gemfiles/Gemfile.fluentd-1.13.3
 - gemfiles/Gemfile.td-agent-3.1.0
 - gemfiles/Gemfile.td-agent-3.1.1
 - gemfiles/Gemfile.td-agent-3.2.0
 - gemfiles/Gemfile.td-agent-3.2.1
 - gemfiles/Gemfile.td-agent-3.3.0
+- gemfiles/Gemfile.td-agent-3.4.0
 - gemfiles/Gemfile.td-agent-3.4.1
 - gemfiles/Gemfile.td-agent-3.5.0
 - gemfiles/Gemfile.td-agent-3.5.1
 - gemfiles/Gemfile.td-agent-3.6.0
+- gemfiles/Gemfile.td-agent-3.7.0
 - gemfiles/Gemfile.td-agent-3.7.1
 - gemfiles/Gemfile.td-agent-3.8.0
+- gemfiles/Gemfile.td-agent-3.8.1
+- gemfiles/Gemfile.td-agent-4.0.0
+- gemfiles/Gemfile.td-agent-4.0.1
+- gemfiles/Gemfile.td-agent-4.1.0
+- gemfiles/Gemfile.td-agent-4.1.1
+- gemfiles/Gemfile.td-agent-4.2.0
 - lib/fluent/plugin/kinesis.rb
 - lib/fluent/plugin/kinesis_helper/aggregator.rb
 - lib/fluent/plugin/kinesis_helper/api.rb
 - lib/fluent/plugin/kinesis_helper/client.rb
+- lib/fluent/plugin/kinesis_helper/compression.rb
 - lib/fluent/plugin/out_kinesis_firehose.rb
 - lib/fluent/plugin/out_kinesis_streams.rb
 - lib/fluent/plugin/out_kinesis_streams_aggregated.rb
@@ -329,7 +332,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin that sends events to Amazon Kinesis.

data/gemfiles/Gemfile.fluentd-1.11.1

@@ -1,20 +0,0 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
-gemspec path: ".."
-
-gem "fluentd", "1.11.1"