fluent-plugin-kinesis 3.2.2 → 3.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f410e3829237caf34033008ff210d278bd9e11e127e48f4611042065b2b5ff2
-  data.tar.gz: 8085f421bf1ca76e6d2cbc1e91c9168ddef470427591c4e386f204f20e9214e9
+  metadata.gz: 90a9718baa3e6849ce6ad7be32104672dcdf8993940c71d2d9fb666ce64b35e2
+  data.tar.gz: 064de8d0bbe66a8b8a8f4102bb686487fc0d7294904399dd745ce641b593c9bf
 SHA512:
-  metadata.gz: 2ea0c5855780a12ff28de449a5def77ca74b21d455a72098b2da8feb3e67847c4ceb4d562739f84dece5fa91f448d6a665838da19e539234458b3a0900bb768b
-  data.tar.gz: '0844b285486b2bb81e5d18bed60ebdee24dce74e4a1f00a28b0d6afa8af69f6c05fa016aa884424e8e61c232dc506c53ec1bf560f3b6ab228ce5f7141e95db17'
+  metadata.gz: 248f0fd2de87171ea58d878de9450edf95eb1dee58e31bfaf1aa716080e3921f68aa8ebdac2cbf052759a7e4262d5e598c5380c0c75edef503c54c8e760f1d3e
+  data.tar.gz: 79128aa4b4d07372ecc8d53d2f7196fa442d2c7c1694e215daca6cac7aae4d097af3b0512ddeae74708c053d6fb92fdce3c87811f4c0a2f5060b2c76649597e4

data/.gitignore

@@ -4,6 +4,7 @@
 /gemfiles/Gemfile*.lock
 /_yardoc/
 /coverage/
+/vendor/
 /doc/
 /pkg/
 /spec/reports/

data/.travis.yml

@@ -2,34 +2,55 @@ language: ruby
 os: linux
 sudo: false
 
-gemfile:
-  - Gemfile
-  - gemfiles/Gemfile.td-agent-3.5.1
-
 # Test with supported td-agent versions
-# https://support.treasuredata.com/hc/en-us/articles/360001479187-The-td-agent-ChangeLog
+# https://docs.fluentd.org/quickstart/support#change-logs
+# https://docs.treasuredata.com/display/public/PD/The+td-agent+Change+Log
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
 matrix:
   include:
-    - rvm: 2.6.5
-      gemfile: gemfiles/Gemfile.fluentd-1.9.1 # Latest Fluentd
-    - rvm: 2.4.1
-      gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
-    - rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L20
-      gemfile: gemfiles/Gemfile.td-agent-3.1.0
-    - rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
-      gemfile: gemfiles/Gemfile.td-agent-3.1.1
-    - rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.2.0
-    - rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.1/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.2.1
-    - rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.3.0
-    - rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.4.1
-    - rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.5.0
-    - rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.1/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.5.1
+    - gemfile: gemfiles/Gemfile.fluentd-1.13.3 # Latest Fluentd / https://www.fluentd.org/download
+      rvm: 3.0.2 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads
+      dist: focal
+    - gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
+      rvm: 2.3.0 # Oldest supported Ruby
+    - gemfile: gemfiles/Gemfile.td-agent-3.1.0
+      rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L20
+    # - gemfile: gemfiles/Gemfile.td-agent-3.1.1
+    #   rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
+    # - gemfile: gemfiles/Gemfile.td-agent-3.2.0
+    #   rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.2.1
+      rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.1/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.3.0
+      rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/config/projects/td-agent3.rb#L22
+    # - gemfile: gemfiles/Gemfile.td-agent-3.4.0
+    #   rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.4.1
+      rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L22
+    # - gemfile: gemfiles/Gemfile.td-agent-3.5.0
+    #   rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.5.1
+      rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.1/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.6.0
+      rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/config/projects/td-agent3.rb#L22
+    # - gemfile: gemfiles/Gemfile.td-agent-3.7.0
+    #   rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/commit/da5cb39ec5396f652cd30bad5bb8ef4ec19071a9#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R22
+    - gemfile: gemfiles/Gemfile.td-agent-3.7.1
+      rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.8.0 # Latest official release of td-agent v3
+      rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.8.1 # Maintenance release of td-agent v3
+      rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R22
+    # - gemfile: gemfiles/Gemfile.td-agent-4.0.0
+    #   rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L16
+    - gemfile: gemfiles/Gemfile.td-agent-4.0.1
+      rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L16
+    # - gemfile: gemfiles/Gemfile.td-agent-4.1.0
+    #   rvm: 2.7.2 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L19
+    - gemfile: gemfiles/Gemfile.td-agent-4.1.1
+      rvm: 2.7.3 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L17
+    - gemfile: gemfiles/Gemfile.td-agent-4.2.0
+      rvm: 2.7.4 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L21
   fast_finish: true
 
 script: bundle exec rake test

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # CHANGELOG
 
+## 3.4.1
+
+- Enhancement - Add stream name to debug logs to identify : [#214](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/214)
+- Enhancement - Add td-agent 4.1.1, 4.2.0 and Fluentd 1.13.3 with Ruby 3.0.2 to test cases
+
+## 3.4.0
+
+- Enhancement - Enable to monitor batch request failure and retries : [#150](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/150) [#211](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/211)
+- Enhancement - Make sleep reliable by measuring actual slept time : [#162](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/162)
+- Enhancement - Add td-agent 4.1.0 and Fluentd 1.12.3 to test cases
+
+## 3.3.0
+
+- Feature - Add web_identity_credentials configuration for IRSA : [#208](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/208) [#209](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/209)
+- Enhancement - Remove strict gem pinning of google-protobuf to support Ruby 2.7 : [#199](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/199) [#206](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/206)
+- Enhancement - Add td-agent v4 (4.0.1) and Fluentd 1.11.2 to test cases
+
+## 3.2.3
+
+- Enhancement - Add placeholder support for delivery_stream_name of kinesis_firehose : [#203](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/203) [#204](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/204)
+
 ## 3.2.2
 
 - Enhancement - Make more strict gem pinning to deal with google-protobuf requiring Ruby 2.5+ : [#199](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/199)

data/CONTRIBUTORS.txt

@@ -4,3 +4,5 @@ to the Fluent Plugin for Amazon Kinesis Streams and Amazon Kinesis Firehose. Giv
 Genki Sugawara
 sgwr_dts@yahoo.co.jp
 https://github.com/winebarrel
+https://github.com/adammw
+https://github.com/cosmo0920

data/README.md

@@ -1,7 +1,6 @@
 # Fluent plugin for Amazon Kinesis
 
-[![Gitter](https://badges.gitter.im/awslabs/aws-fluent-plugin-kinesis.svg)](https://gitter.im/awslabs/aws-fluent-plugin-kinesis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
-[![Build Status](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis)
+[![Build Status](https://api.travis-ci.com/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://app.travis-ci.com/github/awslabs/aws-fluent-plugin-kinesis)
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 [![Gem Downloads](https://img.shields.io/gem/dt/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 
@@ -186,6 +185,50 @@ It should be added to assume_role_credentials configuration stanza in the next f
 
 STS API endpoint url. This can be used to override the default global STS API endpoint of sts.amazonaws.com. Using regional endpoints may be preferred to reduce latency, and are required if utilizing a PrivateLink VPC Endpoint for STS API calls.
 
+
+### web_identity_credentials
+
+Similar to the assume_role_credentials, but for usage in EKS.
+
+    <match *>
+      @type kinesis_streams
+
+      <web_identity_credentials>
+        role_arn          ROLE_ARN
+        role_session_name ROLE_SESSION_NAME
+        web_identity_token_file AWS_WEB_IDENTITY_TOKEN_FILE
+      </web_identity_credentials>
+    </match>
+
+See also:
+
+*   [Using IAM Roles - AWS Identity and Access Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
+*   [IAM Roles For Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html)
+*   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
+*   [Aws::AssumeRoleWebIdentityCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleWebIdentityCredentials.html)
+
+**role_arn (required)**
+
+The Amazon Resource Name (ARN) of the role to assume.
+
+**role_session_name (required)**
+
+An identifier for the assumed role session.
+
+**web_identity_token_file (required)**
+
+The absolute path to the file on disk containing the OIDC token
+
+**policy**
+
+An IAM policy in JSON format.
+
+**duration_seconds**
+
+The duration, in seconds, of the role session. The value can range from
+900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
+is set to 3600 seconds (1 hour).
+
 ### instance_profile_credentials
 
 Retrieve temporary security credentials via HTTP request. This is useful on EC2 instance.
@@ -365,6 +408,16 @@ Default:
 - `kinesis_firehose`: 4 MB
 - `kinesis_streams_aggregated`: 1 MB
 
+### drop_failed_records_after_batch_request_retries
+Boolean, default `true`.
+
+If *drop_failed_records_after_batch_request_retries* is enabled (default), the plugin will drop failed records when batch request fails after retrying max times configured as *retries_on_batch_request*. This dropping can be monitored from [monitor_agent](https://docs.fluentd.org/input/monitor_agent) or [fluent-plugin-prometheus](https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus) as *retry_count* or *num_errors* metrics.
+
+If *drop_failed_records_after_batch_request_retries* is disabled, the plugin will raise error and return chunk to Fluentd buffer when batch request fails after retrying max times. Fluentd will retry to send chunk records according to retry config in [Buffer Section](https://docs.fluentd.org/configuration/buffer-section). Note that this retryng may create duplicate records since [PutRecords API](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html) of Kinesis Data Streams and [PutRecordBatch API](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecordBatch.html) of Kinesis Data Firehose may return a partially successful response.
+
+### monitor_num_of_batch_request_retries
+Boolean, default `false`. If enabled, the plugin will increment *retry_count* monitoring metrics after internal retrying to send batch request. This configuration enables you to monitor [ProvisionedThroughputExceededException](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html) from [monitor_agent](https://docs.fluentd.org/input/monitor_agent) or [fluent-plugin-prometheus](https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus). Note that *retry_count* metrics will be counted by the plugin in addition to original Fluentd buffering mechanism if *monitor_num_of_batch_request_retries* is enabled.
+
 ## Configuration: kinesis_streams
 Here are `kinesis_streams` specific configurations.
 
@@ -404,6 +457,30 @@ Here are `kinesis_firehose` specific configurations.
 ### delivery_stream_name
 Name of the delivery stream to put data.
 
+As of Fluentd v1, built-in placeholders are supported. Now, you can also use built-in placeholders for this parameter.
+
+**NOTE:**
+Built-in placeholders require target key information in your buffer section attributes.
+
+e.g.)
+
+When you specify the following `delivery_stream_name` configuration with built-in placeholder:
+
+```aconf
+delivery_stream_name "${$.kubernetes.annotations.kinesis_firehose_streams}"
+```
+
+you ought to specify the corresponding attributes in buffer section:
+
+```aconf
+# $.kubernetes.annotations.kinesis_firehose_streams needs to be set in buffer attributes
+<buffer $.kubernetes.annotations.kinesis_firehose_streams>
+   # ...
+</buffer>
+```
+
+For more details, refer [Placeholders section in the official Fluentd document](https://docs.fluentd.org/configuration/buffer-section#placeholders).
+
 ### append_new_line
 Boolean. Default `true`. If it is enabled, the plugin adds new line character (`\n`) to each serialized record.  
 Before appending `\n`, plugin calls chomp and removes separator from the end of each record as [chomp_record](#chomp_record) is `true`. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) output with default configuration ([append_new_line](#append_new_line) is `true`). If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).

data/fluent-plugin-kinesis.gemspec

@@ -55,7 +55,7 @@ Gem::Specification.new do |spec|
   #   https://github.com/aws/aws-sdk-ruby/commit/e26577d2a426a4be79cd2d9edc1a4a4176e388ba#diff-10f50e27b30c3dc522b3c25db5782e2e
   spec.add_dependency "aws-sdk-firehose", "~> 1", "!= 1.5", "!= 1.9", "!= 1.15"
 
-  spec.add_dependency "google-protobuf", "~> 3", "< 3.12"
+  spec.add_dependency "google-protobuf", "~> 3"
 
   spec.add_development_dependency "bundler", ">= 1.10"
   spec.add_development_dependency "rake", ">= 10.0"

data/gemfiles/Gemfile.fluentd-0.14.22

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.fluentd-1.13.3

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+gem "fluentd", "1.13.3"

data/gemfiles/Gemfile.td-agent-3.1.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.1.1

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.2.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.2.1

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.3.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.4.0

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.4.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.4.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.149.0"
+gem "aws-sigv4", "1.1.0"
+gem "aws-sdk-core", "3.48.3"
+gem "aws-sdk-kms", "1.16.0"
+gem "aws-sdk-sqs", "1.13.0"
+gem "aws-sdk-s3", "1.36.0"
+gem "fluent-plugin-s3", "1.1.9"

data/gemfiles/Gemfile.td-agent-3.4.1

@@ -1,29 +1,12 @@
-#
-# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
 gemspec path: ".."
 
 # Specify related gems for td-agent v3.4.1
-# td-agent v3.4.0 has the same dependencies
 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L27
-# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/config/projects/td-agent3.rb#L27
 gem "fluentd", "1.4.2"
 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/plugin_gems.rb#L16-L23
-# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/plugin_gems.rb#L16-L23
 gem "jmespath", "1.4.0"
 gem "aws-partitions", "1.149.0"
 gem "aws-sigv4", "1.1.0"

data/gemfiles/Gemfile.td-agent-3.5.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.5.1

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.6.0

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.6.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.9.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.273.0"
+gem "aws-sigv4", "1.1.0"
+gem "aws-sdk-core", "3.90.0"
+gem "aws-sdk-kms", "1.29.0"
+gem "aws-sdk-sqs", "1.23.1"
+gem "aws-sdk-s3", "1.60.2"
+gem "fluent-plugin-s3", "1.3.0"

data/gemfiles/Gemfile.td-agent-3.7.0

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.7.0
+# https://docs.treasuredata.com/display/public/PD/The+td-agent+Change+Log
+gem "fluentd", "1.10.0"
+# https://github.com/treasure-data/omnibus-td-agent/commit/74920cb2a55e40195961a58c6b6de4bd5b3f1f87#diff-8d45eaa8758f7591f884c9e9917ea96105331a1daf5a891b4645b89e120f2d69R16-R23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.288.0"
+gem "aws-sigv4", "1.1.1"
+gem "aws-sdk-core", "3.92.0"
+gem "aws-sdk-kms", "1.30.0"
+gem "aws-sdk-sqs", "1.24.0"
+gem "aws-sdk-s3", "1.61.1"
+gem "fluent-plugin-s3", "1.3.0"

data/gemfiles/Gemfile.td-agent-3.7.1

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.7.1
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.10.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.297.0"
+gem "aws-sigv4", "1.1.1"
+gem "aws-sdk-core", "3.94.0"
+gem "aws-sdk-kms", "1.30.0"
+gem "aws-sdk-sqs", "1.24.0"
+gem "aws-sdk-s3", "1.63.0"
+gem "fluent-plugin-s3", "1.3.1"

data/gemfiles/Gemfile.td-agent-3.8.0

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.8.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.11.1"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.332.0"
+gem "aws-sigv4", "1.2.0"
+gem "aws-sdk-core", "3.100.0"
+gem "aws-sdk-kms", "1.34.1"
+gem "aws-sdk-sqs", "1.28.0"
+gem "aws-sdk-s3", "1.69.1"
+gem "fluent-plugin-s3", "1.3.2"

data/gemfiles/Gemfile.td-agent-3.8.1

@@ -0,0 +1,18 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Maintenance release of td-agent v3
+# Specify related gems for td-agent v3.8.1
+# https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R27
+gem "fluentd", "1.11.5"
+# https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-8d45eaa8758f7591f884c9e9917ea96105331a1daf5a891b4645b89e120f2d69R19-R25
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.399.0"
+gem "aws-sigv4", "1.2.2"
+gem "aws-sdk-core", "3.109.3"
+gem "aws-sdk-kms", "1.39.0"
+gem "aws-sdk-sqs", "1.34.0"
+gem "aws-sdk-s3", "1.85.0"
+gem "fluent-plugin-s3", "1.4.0"

data/gemfiles/Gemfile.td-agent-4.0.0

@@ -0,0 +1,25 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.0.0
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L4
+gem "fluentd", "1.11.1"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L31
+gem "prometheus-client", "0.9.0"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L19-L20
+gem "fluent-plugin-prometheus", "1.8.0"
+gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L28
+gem "jmespath", "1.4.0"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L8-L13
+gem "aws-partitions", "1.337.0"
+gem "aws-sdk-core", "3.102.1"
+gem "aws-sdk-kms", "1.35.0"
+gem "aws-sdk-s3", "1.72.0"
+gem "aws-sdk-sqs", "1.29.0"
+gem "aws-sigv4", "1.2.1"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L23
+gem "fluent-plugin-s3", "1.3.3"

data/gemfiles/Gemfile.td-agent-4.0.1

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.0.1
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L4
+gem "fluentd", "1.11.2"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/gemfiles/linux/Gemfile#L35-L42
+gem "prometheus-client", "0.9.0"
+gem "fluent-plugin-prometheus", "1.8.2"
+gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.352.0"
+gem "aws-sdk-core", "3.104.3"
+gem "aws-sdk-kms", "1.36.0"
+gem "aws-sdk-sqs", "1.30.0"
+gem "aws-sigv4", "1.2.1"
+gem "aws-sdk-s3", "1.75.0"
+gem "fluent-plugin-s3", "1.4.0"

data/gemfiles/Gemfile.td-agent-4.1.0

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.1.0
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L4
+gem "fluentd", "1.12.1"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/Gemfile#L32-L42
+gem "prometheus-client", "0.9.0"
+gem "fluent-plugin-prometheus", "1.8.5"
+gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.427.0"
+gem "aws-sdk-core", "3.112.0"
+gem "aws-sdk-kms", "1.42.0"
+gem "aws-sdk-sqs", "1.36.0"
+gem "aws-sigv4", "1.2.2"
+gem "aws-sdk-s3", "1.88.1"
+gem "fluent-plugin-s3", "1.5.1"

data/gemfiles/Gemfile.td-agent-4.1.1

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.1.1
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L4
+gem "fluentd", "1.12.3"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/Gemfile#L32-L42
+gem "prometheus-client", "0.9.0"
+gem "fluent-plugin-prometheus", "1.8.5"
+gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.446.0"
+gem "aws-sdk-core", "3.114.0"
+gem "aws-sdk-kms", "1.43.0"
+gem "aws-sdk-sqs", "1.38.0"
+gem "aws-sigv4", "1.2.3"
+gem "aws-sdk-s3", "1.93.1"
+gem "fluent-plugin-s3", "1.6.0"

data/gemfiles/Gemfile.td-agent-4.2.0

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.2.0
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L4
+gem "fluentd", "1.13.3"
+# https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/Gemfile#L42-L52
+gem "prometheus-client", "2.1.0"
+gem "fluent-plugin-prometheus", "2.0.1"
+gem "fluent-plugin-prometheus_pushgateway", "0.1.0"
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.478.0"
+gem "aws-sdk-core", "3.117.0"
+gem "aws-sdk-kms", "1.44.0"
+gem "aws-sdk-sqs", "1.40.0"
+gem "aws-sigv4", "1.2.4"
+gem "aws-sdk-s3", "1.96.1"
+gem "fluent-plugin-s3", "1.6.0"

data/lib/fluent/plugin/kinesis.rb

@@ -84,6 +84,10 @@ module Fluent
         true
       end
 
+      def formatted_to_msgpack_binary?
+        true
+      end
+
       private
 
       def data_formatter_create(conf)
@@ -142,15 +146,13 @@ module Fluent
         include Fluent::MessagePackFactory::Mixin
       end
 
-      def write_records_batch(chunk, &block)
+      def write_records_batch(chunk, stream_name, &block)
         unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
-        chunk.open do |io|
-          records = msgpack_unpacker(io).to_enum
-          split_to_batches(records) do |batch, size|
-            log.debug(sprintf "Write chunk %s / %3d records / %4d KB", unique_id, batch.size, size/1024)
-            batch_request_with_retry(batch, &block)
-            log.debug("Finish writing chunk")
-          end
+        records = chunk.to_enum(:msgpack_each)
+        split_to_batches(records) do |batch, size|
+          log.debug(sprintf "%s: Write chunk %s / %3d records / %4d KB", stream_name, unique_id, batch.size, size/1024)
+          batch_request_with_retry(batch, &block)
+          log.debug(sprintf "%s: Finish writing chunk", stream_name)
         end
       end
 

data/lib/fluent/plugin/kinesis_helper/api.rb

@@ -14,6 +14,7 @@
 
 require 'fluent_plugin_kinesis/version'
 require 'fluent/configurable'
+require 'benchmark'
 
 module Fluent
   module Plugin
@@ -40,10 +41,12 @@ module Fluent
         module BatchRequest
           module BatchRequestParams
             include Fluent::Configurable
-            config_param :retries_on_batch_request, :integer, default: 8
-            config_param :reset_backoff_if_success, :bool,    default: true
-            config_param :batch_request_max_count,  :integer, default: nil
-            config_param :batch_request_max_size,   :integer, default: nil
+            config_param :retries_on_batch_request,                        :integer, default: 8
+            config_param :reset_backoff_if_success,                        :bool,    default: true
+            config_param :batch_request_max_count,                         :integer, default: nil
+            config_param :batch_request_max_size,                          :integer, default: nil
+            config_param :drop_failed_records_after_batch_request_retries, :bool,    default: true
+            config_param :monitor_num_of_batch_request_retries,            :bool,    default: false
           end
 
           def self.included(mod)
@@ -96,11 +99,9 @@ module Fluent
                 wait_second = backoff.next
                 msg = 'Retrying to request batch. Retry count: %3d, Retry records: %3d, Wait seconds %3.2f' % [retry_count+1, failed_records.size, wait_second]
                 log.warn(truncate msg)
-                # TODO: sleep() doesn't wait the given seconds sometime.
-                # The root cause is unknown so far, so I'd like to add debug print only. It should be fixed in the future.
-                log.debug("#{Thread.current.object_id} sleep start")
-                sleep(wait_second)
-                log.debug("#{Thread.current.object_id} sleep finish")
+                # Increment num_errors to monitor batch request retries from "monitor_agent" or "fluent-plugin-prometheus"
+                increment_num_errors if @monitor_num_of_batch_request_retries
+                reliable_sleep(wait_second)
                 batch_request_with_retry(retry_records(failed_records), retry_count+1, backoff: backoff, &block)
               else
                 give_up_retries(failed_records)
@@ -108,6 +109,18 @@ module Fluent
             end
           end
 
+          # Sleep seems to not sleep as long as we ask it, our guess is that something wakes up the thread,
+          # so we keep on going to sleep if that happens.
+          # TODO: find out who is causing the sleep to be too short and try to make them stop it instead
+          def reliable_sleep(wait_second)
+            loop do
+              actual = Benchmark.realtime { sleep(wait_second) }
+              break if actual >= wait_second
+              log.error("#{Thread.current.object_id} sleep failed expected #{wait_second} but slept #{actual}")
+              wait_second -= actual
+            end
+          end
+
           def any_records_shipped?(res)
             results(res).size > failed_count(res)
           end
@@ -164,6 +177,49 @@ module Fluent
                 record[:original]
               ])
             }
+
+            if @drop_failed_records_after_batch_request_retries
+              # Increment num_errors to monitor batch request failure from "monitor_agent" or "fluent-plugin-prometheus"
+              increment_num_errors
+            else
+              # Raise error and return chunk to Fluentd for retrying
+              case request_type
+              # @see https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Kinesis/Client.html#put_records-instance_method
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Kinesis/Errors.html
+              when :streams, :streams_aggregated
+                provisioned_throughput_exceeded_records = failed_records.select { |record| record[:error_code] == 'ProvisionedThroughputExceededException' }
+                target_failed_record = provisioned_throughput_exceeded_records.first || failed_records.first
+                target_error = provisioned_throughput_exceeded_records.empty? ?
+                  Aws::Kinesis::Errors::ServiceError :
+                  Aws::Kinesis::Errors::ProvisionedThroughputExceededException
+              # @see https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Firehose/Client.html#put_record_batch-instance_method
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Firehose/Errors.html
+              when :firehose
+                service_unavailable_exception_records = failed_records.select { |record| record[:error_code] == 'ServiceUnavailableException' }
+                target_failed_record = service_unavailable_exception_records.first || failed_records.first
+                target_error = service_unavailable_exception_records.empty? ?
+                  Aws::Firehose::Errors::ServiceError :
+                  Aws::Firehose::Errors::ServiceUnavailableException
+              end
+              log.error("Raise #{target_failed_record[:error_code]} and return chunk to Fluentd buffer for retrying")
+              raise target_error.new(Seahorse::Client::RequestContext.new, target_failed_record[:error_message])
+            end
+          end
+
+          def increment_num_errors
+            # Prepare Fluent::Plugin::Output instance variables to count errors in this method.
+            # These instance variables are initialized here for possible future breaking changes of Fluentd.
+            @num_errors ||= 0
+            # @see https://github.com/fluent/fluentd/commit/d245454658d16170431d276fcd5849fb0d88ab2b
+            if Gem::Version.new(Fluent::VERSION) >= Gem::Version.new('1.7.0')
+              @counter_mutex ||= Mutex.new
+              @counter_mutex.synchronize{ @num_errors += 1 }
+            else
+              @counters_monitor ||= Monitor.new
+              @counters_monitor.synchronize{ @num_errors += 1 }
+            end
           end
 
           class Backoff

data/lib/fluent/plugin/kinesis_helper/client.rb

@@ -46,6 +46,20 @@ module Fluent
             desc "A URL for a regional STS API endpoint, the default is global"
             config_param :sts_endpoint_url, :string, default: nil
           end
+          # Refer to the following link for additional parameters that could be added:
+          # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
+          config_section :web_identity_credentials, multi: false do
+            desc "The Amazon Resource Name (ARN) of the role to assume"
+            config_param :role_arn, :string
+            desc "An identifier for the assumed role session"
+            config_param :role_session_name, :string
+            desc "The absolute path to the file on disk containing the OIDC token"
+            config_param :web_identity_token_file, :string, default: nil #required
+            desc "An IAM policy in JSON format"
+            config_param :policy, :string, default: nil
+            desc "The duration, in seconds, of the role session (900-43200)"
+            config_param :duration_seconds, :time, default: nil
+          end
           config_section :instance_profile_credentials, multi: false do
             desc "Number of times to retry when retrieving credentials"
             config_param :retries, :integer, default: nil
@@ -149,6 +163,17 @@ module Fluent
                 credentials_options[:client] = Aws::STS::Client.new(region: @region)
             end
             options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+          when @web_identity_credentials
+            c = @web_identity_credentials
+            credentials_options[:role_arn] = c.role_arn
+            credentials_options[:role_session_name] = c.role_session_name
+            credentials_options[:web_identity_token_file] = c.web_identity_token_file
+            credentials_options[:policy] = c.policy if c.policy
+            credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
+            if @region
+              credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+            end
+            options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
           when @instance_profile_credentials
             c = @instance_profile_credentials
             credentials_options[:retries] = c.retries if c.retries

data/lib/fluent/plugin/out_kinesis_firehose.rb

@@ -44,12 +44,13 @@ module Fluent
       end
 
       def write(chunk)
-        write_records_batch(chunk) do |batch|
+        delivery_stream_name = extract_placeholders(@delivery_stream_name, chunk)
+        write_records_batch(chunk, delivery_stream_name) do |batch|
           records = batch.map{|(data)|
             { data: data }
           }
           client.put_record_batch(
-            delivery_stream_name: @delivery_stream_name,
+            delivery_stream_name: delivery_stream_name,
             records: records,
           )
         end

data/lib/fluent/plugin/out_kinesis_streams.rb

@@ -42,7 +42,7 @@ module Fluent
 
       def write(chunk)
         stream_name = extract_placeholders(@stream_name, chunk)
-        write_records_batch(chunk) do |batch|
+        write_records_batch(chunk, stream_name) do |batch|
           records = batch.map{|(data, partition_key)|
             { data: data, partition_key: partition_key }
           }

data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb

@@ -44,7 +44,7 @@ module Fluent
 
       def write(chunk)
         stream_name = extract_placeholders(@stream_name, chunk)
-        write_records_batch(chunk) do |batch|
+        write_records_batch(chunk, stream_name) do |batch|
           key = @partition_key_generator.call
           records = batch.map{|(data)|data}
           client.put_records(

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '3.2.2'
+  VERSION = '3.4.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 3.2.2
+  version: 3.4.1
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-19 00:00:00.000000000 Z
+date: 2021-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -101,9 +101,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -111,9 +108,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.12'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -290,15 +284,26 @@ files:
 - benchmark/task.rake
 - fluent-plugin-kinesis.gemspec
 - gemfiles/Gemfile.fluentd-0.14.22
-- gemfiles/Gemfile.fluentd-1.9.1
+- gemfiles/Gemfile.fluentd-1.13.3
 - gemfiles/Gemfile.td-agent-3.1.0
 - gemfiles/Gemfile.td-agent-3.1.1
 - gemfiles/Gemfile.td-agent-3.2.0
 - gemfiles/Gemfile.td-agent-3.2.1
 - gemfiles/Gemfile.td-agent-3.3.0
+- gemfiles/Gemfile.td-agent-3.4.0
 - gemfiles/Gemfile.td-agent-3.4.1
 - gemfiles/Gemfile.td-agent-3.5.0
 - gemfiles/Gemfile.td-agent-3.5.1
+- gemfiles/Gemfile.td-agent-3.6.0
+- gemfiles/Gemfile.td-agent-3.7.0
+- gemfiles/Gemfile.td-agent-3.7.1
+- gemfiles/Gemfile.td-agent-3.8.0
+- gemfiles/Gemfile.td-agent-3.8.1
+- gemfiles/Gemfile.td-agent-4.0.0
+- gemfiles/Gemfile.td-agent-4.0.1
+- gemfiles/Gemfile.td-agent-4.1.0
+- gemfiles/Gemfile.td-agent-4.1.1
+- gemfiles/Gemfile.td-agent-4.2.0
 - lib/fluent/plugin/kinesis.rb
 - lib/fluent/plugin/kinesis_helper/aggregator.rb
 - lib/fluent/plugin/kinesis_helper/api.rb
@@ -326,7 +331,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin that sends events to Amazon Kinesis.

data/gemfiles/Gemfile.fluentd-1.9.1

@@ -1,20 +0,0 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
-gemspec path: ".."
-
-gem "fluentd", "1.9.1"