fluent-plugin-kinesis 3.1.0 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 529a0b5277cab5b82c165d2eaa4339c529381a17
-  data.tar.gz: fcc5161078a9311d511c5ecb95d4d2dfd9f078e4
+SHA256:
+  metadata.gz: 98a12a1569d674676d986f802935ca80df435b4cd2acc4dd15d1951c8b020671
+  data.tar.gz: 63238ffd8abeb4c038af46eefff7c528277fa2225c8ca9affbe2714f433a6417
 SHA512:
-  metadata.gz: 05661c7aa7080a5e693d58f4d16a07bbeb8f426bbddefacc7fcce78dfe411f098f1829ffd9bf0c5e253b79419605a00d49c1baf44248512e9f6190e97e8c4c1f
-  data.tar.gz: 7c4530a60cc37bbef7ca85559a07b0fee5038be2d84c195fd68900a7dba73485ae5b2d29aec1f0b585185a160f4317dc111a190951899a0451437b5978d96764
+  metadata.gz: c49fb3a4a2a6443375c52a7e416686188160e7a8e85d997c4d4105ee18b06c3e9d94bfcd25e42ef971a3128c55575a9296012a6a5343bc3c9d466a5083c8d78c
+  data.tar.gz: 6d2d7fd0012cfcb2282aa6982ff6897333155a0f43c19a90eb365b147f42aac7f16b06dd38924a16403494295031103569529f8fe42055547e077ca558581005

data/.gitignore

@@ -4,6 +4,7 @@
 /gemfiles/Gemfile*.lock
 /_yardoc/
 /coverage/
+/vendor/
 /doc/
 /pkg/
 /spec/reports/
@@ -11,3 +12,4 @@
 /.cache/
 /amazon-kinesis-producer-native-binarie*
 /.ruby-version
+/fluent-plugin-kinesis-*.gem

data/.travis.yml

@@ -3,23 +3,37 @@ os: linux
 sudo: false
 
 gemfile:
-  - Gemfile
-  - gemfiles/Gemfile.td-agent-3.3.0
+  - gemfiles/Gemfile.td-agent-3.8.0
+  - gemfiles/Gemfile.td-agent-4.0.1
 
 # Test with supported td-agent versions
-# https://support.treasuredata.com/hc/en-us/articles/360001479187-The-td-agent-ChangeLog
+# https://tddocs.atlassian.net/wiki/spaces/PD/pages/6291635/The+td-agent+ChangeLog
+# https://docs.fluentd.org/quickstart/td-agent-v2-vs-v3-vs-v4
 matrix:
   include:
-    - rvm: 2.4.1
-      gemfile: gemfiles/Gemfile.fluentd-0.14.10
+    - rvm: 2.7.2 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads/
+      gemfile: gemfiles/Gemfile.fluentd-1.11.2 # Latest Fluentd / https://www.fluentd.org/download
+    - rvm: 2.3.0 # Oldest supported Ruby
+      gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
     - rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
       gemfile: gemfiles/Gemfile.td-agent-3.1.1
-    - rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.2.0
     - rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.1/config/projects/td-agent3.rb#L22
       gemfile: gemfiles/Gemfile.td-agent-3.2.1
     - rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/config/projects/td-agent3.rb#L22
       gemfile: gemfiles/Gemfile.td-agent-3.3.0
+    - rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.4.1
+    - rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.1/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.5.1
+    - rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.6.0
+    - rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.7.1
+    - rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.8.0
+    - rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/td-agent/config.rb#L16
+      gemfile: gemfiles/Gemfile.td-agent-4.0.1
+      
   fast_finish: true
 
 script: bundle exec rake test

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 # CHANGELOG
 
+## 3.3.0
+
+- Feature - Add web_identity_credentials configuration for IRSA : [#208](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/208) [#209](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/209)
+- Enhancement - Remove strict gem pinning of google-protobuf to support Ruby 2.7 : [#199](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/199) [#206](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/206)
+- Enhancement - Add td-agent v4 (4.0.1) and Fluentd 1.11.2 to test cases
+
+## 3.2.3
+
+- Enhancement - Add placeholder support for delivery_stream_name of kinesis_firehose : [#203](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/203) [#204](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/204)
+
+## 3.2.2
+
+- Enhancement - Make more strict gem pinning to deal with google-protobuf requiring Ruby 2.5+ : [#199](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/199)
+- Bug - Fix MissingRegionError when http_proxy and endpoint_url are specified : [#197](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/197)
+
+## 3.2.1
+
+- Enhancement - Use Fluent::MessagePackFactory class methods instead of Mixin with Fluentd >= v1.8 : [#194](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/194) [#195](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/195)
+- Enhancement - Add td-agent 3.5.1 and Fluentd 1.9.1 to test cases
+
+## 3.2.0
+
+- Feature - Add placeholder support for stream names to send records to multiple streams : [#165](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/165) [#174](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/174)
+- Enhancement - Add sts_endpoint_url configuration parameter to support AWS STS regional endpoints : [#186](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/186)
+- Enhancement - Add aws_ses_token configuration parameter to use IAM with MFA and directly provided temporary credentials : [#166](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/166)
+- Dependency - Update gem dependency to Ruby 2.3.0+ and Fluentd 0.14.22+
+- Bug - Fix dependency problem on AWS SDK with td-agent v3.4.1
+
 ## 3.1.0
 
 - Feature - Add process_credentials configuration : [#178](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/178)

data/CONTRIBUTORS.txt

@@ -4,3 +4,5 @@ to the Fluent Plugin for Amazon Kinesis Streams and Amazon Kinesis Firehose. Giv
 Genki Sugawara
 sgwr_dts@yahoo.co.jp
 https://github.com/winebarrel
+https://github.com/adammw
+https://github.com/cosmo0920

data/README.md

@@ -1,11 +1,11 @@
 # Fluent plugin for Amazon Kinesis
 
-[![Gitter](https://badges.gitter.im/awslabs/aws-fluent-plugin-kinesis.svg)](https://gitter.im/awslabs/aws-fluent-plugin-kinesis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 [![Build Status](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis)
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
+[![Gem Downloads](https://img.shields.io/gem/dt/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 
 [Fluentd][fluentd] output plugin
-that sends events to [Amazon Kinesis Data Streams][streams] and [Amazon Kinesis Data Firehose][firehose]. Also it supports [KPL Aggregated record format][kpl]. This gem includes three output plugins respectively:
+that sends events to [Amazon Kinesis Data Streams][streams] and [Amazon Kinesis Data Firehose][firehose]. Also it supports [KPL Aggregated Record Format][kpl]. This gem includes three output plugins respectively:
 
 - `kinesis_streams`
 - `kinesis_firehose`
@@ -24,7 +24,7 @@ Or you can install this plugin for [td-agent][td-agent] as:
 
     td-agent-gem install fluent-plugin-kinesis
 
-If you would like to build by yourself and install, please see the section below. Your need [bundler][bundler] for this.
+If you would like to build by yourself and install, see the section below. Your need [bundler][bundler] for this.
 
 In case of using with Fluentd: Fluentd will be also installed via the process below.
 
@@ -50,8 +50,8 @@ Or just download specify your Ruby library path. Below is the sample for specify
     export RUBYLIB=$RUBYLIB:/path/to/aws-fluent-plugin-kinesis/lib
 
 ## Dependencies
- * Ruby 2.1.0+
- * Fluentd 0.14.10+
+ * Ruby 2.3.0+
+ * Fluentd 0.14.22+ (td-agent v3.1.0+)
 
 ## Basic Usage
 Here are general procedures for using this plugin:
@@ -82,7 +82,7 @@ Assume you use Amazon EC2 instances with Instance profile. If you want to use sp
       stream_name your_stream
       partition_key key  # Otherwise, use random partition key
     </match>
-For more detail, see [Configuration: kinesis_streams](#configuration-kinesis_streams)
+For more details, see [Configuration: kinesis_streams](#configuration-kinesis_streams).
 
 ### kinesis_firehose
     <match your_tag>
@@ -90,7 +90,7 @@ For more detail, see [Configuration: kinesis_streams](#configuration-kinesis_str
       region us-east-1
       delivery_stream_name your_stream
     </match>
-For more detail, see [Configuration: kinesis_firehose](#configuration-kinesis_firehose)
+For more details, see [Configuration: kinesis_firehose](#configuration-kinesis_firehose).
 
 ### kinesis_streams_aggregated
     <match your_tag>
@@ -100,10 +100,10 @@ For more detail, see [Configuration: kinesis_firehose](#configuration-kinesis_fi
       # Unlike kinesis_streams, there is no way to use dynamic partition key.
       # fixed_partition_key or random.
     </match>
-For more detail, see [Configuration: kinesis_streams_aggregated](#configuration-kinesis_streams_aggregated)
+For more details, see [Configuration: kinesis_streams_aggregated](#configuration-kinesis_streams_aggregated).
 
 ### For better throughput
-Add configuration like below:
+Add configurations like below:
 
       flush_interval 1
       chunk_limit_size 1m
@@ -111,12 +111,14 @@ Add configuration like below:
       flush_thread_burst_interval 0.01
       flush_thread_count 15
 
+When you use Fluent v1.0 (td-agent3), write these configurations in buffer section. For more details, see [Config: Buffer Section][fluentd-buffer-section].
+
 Note: Each value should be adjusted to your system by yourself.
 
 ## Configuration: Credentials
 To put records into Amazon Kinesis Data Streams or Firehose, you need to provide AWS security credentials somehow. Without specifying credentials in config file, this plugin automatically fetch credential just following AWS SDK for Ruby does (environment variable, shared profile, and instance profile).
 
-This plugin uses the same configuration in [fluent-plugin-s3][fluent-plugin-s3].
+This plugin uses the same configuration in [fluent-plugin-s3][fluent-plugin-s3], but also supports aws session tokens for temporary credentials. 
 
 **aws_key_id**
 
@@ -126,12 +128,16 @@ AWS access key id. This parameter is required when your agent is not running on
 
 AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role.
 
+**aws_ses_token**
+
+AWS session token. This parameter is optional, but can be provided if using MFA or temporary credentials when your agent is not running on EC2 instance with an IAM Role. 
+
 **aws_iam_retries**
 
 The number of attempts to make (with exponential backoff) when loading instance profile credentials from the EC2 metadata service using an IAM role. Defaults to 5 retries.
 
 ### assume_role_credentials
-Typically, you use AssumeRole for cross-account access or federation.
+Typically, you can use AssumeRole for cross-account access or federation.
 
     <match *>
       @type kinesis_streams
@@ -145,9 +151,9 @@ Typically, you use AssumeRole for cross-account access or federation.
 See also:
 
 *   [Using IAM Roles - AWS Identity and Access
-    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
-*   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
-*   [Aws::AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html)
+    Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
+*   [Aws::STS::Client](https://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
+*   [Aws::AssumeRoleCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html)
 
 **role_arn (required)**
 
@@ -175,6 +181,54 @@ Proxy url for proxying requests to amazon sts service api. This needs to be  set
 It should be added to assume_role_credentials configuration stanza in the next format:
     sts_http_proxy http://[username:password]@hostname:port
 
+**sts_endpoint_url**
+
+STS API endpoint url. This can be used to override the default global STS API endpoint of sts.amazonaws.com. Using regional endpoints may be preferred to reduce latency, and are required if utilizing a PrivateLink VPC Endpoint for STS API calls.
+
+
+### web_identity_credentials
+
+Similar to the assume_role_credentials, but for usage in EKS.
+
+    <match *>
+      @type kinesis_streams
+
+      <web_identity_credentials>
+        role_arn          ROLE_ARN
+        role_session_name ROLE_SESSION_NAME
+        web_identity_token_file AWS_WEB_IDENTITY_TOKEN_FILE
+      </web_identity_credentials>
+    </match>
+
+See also:
+
+*   [Using IAM Roles - AWS Identity and Access Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
+*   [IAM Roles For Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html)
+*   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
+*   [Aws::AssumeRoleWebIdentityCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleWebIdentityCredentials.html)
+
+**role_arn (required)**
+
+The Amazon Resource Name (ARN) of the role to assume.
+
+**role_session_name (required)**
+
+An identifier for the assumed role session.
+
+**web_identity_token_file (required)**
+
+The absolute path to the file on disk containing the OIDC token
+
+**policy**
+
+An IAM policy in JSON format.
+
+**duration_seconds**
+
+The duration, in seconds, of the role session. The value can range from
+900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
+is set to 3600 seconds (1 hour).
+
 ### instance_profile_credentials
 
 Retrieve temporary security credentials via HTTP request. This is useful on EC2 instance.
@@ -190,11 +244,11 @@ Retrieve temporary security credentials via HTTP request. This is useful on EC2
 
 See also:
 
-*   [Aws::InstanceProfileCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/InstanceProfileCredentials.html)
+*   [Aws::InstanceProfileCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/InstanceProfileCredentials.html)
 *   [Temporary Security Credentials - AWS Identity and Access
-    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
+    Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
 *   [Instance Metadata and User Data - Amazon Elastic Compute
-    Cloud](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
+    Cloud](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
 
 **retries**
 
@@ -231,7 +285,7 @@ This loads AWS access credentials from local ini file. This is useful for local
 
 See also:
 
-*   [Aws::SharedCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html)
+*   [Aws::SharedCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html)
 
 **path**
 
@@ -265,7 +319,7 @@ Command to be executed as an external process.
 ## Configuration: Format
 
 ### format (section)
-This plugin use `Fluent::TextFormatter` to serialize record to string. For more detail, see [formatter.rb]. By default, it uses `json` formatter same as specific like below:
+This plugin uses `Fluent::TextFormatter` to serialize record to string. See [formatter.rb] for more details. By default, it uses `json` formatter same as specific like below:
 
     <match *>
       @type kinesis_streams
@@ -275,10 +329,12 @@ This plugin use `Fluent::TextFormatter` to serialize record to string. For more
       </format>
     </match>
 
+For other configurations of `json` formatter, see [json formatter plugin][fluentd-formatter-json].
+
 ### inject (section)
-This plugin use `Fluent::TimeFormatter` and other injection configurations. For more detail, see [inject.rb]. This section only works with Fluentd v0.14.
+This plugin uses `Fluent::TimeFormatter` and other injection configurations. See [inject.rb] for more details.
 
-For example, the config below will add `time` field whose value is eventtime with nanosecond and `tag` field whose value is its tag. (Mostly same as `include_time key true` and `include_tag_key true`)
+For example, the config below will add `time` field whose value is event time with nanosecond and `tag` field whose value is its tag.
 
     <match *>
       @type kinesis_streams
@@ -293,16 +349,6 @@ By default, `time_type string` and `time_format %Y-%m-%dT%H:%M:%S.%N%z` are alre
 
 Also, there are some format related options below:
 
-### include_time_key
-**Notice** With Fluentd v0.14, you should use inject section.
-
-Defalut `false`. If you want to include `time` field in your record, set `true`.
-
-### include_tag_key
-**Notice** With Fluentd v0.14, you should use inject section.
-
-Defalut `false`. If you want to include `tag` field in your record, set `true`.
-
 ### data_key
 If your record contains a field whose string should be sent to Amazon Kinesis directly (without formatter), use this parameter to specify the field. In that case, other fields than **data_key** are thrown away and never sent to Amazon Kinesis. Default `nil`, which means whole record will be formatted and sent.
 
@@ -368,6 +414,30 @@ Here are `kinesis_streams` specific configurations.
 ### stream_name
 Name of the stream to put data.
 
+As of Fluentd v1, built-in placeholders are supported. Now, you can also use built-in placeholders for this parameter.
+
+**NOTE:**
+Built-in placeholders require target key information in your buffer section attributes.
+
+e.g.)
+
+When you specify the following `stream_name` configuration with built-in placeholder:
+
+```aconf
+stream_name "${$.kubernetes.annotations.kinesis_streams}"
+```
+
+you ought to specify the corresponding attributes in buffer section:
+
+```aconf
+# $.kubernetes.annotations.kinesis_streams needs to be set in buffer attributes
+<buffer $.kubernetes.annotations.kinesis_streams>
+   # ...
+</buffer>
+```
+
+For more details, refer [Placeholders section in the official Fluentd document](https://docs.fluentd.org/configuration/buffer-section#placeholders).
+
 ### partition_key
 A key to extract partition key from JSON object. Default `nil`, which means partition key will be generated randomly.
 
@@ -377,6 +447,30 @@ Here are `kinesis_firehose` specific configurations.
 ### delivery_stream_name
 Name of the delivery stream to put data.
 
+As of Fluentd v1, built-in placeholders are supported. Now, you can also use built-in placeholders for this parameter.
+
+**NOTE:**
+Built-in placeholders require target key information in your buffer section attributes.
+
+e.g.)
+
+When you specify the following `delivery_stream_name` configuration with built-in placeholder:
+
+```aconf
+delivery_stream_name "${$.kubernetes.annotations.kinesis_firehose_streams}"
+```
+
+you ought to specify the corresponding attributes in buffer section:
+
+```aconf
+# $.kubernetes.annotations.kinesis_firehose_streams needs to be set in buffer attributes
+<buffer $.kubernetes.annotations.kinesis_firehose_streams>
+   # ...
+</buffer>
+```
+
+For more details, refer [Placeholders section in the official Fluentd document](https://docs.fluentd.org/configuration/buffer-section#placeholders).
+
 ### append_new_line
 Boolean. Default `true`. If it is enabled, the plugin adds new line character (`\n`) to each serialized record.  
 Before appending `\n`, plugin calls chomp and removes separator from the end of each record as [chomp_record](#chomp_record) is `true`. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) output with default configuration ([append_new_line](#append_new_line) is `true`). If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).
@@ -387,6 +481,30 @@ Here are `kinesis_streams_aggregated` specific configurations.
 ### stream_name
 Name of the stream to put data.
 
+As of Fluentd v1, built-in placeholders are supported. Now, you can also use built-in placeholders for this parameter.
+
+**NOTE:**
+Built-in placeholders require target key information in your buffer section attributes.
+
+e.g.)
+
+When you specify the following `stream_name` configuration with built-in placeholder:
+
+```aconf
+stream_name "${$.kubernetes.annotations.kinesis_streams_aggregated}"
+```
+
+you ought to specify the corresponding attributes in buffer section:
+
+```aconf
+# $.kubernetes.annotations.kinesis_streams_aggregated needs to be set in buffer attributes
+<buffer $.kubernetes.annotations.kinesis_streams_aggregated>
+   # ...
+</buffer>
+```
+
+For more details, refer [Placeholders section in the official Fluentd document](https://docs.fluentd.org/configuration/buffer-section#placeholders).
+
 ### fixed_partition_key
 A value of fixed partition key. Default `nil`, which means partition key will be generated randomly.
 
@@ -414,17 +532,18 @@ Bug reports and pull requests are welcome on [GitHub][github].
 * [Amazon Kinesis Data Streams Developer Guide](http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html)
 * [Amazon Kinesis Data Firehose Developer Guide](http://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
 
-[fluentd]: http://fluentd.org/
+[fluentd]: https://www.fluentd.org/
 [streams]: https://aws.amazon.com/kinesis/streams/
 [firehose]: https://aws.amazon.com/kinesis/firehose/
 [kpl]: https://github.com/awslabs/amazon-kinesis-producer/blob/master/aggregation-format.md
-[td-agent]: https://github.com/treasure-data/td-agent
-[bundler]: http://bundler.io/
-[region]: http://docs.aws.amazon.com/general/latest/gr/rande.html#ak_region
-[fluentd_buffer]: http://docs.fluentd.org/articles/buffer-plugin-overview
+[td-agent]: https://github.com/treasure-data/omnibus-td-agent
+[bundler]: https://bundler.io/
+[region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#ak_region
+[fluentd-buffer-section]: https://docs.fluentd.org/configuration/buffer-section
+[fluentd-formatter-json]: https://docs.fluentd.org/formatter/json
 [github]: https://github.com/awslabs/aws-fluent-plugin-kinesis
 [formatter.rb]: https://github.com/fluent/fluentd/blob/master/lib/fluent/formatter.rb
 [inject.rb]: https://github.com/fluent/fluentd/blob/master/lib/fluent/plugin_helper/inject.rb
-[fluentd-doc-kinesis]: http://docs.fluentd.org/articles/kinesis-stream
+[fluentd-doc-kinesis]: https://docs.fluentd.org/how-to-guides/kinesis-stream
 [fluent-plugin-s3]: https://github.com/fluent/fluent-plugin-s3
 [v1-readme]: https://github.com/awslabs/aws-fluent-plugin-kinesis/blob/v1/README.md

data/fluent-plugin-kinesis.gemspec

@@ -28,9 +28,9 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = '>= 2.1'
+  spec.required_ruby_version = '>= 2.3'
 
-  spec.add_dependency "fluentd", ">= 0.14.10", "< 2"
+  spec.add_dependency "fluentd", ">= 0.14.22", "< 2"
 
   # This plugin is sometimes used with s3 plugin, so watch out for conflicts
   #   https://rubygems.org/gems/fluent-plugin-s3
@@ -40,12 +40,20 @@ Gem::Specification.new do |spec|
   #   NoMethodError: undefined method `event=' for #<Seahorse::Model::Shapes::ShapeRef:*>
   #   https://github.com/aws/aws-sdk-ruby/commit/03d60f9d3d821e645bd2a3efca066f37350ef906#diff-c69f15af8ea3eb9ab152659476e04608R401
   #   https://github.com/aws/aws-sdk-ruby/commit/571c2d0e5ff9c24ff72893a08a74790db591fb57#diff-a55155f04aa6559460a0814e264eb0cdR43
-  spec.add_dependency "aws-sdk-kinesis", "~> 1", "!= 1.4", "!= 1.5"
+  # Exclude aws-sdk-kinesis v1.14 to avoid aws-sdk-core dependency problem with td-agent v3.4.1
+  #   LoadError: cannot load such file -- aws-sdk-core/plugins/transfer_encoding.rb
+  #   https://github.com/aws/aws-sdk-ruby/commit/bb61ed0a2fabc6b1f90b757f13f37d5aeae48d8a#diff-b493e941d32289cd2df7eebc3fc5be2cR26
+  #   https://github.com/aws/aws-sdk-ruby/commit/e26577d2a426a4be79cd2d9edc1a4a4176e388ba#diff-10f50e27b30c3dc522b3c25db5782e2e
+  spec.add_dependency "aws-sdk-kinesis", "~> 1", "!= 1.4", "!= 1.5", "!= 1.14"
   # Exclude aws-sdk-firehose v1.9 to avoid aws-sdk-core dependency problem with td-agent v3.2.1
   #   LoadError: cannot load such file -- aws-sdk-core/plugins/endpoint_discovery.rb
   #   https://github.com/aws/aws-sdk-ruby/commit/85d8538a62255e58d9e176ee524a9f94354b51a0#diff-d51486091a10ada65b308b7f45966af1R18
   #   https://github.com/aws/aws-sdk-ruby/commit/7c9584bc6473100df9aec9333ab491ad4faeeca8#diff-be94f87e58e00329a6c0e03e43d5c292
-  spec.add_dependency "aws-sdk-firehose", "~> 1", "!= 1.5", "!= 1.9"
+  # Exclude aws-sdk-firehose v1.15 to avoid aws-sdk-core dependency problem with td-agent v3.4.1
+  #   LoadError: cannot load such file -- aws-sdk-core/plugins/transfer_encoding.rb
+  #   https://github.com/aws/aws-sdk-ruby/commit/bb61ed0a2fabc6b1f90b757f13f37d5aeae48d8a#diff-d51486091a10ada65b308b7f45966af1R26
+  #   https://github.com/aws/aws-sdk-ruby/commit/e26577d2a426a4be79cd2d9edc1a4a4176e388ba#diff-10f50e27b30c3dc522b3c25db5782e2e
+  spec.add_dependency "aws-sdk-firehose", "~> 1", "!= 1.5", "!= 1.9", "!= 1.15"
 
   spec.add_dependency "google-protobuf", "~> 3"
 

data/gemfiles/{Gemfile.fluentd-0.14.10 → Gemfile.fluentd-0.14.22}

@@ -17,4 +17,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
 gemspec path: ".."
 
-gem "fluentd", "0.14.10"
+gem "fluentd", "0.14.22"

data/gemfiles/Gemfile.fluentd-1.11.2

@@ -0,0 +1,20 @@
+#
+# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+gem "fluentd", "1.11.2"

data/gemfiles/Gemfile.td-agent-3.1.0

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.1.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L25
+gem "fluentd", "0.14.25"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.3.1"
+gem "aws-partitions", "1.42.0"
+gem "aws-sigv4", "1.0.2"
+gem "aws-sdk-core", "3.11.0"
+gem "aws-sdk-kms", "1.3.0"
+gem "aws-sdk-sqs", "1.3.0"
+gem "aws-sdk-s3", "1.8.0"
+gem "fluent-plugin-s3", "1.1.0"

data/gemfiles/Gemfile.td-agent-3.4.1

@@ -0,0 +1,34 @@
+#
+# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.4.1
+# td-agent v3.4.0 has the same dependencies
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L27
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.4.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/plugin_gems.rb#L16-L23
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.149.0"
+gem "aws-sigv4", "1.1.0"
+gem "aws-sdk-core", "3.48.3"
+gem "aws-sdk-kms", "1.16.0"
+gem "aws-sdk-sqs", "1.13.0"
+gem "aws-sdk-s3", "1.36.0"
+gem "fluent-plugin-s3", "1.1.9"

data/gemfiles/Gemfile.td-agent-3.5.0

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.5.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.7.0"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.195.0"
+gem "aws-sigv4", "1.1.0"
+gem "aws-sdk-core", "3.61.2"
+gem "aws-sdk-kms", "1.24.0"
+gem "aws-sdk-sqs", "1.20.0"
+gem "aws-sdk-s3", "1.46.0"
+gem "fluent-plugin-s3", "1.1.11"

data/gemfiles/Gemfile.td-agent-3.5.1

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.5.1
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.1/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.7.4"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.1/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.230.0"
+gem "aws-sigv4", "1.1.0"
+gem "aws-sdk-core", "3.72.0"
+gem "aws-sdk-kms", "1.25.0"
+gem "aws-sdk-sqs", "1.23.0"
+gem "aws-sdk-s3", "1.52.0"
+gem "fluent-plugin-s3", "1.2.0"

data/gemfiles/Gemfile.td-agent-3.6.0

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.6.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.9.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.273.0"
+gem "aws-sigv4", "1.1.0"
+gem "aws-sdk-core", "3.90.0"
+gem "aws-sdk-kms", "1.29.0"
+gem "aws-sdk-sqs", "1.23.1"
+gem "aws-sdk-s3", "1.60.2"
+gem "fluent-plugin-s3", "1.3.0"

data/gemfiles/Gemfile.td-agent-3.7.1

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.7.1
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.10.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.297.0"
+gem "aws-sigv4", "1.1.1"
+gem "aws-sdk-core", "3.94.0"
+gem "aws-sdk-kms", "1.30.0"
+gem "aws-sdk-sqs", "1.24.0"
+gem "aws-sdk-s3", "1.63.0"
+gem "fluent-plugin-s3", "1.3.1"

data/gemfiles/Gemfile.td-agent-3.8.0

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.8.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.11.1"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.332.0"
+gem "aws-sigv4", "1.2.0"
+gem "aws-sdk-core", "3.100.0"
+gem "aws-sdk-kms", "1.34.1"
+gem "aws-sdk-sqs", "1.28.0"
+gem "aws-sdk-s3", "1.69.1"
+gem "fluent-plugin-s3", "1.3.2"

data/gemfiles/Gemfile.td-agent-4.0.1

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.0.1
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/td-agent/config.rb#L4
+gem "fluentd", "1.11.1"
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/gemfiles/linux/Gemfile#L35-L42
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.352.0"
+gem "aws-sdk-core", "3.104.3"
+gem "aws-sdk-kms", "1.36.0"
+gem "aws-sdk-sqs", "1.30.0"
+gem "aws-sigv4", "1.2.1"
+gem "aws-sdk-s3", "1.75.0"
+gem "fluent-plugin-s3", "1.4.0"

data/lib/fluent/plugin/kinesis.rb

@@ -12,6 +12,8 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 
+require 'fluent/version'
+require 'fluent/msgpack_factory'
 require 'fluent/plugin/output'
 require 'fluent/plugin/kinesis_helper/client'
 require 'fluent/plugin/kinesis_helper/api'
@@ -20,7 +22,6 @@ require 'zlib'
 module Fluent
   module Plugin
     class KinesisOutput < Fluent::Plugin::Output
-      include Fluent::MessagePackFactory::Mixin
       include KinesisHelper::Client
       include KinesisHelper::API
 
@@ -83,6 +84,10 @@ module Fluent
         true
       end
 
+      def formatted_to_msgpack_binary?
+        true
+      end
+
       private
 
       def data_formatter_create(conf)
@@ -133,15 +138,21 @@ module Fluent
         ''
       end
 
+      if Gem::Version.new(Fluent::VERSION) >= Gem::Version.new('1.8.0')
+        def msgpack_unpacker(*args)
+          Fluent::MessagePackFactory.msgpack_unpacker(*args)
+        end
+      else
+        include Fluent::MessagePackFactory::Mixin
+      end
+
       def write_records_batch(chunk, &block)
         unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
-        chunk.open do |io|
-          records = msgpack_unpacker(io).to_enum
-          split_to_batches(records) do |batch, size|
-            log.debug(sprintf "Write chunk %s / %3d records / %4d KB", unique_id, batch.size, size/1024)
-            batch_request_with_retry(batch, &block)
-            log.debug("Finish writing chunk")
-          end
+        records = chunk.to_enum(:msgpack_each)
+        split_to_batches(records) do |batch, size|
+          log.debug(sprintf "Write chunk %s / %3d records / %4d KB", unique_id, batch.size, size/1024)
+          batch_request_with_retry(batch, &block)
+          log.debug("Finish writing chunk")
         end
       end
 

data/lib/fluent/plugin/kinesis_helper/client.rb

@@ -21,14 +21,15 @@ module Fluent
       module Client
         module ClientParams
           include Fluent::Configurable
-          config_param :region, :string,  default: nil
+          config_param :region,          :string,  default: nil
 
           config_param :http_proxy,      :string, default: nil, secret: true
           config_param :endpoint,        :string, default: nil
           config_param :ssl_verify_peer, :bool,   default: true
 
-          config_param :aws_key_id,  :string, default: nil, secret: true
-          config_param :aws_sec_key, :string, default: nil, secret: true
+          config_param :aws_key_id,      :string, default: nil, secret: true
+          config_param :aws_sec_key,     :string, default: nil, secret: true
+          config_param :aws_ses_token,   :string, default: nil, secret: true
           config_section :assume_role_credentials, multi: false do
             desc "The Amazon Resource Name (ARN) of the role to assume"
             config_param :role_arn, :string, secret: true
@@ -42,6 +43,22 @@ module Fluent
             config_param :external_id, :string, default: nil, secret: true
             desc "A http proxy url for requests to aws sts service"
             config_param :sts_http_proxy, :string, default: nil, secret: true
+            desc "A URL for a regional STS API endpoint, the default is global"
+            config_param :sts_endpoint_url, :string, default: nil
+          end
+          # Refer to the following link for additional parameters that could be added:
+          # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
+          config_section :web_identity_credentials, multi: false do
+            desc "The Amazon Resource Name (ARN) of the role to assume"
+            config_param :role_arn, :string
+            desc "An identifier for the assumed role session"
+            config_param :role_session_name, :string
+            desc "The absolute path to the file on disk containing the OIDC token"
+            config_param :web_identity_token_file, :string, default: nil #required
+            desc "An IAM policy in JSON format"
+            config_param :policy, :string, default: nil
+            desc "The duration, in seconds, of the role session (900-43200)"
+            config_param :duration_seconds, :time, default: nil
           end
           config_section :instance_profile_credentials, multi: false do
             desc "Number of times to retry when retrieving credentials"
@@ -115,6 +132,10 @@ module Fluent
           options = {}
           credentials_options = {}
           case
+          when @aws_key_id && @aws_sec_key && @aws_ses_token
+            options[:access_key_id] = @aws_key_id
+            options[:secret_access_key] = @aws_sec_key
+            options[:session_token] = @aws_ses_token  
           when @aws_key_id && @aws_sec_key
             options[:access_key_id] = @aws_key_id
             options[:secret_access_key] = @aws_sec_key
@@ -125,14 +146,34 @@ module Fluent
             credentials_options[:policy] = c.policy if c.policy
             credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
             credentials_options[:external_id] = c.external_id if c.external_id
-            if c.sts_http_proxy and @region
+            credentials_options[:sts_endpoint_url] = c.sts_endpoint_url if c.sts_endpoint_url
+            if @region and c.sts_http_proxy and c.sts_endpoint_url
+                credentials_options[:client] = Aws::STS::Client.new(region: @region, http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
+            elsif c.sts_http_proxy and c.sts_endpoint_url
+                credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
+            elsif @region and c.sts_http_proxy
                 credentials_options[:client] = Aws::STS::Client.new(region: @region, http_proxy: c.sts_http_proxy)
-            elsif @region
-                credentials_options[:client] = Aws::STS::Client.new(region: @region)
+            elsif @region and c.sts_endpoint_url
+                credentials_options[:client] = Aws::STS::Client.new(region: @region, endpoint: c.sts_endpoint_url)
             elsif c.sts_http_proxy
                 credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy)
+            elsif c.sts_endpoint_url
+                credentials_options[:client] = Aws::STS::Client.new(endpoint: c.sts_endpoint_url)
+            elsif @region
+                credentials_options[:client] = Aws::STS::Client.new(region: @region)
             end
             options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+          when @web_identity_credentials
+            c = @web_identity_credentials
+            credentials_options[:role_arn] = c.role_arn
+            credentials_options[:role_session_name] = c.role_session_name
+            credentials_options[:web_identity_token_file] = c.web_identity_token_file
+            credentials_options[:policy] = c.policy if c.policy
+            credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
+            if @region
+              credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+            end
+            options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
           when @instance_profile_credentials
             c = @instance_profile_credentials
             credentials_options[:retries] = c.retries if c.retries

data/lib/fluent/plugin/out_kinesis_firehose.rb

@@ -44,12 +44,13 @@ module Fluent
       end
 
       def write(chunk)
+        delivery_stream_name = extract_placeholders(@delivery_stream_name, chunk)
         write_records_batch(chunk) do |batch|
           records = batch.map{|(data)|
             { data: data }
           }
           client.put_record_batch(
-            delivery_stream_name: @delivery_stream_name,
+            delivery_stream_name: delivery_stream_name,
             records: records,
           )
         end

data/lib/fluent/plugin/out_kinesis_streams.rb

@@ -41,12 +41,13 @@ module Fluent
       end
 
       def write(chunk)
+        stream_name = extract_placeholders(@stream_name, chunk)
         write_records_batch(chunk) do |batch|
           records = batch.map{|(data, partition_key)|
             { data: data, partition_key: partition_key }
           }
           client.put_records(
-            stream_name: @stream_name,
+            stream_name: stream_name,
             records: records,
           )
         end

data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb

@@ -43,11 +43,12 @@ module Fluent
       end
 
       def write(chunk)
+        stream_name = extract_placeholders(@stream_name, chunk)
         write_records_batch(chunk) do |batch|
           key = @partition_key_generator.call
           records = batch.map{|(data)|data}
           client.put_records(
-            stream_name: @stream_name,
+            stream_name: stream_name,
             records: [{
               partition_key: key,
               data: aggregator.aggregate(records, key),

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '3.1.0'
+  VERSION = '3.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.3.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-18 00:00:00.000000000 Z
+date: 2020-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -43,6 +43,9 @@ dependencies:
     - - "!="
       - !ruby/object:Gem::Version
         version: '1.5'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -56,6 +59,9 @@ dependencies:
     - - "!="
       - !ruby/object:Gem::Version
         version: '1.5'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.14'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-firehose
   requirement: !ruby/object:Gem::Requirement
@@ -69,6 +75,9 @@ dependencies:
     - - "!="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -82,6 +91,9 @@ dependencies:
     - - "!="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.15'
 - !ruby/object:Gem::Dependency
   name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
@@ -271,11 +283,20 @@ files:
 - Rakefile
 - benchmark/task.rake
 - fluent-plugin-kinesis.gemspec
-- gemfiles/Gemfile.fluentd-0.14.10
+- gemfiles/Gemfile.fluentd-0.14.22
+- gemfiles/Gemfile.fluentd-1.11.2
+- gemfiles/Gemfile.td-agent-3.1.0
 - gemfiles/Gemfile.td-agent-3.1.1
 - gemfiles/Gemfile.td-agent-3.2.0
 - gemfiles/Gemfile.td-agent-3.2.1
 - gemfiles/Gemfile.td-agent-3.3.0
+- gemfiles/Gemfile.td-agent-3.4.1
+- gemfiles/Gemfile.td-agent-3.5.0
+- gemfiles/Gemfile.td-agent-3.5.1
+- gemfiles/Gemfile.td-agent-3.6.0
+- gemfiles/Gemfile.td-agent-3.7.1
+- gemfiles/Gemfile.td-agent-3.8.0
+- gemfiles/Gemfile.td-agent-4.0.1
 - lib/fluent/plugin/kinesis.rb
 - lib/fluent/plugin/kinesis_helper/aggregator.rb
 - lib/fluent/plugin/kinesis_helper/api.rb
@@ -296,15 +317,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.1'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin that sends events to Amazon Kinesis.