fluent-plugin-kinesis 3.5.0 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e07039476c9653143c9887e1954b32d1effbff30656ec31db85971f6169ed1cd
-  data.tar.gz: c76f31ddd864fae39083cbc57ce43fe3155a402bab0bd5904fd662ddea6df54e
+  metadata.gz: b5ce5b2ac051faba35c18fdf53601a2a5cdb9e85856992c041d0eaedd5fbdd59
+  data.tar.gz: 834b24ebee7ae29d9ba83c2638437b3b6da32460e278c12543e74b7ee7963434
 SHA512:
-  metadata.gz: c234cf9b6c92c35951efc4e7c8a3d142bbaa989888434ec675ff9570259a0750750e50ec48d3b245d3c441ef0f6b188b0dcdabbdd6b2857bd52391eed8ce8552
-  data.tar.gz: d17eba2ee78464bfd5acda8cd6bbbaa26a0a2b8082c8dd67fa307077fa2ef443414c0482ac3a0531705bc61050cfad055b120609615f7ad4c5601d52804337b7
+  metadata.gz: a13fe3f442273b0db4f1a1ee4678383930691742b12f2c4a7ed12e299da37b6382f8f67d3dd3bd8a1152439dd26f6b961f3738494346a3ab3f3481f3d1c844c1
+  data.tar.gz: 952399dc09b0a17f2d6b2bba38f972ea19a4c1ad03da28c99601bf0440f2653371e13892acd51adc557bc796425b2fa0ed5238015a6a64225d760d8da7347252

data/.github/workflows/test.yml

@@ -2,8 +2,10 @@ name: test
 
 on: [ push, pull_request ]
 
-# Test with supported td-agent versions
+# Test with supported fluent-package versions
+# https://www.fluentd.org/download/fluent_package
 # https://docs.fluentd.org/quickstart/support#change-logs
+# https://github.com/fluent/fluentd/blob/master/CHANGELOG.md
 # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
 # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v4.md
 jobs:
@@ -14,11 +16,11 @@ jobs:
       matrix:
         include:
           - gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
-            ruby-version: 2.4.2 # Oldest supported Ruby
-          - gemfile: gemfiles/Gemfile.fluentd-1.16.3 # Latest Fluentd / https://www.fluentd.org/download
-            ruby-version: 3.3.0 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads
-          - gemfile: gemfiles/Gemfile.td-agent-3.1.0 # Oldest supported td-agent
-            ruby-version: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L20
+            ruby-version: 2.7.1 # Oldest supported Ruby
+          - gemfile: gemfiles/Gemfile.fluentd-1.19.0 # Latest Fluentd / https://rubygems.org/gems/fluentd
+            ruby-version: 3.4.5 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads
+          # - gemfile: gemfiles/Gemfile.td-agent-3.1.0
+          #   ruby-version: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L20
           # - gemfile: gemfiles/Gemfile.td-agent-3.1.1
           #   ruby-version: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
           # - gemfile: gemfiles/Gemfile.td-agent-3.2.0
@@ -43,11 +45,11 @@ jobs:
           #   ruby-version: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/config/projects/td-agent3.rb#L22
           # - gemfile: gemfiles/Gemfile.td-agent-3.8.0
           #   ruby-version: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L22
-          - gemfile: gemfiles/Gemfile.td-agent-3.8.1 # Latest release of td-agent v3
-            ruby-version: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R22
+          # - gemfile: gemfiles/Gemfile.td-agent-3.8.1 # Latest release of td-agent v3
+          #   ruby-version: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R22
           # - gemfile: gemfiles/Gemfile.td-agent-4.0.0
           #   ruby-version: 2.7.1 # https://github.com/fluent/fluent-package-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L16
-          - gemfile: gemfiles/Gemfile.td-agent-4.0.1
+          - gemfile: gemfiles/Gemfile.td-agent-4.0.1 # Oldest supported td-agent
             ruby-version: 2.7.1 # https://github.com/fluent/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L16
           # - gemfile: gemfiles/Gemfile.td-agent-4.1.0
           #   ruby-version: 2.7.2 # https://github.com/fluent/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L19
@@ -71,14 +73,20 @@ jobs:
           #   ruby-version: 2.7.8 # https://github.com/fluent/fluent-package-builder/blob/v4.5.0/td-agent/config.rb#L19
           # - gemfile: gemfiles/Gemfile.td-agent-4.5.1
           #   ruby-version: 2.7.8 # https://github.com/fluent/fluent-package-builder/blob/v4.5.1/td-agent/config.rb#L19
-          - gemfile: gemfiles/Gemfile.td-agent-4.5.2 # Latest release of td-agent v4
-            ruby-version: 2.7.8 # https://github.com/fluent/fluent-package-builder/blob/v4.5.2/td-agent/config.rb#L19
+          # - gemfile: gemfiles/Gemfile.td-agent-4.5.2
+          #   ruby-version: 2.7.8 # https://github.com/fluent/fluent-package-builder/blob/v4.5.2/td-agent/config.rb#L19
+          - gemfile: gemfiles/Gemfile.td-agent-4.5.3 # Latest release of td-agent v4
+            ruby-version: 2.7.8 # https://github.com/fluent/fluent-package-builder/blob/v4.5.3/td-agent/config.rb#L19
           # - gemfile: gemfiles/Gemfile.fluent-package-5.0.0
           #   ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.0/fluent-package/config.rb#L26
           # - gemfile: gemfiles/Gemfile.fluent-package-5.0.1
           #   ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.1/fluent-package/config.rb#L26
-          - gemfile: gemfiles/Gemfile.fluent-package-5.0.2
-            ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.2/fluent-package/config.rb#L26
+          # - gemfile: gemfiles/Gemfile.fluent-package-5.0.2
+          #   ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.2/fluent-package/config.rb#L26
+          - gemfile: gemfiles/Gemfile.fluent-package-5.0.7 # fluent-package LTS
+            ruby-version: 3.2.8 # https://github.com/fluent/fluent-package-builder/blob/v5.0.7/fluent-package/config.rb#L26
+          - gemfile: gemfiles/Gemfile.fluent-package-5.2.0
+            ruby-version: 3.2.6 # https://github.com/fluent/fluent-package-builder/blob/v5.2.0/fluent-package/config.rb#L26
 
     env:
       BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 3.6.0
+
+- Enhancement - Add fluent-package 5.2 and Fluentd 1.19.0 with Ruby 3.4.5 to test cases
+- Dependency - Update google-protobuf dependency to v4
+- Dependency - Update gem dependency to Ruby 2.7.1+
+
 ## 3.5.0
 
 - Enhancement - Add td-agent 4.3-4.5, fluent-package 5.0, and Fluentd 1.16.3 with Ruby 3.3.0 to test cases

data/Makefile

@@ -42,3 +42,7 @@ benchmark:
 
 benchmark-remote:
 	bundle exec rake benchmark:remote
+
+regenerate-protobuf:
+	protoc --proto_path=proto --ruby_out=lib/fluent/plugin/gen kinesis_aggregation.proto
+

data/README.md

@@ -4,139 +4,227 @@
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 [![Gem Downloads](https://img.shields.io/gem/dt/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 
-[Fluentd][fluentd] output plugin
-that sends events to [Amazon Kinesis Data Streams][streams] and [Amazon Kinesis Data Firehose][firehose]. Also it supports [KPL Aggregated Record Format][kpl]. This gem includes three output plugins respectively:
+A fluentd output plugin to send events to [Amazon Kinesis Data Streams][streams] and [Amazon Data Firehose][firehose]. The plugin also supports [KPL Aggregated Record Format][kpl].
+
+This gem includes following three output plugins:
 
 - `kinesis_streams`
 - `kinesis_firehose`
 - `kinesis_streams_aggregated`
 
-Also, there is a [documentation on Fluentd official site][fluentd-doc-kinesis].
+The plugin is also described in [official Fluentd document][fluentd-doc-kinesis].
 
-**Note**: This README is for v3. Plugin v3 is almost compatible with v2. If you use v1, see the [old README][v1-readme].
+**Note**: This README is for the latest v3. Plugin v3 is almost compatible with v2. If you use v1, see [v1 README][v1-readme].
 
 ## Installation
-This Fluentd plugin is available as the `fluent-plugin-kinesis` gem from RubyGems.
-
-    gem install fluent-plugin-kinesis
-
-Or you can install this plugin for [td-agent][td-agent] as:
-
-    td-agent-gem install fluent-plugin-kinesis
-
-If you would like to build by yourself and install, see the section below. Your need [bundler][bundler] for this.
 
-In case of using with Fluentd: Fluentd will be also installed via the process below.
+Simply use RubyGems:
 
-    git clone https://github.com/awslabs/aws-fluent-plugin-kinesis.git
-    cd aws-fluent-plugin-kinesis
-    bundle install
-    bundle exec rake build
-    bundle exec rake install
+    $ gem install fluent-plugin-kinesis --no-document
 
-Also, you can use this plugin with td-agent: You have to install td-agent before installing this plugin.
+If you would like to build by yourself and install, you can build and install as follows:
 
-    git clone https://github.com/awslabs/aws-fluent-plugin-kinesis.git
-    cd aws-fluent-plugin-kinesis
-    bundle install
-    bundle exec rake build
-    fluent-gem install pkg/fluent-plugin-kinesis
+    $ git clone https://github.com/awslabs/aws-fluent-plugin-kinesis.git
+    $ cd aws-fluent-plugin-kinesis
+    $ bundle install
+    $ bundle exec rake build
+    $ bundle exec rake install
 
-Or just download specify your Ruby library path. Below is the sample for specifying your library path via RUBYLIB.
+If using td-agent v4 or lower, use td-agent-gem:
 
-    git clone https://github.com/awslabs/aws-fluent-plugin-kinesis.git
-    cd aws-fluent-plugin-kinesis
-    bundle install
-    export RUBYLIB=$RUBYLIB:/path/to/aws-fluent-plugin-kinesis/lib
+    $ td-agent-gem install fluent-plugin-kinesis
 
-## Dependencies
- * Ruby 2.4.2+
- * Fluentd 0.14.22+ (td-agent v3.1.0+)
+If using fluent-package v5 or higher, use fluent-gem:
 
-## Basic Usage
-Here are general procedures for using this plugin:
+    $ fluent-gem install fluent-plugin-kinesis
 
- 1. Install.
- 1. Edit configuration
- 1. Run Fluentd or td-agent
+## Requirements
 
-You can run this plugin with Fluentd as follows:
+| fluent-plugin-kinesis | fluentd    | ruby     |
+|:---------------------:|:-----------:|:--------:|
+| >= 3.6.0              | >= 0.14.22 | >= 2.7.1 |
+| >= 3.5.0 && < 3.6.0   | >= 0.14.22 | >= 2.4.2 |
+| >= 3.2.0 && < 3.5.0   | >= 0.14.22 | >= 2.3   |
+| >= 3.0.0 && < 3.2.0   | >= 0.14.10 | >= 2.1   |
+| >= 2.0.0 && < 3.0.0   | >= 0.12.35 | >= 2.1   |
+| < 2.0.0               | >= 0.10.58 | >= 2.0   |
 
- 1. Install.
- 1. Edit configuration file and save it as 'fluentd.conf'.
- 1. Then, run `fluentd -c /path/to/fluentd.conf`
+## Getting Started
 
-To run with td-agent, it would be as follows:
+When you run this plugin on Amazon EC2 instances or container services, use [instance profiles](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) to assume role. If you want to use specific credentials, see [Credentials](#configuration-credentials).
 
- 1. Install.
- 1. Edit configuration file provided by td-agent.
- 1. Then, run or restart td-agent.
+### kinesis_streams
 
-## Getting started
-Assume you use Amazon EC2 instances with Instance profile. If you want to use specific credentials, see [Credentials](#configuration-credentials).
+In your Fluentd configuration, use `@type kinesis_streams`. The configuration would look like this:
 
-### kinesis_streams
-    <match your_tag>
+    <match *>
       @type kinesis_streams
       region us-east-1
-      stream_name your_stream
+      stream_name YOUR_STREAM
       partition_key key  # Otherwise, use random partition key
     </match>
+
 For more details, see [Configuration: kinesis_streams](#configuration-kinesis_streams).
 
 ### kinesis_firehose
-    <match your_tag>
+
+In your Fluentd configuration, use `@type kinesis_firehose`. The configuration would look like this:
+
+    <match *>
       @type kinesis_firehose
       region us-east-1
-      delivery_stream_name your_stream
+      delivery_stream_name YOUR_STREAM
     </match>
+
 For more details, see [Configuration: kinesis_firehose](#configuration-kinesis_firehose).
 
 ### kinesis_streams_aggregated
-    <match your_tag>
+
+In your Fluentd configuration, use `@type kinesis_streams_aggregated`. The configuration would look like this:
+
+    <match *>
       @type kinesis_streams_aggregated
       region us-east-1
-      stream_name your_stream
+      stream_name YOUR_STREAM
       # Unlike kinesis_streams, there is no way to use dynamic partition key.
       # fixed_partition_key or random.
     </match>
+
 For more details, see [Configuration: kinesis_streams_aggregated](#configuration-kinesis_streams_aggregated).
 
-### For better throughput
-Add configurations like below:
+## Configuration
+
+- [Configuration: Plugin](#configuration-plugin)
+  - [Configuration: kinesis_streams](#configuration-kinesis_streams)
+  - [Configuration: kinesis_firehose](#configuration-kinesis_firehose)
+  - [Configuration: kinesis_streams_aggregated](#configuration-kinesis_streams_aggregated)
+- [Configuration: Credentials](#configuration-credentials)
+  - [AWS key and secret authentication](#aws-key-and-secret-authentication)
+  - [\<assume_role_credentials\> section](#assume_role_credentials-section)
+  - [\<web_identity_credentials\> section](#web_identity_credentials-section)
+  - [\<instance_profile_credentials\> section](#instance_profile_credentials-section)
+  - [\<shared_credentials\> section](#shared_credentials-section)
+  - [\<process_credentials\> section](#process_credentials-section)
+- [Configuration: Performance](#configuration-performance)
+  - [\<buffer\> section](#buffer-section)
+  - [Configuration: Batch Request](#configuration-batch-request)
+- [Configuration: Format](#configuration-format)
+  - [\<format\> section](#format-section)
+  - [\<inject\> section](#inject-section)
+- [Configuration: AWS SDK](#configuration-aws-sdk)
+
+## Configuration: Plugin
+
+### Configuration: kinesis_streams
+
+The following parameters are `kinesis_streams` specific configurations.
+
+#### stream_name
+
+Name of the stream to put data.
+
+As of Fluentd v1, you can use placeholders for this `stream_name` parameter. Note that chunk keys are required in your buffer section attributes for placeholders to work.
+
+The following configuration shows `kinesis_streams` output plugin that applies extract_placeholders on `stream_name`:
+
+    # chunk_key: tag
+    # ${tag} will be replaced with actual tag string
+    <match *>
+      @type kinesis_streams
+      stream_name ${tag}-stream
+
+      <buffer tag>
+        # ...
+      </buffer>
+    </match>
 
-      flush_interval 1
-      chunk_limit_size 1m
-      flush_thread_interval 0.1
-      flush_thread_burst_interval 0.01
-      flush_thread_count 15
+The value of `timekey` in buffer chunk keys can be extracted using `strptime` placeholders like this:
 
-When you use Fluent v1.0 (td-agent3), write these configurations in buffer section. For more details, see [Config: Buffer Section][fluentd-buffer-section].
+    # chunk_key: tag and time
+    <match *>
+      @type kinesis_streams
+      stream_name ${tag}-stream-%Y%m%d
 
-Note: Each value should be adjusted to your system by yourself.
+      <buffer tag, time>
+        # ...
+      </buffer>
+    </match>
+
+You can also use custom placeholder like this:
+
+    # chunk_key: $.kubernetes.annotations.kinesis_stream
+    <match *>
+      @type kinesis_streams
+      stream_name "${$.kubernetes.annotations.kinesis_stream}"
+
+      <buffer $.kubernetes.annotations.kinesis_stream>
+        # ...
+      </buffer>
+    </match>
+
+For more details, see [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
+
+#### partition_key
+
+A key to extract partition key from JSON object. Default `nil`, which means partition key will be generated randomly.
+
+### Configuration: kinesis_firehose
+
+The following parameters are `kinesis_firehose` specific configurations.
+
+#### delivery_stream_name
+
+Name of the delivery stream to put data.
+
+As of Fluentd v1, placerholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
+
+#### append_new_line
+
+Boolean. Default `true`. If it is enabled, the plugin adds new line character (`\n`) to each serialized record.  
+Before appending `\n`, plugin calls chomp and removes separator from the end of each record as [chomp_record](#chomp_record) is `true`. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) output with default configuration ([append_new_line](#append_new_line) is `true`). If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).
+
+### Configuration: kinesis_streams_aggregated
+
+The following parameters are `kinesis_streams_aggregated` specific configurations.
+
+#### stream_name (kinesis_streams_aggregated)
+
+Name of the stream to put data.
+
+As of Fluentd v1, placerholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
+
+#### fixed_partition_key
+
+A value of fixed partition key. Default `nil`, which means partition key will be generated randomly. Note that all records will go to single shard if you specify this option.
 
 ## Configuration: Credentials
-To put records into Amazon Kinesis Data Streams or Firehose, you need to provide AWS security credentials somehow. Without specifying credentials in config file, this plugin automatically fetch credential just following AWS SDK for Ruby does (environment variable, shared profile, and instance profile).
 
-This plugin uses the same configuration in [fluent-plugin-s3][fluent-plugin-s3], but also supports aws session tokens for temporary credentials. 
+To put records into Amazon Kinesis Data Streams or Amazon Data Firehose, you need to provide AWS security credentials. Without specifying credentials in config file, this plugin automatically fetches credentials just following AWS SDK for Ruby does (environment variable, shared profile, or instance profile).
+
+This plugin uses almost same configurations as [fluent-plugin-s3][fluent-plugin-s3-credentials], but also supports several additional configurations like `aws_ses_token` for temporary credentials.
+
+### AWS key and secret authentication
 
-**aws_key_id**
+These parameters are required when your agent is not running on EC2 instance with an IAM Role. When using an IAM role, make sure to configure `instance_profile_credentials`. Usage can be found below.
 
-AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. When using an IAM role, make sure to configure `instance_profile_credentials`. Usage can be found below.
+#### aws_key_id (required)
 
-**aws_sec_key**
+AWS access key id.
 
-AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role.
+#### aws_sec_key (required)
 
-**aws_ses_token**
+AWS secret key.
+
+#### aws_ses_token
 
 AWS session token. This parameter is optional, but can be provided if using MFA or temporary credentials when your agent is not running on EC2 instance with an IAM Role. 
 
-**aws_iam_retries**
+#### aws_iam_retries
 
 The number of attempts to make (with exponential backoff) when loading instance profile credentials from the EC2 metadata service using an IAM role. Defaults to 5 retries.
 
-### assume_role_credentials
+### \<assume_role_credentials\> section
+
 Typically, you can use AssumeRole for cross-account access or federation.
 
     <match *>
@@ -155,38 +243,39 @@ See also:
 *   [Aws::STS::Client](https://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
 *   [Aws::AssumeRoleCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html)
 
-**role_arn (required)**
+#### role_arn (required)
 
 The Amazon Resource Name (ARN) of the role to assume.
 
-**role_session_name (required)**
+#### role_session_name (required)
 
 An identifier for the assumed role session.
 
-**policy**
+#### policy
 
 An IAM policy in JSON format.
 
-**duration_seconds**
+#### duration_seconds
 
 The duration, in seconds, of the role session. The value can range from 900 seconds (15 minutes) to 3600 seconds (1 hour). By default, the value is set to 3600 seconds.
 
-**external_id**
+#### external_id
 
 A unique identifier that is used by third parties when assuming roles in their customers' accounts.
 
-**sts_http_proxy**
+#### sts_http_proxy
 
 Proxy url for proxying requests to amazon sts service api. This needs to be  set up independently from global http_proxy parameter for the use case in which requests to kinesis api are going via kinesis vpc endpoint but requests to sts api have to go via http proxy.
-It should be added to assume_role_credentials configuration stanza in the next format:
+It should be added to *assume_role_credentials* section in the next format:
+
     sts_http_proxy http://[username:password]@hostname:port
 
-**sts_endpoint_url**
+#### sts_endpoint_url
 
-STS API endpoint url. This can be used to override the default global STS API endpoint of sts.amazonaws.com. Using regional endpoints may be preferred to reduce latency, and are required if utilizing a PrivateLink VPC Endpoint for STS API calls.
+STS API endpoint url. This can be used to override the default global STS API endpoint of *sts.amazonaws.com*. Using regional endpoints may be preferred to reduce latency, and are required if utilizing a PrivateLink VPC Endpoint for STS API calls.
 
 
-### web_identity_credentials
+### \<web_identity_credentials\> section
 
 Similar to the assume_role_credentials, but for usage in EKS.
 
@@ -207,29 +296,27 @@ See also:
 *   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
 *   [Aws::AssumeRoleWebIdentityCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleWebIdentityCredentials.html)
 
-**role_arn (required)**
+#### role_arn (required)
 
 The Amazon Resource Name (ARN) of the role to assume.
 
-**role_session_name (required)**
+#### role_session_name (required)
 
 An identifier for the assumed role session.
 
-**web_identity_token_file (required)**
+#### web_identity_token_file (required)
 
-The absolute path to the file on disk containing the OIDC token
+The absolute path to the file on disk containing the OIDC token.
 
-**policy**
+#### policy
 
 An IAM policy in JSON format.
 
-**duration_seconds**
+#### duration_seconds
 
-The duration, in seconds, of the role session. The value can range from
-900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
-is set to 3600 seconds (1 hour).
+The duration, in seconds, of the role session. The value can range from 900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value is set to 3600 seconds (1 hour).
 
-### instance_profile_credentials
+### \<instance_profile_credentials\> section
 
 Retrieve temporary security credentials via HTTP request. This is useful on EC2 instance.
 
@@ -250,27 +337,27 @@ See also:
 *   [Instance Metadata and User Data - Amazon Elastic Compute
     Cloud](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
 
-**retries**
+#### retries
 
 Number of times to retry when retrieving credentials. Default is 5.
 
-**ip_address**
+#### ip_address
 
 Default is 169.254.169.254.
 
-**port**
+#### port
 
 Default is 80.
 
-**http_open_timeout**
+#### http_open_timeout
 
 Default is 5.
 
-**http_read_timeout**
+#### http_read_timeout
 
 Default is 5.
 
-### shared_credentials
+### \<shared_credentials\> section
 
 This loads AWS access credentials from local ini file. This is useful for local developing.
 
@@ -287,15 +374,15 @@ See also:
 
 *   [Aws::SharedCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html)
 
-**path**
+#### path
 
 Path to the shared file. Defaults to "#{Dir.home}/.aws/credentials".
 
-**profile_name**
+#### profile_name
 
 Defaults to 'default' or `[ENV]('AWS_PROFILE')`.
 
-### process_credentials
+### \<process_credentials\> section
 
 This loads AWS access credentials from an external process.
 
@@ -312,85 +399,42 @@ See also:
 *   [Aws::ProcessCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/ProcessCredentials.html)
 *   [Sourcing Credentials From External Processes](https://docs.aws.amazon.com/cli/latest/topic/config-vars.html#sourcing-credentials-from-external-processes)
 
-**process (required)**
+#### process (required)
 
 Command to be executed as an external process.
 
-## Configuration: Format
+## Configuration: Performance
 
-### format (section)
-This plugin uses `Fluent::TextFormatter` to serialize record to string. See [formatter.rb] for more details. By default, it uses `json` formatter same as specific like below:
+### \<buffer\> section
 
-    <match *>
-      @type kinesis_streams
-
-      <format>
-        @type json
-      </format>
-    </match>
-
-For other configurations of `json` formatter, see [json formatter plugin][fluentd-formatter-json].
-
-### inject (section)
-This plugin uses `Fluent::TimeFormatter` and other injection configurations. See [inject.rb] for more details.
-
-For example, the config below will add `time` field whose value is event time with nanosecond and `tag` field whose value is its tag.
+Use Fluentd buffering and flushing parameters to optimize throughput. When you use Fluent v1+ (td-agent v3+), write these configurations in buffer section like this:
 
     <match *>
       @type kinesis_streams
 
-      <inject>
-        time_key time
-        tag_key tag
-      </inject>
+      <buffer>
+        flush_interval 1
+        chunk_limit_size 1m
+        flush_thread_interval 0.1
+        flush_thread_burst_interval 0.01
+        flush_thread_count 15
+      </buffer>
     </match>
 
-By default, `time_type string` and `time_format %Y-%m-%dT%H:%M:%S.%N%z` are already set to be applicable to Elasticsearch sub-second format. Although, you can use any configuration.
-
-Also, there are some format related options below:
+For more details, see [Config: Buffer Section][fluentd-buffer-section]. Note that each parameter should be adjusted to your system.
 
-### data_key
-If your record contains a field whose string should be sent to Amazon Kinesis directly (without formatter), use this parameter to specify the field. In that case, other fields than **data_key** are thrown away and never sent to Amazon Kinesis. Default `nil`, which means whole record will be formatted and sent.
+### Configuration: Batch Request
 
-### compression
-Specifying compression way for data of each record. Current accepted options are `zlib` and `gzip`. Otherwise, no compression will be preformed.
+#### retries_on_batch_request
 
-### log_truncate_max_size
-Integer, default 1024. When emitting the log entry, the message will be truncated by this size to avoid infinite loop when the log is also sent to Kinesis. The value 0 means no truncation.
-
-### chomp_record
-Boolean. Default `false`. If it is enabled, the plugin calls chomp and removes separator from the end of each record. This option is for compatible format with plugin v2. See [#142](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/142) for more details.  
-When you use [kinesis_firehose](#kinesis_firehose) output, [append_new_line](#append_new_line) option is `true` as default. If [append_new_line](#append_new_line) is enabled, the plugin calls chomp as [chomp_record](#chomp_record) is `true` before appending `\n` to each record. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) with default configuration. If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).
-
-## Configuration: API
-### region
-AWS region of your stream. It should be in form like `us-east-1`, `us-west-2`. Refer to [Regions and Endpoints in AWS General Reference][region] for supported regions.
-
-Default `nil`, which means try to find from environment variable `AWS_REGION`.
-
-### max_record_size
-The upper limit of size of each record. Default is 1 MB which is the limitation of Kinesis.
-
-### http_proxy
-HTTP proxy for API calling. Default `nil`.
-
-### endpoint
-API endpoint URL, for testing. Default `nil`.
-
-### ssl_verify_peer
-Boolean. Disable if you want to verify ssl connection, for testing. Default `true`.
-
-### debug
-Boolean. Enable if you need to debug Amazon Kinesis Data Firehose API call. Default is `false`.
-
-## Configuration: Batch request
-### retries_on_batch_request
 Integer, default is 8. The plugin will put multiple records to Amazon Kinesis Data Streams in batches using PutRecords. A set of records in a batch may fail for reasons documented in the Kinesis Service API Reference for PutRecords. Failed records will be retried **retries_on_batch_request** times. If a record fails all retries an error log will be emitted.
 
-### reset_backoff_if_success
+#### reset_backoff_if_success
+
 Boolean, default `true`. If enabled, when after retrying, the next retrying checks the number of succeeded records on the former batch request and reset exponential backoff if there is any success. Because batch request could be composed by requests across shards, simple exponential backoff for the batch request wouldn't work some cases.
 
-### batch_request_max_count
+#### batch_request_max_count
+
 Integer, default 500. The number of max count of making batch request from record chunk. It can't exceed the default value because it's API limit.
 
 Default:
@@ -399,7 +443,8 @@ Default:
 - `kinesis_firehose`: 500
 - `kinesis_streams_aggregated`: 100,000
 
-### batch_request_max_size
+#### batch_request_max_size
+
 Integer. The number of max size of making batch request from record chunk. It can't exceed the default value because it's API limit.
 
 Default:
@@ -408,117 +453,95 @@ Default:
 - `kinesis_firehose`: 4 MB
 - `kinesis_streams_aggregated`: 1 MB
 
-### drop_failed_records_after_batch_request_retries
+#### drop_failed_records_after_batch_request_retries
+
 Boolean, default `true`.
 
 If *drop_failed_records_after_batch_request_retries* is enabled (default), the plugin will drop failed records when batch request fails after retrying max times configured as *retries_on_batch_request*. This dropping can be monitored from [monitor_agent](https://docs.fluentd.org/input/monitor_agent) or [fluent-plugin-prometheus](https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus) as *retry_count* or *num_errors* metrics.
 
 If *drop_failed_records_after_batch_request_retries* is disabled, the plugin will raise error and return chunk to Fluentd buffer when batch request fails after retrying max times. Fluentd will retry to send chunk records according to retry config in [Buffer Section](https://docs.fluentd.org/configuration/buffer-section). Note that this retryng may create duplicate records since [PutRecords API](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html) of Kinesis Data Streams and [PutRecordBatch API](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecordBatch.html) of Kinesis Data Firehose may return a partially successful response.
 
-### monitor_num_of_batch_request_retries
+#### monitor_num_of_batch_request_retries
+
 Boolean, default `false`. If enabled, the plugin will increment *retry_count* monitoring metrics after internal retrying to send batch request. This configuration enables you to monitor [ProvisionedThroughputExceededException](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html) from [monitor_agent](https://docs.fluentd.org/input/monitor_agent) or [fluent-plugin-prometheus](https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus). Note that *retry_count* metrics will be counted by the plugin in addition to original Fluentd buffering mechanism if *monitor_num_of_batch_request_retries* is enabled.
 
-## Configuration: kinesis_streams
-Here are `kinesis_streams` specific configurations.
+## Configuration: Format
 
-### stream_name
-Name of the stream to put data.
+### \<format\> section
 
-As of Fluentd v1, built-in placeholders are supported. Now, you can also use built-in placeholders for this parameter.
+This plugin uses `Fluent::TextFormatter` to serialize record to string. See [formatter.rb] for more details. By default, it uses `json` formatter same as specific like below:
 
-**NOTE:**
-Built-in placeholders require target key information in your buffer section attributes.
+    <match *>
+      @type kinesis_streams
 
-e.g.)
+      <format>
+        @type json
+      </format>
+    </match>
 
-When you specify the following `stream_name` configuration with built-in placeholder:
+For other configurations of `json` formatter, see [json formatter plugin][fluentd-formatter-json].
 
-```aconf
-stream_name "${$.kubernetes.annotations.kinesis_streams}"
-```
+### \<inject\> section
 
-you ought to specify the corresponding attributes in buffer section:
+This plugin uses `Fluent::TimeFormatter` and other injection configurations. See [inject.rb] for more details.
 
-```aconf
-# $.kubernetes.annotations.kinesis_streams needs to be set in buffer attributes
-<buffer $.kubernetes.annotations.kinesis_streams>
-   # ...
-</buffer>
-```
+For example, the config below will add `time` field whose value is event time with nanosecond and `tag` field whose value is its tag.
 
-For more details, refer [Placeholders section in the official Fluentd document](https://docs.fluentd.org/configuration/buffer-section#placeholders).
+    <match *>
+      @type kinesis_streams
 
-### partition_key
-A key to extract partition key from JSON object. Default `nil`, which means partition key will be generated randomly.
+      <inject>
+        time_key time
+        tag_key tag
+      </inject>
+    </match>
 
-## Configuration: kinesis_firehose
-Here are `kinesis_firehose` specific configurations.
+By default, `time_type string` and `time_format %Y-%m-%dT%H:%M:%S.%N%z` are already set to be applicable to Elasticsearch sub-second format. Although, you can use any configuration.
 
-### delivery_stream_name
-Name of the delivery stream to put data.
+In addition, there are some format related options:
 
-As of Fluentd v1, built-in placeholders are supported. Now, you can also use built-in placeholders for this parameter.
+#### data_key
 
-**NOTE:**
-Built-in placeholders require target key information in your buffer section attributes.
+If your record contains a field whose string should be sent to Amazon Kinesis directly (without formatter), use this parameter to specify the field. In that case, other fields than **data_key** are thrown away and never sent to Amazon Kinesis. Default `nil`, which means whole record will be formatted and sent.
 
-e.g.)
+#### compression
 
-When you specify the following `delivery_stream_name` configuration with built-in placeholder:
+Specifying compression way for data of each record. Current accepted options are `zlib` and `gzip`. Otherwise, no compression will be preformed.
 
-```aconf
-delivery_stream_name "${$.kubernetes.annotations.kinesis_firehose_streams}"
-```
+#### log_truncate_max_size
 
-you ought to specify the corresponding attributes in buffer section:
+Integer, default 1024. When emitting the log entry, the message will be truncated by this size to avoid infinite loop when the log is also sent to Kinesis. The value 0 means no truncation.
 
-```aconf
-# $.kubernetes.annotations.kinesis_firehose_streams needs to be set in buffer attributes
-<buffer $.kubernetes.annotations.kinesis_firehose_streams>
-   # ...
-</buffer>
-```
+#### chomp_record
 
-For more details, refer [Placeholders section in the official Fluentd document](https://docs.fluentd.org/configuration/buffer-section#placeholders).
+Boolean. Default `false`. If it is enabled, the plugin calls chomp and removes separator from the end of each record. This option is for compatible format with plugin v2. See [#142](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/142) for more details.  
+When you use [kinesis_firehose](#kinesis_firehose) output, [append_new_line](#append_new_line) option is `true` as default. If [append_new_line](#append_new_line) is enabled, the plugin calls chomp as [chomp_record](#chomp_record) is `true` before appending `\n` to each record. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) with default configuration. If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).
 
-### append_new_line
-Boolean. Default `true`. If it is enabled, the plugin adds new line character (`\n`) to each serialized record.  
-Before appending `\n`, plugin calls chomp and removes separator from the end of each record as [chomp_record](#chomp_record) is `true`. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) output with default configuration ([append_new_line](#append_new_line) is `true`). If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).
+## Configuration: AWS SDK
 
-## Configuration: kinesis_streams_aggregated
-Here are `kinesis_streams_aggregated` specific configurations.
+#### region
 
-### stream_name
-Name of the stream to put data.
+AWS region of your stream. It should be in form like `us-east-1`, `us-west-2`. Refer to [Regions and Endpoints in AWS General Reference][region] for supported regions. Default `nil`, which means try to find from environment variable `AWS_REGION`.
 
-As of Fluentd v1, built-in placeholders are supported. Now, you can also use built-in placeholders for this parameter.
+#### max_record_size
 
-**NOTE:**
-Built-in placeholders require target key information in your buffer section attributes.
+The upper limit of size of each record. Default is 1 MB which is the limitation of Kinesis.
 
-e.g.)
+#### http_proxy
 
-When you specify the following `stream_name` configuration with built-in placeholder:
+HTTP proxy for API calling. Default `nil`.
 
-```aconf
-stream_name "${$.kubernetes.annotations.kinesis_streams_aggregated}"
-```
+#### endpoint
 
-you ought to specify the corresponding attributes in buffer section:
+API endpoint URL, for testing. Default `nil`.
 
-```aconf
-# $.kubernetes.annotations.kinesis_streams_aggregated needs to be set in buffer attributes
-<buffer $.kubernetes.annotations.kinesis_streams_aggregated>
-   # ...
-</buffer>
-```
+#### ssl_verify_peer
 
-For more details, refer [Placeholders section in the official Fluentd document](https://docs.fluentd.org/configuration/buffer-section#placeholders).
+Boolean. Disable if you want to verify ssl connection, for testing. Default `true`.
 
-### fixed_partition_key
-A value of fixed partition key. Default `nil`, which means partition key will be generated randomly.
+#### debug
 
-Note: if you specified this option, all records go to a single shard.
+Boolean. Enable if you need to debug Amazon Data Firehose API call. Default is `false`.
 
 ## Development
 
@@ -540,20 +563,18 @@ Bug reports and pull requests are welcome on [GitHub][github].
 ## Related Resources
 
 * [Amazon Kinesis Data Streams Developer Guide](http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html)
-* [Amazon Kinesis Data Firehose Developer Guide](http://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
+* [Amazon Data Firehose Developer Guide](http://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
 
-[fluentd]: https://www.fluentd.org/
+[github]: https://github.com/awslabs/aws-fluent-plugin-kinesis
+[fluentd-doc-kinesis]: https://docs.fluentd.org/how-to-guides/kinesis-stream
+[v1-readme]: https://github.com/awslabs/aws-fluent-plugin-kinesis/blob/v1/README.md
 [streams]: https://aws.amazon.com/kinesis/streams/
 [firehose]: https://aws.amazon.com/kinesis/firehose/
 [kpl]: https://github.com/awslabs/amazon-kinesis-producer/blob/master/aggregation-format.md
-[td-agent]: https://github.com/treasure-data/omnibus-td-agent
-[bundler]: https://bundler.io/
 [region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#ak_region
+[fluent-plugin-s3-credentials]: https://github.com/fluent/fluent-plugin-s3/blob/master/docs/credentials.md
 [fluentd-buffer-section]: https://docs.fluentd.org/configuration/buffer-section
+[fluentd-buffer-section-placeholders]: https://docs.fluentd.org/configuration/buffer-section#placeholders
 [fluentd-formatter-json]: https://docs.fluentd.org/formatter/json
-[github]: https://github.com/awslabs/aws-fluent-plugin-kinesis
 [formatter.rb]: https://github.com/fluent/fluentd/blob/master/lib/fluent/formatter.rb
 [inject.rb]: https://github.com/fluent/fluentd/blob/master/lib/fluent/plugin_helper/inject.rb
-[fluentd-doc-kinesis]: https://docs.fluentd.org/how-to-guides/kinesis-stream
-[fluent-plugin-s3]: https://github.com/fluent/fluent-plugin-s3
-[v1-readme]: https://github.com/awslabs/aws-fluent-plugin-kinesis/blob/v1/README.md

data/fluent-plugin-kinesis.gemspec

@@ -55,7 +55,7 @@ Gem::Specification.new do |spec|
   #   https://github.com/aws/aws-sdk-ruby/commit/e26577d2a426a4be79cd2d9edc1a4a4176e388ba#diff-10f50e27b30c3dc522b3c25db5782e2e
   spec.add_dependency "aws-sdk-firehose", "~> 1", "!= 1.5", "!= 1.9", "!= 1.15"
 
-  spec.add_dependency "google-protobuf", "~> 3"
+  spec.add_dependency "google-protobuf", "~> 4"
 
   spec.add_development_dependency "bundler", ">= 1.10"
   spec.add_development_dependency "rake", ">= 10.0"

data/gemfiles/Gemfile.fluent-package-5.0.7

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for fluent-package v5.0.7
+# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent/fluent-package-builder/blob/v5.0.7/fluent-package/config.rb#L10
+gem "fluentd", "1.16.9"
+# https://github.com/fluent/fluent-package-builder/blob/v5.0.7/fluent-package/Gemfile#L57-L67
+gem "prometheus-client", "2.1.0"
+gem "fluent-plugin-prometheus", "2.1.0"
+gem "fluent-plugin-prometheus_pushgateway", "0.1.1"
+gem "jmespath", "1.6.2"
+gem "aws-partitions", "1.785.0"
+gem "aws-sdk-core", "3.178.0"
+gem "aws-sdk-kms", "1.71.0"
+gem "aws-sdk-sqs", "1.61.0"
+gem "aws-sigv4", "1.6.0"
+gem "aws-sdk-s3", "1.129.0"
+gem "fluent-plugin-s3", "1.7.2"

data/gemfiles/Gemfile.fluent-package-5.2.0

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for fluent-package v5.2.0
+# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent/fluent-package-builder/blob/v5.2.0/fluent-package/config.rb#L10
+gem "fluentd", "1.18.0"
+# https://github.com/fluent/fluent-package-builder/blob/v5.2.0/fluent-package/Gemfile#L55-L65
+gem "prometheus-client", "2.1.0"
+gem "fluent-plugin-prometheus", "2.1.0"
+gem "fluent-plugin-prometheus_pushgateway", "0.1.1"
+gem "jmespath", "1.6.2"
+gem "aws-partitions", "1.957.0"
+gem "aws-sdk-core", "3.201.2"
+gem "aws-sdk-kms", "1.88.0"
+gem "aws-sdk-sqs", "1.80.0"
+gem "aws-sigv4", "1.8.0"
+gem "aws-sdk-s3", "1.156.0"
+gem "fluent-plugin-s3", "1.8.1"

data/gemfiles/{Gemfile.fluentd-1.16.3 → Gemfile.fluentd-1.19.0}

@@ -3,4 +3,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
 gemspec path: ".."
 
-gem "fluentd", "1.16.3"
+gem "fluentd", "1.19.0"

data/gemfiles/Gemfile.td-agent-4.5.3

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.5.3
+# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v4.md
+# https://github.com/fluent/fluent-package-builder/blob/v4.5.3/td-agent/config.rb#L4
+gem "fluentd", "1.16.3"
+# https://github.com/fluent/fluent-package-builder/blob/v4.5.2/td-agent/Gemfile#L45-L55
+gem "prometheus-client", "2.1.0"
+gem "fluent-plugin-prometheus", "2.1.0"
+gem "fluent-plugin-prometheus_pushgateway", "0.1.1"
+gem "jmespath", "1.6.2"
+gem "aws-partitions", "1.785.0"
+gem "aws-sdk-core", "3.178.0"
+gem "aws-sdk-kms", "1.71.0"
+gem "aws-sdk-sqs", "1.61.0"
+gem "aws-sigv4", "1.6.0"
+gem "aws-sdk-s3", "1.129.0"
+gem "fluent-plugin-s3", "1.7.2"

data/lib/fluent/plugin/gen/kinesis_aggregation_pb.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: kinesis_aggregation.proto
+
+require 'google/protobuf'
+
+
+descriptor_data = "\n\x19kinesis_aggregation.proto\"j\n\x10\x41ggregatedRecord\x12\x1b\n\x13partition_key_table\x18\x01 \x03(\t\x12\x1f\n\x17\x65xplicit_hash_key_table\x18\x02 \x03(\t\x12\x18\n\x07records\x18\x03 \x03(\x0b\x32\x07.Record\"!\n\x03Tag\x12\x0b\n\x03key\x18\x01 \x02(\t\x12\r\n\x05value\x18\x02 \x01(\t\"h\n\x06Record\x12\x1b\n\x13partition_key_index\x18\x01 \x02(\x04\x12\x1f\n\x17\x65xplicit_hash_key_index\x18\x02 \x01(\x04\x12\x0c\n\x04\x64\x61ta\x18\x03 \x02(\x0c\x12\x12\n\x04tags\x18\x04 \x03(\x0b\x32\x04.Tag"
+
+pool = Google::Protobuf::DescriptorPool.generated_pool
+pool.add_serialized_file(descriptor_data)
+
+AggregatedRecord = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("AggregatedRecord").msgclass
+Tag = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("Tag").msgclass
+Record = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("Record").msgclass

data/lib/fluent/plugin/kinesis_helper/aggregator.rb

@@ -14,24 +14,7 @@
 
 require 'fluent/configurable'
 require 'google/protobuf'
-
-Google::Protobuf::DescriptorPool.generated_pool.build do
-  add_message "AggregatedRecord" do
-    repeated :partition_key_table, :string, 1
-    repeated :explicit_hash_key_table, :string, 2
-    repeated :records, :message, 3, "Record"
-  end
-  add_message "Tag" do
-    optional :key, :string, 1
-    optional :value, :string, 2
-  end
-  add_message "Record" do
-    optional :partition_key_index, :uint64, 1
-    optional :explicit_hash_key_index, :uint64, 2
-    optional :data, :bytes, 3
-    repeated :tags, :message, 4, "Tag"
-  end
-end
+require 'fluent/plugin/gen/kinesis_aggregation_pb'
 
 module Fluent
   module Plugin

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '3.5.0'
+  VERSION = '3.6.0'
 end

data/proto/kinesis_aggregation.proto

@@ -0,0 +1,19 @@
+// Based on https://github.com/awslabs/amazon-kinesis-producer/blob/master/aggregation-format.md
+
+message AggregatedRecord {
+  repeated string partition_key_table     = 1;
+  repeated string explicit_hash_key_table = 2;
+  repeated Record records                 = 3;
+}
+
+message Tag {
+  required string key   = 1;
+  optional string value = 2;
+}
+
+message Record {
+  required uint64 partition_key_index     = 1;
+  optional uint64 explicit_hash_key_index = 2;
+  required bytes  data                    = 3;
+  repeated Tag    tags                    = 4;
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 3.5.0
+  version: 3.6.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-29 00:00:00.000000000 Z
+date: 2025-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -286,8 +286,10 @@ files:
 - gemfiles/Gemfile.fluent-package-5.0.0
 - gemfiles/Gemfile.fluent-package-5.0.1
 - gemfiles/Gemfile.fluent-package-5.0.2
+- gemfiles/Gemfile.fluent-package-5.0.7
+- gemfiles/Gemfile.fluent-package-5.2.0
 - gemfiles/Gemfile.fluentd-0.14.22
-- gemfiles/Gemfile.fluentd-1.16.3
+- gemfiles/Gemfile.fluentd-1.19.0
 - gemfiles/Gemfile.td-agent-3.1.0
 - gemfiles/Gemfile.td-agent-3.1.1
 - gemfiles/Gemfile.td-agent-3.2.0
@@ -316,6 +318,8 @@ files:
 - gemfiles/Gemfile.td-agent-4.5.0
 - gemfiles/Gemfile.td-agent-4.5.1
 - gemfiles/Gemfile.td-agent-4.5.2
+- gemfiles/Gemfile.td-agent-4.5.3
+- lib/fluent/plugin/gen/kinesis_aggregation_pb.rb
 - lib/fluent/plugin/kinesis.rb
 - lib/fluent/plugin/kinesis_helper/aggregator.rb
 - lib/fluent/plugin/kinesis_helper/api.rb
@@ -325,6 +329,7 @@ files:
 - lib/fluent/plugin/out_kinesis_streams.rb
 - lib/fluent/plugin/out_kinesis_streams_aggregated.rb
 - lib/fluent_plugin_kinesis/version.rb
+- proto/kinesis_aggregation.proto
 homepage: https://github.com/awslabs/aws-fluent-plugin-kinesis
 licenses:
 - Apache-2.0