fluent-plugin-kinesis 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 529a0b5277cab5b82c165d2eaa4339c529381a17
-  data.tar.gz: fcc5161078a9311d511c5ecb95d4d2dfd9f078e4
+  metadata.gz: 2390ce24f4d7b54d0271ae6c7f4e90814961aa8b
+  data.tar.gz: 0101a6e0685efb9dfdfb58f78634ebcdf4b75b6c
 SHA512:
-  metadata.gz: 05661c7aa7080a5e693d58f4d16a07bbeb8f426bbddefacc7fcce78dfe411f098f1829ffd9bf0c5e253b79419605a00d49c1baf44248512e9f6190e97e8c4c1f
-  data.tar.gz: 7c4530a60cc37bbef7ca85559a07b0fee5038be2d84c195fd68900a7dba73485ae5b2d29aec1f0b585185a160f4317dc111a190951899a0451437b5978d96764
+  metadata.gz: 021a67078235969418ad03f166a3ffeafdb4559f1893828dfee7b7eaf4b4bd27b591c63729a1ee664a3a9b101422f638ec328454227f8b8f81eb77907ac21a2d
+  data.tar.gz: 86aac9dbe4fe9647659989d3e6591ecf350bf60b503352cbdde722f1131ac03e7c8d455124b47fb0e67b23b09c5b0a01fedcc3259a2691cdd2fd28183ef31d25

data/.travis.yml

@@ -4,14 +4,16 @@ sudo: false
 
 gemfile:
   - Gemfile
-  - gemfiles/Gemfile.td-agent-3.3.0
+  - gemfiles/Gemfile.td-agent-3.5.0
 
 # Test with supported td-agent versions
 # https://support.treasuredata.com/hc/en-us/articles/360001479187-The-td-agent-ChangeLog
 matrix:
   include:
     - rvm: 2.4.1
-      gemfile: gemfiles/Gemfile.fluentd-0.14.10
+      gemfile: gemfiles/Gemfile.fluentd-0.14.22
+    - rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L20
+      gemfile: gemfiles/Gemfile.td-agent-3.1.0
     - rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
       gemfile: gemfiles/Gemfile.td-agent-3.1.1
     - rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.0/config/projects/td-agent3.rb#L22
@@ -20,6 +22,10 @@ matrix:
       gemfile: gemfiles/Gemfile.td-agent-3.2.1
     - rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/config/projects/td-agent3.rb#L22
       gemfile: gemfiles/Gemfile.td-agent-3.3.0
+    - rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.4.1
+    - rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.0/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.5.0
   fast_finish: true
 
 script: bundle exec rake test

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # CHANGELOG
 
+## 3.2.0
+
+- Feature - Add placeholder support for stream names to send records to multiple streams : [#165](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/165) [#174](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/174)
+- Enhancement - Add sts_endpoint_url configuration parameter to support AWS STS regional endpoints : [#186](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/186)
+- Enhancement - Add aws_ses_token configuration parameter to use IAM with MFA and directly provided temporary credentials : [#166](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/166)
+- Dependency - Update gem dependency to Ruby 2.3.0+ and Fluentd 0.14.22+
+- Bug - Fix dependency problem on AWS SDK with td-agent v3.4.1
+
 ## 3.1.0
 
 - Feature - Add process_credentials configuration : [#178](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/178)

data/README.md

@@ -3,9 +3,10 @@
 [![Gitter](https://badges.gitter.im/awslabs/aws-fluent-plugin-kinesis.svg)](https://gitter.im/awslabs/aws-fluent-plugin-kinesis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 [![Build Status](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis)
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
+[![Gem Downloads](https://img.shields.io/gem/dt/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 
 [Fluentd][fluentd] output plugin
-that sends events to [Amazon Kinesis Data Streams][streams] and [Amazon Kinesis Data Firehose][firehose]. Also it supports [KPL Aggregated record format][kpl]. This gem includes three output plugins respectively:
+that sends events to [Amazon Kinesis Data Streams][streams] and [Amazon Kinesis Data Firehose][firehose]. Also it supports [KPL Aggregated Record Format][kpl]. This gem includes three output plugins respectively:
 
 - `kinesis_streams`
 - `kinesis_firehose`
@@ -24,7 +25,7 @@ Or you can install this plugin for [td-agent][td-agent] as:
 
     td-agent-gem install fluent-plugin-kinesis
 
-If you would like to build by yourself and install, please see the section below. Your need [bundler][bundler] for this.
+If you would like to build by yourself and install, see the section below. Your need [bundler][bundler] for this.
 
 In case of using with Fluentd: Fluentd will be also installed via the process below.
 
@@ -50,8 +51,8 @@ Or just download specify your Ruby library path. Below is the sample for specify
     export RUBYLIB=$RUBYLIB:/path/to/aws-fluent-plugin-kinesis/lib
 
 ## Dependencies
- * Ruby 2.1.0+
- * Fluentd 0.14.10+
+ * Ruby 2.3.0+
+ * Fluentd 0.14.22+ (td-agent v3.1.0+)
 
 ## Basic Usage
 Here are general procedures for using this plugin:
@@ -82,7 +83,7 @@ Assume you use Amazon EC2 instances with Instance profile. If you want to use sp
       stream_name your_stream
       partition_key key  # Otherwise, use random partition key
     </match>
-For more detail, see [Configuration: kinesis_streams](#configuration-kinesis_streams)
+For more details, see [Configuration: kinesis_streams](#configuration-kinesis_streams).
 
 ### kinesis_firehose
     <match your_tag>
@@ -90,7 +91,7 @@ For more detail, see [Configuration: kinesis_streams](#configuration-kinesis_str
       region us-east-1
       delivery_stream_name your_stream
     </match>
-For more detail, see [Configuration: kinesis_firehose](#configuration-kinesis_firehose)
+For more details, see [Configuration: kinesis_firehose](#configuration-kinesis_firehose).
 
 ### kinesis_streams_aggregated
     <match your_tag>
@@ -100,10 +101,10 @@ For more detail, see [Configuration: kinesis_firehose](#configuration-kinesis_fi
       # Unlike kinesis_streams, there is no way to use dynamic partition key.
       # fixed_partition_key or random.
     </match>
-For more detail, see [Configuration: kinesis_streams_aggregated](#configuration-kinesis_streams_aggregated)
+For more details, see [Configuration: kinesis_streams_aggregated](#configuration-kinesis_streams_aggregated).
 
 ### For better throughput
-Add configuration like below:
+Add configurations like below:
 
       flush_interval 1
       chunk_limit_size 1m
@@ -111,12 +112,14 @@ Add configuration like below:
       flush_thread_burst_interval 0.01
       flush_thread_count 15
 
+When you use Fluent v1.0 (td-agent3), write these configurations in buffer section. For more details, see [Config: Buffer Section][fluentd-buffer-section].
+
 Note: Each value should be adjusted to your system by yourself.
 
 ## Configuration: Credentials
 To put records into Amazon Kinesis Data Streams or Firehose, you need to provide AWS security credentials somehow. Without specifying credentials in config file, this plugin automatically fetch credential just following AWS SDK for Ruby does (environment variable, shared profile, and instance profile).
 
-This plugin uses the same configuration in [fluent-plugin-s3][fluent-plugin-s3].
+This plugin uses the same configuration in [fluent-plugin-s3][fluent-plugin-s3], but also supports aws session tokens for temporary credentials. 
 
 **aws_key_id**
 
@@ -126,12 +129,16 @@ AWS access key id. This parameter is required when your agent is not running on
 
 AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role.
 
+**aws_ses_token**
+
+AWS session token. This parameter is optional, but can be provided if using MFA or temporary credentials when your agent is not running on EC2 instance with an IAM Role. 
+
 **aws_iam_retries**
 
 The number of attempts to make (with exponential backoff) when loading instance profile credentials from the EC2 metadata service using an IAM role. Defaults to 5 retries.
 
 ### assume_role_credentials
-Typically, you use AssumeRole for cross-account access or federation.
+Typically, you can use AssumeRole for cross-account access or federation.
 
     <match *>
       @type kinesis_streams
@@ -145,9 +152,9 @@ Typically, you use AssumeRole for cross-account access or federation.
 See also:
 
 *   [Using IAM Roles - AWS Identity and Access
-    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
-*   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
-*   [Aws::AssumeRoleCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html)
+    Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
+*   [Aws::STS::Client](https://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
+*   [Aws::AssumeRoleCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html)
 
 **role_arn (required)**
 
@@ -175,6 +182,10 @@ Proxy url for proxying requests to amazon sts service api. This needs to be  set
 It should be added to assume_role_credentials configuration stanza in the next format:
     sts_http_proxy http://[username:password]@hostname:port
 
+**sts_endpoint_url**
+
+STS API endpoint url. This can be used to override the default global STS API endpoint of sts.amazonaws.com. Using regional endpoints may be preferred to reduce latency, and are required if utilizing a PrivateLink VPC Endpoint for STS API calls.
+
 ### instance_profile_credentials
 
 Retrieve temporary security credentials via HTTP request. This is useful on EC2 instance.
@@ -190,11 +201,11 @@ Retrieve temporary security credentials via HTTP request. This is useful on EC2
 
 See also:
 
-*   [Aws::InstanceProfileCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/InstanceProfileCredentials.html)
+*   [Aws::InstanceProfileCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/InstanceProfileCredentials.html)
 *   [Temporary Security Credentials - AWS Identity and Access
-    Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
+    Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
 *   [Instance Metadata and User Data - Amazon Elastic Compute
-    Cloud](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
+    Cloud](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
 
 **retries**
 
@@ -231,7 +242,7 @@ This loads AWS access credentials from local ini file. This is useful for local
 
 See also:
 
-*   [Aws::SharedCredentials](http://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html)
+*   [Aws::SharedCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html)
 
 **path**
 
@@ -265,7 +276,7 @@ Command to be executed as an external process.
 ## Configuration: Format
 
 ### format (section)
-This plugin use `Fluent::TextFormatter` to serialize record to string. For more detail, see [formatter.rb]. By default, it uses `json` formatter same as specific like below:
+This plugin uses `Fluent::TextFormatter` to serialize record to string. See [formatter.rb] for more details. By default, it uses `json` formatter same as specific like below:
 
     <match *>
       @type kinesis_streams
@@ -275,10 +286,12 @@ This plugin use `Fluent::TextFormatter` to serialize record to string. For more
       </format>
     </match>
 
+For other configurations of `json` formatter, see [json formatter plugin][fluentd-formatter-json].
+
 ### inject (section)
-This plugin use `Fluent::TimeFormatter` and other injection configurations. For more detail, see [inject.rb]. This section only works with Fluentd v0.14.
+This plugin uses `Fluent::TimeFormatter` and other injection configurations. See [inject.rb] for more details.
 
-For example, the config below will add `time` field whose value is eventtime with nanosecond and `tag` field whose value is its tag. (Mostly same as `include_time key true` and `include_tag_key true`)
+For example, the config below will add `time` field whose value is event time with nanosecond and `tag` field whose value is its tag.
 
     <match *>
       @type kinesis_streams
@@ -293,16 +306,6 @@ By default, `time_type string` and `time_format %Y-%m-%dT%H:%M:%S.%N%z` are alre
 
 Also, there are some format related options below:
 
-### include_time_key
-**Notice** With Fluentd v0.14, you should use inject section.
-
-Defalut `false`. If you want to include `time` field in your record, set `true`.
-
-### include_tag_key
-**Notice** With Fluentd v0.14, you should use inject section.
-
-Defalut `false`. If you want to include `tag` field in your record, set `true`.
-
 ### data_key
 If your record contains a field whose string should be sent to Amazon Kinesis directly (without formatter), use this parameter to specify the field. In that case, other fields than **data_key** are thrown away and never sent to Amazon Kinesis. Default `nil`, which means whole record will be formatted and sent.
 
@@ -368,6 +371,30 @@ Here are `kinesis_streams` specific configurations.
 ### stream_name
 Name of the stream to put data.
 
+As of Fluentd v1, built-in placeholders are supported. Now, you can also use built-in placeholders for this parameter.
+
+**NOTE:**
+Built-in placeholders require target key information in your buffer section attributes.
+
+e.g.)
+
+When you specify the following `stream_name` configuration with built-in placeholder:
+
+```aconf
+stream_name "${$.kubernetes.annotations.kinesis_streams}"
+```
+
+you ought to specify the corresponding attributes in buffer section:
+
+```aconf
+# $.kubernetes.annotations.kinesis_streams needs to be set in buffer attributes
+<buffer $.kubernetes.annotations.kinesis_streams>
+   # ...
+</buffer>
+```
+
+For more details, refer [Placeholders section in the official Fluentd document](https://docs.fluentd.org/configuration/buffer-section#placeholders).
+
 ### partition_key
 A key to extract partition key from JSON object. Default `nil`, which means partition key will be generated randomly.
 
@@ -387,6 +414,30 @@ Here are `kinesis_streams_aggregated` specific configurations.
 ### stream_name
 Name of the stream to put data.
 
+As of Fluentd v1, built-in placeholders are supported. Now, you can also use built-in placeholders for this parameter.
+
+**NOTE:**
+Built-in placeholders require target key information in your buffer section attributes.
+
+e.g.)
+
+When you specify the following `stream_name` configuration with built-in placeholder:
+
+```aconf
+stream_name "${$.kubernetes.annotations.kinesis_streams_aggregated}"
+```
+
+you ought to specify the corresponding attributes in buffer section:
+
+```aconf
+# $.kubernetes.annotations.kinesis_streams_aggregated needs to be set in buffer attributes
+<buffer $.kubernetes.annotations.kinesis_streams_aggregated>
+   # ...
+</buffer>
+```
+
+For more details, refer [Placeholders section in the official Fluentd document](https://docs.fluentd.org/configuration/buffer-section#placeholders).
+
 ### fixed_partition_key
 A value of fixed partition key. Default `nil`, which means partition key will be generated randomly.
 
@@ -414,17 +465,18 @@ Bug reports and pull requests are welcome on [GitHub][github].
 * [Amazon Kinesis Data Streams Developer Guide](http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html)
 * [Amazon Kinesis Data Firehose Developer Guide](http://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
 
-[fluentd]: http://fluentd.org/
+[fluentd]: https://www.fluentd.org/
 [streams]: https://aws.amazon.com/kinesis/streams/
 [firehose]: https://aws.amazon.com/kinesis/firehose/
 [kpl]: https://github.com/awslabs/amazon-kinesis-producer/blob/master/aggregation-format.md
-[td-agent]: https://github.com/treasure-data/td-agent
-[bundler]: http://bundler.io/
-[region]: http://docs.aws.amazon.com/general/latest/gr/rande.html#ak_region
-[fluentd_buffer]: http://docs.fluentd.org/articles/buffer-plugin-overview
+[td-agent]: https://github.com/treasure-data/omnibus-td-agent
+[bundler]: https://bundler.io/
+[region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#ak_region
+[fluentd-buffer-section]: https://docs.fluentd.org/configuration/buffer-section
+[fluentd-formatter-json]: https://docs.fluentd.org/formatter/json
 [github]: https://github.com/awslabs/aws-fluent-plugin-kinesis
 [formatter.rb]: https://github.com/fluent/fluentd/blob/master/lib/fluent/formatter.rb
 [inject.rb]: https://github.com/fluent/fluentd/blob/master/lib/fluent/plugin_helper/inject.rb
-[fluentd-doc-kinesis]: http://docs.fluentd.org/articles/kinesis-stream
+[fluentd-doc-kinesis]: https://docs.fluentd.org/how-to-guides/kinesis-stream
 [fluent-plugin-s3]: https://github.com/fluent/fluent-plugin-s3
 [v1-readme]: https://github.com/awslabs/aws-fluent-plugin-kinesis/blob/v1/README.md

data/fluent-plugin-kinesis.gemspec

@@ -28,9 +28,9 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = '>= 2.1'
+  spec.required_ruby_version = '>= 2.3'
 
-  spec.add_dependency "fluentd", ">= 0.14.10", "< 2"
+  spec.add_dependency "fluentd", ">= 0.14.22", "< 2"
 
   # This plugin is sometimes used with s3 plugin, so watch out for conflicts
   #   https://rubygems.org/gems/fluent-plugin-s3
@@ -40,12 +40,20 @@ Gem::Specification.new do |spec|
   #   NoMethodError: undefined method `event=' for #<Seahorse::Model::Shapes::ShapeRef:*>
   #   https://github.com/aws/aws-sdk-ruby/commit/03d60f9d3d821e645bd2a3efca066f37350ef906#diff-c69f15af8ea3eb9ab152659476e04608R401
   #   https://github.com/aws/aws-sdk-ruby/commit/571c2d0e5ff9c24ff72893a08a74790db591fb57#diff-a55155f04aa6559460a0814e264eb0cdR43
-  spec.add_dependency "aws-sdk-kinesis", "~> 1", "!= 1.4", "!= 1.5"
+  # Exclude aws-sdk-kinesis v1.14 to avoid aws-sdk-core dependency problem with td-agent v3.4.1
+  #   LoadError: cannot load such file -- aws-sdk-core/plugins/transfer_encoding.rb
+  #   https://github.com/aws/aws-sdk-ruby/commit/bb61ed0a2fabc6b1f90b757f13f37d5aeae48d8a#diff-b493e941d32289cd2df7eebc3fc5be2cR26
+  #   https://github.com/aws/aws-sdk-ruby/commit/e26577d2a426a4be79cd2d9edc1a4a4176e388ba#diff-10f50e27b30c3dc522b3c25db5782e2e
+  spec.add_dependency "aws-sdk-kinesis", "~> 1", "!= 1.4", "!= 1.5", "!= 1.14"
   # Exclude aws-sdk-firehose v1.9 to avoid aws-sdk-core dependency problem with td-agent v3.2.1
   #   LoadError: cannot load such file -- aws-sdk-core/plugins/endpoint_discovery.rb
   #   https://github.com/aws/aws-sdk-ruby/commit/85d8538a62255e58d9e176ee524a9f94354b51a0#diff-d51486091a10ada65b308b7f45966af1R18
   #   https://github.com/aws/aws-sdk-ruby/commit/7c9584bc6473100df9aec9333ab491ad4faeeca8#diff-be94f87e58e00329a6c0e03e43d5c292
-  spec.add_dependency "aws-sdk-firehose", "~> 1", "!= 1.5", "!= 1.9"
+  # Exclude aws-sdk-firehose v1.15 to avoid aws-sdk-core dependency problem with td-agent v3.4.1
+  #   LoadError: cannot load such file -- aws-sdk-core/plugins/transfer_encoding.rb
+  #   https://github.com/aws/aws-sdk-ruby/commit/bb61ed0a2fabc6b1f90b757f13f37d5aeae48d8a#diff-d51486091a10ada65b308b7f45966af1R26
+  #   https://github.com/aws/aws-sdk-ruby/commit/e26577d2a426a4be79cd2d9edc1a4a4176e388ba#diff-10f50e27b30c3dc522b3c25db5782e2e
+  spec.add_dependency "aws-sdk-firehose", "~> 1", "!= 1.5", "!= 1.9", "!= 1.15"
 
   spec.add_dependency "google-protobuf", "~> 3"
 

data/gemfiles/{Gemfile.fluentd-0.14.10 → Gemfile.fluentd-0.14.22}

@@ -17,4 +17,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
 gemspec path: ".."
 
-gem "fluentd", "0.14.10"
+gem "fluentd", "0.14.22"

data/gemfiles/Gemfile.td-agent-3.1.0

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.1.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L25
+gem "fluentd", "0.14.25"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.3.1"
+gem "aws-partitions", "1.42.0"
+gem "aws-sigv4", "1.0.2"
+gem "aws-sdk-core", "3.11.0"
+gem "aws-sdk-kms", "1.3.0"
+gem "aws-sdk-sqs", "1.3.0"
+gem "aws-sdk-s3", "1.8.0"
+gem "fluent-plugin-s3", "1.1.0"

data/gemfiles/Gemfile.td-agent-3.4.1

@@ -0,0 +1,34 @@
+#
+# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.4.1
+# td-agent v3.4.0 has the same dependencies
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L27
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.4.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/plugin_gems.rb#L16-L23
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.149.0"
+gem "aws-sigv4", "1.1.0"
+gem "aws-sdk-core", "3.48.3"
+gem "aws-sdk-kms", "1.16.0"
+gem "aws-sdk-sqs", "1.13.0"
+gem "aws-sdk-s3", "1.36.0"
+gem "fluent-plugin-s3", "1.1.9"

data/gemfiles/Gemfile.td-agent-3.5.0

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.5.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.7.0"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.195.0"
+gem "aws-sigv4", "1.1.0"
+gem "aws-sdk-core", "3.61.2"
+gem "aws-sdk-kms", "1.24.0"
+gem "aws-sdk-sqs", "1.20.0"
+gem "aws-sdk-s3", "1.46.0"
+gem "fluent-plugin-s3", "1.1.11"

data/lib/fluent/plugin/kinesis_helper/client.rb

@@ -21,14 +21,15 @@ module Fluent
       module Client
         module ClientParams
           include Fluent::Configurable
-          config_param :region, :string,  default: nil
+          config_param :region,          :string,  default: nil
 
           config_param :http_proxy,      :string, default: nil, secret: true
           config_param :endpoint,        :string, default: nil
           config_param :ssl_verify_peer, :bool,   default: true
 
-          config_param :aws_key_id,  :string, default: nil, secret: true
-          config_param :aws_sec_key, :string, default: nil, secret: true
+          config_param :aws_key_id,      :string, default: nil, secret: true
+          config_param :aws_sec_key,     :string, default: nil, secret: true
+          config_param :aws_ses_token,   :string, default: nil, secret: true
           config_section :assume_role_credentials, multi: false do
             desc "The Amazon Resource Name (ARN) of the role to assume"
             config_param :role_arn, :string, secret: true
@@ -42,6 +43,8 @@ module Fluent
             config_param :external_id, :string, default: nil, secret: true
             desc "A http proxy url for requests to aws sts service"
             config_param :sts_http_proxy, :string, default: nil, secret: true
+            desc "A URL for a regional STS API endpoint, the default is global"
+            config_param :sts_endpoint_url, :string, default: nil
           end
           config_section :instance_profile_credentials, multi: false do
             desc "Number of times to retry when retrieving credentials"
@@ -115,6 +118,10 @@ module Fluent
           options = {}
           credentials_options = {}
           case
+          when @aws_key_id && @aws_sec_key && @aws_ses_token
+            options[:access_key_id] = @aws_key_id
+            options[:secret_access_key] = @aws_sec_key
+            options[:session_token] = @aws_ses_token  
           when @aws_key_id && @aws_sec_key
             options[:access_key_id] = @aws_key_id
             options[:secret_access_key] = @aws_sec_key
@@ -125,12 +132,19 @@ module Fluent
             credentials_options[:policy] = c.policy if c.policy
             credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
             credentials_options[:external_id] = c.external_id if c.external_id
-            if c.sts_http_proxy and @region
+            credentials_options[:sts_endpoint_url] = c.sts_endpoint_url if c.sts_endpoint_url
+            if c.sts_http_proxy and c.sts_endpoint_url
+                credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
+            elsif @region and c.sts_http_proxy
                 credentials_options[:client] = Aws::STS::Client.new(region: @region, http_proxy: c.sts_http_proxy)
-            elsif @region
-                credentials_options[:client] = Aws::STS::Client.new(region: @region)
+            elsif @region and c.sts_endpoint_url
+                credentials_options[:client] = Aws::STS::Client.new(region: @region, endpoint: c.sts_endpoint_url)
             elsif c.sts_http_proxy
                 credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy)
+            elsif c.sts_endpoint_url
+                credentials_options[:client] = Aws::STS::Client.new(endpoint: c.sts_endpoint_url)
+            elsif @region
+                credentials_options[:client] = Aws::STS::Client.new(region: @region)
             end
             options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
           when @instance_profile_credentials

data/lib/fluent/plugin/out_kinesis_streams.rb

@@ -41,12 +41,13 @@ module Fluent
       end
 
       def write(chunk)
+        stream_name = extract_placeholders(@stream_name, chunk)
         write_records_batch(chunk) do |batch|
           records = batch.map{|(data, partition_key)|
             { data: data, partition_key: partition_key }
           }
           client.put_records(
-            stream_name: @stream_name,
+            stream_name: stream_name,
             records: records,
           )
         end

data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb

@@ -43,11 +43,12 @@ module Fluent
       end
 
       def write(chunk)
+        stream_name = extract_placeholders(@stream_name, chunk)
         write_records_batch(chunk) do |batch|
           key = @partition_key_generator.call
           records = batch.map{|(data)|data}
           client.put_records(
-            stream_name: @stream_name,
+            stream_name: stream_name,
             records: [{
               partition_key: key,
               data: aggregator.aggregate(records, key),

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '3.1.0'
+  VERSION = '3.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-18 00:00:00.000000000 Z
+date: 2019-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 0.14.22
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -43,6 +43,9 @@ dependencies:
     - - "!="
       - !ruby/object:Gem::Version
         version: '1.5'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -56,6 +59,9 @@ dependencies:
     - - "!="
       - !ruby/object:Gem::Version
         version: '1.5'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.14'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-firehose
   requirement: !ruby/object:Gem::Requirement
@@ -69,6 +75,9 @@ dependencies:
     - - "!="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -82,6 +91,9 @@ dependencies:
     - - "!="
       - !ruby/object:Gem::Version
         version: '1.9'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.15'
 - !ruby/object:Gem::Dependency
   name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
@@ -271,11 +283,14 @@ files:
 - Rakefile
 - benchmark/task.rake
 - fluent-plugin-kinesis.gemspec
-- gemfiles/Gemfile.fluentd-0.14.10
+- gemfiles/Gemfile.fluentd-0.14.22
+- gemfiles/Gemfile.td-agent-3.1.0
 - gemfiles/Gemfile.td-agent-3.1.1
 - gemfiles/Gemfile.td-agent-3.2.0
 - gemfiles/Gemfile.td-agent-3.2.1
 - gemfiles/Gemfile.td-agent-3.3.0
+- gemfiles/Gemfile.td-agent-3.4.1
+- gemfiles/Gemfile.td-agent-3.5.0
 - lib/fluent/plugin/kinesis.rb
 - lib/fluent/plugin/kinesis_helper/aggregator.rb
 - lib/fluent/plugin/kinesis_helper/api.rb
@@ -296,7 +311,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.1'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="