fluent-plugin-kinesis 2.2.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8cd92febc826fee7780bd0a1fb27c8a9694b00a1
-  data.tar.gz: 87d3709561d88d0a1fd489f4905d26e62fc631c4
+  metadata.gz: d525a1183f3d3aec1209db62718f8aa412a7c0eb
+  data.tar.gz: 3b197806fe40602e39462232cdce58ccbc3b02d6
 SHA512:
-  metadata.gz: 6727a62233dea478bf23a98494db47471fe7e161ebeef9823b84ff73b1fabd8705957206914d535a01125af53508667db76a51270d5d96da0aad1bf83b05d443
-  data.tar.gz: 8dfb953fab4f31bad804b30594a27a549602b4c1a4f782ee6825aa5bfbe401b91b3999ea724c1b3522e181e6fdf6d9f3481b3d95de86d5b8b39101fae4369f88
+  metadata.gz: a0144ae7446308a043139383e64c2de61875163c4114e2344389a4474c9465fcdb8b5bb1bad7c45d1d1fb56be20601016776315ef8d0fe0657fa4fec1c97dc49
+  data.tar.gz: 593f79bec197980ce5d3e9f3b1e9884575efd19166dd3d850280472b174f199ae4f486c33622d312da2d1f3b861f9c41ff50b791868e9ccbb52d5bbc487622b8

data/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,6 @@
+*Issue #, if available:*
+
+*Description of changes:*
+
+
+By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

data/.travis.yml

@@ -1,14 +1,25 @@
 language: ruby
+os: linux
+sudo: false
+
+gemfile:
+  - Gemfile
+  - gemfiles/Gemfile.td-agent-3.3.0
 
+# Test with supported td-agent versions
+# https://support.treasuredata.com/hc/en-us/articles/360001479187-The-td-agent-ChangeLog
 matrix:
   include:
     - rvm: 2.4.1
-      os: linux
-      gemfile: Gemfile
-    - rvm: 2.1.10
-      os: linux
-      gemfile: gemfiles/Gemfile.td-agent-2.3.5
+      gemfile: gemfiles/Gemfile.fluentd-0.14.10
+    - rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
+      gemfile: gemfiles/Gemfile.td-agent-3.1.1
+    - rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.0/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.2.0
+    - rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.1/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.2.1
+    - rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.3.0
+  fast_finish: true
 
 script: bundle exec rake test
-
-sudo: false

data/CHANGELOG.md

@@ -1,8 +1,15 @@
 # CHANGELOG
 
-## 2.2.0
+## 3.0.0
 
-- Bug - Update protobuf dependendy to fix breaking v2 plugin issue [#183](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/183)
+Plugin v3 is almost compatible with v2. You can use v3 with the same configuration as v2. For more details, please see [README](README.md).
+
+- Enhancement - Use modularized AWS SDK v3 since fluent-plugin-s3 also supports it : [#152](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/152)
+- Enhancement - Remove support for Fluentd v0.12 and use new Plugin API : [#156](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/156) (also fix [#133](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/133))
+- Enhancement / Breaking change - Remove *chomp* method from internal formatter : [#142](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/142) [#144](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/144)
+- Enhancement - Add *chomp_record* option for compatible format with plugin v2 : [#142](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/142)
+- Bug - Fix undefined method error in flushing buffers : [#133](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/133)
+- Bug - Fix dependency problem on AWS SDK : [#161](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/161)
 
 ## 2.1.1
 

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,4 @@
+## Code of Conduct
+This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 
+For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 
+opensource-codeofconduct@amazon.com with any additional questions or comments.

data/CONTRIBUTING.md

@@ -0,0 +1,61 @@
+# Contributing Guidelines
+
+Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 
+documentation, we greatly value feedback and contributions from our community.
+
+Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 
+information to effectively respond to your bug report or contribution.
+
+
+## Reporting Bugs/Feature Requests
+
+We welcome you to use the GitHub issue tracker to report bugs or suggest features.
+
+When filing an issue, please check [existing open](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues), or [recently closed](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already 
+reported the issue. Please try to include as much information as you can. Details like these are incredibly useful:
+
+* A reproducible test case or series of steps
+* The version of our code being used
+* Any modifications you've made relevant to the bug
+* Anything unusual about your environment or deployment
+
+
+## Contributing via Pull Requests
+Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that:
+
+1. You are working against the latest source on the *master* branch.
+2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already.
+3. You open an issue to discuss any significant work - we would hate for your time to be wasted.
+
+To send us a pull request, please:
+
+1. Fork the repository.
+2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change.
+3. Ensure local tests pass.
+4. Commit to your fork using clear commit messages.
+5. Send us a pull request, answering any default questions in the pull request interface.
+6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation.
+
+GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 
+[creating a pull request](https://help.github.com/articles/creating-a-pull-request/).
+
+
+## Finding contributions to work on
+Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/aws-fluent-plugin-kinesis/labels/help%20wanted) issues is a great place to start. 
+
+
+## Code of Conduct
+This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 
+For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 
+opensource-codeofconduct@amazon.com with any additional questions or comments.
+
+
+## Security issue notifications
+If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue.
+
+
+## Licensing
+
+See the [LICENSE](https://github.com/awslabs/aws-fluent-plugin-kinesis/blob/master/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution.
+
+We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes.

data/README.md

@@ -1,9 +1,11 @@
 # Fluent plugin for Amazon Kinesis
 
-[![Gitter](https://badges.gitter.im/awslabs/aws-fluent-plugin-kinesis.svg)](https://gitter.im/awslabs/aws-fluent-plugin-kinesis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Build Status](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis)
+[![Gitter](https://badges.gitter.im/awslabs/aws-fluent-plugin-kinesis.svg)](https://gitter.im/awslabs/aws-fluent-plugin-kinesis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+[![Build Status](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis)
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 
 [Fluentd][fluentd] output plugin
-that sends events to [Amazon Kinesis Streams][streams] and [Amazon Kinesis Firehose][firehose]. Also it supports [KPL Aggregated record format][kpl]. This gem includes three output plugins respectively:
+that sends events to [Amazon Kinesis Data Streams][streams] and [Amazon Kinesis Data Firehose][firehose]. Also it supports [KPL Aggregated record format][kpl]. This gem includes three output plugins respectively:
 
 - `kinesis_streams`
 - `kinesis_firehose`
@@ -11,10 +13,10 @@ that sends events to [Amazon Kinesis Streams][streams] and [Amazon Kinesis Fireh
 
 Also, there is a [documentation on Fluentd official site][fluentd-doc-kinesis].
 
-**Note**: This README is for v2.x. If you use v1.x, see the [old README][v1-readme].
+**Note**: This README is for v3. Plugin v3 is almost compatible with v2. If you use v1, see the [old README][v1-readme].
 
 ## Installation
-This fluentd plugin is available as the `fluent-plugin-kinesis` gem from RubyGems.
+This Fluentd plugin is available as the `fluent-plugin-kinesis` gem from RubyGems.
 
     gem install fluent-plugin-kinesis
 
@@ -49,7 +51,7 @@ Or just download specify your Ruby library path. Below is the sample for specify
 
 ## Dependencies
  * Ruby 2.1.0+
- * Fluentd 0.12.35+ (supporting 0.14.x)
+ * Fluentd 0.14.10+
 
 ## Basic Usage
 Here are general procedures for using this plugin:
@@ -112,26 +114,21 @@ Add configuration like below:
 Note: Each value should be adjusted to your system by yourself.
 
 ## Configuration: Credentials
-To put records into Amazon Kinesis Streams or Firehose, you need to provide AWS security credentials somehow. Without specifiying credentials in config file, this plugin automatically fetch credential just following AWS SDK for Ruby does (environment variable, shared profile, and instance profile).
+To put records into Amazon Kinesis Data Streams or Firehose, you need to provide AWS security credentials somehow. Without specifying credentials in config file, this plugin automatically fetch credential just following AWS SDK for Ruby does (environment variable, shared profile, and instance profile).
 
 This plugin uses the same configuration in [fluent-plugin-s3][fluent-plugin-s3].
 
 **aws_key_id**
 
-AWS access key id. This parameter is required when your agent is not
-running on EC2 instance with an IAM Role. When using an IAM role, make 
-sure to configure `instance_profile_credentials`. Usage can be found below.
+AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. When using an IAM role, make sure to configure `instance_profile_credentials`. Usage can be found below.
 
 **aws_sec_key**
 
-AWS secret key. This parameter is required when your agent is not running
-on EC2 instance with an IAM Role.
+AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role.
 
 **aws_iam_retries**
 
-The number of attempts to make (with exponential backoff) when loading
-instance profile credentials from the EC2 metadata service using an IAM
-role. Defaults to 5 retries.
+The number of attempts to make (with exponential backoff) when loading instance profile credentials from the EC2 metadata service using an IAM role. Defaults to 5 retries.
 
 ### assume_role_credentials
 Typically, you use AssumeRole for cross-account access or federation.
@@ -166,26 +163,21 @@ An IAM policy in JSON format.
 
 **duration_seconds**
 
-The duration, in seconds, of the role session. The value can range from
-900 seconds (15 minutes) to 3600 seconds (1 hour). By default, the value
-is set to 3600 seconds.
+The duration, in seconds, of the role session. The value can range from 900 seconds (15 minutes) to 3600 seconds (1 hour). By default, the value is set to 3600 seconds.
 
 **external_id**
 
-A unique identifier that is used by third parties when assuming roles in
-their customers' accounts.
+A unique identifier that is used by third parties when assuming roles in their customers' accounts.
 
 **sts_http_proxy**
 
-Proxy url for proxying requests to amazon sts service api. This needs to be  set up independently from global http_proxy parameter
-for the use case in which requests to kinesis api are going via kinesis vpc endpoint but requests to sts api have to go via http proxy. 
+Proxy url for proxying requests to amazon sts service api. This needs to be  set up independently from global http_proxy parameter for the use case in which requests to kinesis api are going via kinesis vpc endpoint but requests to sts api have to go via http proxy.
 It should be added to assume_role_credentials configuration stanza in the next format:
     sts_http_proxy http://[username:password]@hostname:port
 
 ### instance_profile_credentials
 
-Retrieve temporary security credentials via HTTP request. This is useful on
-EC2 instance.
+Retrieve temporary security credentials via HTTP request. This is useful on EC2 instance.
 
     <match *>
       @type kinesis_streams
@@ -226,8 +218,7 @@ Default is 5.
 
 ### shared_credentials
 
-This loads AWS access credentials from local ini file. This is useful for
-local developing.
+This loads AWS access credentials from local ini file. This is useful for local developing.
 
     <match *>
       @type kinesis_streams
@@ -250,7 +241,7 @@ Path to the shared file. Defaults to "#{Dir.home}/.aws/credentials".
 
 Defaults to 'default' or `[ENV]('AWS_PROFILE')`.
 
-## Configuraion: Format
+## Configuration: Format
 
 ### format (section)
 This plugin use `Fluent::TextFormatter` to serialize record to string. For more detail, see [formatter.rb]. By default, it uses `json` formatter same as specific like below:
@@ -277,7 +268,7 @@ For example, the config below will add `time` field whose value is eventtime wit
       </inject>
     </match>
 
-By default, `time_type string` and `time_format %Y-%m-%dT%H:%M:%S.%N%z` are already set to be applicable to Elasticsearch subsecond format. Although, you can use any configuration.
+By default, `time_type string` and `time_format %Y-%m-%dT%H:%M:%S.%N%z` are already set to be applicable to Elasticsearch sub-second format. Although, you can use any configuration.
 
 Also, there are some format related options below:
 
@@ -292,7 +283,7 @@ Defalut `false`. If you want to include `time` field in your record, set `true`.
 Defalut `false`. If you want to include `tag` field in your record, set `true`.
 
 ### data_key
-If your record contains a field whose string should be sent to Amazon Kinesis directly (without formatter), use this parameter to specify the field. In that case, other fileds than **data_key** are thrown away and never sent to Amazon Kinesis. Default `nil`, which means whole record will be formatted and sent.
+If your record contains a field whose string should be sent to Amazon Kinesis directly (without formatter), use this parameter to specify the field. In that case, other fields than **data_key** are thrown away and never sent to Amazon Kinesis. Default `nil`, which means whole record will be formatted and sent.
 
 ### compression
 Specifing compression way for data of each record. Current accepted options are `zlib`. Otherwise, no compression will be preformed.
@@ -300,7 +291,11 @@ Specifing compression way for data of each record. Current accepted options are
 ### log_truncate_max_size
 Integer, default 1024. When emitting the log entry, the message will be truncated by this size to avoid infinite loop when the log is also sent to Kinesis. The value 0 means no truncation.
 
-## Configuraion: API
+### chomp_record
+Boolean. Default `false`. If it is enabled, the plugin calls chomp and removes separator from the end of each record. This option is for compatible format with plugin v2. See [#142](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/142) for more details.  
+When you use [kinesis_firehose](#kinesis_firehose) output, [append_new_line](#append_new_line) option is `true` as default. If [append_new_line](#append_new_line) is enabled, the plugin calls chomp as [chomp_record](#chomp_record) is `true` before appending `\n` to each record. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) with default configuration. If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).
+
+## Configuration: API
 ### region
 AWS region of your stream. It should be in form like `us-east-1`, `us-west-2`. Refer to [Regions and Endpoints in AWS General Reference][region] for supported regions.
 
@@ -313,17 +308,17 @@ The upper limit of size of each record. Default is 1 MB which is the limitation
 HTTP proxy for API calling. Default `nil`.
 
 ### endpoint
-API endpoint URL, for testing. Defalut `nil`.
+API endpoint URL, for testing. Default `nil`.
 
 ### ssl_verify_peer
-Boolean. Disable if you want to verify ssl conncetion, for testing. Default `true`.
+Boolean. Disable if you want to verify ssl connection, for testing. Default `true`.
 
 ### debug
-Boolean. Enable if you need to debug Amazon Kinesis Firehose API call. Default is `false`.
+Boolean. Enable if you need to debug Amazon Kinesis Data Firehose API call. Default is `false`.
 
 ## Configuration: Batch request
 ### retries_on_batch_request
-Integer, default is 3. The plugin will put multiple records to Amazon Kinesis Streams in batches using PutRecords. A set of records in a batch may fail for reasons documented in the Kinesis Service API Reference for PutRecords. Failed records will be retried **retries_on_batch_request** times. If a record fails all retries an error log will be emitted.
+Integer, default is 8. The plugin will put multiple records to Amazon Kinesis Data Streams in batches using PutRecords. A set of records in a batch may fail for reasons documented in the Kinesis Service API Reference for PutRecords. Failed records will be retried **retries_on_batch_request** times. If a record fails all retries an error log will be emitted.
 
 ### reset_backoff_if_success
 Boolean, default `true`. If enabled, when after retrying, the next retrying checks the number of succeeded records on the former batch request and reset exponential backoff if there is any success. Because batch request could be composed by requests across shards, simple exponential backoff for the batch request wouldn't work some cases.
@@ -362,7 +357,8 @@ Here are `kinesis_firehose` specific configurations.
 Name of the delivery stream to put data.
 
 ### append_new_line
-Boolean. Default `true`. If it is enabled, the plugin add new line character (`\n`) to each serialized record.
+Boolean. Default `true`. If it is enabled, the plugin adds new line character (`\n`) to each serialized record.  
+Before appending `\n`, plugin calls chomp and removes separator from the end of each record as [chomp_record](#chomp_record) is `true`. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) output with default configuration ([append_new_line](#append_new_line) is `true`). If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).
 
 ## Configuration: kinesis_streams_aggregated
 Here are `kinesis_streams_aggregated` specific configurations.
@@ -384,9 +380,9 @@ To launch `fluentd` process with this plugin for development, follow the steps b
     make # will install gems dependency
     bundle exec fluentd -c /path/to/fluent.conf
 
-To launch using Fluentd v0.12, use `BUNDLE_GEMFILE` environment variable:
+To launch using specified version of Fluentd, use `BUNDLE_GEMFILE` environment variable:
 
-    BUNDLE_GEMFILE=$PWD/gemfiles/Gemfile.td-agent-2.3.5 bundle exec fluentd -c /path/to/fluent.conf
+    BUNDLE_GEMFILE=$PWD/gemfiles/Gemfile.td-agent-3.3.0 bundle exec fluentd -c /path/to/fluent.conf
 
 ## Contributing
 
@@ -394,8 +390,8 @@ Bug reports and pull requests are welcome on [GitHub][github].
 
 ## Related Resources
 
-* [Amazon Kinesis Streams Developer Guide](http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html)
-* [Amazon Kinesis Firehose Developer Guide](http://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
+* [Amazon Kinesis Data Streams Developer Guide](http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html)
+* [Amazon Kinesis Data Firehose Developer Guide](http://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
 
 [fluentd]: http://fluentd.org/
 [streams]: https://aws.amazon.com/kinesis/streams/

data/benchmark/task.rake

@@ -78,11 +78,11 @@ class KinesisBenchmark
   def create_driver(type, conf = default_config)
     klass = case type
             when :streams
-              Fluent::KinesisStreamsOutput
+              Fluent::Plugin::KinesisStreamsOutput
             when :streams_aggregated
-              Fluent::KinesisStreamsAggregatedOutput
+              Fluent::Plugin::KinesisStreamsAggregatedOutput
             when :firehose
-              Fluent::KinesisFirehoseOutput
+              Fluent::Plugin::KinesisFirehoseOutput
             end
     conf += case type
             when :streams, :streams_aggregated
@@ -90,13 +90,9 @@ class KinesisBenchmark
             when :firehose
               "delivery_stream_name fluent-plugin-test"
             end
-    if fluentd_v0_12?
-      Fluent::Test::BufferedOutputTestDriver.new(klass) do
-      end.configure(conf)
-    else
-      Fluent::Test::Driver::Output.new(klass) do
-      end.configure(conf)
-    end
+
+    Fluent::Test::Driver::Output.new(klass) do
+    end.configure(conf)
   end
 
   def benchmark(size, count)

data/fluent-plugin-kinesis.gemspec

@@ -30,20 +30,27 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.required_ruby_version = '>= 2.1'
 
-  spec.add_dependency "fluentd", ">= 0.12.35", "< 2"
-  spec.add_dependency "aws-sdk", ">= 2.9.9", "< 4"
-  # TODO: fluent-plugin-s3 depends on v2 only.
-  # https://github.com/fluent/fluent-plugin-s3/issues/208
-  #
-  # This plugin is sometimes used with s3 plugin.
-  # Unless s3 plugin is updated to be available with v3,
-  # this plugin should depend on v2 only.
-  # spec.add_dependency "aws-sdk-kinesis", "~> 1"
-  # spec.add_dependency "aws-sdk-firehose", "~> 1"
-  spec.add_dependency "google-protobuf", "~> 3", "< 3.7"
+  spec.add_dependency "fluentd", ">= 0.14.10", "< 2"
 
-  spec.add_development_dependency "bundler", "~> 1.10"
-  spec.add_development_dependency "rake", "~> 10.0"
+  # This plugin is sometimes used with s3 plugin, so watch out for conflicts
+  #   https://rubygems.org/gems/fluent-plugin-s3
+  # Exclude v1.5 to avoid aws-sdk dependency problem due to this issue
+  #   https://github.com/aws/aws-sdk-ruby/issues/1872
+  # Exclude aws-sdk-kinesis v1.4 to avoid aws-sdk-core dependency problem with td-agent v3.1.1
+  #   NoMethodError: undefined method `event=' for #<Seahorse::Model::Shapes::ShapeRef:*>
+  #   https://github.com/aws/aws-sdk-ruby/commit/03d60f9d3d821e645bd2a3efca066f37350ef906#diff-c69f15af8ea3eb9ab152659476e04608R401
+  #   https://github.com/aws/aws-sdk-ruby/commit/571c2d0e5ff9c24ff72893a08a74790db591fb57#diff-a55155f04aa6559460a0814e264eb0cdR43
+  spec.add_dependency "aws-sdk-kinesis", "~> 1", "!= 1.4", "!= 1.5"
+  # Exclude aws-sdk-firehose v1.9 to avoid aws-sdk-core dependency problem with td-agent v3.2.1
+  #   LoadError: cannot load such file -- aws-sdk-core/plugins/endpoint_discovery.rb
+  #   https://github.com/aws/aws-sdk-ruby/commit/85d8538a62255e58d9e176ee524a9f94354b51a0#diff-d51486091a10ada65b308b7f45966af1R18
+  #   https://github.com/aws/aws-sdk-ruby/commit/7c9584bc6473100df9aec9333ab491ad4faeeca8#diff-be94f87e58e00329a6c0e03e43d5c292
+  spec.add_dependency "aws-sdk-firehose", "~> 1", "!= 1.5", "!= 1.9"
+
+  spec.add_dependency "google-protobuf", "~> 3"
+
+  spec.add_development_dependency "bundler", ">= 1.10"
+  spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "test-unit", ">= 3.0.8"
   spec.add_development_dependency "test-unit-rr", ">= 1.0.3"
   spec.add_development_dependency "pry", ">= 0.10.1"

data/gemfiles/{Gemfile.td-agent-2.3.5 → Gemfile.fluentd-0.14.10}

@@ -17,9 +17,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
 gemspec path: ".."
 
-# Specify related gems for td-agent v2.3.5
-# https://github.com/treasure-data/omnibus-td-agent/blob/release-2.3.5/config/projects/td-agent2.rb#L23
-gem "fluentd", "0.12.35"
-# https://github.com/treasure-data/omnibus-td-agent/blob/release-2.3.5/plugin_gems.rb#L13-L15
-gem "fluent-plugin-s3", "0.8.2"
-gem "aws-sdk", "2.9.9"
+gem "fluentd", "0.14.10"

data/gemfiles/Gemfile.td-agent-3.1.1

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.1.1
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L22
+gem "fluentd", "1.0.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.3.1"
+gem "aws-partitions", "1.42.0"
+gem "aws-sigv4", "1.0.2"
+gem "aws-sdk-core", "3.11.0"
+gem "aws-sdk-kms", "1.3.0"
+gem "aws-sdk-sqs", "1.3.0"
+gem "aws-sdk-s3", "1.8.0"
+gem "fluent-plugin-s3", "1.1.0"