fluent-plugin-kinesis 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 98a12a1569d674676d986f802935ca80df435b4cd2acc4dd15d1951c8b020671
-  data.tar.gz: 63238ffd8abeb4c038af46eefff7c528277fa2225c8ca9affbe2714f433a6417
+  metadata.gz: 7a56428085f3d1f60d243071976cf17ef81ae64cc02176aa9106e0f38763e8c2
+  data.tar.gz: d7176edca0e7fd26c7e6724a1c63c8817c7104666e1ed7108c25ef91e7449662
 SHA512:
-  metadata.gz: c49fb3a4a2a6443375c52a7e416686188160e7a8e85d997c4d4105ee18b06c3e9d94bfcd25e42ef971a3128c55575a9296012a6a5343bc3c9d466a5083c8d78c
-  data.tar.gz: 6d2d7fd0012cfcb2282aa6982ff6897333155a0f43c19a90eb365b147f42aac7f16b06dd38924a16403494295031103569529f8fe42055547e077ca558581005
+  metadata.gz: 332e9a47297b9ee84f03527119b6e286301378ffce699c5fa3d0f23549f4e345754e439f2afc540190415bc02cb4bd466b15a17498560c564701115dc2698a51
+  data.tar.gz: 7979f2ea6cdbda52973fd13f1b403f99ae3852ce9a1e3112b144a45e63af21b43b9bc3f6591f5f5004a5fe8df3c3f77cc7d85fbc4aa2150a92563ac1dc3fc4a7

data/.travis.yml

@@ -2,38 +2,50 @@ language: ruby
 os: linux
 sudo: false
 
-gemfile:
-  - gemfiles/Gemfile.td-agent-3.8.0
-  - gemfiles/Gemfile.td-agent-4.0.1
-
 # Test with supported td-agent versions
-# https://tddocs.atlassian.net/wiki/spaces/PD/pages/6291635/The+td-agent+ChangeLog
-# https://docs.fluentd.org/quickstart/td-agent-v2-vs-v3-vs-v4
+# https://docs.fluentd.org/quickstart/support#change-logs
+# https://docs.treasuredata.com/display/public/PD/The+td-agent+Change+Log
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
 matrix:
   include:
-    - rvm: 2.7.2 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads/
-      gemfile: gemfiles/Gemfile.fluentd-1.11.2 # Latest Fluentd / https://www.fluentd.org/download
-    - rvm: 2.3.0 # Oldest supported Ruby
-      gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
-    - rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
-      gemfile: gemfiles/Gemfile.td-agent-3.1.1
-    - rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.1/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.2.1
-    - rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.3.0
-    - rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.4.1
-    - rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.1/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.5.1
-    - rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.6.0
-    - rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.7.1
-    - rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.8.0
-    - rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/td-agent/config.rb#L16
-      gemfile: gemfiles/Gemfile.td-agent-4.0.1
-      
+    - gemfile: gemfiles/Gemfile.fluentd-1.12.3 # Latest Fluentd / https://www.fluentd.org/download
+      rvm: 2.7.3 # Latest supported Ruby (less than v3.0.0) / https://www.ruby-lang.org/en/downloads
+    - gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
+      rvm: 2.3.0 # Oldest supported Ruby
+    - gemfile: gemfiles/Gemfile.td-agent-3.1.0
+      rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L20
+    # - gemfile: gemfiles/Gemfile.td-agent-3.1.1
+    #   rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
+    # - gemfile: gemfiles/Gemfile.td-agent-3.2.0
+    #   rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.2.1
+      rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.1/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.3.0
+      rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/config/projects/td-agent3.rb#L22
+    # - gemfile: gemfiles/Gemfile.td-agent-3.4.0
+    #   rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.4.1
+      rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L22
+    # - gemfile: gemfiles/Gemfile.td-agent-3.5.0
+    #   rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.5.1
+      rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.1/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.6.0
+      rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/config/projects/td-agent3.rb#L22
+    # - gemfile: gemfiles/Gemfile.td-agent-3.7.0
+    #   rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/commit/da5cb39ec5396f652cd30bad5bb8ef4ec19071a9#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R22
+    - gemfile: gemfiles/Gemfile.td-agent-3.7.1
+      rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.8.0 # Latest official release of td-agent v3
+      rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L22
+    - gemfile: gemfiles/Gemfile.td-agent-3.8.1 # Maintenance release of td-agent v3
+      rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R22
+    # - gemfile: gemfiles/Gemfile.td-agent-4.0.0
+    #   rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L16
+    - gemfile: gemfiles/Gemfile.td-agent-4.0.1
+      rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/td-agent/config.rb#L16
+    - gemfile: gemfiles/Gemfile.td-agent-4.1.0
+      rvm: 2.7.2 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.1.0/td-agent/config.rb#L19
   fast_finish: true
 
 script: bundle exec rake test

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 3.4.0
+
+- Enhancement - Enable to monitor batch request failure and retries : [#150](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/150) [#211](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/211)
+- Enhancement - Make sleep reliable by measuring actual slept time : [#162](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/162)
+- Enhancement - Add td-agent 4.1.0 and Fluentd 1.12.3 to test cases
+
 ## 3.3.0
 
 - Feature - Add web_identity_credentials configuration for IRSA : [#208](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/208) [#209](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/209)

data/README.md

@@ -1,6 +1,6 @@
 # Fluent plugin for Amazon Kinesis
 
-[![Build Status](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis)
+[![Build Status](https://travis-ci.com/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://travis-ci.com/awslabs/aws-fluent-plugin-kinesis)
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 [![Gem Downloads](https://img.shields.io/gem/dt/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 
@@ -408,6 +408,16 @@ Default:
 - `kinesis_firehose`: 4 MB
 - `kinesis_streams_aggregated`: 1 MB
 
+### drop_failed_records_after_batch_request_retries
+Boolean, default `true`.
+
+If *drop_failed_records_after_batch_request_retries* is enabled (default), the plugin will drop failed records when batch request fails after retrying max times configured as *retries_on_batch_request*. This dropping can be monitored from [monitor_agent](https://docs.fluentd.org/input/monitor_agent) or [fluent-plugin-prometheus](https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus) as *retry_count* or *num_errors* metrics.
+
+If *drop_failed_records_after_batch_request_retries* is disabled, the plugin will raise error and return chunk to Fluentd buffer when batch request fails after retrying max times. Fluentd will retry to send chunk records according to retry config in [Buffer Section](https://docs.fluentd.org/configuration/buffer-section). Note that this retryng may create duplicate records since [PutRecords API](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html) of Kinesis Data Streams and [PutRecordBatch API](https://docs.aws.amazon.com/firehose/latest/APIReference/API_PutRecordBatch.html) of Kinesis Data Firehose may return a partially successful response.
+
+### monitor_num_of_batch_request_retries
+Boolean, default `false`. If enabled, the plugin will increment *retry_count* monitoring metrics after internal retrying to send batch request. This configuration enables you to monitor [ProvisionedThroughputExceededException](https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html) from [monitor_agent](https://docs.fluentd.org/input/monitor_agent) or [fluent-plugin-prometheus](https://docs.fluentd.org/monitoring-fluentd/monitoring-prometheus). Note that *retry_count* metrics will be counted by the plugin in addition to original Fluentd buffering mechanism if *monitor_num_of_batch_request_retries* is enabled.
+
 ## Configuration: kinesis_streams
 Here are `kinesis_streams` specific configurations.
 

data/gemfiles/Gemfile.fluentd-0.14.22

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.fluentd-1.12.3

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+gem "fluentd", "1.12.3"

data/gemfiles/Gemfile.td-agent-3.1.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.1.1

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.2.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.2.1

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.3.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.4.0

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.4.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.4.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.149.0"
+gem "aws-sigv4", "1.1.0"
+gem "aws-sdk-core", "3.48.3"
+gem "aws-sdk-kms", "1.16.0"
+gem "aws-sdk-sqs", "1.13.0"
+gem "aws-sdk-s3", "1.36.0"
+gem "fluent-plugin-s3", "1.1.9"

data/gemfiles/Gemfile.td-agent-3.4.1

@@ -1,29 +1,12 @@
-#
-# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
 gemspec path: ".."
 
 # Specify related gems for td-agent v3.4.1
-# td-agent v3.4.0 has the same dependencies
 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L27
-# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/config/projects/td-agent3.rb#L27
 gem "fluentd", "1.4.2"
 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/plugin_gems.rb#L16-L23
-# https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.0/plugin_gems.rb#L16-L23
 gem "jmespath", "1.4.0"
 gem "aws-partitions", "1.149.0"
 gem "aws-sigv4", "1.1.0"

data/gemfiles/Gemfile.td-agent-3.5.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.5.1

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.6.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.7.0

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.7.0
+# https://docs.treasuredata.com/display/public/PD/The+td-agent+Change+Log
+gem "fluentd", "1.10.0"
+# https://github.com/treasure-data/omnibus-td-agent/commit/74920cb2a55e40195961a58c6b6de4bd5b3f1f87#diff-8d45eaa8758f7591f884c9e9917ea96105331a1daf5a891b4645b89e120f2d69R16-R23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.288.0"
+gem "aws-sigv4", "1.1.1"
+gem "aws-sdk-core", "3.92.0"
+gem "aws-sdk-kms", "1.30.0"
+gem "aws-sdk-sqs", "1.24.0"
+gem "aws-sdk-s3", "1.61.1"
+gem "fluent-plugin-s3", "1.3.0"

data/gemfiles/Gemfile.td-agent-3.7.1

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.8.0

@@ -1,17 +1,3 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec

data/gemfiles/Gemfile.td-agent-3.8.1

@@ -0,0 +1,18 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Maintenance release of td-agent v3
+# Specify related gems for td-agent v3.8.1
+# https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R27
+gem "fluentd", "1.11.5"
+# https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-8d45eaa8758f7591f884c9e9917ea96105331a1daf5a891b4645b89e120f2d69R19-R25
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.399.0"
+gem "aws-sigv4", "1.2.2"
+gem "aws-sdk-core", "3.109.3"
+gem "aws-sdk-kms", "1.39.0"
+gem "aws-sdk-sqs", "1.34.0"
+gem "aws-sdk-s3", "1.85.0"
+gem "fluent-plugin-s3", "1.4.0"

data/gemfiles/Gemfile.td-agent-4.0.0

@@ -0,0 +1,25 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.0.0
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L4
+gem "fluentd", "1.11.1"
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L31
+gem "prometheus-client", "0.9.0"
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L19-L20
+gem "fluent-plugin-prometheus", "1.8.0"
+gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L28
+gem "jmespath", "1.4.0"
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L8-L13
+gem "aws-partitions", "1.337.0"
+gem "aws-sdk-core", "3.102.1"
+gem "aws-sdk-kms", "1.35.0"
+gem "aws-sdk-s3", "1.72.0"
+gem "aws-sdk-sqs", "1.29.0"
+gem "aws-sigv4", "1.2.1"
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L23
+gem "fluent-plugin-s3", "1.3.3"

data/gemfiles/Gemfile.td-agent-4.0.1

@@ -1,26 +1,16 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
 gemspec path: ".."
 
 # Specify related gems for td-agent v4.0.1
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/td-agent/config.rb#L4
-gem "fluentd", "1.11.1"
+gem "fluentd", "1.11.2"
 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/gemfiles/linux/Gemfile#L35-L42
+gem "prometheus-client", "0.9.0"
+gem "fluent-plugin-prometheus", "1.8.2"
+gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
 gem "jmespath", "1.4.0"
 gem "aws-partitions", "1.352.0"
 gem "aws-sdk-core", "3.104.3"

data/gemfiles/Gemfile.td-agent-4.1.0

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.1.0
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.1.0/td-agent/config.rb#L4
+gem "fluentd", "1.12.1"
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.1.0/td-agent/Gemfile#L32-L42
+gem "prometheus-client", "0.9.0"
+gem "fluent-plugin-prometheus", "1.8.5"
+gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.427.0"
+gem "aws-sdk-core", "3.112.0"
+gem "aws-sdk-kms", "1.42.0"
+gem "aws-sdk-sqs", "1.36.0"
+gem "aws-sigv4", "1.2.2"
+gem "aws-sdk-s3", "1.88.1"
+gem "fluent-plugin-s3", "1.5.1"

data/lib/fluent/plugin/kinesis_helper/api.rb

@@ -14,6 +14,7 @@
 
 require 'fluent_plugin_kinesis/version'
 require 'fluent/configurable'
+require 'benchmark'
 
 module Fluent
   module Plugin
@@ -40,10 +41,12 @@ module Fluent
         module BatchRequest
           module BatchRequestParams
             include Fluent::Configurable
-            config_param :retries_on_batch_request, :integer, default: 8
-            config_param :reset_backoff_if_success, :bool,    default: true
-            config_param :batch_request_max_count,  :integer, default: nil
-            config_param :batch_request_max_size,   :integer, default: nil
+            config_param :retries_on_batch_request,                        :integer, default: 8
+            config_param :reset_backoff_if_success,                        :bool,    default: true
+            config_param :batch_request_max_count,                         :integer, default: nil
+            config_param :batch_request_max_size,                          :integer, default: nil
+            config_param :drop_failed_records_after_batch_request_retries, :bool,    default: true
+            config_param :monitor_num_of_batch_request_retries,            :bool,    default: false
           end
 
           def self.included(mod)
@@ -96,11 +99,9 @@ module Fluent
                 wait_second = backoff.next
                 msg = 'Retrying to request batch. Retry count: %3d, Retry records: %3d, Wait seconds %3.2f' % [retry_count+1, failed_records.size, wait_second]
                 log.warn(truncate msg)
-                # TODO: sleep() doesn't wait the given seconds sometime.
-                # The root cause is unknown so far, so I'd like to add debug print only. It should be fixed in the future.
-                log.debug("#{Thread.current.object_id} sleep start")
-                sleep(wait_second)
-                log.debug("#{Thread.current.object_id} sleep finish")
+                # Increment num_errors to monitor batch request retries from "monitor_agent" or "fluent-plugin-prometheus"
+                increment_num_errors if @monitor_num_of_batch_request_retries
+                reliable_sleep(wait_second)
                 batch_request_with_retry(retry_records(failed_records), retry_count+1, backoff: backoff, &block)
               else
                 give_up_retries(failed_records)
@@ -108,6 +109,18 @@ module Fluent
             end
           end
 
+          # Sleep seems to not sleep as long as we ask it, our guess is that something wakes up the thread,
+          # so we keep on going to sleep if that happens.
+          # TODO: find out who is causing the sleep to be too short and try to make them stop it instead
+          def reliable_sleep(wait_second)
+            loop do
+              actual = Benchmark.realtime { sleep(wait_second) }
+              break if actual >= wait_second
+              log.error("#{Thread.current.object_id} sleep failed expected #{wait_second} but slept #{actual}")
+              wait_second -= actual
+            end
+          end
+
           def any_records_shipped?(res)
             results(res).size > failed_count(res)
           end
@@ -164,6 +177,49 @@ module Fluent
                 record[:original]
               ])
             }
+
+            if @drop_failed_records_after_batch_request_retries
+              # Increment num_errors to monitor batch request failure from "monitor_agent" or "fluent-plugin-prometheus"
+              increment_num_errors
+            else
+              # Raise error and return chunk to Fluentd for retrying
+              case request_type
+              # @see https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Kinesis/Client.html#put_records-instance_method
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Kinesis/Errors.html
+              when :streams, :streams_aggregated
+                provisioned_throughput_exceeded_records = failed_records.select { |record| record[:error_code] == 'ProvisionedThroughputExceededException' }
+                target_failed_record = provisioned_throughput_exceeded_records.first || failed_records.first
+                target_error = provisioned_throughput_exceeded_records.empty? ?
+                  Aws::Kinesis::Errors::ServiceError :
+                  Aws::Kinesis::Errors::ProvisionedThroughputExceededException
+              # @see https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Firehose/Client.html#put_record_batch-instance_method
+              # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Firehose/Errors.html
+              when :firehose
+                service_unavailable_exception_records = failed_records.select { |record| record[:error_code] == 'ServiceUnavailableException' }
+                target_failed_record = service_unavailable_exception_records.first || failed_records.first
+                target_error = service_unavailable_exception_records.empty? ?
+                  Aws::Firehose::Errors::ServiceError :
+                  Aws::Firehose::Errors::ServiceUnavailableException
+              end
+              log.error("Raise #{target_failed_record[:error_code]} and return chunk to Fluentd buffer for retrying")
+              raise target_error.new(Seahorse::Client::RequestContext.new, target_failed_record[:error_message])
+            end
+          end
+
+          def increment_num_errors
+            # Prepare Fluent::Plugin::Output instance variables to count errors in this method.
+            # These instance variables are initialized here for possible future breaking changes of Fluentd.
+            @num_errors ||= 0
+            # @see https://github.com/fluent/fluentd/commit/d245454658d16170431d276fcd5849fb0d88ab2b
+            if Gem::Version.new(Fluent::VERSION) >= Gem::Version.new('1.7.0')
+              @counter_mutex ||= Mutex.new
+              @counter_mutex.synchronize{ @num_errors += 1 }
+            else
+              @counters_monitor ||= Monitor.new
+              @counters_monitor.synchronize{ @num_errors += 1 }
+            end
           end
 
           class Backoff

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '3.3.0'
+  VERSION = '3.4.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-02 00:00:00.000000000 Z
+date: 2021-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -284,19 +284,24 @@ files:
 - benchmark/task.rake
 - fluent-plugin-kinesis.gemspec
 - gemfiles/Gemfile.fluentd-0.14.22
-- gemfiles/Gemfile.fluentd-1.11.2
+- gemfiles/Gemfile.fluentd-1.12.3
 - gemfiles/Gemfile.td-agent-3.1.0
 - gemfiles/Gemfile.td-agent-3.1.1
 - gemfiles/Gemfile.td-agent-3.2.0
 - gemfiles/Gemfile.td-agent-3.2.1
 - gemfiles/Gemfile.td-agent-3.3.0
+- gemfiles/Gemfile.td-agent-3.4.0
 - gemfiles/Gemfile.td-agent-3.4.1
 - gemfiles/Gemfile.td-agent-3.5.0
 - gemfiles/Gemfile.td-agent-3.5.1
 - gemfiles/Gemfile.td-agent-3.6.0
+- gemfiles/Gemfile.td-agent-3.7.0
 - gemfiles/Gemfile.td-agent-3.7.1
 - gemfiles/Gemfile.td-agent-3.8.0
+- gemfiles/Gemfile.td-agent-3.8.1
+- gemfiles/Gemfile.td-agent-4.0.0
 - gemfiles/Gemfile.td-agent-4.0.1
+- gemfiles/Gemfile.td-agent-4.1.0
 - lib/fluent/plugin/kinesis.rb
 - lib/fluent/plugin/kinesis_helper/aggregator.rb
 - lib/fluent/plugin/kinesis_helper/api.rb
@@ -324,7 +329,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin that sends events to Amazon Kinesis.

data/gemfiles/Gemfile.fluentd-1.11.2

@@ -1,20 +0,0 @@
-#
-# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
-
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
-gemspec path: ".."
-
-gem "fluentd", "1.11.2"