fluent-plugin-kinesis 3.2.3 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d1bd84d96b64e32b8aac649672adeb5333c7931cb7a7b5a4ec13cd6a8031f1be
-  data.tar.gz: 657ed567a713283fbc40c6ea7ed1fe7cc940f7898547e584aa4ed447fbfabdd6
+  metadata.gz: 98a12a1569d674676d986f802935ca80df435b4cd2acc4dd15d1951c8b020671
+  data.tar.gz: 63238ffd8abeb4c038af46eefff7c528277fa2225c8ca9affbe2714f433a6417
 SHA512:
-  metadata.gz: 6da3c960496b64b446718ff79cd95523c3058372893e313a5049ca3fad51895a613bc06b5ce5e8dc6a71966b5c0dbe00fcfd9b0b5ff7438813f0be581efc1122
-  data.tar.gz: e5fff3586fd0f1519a9f686ccaca4212a45b30c9f7199fb82460bbd04d4a98c080fe1c93bf1dc33099b432dab9da4118a943c619e31069b082ce98ac366b80e2
+  metadata.gz: c49fb3a4a2a6443375c52a7e416686188160e7a8e85d997c4d4105ee18b06c3e9d94bfcd25e42ef971a3128c55575a9296012a6a5343bc3c9d466a5083c8d78c
+  data.tar.gz: 6d2d7fd0012cfcb2282aa6982ff6897333155a0f43c19a90eb365b147f42aac7f16b06dd38924a16403494295031103569529f8fe42055547e077ca558581005

data/.gitignore

@@ -4,6 +4,7 @@
 /gemfiles/Gemfile*.lock
 /_yardoc/
 /coverage/
+/vendor/
 /doc/
 /pkg/
 /spec/reports/

data/.travis.yml

@@ -3,16 +3,16 @@ os: linux
 sudo: false
 
 gemfile:
-  - Gemfile
   - gemfiles/Gemfile.td-agent-3.8.0
+  - gemfiles/Gemfile.td-agent-4.0.1
 
 # Test with supported td-agent versions
 # https://tddocs.atlassian.net/wiki/spaces/PD/pages/6291635/The+td-agent+ChangeLog
+# https://docs.fluentd.org/quickstart/td-agent-v2-vs-v3-vs-v4
 matrix:
   include:
-    - rvm: 2.6.6 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads/
-      # Waiting protobuf support for Ruby 2.7 / https://github.com/protocolbuffers/protobuf/issues/7070
-      gemfile: gemfiles/Gemfile.fluentd-1.11.1 # Latest Fluentd / https://www.fluentd.org/download
+    - rvm: 2.7.2 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads/
+      gemfile: gemfiles/Gemfile.fluentd-1.11.2 # Latest Fluentd / https://www.fluentd.org/download
     - rvm: 2.3.0 # Oldest supported Ruby
       gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
     - rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
@@ -31,6 +31,9 @@ matrix:
       gemfile: gemfiles/Gemfile.td-agent-3.7.1
     - rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L22
       gemfile: gemfiles/Gemfile.td-agent-3.8.0
+    - rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/td-agent/config.rb#L16
+      gemfile: gemfiles/Gemfile.td-agent-4.0.1
+      
   fast_finish: true
 
 script: bundle exec rake test

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # CHANGELOG
 
+## 3.3.0
+
+- Feature - Add web_identity_credentials configuration for IRSA : [#208](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/208) [#209](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/209)
+- Enhancement - Remove strict gem pinning of google-protobuf to support Ruby 2.7 : [#199](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/199) [#206](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/206)
+- Enhancement - Add td-agent v4 (4.0.1) and Fluentd 1.11.2 to test cases
+
 ## 3.2.3
 
 - Enhancement - Add placeholder support for delivery_stream_name of kinesis_firehose : [#203](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/203) [#204](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/204)

data/CONTRIBUTORS.txt

@@ -4,3 +4,5 @@ to the Fluent Plugin for Amazon Kinesis Streams and Amazon Kinesis Firehose. Giv
 Genki Sugawara
 sgwr_dts@yahoo.co.jp
 https://github.com/winebarrel
+https://github.com/adammw
+https://github.com/cosmo0920

data/README.md

@@ -1,6 +1,5 @@
 # Fluent plugin for Amazon Kinesis
 
-[![Gitter](https://badges.gitter.im/awslabs/aws-fluent-plugin-kinesis.svg)](https://gitter.im/awslabs/aws-fluent-plugin-kinesis?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 [![Build Status](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://travis-ci.org/awslabs/aws-fluent-plugin-kinesis)
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
 [![Gem Downloads](https://img.shields.io/gem/dt/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
@@ -186,6 +185,50 @@ It should be added to assume_role_credentials configuration stanza in the next f
 
 STS API endpoint url. This can be used to override the default global STS API endpoint of sts.amazonaws.com. Using regional endpoints may be preferred to reduce latency, and are required if utilizing a PrivateLink VPC Endpoint for STS API calls.
 
+
+### web_identity_credentials
+
+Similar to the assume_role_credentials, but for usage in EKS.
+
+    <match *>
+      @type kinesis_streams
+
+      <web_identity_credentials>
+        role_arn          ROLE_ARN
+        role_session_name ROLE_SESSION_NAME
+        web_identity_token_file AWS_WEB_IDENTITY_TOKEN_FILE
+      </web_identity_credentials>
+    </match>
+
+See also:
+
+*   [Using IAM Roles - AWS Identity and Access Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
+*   [IAM Roles For Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html)
+*   [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
+*   [Aws::AssumeRoleWebIdentityCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleWebIdentityCredentials.html)
+
+**role_arn (required)**
+
+The Amazon Resource Name (ARN) of the role to assume.
+
+**role_session_name (required)**
+
+An identifier for the assumed role session.
+
+**web_identity_token_file (required)**
+
+The absolute path to the file on disk containing the OIDC token
+
+**policy**
+
+An IAM policy in JSON format.
+
+**duration_seconds**
+
+The duration, in seconds, of the role session. The value can range from
+900 seconds (15 minutes) to 43200 seconds (12 hours). By default, the value
+is set to 3600 seconds (1 hour).
+
 ### instance_profile_credentials
 
 Retrieve temporary security credentials via HTTP request. This is useful on EC2 instance.

data/fluent-plugin-kinesis.gemspec

@@ -55,7 +55,7 @@ Gem::Specification.new do |spec|
   #   https://github.com/aws/aws-sdk-ruby/commit/e26577d2a426a4be79cd2d9edc1a4a4176e388ba#diff-10f50e27b30c3dc522b3c25db5782e2e
   spec.add_dependency "aws-sdk-firehose", "~> 1", "!= 1.5", "!= 1.9", "!= 1.15"
 
-  spec.add_dependency "google-protobuf", "~> 3", "< 3.12"
+  spec.add_dependency "google-protobuf", "~> 3"
 
   spec.add_development_dependency "bundler", ">= 1.10"
   spec.add_development_dependency "rake", ">= 10.0"

data/gemfiles/{Gemfile.fluentd-1.11.1 → Gemfile.fluentd-1.11.2}

@@ -17,4 +17,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
 gemspec path: ".."
 
-gem "fluentd", "1.11.1"
+gem "fluentd", "1.11.2"

data/gemfiles/Gemfile.td-agent-4.0.1

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v4.0.1
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/td-agent/config.rb#L4
+gem "fluentd", "1.11.1"
+# https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/gemfiles/linux/Gemfile#L35-L42
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.352.0"
+gem "aws-sdk-core", "3.104.3"
+gem "aws-sdk-kms", "1.36.0"
+gem "aws-sdk-sqs", "1.30.0"
+gem "aws-sigv4", "1.2.1"
+gem "aws-sdk-s3", "1.75.0"
+gem "fluent-plugin-s3", "1.4.0"

data/lib/fluent/plugin/kinesis.rb

@@ -84,6 +84,10 @@ module Fluent
         true
       end
 
+      def formatted_to_msgpack_binary?
+        true
+      end
+
       private
 
       def data_formatter_create(conf)
@@ -144,13 +148,11 @@ module Fluent
 
       def write_records_batch(chunk, &block)
         unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
-        chunk.open do |io|
-          records = msgpack_unpacker(io).to_enum
-          split_to_batches(records) do |batch, size|
-            log.debug(sprintf "Write chunk %s / %3d records / %4d KB", unique_id, batch.size, size/1024)
-            batch_request_with_retry(batch, &block)
-            log.debug("Finish writing chunk")
-          end
+        records = chunk.to_enum(:msgpack_each)
+        split_to_batches(records) do |batch, size|
+          log.debug(sprintf "Write chunk %s / %3d records / %4d KB", unique_id, batch.size, size/1024)
+          batch_request_with_retry(batch, &block)
+          log.debug("Finish writing chunk")
         end
       end
 

data/lib/fluent/plugin/kinesis_helper/client.rb

@@ -46,6 +46,20 @@ module Fluent
             desc "A URL for a regional STS API endpoint, the default is global"
             config_param :sts_endpoint_url, :string, default: nil
           end
+          # Refer to the following link for additional parameters that could be added:
+          # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
+          config_section :web_identity_credentials, multi: false do
+            desc "The Amazon Resource Name (ARN) of the role to assume"
+            config_param :role_arn, :string
+            desc "An identifier for the assumed role session"
+            config_param :role_session_name, :string
+            desc "The absolute path to the file on disk containing the OIDC token"
+            config_param :web_identity_token_file, :string, default: nil #required
+            desc "An IAM policy in JSON format"
+            config_param :policy, :string, default: nil
+            desc "The duration, in seconds, of the role session (900-43200)"
+            config_param :duration_seconds, :time, default: nil
+          end
           config_section :instance_profile_credentials, multi: false do
             desc "Number of times to retry when retrieving credentials"
             config_param :retries, :integer, default: nil
@@ -149,6 +163,17 @@ module Fluent
                 credentials_options[:client] = Aws::STS::Client.new(region: @region)
             end
             options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
+          when @web_identity_credentials
+            c = @web_identity_credentials
+            credentials_options[:role_arn] = c.role_arn
+            credentials_options[:role_session_name] = c.role_session_name
+            credentials_options[:web_identity_token_file] = c.web_identity_token_file
+            credentials_options[:policy] = c.policy if c.policy
+            credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
+            if @region
+              credentials_options[:client] = Aws::STS::Client.new(:region => @region)
+            end
+            options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
           when @instance_profile_credentials
             c = @instance_profile_credentials
             credentials_options[:retries] = c.retries if c.retries

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '3.2.3'
+  VERSION = '3.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 3.2.3
+  version: 3.3.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-15 00:00:00.000000000 Z
+date: 2020-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -101,9 +101,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -111,9 +108,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '3.12'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -290,7 +284,7 @@ files:
 - benchmark/task.rake
 - fluent-plugin-kinesis.gemspec
 - gemfiles/Gemfile.fluentd-0.14.22
-- gemfiles/Gemfile.fluentd-1.11.1
+- gemfiles/Gemfile.fluentd-1.11.2
 - gemfiles/Gemfile.td-agent-3.1.0
 - gemfiles/Gemfile.td-agent-3.1.1
 - gemfiles/Gemfile.td-agent-3.2.0
@@ -302,6 +296,7 @@ files:
 - gemfiles/Gemfile.td-agent-3.6.0
 - gemfiles/Gemfile.td-agent-3.7.1
 - gemfiles/Gemfile.td-agent-3.8.0
+- gemfiles/Gemfile.td-agent-4.0.1
 - lib/fluent/plugin/kinesis.rb
 - lib/fluent/plugin/kinesis_helper/aggregator.rb
 - lib/fluent/plugin/kinesis_helper/api.rb