fluent-plugin-kinesis 3.2.2 → 3.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f410e3829237caf34033008ff210d278bd9e11e127e48f4611042065b2b5ff2
-  data.tar.gz: 8085f421bf1ca76e6d2cbc1e91c9168ddef470427591c4e386f204f20e9214e9
+  metadata.gz: d1bd84d96b64e32b8aac649672adeb5333c7931cb7a7b5a4ec13cd6a8031f1be
+  data.tar.gz: 657ed567a713283fbc40c6ea7ed1fe7cc940f7898547e584aa4ed447fbfabdd6
 SHA512:
-  metadata.gz: 2ea0c5855780a12ff28de449a5def77ca74b21d455a72098b2da8feb3e67847c4ceb4d562739f84dece5fa91f448d6a665838da19e539234458b3a0900bb768b
-  data.tar.gz: '0844b285486b2bb81e5d18bed60ebdee24dce74e4a1f00a28b0d6afa8af69f6c05fa016aa884424e8e61c232dc506c53ec1bf560f3b6ab228ce5f7141e95db17'
+  metadata.gz: 6da3c960496b64b446718ff79cd95523c3058372893e313a5049ca3fad51895a613bc06b5ce5e8dc6a71966b5c0dbe00fcfd9b0b5ff7438813f0be581efc1122
+  data.tar.gz: e5fff3586fd0f1519a9f686ccaca4212a45b30c9f7199fb82460bbd04d4a98c080fe1c93bf1dc33099b432dab9da4118a943c619e31069b082ce98ac366b80e2

data/.travis.yml

@@ -4,32 +4,33 @@ sudo: false
 
 gemfile:
   - Gemfile
-  - gemfiles/Gemfile.td-agent-3.5.1
+  - gemfiles/Gemfile.td-agent-3.8.0
 
 # Test with supported td-agent versions
-# https://support.treasuredata.com/hc/en-us/articles/360001479187-The-td-agent-ChangeLog
+# https://tddocs.atlassian.net/wiki/spaces/PD/pages/6291635/The+td-agent+ChangeLog
 matrix:
   include:
-    - rvm: 2.6.5
-      gemfile: gemfiles/Gemfile.fluentd-1.9.1 # Latest Fluentd
-    - rvm: 2.4.1
+    - rvm: 2.6.6 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads/
+      # Waiting protobuf support for Ruby 2.7 / https://github.com/protocolbuffers/protobuf/issues/7070
+      gemfile: gemfiles/Gemfile.fluentd-1.11.1 # Latest Fluentd / https://www.fluentd.org/download
+    - rvm: 2.3.0 # Oldest supported Ruby
       gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
-    - rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L20
-      gemfile: gemfiles/Gemfile.td-agent-3.1.0
     - rvm: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.1/config/projects/td-agent3.rb#L17
       gemfile: gemfiles/Gemfile.td-agent-3.1.1
-    - rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.2.0
     - rvm: 2.4.4 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.1/config/projects/td-agent3.rb#L22
       gemfile: gemfiles/Gemfile.td-agent-3.2.1
     - rvm: 2.4.5 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/config/projects/td-agent3.rb#L22
       gemfile: gemfiles/Gemfile.td-agent-3.3.0
     - rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.4.1/config/projects/td-agent3.rb#L22
       gemfile: gemfiles/Gemfile.td-agent-3.4.1
-    - rvm: 2.4.6 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.0/config/projects/td-agent3.rb#L22
-      gemfile: gemfiles/Gemfile.td-agent-3.5.0
     - rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.5.1/config/projects/td-agent3.rb#L22
       gemfile: gemfiles/Gemfile.td-agent-3.5.1
+    - rvm: 2.4.9 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.6.0
+    - rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.7.1
+    - rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L22
+      gemfile: gemfiles/Gemfile.td-agent-3.8.0
   fast_finish: true
 
 script: bundle exec rake test

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## 3.2.3
+
+- Enhancement - Add placeholder support for delivery_stream_name of kinesis_firehose : [#203](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/203) [#204](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/204)
+
 ## 3.2.2
 
 - Enhancement - Make more strict gem pinning to deal with google-protobuf requiring Ruby 2.5+ : [#199](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/199)

data/README.md

@@ -404,6 +404,30 @@ Here are `kinesis_firehose` specific configurations.
 ### delivery_stream_name
 Name of the delivery stream to put data.
 
+As of Fluentd v1, built-in placeholders are supported. Now, you can also use built-in placeholders for this parameter.
+
+**NOTE:**
+Built-in placeholders require target key information in your buffer section attributes.
+
+e.g.)
+
+When you specify the following `delivery_stream_name` configuration with built-in placeholder:
+
+```aconf
+delivery_stream_name "${$.kubernetes.annotations.kinesis_firehose_streams}"
+```
+
+you ought to specify the corresponding attributes in buffer section:
+
+```aconf
+# $.kubernetes.annotations.kinesis_firehose_streams needs to be set in buffer attributes
+<buffer $.kubernetes.annotations.kinesis_firehose_streams>
+   # ...
+</buffer>
+```
+
+For more details, refer [Placeholders section in the official Fluentd document](https://docs.fluentd.org/configuration/buffer-section#placeholders).
+
 ### append_new_line
 Boolean. Default `true`. If it is enabled, the plugin adds new line character (`\n`) to each serialized record.  
 Before appending `\n`, plugin calls chomp and removes separator from the end of each record as [chomp_record](#chomp_record) is `true`. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) output with default configuration ([append_new_line](#append_new_line) is `true`). If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).

data/gemfiles/{Gemfile.fluentd-1.9.1 → Gemfile.fluentd-1.11.1}

@@ -17,4 +17,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
 gemspec path: ".."
 
-gem "fluentd", "1.9.1"
+gem "fluentd", "1.11.1"

data/gemfiles/Gemfile.td-agent-3.6.0

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.6.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.9.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.6.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.273.0"
+gem "aws-sigv4", "1.1.0"
+gem "aws-sdk-core", "3.90.0"
+gem "aws-sdk-kms", "1.29.0"
+gem "aws-sdk-sqs", "1.23.1"
+gem "aws-sdk-s3", "1.60.2"
+gem "fluent-plugin-s3", "1.3.0"

data/gemfiles/Gemfile.td-agent-3.7.1

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.7.1
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.10.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.7.1/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.297.0"
+gem "aws-sigv4", "1.1.1"
+gem "aws-sdk-core", "3.94.0"
+gem "aws-sdk-kms", "1.30.0"
+gem "aws-sdk-sqs", "1.24.0"
+gem "aws-sdk-s3", "1.63.0"
+gem "fluent-plugin-s3", "1.3.1"

data/gemfiles/Gemfile.td-agent-3.8.0

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.8.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.11.1"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.8.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.332.0"
+gem "aws-sigv4", "1.2.0"
+gem "aws-sdk-core", "3.100.0"
+gem "aws-sdk-kms", "1.34.1"
+gem "aws-sdk-sqs", "1.28.0"
+gem "aws-sdk-s3", "1.69.1"
+gem "fluent-plugin-s3", "1.3.2"

data/lib/fluent/plugin/out_kinesis_firehose.rb

@@ -44,12 +44,13 @@ module Fluent
       end
 
       def write(chunk)
+        delivery_stream_name = extract_placeholders(@delivery_stream_name, chunk)
         write_records_batch(chunk) do |batch|
           records = batch.map{|(data)|
             { data: data }
           }
           client.put_record_batch(
-            delivery_stream_name: @delivery_stream_name,
+            delivery_stream_name: delivery_stream_name,
             records: records,
           )
         end

data/lib/fluent_plugin_kinesis/version.rb

@@ -13,5 +13,5 @@
 # language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '3.2.2'
+  VERSION = '3.2.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 3.2.2
+  version: 3.2.3
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-19 00:00:00.000000000 Z
+date: 2020-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -290,7 +290,7 @@ files:
 - benchmark/task.rake
 - fluent-plugin-kinesis.gemspec
 - gemfiles/Gemfile.fluentd-0.14.22
-- gemfiles/Gemfile.fluentd-1.9.1
+- gemfiles/Gemfile.fluentd-1.11.1
 - gemfiles/Gemfile.td-agent-3.1.0
 - gemfiles/Gemfile.td-agent-3.1.1
 - gemfiles/Gemfile.td-agent-3.2.0
@@ -299,6 +299,9 @@ files:
 - gemfiles/Gemfile.td-agent-3.4.1
 - gemfiles/Gemfile.td-agent-3.5.0
 - gemfiles/Gemfile.td-agent-3.5.1
+- gemfiles/Gemfile.td-agent-3.6.0
+- gemfiles/Gemfile.td-agent-3.7.1
+- gemfiles/Gemfile.td-agent-3.8.0
 - lib/fluent/plugin/kinesis.rb
 - lib/fluent/plugin/kinesis_helper/aggregator.rb
 - lib/fluent/plugin/kinesis_helper/api.rb