fluent-plugin-kinesis 3.4.0 → 3.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +12 -7
  3. data/CHANGELOG.md +5 -0
  4. data/README.md +1 -1
  5. data/gemfiles/{Gemfile.fluentd-1.12.3 → Gemfile.fluentd-1.13.3} +1 -1
  6. data/gemfiles/Gemfile.td-agent-4.0.0 +7 -7
  7. data/gemfiles/Gemfile.td-agent-4.0.1 +3 -3
  8. data/gemfiles/Gemfile.td-agent-4.1.0 +3 -3
  9. data/gemfiles/Gemfile.td-agent-4.1.1 +21 -0
  10. data/gemfiles/Gemfile.td-agent-4.2.0 +21 -0
  11. data/lib/fluent/plugin/kinesis.rb +3 -3
  12. data/lib/fluent/plugin/out_kinesis_firehose.rb +1 -1
  13. data/lib/fluent/plugin/out_kinesis_streams.rb +1 -1
  14. data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +1 -1
  15. data/lib/fluent_plugin_kinesis/version.rb +1 -1
  16. metadata +5 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7a56428085f3d1f60d243071976cf17ef81ae64cc02176aa9106e0f38763e8c2
4
- data.tar.gz: d7176edca0e7fd26c7e6724a1c63c8817c7104666e1ed7108c25ef91e7449662
3
+ metadata.gz: 90a9718baa3e6849ce6ad7be32104672dcdf8993940c71d2d9fb666ce64b35e2
4
+ data.tar.gz: 064de8d0bbe66a8b8a8f4102bb686487fc0d7294904399dd745ce641b593c9bf
5
5
  SHA512:
6
- metadata.gz: 332e9a47297b9ee84f03527119b6e286301378ffce699c5fa3d0f23549f4e345754e439f2afc540190415bc02cb4bd466b15a17498560c564701115dc2698a51
7
- data.tar.gz: 7979f2ea6cdbda52973fd13f1b403f99ae3852ce9a1e3112b144a45e63af21b43b9bc3f6591f5f5004a5fe8df3c3f77cc7d85fbc4aa2150a92563ac1dc3fc4a7
6
+ metadata.gz: 248f0fd2de87171ea58d878de9450edf95eb1dee58e31bfaf1aa716080e3921f68aa8ebdac2cbf052759a7e4262d5e598c5380c0c75edef503c54c8e760f1d3e
7
+ data.tar.gz: 79128aa4b4d07372ecc8d53d2f7196fa442d2c7c1694e215daca6cac7aae4d097af3b0512ddeae74708c053d6fb92fdce3c87811f4c0a2f5060b2c76649597e4
data/.travis.yml CHANGED
@@ -5,11 +5,12 @@ sudo: false
5
5
  # Test with supported td-agent versions
6
6
  # https://docs.fluentd.org/quickstart/support#change-logs
7
7
  # https://docs.treasuredata.com/display/public/PD/The+td-agent+Change+Log
8
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
9
9
  matrix:
10
10
  include:
11
- - gemfile: gemfiles/Gemfile.fluentd-1.12.3 # Latest Fluentd / https://www.fluentd.org/download
12
- rvm: 2.7.3 # Latest supported Ruby (less than v3.0.0) / https://www.ruby-lang.org/en/downloads
11
+ - gemfile: gemfiles/Gemfile.fluentd-1.13.3 # Latest Fluentd / https://www.fluentd.org/download
12
+ rvm: 3.0.2 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads
13
+ dist: focal
13
14
  - gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
14
15
  rvm: 2.3.0 # Oldest supported Ruby
15
16
  - gemfile: gemfiles/Gemfile.td-agent-3.1.0
@@ -41,11 +42,15 @@ matrix:
41
42
  - gemfile: gemfiles/Gemfile.td-agent-3.8.1 # Maintenance release of td-agent v3
42
43
  rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R22
43
44
  # - gemfile: gemfiles/Gemfile.td-agent-4.0.0
44
- # rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L16
45
+ # rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L16
45
46
  - gemfile: gemfiles/Gemfile.td-agent-4.0.1
46
- rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/td-agent/config.rb#L16
47
- - gemfile: gemfiles/Gemfile.td-agent-4.1.0
48
- rvm: 2.7.2 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.1.0/td-agent/config.rb#L19
47
+ rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L16
48
+ # - gemfile: gemfiles/Gemfile.td-agent-4.1.0
49
+ # rvm: 2.7.2 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L19
50
+ - gemfile: gemfiles/Gemfile.td-agent-4.1.1
51
+ rvm: 2.7.3 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L17
52
+ - gemfile: gemfiles/Gemfile.td-agent-4.2.0
53
+ rvm: 2.7.4 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L21
49
54
  fast_finish: true
50
55
 
51
56
  script: bundle exec rake test
data/CHANGELOG.md CHANGED
@@ -1,5 +1,10 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## 3.4.1
4
+
5
+ - Enhancement - Add stream name to debug logs to identify : [#214](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/214)
6
+ - Enhancement - Add td-agent 4.1.1, 4.2.0 and Fluentd 1.13.3 with Ruby 3.0.2 to test cases
7
+
3
8
  ## 3.4.0
4
9
 
5
10
  - Enhancement - Enable to monitor batch request failure and retries : [#150](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/150) [#211](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/211)
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Fluent plugin for Amazon Kinesis
2
2
 
3
- [![Build Status](https://travis-ci.com/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://travis-ci.com/awslabs/aws-fluent-plugin-kinesis)
3
+ [![Build Status](https://api.travis-ci.com/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://app.travis-ci.com/github/awslabs/aws-fluent-plugin-kinesis)
4
4
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
5
5
  [![Gem Downloads](https://img.shields.io/gem/dt/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
6
6
 
data/gemfiles/{Gemfile.fluentd-1.12.3 → Gemfile.fluentd-1.13.3} RENAMED
@@ -3,4 +3,4 @@ source 'https://rubygems.org'
3
3
  # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
4
  gemspec path: ".."
5
5
 
6
- gem "fluentd", "1.12.3"
6
+ gem "fluentd", "1.13.3"
data/gemfiles/Gemfile.td-agent-4.0.0 CHANGED
@@ -4,22 +4,22 @@ source 'https://rubygems.org'
4
4
  gemspec path: ".."
5
5
 
6
6
  # Specify related gems for td-agent v4.0.0
7
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
8
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L4
7
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L4
9
9
  gem "fluentd", "1.11.1"
10
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L31
10
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L31
11
11
  gem "prometheus-client", "0.9.0"
12
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L19-L20
12
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L19-L20
13
13
  gem "fluent-plugin-prometheus", "1.8.0"
14
14
  gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
15
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L28
15
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L28
16
16
  gem "jmespath", "1.4.0"
17
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L8-L13
17
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L8-L13
18
18
  gem "aws-partitions", "1.337.0"
19
19
  gem "aws-sdk-core", "3.102.1"
20
20
  gem "aws-sdk-kms", "1.35.0"
21
21
  gem "aws-sdk-s3", "1.72.0"
22
22
  gem "aws-sdk-sqs", "1.29.0"
23
23
  gem "aws-sigv4", "1.2.1"
24
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L23
24
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L23
25
25
  gem "fluent-plugin-s3", "1.3.3"
data/gemfiles/Gemfile.td-agent-4.0.1 CHANGED
@@ -4,10 +4,10 @@ source 'https://rubygems.org'
4
4
  gemspec path: ".."
5
5
 
6
6
  # Specify related gems for td-agent v4.0.1
7
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
8
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/td-agent/config.rb#L4
7
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L4
9
9
  gem "fluentd", "1.11.2"
10
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/gemfiles/linux/Gemfile#L35-L42
10
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/gemfiles/linux/Gemfile#L35-L42
11
11
  gem "prometheus-client", "0.9.0"
12
12
  gem "fluent-plugin-prometheus", "1.8.2"
13
13
  gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
data/gemfiles/Gemfile.td-agent-4.1.0 CHANGED
@@ -4,10 +4,10 @@ source 'https://rubygems.org'
4
4
  gemspec path: ".."
5
5
 
6
6
  # Specify related gems for td-agent v4.1.0
7
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
8
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.1.0/td-agent/config.rb#L4
7
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L4
9
9
  gem "fluentd", "1.12.1"
10
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.1.0/td-agent/Gemfile#L32-L42
10
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/Gemfile#L32-L42
11
11
  gem "prometheus-client", "0.9.0"
12
12
  gem "fluent-plugin-prometheus", "1.8.5"
13
13
  gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
data/gemfiles/Gemfile.td-agent-4.1.1 ADDED
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for td-agent v4.1.1
7
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L4
9
+ gem "fluentd", "1.12.3"
10
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/Gemfile#L32-L42
11
+ gem "prometheus-client", "0.9.0"
12
+ gem "fluent-plugin-prometheus", "1.8.5"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
14
+ gem "jmespath", "1.4.0"
15
+ gem "aws-partitions", "1.446.0"
16
+ gem "aws-sdk-core", "3.114.0"
17
+ gem "aws-sdk-kms", "1.43.0"
18
+ gem "aws-sdk-sqs", "1.38.0"
19
+ gem "aws-sigv4", "1.2.3"
20
+ gem "aws-sdk-s3", "1.93.1"
21
+ gem "fluent-plugin-s3", "1.6.0"
data/gemfiles/Gemfile.td-agent-4.2.0 ADDED
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for td-agent v4.2.0
7
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L4
9
+ gem "fluentd", "1.13.3"
10
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/Gemfile#L42-L52
11
+ gem "prometheus-client", "2.1.0"
12
+ gem "fluent-plugin-prometheus", "2.0.1"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.1.0"
14
+ gem "jmespath", "1.4.0"
15
+ gem "aws-partitions", "1.478.0"
16
+ gem "aws-sdk-core", "3.117.0"
17
+ gem "aws-sdk-kms", "1.44.0"
18
+ gem "aws-sdk-sqs", "1.40.0"
19
+ gem "aws-sigv4", "1.2.4"
20
+ gem "aws-sdk-s3", "1.96.1"
21
+ gem "fluent-plugin-s3", "1.6.0"
data/lib/fluent/plugin/kinesis.rb CHANGED
@@ -146,13 +146,13 @@ module Fluent
146
146
  include Fluent::MessagePackFactory::Mixin
147
147
  end
148
148
 
149
- def write_records_batch(chunk, &block)
149
+ def write_records_batch(chunk, stream_name, &block)
150
150
  unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
151
151
  records = chunk.to_enum(:msgpack_each)
152
152
  split_to_batches(records) do |batch, size|
153
- log.debug(sprintf "Write chunk %s / %3d records / %4d KB", unique_id, batch.size, size/1024)
153
+ log.debug(sprintf "%s: Write chunk %s / %3d records / %4d KB", stream_name, unique_id, batch.size, size/1024)
154
154
  batch_request_with_retry(batch, &block)
155
- log.debug("Finish writing chunk")
155
+ log.debug(sprintf "%s: Finish writing chunk", stream_name)
156
156
  end
157
157
  end
158
158
 
data/lib/fluent/plugin/out_kinesis_firehose.rb CHANGED
@@ -45,7 +45,7 @@ module Fluent
45
45
 
46
46
  def write(chunk)
47
47
  delivery_stream_name = extract_placeholders(@delivery_stream_name, chunk)
48
- write_records_batch(chunk) do |batch|
48
+ write_records_batch(chunk, delivery_stream_name) do |batch|
49
49
  records = batch.map{|(data)|
50
50
  { data: data }
51
51
  }
data/lib/fluent/plugin/out_kinesis_streams.rb CHANGED
@@ -42,7 +42,7 @@ module Fluent
42
42
 
43
43
  def write(chunk)
44
44
  stream_name = extract_placeholders(@stream_name, chunk)
45
- write_records_batch(chunk) do |batch|
45
+ write_records_batch(chunk, stream_name) do |batch|
46
46
  records = batch.map{|(data, partition_key)|
47
47
  { data: data, partition_key: partition_key }
48
48
  }
data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb CHANGED
@@ -44,7 +44,7 @@ module Fluent
44
44
 
45
45
  def write(chunk)
46
46
  stream_name = extract_placeholders(@stream_name, chunk)
47
- write_records_batch(chunk) do |batch|
47
+ write_records_batch(chunk, stream_name) do |batch|
48
48
  key = @partition_key_generator.call
49
49
  records = batch.map{|(data)|data}
50
50
  client.put_records(
data/lib/fluent_plugin_kinesis/version.rb CHANGED
@@ -13,5 +13,5 @@
13
13
  # language governing permissions and limitations under the License.
14
14
 
15
15
  module FluentPluginKinesis
16
- VERSION = '3.4.0'
16
+ VERSION = '3.4.1'
17
17
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kinesis
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.4.0
4
+ version: 3.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon Web Services
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-05-03 00:00:00.000000000 Z
11
+ date: 2021-10-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -284,7 +284,7 @@ files:
284
284
  - benchmark/task.rake
285
285
  - fluent-plugin-kinesis.gemspec
286
286
  - gemfiles/Gemfile.fluentd-0.14.22
287
- - gemfiles/Gemfile.fluentd-1.12.3
287
+ - gemfiles/Gemfile.fluentd-1.13.3
288
288
  - gemfiles/Gemfile.td-agent-3.1.0
289
289
  - gemfiles/Gemfile.td-agent-3.1.1
290
290
  - gemfiles/Gemfile.td-agent-3.2.0
@@ -302,6 +302,8 @@ files:
302
302
  - gemfiles/Gemfile.td-agent-4.0.0
303
303
  - gemfiles/Gemfile.td-agent-4.0.1
304
304
  - gemfiles/Gemfile.td-agent-4.1.0
305
+ - gemfiles/Gemfile.td-agent-4.1.1
306
+ - gemfiles/Gemfile.td-agent-4.2.0
305
307
  - lib/fluent/plugin/kinesis.rb
306
308
  - lib/fluent/plugin/kinesis_helper/aggregator.rb
307
309
  - lib/fluent/plugin/kinesis_helper/api.rb