fluent-plugin-kinesis 3.4.0 → 3.4.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (16) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +12 -7
  3. data/CHANGELOG.md +5 -0
  4. data/README.md +1 -1
  5. data/gemfiles/{Gemfile.fluentd-1.12.3 → Gemfile.fluentd-1.13.3} +1 -1
  6. data/gemfiles/Gemfile.td-agent-4.0.0 +7 -7
  7. data/gemfiles/Gemfile.td-agent-4.0.1 +3 -3
  8. data/gemfiles/Gemfile.td-agent-4.1.0 +3 -3
  9. data/gemfiles/Gemfile.td-agent-4.1.1 +21 -0
  10. data/gemfiles/Gemfile.td-agent-4.2.0 +21 -0
  11. data/lib/fluent/plugin/kinesis.rb +3 -3
  12. data/lib/fluent/plugin/out_kinesis_firehose.rb +1 -1
  13. data/lib/fluent/plugin/out_kinesis_streams.rb +1 -1
  14. data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +1 -1
  15. data/lib/fluent_plugin_kinesis/version.rb +1 -1
  16. metadata +5 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 7a56428085f3d1f60d243071976cf17ef81ae64cc02176aa9106e0f38763e8c2
4
- data.tar.gz: d7176edca0e7fd26c7e6724a1c63c8817c7104666e1ed7108c25ef91e7449662
3
+ metadata.gz: 90a9718baa3e6849ce6ad7be32104672dcdf8993940c71d2d9fb666ce64b35e2
4
+ data.tar.gz: 064de8d0bbe66a8b8a8f4102bb686487fc0d7294904399dd745ce641b593c9bf
5
5
  SHA512:
6
- metadata.gz: 332e9a47297b9ee84f03527119b6e286301378ffce699c5fa3d0f23549f4e345754e439f2afc540190415bc02cb4bd466b15a17498560c564701115dc2698a51
7
- data.tar.gz: 7979f2ea6cdbda52973fd13f1b403f99ae3852ce9a1e3112b144a45e63af21b43b9bc3f6591f5f5004a5fe8df3c3f77cc7d85fbc4aa2150a92563ac1dc3fc4a7
6
+ metadata.gz: 248f0fd2de87171ea58d878de9450edf95eb1dee58e31bfaf1aa716080e3921f68aa8ebdac2cbf052759a7e4262d5e598c5380c0c75edef503c54c8e760f1d3e
7
+ data.tar.gz: 79128aa4b4d07372ecc8d53d2f7196fa442d2c7c1694e215daca6cac7aae4d097af3b0512ddeae74708c053d6fb92fdce3c87811f4c0a2f5060b2c76649597e4
data/.travis.yml CHANGED
@@ -5,11 +5,12 @@ sudo: false
5
5
  # Test with supported td-agent versions
6
6
  # https://docs.fluentd.org/quickstart/support#change-logs
7
7
  # https://docs.treasuredata.com/display/public/PD/The+td-agent+Change+Log
8
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
9
9
  matrix:
10
10
  include:
11
- - gemfile: gemfiles/Gemfile.fluentd-1.12.3 # Latest Fluentd / https://www.fluentd.org/download
12
- rvm: 2.7.3 # Latest supported Ruby (less than v3.0.0) / https://www.ruby-lang.org/en/downloads
11
+ - gemfile: gemfiles/Gemfile.fluentd-1.13.3 # Latest Fluentd / https://www.fluentd.org/download
12
+ rvm: 3.0.2 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads
13
+ dist: focal
13
14
  - gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
14
15
  rvm: 2.3.0 # Oldest supported Ruby
15
16
  - gemfile: gemfiles/Gemfile.td-agent-3.1.0
@@ -41,11 +42,15 @@ matrix:
41
42
  - gemfile: gemfiles/Gemfile.td-agent-3.8.1 # Maintenance release of td-agent v3
42
43
  rvm: 2.4.10 # https://github.com/treasure-data/omnibus-td-agent/pull/273/files#diff-f73e763ace61e7c4bf48bf94bd6e295bc2cfd1ce5f9e38a4d19e266353e37498R22
43
44
  # - gemfile: gemfiles/Gemfile.td-agent-4.0.0
44
- # rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L16
45
+ # rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L16
45
46
  - gemfile: gemfiles/Gemfile.td-agent-4.0.1
46
- rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/td-agent/config.rb#L16
47
- - gemfile: gemfiles/Gemfile.td-agent-4.1.0
48
- rvm: 2.7.2 # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.1.0/td-agent/config.rb#L19
47
+ rvm: 2.7.1 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L16
48
+ # - gemfile: gemfiles/Gemfile.td-agent-4.1.0
49
+ # rvm: 2.7.2 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L19
50
+ - gemfile: gemfiles/Gemfile.td-agent-4.1.1
51
+ rvm: 2.7.3 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L17
52
+ - gemfile: gemfiles/Gemfile.td-agent-4.2.0
53
+ rvm: 2.7.4 # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L21
49
54
  fast_finish: true
50
55
 
51
56
  script: bundle exec rake test
data/CHANGELOG.md CHANGED
@@ -1,5 +1,10 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## 3.4.1
4
+
5
+ - Enhancement - Add stream name to debug logs to identify : [#214](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/214)
6
+ - Enhancement - Add td-agent 4.1.1, 4.2.0 and Fluentd 1.13.3 with Ruby 3.0.2 to test cases
7
+
3
8
  ## 3.4.0
4
9
 
5
10
  - Enhancement - Enable to monitor batch request failure and retries : [#150](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/150) [#211](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/211)
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Fluent plugin for Amazon Kinesis
2
2
 
3
- [![Build Status](https://travis-ci.com/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://travis-ci.com/awslabs/aws-fluent-plugin-kinesis)
3
+ [![Build Status](https://api.travis-ci.com/awslabs/aws-fluent-plugin-kinesis.svg?branch=master)](https://app.travis-ci.com/github/awslabs/aws-fluent-plugin-kinesis)
4
4
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
5
5
  [![Gem Downloads](https://img.shields.io/gem/dt/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
6
6
 
data/gemfiles/{Gemfile.fluentd-1.12.3 → Gemfile.fluentd-1.13.3} RENAMED
@@ -3,4 +3,4 @@ source 'https://rubygems.org'
3
3
  # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
4
  gemspec path: ".."
5
5
 
6
- gem "fluentd", "1.12.3"
6
+ gem "fluentd", "1.13.3"
data/gemfiles/Gemfile.td-agent-4.0.0 CHANGED
@@ -4,22 +4,22 @@ source 'https://rubygems.org'
4
4
  gemspec path: ".."
5
5
 
6
6
  # Specify related gems for td-agent v4.0.0
7
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
8
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L4
7
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L4
9
9
  gem "fluentd", "1.11.1"
10
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L31
10
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L31
11
11
  gem "prometheus-client", "0.9.0"
12
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L19-L20
12
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L19-L20
13
13
  gem "fluent-plugin-prometheus", "1.8.0"
14
14
  gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
15
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L28
15
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L28
16
16
  gem "jmespath", "1.4.0"
17
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L8-L13
17
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L8-L13
18
18
  gem "aws-partitions", "1.337.0"
19
19
  gem "aws-sdk-core", "3.102.1"
20
20
  gem "aws-sdk-kms", "1.35.0"
21
21
  gem "aws-sdk-s3", "1.72.0"
22
22
  gem "aws-sdk-sqs", "1.29.0"
23
23
  gem "aws-sigv4", "1.2.1"
24
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L23
24
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/8a8721c2a0b05bc487fb81a04aa079c8bb4e47fb/gemfiles/linux/Gemfile#L23
25
25
  gem "fluent-plugin-s3", "1.3.3"
data/gemfiles/Gemfile.td-agent-4.0.1 CHANGED
@@ -4,10 +4,10 @@ source 'https://rubygems.org'
4
4
  gemspec path: ".."
5
5
 
6
6
  # Specify related gems for td-agent v4.0.1
7
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
8
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/td-agent/config.rb#L4
7
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L4
9
9
  gem "fluentd", "1.11.2"
10
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.0.1/gemfiles/linux/Gemfile#L35-L42
10
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.0.1/gemfiles/linux/Gemfile#L35-L42
11
11
  gem "prometheus-client", "0.9.0"
12
12
  gem "fluent-plugin-prometheus", "1.8.2"
13
13
  gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
data/gemfiles/Gemfile.td-agent-4.1.0 CHANGED
@@ -4,10 +4,10 @@ source 'https://rubygems.org'
4
4
  gemspec path: ".."
5
5
 
6
6
  # Specify related gems for td-agent v4.1.0
7
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/master/CHANGELOG.md
8
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.1.0/td-agent/config.rb#L4
7
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L4
9
9
  gem "fluentd", "1.12.1"
10
- # https://github.com/fluent-plugins-nursery/td-agent-builder/blob/v4.1.0/td-agent/Gemfile#L32-L42
10
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.0/td-agent/Gemfile#L32-L42
11
11
  gem "prometheus-client", "0.9.0"
12
12
  gem "fluent-plugin-prometheus", "1.8.5"
13
13
  gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
data/gemfiles/Gemfile.td-agent-4.1.1 ADDED
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for td-agent v4.1.1
7
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L4
9
+ gem "fluentd", "1.12.3"
10
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.1.1/td-agent/Gemfile#L32-L42
11
+ gem "prometheus-client", "0.9.0"
12
+ gem "fluent-plugin-prometheus", "1.8.5"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.0.2"
14
+ gem "jmespath", "1.4.0"
15
+ gem "aws-partitions", "1.446.0"
16
+ gem "aws-sdk-core", "3.114.0"
17
+ gem "aws-sdk-kms", "1.43.0"
18
+ gem "aws-sdk-sqs", "1.38.0"
19
+ gem "aws-sigv4", "1.2.3"
20
+ gem "aws-sdk-s3", "1.93.1"
21
+ gem "fluent-plugin-s3", "1.6.0"
data/gemfiles/Gemfile.td-agent-4.2.0 ADDED
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for td-agent v4.2.0
7
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L4
9
+ gem "fluentd", "1.13.3"
10
+ # https://github.com/fluent-plugins-nursery/fluent-package-builder/blob/v4.2.0/td-agent/Gemfile#L42-L52
11
+ gem "prometheus-client", "2.1.0"
12
+ gem "fluent-plugin-prometheus", "2.0.1"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.1.0"
14
+ gem "jmespath", "1.4.0"
15
+ gem "aws-partitions", "1.478.0"
16
+ gem "aws-sdk-core", "3.117.0"
17
+ gem "aws-sdk-kms", "1.44.0"
18
+ gem "aws-sdk-sqs", "1.40.0"
19
+ gem "aws-sigv4", "1.2.4"
20
+ gem "aws-sdk-s3", "1.96.1"
21
+ gem "fluent-plugin-s3", "1.6.0"
data/lib/fluent/plugin/kinesis.rb CHANGED
@@ -146,13 +146,13 @@ module Fluent
146
146
  include Fluent::MessagePackFactory::Mixin
147
147
  end
148
148
 
149
- def write_records_batch(chunk, &block)
149
+ def write_records_batch(chunk, stream_name, &block)
150
150
  unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
151
151
  records = chunk.to_enum(:msgpack_each)
152
152
  split_to_batches(records) do |batch, size|
153
- log.debug(sprintf "Write chunk %s / %3d records / %4d KB", unique_id, batch.size, size/1024)
153
+ log.debug(sprintf "%s: Write chunk %s / %3d records / %4d KB", stream_name, unique_id, batch.size, size/1024)
154
154
  batch_request_with_retry(batch, &block)
155
- log.debug("Finish writing chunk")
155
+ log.debug(sprintf "%s: Finish writing chunk", stream_name)
156
156
  end
157
157
  end
158
158
 
data/lib/fluent/plugin/out_kinesis_firehose.rb CHANGED
@@ -45,7 +45,7 @@ module Fluent
45
45
 
46
46
  def write(chunk)
47
47
  delivery_stream_name = extract_placeholders(@delivery_stream_name, chunk)
48
- write_records_batch(chunk) do |batch|
48
+ write_records_batch(chunk, delivery_stream_name) do |batch|
49
49
  records = batch.map{|(data)|
50
50
  { data: data }
51
51
  }
data/lib/fluent/plugin/out_kinesis_streams.rb CHANGED
@@ -42,7 +42,7 @@ module Fluent
42
42
 
43
43
  def write(chunk)
44
44
  stream_name = extract_placeholders(@stream_name, chunk)
45
- write_records_batch(chunk) do |batch|
45
+ write_records_batch(chunk, stream_name) do |batch|
46
46
  records = batch.map{|(data, partition_key)|
47
47
  { data: data, partition_key: partition_key }
48
48
  }
data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb CHANGED
@@ -44,7 +44,7 @@ module Fluent
44
44
 
45
45
  def write(chunk)
46
46
  stream_name = extract_placeholders(@stream_name, chunk)
47
- write_records_batch(chunk) do |batch|
47
+ write_records_batch(chunk, stream_name) do |batch|
48
48
  key = @partition_key_generator.call
49
49
  records = batch.map{|(data)|data}
50
50
  client.put_records(
data/lib/fluent_plugin_kinesis/version.rb CHANGED
@@ -13,5 +13,5 @@
13
13
  # language governing permissions and limitations under the License.
14
14
 
15
15
  module FluentPluginKinesis
16
- VERSION = '3.4.0'
16
+ VERSION = '3.4.1'
17
17
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kinesis
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.4.0
4
+ version: 3.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon Web Services
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-05-03 00:00:00.000000000 Z
11
+ date: 2021-10-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -284,7 +284,7 @@ files:
284
284
  - benchmark/task.rake
285
285
  - fluent-plugin-kinesis.gemspec
286
286
  - gemfiles/Gemfile.fluentd-0.14.22
287
- - gemfiles/Gemfile.fluentd-1.12.3
287
+ - gemfiles/Gemfile.fluentd-1.13.3
288
288
  - gemfiles/Gemfile.td-agent-3.1.0
289
289
  - gemfiles/Gemfile.td-agent-3.1.1
290
290
  - gemfiles/Gemfile.td-agent-3.2.0
@@ -302,6 +302,8 @@ files:
302
302
  - gemfiles/Gemfile.td-agent-4.0.0
303
303
  - gemfiles/Gemfile.td-agent-4.0.1
304
304
  - gemfiles/Gemfile.td-agent-4.1.0
305
+ - gemfiles/Gemfile.td-agent-4.1.1
306
+ - gemfiles/Gemfile.td-agent-4.2.0
305
307
  - lib/fluent/plugin/kinesis.rb
306
308
  - lib/fluent/plugin/kinesis_helper/aggregator.rb
307
309
  - lib/fluent/plugin/kinesis_helper/api.rb