fluent-plugin-kinesis 3.6.1 → 3.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (15) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +25 -14
  3. data/CHANGELOG.md +9 -0
  4. data/README.md +25 -16
  5. data/gemfiles/Gemfile.fluent-package-5.0.8 +21 -0
  6. data/gemfiles/Gemfile.fluent-package-5.0.9 +21 -0
  7. data/gemfiles/Gemfile.fluent-package-6.0.0 +21 -0
  8. data/gemfiles/Gemfile.fluent-package-6.0.1 +21 -0
  9. data/gemfiles/Gemfile.fluent-package-6.0.2 +21 -0
  10. data/gemfiles/Gemfile.fluentd-1.19.2 +6 -0
  11. data/lib/fluent/plugin/kinesis.rb +13 -2
  12. data/lib/fluent/plugin/kinesis_helper/client.rb +14 -2
  13. data/lib/fluent/plugin/out_kinesis_firehose.rb +60 -8
  14. data/lib/fluent_plugin_kinesis/version.rb +1 -1
  15. metadata +8 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d2fb86867279f041ba7c6fc4892295bfb06ac0508327033a4f5f3b310dcb5dc5
4
- data.tar.gz: 72ebdd6da44b695e66a51b73d6c1b97b6fe458723e34ddeef102b50bdba37045
3
+ metadata.gz: dfa73c2eb5e85d404cc2e96f16e08d32899a8a4941af2cf61614d148335de9f3
4
+ data.tar.gz: 84935604016758b58f59d93a7170a1569019dc30b2e067e59e3460c6cc460577
5
5
  SHA512:
6
- metadata.gz: 20c6c0ff4453ae366ebf13e6c3448ab876f71f04a8247af9ae7e3c71c419c99db0703d488a920141b87748240811ae0868833add58edc7d4d9428784fdf4a6f5
7
- data.tar.gz: 7d50da99e747b17192cd769f6537cf07063821a14608f29f3e77b1dc6e98873f8867189332bee18e46ac7509ddf27e2d9e93b281902e7daa61dcbd9fea080694
6
+ metadata.gz: d67942cccb042d0f1d887d9470856681d0c58b16abd037e8fa83cbaead7e2d29acbcac4c428ee0d8e050dee3a10efc8efe3faf45ea624f39f4cacbc35a9570c0
7
+ data.tar.gz: 75d7c18f7a2996efa05dfab47c0e64178a1d5c3412411fa8cdaaeac31dbc59a456f0ff573f3f872535f443559e747fd7cdd22e98d9d5120e06e9080b6d3dfb57
data/.github/workflows/test.yml CHANGED
@@ -7,6 +7,7 @@ on: [ push, pull_request ]
7
7
  # https://docs.fluentd.org/quickstart/support#change-logs
8
8
  # https://github.com/fluent/fluentd/blob/master/CHANGELOG.md
9
9
  # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
10
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v5.md
10
11
  # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v4.md
11
12
  jobs:
12
13
  build:
@@ -17,8 +18,8 @@ jobs:
17
18
  include:
18
19
  - gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
19
20
  ruby-version: 2.7.1 # Oldest supported Ruby
20
- - gemfile: gemfiles/Gemfile.fluentd-1.19.0 # Latest Fluentd / https://rubygems.org/gems/fluentd
21
- ruby-version: 3.4.5 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads
21
+ - gemfile: gemfiles/Gemfile.fluentd-1.19.2 # Latest Fluentd / https://rubygems.org/gems/fluentd
22
+ ruby-version: 3.4.8 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads
22
23
  # - gemfile: gemfiles/Gemfile.td-agent-3.1.0
23
24
  # ruby-version: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L20
24
25
  # - gemfile: gemfiles/Gemfile.td-agent-3.1.1
@@ -53,22 +54,22 @@ jobs:
53
54
  ruby-version: 2.7.1 # https://github.com/fluent/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L16
54
55
  # - gemfile: gemfiles/Gemfile.td-agent-4.1.0
55
56
  # ruby-version: 2.7.2 # https://github.com/fluent/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L19
56
- - gemfile: gemfiles/Gemfile.td-agent-4.1.1
57
- ruby-version: 2.7.3 # https://github.com/fluent/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L17
58
- - gemfile: gemfiles/Gemfile.td-agent-4.2.0
59
- ruby-version: 2.7.4 # https://github.com/fluent/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L21
57
+ # - gemfile: gemfiles/Gemfile.td-agent-4.1.1
58
+ # ruby-version: 2.7.3 # https://github.com/fluent/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L17
59
+ # - gemfile: gemfiles/Gemfile.td-agent-4.2.0
60
+ # ruby-version: 2.7.4 # https://github.com/fluent/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L21
60
61
  # - gemfile: gemfiles/Gemfile.td-agent-4.3.0
61
62
  # ruby-version: 2.7.5 # https://github.com/fluent/fluent-package-builder/blob/v4.3.0/td-agent/config.rb#L21
62
63
  # - gemfile: gemfiles/Gemfile.td-agent-4.3.1
63
64
  # ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.3.1/td-agent/config.rb#L19
64
- - gemfile: gemfiles/Gemfile.td-agent-4.3.2
65
- ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.3.2/td-agent/config.rb#L23
65
+ # - gemfile: gemfiles/Gemfile.td-agent-4.3.2
66
+ # ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.3.2/td-agent/config.rb#L23
66
67
  # - gemfile: gemfiles/Gemfile.td-agent-4.4.0
67
68
  # ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.4.0/td-agent/config.rb#L23
68
69
  # - gemfile: gemfiles/Gemfile.td-agent-4.4.1
69
70
  # ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.4.1/td-agent/config.rb#L23
70
- - gemfile: gemfiles/Gemfile.td-agent-4.4.2
71
- ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.4.2/td-agent/config.rb#L23
71
+ # - gemfile: gemfiles/Gemfile.td-agent-4.4.2
72
+ # ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.4.2/td-agent/config.rb#L23
72
73
  # - gemfile: gemfiles/Gemfile.td-agent-4.5.0
73
74
  # ruby-version: 2.7.8 # https://github.com/fluent/fluent-package-builder/blob/v4.5.0/td-agent/config.rb#L19
74
75
  # - gemfile: gemfiles/Gemfile.td-agent-4.5.1
@@ -83,10 +84,20 @@ jobs:
83
84
  # ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.1/fluent-package/config.rb#L26
84
85
  # - gemfile: gemfiles/Gemfile.fluent-package-5.0.2
85
86
  # ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.2/fluent-package/config.rb#L26
86
- - gemfile: gemfiles/Gemfile.fluent-package-5.0.7 # fluent-package LTS
87
- ruby-version: 3.2.8 # https://github.com/fluent/fluent-package-builder/blob/v5.0.7/fluent-package/config.rb#L26
88
- - gemfile: gemfiles/Gemfile.fluent-package-5.2.0
89
- ruby-version: 3.2.6 # https://github.com/fluent/fluent-package-builder/blob/v5.2.0/fluent-package/config.rb#L26
87
+ # - gemfile: gemfiles/Gemfile.fluent-package-5.0.7
88
+ # ruby-version: 3.2.8 # https://github.com/fluent/fluent-package-builder/blob/v5.0.7/fluent-package/config.rb#L26
89
+ # - gemfile: gemfiles/Gemfile.fluent-package-5.0.8
90
+ # ruby-version: 3.2.9 # https://github.com/fluent/fluent-package-builder/blob/v5.0.8/fluent-package/config.rb#L26
91
+ - gemfile: gemfiles/Gemfile.fluent-package-5.0.9 # Latest release of fluent-package v5
92
+ ruby-version: 3.2.9 # https://github.com/fluent/fluent-package-builder/blob/v5.0.9/fluent-package/config.rb#L26
93
+ # - gemfile: gemfiles/Gemfile.fluent-package-5.2.0
94
+ # ruby-version: 3.2.6 # https://github.com/fluent/fluent-package-builder/blob/v5.2.0/fluent-package/config.rb#L26
95
+ # - gemfile: gemfiles/Gemfile.fluent-package-6.0.0
96
+ # ruby-version: 3.4.5 # https://github.com/fluent/fluent-package-builder/blob/v6.0.0/fluent-package/config.rb#L26
97
+ # - gemfile: gemfiles/Gemfile.fluent-package-6.0.1
98
+ # ruby-version: 3.4.7 # https://github.com/fluent/fluent-package-builder/blob/v6.0.1/fluent-package/config.rb#L26
99
+ - gemfile: gemfiles/Gemfile.fluent-package-6.0.2 # fluent-package v6 LTS
100
+ ruby-version: 3.4.8 # https://github.com/fluent/fluent-package-builder/blob/v6.0.2/fluent-package/config.rb#L26
90
101
 
91
102
  env:
92
103
  BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}
data/CHANGELOG.md CHANGED
@@ -1,5 +1,14 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## 3.7.0
4
+
5
+ - Feature - Add `use_fips_endpoint` configuration parameter for FIPS-compliant endpoints : [#227](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/227)
6
+ - Feature - Add `aggregated_record_size` configuration parameter for `kinesis_firehose` to reduce Firehose 5 KB per-record billing roundup : [#193](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/193)
7
+ - Bug fix - Fix binary data appearing in error log when compression is enabled : [#231](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/231)
8
+ - Bug fix - Suppress `ProcessCredentials` insecure string warning from newer aws-sdk-core
9
+ - Enhancement - Add fluent-package v6 LTS and Fluentd 1.19.2 with Ruby 3.4.8 to test cases
10
+ - Enhancement - Fix typos and update links in README
11
+
3
12
  ## 3.6.1
4
13
 
5
14
  - Dependency - Update gem dependency to Ruby 2.7.1+
data/README.md CHANGED
@@ -30,7 +30,7 @@ If you would like to build by yourself and install, you can build and install as
30
30
  $ bundle exec rake build
31
31
  $ bundle exec rake install
32
32
 
33
- If using td-agent v4 or lower, use td-agent-gem:
33
+ If using td-agent v4 or lower (EOL), use td-agent-gem:
34
34
 
35
35
  $ td-agent-gem install fluent-plugin-kinesis
36
36
 
@@ -42,7 +42,8 @@ If using fluent-package v5 or higher, use fluent-gem:
42
42
 
43
43
  | fluent-plugin-kinesis | fluentd | ruby |
44
44
  |:---------------------:|:-----------:|:--------:|
45
- | >= 3.6.0 | >= 0.14.22 | >= 2.7.1 |
45
+ | >= 3.7.0 | >= 0.14.22 | >= 2.7.1 |
46
+ | >= 3.6.0 && < 3.7.0 | >= 0.14.22 | >= 2.7.1 |
46
47
  | >= 3.5.0 && < 3.6.0 | >= 0.14.22 | >= 2.4.2 |
47
48
  | >= 3.2.0 && < 3.5.0 | >= 0.14.22 | >= 2.3 |
48
49
  | >= 3.0.0 && < 3.2.0 | >= 0.14.10 | >= 2.1 |
@@ -176,13 +177,17 @@ The following parameters are `kinesis_firehose` specific configurations.
176
177
 
177
178
  Name of the delivery stream to put data.
178
179
 
179
- As of Fluentd v1, placerholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
180
+ As of Fluentd v1, placeholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
180
181
 
181
182
  #### append_new_line
182
183
 
183
184
  Boolean. Default `true`. If it is enabled, the plugin adds new line character (`\n`) to each serialized record.
184
185
  Before appending `\n`, plugin calls chomp and removes separator from the end of each record as [chomp_record](#chomp_record) is `true`. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) output with default configuration ([append_new_line](#append_new_line) is `true`). If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).
185
186
 
187
+ #### aggregated_record_size
188
+
189
+ Integer. Default `0` (no aggregation). When set to a value greater than 0, the plugin concatenates multiple small records into a single larger record up to the specified size (in bytes) before sending to Firehose via PutRecordBatch. This can reduce costs when using Direct PUT ingestion, which applies a 5 KB per-record billing roundup. The value cannot exceed [max_record_size](#max_record_size) (default 1 MB). This is similar to the `aggregatedRecordSizeBytes` configuration in [Kinesis Agent](https://docs.aws.amazon.com/firehose/latest/dev/agent-config-settings.html).
190
+
186
191
  ### Configuration: kinesis_streams_aggregated
187
192
 
188
193
  The following parameters are `kinesis_streams_aggregated` specific configurations.
@@ -191,7 +196,7 @@ The following parameters are `kinesis_streams_aggregated` specific configuration
191
196
 
192
197
  Name of the stream to put data.
193
198
 
194
- As of Fluentd v1, placerholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
199
+ As of Fluentd v1, placeholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
195
200
 
196
201
  #### fixed_partition_key
197
202
 
@@ -240,8 +245,8 @@ See also:
240
245
 
241
246
  * [Using IAM Roles - AWS Identity and Access
242
247
  Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
243
- * [Aws::STS::Client](https://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
244
- * [Aws::AssumeRoleCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html)
248
+ * [Aws::STS::Client](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html)
249
+ * [Aws::AssumeRoleCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleCredentials.html)
245
250
 
246
251
  #### role_arn (required)
247
252
 
@@ -291,9 +296,9 @@ Similar to the assume_role_credentials, but for usage in EKS.
291
296
 
292
297
  See also:
293
298
 
294
- * [Using IAM Roles - AWS Identity and Access Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
299
+ * [Using IAM Roles - AWS Identity and Access Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
295
300
  * [IAM Roles For Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html)
296
- * [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
301
+ * [Aws::STS::Client](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html)
297
302
  * [Aws::AssumeRoleWebIdentityCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleWebIdentityCredentials.html)
298
303
 
299
304
  #### role_arn (required)
@@ -331,7 +336,7 @@ Retrieve temporary security credentials via HTTP request. This is useful on EC2
331
336
 
332
337
  See also:
333
338
 
334
- * [Aws::InstanceProfileCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/InstanceProfileCredentials.html)
339
+ * [Aws::InstanceProfileCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/InstanceProfileCredentials.html)
335
340
  * [Temporary Security Credentials - AWS Identity and Access
336
341
  Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
337
342
  * [Instance Metadata and User Data - Amazon Elastic Compute
@@ -372,7 +377,7 @@ This loads AWS access credentials from local ini file. This is useful for local
372
377
 
373
378
  See also:
374
379
 
375
- * [Aws::SharedCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html)
380
+ * [Aws::SharedCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SharedCredentials.html)
376
381
 
377
382
  #### path
378
383
 
@@ -506,7 +511,7 @@ If your record contains a field whose string should be sent to Amazon Kinesis di
506
511
 
507
512
  #### compression
508
513
 
509
- Specifying compression way for data of each record. Current accepted options are `zlib` and `gzip`. Otherwise, no compression will be preformed.
514
+ Specifying compression way for data of each record. Current accepted options are `zlib` and `gzip`. Otherwise, no compression will be performed.
510
515
 
511
516
  #### log_truncate_max_size
512
517
 
@@ -535,13 +540,17 @@ HTTP proxy for API calling. Default `nil`.
535
540
 
536
541
  API endpoint URL, for testing. Default `nil`.
537
542
 
543
+ #### use_fips_endpoint
544
+
545
+ Boolean. Default `nil`. If `true`, the plugin uses FIPS-compliant endpoints for Kinesis Data Streams and Amazon Data Firehose API calls. This is required for environments that must comply with Federal Information Processing Standards (FIPS). When `nil`, the SDK's default behavior applies (which respects the `AWS_USE_FIPS_ENDPOINT` environment variable).
546
+
538
547
  #### ssl_verify_peer
539
548
 
540
- Boolean. Disable if you want to verify ssl connection, for testing. Default `true`.
549
+ Boolean. Set to `false` to disable SSL certificate verification, for testing. Default `true`.
541
550
 
542
551
  #### debug
543
552
 
544
- Boolean. Enable if you need to debug Amazon Data Firehose API call. Default is `false`.
553
+ Boolean. Enable if you need to debug API calls. Default is `false`.
545
554
 
546
555
  ## Development
547
556
 
@@ -554,7 +563,7 @@ To launch `fluentd` process with this plugin for development, follow the steps b
554
563
 
555
564
  To launch using specified version of Fluentd, use `BUNDLE_GEMFILE` environment variable:
556
565
 
557
- BUNDLE_GEMFILE=$PWD/gemfiles/Gemfile.td-agent-3.3.0 bundle exec fluentd -c /path/to/fluent.conf
566
+ BUNDLE_GEMFILE=$PWD/gemfiles/Gemfile.fluent-package-5.0.9 bundle exec fluentd -c /path/to/fluent.conf
558
567
 
559
568
  ## Contributing
560
569
 
@@ -562,8 +571,8 @@ Bug reports and pull requests are welcome on [GitHub][github].
562
571
 
563
572
  ## Related Resources
564
573
 
565
- * [Amazon Kinesis Data Streams Developer Guide](http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html)
566
- * [Amazon Data Firehose Developer Guide](http://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
574
+ * [Amazon Kinesis Data Streams Developer Guide](https://docs.aws.amazon.com/kinesis/latest/dev/introduction.html)
575
+ * [Amazon Data Firehose Developer Guide](https://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
567
576
 
568
577
  [github]: https://github.com/awslabs/aws-fluent-plugin-kinesis
569
578
  [fluentd-doc-kinesis]: https://docs.fluentd.org/how-to-guides/kinesis-stream
data/gemfiles/Gemfile.fluent-package-5.0.8 ADDED
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v5.0.8
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v5.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v5.0.8/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.16.10"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v5.0.8/fluent-package/Gemfile#L57-L67
11
+ gem "prometheus-client", "2.1.0"
12
+ gem "fluent-plugin-prometheus", "2.1.0"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.1.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.785.0"
16
+ gem "aws-sdk-core", "3.178.0"
17
+ gem "aws-sdk-kms", "1.71.0"
18
+ gem "aws-sdk-sqs", "1.61.0"
19
+ gem "aws-sigv4", "1.6.0"
20
+ gem "aws-sdk-s3", "1.129.0"
21
+ gem "fluent-plugin-s3", "1.7.2"
data/gemfiles/Gemfile.fluent-package-5.0.9 ADDED
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v5.0.9
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v5.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v5.0.9/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.16.11"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v5.0.9/fluent-package/Gemfile#L57-L67
11
+ gem "prometheus-client", "2.1.0"
12
+ gem "fluent-plugin-prometheus", "2.1.0"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.1.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.785.0"
16
+ gem "aws-sdk-core", "3.178.0"
17
+ gem "aws-sdk-kms", "1.71.0"
18
+ gem "aws-sdk-sqs", "1.61.0"
19
+ gem "aws-sigv4", "1.6.0"
20
+ gem "aws-sdk-s3", "1.129.0"
21
+ gem "fluent-plugin-s3", "1.7.2"
data/gemfiles/Gemfile.fluent-package-6.0.0 ADDED
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v6.0.0
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.0/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.19.0"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.0/fluent-package/Gemfile#L55-L65
11
+ gem "prometheus-client", "4.2.5"
12
+ gem "fluent-plugin-prometheus", "2.2.1"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.1150.0"
16
+ gem "aws-sdk-core", "3.230.0"
17
+ gem "aws-sdk-kms", "1.110.0"
18
+ gem "aws-sdk-sqs", "1.101.0"
19
+ gem "aws-sigv4", "1.12.1"
20
+ gem "aws-sdk-s3", "1.197.0"
21
+ gem "fluent-plugin-s3", "1.8.3"
data/gemfiles/Gemfile.fluent-package-6.0.1 ADDED
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v6.0.1
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.1/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.19.1"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.1/fluent-package/Gemfile#L55-L65
11
+ gem "prometheus-client", "4.2.5"
12
+ gem "fluent-plugin-prometheus", "2.2.1"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.1150.0"
16
+ gem "aws-sdk-core", "3.230.0"
17
+ gem "aws-sdk-kms", "1.110.0"
18
+ gem "aws-sdk-sqs", "1.101.0"
19
+ gem "aws-sigv4", "1.12.1"
20
+ gem "aws-sdk-s3", "1.197.0"
21
+ gem "fluent-plugin-s3", "1.8.3"
data/gemfiles/Gemfile.fluent-package-6.0.2 ADDED
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v6.0.2
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.2/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.19.2"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.2/fluent-package/Gemfile#L55-L65
11
+ gem "prometheus-client", "4.2.5"
12
+ gem "fluent-plugin-prometheus", "2.2.1"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.1150.0"
16
+ gem "aws-sdk-core", "3.234.0"
17
+ gem "aws-sdk-kms", "1.110.0"
18
+ gem "aws-sdk-sqs", "1.101.0"
19
+ gem "aws-sigv4", "1.12.1"
20
+ gem "aws-sdk-s3", "1.208.0"
21
+ gem "fluent-plugin-s3", "1.8.3"
data/gemfiles/Gemfile.fluentd-1.19.2 ADDED
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ gem "fluentd", "1.19.2"
data/lib/fluent/plugin/kinesis.rb CHANGED
@@ -30,15 +30,26 @@ module Fluent
30
30
  def initialize(message, record)
31
31
  super message
32
32
  @record_message = if record.is_a? Array
33
- record.reverse.map(&:to_s).join(', ')
33
+ record.reverse.map{|r| format_record(r) }.join(', ')
34
34
  else
35
- record.to_s
35
+ format_record(record)
36
36
  end
37
37
  end
38
38
 
39
39
  def to_s
40
40
  super + ": " + @record_message
41
41
  end
42
+
43
+ private
44
+
45
+ def format_record(record)
46
+ str = record.to_s
47
+ if str.encoding == Encoding::ASCII_8BIT || !str.valid_encoding?
48
+ str.unpack1('H*')
49
+ else
50
+ str
51
+ end
52
+ end
42
53
  end
43
54
  class KeyNotFoundError < SkipRecordError
44
55
  def initialize(key, record)
data/lib/fluent/plugin/kinesis_helper/client.rb CHANGED
@@ -26,6 +26,7 @@ module Fluent
26
26
  config_param :http_proxy, :string, default: nil, secret: true
27
27
  config_param :endpoint, :string, default: nil
28
28
  config_param :ssl_verify_peer, :bool, default: true
29
+ config_param :use_fips_endpoint, :bool, default: nil
29
30
 
30
31
  config_param :aws_key_id, :string, default: nil, secret: true
31
32
  config_param :aws_sec_key, :string, default: nil, secret: true
@@ -121,6 +122,10 @@ module Fluent
121
122
  options.update(http_proxy: @http_proxy) unless @http_proxy.nil?
122
123
  options.update(endpoint: @endpoint) unless @endpoint.nil?
123
124
  options.update(ssl_verify_peer: @ssl_verify_peer) unless @ssl_verify_peer.nil?
125
+ if !@use_fips_endpoint.nil? && client_class.method_defined?(:config) &&
126
+ Gem::Version.new(Aws::CORE_GEM_VERSION) >= Gem::Version.new('3.122.0')
127
+ options.update(use_fips_endpoint: @use_fips_endpoint)
128
+ end
124
129
  if @debug
125
130
  options.update(logger: Logger.new(log.out))
126
131
  options.update(log_level: :debug)
@@ -197,10 +202,17 @@ module Fluent
197
202
  end
198
203
  c = @process_credentials
199
204
  process = c.process
200
- options[:credentials] = Aws::ProcessCredentials.new(process)
205
+ # Suppress insecure string warning from newer aws-sdk-core.
206
+ # Using array form would break older versions that use Open3.popen3.
207
+ options[:credentials] = begin
208
+ verbose, $VERBOSE = $VERBOSE, nil
209
+ Aws::ProcessCredentials.new(process)
210
+ ensure
211
+ $VERBOSE = verbose
212
+ end
201
213
  else
202
214
  # Use default credentials
203
- # See http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html
215
+ # See https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html
204
216
  end
205
217
  options
206
218
  end
data/lib/fluent/plugin/out_kinesis_firehose.rb CHANGED
@@ -26,6 +26,7 @@ module Fluent
26
26
 
27
27
  config_param :delivery_stream_name, :string
28
28
  config_param :append_new_line, :bool, default: true
29
+ config_param :aggregated_record_size, :integer, default: 0
29
30
 
30
31
  def configure(conf)
31
32
  super
@@ -35,6 +36,9 @@ module Fluent
35
36
  org_data_formatter.call(tag, time, record).chomp + "\n"
36
37
  }
37
38
  end
39
+ if @aggregated_record_size > @max_record_size
40
+ raise Fluent::ConfigError, "aggregated_record_size (#{@aggregated_record_size}) can't be greater than max_record_size (#{@max_record_size})."
41
+ end
38
42
  end
39
43
 
40
44
  def format(tag, time, record)
@@ -45,14 +49,62 @@ module Fluent
45
49
 
46
50
  def write(chunk)
47
51
  delivery_stream_name = extract_placeholders(@delivery_stream_name, chunk)
48
- write_records_batch(chunk, delivery_stream_name) do |batch|
49
- records = batch.map{|(data)|
50
- { data: data }
51
- }
52
- client.put_record_batch(
53
- delivery_stream_name: delivery_stream_name,
54
- records: records,
55
- )
52
+ if @aggregated_record_size > 0
53
+ write_records_batch_aggregated(chunk, delivery_stream_name) do |batch|
54
+ records = batch.map{|(data)|
55
+ { data: data }
56
+ }
57
+ client.put_record_batch(
58
+ delivery_stream_name: delivery_stream_name,
59
+ records: records,
60
+ )
61
+ end
62
+ else
63
+ write_records_batch(chunk, delivery_stream_name) do |batch|
64
+ records = batch.map{|(data)|
65
+ { data: data }
66
+ }
67
+ client.put_record_batch(
68
+ delivery_stream_name: delivery_stream_name,
69
+ records: records,
70
+ )
71
+ end
72
+ end
73
+ end
74
+
75
+ private
76
+
77
+ def write_records_batch_aggregated(chunk, stream_name, &block)
78
+ unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
79
+ records = chunk.to_enum(:msgpack_each)
80
+ aggregated = aggregate_records(records)
81
+ split_to_batches(aggregated) do |batch, size|
82
+ log.debug(sprintf "%s: Write chunk %s / %3d records / %4d KB", stream_name, unique_id, batch.size, size/1024)
83
+ batch_request_with_retry(batch, &block)
84
+ log.debug(sprintf "%s: Finish writing chunk", stream_name)
85
+ end
86
+ end
87
+
88
+ def aggregate_records(records)
89
+ Enumerator.new do |y|
90
+ current = nil
91
+ current_size = 0
92
+ records.each do |record|
93
+ data = record.first
94
+ data_size = data.bytesize
95
+ if current.nil?
96
+ current = data
97
+ current_size = data_size
98
+ elsif current_size + data_size <= @aggregated_record_size
99
+ current = current + data
100
+ current_size += data_size
101
+ else
102
+ y.yield [current]
103
+ current = data
104
+ current_size = data_size
105
+ end
106
+ end
107
+ y.yield [current] unless current.nil?
56
108
  end
57
109
  end
58
110
  end
data/lib/fluent_plugin_kinesis/version.rb CHANGED
@@ -13,5 +13,5 @@
13
13
  # language governing permissions and limitations under the License.
14
14
 
15
15
  module FluentPluginKinesis
16
- VERSION = '3.6.1'
16
+ VERSION = '3.7.0'
17
17
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kinesis
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.6.1
4
+ version: 3.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon Web Services
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2025-08-01 00:00:00.000000000 Z
11
+ date: 2026-04-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -287,9 +287,15 @@ files:
287
287
  - gemfiles/Gemfile.fluent-package-5.0.1
288
288
  - gemfiles/Gemfile.fluent-package-5.0.2
289
289
  - gemfiles/Gemfile.fluent-package-5.0.7
290
+ - gemfiles/Gemfile.fluent-package-5.0.8
291
+ - gemfiles/Gemfile.fluent-package-5.0.9
290
292
  - gemfiles/Gemfile.fluent-package-5.2.0
293
+ - gemfiles/Gemfile.fluent-package-6.0.0
294
+ - gemfiles/Gemfile.fluent-package-6.0.1
295
+ - gemfiles/Gemfile.fluent-package-6.0.2
291
296
  - gemfiles/Gemfile.fluentd-0.14.22
292
297
  - gemfiles/Gemfile.fluentd-1.19.0
298
+ - gemfiles/Gemfile.fluentd-1.19.2
293
299
  - gemfiles/Gemfile.td-agent-3.1.0
294
300
  - gemfiles/Gemfile.td-agent-3.1.1
295
301
  - gemfiles/Gemfile.td-agent-3.2.0