fluent-plugin-kinesis 3.6.1 → 3.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d2fb86867279f041ba7c6fc4892295bfb06ac0508327033a4f5f3b310dcb5dc5
4
- data.tar.gz: 72ebdd6da44b695e66a51b73d6c1b97b6fe458723e34ddeef102b50bdba37045
3
+ metadata.gz: 406d3b87604e265677a3df5a65645b9d9f45d959c7eb2431c03825884420cba8
4
+ data.tar.gz: 1ecc97af6d59ba885829719c076c3a2243fed7baf0302c11e72d3c126099e37a
5
5
  SHA512:
6
- metadata.gz: 20c6c0ff4453ae366ebf13e6c3448ab876f71f04a8247af9ae7e3c71c419c99db0703d488a920141b87748240811ae0868833add58edc7d4d9428784fdf4a6f5
7
- data.tar.gz: 7d50da99e747b17192cd769f6537cf07063821a14608f29f3e77b1dc6e98873f8867189332bee18e46ac7509ddf27e2d9e93b281902e7daa61dcbd9fea080694
6
+ metadata.gz: 643b74ada4bdd6d5006620ce3f94b54cbf391c5bac66317ba7447f65bcbce17298ecf130d47541484be1821f9cfcb148b5371fb077e76ecc2035fca696c60ace
7
+ data.tar.gz: 16df0f4ce732bbd8cdda57109e491b027668f172a548043e47bcd0804e183801090e7d6d5463a7eeb548523521475c61e1700f840c2626d8bb5ffec1ebdf56c5
@@ -1,12 +1,19 @@
1
1
  name: test
2
2
 
3
- on: [ push, pull_request ]
3
+ on:
4
+ push:
5
+ pull_request:
6
+ # Run weekly to detect breakage by new releases of dependencies such as Fluentd and AWS SDK
7
+ schedule:
8
+ - cron: '0 0 * * 1'
9
+ workflow_dispatch:
4
10
 
5
11
  # Test with supported fluent-package versions
6
12
  # https://www.fluentd.org/download/fluent_package
7
13
  # https://docs.fluentd.org/quickstart/support#change-logs
8
14
  # https://github.com/fluent/fluentd/blob/master/CHANGELOG.md
9
15
  # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
16
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v5.md
10
17
  # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v4.md
11
18
  jobs:
12
19
  build:
@@ -16,9 +23,9 @@ jobs:
16
23
  matrix:
17
24
  include:
18
25
  - gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
19
- ruby-version: 2.7.1 # Oldest supported Ruby
20
- - gemfile: gemfiles/Gemfile.fluentd-1.19.0 # Latest Fluentd / https://rubygems.org/gems/fluentd
21
- ruby-version: 3.4.5 # Latest supported Ruby / https://www.ruby-lang.org/en/downloads
26
+ ruby-version: '2.7' # Oldest supported Ruby
27
+ - gemfile: gemfiles/Gemfile.fluentd-1.20.1 # Latest Fluentd / https://rubygems.org/gems/fluentd
28
+ ruby-version: '4.0' # Latest supported Ruby / https://www.ruby-lang.org/en/downloads
22
29
  # - gemfile: gemfiles/Gemfile.td-agent-3.1.0
23
30
  # ruby-version: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L20
24
31
  # - gemfile: gemfiles/Gemfile.td-agent-3.1.1
@@ -50,25 +57,25 @@ jobs:
50
57
  # - gemfile: gemfiles/Gemfile.td-agent-4.0.0
51
58
  # ruby-version: 2.7.1 # https://github.com/fluent/fluent-package-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L16
52
59
  - gemfile: gemfiles/Gemfile.td-agent-4.0.1 # Oldest supported td-agent
53
- ruby-version: 2.7.1 # https://github.com/fluent/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L16
60
+ ruby-version: '2.7' # https://github.com/fluent/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L16
54
61
  # - gemfile: gemfiles/Gemfile.td-agent-4.1.0
55
62
  # ruby-version: 2.7.2 # https://github.com/fluent/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L19
56
- - gemfile: gemfiles/Gemfile.td-agent-4.1.1
57
- ruby-version: 2.7.3 # https://github.com/fluent/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L17
58
- - gemfile: gemfiles/Gemfile.td-agent-4.2.0
59
- ruby-version: 2.7.4 # https://github.com/fluent/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L21
63
+ # - gemfile: gemfiles/Gemfile.td-agent-4.1.1
64
+ # ruby-version: 2.7.3 # https://github.com/fluent/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L17
65
+ # - gemfile: gemfiles/Gemfile.td-agent-4.2.0
66
+ # ruby-version: 2.7.4 # https://github.com/fluent/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L21
60
67
  # - gemfile: gemfiles/Gemfile.td-agent-4.3.0
61
68
  # ruby-version: 2.7.5 # https://github.com/fluent/fluent-package-builder/blob/v4.3.0/td-agent/config.rb#L21
62
69
  # - gemfile: gemfiles/Gemfile.td-agent-4.3.1
63
70
  # ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.3.1/td-agent/config.rb#L19
64
- - gemfile: gemfiles/Gemfile.td-agent-4.3.2
65
- ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.3.2/td-agent/config.rb#L23
71
+ # - gemfile: gemfiles/Gemfile.td-agent-4.3.2
72
+ # ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.3.2/td-agent/config.rb#L23
66
73
  # - gemfile: gemfiles/Gemfile.td-agent-4.4.0
67
74
  # ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.4.0/td-agent/config.rb#L23
68
75
  # - gemfile: gemfiles/Gemfile.td-agent-4.4.1
69
76
  # ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.4.1/td-agent/config.rb#L23
70
- - gemfile: gemfiles/Gemfile.td-agent-4.4.2
71
- ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.4.2/td-agent/config.rb#L23
77
+ # - gemfile: gemfiles/Gemfile.td-agent-4.4.2
78
+ # ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.4.2/td-agent/config.rb#L23
72
79
  # - gemfile: gemfiles/Gemfile.td-agent-4.5.0
73
80
  # ruby-version: 2.7.8 # https://github.com/fluent/fluent-package-builder/blob/v4.5.0/td-agent/config.rb#L19
74
81
  # - gemfile: gemfiles/Gemfile.td-agent-4.5.1
@@ -76,17 +83,35 @@ jobs:
76
83
  # - gemfile: gemfiles/Gemfile.td-agent-4.5.2
77
84
  # ruby-version: 2.7.8 # https://github.com/fluent/fluent-package-builder/blob/v4.5.2/td-agent/config.rb#L19
78
85
  - gemfile: gemfiles/Gemfile.td-agent-4.5.3 # Latest release of td-agent v4
79
- ruby-version: 2.7.8 # https://github.com/fluent/fluent-package-builder/blob/v4.5.3/td-agent/config.rb#L19
86
+ ruby-version: '2.7' # https://github.com/fluent/fluent-package-builder/blob/v4.5.3/td-agent/config.rb#L19
80
87
  # - gemfile: gemfiles/Gemfile.fluent-package-5.0.0
81
88
  # ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.0/fluent-package/config.rb#L26
82
89
  # - gemfile: gemfiles/Gemfile.fluent-package-5.0.1
83
90
  # ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.1/fluent-package/config.rb#L26
84
91
  # - gemfile: gemfiles/Gemfile.fluent-package-5.0.2
85
92
  # ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.2/fluent-package/config.rb#L26
86
- - gemfile: gemfiles/Gemfile.fluent-package-5.0.7 # fluent-package LTS
87
- ruby-version: 3.2.8 # https://github.com/fluent/fluent-package-builder/blob/v5.0.7/fluent-package/config.rb#L26
88
- - gemfile: gemfiles/Gemfile.fluent-package-5.2.0
89
- ruby-version: 3.2.6 # https://github.com/fluent/fluent-package-builder/blob/v5.2.0/fluent-package/config.rb#L26
93
+ # - gemfile: gemfiles/Gemfile.fluent-package-5.0.7
94
+ # ruby-version: 3.2.8 # https://github.com/fluent/fluent-package-builder/blob/v5.0.7/fluent-package/config.rb#L26
95
+ # - gemfile: gemfiles/Gemfile.fluent-package-5.0.8
96
+ # ruby-version: 3.2.9 # https://github.com/fluent/fluent-package-builder/blob/v5.0.8/fluent-package/config.rb#L26
97
+ - gemfile: gemfiles/Gemfile.fluent-package-5.0.9 # Latest release of fluent-package v5
98
+ ruby-version: '3.2' # https://github.com/fluent/fluent-package-builder/blob/v5.0.9/fluent-package/config.rb#L26
99
+ # - gemfile: gemfiles/Gemfile.fluent-package-5.2.0
100
+ # ruby-version: 3.2.6 # https://github.com/fluent/fluent-package-builder/blob/v5.2.0/fluent-package/config.rb#L26
101
+ # - gemfile: gemfiles/Gemfile.fluent-package-6.0.0
102
+ # ruby-version: 3.4.5 # https://github.com/fluent/fluent-package-builder/blob/v6.0.0/fluent-package/config.rb#L26
103
+ # - gemfile: gemfiles/Gemfile.fluent-package-6.0.1
104
+ # ruby-version: 3.4.7 # https://github.com/fluent/fluent-package-builder/blob/v6.0.1/fluent-package/config.rb#L26
105
+ # - gemfile: gemfiles/Gemfile.fluent-package-6.0.2
106
+ # ruby-version: 3.4.8 # https://github.com/fluent/fluent-package-builder/blob/v6.0.2/fluent-package/config.rb#L26
107
+ # - gemfile: gemfiles/Gemfile.fluent-package-6.0.3
108
+ # ruby-version: 3.4.9 # https://github.com/fluent/fluent-package-builder/blob/v6.0.3/fluent-package/config.rb#L26
109
+ # - gemfile: gemfiles/Gemfile.fluent-package-6.0.4
110
+ # ruby-version: 3.4.9 # https://github.com/fluent/fluent-package-builder/blob/v6.0.4/fluent-package/config.rb#L26
111
+ - gemfile: gemfiles/Gemfile.fluent-package-6.0.5 # Latest release of fluent-package v6 LTS
112
+ ruby-version: '3.4' # https://github.com/fluent/fluent-package-builder/blob/v6.0.5/fluent-package/config.rb#L26
113
+ - gemfile: gemfiles/Gemfile.fluent-package-6.1.0 # Latest release of fluent-package
114
+ ruby-version: '3.4' # https://github.com/fluent/fluent-package-builder/blob/v6.1.0/fluent-package/config.rb#L29
90
115
 
91
116
  env:
92
117
  BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}
@@ -99,4 +124,54 @@ jobs:
99
124
  ruby-version: ${{ matrix.ruby-version }}
100
125
  bundler-cache: true
101
126
  - name: Run tests
102
- run: bundle exec rake test
127
+ run: bundle exec rake test
128
+ aws-emulator:
129
+ # Run the tests including integration tests against AWS Emulator (https://aws.amazon.com/products/developer-tools/emulator/)
130
+ runs-on: ubuntu-latest
131
+ env:
132
+ BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/Gemfile.fluentd-1.20.1 # Latest Fluentd
133
+ AWS_EMULATOR: 'true'
134
+
135
+ steps:
136
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
137
+ - name: Install AWS Emulator
138
+ working-directory: ${{ runner.temp }}
139
+ run: |
140
+ PACKAGE_FILE=aws-emulator_amd64.deb
141
+ curl -fsSLO https://downloads.aws.com/aws-emulator/packages/aws-emulator/LTS/latest/$PACKAGE_FILE
142
+ curl -fsSLO https://downloads.aws.com/aws-emulator/packages/aws-emulator/LTS/latest/$PACKAGE_FILE.sig
143
+ curl -fsSLo aws-publishing.asc https://downloads.aws.com/keys/gpg/current.asc
144
+ gpg --import aws-publishing.asc
145
+ # Verify the package with the documented fingerprint of the AWS publishing key
146
+ gpg --with-colons --fingerprint ssm-publishing@amazon.com | grep -q '^fpr:::::::::98BCF336A3BE53B734E967C52E5BF49BD940F624:'
147
+ gpg --verify $PACKAGE_FILE.sig $PACKAGE_FILE
148
+ sudo apt-get install -y ./$PACKAGE_FILE
149
+ aws-emulator -v
150
+ - name: Start AWS Emulator
151
+ run: |
152
+ nohup aws-emulator > "$RUNNER_TEMP/aws-emulator.log" 2>&1 &
153
+ EMULATOR_DIR="$HOME/.aws/emulator/9297"
154
+ for i in $(seq 30); do
155
+ curl -sf --cacert "$EMULATOR_DIR/ca.pem" https://127.0.0.1:9297/__admin/health && break
156
+ sleep 1
157
+ done
158
+ curl -sf --cacert "$EMULATOR_DIR/ca.pem" https://127.0.0.1:9297/__admin/health
159
+ # AWS_EC2_METADATA_DISABLED is not set here, since unit tests of instance profile credentials mock the metadata endpoint.
160
+ # Integration tests set it while they run.
161
+ {
162
+ echo "AWS_ENDPOINT_URL=https://127.0.0.1:9297"
163
+ echo "AWS_SHARED_CREDENTIALS_FILE=$EMULATOR_DIR/emulator_credentials"
164
+ echo "AWS_CONFIG_FILE=$EMULATOR_DIR/config"
165
+ echo "AWS_CA_BUNDLE=$EMULATOR_DIR/ca.pem"
166
+ echo "AWS_REGION=us-east-1"
167
+ } >> "$GITHUB_ENV"
168
+ - name: Set up Ruby
169
+ uses: ruby/setup-ruby@b3aaceaef7e0ee231b499e7b5f49c03fc1d32199 # v1.338.0
170
+ with:
171
+ ruby-version: '4.0' # Latest supported Ruby
172
+ bundler-cache: true
173
+ - name: Run tests including integration tests with AWS Emulator
174
+ run: bundle exec rake test
175
+ - name: Show AWS Emulator log
176
+ if: failure()
177
+ run: tail -n 200 "$RUNNER_TEMP/aws-emulator.log"
data/CHANGELOG.md CHANGED
@@ -1,5 +1,20 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## 3.7.1
4
+
5
+ - Bug fix - Fix loading the plugin with Bundler on Ruby 4.0, where benchmark is no longer a default gem
6
+ - Enhancement - Add fluent-package 6.0.5 and 6.1.0, and Fluentd 1.20.1 with Ruby 3.4 and 4.0 to test cases
7
+ - Enhancement - Add integration tests with AWS Emulator
8
+
9
+ ## 3.7.0
10
+
11
+ - Feature - Add `use_fips_endpoint` configuration parameter for FIPS-compliant endpoints : [#227](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/227)
12
+ - Feature - Add `aggregated_record_size` configuration parameter for `kinesis_firehose` to reduce Firehose 5 KB per-record billing roundup : [#193](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/193)
13
+ - Bug fix - Fix binary data appearing in error log when compression is enabled : [#231](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/231)
14
+ - Bug fix - Suppress `ProcessCredentials` insecure string warning from newer aws-sdk-core
15
+ - Enhancement - Add fluent-package v6 LTS and Fluentd 1.19.2 with Ruby 3.4.8 to test cases
16
+ - Enhancement - Fix typos and update links in README
17
+
3
18
  ## 3.6.1
4
19
 
5
20
  - Dependency - Update gem dependency to Ruby 2.7.1+
data/README.md CHANGED
@@ -1,8 +1,8 @@
1
1
  # Fluent plugin for Amazon Kinesis
2
2
 
3
- [![Build Status](https://github.com/awslabs/aws-fluent-plugin-kinesis/actions/workflows/test.yml/badge.svg)](https://github.com/awslabs/aws-fluent-plugin-kinesis/actions/workflows/test.yml)
4
3
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
5
4
  [![Gem Downloads](https://img.shields.io/gem/dt/fluent-plugin-kinesis.svg)](https://rubygems.org/gems/fluent-plugin-kinesis)
5
+ [![Build Status](https://github.com/awslabs/aws-fluent-plugin-kinesis/actions/workflows/test.yml/badge.svg)](https://github.com/awslabs/aws-fluent-plugin-kinesis/actions/workflows/test.yml)
6
6
 
7
7
  A fluentd output plugin to send events to [Amazon Kinesis Data Streams][streams] and [Amazon Data Firehose][firehose]. The plugin also supports [KPL Aggregated Record Format][kpl].
8
8
 
@@ -30,7 +30,7 @@ If you would like to build by yourself and install, you can build and install as
30
30
  $ bundle exec rake build
31
31
  $ bundle exec rake install
32
32
 
33
- If using td-agent v4 or lower, use td-agent-gem:
33
+ If using td-agent v4 or lower (EOL), use td-agent-gem:
34
34
 
35
35
  $ td-agent-gem install fluent-plugin-kinesis
36
36
 
@@ -42,7 +42,8 @@ If using fluent-package v5 or higher, use fluent-gem:
42
42
 
43
43
  | fluent-plugin-kinesis | fluentd | ruby |
44
44
  |:---------------------:|:-----------:|:--------:|
45
- | >= 3.6.0 | >= 0.14.22 | >= 2.7.1 |
45
+ | >= 3.7.0 | >= 0.14.22 | >= 2.7.1 |
46
+ | >= 3.6.0 && < 3.7.0 | >= 0.14.22 | >= 2.7.1 |
46
47
  | >= 3.5.0 && < 3.6.0 | >= 0.14.22 | >= 2.4.2 |
47
48
  | >= 3.2.0 && < 3.5.0 | >= 0.14.22 | >= 2.3 |
48
49
  | >= 3.0.0 && < 3.2.0 | >= 0.14.10 | >= 2.1 |
@@ -176,13 +177,17 @@ The following parameters are `kinesis_firehose` specific configurations.
176
177
 
177
178
  Name of the delivery stream to put data.
178
179
 
179
- As of Fluentd v1, placerholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
180
+ As of Fluentd v1, placeholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
180
181
 
181
182
  #### append_new_line
182
183
 
183
184
  Boolean. Default `true`. If it is enabled, the plugin adds new line character (`\n`) to each serialized record.
184
185
  Before appending `\n`, plugin calls chomp and removes separator from the end of each record as [chomp_record](#chomp_record) is `true`. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) output with default configuration ([append_new_line](#append_new_line) is `true`). If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).
185
186
 
187
+ #### aggregated_record_size
188
+
189
+ Integer. Default `0` (no aggregation). When set to a value greater than 0, the plugin concatenates multiple small records into a single larger record up to the specified size (in bytes) before sending to Firehose via PutRecordBatch. This can reduce costs when using Direct PUT ingestion, which applies a 5 KB per-record billing roundup. The value cannot exceed [max_record_size](#max_record_size) (default 1 MB). This is similar to the `aggregatedRecordSizeBytes` configuration in [Kinesis Agent](https://docs.aws.amazon.com/firehose/latest/dev/agent-config-settings.html).
190
+
186
191
  ### Configuration: kinesis_streams_aggregated
187
192
 
188
193
  The following parameters are `kinesis_streams_aggregated` specific configurations.
@@ -191,7 +196,7 @@ The following parameters are `kinesis_streams_aggregated` specific configuration
191
196
 
192
197
  Name of the stream to put data.
193
198
 
194
- As of Fluentd v1, placerholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
199
+ As of Fluentd v1, placeholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
195
200
 
196
201
  #### fixed_partition_key
197
202
 
@@ -240,8 +245,8 @@ See also:
240
245
 
241
246
  * [Using IAM Roles - AWS Identity and Access
242
247
  Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
243
- * [Aws::STS::Client](https://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
244
- * [Aws::AssumeRoleCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/AssumeRoleCredentials.html)
248
+ * [Aws::STS::Client](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html)
249
+ * [Aws::AssumeRoleCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleCredentials.html)
245
250
 
246
251
  #### role_arn (required)
247
252
 
@@ -291,9 +296,9 @@ Similar to the assume_role_credentials, but for usage in EKS.
291
296
 
292
297
  See also:
293
298
 
294
- * [Using IAM Roles - AWS Identity and Access Management](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
299
+ * [Using IAM Roles - AWS Identity and Access Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
295
300
  * [IAM Roles For Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html)
296
- * [Aws::STS::Client](http://docs.aws.amazon.com/sdkforruby/api/Aws/STS/Client.html)
301
+ * [Aws::STS::Client](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html)
297
302
  * [Aws::AssumeRoleWebIdentityCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleWebIdentityCredentials.html)
298
303
 
299
304
  #### role_arn (required)
@@ -331,7 +336,7 @@ Retrieve temporary security credentials via HTTP request. This is useful on EC2
331
336
 
332
337
  See also:
333
338
 
334
- * [Aws::InstanceProfileCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/InstanceProfileCredentials.html)
339
+ * [Aws::InstanceProfileCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/InstanceProfileCredentials.html)
335
340
  * [Temporary Security Credentials - AWS Identity and Access
336
341
  Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
337
342
  * [Instance Metadata and User Data - Amazon Elastic Compute
@@ -372,7 +377,7 @@ This loads AWS access credentials from local ini file. This is useful for local
372
377
 
373
378
  See also:
374
379
 
375
- * [Aws::SharedCredentials](https://docs.aws.amazon.com/sdkforruby/api/Aws/SharedCredentials.html)
380
+ * [Aws::SharedCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SharedCredentials.html)
376
381
 
377
382
  #### path
378
383
 
@@ -506,7 +511,7 @@ If your record contains a field whose string should be sent to Amazon Kinesis di
506
511
 
507
512
  #### compression
508
513
 
509
- Specifying compression way for data of each record. Current accepted options are `zlib` and `gzip`. Otherwise, no compression will be preformed.
514
+ Specifying compression way for data of each record. Current accepted options are `zlib` and `gzip`. Otherwise, no compression will be performed.
510
515
 
511
516
  #### log_truncate_max_size
512
517
 
@@ -535,13 +540,17 @@ HTTP proxy for API calling. Default `nil`.
535
540
 
536
541
  API endpoint URL, for testing. Default `nil`.
537
542
 
543
+ #### use_fips_endpoint
544
+
545
+ Boolean. Default `nil`. If `true`, the plugin uses FIPS-compliant endpoints for Kinesis Data Streams and Amazon Data Firehose API calls. This is required for environments that must comply with Federal Information Processing Standards (FIPS). When `nil`, the SDK's default behavior applies (which respects the `AWS_USE_FIPS_ENDPOINT` environment variable).
546
+
538
547
  #### ssl_verify_peer
539
548
 
540
- Boolean. Disable if you want to verify ssl connection, for testing. Default `true`.
549
+ Boolean. Set to `false` to disable SSL certificate verification, for testing. Default `true`.
541
550
 
542
551
  #### debug
543
552
 
544
- Boolean. Enable if you need to debug Amazon Data Firehose API call. Default is `false`.
553
+ Boolean. Enable if you need to debug API calls. Default is `false`.
545
554
 
546
555
  ## Development
547
556
 
@@ -554,7 +563,16 @@ To launch `fluentd` process with this plugin for development, follow the steps b
554
563
 
555
564
  To launch using specified version of Fluentd, use `BUNDLE_GEMFILE` environment variable:
556
565
 
557
- BUNDLE_GEMFILE=$PWD/gemfiles/Gemfile.td-agent-3.3.0 bundle exec fluentd -c /path/to/fluent.conf
566
+ BUNDLE_GEMFILE=$PWD/gemfiles/Gemfile.fluent-package-5.0.9 bundle exec fluentd -c /path/to/fluent.conf
567
+
568
+ To run tests, use `bundle exec rake test`. Integration tests in `test/integration` run only with `AWS_EMULATOR=true` against [AWS Emulator](https://aws.amazon.com/products/developer-tools/emulator/). Start the emulator and point the AWS SDK to it (see the `aws-emulator` job in [.github/workflows/test.yml](.github/workflows/test.yml)):
569
+
570
+ export AWS_ENDPOINT_URL=https://127.0.0.1:9297
571
+ export AWS_SHARED_CREDENTIALS_FILE=$HOME/.aws/emulator/9297/emulator_credentials
572
+ export AWS_CONFIG_FILE=$HOME/.aws/emulator/9297/config
573
+ export AWS_CA_BUNDLE=$HOME/.aws/emulator/9297/ca.pem
574
+ export AWS_REGION=us-east-1
575
+ AWS_EMULATOR=true bundle exec rake test
558
576
 
559
577
  ## Contributing
560
578
 
@@ -562,8 +580,8 @@ Bug reports and pull requests are welcome on [GitHub][github].
562
580
 
563
581
  ## Related Resources
564
582
 
565
- * [Amazon Kinesis Data Streams Developer Guide](http://docs.aws.amazon.com/kinesis/latest/dev/introduction.html)
566
- * [Amazon Data Firehose Developer Guide](http://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
583
+ * [Amazon Kinesis Data Streams Developer Guide](https://docs.aws.amazon.com/kinesis/latest/dev/introduction.html)
584
+ * [Amazon Data Firehose Developer Guide](https://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
567
585
 
568
586
  [github]: https://github.com/awslabs/aws-fluent-plugin-kinesis
569
587
  [fluentd-doc-kinesis]: https://docs.fluentd.org/how-to-guides/kinesis-stream
@@ -59,6 +59,8 @@ Gem::Specification.new do |spec|
59
59
 
60
60
  spec.add_development_dependency "bundler", ">= 1.10"
61
61
  spec.add_development_dependency "rake", ">= 10.0"
62
+ # Used by the benchmark tasks; benchmark is a bundled gem as of Ruby 4.0
63
+ spec.add_development_dependency "benchmark"
62
64
  spec.add_development_dependency "test-unit", ">= 3.0.8"
63
65
  spec.add_development_dependency "test-unit-rr", ">= 1.0.3"
64
66
  spec.add_development_dependency "pry", ">= 0.10.1"
@@ -68,4 +70,7 @@ Gem::Specification.new do |spec|
68
70
  spec.add_development_dependency "mocha", ">= 1.1.0"
69
71
  spec.add_development_dependency "webmock", ">= 1.24.2"
70
72
  spec.add_development_dependency "fakefs", ">= 0.8.1"
73
+ # Used by integration tests with AWS Emulator to set up and verify Firehose delivery to S3
74
+ spec.add_development_dependency "aws-sdk-iam", "~> 1"
75
+ spec.add_development_dependency "aws-sdk-s3", "~> 1"
71
76
  end
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v5.0.8
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v5.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v5.0.8/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.16.10"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v5.0.8/fluent-package/Gemfile#L57-L67
11
+ gem "prometheus-client", "2.1.0"
12
+ gem "fluent-plugin-prometheus", "2.1.0"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.1.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.785.0"
16
+ gem "aws-sdk-core", "3.178.0"
17
+ gem "aws-sdk-kms", "1.71.0"
18
+ gem "aws-sdk-sqs", "1.61.0"
19
+ gem "aws-sigv4", "1.6.0"
20
+ gem "aws-sdk-s3", "1.129.0"
21
+ gem "fluent-plugin-s3", "1.7.2"
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v5.0.9
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v5.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v5.0.9/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.16.11"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v5.0.9/fluent-package/Gemfile#L57-L67
11
+ gem "prometheus-client", "2.1.0"
12
+ gem "fluent-plugin-prometheus", "2.1.0"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.1.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.785.0"
16
+ gem "aws-sdk-core", "3.178.0"
17
+ gem "aws-sdk-kms", "1.71.0"
18
+ gem "aws-sdk-sqs", "1.61.0"
19
+ gem "aws-sigv4", "1.6.0"
20
+ gem "aws-sdk-s3", "1.129.0"
21
+ gem "fluent-plugin-s3", "1.7.2"
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v6.0.0
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.0/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.19.0"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.0/fluent-package/Gemfile#L55-L65
11
+ gem "prometheus-client", "4.2.5"
12
+ gem "fluent-plugin-prometheus", "2.2.1"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.1150.0"
16
+ gem "aws-sdk-core", "3.230.0"
17
+ gem "aws-sdk-kms", "1.110.0"
18
+ gem "aws-sdk-sqs", "1.101.0"
19
+ gem "aws-sigv4", "1.12.1"
20
+ gem "aws-sdk-s3", "1.197.0"
21
+ gem "fluent-plugin-s3", "1.8.3"
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v6.0.1
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.1/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.19.1"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.1/fluent-package/Gemfile#L55-L65
11
+ gem "prometheus-client", "4.2.5"
12
+ gem "fluent-plugin-prometheus", "2.2.1"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.1150.0"
16
+ gem "aws-sdk-core", "3.230.0"
17
+ gem "aws-sdk-kms", "1.110.0"
18
+ gem "aws-sdk-sqs", "1.101.0"
19
+ gem "aws-sigv4", "1.12.1"
20
+ gem "aws-sdk-s3", "1.197.0"
21
+ gem "fluent-plugin-s3", "1.8.3"
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v6.0.2
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.2/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.19.2"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.2/fluent-package/Gemfile#L55-L65
11
+ gem "prometheus-client", "4.2.5"
12
+ gem "fluent-plugin-prometheus", "2.2.1"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.1150.0"
16
+ gem "aws-sdk-core", "3.234.0"
17
+ gem "aws-sdk-kms", "1.110.0"
18
+ gem "aws-sdk-sqs", "1.101.0"
19
+ gem "aws-sigv4", "1.12.1"
20
+ gem "aws-sdk-s3", "1.208.0"
21
+ gem "fluent-plugin-s3", "1.8.3"
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v6.0.3
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.3/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.19.2"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.3/fluent-package/Gemfile#L62-L72
11
+ gem "prometheus-client", "4.2.5"
12
+ gem "fluent-plugin-prometheus", "2.2.1"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.1150.0"
16
+ gem "aws-sdk-core", "3.234.0"
17
+ gem "aws-sdk-kms", "1.110.0"
18
+ gem "aws-sdk-sqs", "1.101.0"
19
+ gem "aws-sigv4", "1.12.1"
20
+ gem "aws-sdk-s3", "1.208.0"
21
+ gem "fluent-plugin-s3", "1.8.4"
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v6.0.4
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.4/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.19.3"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.4/fluent-package/Gemfile#L70-L80
11
+ gem "prometheus-client", "4.2.5"
12
+ gem "fluent-plugin-prometheus", "2.2.2"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.1150.0"
16
+ gem "aws-sdk-core", "3.234.0"
17
+ gem "aws-sdk-kms", "1.110.0"
18
+ gem "aws-sdk-sqs", "1.101.0"
19
+ gem "aws-sigv4", "1.12.1"
20
+ gem "aws-sdk-s3", "1.208.0"
21
+ gem "fluent-plugin-s3", "1.8.5"
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v6.0.5
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.5/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.19.4"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v6.0.5/fluent-package/Gemfile#L70-L80
11
+ gem "prometheus-client", "4.2.5"
12
+ gem "fluent-plugin-prometheus", "2.3.0"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.1150.0"
16
+ gem "aws-sdk-core", "3.234.0"
17
+ gem "aws-sdk-kms", "1.110.0"
18
+ gem "aws-sdk-sqs", "1.101.0"
19
+ gem "aws-sigv4", "1.12.1"
20
+ gem "aws-sdk-s3", "1.208.0"
21
+ gem "fluent-plugin-s3", "1.8.6"
@@ -0,0 +1,21 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ # Specify related gems for fluent-package v6.1.0
7
+ # https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
8
+ # https://github.com/fluent/fluent-package-builder/blob/v6.1.0/fluent-package/config.rb#L10
9
+ gem "fluentd", "1.20.1"
10
+ # https://github.com/fluent/fluent-package-builder/blob/v6.1.0/fluent-package/Gemfile#L69-L79
11
+ gem "prometheus-client", "5.0.0"
12
+ gem "fluent-plugin-prometheus", "2.3.0"
13
+ gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
14
+ gem "jmespath", "1.6.2"
15
+ gem "aws-partitions", "1.1298.0"
16
+ gem "aws-sdk-core", "3.260.0"
17
+ gem "aws-sdk-kms", "1.135.0"
18
+ gem "aws-sdk-sqs", "1.122.0"
19
+ gem "aws-sigv4", "1.12.1"
20
+ gem "aws-sdk-s3", "1.235.0"
21
+ gem "fluent-plugin-s3", "1.8.6"
@@ -0,0 +1,6 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
4
+ gemspec path: ".."
5
+
6
+ gem "fluentd", "1.20.1"
@@ -30,15 +30,26 @@ module Fluent
30
30
  def initialize(message, record)
31
31
  super message
32
32
  @record_message = if record.is_a? Array
33
- record.reverse.map(&:to_s).join(', ')
33
+ record.reverse.map{|r| format_record(r) }.join(', ')
34
34
  else
35
- record.to_s
35
+ format_record(record)
36
36
  end
37
37
  end
38
38
 
39
39
  def to_s
40
40
  super + ": " + @record_message
41
41
  end
42
+
43
+ private
44
+
45
+ def format_record(record)
46
+ str = record.to_s
47
+ if str.encoding == Encoding::ASCII_8BIT || !str.valid_encoding?
48
+ str.unpack1('H*')
49
+ else
50
+ str
51
+ end
52
+ end
42
53
  end
43
54
  class KeyNotFoundError < SkipRecordError
44
55
  def initialize(key, record)
@@ -14,7 +14,6 @@
14
14
 
15
15
  require 'fluent_plugin_kinesis/version'
16
16
  require 'fluent/configurable'
17
- require 'benchmark'
18
17
 
19
18
  module Fluent
20
19
  module Plugin
@@ -114,7 +113,9 @@ module Fluent
114
113
  # TODO: find out who is causing the sleep to be too short and try to make them stop it instead
115
114
  def reliable_sleep(wait_second)
116
115
  loop do
117
- actual = Benchmark.realtime { sleep(wait_second) }
116
+ started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
117
+ sleep(wait_second)
118
+ actual = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
118
119
  break if actual >= wait_second
119
120
  log.error("#{Thread.current.object_id} sleep failed expected #{wait_second} but slept #{actual}")
120
121
  wait_second -= actual
@@ -26,6 +26,7 @@ module Fluent
26
26
  config_param :http_proxy, :string, default: nil, secret: true
27
27
  config_param :endpoint, :string, default: nil
28
28
  config_param :ssl_verify_peer, :bool, default: true
29
+ config_param :use_fips_endpoint, :bool, default: nil
29
30
 
30
31
  config_param :aws_key_id, :string, default: nil, secret: true
31
32
  config_param :aws_sec_key, :string, default: nil, secret: true
@@ -121,6 +122,10 @@ module Fluent
121
122
  options.update(http_proxy: @http_proxy) unless @http_proxy.nil?
122
123
  options.update(endpoint: @endpoint) unless @endpoint.nil?
123
124
  options.update(ssl_verify_peer: @ssl_verify_peer) unless @ssl_verify_peer.nil?
125
+ if !@use_fips_endpoint.nil? && client_class.method_defined?(:config) &&
126
+ Gem::Version.new(Aws::CORE_GEM_VERSION) >= Gem::Version.new('3.122.0')
127
+ options.update(use_fips_endpoint: @use_fips_endpoint)
128
+ end
124
129
  if @debug
125
130
  options.update(logger: Logger.new(log.out))
126
131
  options.update(log_level: :debug)
@@ -197,10 +202,17 @@ module Fluent
197
202
  end
198
203
  c = @process_credentials
199
204
  process = c.process
200
- options[:credentials] = Aws::ProcessCredentials.new(process)
205
+ # Suppress insecure string warning from newer aws-sdk-core.
206
+ # Using array form would break older versions that use Open3.popen3.
207
+ options[:credentials] = begin
208
+ verbose, $VERBOSE = $VERBOSE, nil
209
+ Aws::ProcessCredentials.new(process)
210
+ ensure
211
+ $VERBOSE = verbose
212
+ end
201
213
  else
202
214
  # Use default credentials
203
- # See http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html
215
+ # See https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html
204
216
  end
205
217
  options
206
218
  end
@@ -26,6 +26,7 @@ module Fluent
26
26
 
27
27
  config_param :delivery_stream_name, :string
28
28
  config_param :append_new_line, :bool, default: true
29
+ config_param :aggregated_record_size, :integer, default: 0
29
30
 
30
31
  def configure(conf)
31
32
  super
@@ -35,6 +36,9 @@ module Fluent
35
36
  org_data_formatter.call(tag, time, record).chomp + "\n"
36
37
  }
37
38
  end
39
+ if @aggregated_record_size > @max_record_size
40
+ raise Fluent::ConfigError, "aggregated_record_size (#{@aggregated_record_size}) can't be greater than max_record_size (#{@max_record_size})."
41
+ end
38
42
  end
39
43
 
40
44
  def format(tag, time, record)
@@ -45,14 +49,62 @@ module Fluent
45
49
 
46
50
  def write(chunk)
47
51
  delivery_stream_name = extract_placeholders(@delivery_stream_name, chunk)
48
- write_records_batch(chunk, delivery_stream_name) do |batch|
49
- records = batch.map{|(data)|
50
- { data: data }
51
- }
52
- client.put_record_batch(
53
- delivery_stream_name: delivery_stream_name,
54
- records: records,
55
- )
52
+ if @aggregated_record_size > 0
53
+ write_records_batch_aggregated(chunk, delivery_stream_name) do |batch|
54
+ records = batch.map{|(data)|
55
+ { data: data }
56
+ }
57
+ client.put_record_batch(
58
+ delivery_stream_name: delivery_stream_name,
59
+ records: records,
60
+ )
61
+ end
62
+ else
63
+ write_records_batch(chunk, delivery_stream_name) do |batch|
64
+ records = batch.map{|(data)|
65
+ { data: data }
66
+ }
67
+ client.put_record_batch(
68
+ delivery_stream_name: delivery_stream_name,
69
+ records: records,
70
+ )
71
+ end
72
+ end
73
+ end
74
+
75
+ private
76
+
77
+ def write_records_batch_aggregated(chunk, stream_name, &block)
78
+ unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
79
+ records = chunk.to_enum(:msgpack_each)
80
+ aggregated = aggregate_records(records)
81
+ split_to_batches(aggregated) do |batch, size|
82
+ log.debug(sprintf "%s: Write chunk %s / %3d records / %4d KB", stream_name, unique_id, batch.size, size/1024)
83
+ batch_request_with_retry(batch, &block)
84
+ log.debug(sprintf "%s: Finish writing chunk", stream_name)
85
+ end
86
+ end
87
+
88
+ def aggregate_records(records)
89
+ Enumerator.new do |y|
90
+ current = nil
91
+ current_size = 0
92
+ records.each do |record|
93
+ data = record.first
94
+ data_size = data.bytesize
95
+ if current.nil?
96
+ current = data
97
+ current_size = data_size
98
+ elsif current_size + data_size <= @aggregated_record_size
99
+ current = current + data
100
+ current_size += data_size
101
+ else
102
+ y.yield [current]
103
+ current = data
104
+ current_size = data_size
105
+ end
106
+ end
107
+ y.yield [current] unless current.nil?
56
108
  end
57
109
  end
58
110
  end
@@ -13,5 +13,5 @@
13
13
  # language governing permissions and limitations under the License.
14
14
 
15
15
  module FluentPluginKinesis
16
- VERSION = '3.6.1'
16
+ VERSION = '3.7.1'
17
17
  end
metadata CHANGED
@@ -1,14 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kinesis
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.6.1
4
+ version: 3.7.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon Web Services
8
- autorequire:
9
8
  bindir: bin
10
9
  cert_chain: []
11
- date: 2025-08-01 00:00:00.000000000 Z
10
+ date: 1980-01-02 00:00:00.000000000 Z
12
11
  dependencies:
13
12
  - !ruby/object:Gem::Dependency
14
13
  name: fluentd
@@ -136,6 +135,20 @@ dependencies:
136
135
  - - ">="
137
136
  - !ruby/object:Gem::Version
138
137
  version: '10.0'
138
+ - !ruby/object:Gem::Dependency
139
+ name: benchmark
140
+ requirement: !ruby/object:Gem::Requirement
141
+ requirements:
142
+ - - ">="
143
+ - !ruby/object:Gem::Version
144
+ version: '0'
145
+ type: :development
146
+ prerelease: false
147
+ version_requirements: !ruby/object:Gem::Requirement
148
+ requirements:
149
+ - - ">="
150
+ - !ruby/object:Gem::Version
151
+ version: '0'
139
152
  - !ruby/object:Gem::Dependency
140
153
  name: test-unit
141
154
  requirement: !ruby/object:Gem::Requirement
@@ -262,8 +275,34 @@ dependencies:
262
275
  - - ">="
263
276
  - !ruby/object:Gem::Version
264
277
  version: 0.8.1
265
- description:
266
- email:
278
+ - !ruby/object:Gem::Dependency
279
+ name: aws-sdk-iam
280
+ requirement: !ruby/object:Gem::Requirement
281
+ requirements:
282
+ - - "~>"
283
+ - !ruby/object:Gem::Version
284
+ version: '1'
285
+ type: :development
286
+ prerelease: false
287
+ version_requirements: !ruby/object:Gem::Requirement
288
+ requirements:
289
+ - - "~>"
290
+ - !ruby/object:Gem::Version
291
+ version: '1'
292
+ - !ruby/object:Gem::Dependency
293
+ name: aws-sdk-s3
294
+ requirement: !ruby/object:Gem::Requirement
295
+ requirements:
296
+ - - "~>"
297
+ - !ruby/object:Gem::Version
298
+ version: '1'
299
+ type: :development
300
+ prerelease: false
301
+ version_requirements: !ruby/object:Gem::Requirement
302
+ requirements:
303
+ - - "~>"
304
+ - !ruby/object:Gem::Version
305
+ version: '1'
267
306
  executables: []
268
307
  extensions: []
269
308
  extra_rdoc_files: []
@@ -287,9 +326,19 @@ files:
287
326
  - gemfiles/Gemfile.fluent-package-5.0.1
288
327
  - gemfiles/Gemfile.fluent-package-5.0.2
289
328
  - gemfiles/Gemfile.fluent-package-5.0.7
329
+ - gemfiles/Gemfile.fluent-package-5.0.8
330
+ - gemfiles/Gemfile.fluent-package-5.0.9
290
331
  - gemfiles/Gemfile.fluent-package-5.2.0
332
+ - gemfiles/Gemfile.fluent-package-6.0.0
333
+ - gemfiles/Gemfile.fluent-package-6.0.1
334
+ - gemfiles/Gemfile.fluent-package-6.0.2
335
+ - gemfiles/Gemfile.fluent-package-6.0.3
336
+ - gemfiles/Gemfile.fluent-package-6.0.4
337
+ - gemfiles/Gemfile.fluent-package-6.0.5
338
+ - gemfiles/Gemfile.fluent-package-6.1.0
291
339
  - gemfiles/Gemfile.fluentd-0.14.22
292
340
  - gemfiles/Gemfile.fluentd-1.19.0
341
+ - gemfiles/Gemfile.fluentd-1.20.1
293
342
  - gemfiles/Gemfile.td-agent-3.1.0
294
343
  - gemfiles/Gemfile.td-agent-3.1.1
295
344
  - gemfiles/Gemfile.td-agent-3.2.0
@@ -334,7 +383,6 @@ homepage: https://github.com/awslabs/aws-fluent-plugin-kinesis
334
383
  licenses:
335
384
  - Apache-2.0
336
385
  metadata: {}
337
- post_install_message:
338
386
  rdoc_options: []
339
387
  require_paths:
340
388
  - lib
@@ -349,8 +397,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
349
397
  - !ruby/object:Gem::Version
350
398
  version: '0'
351
399
  requirements: []
352
- rubygems_version: 3.2.33
353
- signing_key:
400
+ rubygems_version: 4.0.3
354
401
  specification_version: 4
355
402
  summary: Fluentd output plugin that sends events to Amazon Kinesis.
356
403
  test_files: []