fluent-plugin-kinesis 3.6.1 → 3.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +94 -19
- data/CHANGELOG.md +15 -0
- data/README.md +35 -17
- data/fluent-plugin-kinesis.gemspec +5 -0
- data/gemfiles/Gemfile.fluent-package-5.0.8 +21 -0
- data/gemfiles/Gemfile.fluent-package-5.0.9 +21 -0
- data/gemfiles/Gemfile.fluent-package-6.0.0 +21 -0
- data/gemfiles/Gemfile.fluent-package-6.0.1 +21 -0
- data/gemfiles/Gemfile.fluent-package-6.0.2 +21 -0
- data/gemfiles/Gemfile.fluent-package-6.0.3 +21 -0
- data/gemfiles/Gemfile.fluent-package-6.0.4 +21 -0
- data/gemfiles/Gemfile.fluent-package-6.0.5 +21 -0
- data/gemfiles/Gemfile.fluent-package-6.1.0 +21 -0
- data/gemfiles/Gemfile.fluentd-1.20.1 +6 -0
- data/lib/fluent/plugin/kinesis.rb +13 -2
- data/lib/fluent/plugin/kinesis_helper/api.rb +3 -2
- data/lib/fluent/plugin/kinesis_helper/client.rb +14 -2
- data/lib/fluent/plugin/out_kinesis_firehose.rb +60 -8
- data/lib/fluent_plugin_kinesis/version.rb +1 -1
- metadata +55 -8
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 406d3b87604e265677a3df5a65645b9d9f45d959c7eb2431c03825884420cba8
|
|
4
|
+
data.tar.gz: 1ecc97af6d59ba885829719c076c3a2243fed7baf0302c11e72d3c126099e37a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 643b74ada4bdd6d5006620ce3f94b54cbf391c5bac66317ba7447f65bcbce17298ecf130d47541484be1821f9cfcb148b5371fb077e76ecc2035fca696c60ace
|
|
7
|
+
data.tar.gz: 16df0f4ce732bbd8cdda57109e491b027668f172a548043e47bcd0804e183801090e7d6d5463a7eeb548523521475c61e1700f840c2626d8bb5ffec1ebdf56c5
|
data/.github/workflows/test.yml
CHANGED
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
name: test
|
|
2
2
|
|
|
3
|
-
on:
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
# Run weekly to detect breakage by new releases of dependencies such as Fluentd and AWS SDK
|
|
7
|
+
schedule:
|
|
8
|
+
- cron: '0 0 * * 1'
|
|
9
|
+
workflow_dispatch:
|
|
4
10
|
|
|
5
11
|
# Test with supported fluent-package versions
|
|
6
12
|
# https://www.fluentd.org/download/fluent_package
|
|
7
13
|
# https://docs.fluentd.org/quickstart/support#change-logs
|
|
8
14
|
# https://github.com/fluent/fluentd/blob/master/CHANGELOG.md
|
|
9
15
|
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
|
|
16
|
+
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v5.md
|
|
10
17
|
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v4.md
|
|
11
18
|
jobs:
|
|
12
19
|
build:
|
|
@@ -16,9 +23,9 @@ jobs:
|
|
|
16
23
|
matrix:
|
|
17
24
|
include:
|
|
18
25
|
- gemfile: gemfiles/Gemfile.fluentd-0.14.22 # Oldest supported Fluentd
|
|
19
|
-
ruby-version: 2.7
|
|
20
|
-
- gemfile: gemfiles/Gemfile.fluentd-1.
|
|
21
|
-
ruby-version:
|
|
26
|
+
ruby-version: '2.7' # Oldest supported Ruby
|
|
27
|
+
- gemfile: gemfiles/Gemfile.fluentd-1.20.1 # Latest Fluentd / https://rubygems.org/gems/fluentd
|
|
28
|
+
ruby-version: '4.0' # Latest supported Ruby / https://www.ruby-lang.org/en/downloads
|
|
22
29
|
# - gemfile: gemfiles/Gemfile.td-agent-3.1.0
|
|
23
30
|
# ruby-version: 2.4.2 # https://github.com/treasure-data/omnibus-td-agent/blob/v3.1.0/config/projects/td-agent3.rb#L20
|
|
24
31
|
# - gemfile: gemfiles/Gemfile.td-agent-3.1.1
|
|
@@ -50,25 +57,25 @@ jobs:
|
|
|
50
57
|
# - gemfile: gemfiles/Gemfile.td-agent-4.0.0
|
|
51
58
|
# ruby-version: 2.7.1 # https://github.com/fluent/fluent-package-builder/blob/41b3436d43c21e48a9cf34c58ae640f0e3dfb44c/td-agent/config.rb#L16
|
|
52
59
|
- gemfile: gemfiles/Gemfile.td-agent-4.0.1 # Oldest supported td-agent
|
|
53
|
-
ruby-version: 2.7
|
|
60
|
+
ruby-version: '2.7' # https://github.com/fluent/fluent-package-builder/blob/v4.0.1/td-agent/config.rb#L16
|
|
54
61
|
# - gemfile: gemfiles/Gemfile.td-agent-4.1.0
|
|
55
62
|
# ruby-version: 2.7.2 # https://github.com/fluent/fluent-package-builder/blob/v4.1.0/td-agent/config.rb#L19
|
|
56
|
-
- gemfile: gemfiles/Gemfile.td-agent-4.1.1
|
|
57
|
-
|
|
58
|
-
- gemfile: gemfiles/Gemfile.td-agent-4.2.0
|
|
59
|
-
|
|
63
|
+
# - gemfile: gemfiles/Gemfile.td-agent-4.1.1
|
|
64
|
+
# ruby-version: 2.7.3 # https://github.com/fluent/fluent-package-builder/blob/v4.1.1/td-agent/config.rb#L17
|
|
65
|
+
# - gemfile: gemfiles/Gemfile.td-agent-4.2.0
|
|
66
|
+
# ruby-version: 2.7.4 # https://github.com/fluent/fluent-package-builder/blob/v4.2.0/td-agent/config.rb#L21
|
|
60
67
|
# - gemfile: gemfiles/Gemfile.td-agent-4.3.0
|
|
61
68
|
# ruby-version: 2.7.5 # https://github.com/fluent/fluent-package-builder/blob/v4.3.0/td-agent/config.rb#L21
|
|
62
69
|
# - gemfile: gemfiles/Gemfile.td-agent-4.3.1
|
|
63
70
|
# ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.3.1/td-agent/config.rb#L19
|
|
64
|
-
- gemfile: gemfiles/Gemfile.td-agent-4.3.2
|
|
65
|
-
|
|
71
|
+
# - gemfile: gemfiles/Gemfile.td-agent-4.3.2
|
|
72
|
+
# ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.3.2/td-agent/config.rb#L23
|
|
66
73
|
# - gemfile: gemfiles/Gemfile.td-agent-4.4.0
|
|
67
74
|
# ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.4.0/td-agent/config.rb#L23
|
|
68
75
|
# - gemfile: gemfiles/Gemfile.td-agent-4.4.1
|
|
69
76
|
# ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.4.1/td-agent/config.rb#L23
|
|
70
|
-
- gemfile: gemfiles/Gemfile.td-agent-4.4.2
|
|
71
|
-
|
|
77
|
+
# - gemfile: gemfiles/Gemfile.td-agent-4.4.2
|
|
78
|
+
# ruby-version: 2.7.6 # https://github.com/fluent/fluent-package-builder/blob/v4.4.2/td-agent/config.rb#L23
|
|
72
79
|
# - gemfile: gemfiles/Gemfile.td-agent-4.5.0
|
|
73
80
|
# ruby-version: 2.7.8 # https://github.com/fluent/fluent-package-builder/blob/v4.5.0/td-agent/config.rb#L19
|
|
74
81
|
# - gemfile: gemfiles/Gemfile.td-agent-4.5.1
|
|
@@ -76,17 +83,35 @@ jobs:
|
|
|
76
83
|
# - gemfile: gemfiles/Gemfile.td-agent-4.5.2
|
|
77
84
|
# ruby-version: 2.7.8 # https://github.com/fluent/fluent-package-builder/blob/v4.5.2/td-agent/config.rb#L19
|
|
78
85
|
- gemfile: gemfiles/Gemfile.td-agent-4.5.3 # Latest release of td-agent v4
|
|
79
|
-
ruby-version: 2.7
|
|
86
|
+
ruby-version: '2.7' # https://github.com/fluent/fluent-package-builder/blob/v4.5.3/td-agent/config.rb#L19
|
|
80
87
|
# - gemfile: gemfiles/Gemfile.fluent-package-5.0.0
|
|
81
88
|
# ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.0/fluent-package/config.rb#L26
|
|
82
89
|
# - gemfile: gemfiles/Gemfile.fluent-package-5.0.1
|
|
83
90
|
# ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.1/fluent-package/config.rb#L26
|
|
84
91
|
# - gemfile: gemfiles/Gemfile.fluent-package-5.0.2
|
|
85
92
|
# ruby-version: 3.2.2 # https://github.com/fluent/fluent-package-builder/blob/v5.0.2/fluent-package/config.rb#L26
|
|
86
|
-
- gemfile: gemfiles/Gemfile.fluent-package-5.0.7
|
|
87
|
-
|
|
88
|
-
- gemfile: gemfiles/Gemfile.fluent-package-5.
|
|
89
|
-
|
|
93
|
+
# - gemfile: gemfiles/Gemfile.fluent-package-5.0.7
|
|
94
|
+
# ruby-version: 3.2.8 # https://github.com/fluent/fluent-package-builder/blob/v5.0.7/fluent-package/config.rb#L26
|
|
95
|
+
# - gemfile: gemfiles/Gemfile.fluent-package-5.0.8
|
|
96
|
+
# ruby-version: 3.2.9 # https://github.com/fluent/fluent-package-builder/blob/v5.0.8/fluent-package/config.rb#L26
|
|
97
|
+
- gemfile: gemfiles/Gemfile.fluent-package-5.0.9 # Latest release of fluent-package v5
|
|
98
|
+
ruby-version: '3.2' # https://github.com/fluent/fluent-package-builder/blob/v5.0.9/fluent-package/config.rb#L26
|
|
99
|
+
# - gemfile: gemfiles/Gemfile.fluent-package-5.2.0
|
|
100
|
+
# ruby-version: 3.2.6 # https://github.com/fluent/fluent-package-builder/blob/v5.2.0/fluent-package/config.rb#L26
|
|
101
|
+
# - gemfile: gemfiles/Gemfile.fluent-package-6.0.0
|
|
102
|
+
# ruby-version: 3.4.5 # https://github.com/fluent/fluent-package-builder/blob/v6.0.0/fluent-package/config.rb#L26
|
|
103
|
+
# - gemfile: gemfiles/Gemfile.fluent-package-6.0.1
|
|
104
|
+
# ruby-version: 3.4.7 # https://github.com/fluent/fluent-package-builder/blob/v6.0.1/fluent-package/config.rb#L26
|
|
105
|
+
# - gemfile: gemfiles/Gemfile.fluent-package-6.0.2
|
|
106
|
+
# ruby-version: 3.4.8 # https://github.com/fluent/fluent-package-builder/blob/v6.0.2/fluent-package/config.rb#L26
|
|
107
|
+
# - gemfile: gemfiles/Gemfile.fluent-package-6.0.3
|
|
108
|
+
# ruby-version: 3.4.9 # https://github.com/fluent/fluent-package-builder/blob/v6.0.3/fluent-package/config.rb#L26
|
|
109
|
+
# - gemfile: gemfiles/Gemfile.fluent-package-6.0.4
|
|
110
|
+
# ruby-version: 3.4.9 # https://github.com/fluent/fluent-package-builder/blob/v6.0.4/fluent-package/config.rb#L26
|
|
111
|
+
- gemfile: gemfiles/Gemfile.fluent-package-6.0.5 # Latest release of fluent-package v6 LTS
|
|
112
|
+
ruby-version: '3.4' # https://github.com/fluent/fluent-package-builder/blob/v6.0.5/fluent-package/config.rb#L26
|
|
113
|
+
- gemfile: gemfiles/Gemfile.fluent-package-6.1.0 # Latest release of fluent-package
|
|
114
|
+
ruby-version: '3.4' # https://github.com/fluent/fluent-package-builder/blob/v6.1.0/fluent-package/config.rb#L29
|
|
90
115
|
|
|
91
116
|
env:
|
|
92
117
|
BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}
|
|
@@ -99,4 +124,54 @@ jobs:
|
|
|
99
124
|
ruby-version: ${{ matrix.ruby-version }}
|
|
100
125
|
bundler-cache: true
|
|
101
126
|
- name: Run tests
|
|
102
|
-
run: bundle exec rake test
|
|
127
|
+
run: bundle exec rake test
|
|
128
|
+
aws-emulator:
|
|
129
|
+
# Run the tests including integration tests against AWS Emulator (https://aws.amazon.com/products/developer-tools/emulator/)
|
|
130
|
+
runs-on: ubuntu-latest
|
|
131
|
+
env:
|
|
132
|
+
BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/Gemfile.fluentd-1.20.1 # Latest Fluentd
|
|
133
|
+
AWS_EMULATOR: 'true'
|
|
134
|
+
|
|
135
|
+
steps:
|
|
136
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
137
|
+
- name: Install AWS Emulator
|
|
138
|
+
working-directory: ${{ runner.temp }}
|
|
139
|
+
run: |
|
|
140
|
+
PACKAGE_FILE=aws-emulator_amd64.deb
|
|
141
|
+
curl -fsSLO https://downloads.aws.com/aws-emulator/packages/aws-emulator/LTS/latest/$PACKAGE_FILE
|
|
142
|
+
curl -fsSLO https://downloads.aws.com/aws-emulator/packages/aws-emulator/LTS/latest/$PACKAGE_FILE.sig
|
|
143
|
+
curl -fsSLo aws-publishing.asc https://downloads.aws.com/keys/gpg/current.asc
|
|
144
|
+
gpg --import aws-publishing.asc
|
|
145
|
+
# Verify the package with the documented fingerprint of the AWS publishing key
|
|
146
|
+
gpg --with-colons --fingerprint ssm-publishing@amazon.com | grep -q '^fpr:::::::::98BCF336A3BE53B734E967C52E5BF49BD940F624:'
|
|
147
|
+
gpg --verify $PACKAGE_FILE.sig $PACKAGE_FILE
|
|
148
|
+
sudo apt-get install -y ./$PACKAGE_FILE
|
|
149
|
+
aws-emulator -v
|
|
150
|
+
- name: Start AWS Emulator
|
|
151
|
+
run: |
|
|
152
|
+
nohup aws-emulator > "$RUNNER_TEMP/aws-emulator.log" 2>&1 &
|
|
153
|
+
EMULATOR_DIR="$HOME/.aws/emulator/9297"
|
|
154
|
+
for i in $(seq 30); do
|
|
155
|
+
curl -sf --cacert "$EMULATOR_DIR/ca.pem" https://127.0.0.1:9297/__admin/health && break
|
|
156
|
+
sleep 1
|
|
157
|
+
done
|
|
158
|
+
curl -sf --cacert "$EMULATOR_DIR/ca.pem" https://127.0.0.1:9297/__admin/health
|
|
159
|
+
# AWS_EC2_METADATA_DISABLED is not set here, since unit tests of instance profile credentials mock the metadata endpoint.
|
|
160
|
+
# Integration tests set it while they run.
|
|
161
|
+
{
|
|
162
|
+
echo "AWS_ENDPOINT_URL=https://127.0.0.1:9297"
|
|
163
|
+
echo "AWS_SHARED_CREDENTIALS_FILE=$EMULATOR_DIR/emulator_credentials"
|
|
164
|
+
echo "AWS_CONFIG_FILE=$EMULATOR_DIR/config"
|
|
165
|
+
echo "AWS_CA_BUNDLE=$EMULATOR_DIR/ca.pem"
|
|
166
|
+
echo "AWS_REGION=us-east-1"
|
|
167
|
+
} >> "$GITHUB_ENV"
|
|
168
|
+
- name: Set up Ruby
|
|
169
|
+
uses: ruby/setup-ruby@b3aaceaef7e0ee231b499e7b5f49c03fc1d32199 # v1.338.0
|
|
170
|
+
with:
|
|
171
|
+
ruby-version: '4.0' # Latest supported Ruby
|
|
172
|
+
bundler-cache: true
|
|
173
|
+
- name: Run tests including integration tests with AWS Emulator
|
|
174
|
+
run: bundle exec rake test
|
|
175
|
+
- name: Show AWS Emulator log
|
|
176
|
+
if: failure()
|
|
177
|
+
run: tail -n 200 "$RUNNER_TEMP/aws-emulator.log"
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# CHANGELOG
|
|
2
2
|
|
|
3
|
+
## 3.7.1
|
|
4
|
+
|
|
5
|
+
- Bug fix - Fix loading the plugin with Bundler on Ruby 4.0, where benchmark is no longer a default gem
|
|
6
|
+
- Enhancement - Add fluent-package 6.0.5 and 6.1.0, and Fluentd 1.20.1 with Ruby 3.4 and 4.0 to test cases
|
|
7
|
+
- Enhancement - Add integration tests with AWS Emulator
|
|
8
|
+
|
|
9
|
+
## 3.7.0
|
|
10
|
+
|
|
11
|
+
- Feature - Add `use_fips_endpoint` configuration parameter for FIPS-compliant endpoints : [#227](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/227)
|
|
12
|
+
- Feature - Add `aggregated_record_size` configuration parameter for `kinesis_firehose` to reduce Firehose 5 KB per-record billing roundup : [#193](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/193)
|
|
13
|
+
- Bug fix - Fix binary data appearing in error log when compression is enabled : [#231](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/231)
|
|
14
|
+
- Bug fix - Suppress `ProcessCredentials` insecure string warning from newer aws-sdk-core
|
|
15
|
+
- Enhancement - Add fluent-package v6 LTS and Fluentd 1.19.2 with Ruby 3.4.8 to test cases
|
|
16
|
+
- Enhancement - Fix typos and update links in README
|
|
17
|
+
|
|
3
18
|
## 3.6.1
|
|
4
19
|
|
|
5
20
|
- Dependency - Update gem dependency to Ruby 2.7.1+
|
data/README.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# Fluent plugin for Amazon Kinesis
|
|
2
2
|
|
|
3
|
-
[](https://github.com/awslabs/aws-fluent-plugin-kinesis/actions/workflows/test.yml)
|
|
4
3
|
[](https://rubygems.org/gems/fluent-plugin-kinesis)
|
|
5
4
|
[](https://rubygems.org/gems/fluent-plugin-kinesis)
|
|
5
|
+
[](https://github.com/awslabs/aws-fluent-plugin-kinesis/actions/workflows/test.yml)
|
|
6
6
|
|
|
7
7
|
A fluentd output plugin to send events to [Amazon Kinesis Data Streams][streams] and [Amazon Data Firehose][firehose]. The plugin also supports [KPL Aggregated Record Format][kpl].
|
|
8
8
|
|
|
@@ -30,7 +30,7 @@ If you would like to build by yourself and install, you can build and install as
|
|
|
30
30
|
$ bundle exec rake build
|
|
31
31
|
$ bundle exec rake install
|
|
32
32
|
|
|
33
|
-
If using td-agent v4 or lower, use td-agent-gem:
|
|
33
|
+
If using td-agent v4 or lower (EOL), use td-agent-gem:
|
|
34
34
|
|
|
35
35
|
$ td-agent-gem install fluent-plugin-kinesis
|
|
36
36
|
|
|
@@ -42,7 +42,8 @@ If using fluent-package v5 or higher, use fluent-gem:
|
|
|
42
42
|
|
|
43
43
|
| fluent-plugin-kinesis | fluentd | ruby |
|
|
44
44
|
|:---------------------:|:-----------:|:--------:|
|
|
45
|
-
| >= 3.
|
|
45
|
+
| >= 3.7.0 | >= 0.14.22 | >= 2.7.1 |
|
|
46
|
+
| >= 3.6.0 && < 3.7.0 | >= 0.14.22 | >= 2.7.1 |
|
|
46
47
|
| >= 3.5.0 && < 3.6.0 | >= 0.14.22 | >= 2.4.2 |
|
|
47
48
|
| >= 3.2.0 && < 3.5.0 | >= 0.14.22 | >= 2.3 |
|
|
48
49
|
| >= 3.0.0 && < 3.2.0 | >= 0.14.10 | >= 2.1 |
|
|
@@ -176,13 +177,17 @@ The following parameters are `kinesis_firehose` specific configurations.
|
|
|
176
177
|
|
|
177
178
|
Name of the delivery stream to put data.
|
|
178
179
|
|
|
179
|
-
As of Fluentd v1,
|
|
180
|
+
As of Fluentd v1, placeholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
|
|
180
181
|
|
|
181
182
|
#### append_new_line
|
|
182
183
|
|
|
183
184
|
Boolean. Default `true`. If it is enabled, the plugin adds new line character (`\n`) to each serialized record.
|
|
184
185
|
Before appending `\n`, plugin calls chomp and removes separator from the end of each record as [chomp_record](#chomp_record) is `true`. Therefore, you don't need to enable [chomp_record](#chomp_record) option when you use [kinesis_firehose](#kinesis_firehose) output with default configuration ([append_new_line](#append_new_line) is `true`). If you want to set [append_new_line](#append_new_line) `false`, you can choose [chomp_record](#chomp_record) `false` (default) or `true` (compatible format with plugin v2).
|
|
185
186
|
|
|
187
|
+
#### aggregated_record_size
|
|
188
|
+
|
|
189
|
+
Integer. Default `0` (no aggregation). When set to a value greater than 0, the plugin concatenates multiple small records into a single larger record up to the specified size (in bytes) before sending to Firehose via PutRecordBatch. This can reduce costs when using Direct PUT ingestion, which applies a 5 KB per-record billing roundup. The value cannot exceed [max_record_size](#max_record_size) (default 1 MB). This is similar to the `aggregatedRecordSizeBytes` configuration in [Kinesis Agent](https://docs.aws.amazon.com/firehose/latest/dev/agent-config-settings.html).
|
|
190
|
+
|
|
186
191
|
### Configuration: kinesis_streams_aggregated
|
|
187
192
|
|
|
188
193
|
The following parameters are `kinesis_streams_aggregated` specific configurations.
|
|
@@ -191,7 +196,7 @@ The following parameters are `kinesis_streams_aggregated` specific configuration
|
|
|
191
196
|
|
|
192
197
|
Name of the stream to put data.
|
|
193
198
|
|
|
194
|
-
As of Fluentd v1,
|
|
199
|
+
As of Fluentd v1, placeholders are supported. For more details, see [stream_name for kinesis_streams plugin](#stream_name) and [Placeholders in Config: Buffer Section][fluentd-buffer-section-placeholders].
|
|
195
200
|
|
|
196
201
|
#### fixed_partition_key
|
|
197
202
|
|
|
@@ -240,8 +245,8 @@ See also:
|
|
|
240
245
|
|
|
241
246
|
* [Using IAM Roles - AWS Identity and Access
|
|
242
247
|
Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
|
|
243
|
-
* [Aws::STS::Client](https://docs.aws.amazon.com/
|
|
244
|
-
* [Aws::AssumeRoleCredentials](https://docs.aws.amazon.com/
|
|
248
|
+
* [Aws::STS::Client](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html)
|
|
249
|
+
* [Aws::AssumeRoleCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleCredentials.html)
|
|
245
250
|
|
|
246
251
|
#### role_arn (required)
|
|
247
252
|
|
|
@@ -291,9 +296,9 @@ Similar to the assume_role_credentials, but for usage in EKS.
|
|
|
291
296
|
|
|
292
297
|
See also:
|
|
293
298
|
|
|
294
|
-
* [Using IAM Roles - AWS Identity and Access Management](
|
|
299
|
+
* [Using IAM Roles - AWS Identity and Access Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use.html)
|
|
295
300
|
* [IAM Roles For Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html)
|
|
296
|
-
* [Aws::STS::Client](
|
|
301
|
+
* [Aws::STS::Client](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html)
|
|
297
302
|
* [Aws::AssumeRoleWebIdentityCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/AssumeRoleWebIdentityCredentials.html)
|
|
298
303
|
|
|
299
304
|
#### role_arn (required)
|
|
@@ -331,7 +336,7 @@ Retrieve temporary security credentials via HTTP request. This is useful on EC2
|
|
|
331
336
|
|
|
332
337
|
See also:
|
|
333
338
|
|
|
334
|
-
* [Aws::InstanceProfileCredentials](https://docs.aws.amazon.com/
|
|
339
|
+
* [Aws::InstanceProfileCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/InstanceProfileCredentials.html)
|
|
335
340
|
* [Temporary Security Credentials - AWS Identity and Access
|
|
336
341
|
Management](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp.html)
|
|
337
342
|
* [Instance Metadata and User Data - Amazon Elastic Compute
|
|
@@ -372,7 +377,7 @@ This loads AWS access credentials from local ini file. This is useful for local
|
|
|
372
377
|
|
|
373
378
|
See also:
|
|
374
379
|
|
|
375
|
-
* [Aws::SharedCredentials](https://docs.aws.amazon.com/
|
|
380
|
+
* [Aws::SharedCredentials](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/SharedCredentials.html)
|
|
376
381
|
|
|
377
382
|
#### path
|
|
378
383
|
|
|
@@ -506,7 +511,7 @@ If your record contains a field whose string should be sent to Amazon Kinesis di
|
|
|
506
511
|
|
|
507
512
|
#### compression
|
|
508
513
|
|
|
509
|
-
Specifying compression way for data of each record. Current accepted options are `zlib` and `gzip`. Otherwise, no compression will be
|
|
514
|
+
Specifying compression way for data of each record. Current accepted options are `zlib` and `gzip`. Otherwise, no compression will be performed.
|
|
510
515
|
|
|
511
516
|
#### log_truncate_max_size
|
|
512
517
|
|
|
@@ -535,13 +540,17 @@ HTTP proxy for API calling. Default `nil`.
|
|
|
535
540
|
|
|
536
541
|
API endpoint URL, for testing. Default `nil`.
|
|
537
542
|
|
|
543
|
+
#### use_fips_endpoint
|
|
544
|
+
|
|
545
|
+
Boolean. Default `nil`. If `true`, the plugin uses FIPS-compliant endpoints for Kinesis Data Streams and Amazon Data Firehose API calls. This is required for environments that must comply with Federal Information Processing Standards (FIPS). When `nil`, the SDK's default behavior applies (which respects the `AWS_USE_FIPS_ENDPOINT` environment variable).
|
|
546
|
+
|
|
538
547
|
#### ssl_verify_peer
|
|
539
548
|
|
|
540
|
-
Boolean.
|
|
549
|
+
Boolean. Set to `false` to disable SSL certificate verification, for testing. Default `true`.
|
|
541
550
|
|
|
542
551
|
#### debug
|
|
543
552
|
|
|
544
|
-
Boolean. Enable if you need to debug
|
|
553
|
+
Boolean. Enable if you need to debug API calls. Default is `false`.
|
|
545
554
|
|
|
546
555
|
## Development
|
|
547
556
|
|
|
@@ -554,7 +563,16 @@ To launch `fluentd` process with this plugin for development, follow the steps b
|
|
|
554
563
|
|
|
555
564
|
To launch using specified version of Fluentd, use `BUNDLE_GEMFILE` environment variable:
|
|
556
565
|
|
|
557
|
-
BUNDLE_GEMFILE=$PWD/gemfiles/Gemfile.
|
|
566
|
+
BUNDLE_GEMFILE=$PWD/gemfiles/Gemfile.fluent-package-5.0.9 bundle exec fluentd -c /path/to/fluent.conf
|
|
567
|
+
|
|
568
|
+
To run tests, use `bundle exec rake test`. Integration tests in `test/integration` run only with `AWS_EMULATOR=true` against [AWS Emulator](https://aws.amazon.com/products/developer-tools/emulator/). Start the emulator and point the AWS SDK to it (see the `aws-emulator` job in [.github/workflows/test.yml](.github/workflows/test.yml)):
|
|
569
|
+
|
|
570
|
+
export AWS_ENDPOINT_URL=https://127.0.0.1:9297
|
|
571
|
+
export AWS_SHARED_CREDENTIALS_FILE=$HOME/.aws/emulator/9297/emulator_credentials
|
|
572
|
+
export AWS_CONFIG_FILE=$HOME/.aws/emulator/9297/config
|
|
573
|
+
export AWS_CA_BUNDLE=$HOME/.aws/emulator/9297/ca.pem
|
|
574
|
+
export AWS_REGION=us-east-1
|
|
575
|
+
AWS_EMULATOR=true bundle exec rake test
|
|
558
576
|
|
|
559
577
|
## Contributing
|
|
560
578
|
|
|
@@ -562,8 +580,8 @@ Bug reports and pull requests are welcome on [GitHub][github].
|
|
|
562
580
|
|
|
563
581
|
## Related Resources
|
|
564
582
|
|
|
565
|
-
* [Amazon Kinesis Data Streams Developer Guide](
|
|
566
|
-
* [Amazon Data Firehose Developer Guide](
|
|
583
|
+
* [Amazon Kinesis Data Streams Developer Guide](https://docs.aws.amazon.com/kinesis/latest/dev/introduction.html)
|
|
584
|
+
* [Amazon Data Firehose Developer Guide](https://docs.aws.amazon.com/firehose/latest/dev/what-is-this-service.html)
|
|
567
585
|
|
|
568
586
|
[github]: https://github.com/awslabs/aws-fluent-plugin-kinesis
|
|
569
587
|
[fluentd-doc-kinesis]: https://docs.fluentd.org/how-to-guides/kinesis-stream
|
|
@@ -59,6 +59,8 @@ Gem::Specification.new do |spec|
|
|
|
59
59
|
|
|
60
60
|
spec.add_development_dependency "bundler", ">= 1.10"
|
|
61
61
|
spec.add_development_dependency "rake", ">= 10.0"
|
|
62
|
+
# Used by the benchmark tasks; benchmark is a bundled gem as of Ruby 4.0
|
|
63
|
+
spec.add_development_dependency "benchmark"
|
|
62
64
|
spec.add_development_dependency "test-unit", ">= 3.0.8"
|
|
63
65
|
spec.add_development_dependency "test-unit-rr", ">= 1.0.3"
|
|
64
66
|
spec.add_development_dependency "pry", ">= 0.10.1"
|
|
@@ -68,4 +70,7 @@ Gem::Specification.new do |spec|
|
|
|
68
70
|
spec.add_development_dependency "mocha", ">= 1.1.0"
|
|
69
71
|
spec.add_development_dependency "webmock", ">= 1.24.2"
|
|
70
72
|
spec.add_development_dependency "fakefs", ">= 0.8.1"
|
|
73
|
+
# Used by integration tests with AWS Emulator to set up and verify Firehose delivery to S3
|
|
74
|
+
spec.add_development_dependency "aws-sdk-iam", "~> 1"
|
|
75
|
+
spec.add_development_dependency "aws-sdk-s3", "~> 1"
|
|
71
76
|
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
source 'https://rubygems.org'
|
|
2
|
+
|
|
3
|
+
# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
|
|
4
|
+
gemspec path: ".."
|
|
5
|
+
|
|
6
|
+
# Specify related gems for fluent-package v5.0.8
|
|
7
|
+
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v5.md
|
|
8
|
+
# https://github.com/fluent/fluent-package-builder/blob/v5.0.8/fluent-package/config.rb#L10
|
|
9
|
+
gem "fluentd", "1.16.10"
|
|
10
|
+
# https://github.com/fluent/fluent-package-builder/blob/v5.0.8/fluent-package/Gemfile#L57-L67
|
|
11
|
+
gem "prometheus-client", "2.1.0"
|
|
12
|
+
gem "fluent-plugin-prometheus", "2.1.0"
|
|
13
|
+
gem "fluent-plugin-prometheus_pushgateway", "0.1.1"
|
|
14
|
+
gem "jmespath", "1.6.2"
|
|
15
|
+
gem "aws-partitions", "1.785.0"
|
|
16
|
+
gem "aws-sdk-core", "3.178.0"
|
|
17
|
+
gem "aws-sdk-kms", "1.71.0"
|
|
18
|
+
gem "aws-sdk-sqs", "1.61.0"
|
|
19
|
+
gem "aws-sigv4", "1.6.0"
|
|
20
|
+
gem "aws-sdk-s3", "1.129.0"
|
|
21
|
+
gem "fluent-plugin-s3", "1.7.2"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
source 'https://rubygems.org'
|
|
2
|
+
|
|
3
|
+
# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
|
|
4
|
+
gemspec path: ".."
|
|
5
|
+
|
|
6
|
+
# Specify related gems for fluent-package v5.0.9
|
|
7
|
+
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG-v5.md
|
|
8
|
+
# https://github.com/fluent/fluent-package-builder/blob/v5.0.9/fluent-package/config.rb#L10
|
|
9
|
+
gem "fluentd", "1.16.11"
|
|
10
|
+
# https://github.com/fluent/fluent-package-builder/blob/v5.0.9/fluent-package/Gemfile#L57-L67
|
|
11
|
+
gem "prometheus-client", "2.1.0"
|
|
12
|
+
gem "fluent-plugin-prometheus", "2.1.0"
|
|
13
|
+
gem "fluent-plugin-prometheus_pushgateway", "0.1.1"
|
|
14
|
+
gem "jmespath", "1.6.2"
|
|
15
|
+
gem "aws-partitions", "1.785.0"
|
|
16
|
+
gem "aws-sdk-core", "3.178.0"
|
|
17
|
+
gem "aws-sdk-kms", "1.71.0"
|
|
18
|
+
gem "aws-sdk-sqs", "1.61.0"
|
|
19
|
+
gem "aws-sigv4", "1.6.0"
|
|
20
|
+
gem "aws-sdk-s3", "1.129.0"
|
|
21
|
+
gem "fluent-plugin-s3", "1.7.2"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
source 'https://rubygems.org'
|
|
2
|
+
|
|
3
|
+
# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
|
|
4
|
+
gemspec path: ".."
|
|
5
|
+
|
|
6
|
+
# Specify related gems for fluent-package v6.0.0
|
|
7
|
+
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
|
|
8
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.0/fluent-package/config.rb#L10
|
|
9
|
+
gem "fluentd", "1.19.0"
|
|
10
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.0/fluent-package/Gemfile#L55-L65
|
|
11
|
+
gem "prometheus-client", "4.2.5"
|
|
12
|
+
gem "fluent-plugin-prometheus", "2.2.1"
|
|
13
|
+
gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
|
|
14
|
+
gem "jmespath", "1.6.2"
|
|
15
|
+
gem "aws-partitions", "1.1150.0"
|
|
16
|
+
gem "aws-sdk-core", "3.230.0"
|
|
17
|
+
gem "aws-sdk-kms", "1.110.0"
|
|
18
|
+
gem "aws-sdk-sqs", "1.101.0"
|
|
19
|
+
gem "aws-sigv4", "1.12.1"
|
|
20
|
+
gem "aws-sdk-s3", "1.197.0"
|
|
21
|
+
gem "fluent-plugin-s3", "1.8.3"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
source 'https://rubygems.org'
|
|
2
|
+
|
|
3
|
+
# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
|
|
4
|
+
gemspec path: ".."
|
|
5
|
+
|
|
6
|
+
# Specify related gems for fluent-package v6.0.1
|
|
7
|
+
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
|
|
8
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.1/fluent-package/config.rb#L10
|
|
9
|
+
gem "fluentd", "1.19.1"
|
|
10
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.1/fluent-package/Gemfile#L55-L65
|
|
11
|
+
gem "prometheus-client", "4.2.5"
|
|
12
|
+
gem "fluent-plugin-prometheus", "2.2.1"
|
|
13
|
+
gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
|
|
14
|
+
gem "jmespath", "1.6.2"
|
|
15
|
+
gem "aws-partitions", "1.1150.0"
|
|
16
|
+
gem "aws-sdk-core", "3.230.0"
|
|
17
|
+
gem "aws-sdk-kms", "1.110.0"
|
|
18
|
+
gem "aws-sdk-sqs", "1.101.0"
|
|
19
|
+
gem "aws-sigv4", "1.12.1"
|
|
20
|
+
gem "aws-sdk-s3", "1.197.0"
|
|
21
|
+
gem "fluent-plugin-s3", "1.8.3"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
source 'https://rubygems.org'
|
|
2
|
+
|
|
3
|
+
# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
|
|
4
|
+
gemspec path: ".."
|
|
5
|
+
|
|
6
|
+
# Specify related gems for fluent-package v6.0.2
|
|
7
|
+
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
|
|
8
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.2/fluent-package/config.rb#L10
|
|
9
|
+
gem "fluentd", "1.19.2"
|
|
10
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.2/fluent-package/Gemfile#L55-L65
|
|
11
|
+
gem "prometheus-client", "4.2.5"
|
|
12
|
+
gem "fluent-plugin-prometheus", "2.2.1"
|
|
13
|
+
gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
|
|
14
|
+
gem "jmespath", "1.6.2"
|
|
15
|
+
gem "aws-partitions", "1.1150.0"
|
|
16
|
+
gem "aws-sdk-core", "3.234.0"
|
|
17
|
+
gem "aws-sdk-kms", "1.110.0"
|
|
18
|
+
gem "aws-sdk-sqs", "1.101.0"
|
|
19
|
+
gem "aws-sigv4", "1.12.1"
|
|
20
|
+
gem "aws-sdk-s3", "1.208.0"
|
|
21
|
+
gem "fluent-plugin-s3", "1.8.3"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
source 'https://rubygems.org'
|
|
2
|
+
|
|
3
|
+
# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
|
|
4
|
+
gemspec path: ".."
|
|
5
|
+
|
|
6
|
+
# Specify related gems for fluent-package v6.0.3
|
|
7
|
+
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
|
|
8
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.3/fluent-package/config.rb#L10
|
|
9
|
+
gem "fluentd", "1.19.2"
|
|
10
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.3/fluent-package/Gemfile#L62-L72
|
|
11
|
+
gem "prometheus-client", "4.2.5"
|
|
12
|
+
gem "fluent-plugin-prometheus", "2.2.1"
|
|
13
|
+
gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
|
|
14
|
+
gem "jmespath", "1.6.2"
|
|
15
|
+
gem "aws-partitions", "1.1150.0"
|
|
16
|
+
gem "aws-sdk-core", "3.234.0"
|
|
17
|
+
gem "aws-sdk-kms", "1.110.0"
|
|
18
|
+
gem "aws-sdk-sqs", "1.101.0"
|
|
19
|
+
gem "aws-sigv4", "1.12.1"
|
|
20
|
+
gem "aws-sdk-s3", "1.208.0"
|
|
21
|
+
gem "fluent-plugin-s3", "1.8.4"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
source 'https://rubygems.org'
|
|
2
|
+
|
|
3
|
+
# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
|
|
4
|
+
gemspec path: ".."
|
|
5
|
+
|
|
6
|
+
# Specify related gems for fluent-package v6.0.4
|
|
7
|
+
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
|
|
8
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.4/fluent-package/config.rb#L10
|
|
9
|
+
gem "fluentd", "1.19.3"
|
|
10
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.4/fluent-package/Gemfile#L70-L80
|
|
11
|
+
gem "prometheus-client", "4.2.5"
|
|
12
|
+
gem "fluent-plugin-prometheus", "2.2.2"
|
|
13
|
+
gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
|
|
14
|
+
gem "jmespath", "1.6.2"
|
|
15
|
+
gem "aws-partitions", "1.1150.0"
|
|
16
|
+
gem "aws-sdk-core", "3.234.0"
|
|
17
|
+
gem "aws-sdk-kms", "1.110.0"
|
|
18
|
+
gem "aws-sdk-sqs", "1.101.0"
|
|
19
|
+
gem "aws-sigv4", "1.12.1"
|
|
20
|
+
gem "aws-sdk-s3", "1.208.0"
|
|
21
|
+
gem "fluent-plugin-s3", "1.8.5"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
source 'https://rubygems.org'
|
|
2
|
+
|
|
3
|
+
# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
|
|
4
|
+
gemspec path: ".."
|
|
5
|
+
|
|
6
|
+
# Specify related gems for fluent-package v6.0.5
|
|
7
|
+
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
|
|
8
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.5/fluent-package/config.rb#L10
|
|
9
|
+
gem "fluentd", "1.19.4"
|
|
10
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.0.5/fluent-package/Gemfile#L70-L80
|
|
11
|
+
gem "prometheus-client", "4.2.5"
|
|
12
|
+
gem "fluent-plugin-prometheus", "2.3.0"
|
|
13
|
+
gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
|
|
14
|
+
gem "jmespath", "1.6.2"
|
|
15
|
+
gem "aws-partitions", "1.1150.0"
|
|
16
|
+
gem "aws-sdk-core", "3.234.0"
|
|
17
|
+
gem "aws-sdk-kms", "1.110.0"
|
|
18
|
+
gem "aws-sdk-sqs", "1.101.0"
|
|
19
|
+
gem "aws-sigv4", "1.12.1"
|
|
20
|
+
gem "aws-sdk-s3", "1.208.0"
|
|
21
|
+
gem "fluent-plugin-s3", "1.8.6"
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
source 'https://rubygems.org'
|
|
2
|
+
|
|
3
|
+
# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
|
|
4
|
+
gemspec path: ".."
|
|
5
|
+
|
|
6
|
+
# Specify related gems for fluent-package v6.1.0
|
|
7
|
+
# https://github.com/fluent/fluent-package-builder/blob/master/CHANGELOG.md
|
|
8
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.1.0/fluent-package/config.rb#L10
|
|
9
|
+
gem "fluentd", "1.20.1"
|
|
10
|
+
# https://github.com/fluent/fluent-package-builder/blob/v6.1.0/fluent-package/Gemfile#L69-L79
|
|
11
|
+
gem "prometheus-client", "5.0.0"
|
|
12
|
+
gem "fluent-plugin-prometheus", "2.3.0"
|
|
13
|
+
gem "fluent-plugin-prometheus_pushgateway", "0.2.1"
|
|
14
|
+
gem "jmespath", "1.6.2"
|
|
15
|
+
gem "aws-partitions", "1.1298.0"
|
|
16
|
+
gem "aws-sdk-core", "3.260.0"
|
|
17
|
+
gem "aws-sdk-kms", "1.135.0"
|
|
18
|
+
gem "aws-sdk-sqs", "1.122.0"
|
|
19
|
+
gem "aws-sigv4", "1.12.1"
|
|
20
|
+
gem "aws-sdk-s3", "1.235.0"
|
|
21
|
+
gem "fluent-plugin-s3", "1.8.6"
|
|
@@ -30,15 +30,26 @@ module Fluent
|
|
|
30
30
|
def initialize(message, record)
|
|
31
31
|
super message
|
|
32
32
|
@record_message = if record.is_a? Array
|
|
33
|
-
record.reverse.map(
|
|
33
|
+
record.reverse.map{|r| format_record(r) }.join(', ')
|
|
34
34
|
else
|
|
35
|
-
record
|
|
35
|
+
format_record(record)
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
def to_s
|
|
40
40
|
super + ": " + @record_message
|
|
41
41
|
end
|
|
42
|
+
|
|
43
|
+
private
|
|
44
|
+
|
|
45
|
+
def format_record(record)
|
|
46
|
+
str = record.to_s
|
|
47
|
+
if str.encoding == Encoding::ASCII_8BIT || !str.valid_encoding?
|
|
48
|
+
str.unpack1('H*')
|
|
49
|
+
else
|
|
50
|
+
str
|
|
51
|
+
end
|
|
52
|
+
end
|
|
42
53
|
end
|
|
43
54
|
class KeyNotFoundError < SkipRecordError
|
|
44
55
|
def initialize(key, record)
|
|
@@ -14,7 +14,6 @@
|
|
|
14
14
|
|
|
15
15
|
require 'fluent_plugin_kinesis/version'
|
|
16
16
|
require 'fluent/configurable'
|
|
17
|
-
require 'benchmark'
|
|
18
17
|
|
|
19
18
|
module Fluent
|
|
20
19
|
module Plugin
|
|
@@ -114,7 +113,9 @@ module Fluent
|
|
|
114
113
|
# TODO: find out who is causing the sleep to be too short and try to make them stop it instead
|
|
115
114
|
def reliable_sleep(wait_second)
|
|
116
115
|
loop do
|
|
117
|
-
|
|
116
|
+
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
117
|
+
sleep(wait_second)
|
|
118
|
+
actual = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
|
|
118
119
|
break if actual >= wait_second
|
|
119
120
|
log.error("#{Thread.current.object_id} sleep failed expected #{wait_second} but slept #{actual}")
|
|
120
121
|
wait_second -= actual
|
|
@@ -26,6 +26,7 @@ module Fluent
|
|
|
26
26
|
config_param :http_proxy, :string, default: nil, secret: true
|
|
27
27
|
config_param :endpoint, :string, default: nil
|
|
28
28
|
config_param :ssl_verify_peer, :bool, default: true
|
|
29
|
+
config_param :use_fips_endpoint, :bool, default: nil
|
|
29
30
|
|
|
30
31
|
config_param :aws_key_id, :string, default: nil, secret: true
|
|
31
32
|
config_param :aws_sec_key, :string, default: nil, secret: true
|
|
@@ -121,6 +122,10 @@ module Fluent
|
|
|
121
122
|
options.update(http_proxy: @http_proxy) unless @http_proxy.nil?
|
|
122
123
|
options.update(endpoint: @endpoint) unless @endpoint.nil?
|
|
123
124
|
options.update(ssl_verify_peer: @ssl_verify_peer) unless @ssl_verify_peer.nil?
|
|
125
|
+
if !@use_fips_endpoint.nil? && client_class.method_defined?(:config) &&
|
|
126
|
+
Gem::Version.new(Aws::CORE_GEM_VERSION) >= Gem::Version.new('3.122.0')
|
|
127
|
+
options.update(use_fips_endpoint: @use_fips_endpoint)
|
|
128
|
+
end
|
|
124
129
|
if @debug
|
|
125
130
|
options.update(logger: Logger.new(log.out))
|
|
126
131
|
options.update(log_level: :debug)
|
|
@@ -197,10 +202,17 @@ module Fluent
|
|
|
197
202
|
end
|
|
198
203
|
c = @process_credentials
|
|
199
204
|
process = c.process
|
|
200
|
-
|
|
205
|
+
# Suppress insecure string warning from newer aws-sdk-core.
|
|
206
|
+
# Using array form would break older versions that use Open3.popen3.
|
|
207
|
+
options[:credentials] = begin
|
|
208
|
+
verbose, $VERBOSE = $VERBOSE, nil
|
|
209
|
+
Aws::ProcessCredentials.new(process)
|
|
210
|
+
ensure
|
|
211
|
+
$VERBOSE = verbose
|
|
212
|
+
end
|
|
201
213
|
else
|
|
202
214
|
# Use default credentials
|
|
203
|
-
# See
|
|
215
|
+
# See https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html
|
|
204
216
|
end
|
|
205
217
|
options
|
|
206
218
|
end
|
|
@@ -26,6 +26,7 @@ module Fluent
|
|
|
26
26
|
|
|
27
27
|
config_param :delivery_stream_name, :string
|
|
28
28
|
config_param :append_new_line, :bool, default: true
|
|
29
|
+
config_param :aggregated_record_size, :integer, default: 0
|
|
29
30
|
|
|
30
31
|
def configure(conf)
|
|
31
32
|
super
|
|
@@ -35,6 +36,9 @@ module Fluent
|
|
|
35
36
|
org_data_formatter.call(tag, time, record).chomp + "\n"
|
|
36
37
|
}
|
|
37
38
|
end
|
|
39
|
+
if @aggregated_record_size > @max_record_size
|
|
40
|
+
raise Fluent::ConfigError, "aggregated_record_size (#{@aggregated_record_size}) can't be greater than max_record_size (#{@max_record_size})."
|
|
41
|
+
end
|
|
38
42
|
end
|
|
39
43
|
|
|
40
44
|
def format(tag, time, record)
|
|
@@ -45,14 +49,62 @@ module Fluent
|
|
|
45
49
|
|
|
46
50
|
def write(chunk)
|
|
47
51
|
delivery_stream_name = extract_placeholders(@delivery_stream_name, chunk)
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
52
|
+
if @aggregated_record_size > 0
|
|
53
|
+
write_records_batch_aggregated(chunk, delivery_stream_name) do |batch|
|
|
54
|
+
records = batch.map{|(data)|
|
|
55
|
+
{ data: data }
|
|
56
|
+
}
|
|
57
|
+
client.put_record_batch(
|
|
58
|
+
delivery_stream_name: delivery_stream_name,
|
|
59
|
+
records: records,
|
|
60
|
+
)
|
|
61
|
+
end
|
|
62
|
+
else
|
|
63
|
+
write_records_batch(chunk, delivery_stream_name) do |batch|
|
|
64
|
+
records = batch.map{|(data)|
|
|
65
|
+
{ data: data }
|
|
66
|
+
}
|
|
67
|
+
client.put_record_batch(
|
|
68
|
+
delivery_stream_name: delivery_stream_name,
|
|
69
|
+
records: records,
|
|
70
|
+
)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
private
|
|
76
|
+
|
|
77
|
+
def write_records_batch_aggregated(chunk, stream_name, &block)
|
|
78
|
+
unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
|
|
79
|
+
records = chunk.to_enum(:msgpack_each)
|
|
80
|
+
aggregated = aggregate_records(records)
|
|
81
|
+
split_to_batches(aggregated) do |batch, size|
|
|
82
|
+
log.debug(sprintf "%s: Write chunk %s / %3d records / %4d KB", stream_name, unique_id, batch.size, size/1024)
|
|
83
|
+
batch_request_with_retry(batch, &block)
|
|
84
|
+
log.debug(sprintf "%s: Finish writing chunk", stream_name)
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def aggregate_records(records)
|
|
89
|
+
Enumerator.new do |y|
|
|
90
|
+
current = nil
|
|
91
|
+
current_size = 0
|
|
92
|
+
records.each do |record|
|
|
93
|
+
data = record.first
|
|
94
|
+
data_size = data.bytesize
|
|
95
|
+
if current.nil?
|
|
96
|
+
current = data
|
|
97
|
+
current_size = data_size
|
|
98
|
+
elsif current_size + data_size <= @aggregated_record_size
|
|
99
|
+
current = current + data
|
|
100
|
+
current_size += data_size
|
|
101
|
+
else
|
|
102
|
+
y.yield [current]
|
|
103
|
+
current = data
|
|
104
|
+
current_size = data_size
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
y.yield [current] unless current.nil?
|
|
56
108
|
end
|
|
57
109
|
end
|
|
58
110
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-kinesis
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 3.
|
|
4
|
+
version: 3.7.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Amazon Web Services
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: bin
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
12
11
|
dependencies:
|
|
13
12
|
- !ruby/object:Gem::Dependency
|
|
14
13
|
name: fluentd
|
|
@@ -136,6 +135,20 @@ dependencies:
|
|
|
136
135
|
- - ">="
|
|
137
136
|
- !ruby/object:Gem::Version
|
|
138
137
|
version: '10.0'
|
|
138
|
+
- !ruby/object:Gem::Dependency
|
|
139
|
+
name: benchmark
|
|
140
|
+
requirement: !ruby/object:Gem::Requirement
|
|
141
|
+
requirements:
|
|
142
|
+
- - ">="
|
|
143
|
+
- !ruby/object:Gem::Version
|
|
144
|
+
version: '0'
|
|
145
|
+
type: :development
|
|
146
|
+
prerelease: false
|
|
147
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
148
|
+
requirements:
|
|
149
|
+
- - ">="
|
|
150
|
+
- !ruby/object:Gem::Version
|
|
151
|
+
version: '0'
|
|
139
152
|
- !ruby/object:Gem::Dependency
|
|
140
153
|
name: test-unit
|
|
141
154
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -262,8 +275,34 @@ dependencies:
|
|
|
262
275
|
- - ">="
|
|
263
276
|
- !ruby/object:Gem::Version
|
|
264
277
|
version: 0.8.1
|
|
265
|
-
|
|
266
|
-
|
|
278
|
+
- !ruby/object:Gem::Dependency
|
|
279
|
+
name: aws-sdk-iam
|
|
280
|
+
requirement: !ruby/object:Gem::Requirement
|
|
281
|
+
requirements:
|
|
282
|
+
- - "~>"
|
|
283
|
+
- !ruby/object:Gem::Version
|
|
284
|
+
version: '1'
|
|
285
|
+
type: :development
|
|
286
|
+
prerelease: false
|
|
287
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
288
|
+
requirements:
|
|
289
|
+
- - "~>"
|
|
290
|
+
- !ruby/object:Gem::Version
|
|
291
|
+
version: '1'
|
|
292
|
+
- !ruby/object:Gem::Dependency
|
|
293
|
+
name: aws-sdk-s3
|
|
294
|
+
requirement: !ruby/object:Gem::Requirement
|
|
295
|
+
requirements:
|
|
296
|
+
- - "~>"
|
|
297
|
+
- !ruby/object:Gem::Version
|
|
298
|
+
version: '1'
|
|
299
|
+
type: :development
|
|
300
|
+
prerelease: false
|
|
301
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
302
|
+
requirements:
|
|
303
|
+
- - "~>"
|
|
304
|
+
- !ruby/object:Gem::Version
|
|
305
|
+
version: '1'
|
|
267
306
|
executables: []
|
|
268
307
|
extensions: []
|
|
269
308
|
extra_rdoc_files: []
|
|
@@ -287,9 +326,19 @@ files:
|
|
|
287
326
|
- gemfiles/Gemfile.fluent-package-5.0.1
|
|
288
327
|
- gemfiles/Gemfile.fluent-package-5.0.2
|
|
289
328
|
- gemfiles/Gemfile.fluent-package-5.0.7
|
|
329
|
+
- gemfiles/Gemfile.fluent-package-5.0.8
|
|
330
|
+
- gemfiles/Gemfile.fluent-package-5.0.9
|
|
290
331
|
- gemfiles/Gemfile.fluent-package-5.2.0
|
|
332
|
+
- gemfiles/Gemfile.fluent-package-6.0.0
|
|
333
|
+
- gemfiles/Gemfile.fluent-package-6.0.1
|
|
334
|
+
- gemfiles/Gemfile.fluent-package-6.0.2
|
|
335
|
+
- gemfiles/Gemfile.fluent-package-6.0.3
|
|
336
|
+
- gemfiles/Gemfile.fluent-package-6.0.4
|
|
337
|
+
- gemfiles/Gemfile.fluent-package-6.0.5
|
|
338
|
+
- gemfiles/Gemfile.fluent-package-6.1.0
|
|
291
339
|
- gemfiles/Gemfile.fluentd-0.14.22
|
|
292
340
|
- gemfiles/Gemfile.fluentd-1.19.0
|
|
341
|
+
- gemfiles/Gemfile.fluentd-1.20.1
|
|
293
342
|
- gemfiles/Gemfile.td-agent-3.1.0
|
|
294
343
|
- gemfiles/Gemfile.td-agent-3.1.1
|
|
295
344
|
- gemfiles/Gemfile.td-agent-3.2.0
|
|
@@ -334,7 +383,6 @@ homepage: https://github.com/awslabs/aws-fluent-plugin-kinesis
|
|
|
334
383
|
licenses:
|
|
335
384
|
- Apache-2.0
|
|
336
385
|
metadata: {}
|
|
337
|
-
post_install_message:
|
|
338
386
|
rdoc_options: []
|
|
339
387
|
require_paths:
|
|
340
388
|
- lib
|
|
@@ -349,8 +397,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
349
397
|
- !ruby/object:Gem::Version
|
|
350
398
|
version: '0'
|
|
351
399
|
requirements: []
|
|
352
|
-
rubygems_version:
|
|
353
|
-
signing_key:
|
|
400
|
+
rubygems_version: 4.0.3
|
|
354
401
|
specification_version: 4
|
|
355
402
|
summary: Fluentd output plugin that sends events to Amazon Kinesis.
|
|
356
403
|
test_files: []
|