adp-fluent-plugin-kinesis 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +6 -0
- data/.gitignore +15 -0
- data/.travis.yml +56 -0
- data/CHANGELOG.md +172 -0
- data/CODE_OF_CONDUCT.md +4 -0
- data/CONTRIBUTING.md +61 -0
- data/CONTRIBUTORS.txt +8 -0
- data/Gemfile +18 -0
- data/LICENSE.txt +201 -0
- data/Makefile +44 -0
- data/NOTICE.txt +2 -0
- data/README.md +559 -0
- data/Rakefile +26 -0
- data/adp-fluent-plugin-kinesis.gemspec +71 -0
- data/benchmark/task.rake +106 -0
- data/gemfiles/Gemfile.fluentd-0.14.22 +6 -0
- data/gemfiles/Gemfile.fluentd-1.13.3 +6 -0
- data/gemfiles/Gemfile.td-agent-3.1.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.1.1 +17 -0
- data/gemfiles/Gemfile.td-agent-3.2.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.2.1 +17 -0
- data/gemfiles/Gemfile.td-agent-3.3.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.4.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.4.1 +17 -0
- data/gemfiles/Gemfile.td-agent-3.5.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.5.1 +17 -0
- data/gemfiles/Gemfile.td-agent-3.6.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.7.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.7.1 +17 -0
- data/gemfiles/Gemfile.td-agent-3.8.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.8.1 +18 -0
- data/gemfiles/Gemfile.td-agent-4.0.0 +25 -0
- data/gemfiles/Gemfile.td-agent-4.0.1 +21 -0
- data/gemfiles/Gemfile.td-agent-4.1.0 +21 -0
- data/gemfiles/Gemfile.td-agent-4.1.1 +21 -0
- data/gemfiles/Gemfile.td-agent-4.2.0 +21 -0
- data/lib/fluent/plugin/kinesis.rb +174 -0
- data/lib/fluent/plugin/kinesis_helper/aggregator.rb +101 -0
- data/lib/fluent/plugin/kinesis_helper/api.rb +254 -0
- data/lib/fluent/plugin/kinesis_helper/client.rb +210 -0
- data/lib/fluent/plugin/kinesis_helper/compression.rb +27 -0
- data/lib/fluent/plugin/out_kinesis_firehose.rb +60 -0
- data/lib/fluent/plugin/out_kinesis_streams.rb +72 -0
- data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +79 -0
- data/lib/fluent_plugin_kinesis/version.rb +17 -0
- metadata +339 -0
@@ -0,0 +1,210 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
5
|
+
# may not use this file except in compliance with the License. A copy of
|
6
|
+
# the License is located at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is
|
11
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
12
|
+
# ANY KIND, either express or implied. See the License for the specific
|
13
|
+
# language governing permissions and limitations under the License.
|
14
|
+
|
15
|
+
require 'fluent/configurable'
|
16
|
+
require 'aws-sdk-core'
|
17
|
+
|
18
|
+
module Fluent
|
19
|
+
module Plugin
|
20
|
+
module KinesisHelper
|
21
|
+
module Client
|
22
|
+
module ClientParams
|
23
|
+
include Fluent::Configurable
|
24
|
+
config_param :region, :string, default: nil
|
25
|
+
|
26
|
+
config_param :http_proxy, :string, default: nil, secret: true
|
27
|
+
config_param :endpoint, :string, default: nil
|
28
|
+
config_param :ssl_verify_peer, :bool, default: true
|
29
|
+
|
30
|
+
config_param :aws_key_id, :string, default: nil, secret: true
|
31
|
+
config_param :aws_sec_key, :string, default: nil, secret: true
|
32
|
+
config_param :aws_ses_token, :string, default: nil, secret: true
|
33
|
+
config_section :assume_role_credentials, multi: false do
|
34
|
+
desc "The Amazon Resource Name (ARN) of the role to assume"
|
35
|
+
config_param :role_arn, :string, secret: true
|
36
|
+
desc "An identifier for the assumed role session"
|
37
|
+
config_param :role_session_name, :string
|
38
|
+
desc "An IAM policy in JSON format"
|
39
|
+
config_param :policy, :string, default: nil
|
40
|
+
desc "The duration, in seconds, of the role session (900-3600)"
|
41
|
+
config_param :duration_seconds, :integer, default: nil
|
42
|
+
desc "A unique identifier that is used by third parties when assuming roles in their customers' accounts."
|
43
|
+
config_param :external_id, :string, default: nil, secret: true
|
44
|
+
desc "A http proxy url for requests to aws sts service"
|
45
|
+
config_param :sts_http_proxy, :string, default: nil, secret: true
|
46
|
+
desc "A URL for a regional STS API endpoint, the default is global"
|
47
|
+
config_param :sts_endpoint_url, :string, default: nil
|
48
|
+
end
|
49
|
+
# Refer to the following link for additional parameters that could be added:
|
50
|
+
# https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
|
51
|
+
config_section :web_identity_credentials, multi: false do
|
52
|
+
desc "The Amazon Resource Name (ARN) of the role to assume"
|
53
|
+
config_param :role_arn, :string
|
54
|
+
desc "An identifier for the assumed role session"
|
55
|
+
config_param :role_session_name, :string
|
56
|
+
desc "The absolute path to the file on disk containing the OIDC token"
|
57
|
+
config_param :web_identity_token_file, :string, default: nil #required
|
58
|
+
desc "An IAM policy in JSON format"
|
59
|
+
config_param :policy, :string, default: nil
|
60
|
+
desc "The duration, in seconds, of the role session (900-43200)"
|
61
|
+
config_param :duration_seconds, :time, default: nil
|
62
|
+
end
|
63
|
+
config_section :instance_profile_credentials, multi: false do
|
64
|
+
desc "Number of times to retry when retrieving credentials"
|
65
|
+
config_param :retries, :integer, default: nil
|
66
|
+
desc "IP address (default:169.254.169.254)"
|
67
|
+
config_param :ip_address, :string, default: nil
|
68
|
+
desc "Port number (default:80)"
|
69
|
+
config_param :port, :integer, default: nil
|
70
|
+
desc "Number of seconds to wait for the connection to open"
|
71
|
+
config_param :http_open_timeout, :float, default: nil
|
72
|
+
desc "Number of seconds to wait for one block to be read"
|
73
|
+
config_param :http_read_timeout, :float, default: nil
|
74
|
+
# config_param :delay, :integer or :proc, :default => nil
|
75
|
+
# config_param :http_degub_output, :io, :default => nil
|
76
|
+
end
|
77
|
+
config_section :shared_credentials, multi: false do
|
78
|
+
desc "Path to the shared file. (default: $HOME/.aws/credentials)"
|
79
|
+
config_param :path, :string, default: nil
|
80
|
+
desc "Profile name. Default to 'default' or ENV['AWS_PROFILE']"
|
81
|
+
config_param :profile_name, :string, default: nil
|
82
|
+
end
|
83
|
+
config_section :process_credentials, multi: false do
|
84
|
+
desc "External process to execute."
|
85
|
+
config_param :process, :string
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.included(mod)
|
90
|
+
mod.include ClientParams
|
91
|
+
end
|
92
|
+
|
93
|
+
def configure(conf)
|
94
|
+
super
|
95
|
+
@region = client.config.region if @region.nil?
|
96
|
+
end
|
97
|
+
|
98
|
+
def client
|
99
|
+
@client ||= client_class.new(client_options)
|
100
|
+
end
|
101
|
+
|
102
|
+
private
|
103
|
+
|
104
|
+
def client_class
|
105
|
+
case request_type
|
106
|
+
when :streams, :streams_aggregated
|
107
|
+
require 'aws-sdk-kinesis'
|
108
|
+
Aws::Kinesis::Client
|
109
|
+
when :firehose
|
110
|
+
require 'aws-sdk-firehose'
|
111
|
+
Aws::Firehose::Client
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def client_options
|
116
|
+
options = setup_credentials
|
117
|
+
options.update(
|
118
|
+
user_agent_suffix: "fluent-plugin-kinesis/#{request_type}/#{FluentPluginKinesis::VERSION}"
|
119
|
+
)
|
120
|
+
options.update(region: @region) unless @region.nil?
|
121
|
+
options.update(http_proxy: @http_proxy) unless @http_proxy.nil?
|
122
|
+
options.update(endpoint: @endpoint) unless @endpoint.nil?
|
123
|
+
options.update(ssl_verify_peer: @ssl_verify_peer) unless @ssl_verify_peer.nil?
|
124
|
+
if @debug
|
125
|
+
options.update(logger: Logger.new(log.out))
|
126
|
+
options.update(log_level: :debug)
|
127
|
+
end
|
128
|
+
options
|
129
|
+
end
|
130
|
+
|
131
|
+
def setup_credentials
|
132
|
+
options = {}
|
133
|
+
credentials_options = {}
|
134
|
+
case
|
135
|
+
when @aws_key_id && @aws_sec_key && @aws_ses_token
|
136
|
+
options[:access_key_id] = @aws_key_id
|
137
|
+
options[:secret_access_key] = @aws_sec_key
|
138
|
+
options[:session_token] = @aws_ses_token
|
139
|
+
when @aws_key_id && @aws_sec_key
|
140
|
+
options[:access_key_id] = @aws_key_id
|
141
|
+
options[:secret_access_key] = @aws_sec_key
|
142
|
+
when @assume_role_credentials
|
143
|
+
c = @assume_role_credentials
|
144
|
+
credentials_options[:role_arn] = c.role_arn
|
145
|
+
credentials_options[:role_session_name] = c.role_session_name
|
146
|
+
credentials_options[:policy] = c.policy if c.policy
|
147
|
+
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
148
|
+
credentials_options[:external_id] = c.external_id if c.external_id
|
149
|
+
credentials_options[:sts_endpoint_url] = c.sts_endpoint_url if c.sts_endpoint_url
|
150
|
+
if @region and c.sts_http_proxy and c.sts_endpoint_url
|
151
|
+
credentials_options[:client] = Aws::STS::Client.new(region: @region, http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
|
152
|
+
elsif c.sts_http_proxy and c.sts_endpoint_url
|
153
|
+
credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
|
154
|
+
elsif @region and c.sts_http_proxy
|
155
|
+
credentials_options[:client] = Aws::STS::Client.new(region: @region, http_proxy: c.sts_http_proxy)
|
156
|
+
elsif @region and c.sts_endpoint_url
|
157
|
+
credentials_options[:client] = Aws::STS::Client.new(region: @region, endpoint: c.sts_endpoint_url)
|
158
|
+
elsif c.sts_http_proxy
|
159
|
+
credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy)
|
160
|
+
elsif c.sts_endpoint_url
|
161
|
+
credentials_options[:client] = Aws::STS::Client.new(endpoint: c.sts_endpoint_url)
|
162
|
+
elsif @region
|
163
|
+
credentials_options[:client] = Aws::STS::Client.new(region: @region)
|
164
|
+
end
|
165
|
+
options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
|
166
|
+
when @web_identity_credentials
|
167
|
+
c = @web_identity_credentials
|
168
|
+
credentials_options[:role_arn] = c.role_arn
|
169
|
+
credentials_options[:role_session_name] = c.role_session_name
|
170
|
+
credentials_options[:web_identity_token_file] = c.web_identity_token_file
|
171
|
+
credentials_options[:policy] = c.policy if c.policy
|
172
|
+
credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
|
173
|
+
if @region
|
174
|
+
credentials_options[:client] = Aws::STS::Client.new(:region => @region)
|
175
|
+
end
|
176
|
+
options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
|
177
|
+
when @instance_profile_credentials
|
178
|
+
c = @instance_profile_credentials
|
179
|
+
credentials_options[:retries] = c.retries if c.retries
|
180
|
+
credentials_options[:ip_address] = c.ip_address if c.ip_address
|
181
|
+
credentials_options[:port] = c.port if c.port
|
182
|
+
credentials_options[:http_open_timeout] = c.http_open_timeout if c.http_open_timeout
|
183
|
+
credentials_options[:http_read_timeout] = c.http_read_timeout if c.http_read_timeout
|
184
|
+
if ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
|
185
|
+
options[:credentials] = Aws::ECSCredentials.new(credentials_options)
|
186
|
+
else
|
187
|
+
options[:credentials] = Aws::InstanceProfileCredentials.new(credentials_options)
|
188
|
+
end
|
189
|
+
when @shared_credentials
|
190
|
+
c = @shared_credentials
|
191
|
+
credentials_options[:path] = c.path if c.path
|
192
|
+
credentials_options[:profile_name] = c.profile_name if c.profile_name
|
193
|
+
options[:credentials] = Aws::SharedCredentials.new(credentials_options)
|
194
|
+
when @process_credentials
|
195
|
+
if Gem::Version.new(Aws::CORE_GEM_VERSION) < Gem::Version.new('3.24.0')
|
196
|
+
raise Fluent::ConfigError, "Config process_credentials requires aws-sdk-core >= 3.24.0. Found aws-sdk-core #{Aws::CORE_GEM_VERSION} instead."
|
197
|
+
end
|
198
|
+
c = @process_credentials
|
199
|
+
process = c.process
|
200
|
+
options[:credentials] = Aws::ProcessCredentials.new(process)
|
201
|
+
else
|
202
|
+
# Use default credentials
|
203
|
+
# See http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html
|
204
|
+
end
|
205
|
+
options
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require "stringio"
|
2
|
+
require "zlib"
|
3
|
+
|
4
|
+
class Stream < StringIO
|
5
|
+
def initialize(*)
|
6
|
+
super
|
7
|
+
set_encoding "BINARY"
|
8
|
+
end
|
9
|
+
|
10
|
+
def close
|
11
|
+
rewind;
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
class Gzip
|
16
|
+
def self.compress(string, level = Zlib::DEFAULT_COMPRESSION, strategy = Zlib::DEFAULT_STRATEGY)
|
17
|
+
output = Stream.new
|
18
|
+
gz = Zlib::GzipWriter.new(output, level, strategy)
|
19
|
+
gz.write(string)
|
20
|
+
gz.close
|
21
|
+
output.string
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.decompress(string)
|
25
|
+
Zlib::GzipReader.wrap(StringIO.new(string), &:read)
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
5
|
+
# may not use this file except in compliance with the License. A copy of
|
6
|
+
# the License is located at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is
|
11
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
12
|
+
# ANY KIND, either express or implied. See the License for the specific
|
13
|
+
# language governing permissions and limitations under the License.
|
14
|
+
|
15
|
+
require 'fluent/plugin/kinesis'
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
module Plugin
|
19
|
+
class KinesisFirehoseOutput < KinesisOutput
|
20
|
+
Fluent::Plugin.register_output('kinesis_firehose', self)
|
21
|
+
|
22
|
+
RequestType = :firehose
|
23
|
+
BatchRequestLimitCount = 500
|
24
|
+
BatchRequestLimitSize = 4 * 1024 * 1024
|
25
|
+
include KinesisHelper::API::BatchRequest
|
26
|
+
|
27
|
+
config_param :delivery_stream_name, :string
|
28
|
+
config_param :append_new_line, :bool, default: true
|
29
|
+
|
30
|
+
def configure(conf)
|
31
|
+
super
|
32
|
+
if @append_new_line
|
33
|
+
org_data_formatter = @data_formatter
|
34
|
+
@data_formatter = ->(tag, time, record) {
|
35
|
+
org_data_formatter.call(tag, time, record).chomp + "\n"
|
36
|
+
}
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def format(tag, time, record)
|
41
|
+
format_for_api do
|
42
|
+
[@data_formatter.call(tag, time, record)]
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def write(chunk)
|
47
|
+
delivery_stream_name = extract_placeholders(@delivery_stream_name, chunk)
|
48
|
+
write_records_batch(chunk, delivery_stream_name) do |batch|
|
49
|
+
records = batch.map{|(data)|
|
50
|
+
{ data: data }
|
51
|
+
}
|
52
|
+
client.put_record_batch(
|
53
|
+
delivery_stream_name: delivery_stream_name,
|
54
|
+
records: records,
|
55
|
+
)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
5
|
+
# may not use this file except in compliance with the License. A copy of
|
6
|
+
# the License is located at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is
|
11
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
12
|
+
# ANY KIND, either express or implied. See the License for the specific
|
13
|
+
# language governing permissions and limitations under the License.
|
14
|
+
|
15
|
+
require 'fluent/plugin/kinesis'
|
16
|
+
|
17
|
+
module Fluent
|
18
|
+
module Plugin
|
19
|
+
class KinesisStreamsOutput < KinesisOutput
|
20
|
+
Fluent::Plugin.register_output('kinesis_streams', self)
|
21
|
+
|
22
|
+
RequestType = :streams
|
23
|
+
BatchRequestLimitCount = 500
|
24
|
+
BatchRequestLimitSize = 5 * 1024 * 1024
|
25
|
+
include KinesisHelper::API::BatchRequest
|
26
|
+
|
27
|
+
config_param :stream_name, :string
|
28
|
+
config_param :partition_key, :string, default: nil
|
29
|
+
|
30
|
+
def configure(conf)
|
31
|
+
super
|
32
|
+
@key_formatter = key_formatter_create
|
33
|
+
end
|
34
|
+
|
35
|
+
def format(tag, time, record)
|
36
|
+
format_for_api do
|
37
|
+
data = @data_formatter.call(tag, time, record)
|
38
|
+
key = @key_formatter.call(record)
|
39
|
+
[data, key]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def write(chunk)
|
44
|
+
stream_name = extract_placeholders(@stream_name, chunk)
|
45
|
+
write_records_batch(chunk, stream_name) do |batch|
|
46
|
+
records = batch.map{|(data, partition_key)|
|
47
|
+
{ data: data, partition_key: partition_key }
|
48
|
+
}
|
49
|
+
client.put_records(
|
50
|
+
stream_name: stream_name,
|
51
|
+
records: records,
|
52
|
+
)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def key_formatter_create
|
59
|
+
if @partition_key.nil?
|
60
|
+
->(record) { SecureRandom.hex(16) }
|
61
|
+
else
|
62
|
+
->(record) {
|
63
|
+
if !record.key?(@partition_key)
|
64
|
+
raise KeyNotFoundError.new(@partition_key, record)
|
65
|
+
end
|
66
|
+
record[@partition_key]
|
67
|
+
}
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
5
|
+
# may not use this file except in compliance with the License. A copy of
|
6
|
+
# the License is located at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is
|
11
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
12
|
+
# ANY KIND, either express or implied. See the License for the specific
|
13
|
+
# language governing permissions and limitations under the License.
|
14
|
+
|
15
|
+
require 'fluent/plugin/kinesis'
|
16
|
+
require 'fluent/plugin/kinesis_helper/aggregator'
|
17
|
+
|
18
|
+
module Fluent
|
19
|
+
module Plugin
|
20
|
+
class KinesisStreamsAggregatedOutput < KinesisOutput
|
21
|
+
Fluent::Plugin.register_output('kinesis_streams_aggregated', self)
|
22
|
+
include KinesisHelper::Aggregator::Mixin
|
23
|
+
|
24
|
+
RequestType = :streams_aggregated
|
25
|
+
BatchRequestLimitCount = 100_000
|
26
|
+
BatchRequestLimitSize = 1024 * 1024
|
27
|
+
include KinesisHelper::API::BatchRequest
|
28
|
+
|
29
|
+
config_param :stream_name, :string
|
30
|
+
config_param :fixed_partition_key, :string, default: nil
|
31
|
+
|
32
|
+
def configure(conf)
|
33
|
+
super
|
34
|
+
@partition_key_generator = create_partition_key_generator
|
35
|
+
@batch_request_max_size -= offset
|
36
|
+
@max_record_size -= offset
|
37
|
+
end
|
38
|
+
|
39
|
+
def format(tag, time, record)
|
40
|
+
format_for_api do
|
41
|
+
[@data_formatter.call(tag, time, record)]
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def write(chunk)
|
46
|
+
stream_name = extract_placeholders(@stream_name, chunk)
|
47
|
+
write_records_batch(chunk, stream_name) do |batch|
|
48
|
+
key = @partition_key_generator.call
|
49
|
+
records = batch.map{|(data)|data}
|
50
|
+
client.put_records(
|
51
|
+
stream_name: stream_name,
|
52
|
+
records: [{
|
53
|
+
partition_key: key,
|
54
|
+
data: aggregator.aggregate(records, key),
|
55
|
+
}],
|
56
|
+
)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def offset
|
61
|
+
@offset ||= AggregateOffset + @partition_key_generator.call.size*2
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
def size_of_values(record)
|
67
|
+
super(record) + RecordOffset
|
68
|
+
end
|
69
|
+
|
70
|
+
def create_partition_key_generator
|
71
|
+
if @fixed_partition_key.nil?
|
72
|
+
->() { SecureRandom.hex(16) }
|
73
|
+
else
|
74
|
+
->() { @fixed_partition_key }
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
5
|
+
# may not use this file except in compliance with the License. A copy of
|
6
|
+
# the License is located at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is
|
11
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
12
|
+
# ANY KIND, either express or implied. See the License for the specific
|
13
|
+
# language governing permissions and limitations under the License.
|
14
|
+
|
15
|
+
module FluentPluginKinesis
|
16
|
+
VERSION = '0.0.1'
|
17
|
+
end
|