adp-fluent-plugin-kinesis 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/.github/PULL_REQUEST_TEMPLATE.md +6 -0
  3. data/.gitignore +15 -0
  4. data/.travis.yml +56 -0
  5. data/CHANGELOG.md +172 -0
  6. data/CODE_OF_CONDUCT.md +4 -0
  7. data/CONTRIBUTING.md +61 -0
  8. data/CONTRIBUTORS.txt +8 -0
  9. data/Gemfile +18 -0
  10. data/LICENSE.txt +201 -0
  11. data/Makefile +44 -0
  12. data/NOTICE.txt +2 -0
  13. data/README.md +559 -0
  14. data/Rakefile +26 -0
  15. data/adp-fluent-plugin-kinesis.gemspec +71 -0
  16. data/benchmark/task.rake +106 -0
  17. data/gemfiles/Gemfile.fluentd-0.14.22 +6 -0
  18. data/gemfiles/Gemfile.fluentd-1.13.3 +6 -0
  19. data/gemfiles/Gemfile.td-agent-3.1.0 +17 -0
  20. data/gemfiles/Gemfile.td-agent-3.1.1 +17 -0
  21. data/gemfiles/Gemfile.td-agent-3.2.0 +17 -0
  22. data/gemfiles/Gemfile.td-agent-3.2.1 +17 -0
  23. data/gemfiles/Gemfile.td-agent-3.3.0 +17 -0
  24. data/gemfiles/Gemfile.td-agent-3.4.0 +17 -0
  25. data/gemfiles/Gemfile.td-agent-3.4.1 +17 -0
  26. data/gemfiles/Gemfile.td-agent-3.5.0 +17 -0
  27. data/gemfiles/Gemfile.td-agent-3.5.1 +17 -0
  28. data/gemfiles/Gemfile.td-agent-3.6.0 +17 -0
  29. data/gemfiles/Gemfile.td-agent-3.7.0 +17 -0
  30. data/gemfiles/Gemfile.td-agent-3.7.1 +17 -0
  31. data/gemfiles/Gemfile.td-agent-3.8.0 +17 -0
  32. data/gemfiles/Gemfile.td-agent-3.8.1 +18 -0
  33. data/gemfiles/Gemfile.td-agent-4.0.0 +25 -0
  34. data/gemfiles/Gemfile.td-agent-4.0.1 +21 -0
  35. data/gemfiles/Gemfile.td-agent-4.1.0 +21 -0
  36. data/gemfiles/Gemfile.td-agent-4.1.1 +21 -0
  37. data/gemfiles/Gemfile.td-agent-4.2.0 +21 -0
  38. data/lib/fluent/plugin/kinesis.rb +174 -0
  39. data/lib/fluent/plugin/kinesis_helper/aggregator.rb +101 -0
  40. data/lib/fluent/plugin/kinesis_helper/api.rb +254 -0
  41. data/lib/fluent/plugin/kinesis_helper/client.rb +210 -0
  42. data/lib/fluent/plugin/kinesis_helper/compression.rb +27 -0
  43. data/lib/fluent/plugin/out_kinesis_firehose.rb +60 -0
  44. data/lib/fluent/plugin/out_kinesis_streams.rb +72 -0
  45. data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +79 -0
  46. data/lib/fluent_plugin_kinesis/version.rb +17 -0
  47. metadata +339 -0
@@ -0,0 +1,210 @@
1
+ #
2
+ # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License"). You
5
+ # may not use this file except in compliance with the License. A copy of
6
+ # the License is located at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is
11
+ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
12
+ # ANY KIND, either express or implied. See the License for the specific
13
+ # language governing permissions and limitations under the License.
14
+
15
+ require 'fluent/configurable'
16
+ require 'aws-sdk-core'
17
+
18
+ module Fluent
19
+ module Plugin
20
+ module KinesisHelper
21
+ module Client
22
+ module ClientParams
23
+ include Fluent::Configurable
24
+ config_param :region, :string, default: nil
25
+
26
+ config_param :http_proxy, :string, default: nil, secret: true
27
+ config_param :endpoint, :string, default: nil
28
+ config_param :ssl_verify_peer, :bool, default: true
29
+
30
+ config_param :aws_key_id, :string, default: nil, secret: true
31
+ config_param :aws_sec_key, :string, default: nil, secret: true
32
+ config_param :aws_ses_token, :string, default: nil, secret: true
33
+ config_section :assume_role_credentials, multi: false do
34
+ desc "The Amazon Resource Name (ARN) of the role to assume"
35
+ config_param :role_arn, :string, secret: true
36
+ desc "An identifier for the assumed role session"
37
+ config_param :role_session_name, :string
38
+ desc "An IAM policy in JSON format"
39
+ config_param :policy, :string, default: nil
40
+ desc "The duration, in seconds, of the role session (900-3600)"
41
+ config_param :duration_seconds, :integer, default: nil
42
+ desc "A unique identifier that is used by third parties when assuming roles in their customers' accounts."
43
+ config_param :external_id, :string, default: nil, secret: true
44
+ desc "A http proxy url for requests to aws sts service"
45
+ config_param :sts_http_proxy, :string, default: nil, secret: true
46
+ desc "A URL for a regional STS API endpoint, the default is global"
47
+ config_param :sts_endpoint_url, :string, default: nil
48
+ end
49
+ # Refer to the following link for additional parameters that could be added:
50
+ # https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/STS/Client.html#assume_role_with_web_identity-instance_method
51
+ config_section :web_identity_credentials, multi: false do
52
+ desc "The Amazon Resource Name (ARN) of the role to assume"
53
+ config_param :role_arn, :string
54
+ desc "An identifier for the assumed role session"
55
+ config_param :role_session_name, :string
56
+ desc "The absolute path to the file on disk containing the OIDC token"
57
+ config_param :web_identity_token_file, :string, default: nil #required
58
+ desc "An IAM policy in JSON format"
59
+ config_param :policy, :string, default: nil
60
+ desc "The duration, in seconds, of the role session (900-43200)"
61
+ config_param :duration_seconds, :time, default: nil
62
+ end
63
+ config_section :instance_profile_credentials, multi: false do
64
+ desc "Number of times to retry when retrieving credentials"
65
+ config_param :retries, :integer, default: nil
66
+ desc "IP address (default:169.254.169.254)"
67
+ config_param :ip_address, :string, default: nil
68
+ desc "Port number (default:80)"
69
+ config_param :port, :integer, default: nil
70
+ desc "Number of seconds to wait for the connection to open"
71
+ config_param :http_open_timeout, :float, default: nil
72
+ desc "Number of seconds to wait for one block to be read"
73
+ config_param :http_read_timeout, :float, default: nil
74
+ # config_param :delay, :integer or :proc, :default => nil
75
+ # config_param :http_degub_output, :io, :default => nil
76
+ end
77
+ config_section :shared_credentials, multi: false do
78
+ desc "Path to the shared file. (default: $HOME/.aws/credentials)"
79
+ config_param :path, :string, default: nil
80
+ desc "Profile name. Default to 'default' or ENV['AWS_PROFILE']"
81
+ config_param :profile_name, :string, default: nil
82
+ end
83
+ config_section :process_credentials, multi: false do
84
+ desc "External process to execute."
85
+ config_param :process, :string
86
+ end
87
+ end
88
+
89
+ def self.included(mod)
90
+ mod.include ClientParams
91
+ end
92
+
93
+ def configure(conf)
94
+ super
95
+ @region = client.config.region if @region.nil?
96
+ end
97
+
98
+ def client
99
+ @client ||= client_class.new(client_options)
100
+ end
101
+
102
+ private
103
+
104
+ def client_class
105
+ case request_type
106
+ when :streams, :streams_aggregated
107
+ require 'aws-sdk-kinesis'
108
+ Aws::Kinesis::Client
109
+ when :firehose
110
+ require 'aws-sdk-firehose'
111
+ Aws::Firehose::Client
112
+ end
113
+ end
114
+
115
+ def client_options
116
+ options = setup_credentials
117
+ options.update(
118
+ user_agent_suffix: "fluent-plugin-kinesis/#{request_type}/#{FluentPluginKinesis::VERSION}"
119
+ )
120
+ options.update(region: @region) unless @region.nil?
121
+ options.update(http_proxy: @http_proxy) unless @http_proxy.nil?
122
+ options.update(endpoint: @endpoint) unless @endpoint.nil?
123
+ options.update(ssl_verify_peer: @ssl_verify_peer) unless @ssl_verify_peer.nil?
124
+ if @debug
125
+ options.update(logger: Logger.new(log.out))
126
+ options.update(log_level: :debug)
127
+ end
128
+ options
129
+ end
130
+
131
+ def setup_credentials
132
+ options = {}
133
+ credentials_options = {}
134
+ case
135
+ when @aws_key_id && @aws_sec_key && @aws_ses_token
136
+ options[:access_key_id] = @aws_key_id
137
+ options[:secret_access_key] = @aws_sec_key
138
+ options[:session_token] = @aws_ses_token
139
+ when @aws_key_id && @aws_sec_key
140
+ options[:access_key_id] = @aws_key_id
141
+ options[:secret_access_key] = @aws_sec_key
142
+ when @assume_role_credentials
143
+ c = @assume_role_credentials
144
+ credentials_options[:role_arn] = c.role_arn
145
+ credentials_options[:role_session_name] = c.role_session_name
146
+ credentials_options[:policy] = c.policy if c.policy
147
+ credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
148
+ credentials_options[:external_id] = c.external_id if c.external_id
149
+ credentials_options[:sts_endpoint_url] = c.sts_endpoint_url if c.sts_endpoint_url
150
+ if @region and c.sts_http_proxy and c.sts_endpoint_url
151
+ credentials_options[:client] = Aws::STS::Client.new(region: @region, http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
152
+ elsif c.sts_http_proxy and c.sts_endpoint_url
153
+ credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy, endpoint: c.sts_endpoint_url)
154
+ elsif @region and c.sts_http_proxy
155
+ credentials_options[:client] = Aws::STS::Client.new(region: @region, http_proxy: c.sts_http_proxy)
156
+ elsif @region and c.sts_endpoint_url
157
+ credentials_options[:client] = Aws::STS::Client.new(region: @region, endpoint: c.sts_endpoint_url)
158
+ elsif c.sts_http_proxy
159
+ credentials_options[:client] = Aws::STS::Client.new(http_proxy: c.sts_http_proxy)
160
+ elsif c.sts_endpoint_url
161
+ credentials_options[:client] = Aws::STS::Client.new(endpoint: c.sts_endpoint_url)
162
+ elsif @region
163
+ credentials_options[:client] = Aws::STS::Client.new(region: @region)
164
+ end
165
+ options[:credentials] = Aws::AssumeRoleCredentials.new(credentials_options)
166
+ when @web_identity_credentials
167
+ c = @web_identity_credentials
168
+ credentials_options[:role_arn] = c.role_arn
169
+ credentials_options[:role_session_name] = c.role_session_name
170
+ credentials_options[:web_identity_token_file] = c.web_identity_token_file
171
+ credentials_options[:policy] = c.policy if c.policy
172
+ credentials_options[:duration_seconds] = c.duration_seconds if c.duration_seconds
173
+ if @region
174
+ credentials_options[:client] = Aws::STS::Client.new(:region => @region)
175
+ end
176
+ options[:credentials] = Aws::AssumeRoleWebIdentityCredentials.new(credentials_options)
177
+ when @instance_profile_credentials
178
+ c = @instance_profile_credentials
179
+ credentials_options[:retries] = c.retries if c.retries
180
+ credentials_options[:ip_address] = c.ip_address if c.ip_address
181
+ credentials_options[:port] = c.port if c.port
182
+ credentials_options[:http_open_timeout] = c.http_open_timeout if c.http_open_timeout
183
+ credentials_options[:http_read_timeout] = c.http_read_timeout if c.http_read_timeout
184
+ if ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"]
185
+ options[:credentials] = Aws::ECSCredentials.new(credentials_options)
186
+ else
187
+ options[:credentials] = Aws::InstanceProfileCredentials.new(credentials_options)
188
+ end
189
+ when @shared_credentials
190
+ c = @shared_credentials
191
+ credentials_options[:path] = c.path if c.path
192
+ credentials_options[:profile_name] = c.profile_name if c.profile_name
193
+ options[:credentials] = Aws::SharedCredentials.new(credentials_options)
194
+ when @process_credentials
195
+ if Gem::Version.new(Aws::CORE_GEM_VERSION) < Gem::Version.new('3.24.0')
196
+ raise Fluent::ConfigError, "Config process_credentials requires aws-sdk-core >= 3.24.0. Found aws-sdk-core #{Aws::CORE_GEM_VERSION} instead."
197
+ end
198
+ c = @process_credentials
199
+ process = c.process
200
+ options[:credentials] = Aws::ProcessCredentials.new(process)
201
+ else
202
+ # Use default credentials
203
+ # See http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html
204
+ end
205
+ options
206
+ end
207
+ end
208
+ end
209
+ end
210
+ end
@@ -0,0 +1,27 @@
1
+ require "stringio"
2
+ require "zlib"
3
+
4
+ class Stream < StringIO
5
+ def initialize(*)
6
+ super
7
+ set_encoding "BINARY"
8
+ end
9
+
10
+ def close
11
+ rewind;
12
+ end
13
+ end
14
+
15
+ class Gzip
16
+ def self.compress(string, level = Zlib::DEFAULT_COMPRESSION, strategy = Zlib::DEFAULT_STRATEGY)
17
+ output = Stream.new
18
+ gz = Zlib::GzipWriter.new(output, level, strategy)
19
+ gz.write(string)
20
+ gz.close
21
+ output.string
22
+ end
23
+
24
+ def self.decompress(string)
25
+ Zlib::GzipReader.wrap(StringIO.new(string), &:read)
26
+ end
27
+ end
@@ -0,0 +1,60 @@
1
+ #
2
+ # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License"). You
5
+ # may not use this file except in compliance with the License. A copy of
6
+ # the License is located at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is
11
+ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
12
+ # ANY KIND, either express or implied. See the License for the specific
13
+ # language governing permissions and limitations under the License.
14
+
15
+ require 'fluent/plugin/kinesis'
16
+
17
+ module Fluent
18
+ module Plugin
19
+ class KinesisFirehoseOutput < KinesisOutput
20
+ Fluent::Plugin.register_output('kinesis_firehose', self)
21
+
22
+ RequestType = :firehose
23
+ BatchRequestLimitCount = 500
24
+ BatchRequestLimitSize = 4 * 1024 * 1024
25
+ include KinesisHelper::API::BatchRequest
26
+
27
+ config_param :delivery_stream_name, :string
28
+ config_param :append_new_line, :bool, default: true
29
+
30
+ def configure(conf)
31
+ super
32
+ if @append_new_line
33
+ org_data_formatter = @data_formatter
34
+ @data_formatter = ->(tag, time, record) {
35
+ org_data_formatter.call(tag, time, record).chomp + "\n"
36
+ }
37
+ end
38
+ end
39
+
40
+ def format(tag, time, record)
41
+ format_for_api do
42
+ [@data_formatter.call(tag, time, record)]
43
+ end
44
+ end
45
+
46
+ def write(chunk)
47
+ delivery_stream_name = extract_placeholders(@delivery_stream_name, chunk)
48
+ write_records_batch(chunk, delivery_stream_name) do |batch|
49
+ records = batch.map{|(data)|
50
+ { data: data }
51
+ }
52
+ client.put_record_batch(
53
+ delivery_stream_name: delivery_stream_name,
54
+ records: records,
55
+ )
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,72 @@
1
+ #
2
+ # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License"). You
5
+ # may not use this file except in compliance with the License. A copy of
6
+ # the License is located at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is
11
+ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
12
+ # ANY KIND, either express or implied. See the License for the specific
13
+ # language governing permissions and limitations under the License.
14
+
15
+ require 'fluent/plugin/kinesis'
16
+
17
+ module Fluent
18
+ module Plugin
19
+ class KinesisStreamsOutput < KinesisOutput
20
+ Fluent::Plugin.register_output('kinesis_streams', self)
21
+
22
+ RequestType = :streams
23
+ BatchRequestLimitCount = 500
24
+ BatchRequestLimitSize = 5 * 1024 * 1024
25
+ include KinesisHelper::API::BatchRequest
26
+
27
+ config_param :stream_name, :string
28
+ config_param :partition_key, :string, default: nil
29
+
30
+ def configure(conf)
31
+ super
32
+ @key_formatter = key_formatter_create
33
+ end
34
+
35
+ def format(tag, time, record)
36
+ format_for_api do
37
+ data = @data_formatter.call(tag, time, record)
38
+ key = @key_formatter.call(record)
39
+ [data, key]
40
+ end
41
+ end
42
+
43
+ def write(chunk)
44
+ stream_name = extract_placeholders(@stream_name, chunk)
45
+ write_records_batch(chunk, stream_name) do |batch|
46
+ records = batch.map{|(data, partition_key)|
47
+ { data: data, partition_key: partition_key }
48
+ }
49
+ client.put_records(
50
+ stream_name: stream_name,
51
+ records: records,
52
+ )
53
+ end
54
+ end
55
+
56
+ private
57
+
58
+ def key_formatter_create
59
+ if @partition_key.nil?
60
+ ->(record) { SecureRandom.hex(16) }
61
+ else
62
+ ->(record) {
63
+ if !record.key?(@partition_key)
64
+ raise KeyNotFoundError.new(@partition_key, record)
65
+ end
66
+ record[@partition_key]
67
+ }
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,79 @@
1
+ #
2
+ # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License"). You
5
+ # may not use this file except in compliance with the License. A copy of
6
+ # the License is located at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is
11
+ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
12
+ # ANY KIND, either express or implied. See the License for the specific
13
+ # language governing permissions and limitations under the License.
14
+
15
+ require 'fluent/plugin/kinesis'
16
+ require 'fluent/plugin/kinesis_helper/aggregator'
17
+
18
+ module Fluent
19
+ module Plugin
20
+ class KinesisStreamsAggregatedOutput < KinesisOutput
21
+ Fluent::Plugin.register_output('kinesis_streams_aggregated', self)
22
+ include KinesisHelper::Aggregator::Mixin
23
+
24
+ RequestType = :streams_aggregated
25
+ BatchRequestLimitCount = 100_000
26
+ BatchRequestLimitSize = 1024 * 1024
27
+ include KinesisHelper::API::BatchRequest
28
+
29
+ config_param :stream_name, :string
30
+ config_param :fixed_partition_key, :string, default: nil
31
+
32
+ def configure(conf)
33
+ super
34
+ @partition_key_generator = create_partition_key_generator
35
+ @batch_request_max_size -= offset
36
+ @max_record_size -= offset
37
+ end
38
+
39
+ def format(tag, time, record)
40
+ format_for_api do
41
+ [@data_formatter.call(tag, time, record)]
42
+ end
43
+ end
44
+
45
+ def write(chunk)
46
+ stream_name = extract_placeholders(@stream_name, chunk)
47
+ write_records_batch(chunk, stream_name) do |batch|
48
+ key = @partition_key_generator.call
49
+ records = batch.map{|(data)|data}
50
+ client.put_records(
51
+ stream_name: stream_name,
52
+ records: [{
53
+ partition_key: key,
54
+ data: aggregator.aggregate(records, key),
55
+ }],
56
+ )
57
+ end
58
+ end
59
+
60
+ def offset
61
+ @offset ||= AggregateOffset + @partition_key_generator.call.size*2
62
+ end
63
+
64
+ private
65
+
66
+ def size_of_values(record)
67
+ super(record) + RecordOffset
68
+ end
69
+
70
+ def create_partition_key_generator
71
+ if @fixed_partition_key.nil?
72
+ ->() { SecureRandom.hex(16) }
73
+ else
74
+ ->() { @fixed_partition_key }
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
@@ -0,0 +1,17 @@
1
+ #
2
+ # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License"). You
5
+ # may not use this file except in compliance with the License. A copy of
6
+ # the License is located at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is
11
+ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
12
+ # ANY KIND, either express or implied. See the License for the specific
13
+ # language governing permissions and limitations under the License.
14
+
15
+ module FluentPluginKinesis
16
+ VERSION = '0.0.1'
17
+ end