adp-fluent-plugin-kinesis 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +7 -0
  2. data/.github/PULL_REQUEST_TEMPLATE.md +6 -0
  3. data/.gitignore +15 -0
  4. data/.travis.yml +56 -0
  5. data/CHANGELOG.md +172 -0
  6. data/CODE_OF_CONDUCT.md +4 -0
  7. data/CONTRIBUTING.md +61 -0
  8. data/CONTRIBUTORS.txt +8 -0
  9. data/Gemfile +18 -0
  10. data/LICENSE.txt +201 -0
  11. data/Makefile +44 -0
  12. data/NOTICE.txt +2 -0
  13. data/README.md +559 -0
  14. data/Rakefile +26 -0
  15. data/adp-fluent-plugin-kinesis.gemspec +71 -0
  16. data/benchmark/task.rake +106 -0
  17. data/gemfiles/Gemfile.fluentd-0.14.22 +6 -0
  18. data/gemfiles/Gemfile.fluentd-1.13.3 +6 -0
  19. data/gemfiles/Gemfile.td-agent-3.1.0 +17 -0
  20. data/gemfiles/Gemfile.td-agent-3.1.1 +17 -0
  21. data/gemfiles/Gemfile.td-agent-3.2.0 +17 -0
  22. data/gemfiles/Gemfile.td-agent-3.2.1 +17 -0
  23. data/gemfiles/Gemfile.td-agent-3.3.0 +17 -0
  24. data/gemfiles/Gemfile.td-agent-3.4.0 +17 -0
  25. data/gemfiles/Gemfile.td-agent-3.4.1 +17 -0
  26. data/gemfiles/Gemfile.td-agent-3.5.0 +17 -0
  27. data/gemfiles/Gemfile.td-agent-3.5.1 +17 -0
  28. data/gemfiles/Gemfile.td-agent-3.6.0 +17 -0
  29. data/gemfiles/Gemfile.td-agent-3.7.0 +17 -0
  30. data/gemfiles/Gemfile.td-agent-3.7.1 +17 -0
  31. data/gemfiles/Gemfile.td-agent-3.8.0 +17 -0
  32. data/gemfiles/Gemfile.td-agent-3.8.1 +18 -0
  33. data/gemfiles/Gemfile.td-agent-4.0.0 +25 -0
  34. data/gemfiles/Gemfile.td-agent-4.0.1 +21 -0
  35. data/gemfiles/Gemfile.td-agent-4.1.0 +21 -0
  36. data/gemfiles/Gemfile.td-agent-4.1.1 +21 -0
  37. data/gemfiles/Gemfile.td-agent-4.2.0 +21 -0
  38. data/lib/fluent/plugin/kinesis.rb +174 -0
  39. data/lib/fluent/plugin/kinesis_helper/aggregator.rb +101 -0
  40. data/lib/fluent/plugin/kinesis_helper/api.rb +254 -0
  41. data/lib/fluent/plugin/kinesis_helper/client.rb +210 -0
  42. data/lib/fluent/plugin/kinesis_helper/compression.rb +27 -0
  43. data/lib/fluent/plugin/out_kinesis_firehose.rb +60 -0
  44. data/lib/fluent/plugin/out_kinesis_streams.rb +72 -0
  45. data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +79 -0
  46. data/lib/fluent_plugin_kinesis/version.rb +17 -0
  47. metadata +339 -0
@@ -0,0 +1,174 @@
1
+ #
2
+ # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License"). You
5
+ # may not use this file except in compliance with the License. A copy of
6
+ # the License is located at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is
11
+ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
12
+ # ANY KIND, either express or implied. See the License for the specific
13
+ # language governing permissions and limitations under the License.
14
+
15
+ require 'fluent/version'
16
+ require 'fluent/msgpack_factory'
17
+ require 'fluent/plugin/output'
18
+ require 'fluent/plugin/kinesis_helper/client'
19
+ require 'fluent/plugin/kinesis_helper/api'
20
+ require 'zlib'
21
+
22
+ module Fluent
23
+ module Plugin
24
+ class KinesisOutput < Fluent::Plugin::Output
25
+ include KinesisHelper::Client
26
+ include KinesisHelper::API
27
+
28
+ class SkipRecordError < ::StandardError
29
+ def initialize(message, record)
30
+ super message
31
+ @record_message = if record.is_a? Array
32
+ record.reverse.map(&:to_s).join(', ')
33
+ else
34
+ record.to_s
35
+ end
36
+ end
37
+
38
+ def to_s
39
+ super + ": " + @record_message
40
+ end
41
+ end
42
+ class KeyNotFoundError < SkipRecordError
43
+ def initialize(key, record)
44
+ super "Key '#{key}' doesn't exist", record
45
+ end
46
+ end
47
+ class ExceedMaxRecordSizeError < SkipRecordError
48
+ def initialize(size, record)
49
+ super "Record size limit exceeded in #{size/1024} KB", record
50
+ end
51
+ end
52
+ class InvalidRecordError < SkipRecordError
53
+ def initialize(record)
54
+ super "Invalid type of record", record
55
+ end
56
+ end
57
+
58
+ config_param :data_key, :string, default: nil
59
+ config_param :log_truncate_max_size, :integer, default: 1024
60
+ config_param :compression, :string, default: nil
61
+
62
+ desc "Formatter calls chomp and removes separator from the end of each record. This option is for compatible format with plugin v2. (default: false)"
63
+ # https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/142
64
+ config_param :chomp_record, :bool, default: false
65
+
66
+ config_section :format do
67
+ config_set_default :@type, 'json'
68
+ end
69
+ config_section :inject do
70
+ config_set_default :time_type, 'string'
71
+ config_set_default :time_format, '%Y-%m-%dT%H:%M:%S.%N%z'
72
+ end
73
+
74
+ config_param :debug, :bool, default: false
75
+
76
+ helpers :formatter, :inject
77
+
78
+ def configure(conf)
79
+ super
80
+ @data_formatter = data_formatter_create(conf)
81
+ end
82
+
83
+ def multi_workers_ready?
84
+ true
85
+ end
86
+
87
+ def formatted_to_msgpack_binary?
88
+ true
89
+ end
90
+
91
+ private
92
+
93
+ def data_formatter_create(conf)
94
+ formatter = formatter_create
95
+ compressor = compressor_create
96
+ if @data_key.nil?
97
+ if @chomp_record
98
+ ->(tag, time, record) {
99
+ record = inject_values_to_record(tag, time, record)
100
+ # Formatter calls chomp and removes separator from the end of each record.
101
+ # This option is for compatible format with plugin v2.
102
+ # https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/142
103
+ compressor.call(formatter.format(tag, time, record).chomp.b)
104
+ }
105
+ else
106
+ ->(tag, time, record) {
107
+ record = inject_values_to_record(tag, time, record)
108
+ compressor.call(formatter.format(tag, time, record).b)
109
+ }
110
+ end
111
+ else
112
+ ->(tag, time, record) {
113
+ raise InvalidRecordError, record unless record.is_a? Hash
114
+ raise KeyNotFoundError.new(@data_key, record) if record[@data_key].nil?
115
+ compressor.call(record[@data_key].to_s.b)
116
+ }
117
+ end
118
+ end
119
+
120
+ def compressor_create
121
+ case @compression
122
+ when "zlib"
123
+ ->(data) { Zlib::Deflate.deflate(data) }
124
+ when "gzip"
125
+ ->(data) { Gzip.compress(data) }
126
+ else
127
+ ->(data) { data }
128
+ end
129
+ end
130
+
131
+ def format_for_api(&block)
132
+ converted = block.call
133
+ size = size_of_values(converted)
134
+ if size > @max_record_size
135
+ raise ExceedMaxRecordSizeError.new(size, converted)
136
+ end
137
+ converted.to_msgpack
138
+ rescue SkipRecordError => e
139
+ log.error(truncate e)
140
+ ''
141
+ end
142
+
143
+ if Gem::Version.new(Fluent::VERSION) >= Gem::Version.new('1.8.0')
144
+ def msgpack_unpacker(*args)
145
+ Fluent::MessagePackFactory.msgpack_unpacker(*args)
146
+ end
147
+ else
148
+ include Fluent::MessagePackFactory::Mixin
149
+ end
150
+
151
+ def write_records_batch(chunk, stream_name, &block)
152
+ unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
153
+ records = chunk.to_enum(:msgpack_each)
154
+ split_to_batches(records) do |batch, size|
155
+ log.debug(sprintf "%s: Write chunk %s / %3d records / %4d KB", stream_name, unique_id, batch.size, size/1024)
156
+ batch_request_with_retry(batch, &block)
157
+ log.debug(sprintf "%s: Finish writing chunk", stream_name)
158
+ end
159
+ end
160
+
161
+ def request_type
162
+ self.class::RequestType
163
+ end
164
+
165
+ def truncate(msg)
166
+ if @log_truncate_max_size == 0 or (msg.to_s.size <= @log_truncate_max_size)
167
+ msg.to_s
168
+ else
169
+ msg.to_s[0...@log_truncate_max_size]
170
+ end
171
+ end
172
+ end
173
+ end
174
+ end
@@ -0,0 +1,101 @@
1
+ #
2
+ # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License"). You
5
+ # may not use this file except in compliance with the License. A copy of
6
+ # the License is located at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is
11
+ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
12
+ # ANY KIND, either express or implied. See the License for the specific
13
+ # language governing permissions and limitations under the License.
14
+
15
+ require 'fluent/configurable'
16
+ require 'google/protobuf'
17
+
18
+ Google::Protobuf::DescriptorPool.generated_pool.build do
19
+ add_message "AggregatedRecord" do
20
+ repeated :partition_key_table, :string, 1
21
+ repeated :explicit_hash_key_table, :string, 2
22
+ repeated :records, :message, 3, "Record"
23
+ end
24
+ add_message "Tag" do
25
+ optional :key, :string, 1
26
+ optional :value, :string, 2
27
+ end
28
+ add_message "Record" do
29
+ optional :partition_key_index, :uint64, 1
30
+ optional :explicit_hash_key_index, :uint64, 2
31
+ optional :data, :bytes, 3
32
+ repeated :tags, :message, 4, "Tag"
33
+ end
34
+ end
35
+
36
+ module Fluent
37
+ module Plugin
38
+ module KinesisHelper
39
+ class Aggregator
40
+ AggregatedRecord = Google::Protobuf::DescriptorPool.generated_pool.lookup("AggregatedRecord").msgclass
41
+ Tag = Google::Protobuf::DescriptorPool.generated_pool.lookup("Tag").msgclass
42
+ Record = Google::Protobuf::DescriptorPool.generated_pool.lookup("Record").msgclass
43
+
44
+ class InvalidEncodingError < ::StandardError; end
45
+
46
+ MagicNumber = ['F3899AC2'].pack('H*')
47
+
48
+ def aggregate(records, partition_key)
49
+ message = AggregatedRecord.encode(AggregatedRecord.new(
50
+ partition_key_table: ['a', partition_key],
51
+ records: records.map{|data|
52
+ Record.new(partition_key_index: 1, data: data)
53
+ },
54
+ ))
55
+ [MagicNumber, message, Digest::MD5.digest(message)].pack("A4A*A16")
56
+ end
57
+
58
+ def deaggregate(encoded)
59
+ unless aggregated?(encoded)
60
+ raise InvalidEncodingError, "Invalid MagicNumber #{encoded[0..3]}}"
61
+ end
62
+ message, digest = encoded[4..encoded.length-17], encoded[encoded.length-16..-1]
63
+ if Digest::MD5.digest(message) != digest
64
+ raise InvalidEncodingError, "Digest mismatch #{digest}"
65
+ end
66
+ decoded = AggregatedRecord.decode(message)
67
+ records = decoded.records.map(&:data)
68
+ partition_key = decoded.partition_key_table[1]
69
+ [records, partition_key]
70
+ end
71
+
72
+ def aggregated?(encoded)
73
+ encoded[0..3] == MagicNumber
74
+ end
75
+
76
+ def aggregated_size_offset(partition_key)
77
+ data = 'd'
78
+ encoded = aggregate([record(data)], partition_key)
79
+ finalize(encoded).size - data.size
80
+ end
81
+
82
+ module Mixin
83
+ AggregateOffset = 25
84
+ RecordOffset = 10
85
+
86
+ module Params
87
+ include Fluent::Configurable
88
+ end
89
+
90
+ def self.included(mod)
91
+ mod.include Params
92
+ end
93
+
94
+ def aggregator
95
+ @aggregator ||= Aggregator.new
96
+ end
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
@@ -0,0 +1,254 @@
1
+ #
2
+ # Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License"). You
5
+ # may not use this file except in compliance with the License. A copy of
6
+ # the License is located at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is
11
+ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
12
+ # ANY KIND, either express or implied. See the License for the specific
13
+ # language governing permissions and limitations under the License.
14
+
15
+ require 'fluent_plugin_kinesis/version'
16
+ require 'fluent/configurable'
17
+ require 'benchmark'
18
+
19
+ module Fluent
20
+ module Plugin
21
+ module KinesisHelper
22
+ module API
23
+ MaxRecordSize = 1024 * 1024 # 1 MB
24
+
25
+ module APIParams
26
+ include Fluent::Configurable
27
+ config_param :max_record_size, :integer, default: MaxRecordSize
28
+ end
29
+
30
+ def self.included(mod)
31
+ mod.include APIParams
32
+ end
33
+
34
+ def configure(conf)
35
+ super
36
+ if @max_record_size > MaxRecordSize
37
+ raise ConfigError, "max_record_size can't be grater than #{MaxRecordSize/1024} KB."
38
+ end
39
+ end
40
+
41
+ module BatchRequest
42
+ module BatchRequestParams
43
+ include Fluent::Configurable
44
+ config_param :retries_on_batch_request, :integer, default: 8
45
+ config_param :reset_backoff_if_success, :bool, default: true
46
+ config_param :batch_request_max_count, :integer, default: nil
47
+ config_param :batch_request_max_size, :integer, default: nil
48
+ config_param :drop_failed_records_after_batch_request_retries, :bool, default: true
49
+ config_param :monitor_num_of_batch_request_retries, :bool, default: false
50
+ end
51
+
52
+ def self.included(mod)
53
+ mod.include BatchRequestParams
54
+ end
55
+
56
+ def configure(conf)
57
+ super
58
+ if @batch_request_max_count.nil?
59
+ @batch_request_max_count = self.class::BatchRequestLimitCount
60
+ elsif @batch_request_max_count > self.class::BatchRequestLimitCount
61
+ raise ConfigError, "batch_request_max_count can't be grater than #{self.class::BatchRequestLimitCount}."
62
+ end
63
+ if @batch_request_max_size.nil?
64
+ @batch_request_max_size = self.class::BatchRequestLimitSize
65
+ elsif @batch_request_max_size > self.class::BatchRequestLimitSize
66
+ raise ConfigError, "batch_request_max_size can't be grater than #{self.class::BatchRequestLimitSize}."
67
+ end
68
+ end
69
+
70
+ def size_of_values(record)
71
+ record.compact.map(&:size).inject(:+) || 0
72
+ end
73
+
74
+ private
75
+
76
+ def split_to_batches(records, &block)
77
+ batch = []
78
+ size = 0
79
+ records.each do |record|
80
+ record_size = size_of_values(record)
81
+ if batch.size+1 > @batch_request_max_count or size+record_size > @batch_request_max_size
82
+ yield(batch, size)
83
+ batch = []
84
+ size = 0
85
+ end
86
+ batch << record
87
+ size += record_size
88
+ end
89
+ yield(batch, size) if batch.size > 0
90
+ end
91
+
92
+ def batch_request_with_retry(batch, retry_count=0, backoff: nil, &block)
93
+ backoff ||= Backoff.new
94
+ res = yield(batch)
95
+ if failed_count(res) > 0
96
+ failed_records = collect_failed_records(batch, res)
97
+ if retry_count < @retries_on_batch_request
98
+ backoff.reset if @reset_backoff_if_success and any_records_shipped?(res)
99
+ wait_second = backoff.next
100
+ msg = 'Retrying to request batch. Retry count: %3d, Retry records: %3d, Wait seconds %3.2f' % [retry_count+1, failed_records.size, wait_second]
101
+ log.warn(truncate msg)
102
+ # Increment num_errors to monitor batch request retries from "monitor_agent" or "fluent-plugin-prometheus"
103
+ increment_num_errors if @monitor_num_of_batch_request_retries
104
+ reliable_sleep(wait_second)
105
+ batch_request_with_retry(retry_records(failed_records), retry_count+1, backoff: backoff, &block)
106
+ else
107
+ give_up_retries(failed_records)
108
+ end
109
+ end
110
+ end
111
+
112
+ # Sleep seems to not sleep as long as we ask it, our guess is that something wakes up the thread,
113
+ # so we keep on going to sleep if that happens.
114
+ # TODO: find out who is causing the sleep to be too short and try to make them stop it instead
115
+ def reliable_sleep(wait_second)
116
+ loop do
117
+ actual = Benchmark.realtime { sleep(wait_second) }
118
+ break if actual >= wait_second
119
+ log.error("#{Thread.current.object_id} sleep failed expected #{wait_second} but slept #{actual}")
120
+ wait_second -= actual
121
+ end
122
+ end
123
+
124
+ def any_records_shipped?(res)
125
+ results(res).size > failed_count(res)
126
+ end
127
+
128
+ def collect_failed_records(records, res)
129
+ failed_records = []
130
+ results(res).each_with_index do |record, index|
131
+ next unless record[:error_code]
132
+ original = case request_type
133
+ when :streams, :firehose; records[index]
134
+ when :streams_aggregated; records
135
+ end
136
+ failed_records.push(
137
+ original: original,
138
+ error_code: record[:error_code],
139
+ error_message: record[:error_message]
140
+ )
141
+ end
142
+ failed_records
143
+ end
144
+
145
+ def retry_records(failed_records)
146
+ case request_type
147
+ when :streams, :firehose
148
+ failed_records.map{|r| r[:original] }
149
+ when :streams_aggregated
150
+ failed_records.first[:original]
151
+ end
152
+ end
153
+
154
+ def failed_count(res)
155
+ failed_field = case request_type
156
+ when :streams; :failed_record_count
157
+ when :streams_aggregated; :failed_record_count
158
+ when :firehose; :failed_put_count
159
+ end
160
+ res[failed_field]
161
+ end
162
+
163
+ def results(res)
164
+ result_field = case request_type
165
+ when :streams; :records
166
+ when :streams_aggregated; :records
167
+ when :firehose; :request_responses
168
+ end
169
+ res[result_field]
170
+ end
171
+
172
+ def give_up_retries(failed_records)
173
+ failed_records.each {|record|
174
+ log.error(truncate 'Could not put record, Error: %s/%s, Record: %s' % [
175
+ record[:error_code],
176
+ record[:error_message],
177
+ record[:original]
178
+ ])
179
+ }
180
+
181
+ if @drop_failed_records_after_batch_request_retries
182
+ # Increment num_errors to monitor batch request failure from "monitor_agent" or "fluent-plugin-prometheus"
183
+ increment_num_errors
184
+ else
185
+ # Raise error and return chunk to Fluentd for retrying
186
+ case request_type
187
+ # @see https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
188
+ # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Kinesis/Client.html#put_records-instance_method
189
+ # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Kinesis/Errors.html
190
+ when :streams, :streams_aggregated
191
+ provisioned_throughput_exceeded_records = failed_records.select { |record| record[:error_code] == 'ProvisionedThroughputExceededException' }
192
+ target_failed_record = provisioned_throughput_exceeded_records.first || failed_records.first
193
+ target_error = provisioned_throughput_exceeded_records.empty? ?
194
+ Aws::Kinesis::Errors::ServiceError :
195
+ Aws::Kinesis::Errors::ProvisionedThroughputExceededException
196
+ # @see https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
197
+ # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Firehose/Client.html#put_record_batch-instance_method
198
+ # @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Firehose/Errors.html
199
+ when :firehose
200
+ service_unavailable_exception_records = failed_records.select { |record| record[:error_code] == 'ServiceUnavailableException' }
201
+ target_failed_record = service_unavailable_exception_records.first || failed_records.first
202
+ target_error = service_unavailable_exception_records.empty? ?
203
+ Aws::Firehose::Errors::ServiceError :
204
+ Aws::Firehose::Errors::ServiceUnavailableException
205
+ end
206
+ log.error("Raise #{target_failed_record[:error_code]} and return chunk to Fluentd buffer for retrying")
207
+ raise target_error.new(Seahorse::Client::RequestContext.new, target_failed_record[:error_message])
208
+ end
209
+ end
210
+
211
+ def increment_num_errors
212
+ # Prepare Fluent::Plugin::Output instance variables to count errors in this method.
213
+ # These instance variables are initialized here for possible future breaking changes of Fluentd.
214
+ @num_errors ||= 0
215
+ # @see https://github.com/fluent/fluentd/commit/d245454658d16170431d276fcd5849fb0d88ab2b
216
+ if Gem::Version.new(Fluent::VERSION) >= Gem::Version.new('1.7.0')
217
+ @counter_mutex ||= Mutex.new
218
+ @counter_mutex.synchronize{ @num_errors += 1 }
219
+ else
220
+ @counters_monitor ||= Monitor.new
221
+ @counters_monitor.synchronize{ @num_errors += 1 }
222
+ end
223
+ end
224
+
225
+ class Backoff
226
+ def initialize
227
+ @count = 0
228
+ end
229
+
230
+ def next
231
+ value = calc(@count)
232
+ @count += 1
233
+ value
234
+ end
235
+
236
+ def reset
237
+ @count = 0
238
+ end
239
+
240
+ private
241
+
242
+ def calc(count)
243
+ (2 ** count) * scaling_factor
244
+ end
245
+
246
+ def scaling_factor
247
+ 0.3 + (0.5-rand) * 0.1
248
+ end
249
+ end
250
+ end
251
+ end
252
+ end
253
+ end
254
+ end