adp-fluent-plugin-kinesis 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +6 -0
- data/.gitignore +15 -0
- data/.travis.yml +56 -0
- data/CHANGELOG.md +172 -0
- data/CODE_OF_CONDUCT.md +4 -0
- data/CONTRIBUTING.md +61 -0
- data/CONTRIBUTORS.txt +8 -0
- data/Gemfile +18 -0
- data/LICENSE.txt +201 -0
- data/Makefile +44 -0
- data/NOTICE.txt +2 -0
- data/README.md +559 -0
- data/Rakefile +26 -0
- data/adp-fluent-plugin-kinesis.gemspec +71 -0
- data/benchmark/task.rake +106 -0
- data/gemfiles/Gemfile.fluentd-0.14.22 +6 -0
- data/gemfiles/Gemfile.fluentd-1.13.3 +6 -0
- data/gemfiles/Gemfile.td-agent-3.1.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.1.1 +17 -0
- data/gemfiles/Gemfile.td-agent-3.2.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.2.1 +17 -0
- data/gemfiles/Gemfile.td-agent-3.3.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.4.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.4.1 +17 -0
- data/gemfiles/Gemfile.td-agent-3.5.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.5.1 +17 -0
- data/gemfiles/Gemfile.td-agent-3.6.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.7.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.7.1 +17 -0
- data/gemfiles/Gemfile.td-agent-3.8.0 +17 -0
- data/gemfiles/Gemfile.td-agent-3.8.1 +18 -0
- data/gemfiles/Gemfile.td-agent-4.0.0 +25 -0
- data/gemfiles/Gemfile.td-agent-4.0.1 +21 -0
- data/gemfiles/Gemfile.td-agent-4.1.0 +21 -0
- data/gemfiles/Gemfile.td-agent-4.1.1 +21 -0
- data/gemfiles/Gemfile.td-agent-4.2.0 +21 -0
- data/lib/fluent/plugin/kinesis.rb +174 -0
- data/lib/fluent/plugin/kinesis_helper/aggregator.rb +101 -0
- data/lib/fluent/plugin/kinesis_helper/api.rb +254 -0
- data/lib/fluent/plugin/kinesis_helper/client.rb +210 -0
- data/lib/fluent/plugin/kinesis_helper/compression.rb +27 -0
- data/lib/fluent/plugin/out_kinesis_firehose.rb +60 -0
- data/lib/fluent/plugin/out_kinesis_streams.rb +72 -0
- data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +79 -0
- data/lib/fluent_plugin_kinesis/version.rb +17 -0
- metadata +339 -0
@@ -0,0 +1,174 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
5
|
+
# may not use this file except in compliance with the License. A copy of
|
6
|
+
# the License is located at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is
|
11
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
12
|
+
# ANY KIND, either express or implied. See the License for the specific
|
13
|
+
# language governing permissions and limitations under the License.
|
14
|
+
|
15
|
+
require 'fluent/version'
|
16
|
+
require 'fluent/msgpack_factory'
|
17
|
+
require 'fluent/plugin/output'
|
18
|
+
require 'fluent/plugin/kinesis_helper/client'
|
19
|
+
require 'fluent/plugin/kinesis_helper/api'
|
20
|
+
require 'zlib'
|
21
|
+
|
22
|
+
module Fluent
|
23
|
+
module Plugin
|
24
|
+
class KinesisOutput < Fluent::Plugin::Output
|
25
|
+
include KinesisHelper::Client
|
26
|
+
include KinesisHelper::API
|
27
|
+
|
28
|
+
class SkipRecordError < ::StandardError
|
29
|
+
def initialize(message, record)
|
30
|
+
super message
|
31
|
+
@record_message = if record.is_a? Array
|
32
|
+
record.reverse.map(&:to_s).join(', ')
|
33
|
+
else
|
34
|
+
record.to_s
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def to_s
|
39
|
+
super + ": " + @record_message
|
40
|
+
end
|
41
|
+
end
|
42
|
+
class KeyNotFoundError < SkipRecordError
|
43
|
+
def initialize(key, record)
|
44
|
+
super "Key '#{key}' doesn't exist", record
|
45
|
+
end
|
46
|
+
end
|
47
|
+
class ExceedMaxRecordSizeError < SkipRecordError
|
48
|
+
def initialize(size, record)
|
49
|
+
super "Record size limit exceeded in #{size/1024} KB", record
|
50
|
+
end
|
51
|
+
end
|
52
|
+
class InvalidRecordError < SkipRecordError
|
53
|
+
def initialize(record)
|
54
|
+
super "Invalid type of record", record
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
config_param :data_key, :string, default: nil
|
59
|
+
config_param :log_truncate_max_size, :integer, default: 1024
|
60
|
+
config_param :compression, :string, default: nil
|
61
|
+
|
62
|
+
desc "Formatter calls chomp and removes separator from the end of each record. This option is for compatible format with plugin v2. (default: false)"
|
63
|
+
# https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/142
|
64
|
+
config_param :chomp_record, :bool, default: false
|
65
|
+
|
66
|
+
config_section :format do
|
67
|
+
config_set_default :@type, 'json'
|
68
|
+
end
|
69
|
+
config_section :inject do
|
70
|
+
config_set_default :time_type, 'string'
|
71
|
+
config_set_default :time_format, '%Y-%m-%dT%H:%M:%S.%N%z'
|
72
|
+
end
|
73
|
+
|
74
|
+
config_param :debug, :bool, default: false
|
75
|
+
|
76
|
+
helpers :formatter, :inject
|
77
|
+
|
78
|
+
def configure(conf)
|
79
|
+
super
|
80
|
+
@data_formatter = data_formatter_create(conf)
|
81
|
+
end
|
82
|
+
|
83
|
+
def multi_workers_ready?
|
84
|
+
true
|
85
|
+
end
|
86
|
+
|
87
|
+
def formatted_to_msgpack_binary?
|
88
|
+
true
|
89
|
+
end
|
90
|
+
|
91
|
+
private
|
92
|
+
|
93
|
+
def data_formatter_create(conf)
|
94
|
+
formatter = formatter_create
|
95
|
+
compressor = compressor_create
|
96
|
+
if @data_key.nil?
|
97
|
+
if @chomp_record
|
98
|
+
->(tag, time, record) {
|
99
|
+
record = inject_values_to_record(tag, time, record)
|
100
|
+
# Formatter calls chomp and removes separator from the end of each record.
|
101
|
+
# This option is for compatible format with plugin v2.
|
102
|
+
# https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/142
|
103
|
+
compressor.call(formatter.format(tag, time, record).chomp.b)
|
104
|
+
}
|
105
|
+
else
|
106
|
+
->(tag, time, record) {
|
107
|
+
record = inject_values_to_record(tag, time, record)
|
108
|
+
compressor.call(formatter.format(tag, time, record).b)
|
109
|
+
}
|
110
|
+
end
|
111
|
+
else
|
112
|
+
->(tag, time, record) {
|
113
|
+
raise InvalidRecordError, record unless record.is_a? Hash
|
114
|
+
raise KeyNotFoundError.new(@data_key, record) if record[@data_key].nil?
|
115
|
+
compressor.call(record[@data_key].to_s.b)
|
116
|
+
}
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def compressor_create
|
121
|
+
case @compression
|
122
|
+
when "zlib"
|
123
|
+
->(data) { Zlib::Deflate.deflate(data) }
|
124
|
+
when "gzip"
|
125
|
+
->(data) { Gzip.compress(data) }
|
126
|
+
else
|
127
|
+
->(data) { data }
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def format_for_api(&block)
|
132
|
+
converted = block.call
|
133
|
+
size = size_of_values(converted)
|
134
|
+
if size > @max_record_size
|
135
|
+
raise ExceedMaxRecordSizeError.new(size, converted)
|
136
|
+
end
|
137
|
+
converted.to_msgpack
|
138
|
+
rescue SkipRecordError => e
|
139
|
+
log.error(truncate e)
|
140
|
+
''
|
141
|
+
end
|
142
|
+
|
143
|
+
if Gem::Version.new(Fluent::VERSION) >= Gem::Version.new('1.8.0')
|
144
|
+
def msgpack_unpacker(*args)
|
145
|
+
Fluent::MessagePackFactory.msgpack_unpacker(*args)
|
146
|
+
end
|
147
|
+
else
|
148
|
+
include Fluent::MessagePackFactory::Mixin
|
149
|
+
end
|
150
|
+
|
151
|
+
def write_records_batch(chunk, stream_name, &block)
|
152
|
+
unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
|
153
|
+
records = chunk.to_enum(:msgpack_each)
|
154
|
+
split_to_batches(records) do |batch, size|
|
155
|
+
log.debug(sprintf "%s: Write chunk %s / %3d records / %4d KB", stream_name, unique_id, batch.size, size/1024)
|
156
|
+
batch_request_with_retry(batch, &block)
|
157
|
+
log.debug(sprintf "%s: Finish writing chunk", stream_name)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def request_type
|
162
|
+
self.class::RequestType
|
163
|
+
end
|
164
|
+
|
165
|
+
def truncate(msg)
|
166
|
+
if @log_truncate_max_size == 0 or (msg.to_s.size <= @log_truncate_max_size)
|
167
|
+
msg.to_s
|
168
|
+
else
|
169
|
+
msg.to_s[0...@log_truncate_max_size]
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
@@ -0,0 +1,101 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
5
|
+
# may not use this file except in compliance with the License. A copy of
|
6
|
+
# the License is located at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is
|
11
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
12
|
+
# ANY KIND, either express or implied. See the License for the specific
|
13
|
+
# language governing permissions and limitations under the License.
|
14
|
+
|
15
|
+
require 'fluent/configurable'
|
16
|
+
require 'google/protobuf'
|
17
|
+
|
18
|
+
Google::Protobuf::DescriptorPool.generated_pool.build do
|
19
|
+
add_message "AggregatedRecord" do
|
20
|
+
repeated :partition_key_table, :string, 1
|
21
|
+
repeated :explicit_hash_key_table, :string, 2
|
22
|
+
repeated :records, :message, 3, "Record"
|
23
|
+
end
|
24
|
+
add_message "Tag" do
|
25
|
+
optional :key, :string, 1
|
26
|
+
optional :value, :string, 2
|
27
|
+
end
|
28
|
+
add_message "Record" do
|
29
|
+
optional :partition_key_index, :uint64, 1
|
30
|
+
optional :explicit_hash_key_index, :uint64, 2
|
31
|
+
optional :data, :bytes, 3
|
32
|
+
repeated :tags, :message, 4, "Tag"
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
module Fluent
|
37
|
+
module Plugin
|
38
|
+
module KinesisHelper
|
39
|
+
class Aggregator
|
40
|
+
AggregatedRecord = Google::Protobuf::DescriptorPool.generated_pool.lookup("AggregatedRecord").msgclass
|
41
|
+
Tag = Google::Protobuf::DescriptorPool.generated_pool.lookup("Tag").msgclass
|
42
|
+
Record = Google::Protobuf::DescriptorPool.generated_pool.lookup("Record").msgclass
|
43
|
+
|
44
|
+
class InvalidEncodingError < ::StandardError; end
|
45
|
+
|
46
|
+
MagicNumber = ['F3899AC2'].pack('H*')
|
47
|
+
|
48
|
+
def aggregate(records, partition_key)
|
49
|
+
message = AggregatedRecord.encode(AggregatedRecord.new(
|
50
|
+
partition_key_table: ['a', partition_key],
|
51
|
+
records: records.map{|data|
|
52
|
+
Record.new(partition_key_index: 1, data: data)
|
53
|
+
},
|
54
|
+
))
|
55
|
+
[MagicNumber, message, Digest::MD5.digest(message)].pack("A4A*A16")
|
56
|
+
end
|
57
|
+
|
58
|
+
def deaggregate(encoded)
|
59
|
+
unless aggregated?(encoded)
|
60
|
+
raise InvalidEncodingError, "Invalid MagicNumber #{encoded[0..3]}}"
|
61
|
+
end
|
62
|
+
message, digest = encoded[4..encoded.length-17], encoded[encoded.length-16..-1]
|
63
|
+
if Digest::MD5.digest(message) != digest
|
64
|
+
raise InvalidEncodingError, "Digest mismatch #{digest}"
|
65
|
+
end
|
66
|
+
decoded = AggregatedRecord.decode(message)
|
67
|
+
records = decoded.records.map(&:data)
|
68
|
+
partition_key = decoded.partition_key_table[1]
|
69
|
+
[records, partition_key]
|
70
|
+
end
|
71
|
+
|
72
|
+
def aggregated?(encoded)
|
73
|
+
encoded[0..3] == MagicNumber
|
74
|
+
end
|
75
|
+
|
76
|
+
def aggregated_size_offset(partition_key)
|
77
|
+
data = 'd'
|
78
|
+
encoded = aggregate([record(data)], partition_key)
|
79
|
+
finalize(encoded).size - data.size
|
80
|
+
end
|
81
|
+
|
82
|
+
module Mixin
|
83
|
+
AggregateOffset = 25
|
84
|
+
RecordOffset = 10
|
85
|
+
|
86
|
+
module Params
|
87
|
+
include Fluent::Configurable
|
88
|
+
end
|
89
|
+
|
90
|
+
def self.included(mod)
|
91
|
+
mod.include Params
|
92
|
+
end
|
93
|
+
|
94
|
+
def aggregator
|
95
|
+
@aggregator ||= Aggregator.new
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
@@ -0,0 +1,254 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License"). You
|
5
|
+
# may not use this file except in compliance with the License. A copy of
|
6
|
+
# the License is located at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is
|
11
|
+
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
|
12
|
+
# ANY KIND, either express or implied. See the License for the specific
|
13
|
+
# language governing permissions and limitations under the License.
|
14
|
+
|
15
|
+
require 'fluent_plugin_kinesis/version'
|
16
|
+
require 'fluent/configurable'
|
17
|
+
require 'benchmark'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
module Plugin
|
21
|
+
module KinesisHelper
|
22
|
+
module API
|
23
|
+
MaxRecordSize = 1024 * 1024 # 1 MB
|
24
|
+
|
25
|
+
module APIParams
|
26
|
+
include Fluent::Configurable
|
27
|
+
config_param :max_record_size, :integer, default: MaxRecordSize
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.included(mod)
|
31
|
+
mod.include APIParams
|
32
|
+
end
|
33
|
+
|
34
|
+
def configure(conf)
|
35
|
+
super
|
36
|
+
if @max_record_size > MaxRecordSize
|
37
|
+
raise ConfigError, "max_record_size can't be grater than #{MaxRecordSize/1024} KB."
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
module BatchRequest
|
42
|
+
module BatchRequestParams
|
43
|
+
include Fluent::Configurable
|
44
|
+
config_param :retries_on_batch_request, :integer, default: 8
|
45
|
+
config_param :reset_backoff_if_success, :bool, default: true
|
46
|
+
config_param :batch_request_max_count, :integer, default: nil
|
47
|
+
config_param :batch_request_max_size, :integer, default: nil
|
48
|
+
config_param :drop_failed_records_after_batch_request_retries, :bool, default: true
|
49
|
+
config_param :monitor_num_of_batch_request_retries, :bool, default: false
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.included(mod)
|
53
|
+
mod.include BatchRequestParams
|
54
|
+
end
|
55
|
+
|
56
|
+
def configure(conf)
|
57
|
+
super
|
58
|
+
if @batch_request_max_count.nil?
|
59
|
+
@batch_request_max_count = self.class::BatchRequestLimitCount
|
60
|
+
elsif @batch_request_max_count > self.class::BatchRequestLimitCount
|
61
|
+
raise ConfigError, "batch_request_max_count can't be grater than #{self.class::BatchRequestLimitCount}."
|
62
|
+
end
|
63
|
+
if @batch_request_max_size.nil?
|
64
|
+
@batch_request_max_size = self.class::BatchRequestLimitSize
|
65
|
+
elsif @batch_request_max_size > self.class::BatchRequestLimitSize
|
66
|
+
raise ConfigError, "batch_request_max_size can't be grater than #{self.class::BatchRequestLimitSize}."
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def size_of_values(record)
|
71
|
+
record.compact.map(&:size).inject(:+) || 0
|
72
|
+
end
|
73
|
+
|
74
|
+
private
|
75
|
+
|
76
|
+
def split_to_batches(records, &block)
|
77
|
+
batch = []
|
78
|
+
size = 0
|
79
|
+
records.each do |record|
|
80
|
+
record_size = size_of_values(record)
|
81
|
+
if batch.size+1 > @batch_request_max_count or size+record_size > @batch_request_max_size
|
82
|
+
yield(batch, size)
|
83
|
+
batch = []
|
84
|
+
size = 0
|
85
|
+
end
|
86
|
+
batch << record
|
87
|
+
size += record_size
|
88
|
+
end
|
89
|
+
yield(batch, size) if batch.size > 0
|
90
|
+
end
|
91
|
+
|
92
|
+
def batch_request_with_retry(batch, retry_count=0, backoff: nil, &block)
|
93
|
+
backoff ||= Backoff.new
|
94
|
+
res = yield(batch)
|
95
|
+
if failed_count(res) > 0
|
96
|
+
failed_records = collect_failed_records(batch, res)
|
97
|
+
if retry_count < @retries_on_batch_request
|
98
|
+
backoff.reset if @reset_backoff_if_success and any_records_shipped?(res)
|
99
|
+
wait_second = backoff.next
|
100
|
+
msg = 'Retrying to request batch. Retry count: %3d, Retry records: %3d, Wait seconds %3.2f' % [retry_count+1, failed_records.size, wait_second]
|
101
|
+
log.warn(truncate msg)
|
102
|
+
# Increment num_errors to monitor batch request retries from "monitor_agent" or "fluent-plugin-prometheus"
|
103
|
+
increment_num_errors if @monitor_num_of_batch_request_retries
|
104
|
+
reliable_sleep(wait_second)
|
105
|
+
batch_request_with_retry(retry_records(failed_records), retry_count+1, backoff: backoff, &block)
|
106
|
+
else
|
107
|
+
give_up_retries(failed_records)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
# Sleep seems to not sleep as long as we ask it, our guess is that something wakes up the thread,
|
113
|
+
# so we keep on going to sleep if that happens.
|
114
|
+
# TODO: find out who is causing the sleep to be too short and try to make them stop it instead
|
115
|
+
def reliable_sleep(wait_second)
|
116
|
+
loop do
|
117
|
+
actual = Benchmark.realtime { sleep(wait_second) }
|
118
|
+
break if actual >= wait_second
|
119
|
+
log.error("#{Thread.current.object_id} sleep failed expected #{wait_second} but slept #{actual}")
|
120
|
+
wait_second -= actual
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def any_records_shipped?(res)
|
125
|
+
results(res).size > failed_count(res)
|
126
|
+
end
|
127
|
+
|
128
|
+
def collect_failed_records(records, res)
|
129
|
+
failed_records = []
|
130
|
+
results(res).each_with_index do |record, index|
|
131
|
+
next unless record[:error_code]
|
132
|
+
original = case request_type
|
133
|
+
when :streams, :firehose; records[index]
|
134
|
+
when :streams_aggregated; records
|
135
|
+
end
|
136
|
+
failed_records.push(
|
137
|
+
original: original,
|
138
|
+
error_code: record[:error_code],
|
139
|
+
error_message: record[:error_message]
|
140
|
+
)
|
141
|
+
end
|
142
|
+
failed_records
|
143
|
+
end
|
144
|
+
|
145
|
+
def retry_records(failed_records)
|
146
|
+
case request_type
|
147
|
+
when :streams, :firehose
|
148
|
+
failed_records.map{|r| r[:original] }
|
149
|
+
when :streams_aggregated
|
150
|
+
failed_records.first[:original]
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
def failed_count(res)
|
155
|
+
failed_field = case request_type
|
156
|
+
when :streams; :failed_record_count
|
157
|
+
when :streams_aggregated; :failed_record_count
|
158
|
+
when :firehose; :failed_put_count
|
159
|
+
end
|
160
|
+
res[failed_field]
|
161
|
+
end
|
162
|
+
|
163
|
+
def results(res)
|
164
|
+
result_field = case request_type
|
165
|
+
when :streams; :records
|
166
|
+
when :streams_aggregated; :records
|
167
|
+
when :firehose; :request_responses
|
168
|
+
end
|
169
|
+
res[result_field]
|
170
|
+
end
|
171
|
+
|
172
|
+
def give_up_retries(failed_records)
|
173
|
+
failed_records.each {|record|
|
174
|
+
log.error(truncate 'Could not put record, Error: %s/%s, Record: %s' % [
|
175
|
+
record[:error_code],
|
176
|
+
record[:error_message],
|
177
|
+
record[:original]
|
178
|
+
])
|
179
|
+
}
|
180
|
+
|
181
|
+
if @drop_failed_records_after_batch_request_retries
|
182
|
+
# Increment num_errors to monitor batch request failure from "monitor_agent" or "fluent-plugin-prometheus"
|
183
|
+
increment_num_errors
|
184
|
+
else
|
185
|
+
# Raise error and return chunk to Fluentd for retrying
|
186
|
+
case request_type
|
187
|
+
# @see https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
|
188
|
+
# @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Kinesis/Client.html#put_records-instance_method
|
189
|
+
# @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Kinesis/Errors.html
|
190
|
+
when :streams, :streams_aggregated
|
191
|
+
provisioned_throughput_exceeded_records = failed_records.select { |record| record[:error_code] == 'ProvisionedThroughputExceededException' }
|
192
|
+
target_failed_record = provisioned_throughput_exceeded_records.first || failed_records.first
|
193
|
+
target_error = provisioned_throughput_exceeded_records.empty? ?
|
194
|
+
Aws::Kinesis::Errors::ServiceError :
|
195
|
+
Aws::Kinesis::Errors::ProvisionedThroughputExceededException
|
196
|
+
# @see https://docs.aws.amazon.com/kinesis/latest/APIReference/API_PutRecords.html
|
197
|
+
# @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Firehose/Client.html#put_record_batch-instance_method
|
198
|
+
# @see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/Firehose/Errors.html
|
199
|
+
when :firehose
|
200
|
+
service_unavailable_exception_records = failed_records.select { |record| record[:error_code] == 'ServiceUnavailableException' }
|
201
|
+
target_failed_record = service_unavailable_exception_records.first || failed_records.first
|
202
|
+
target_error = service_unavailable_exception_records.empty? ?
|
203
|
+
Aws::Firehose::Errors::ServiceError :
|
204
|
+
Aws::Firehose::Errors::ServiceUnavailableException
|
205
|
+
end
|
206
|
+
log.error("Raise #{target_failed_record[:error_code]} and return chunk to Fluentd buffer for retrying")
|
207
|
+
raise target_error.new(Seahorse::Client::RequestContext.new, target_failed_record[:error_message])
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
def increment_num_errors
|
212
|
+
# Prepare Fluent::Plugin::Output instance variables to count errors in this method.
|
213
|
+
# These instance variables are initialized here for possible future breaking changes of Fluentd.
|
214
|
+
@num_errors ||= 0
|
215
|
+
# @see https://github.com/fluent/fluentd/commit/d245454658d16170431d276fcd5849fb0d88ab2b
|
216
|
+
if Gem::Version.new(Fluent::VERSION) >= Gem::Version.new('1.7.0')
|
217
|
+
@counter_mutex ||= Mutex.new
|
218
|
+
@counter_mutex.synchronize{ @num_errors += 1 }
|
219
|
+
else
|
220
|
+
@counters_monitor ||= Monitor.new
|
221
|
+
@counters_monitor.synchronize{ @num_errors += 1 }
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
class Backoff
|
226
|
+
def initialize
|
227
|
+
@count = 0
|
228
|
+
end
|
229
|
+
|
230
|
+
def next
|
231
|
+
value = calc(@count)
|
232
|
+
@count += 1
|
233
|
+
value
|
234
|
+
end
|
235
|
+
|
236
|
+
def reset
|
237
|
+
@count = 0
|
238
|
+
end
|
239
|
+
|
240
|
+
private
|
241
|
+
|
242
|
+
def calc(count)
|
243
|
+
(2 ** count) * scaling_factor
|
244
|
+
end
|
245
|
+
|
246
|
+
def scaling_factor
|
247
|
+
0.3 + (0.5-rand) * 0.1
|
248
|
+
end
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
254
|
+
end
|