fluent-plugin-deis-kafka 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -1
- data/lib/fluent/plugin/out_deis.rb +171 -179
- metadata +5 -47
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c414fda72d442beb154378e4c9eb6c1b5d87a648
|
|
4
|
+
data.tar.gz: 16bd58fb7bb0a822bb3574a0ec037d7cf10f33fd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4fb84eaaae243d26775b3e20b0ca0a0725f586d914db8332ccb036ff07f56a0470cd90603572c262fb9335739a22c11b1aca6170a4459f9da6c8cb28fe24aec9
|
|
7
|
+
data.tar.gz: 0df1e72c2a52b1242018dee56bf75eeacd1fafba515594977384bfbcd7caa080c46b19ca9a30e2d57144a46ed794bc7abfee83b8868329d7ed21a6b8a22ae350
|
data/README.md
CHANGED
|
@@ -1 +1,2 @@
|
|
|
1
|
-
# fluentd-plugin-deis-kafka
|
|
1
|
+
# fluentd-plugin-deis-kafka
|
|
2
|
+
[](https://travis-ci.org/topfreegames/fluent-plugin-deis-kafka) [](https://gemnasium.com/github.com/topfreegames/fluent-plugin-deis-kafka) [](https://badge.fury.io/rb/fluentd)
|
|
@@ -1,214 +1,206 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'fluent/mixin/config_placeholders'
|
|
4
|
-
require 'fluent/mixin/plaintextformatter'
|
|
5
|
-
require 'fluent/mixin/rewrite_tag_name'
|
|
6
3
|
require 'fluent/mixin/deis'
|
|
7
|
-
require 'fluent/output'
|
|
4
|
+
require 'fluent/plugin/output'
|
|
5
|
+
require 'fluent/plugin_helper/compat_parameters'
|
|
8
6
|
require 'influxdb'
|
|
9
7
|
|
|
10
8
|
module Fluent
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
@producers = {}
|
|
55
|
-
@producers_mutex = Mutex.new
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
def start
|
|
59
|
-
super
|
|
60
|
-
refresh_client
|
|
61
|
-
end
|
|
9
|
+
module Plugin
|
|
10
|
+
class DeisOutput < Output
|
|
11
|
+
Fluent::Plugin.register_output('deis', self)
|
|
12
|
+
|
|
13
|
+
include Fluent::Mixin::Deis
|
|
14
|
+
config_param :brokers, :string, default: 'localhost:9092',
|
|
15
|
+
desc: <<-DESC
|
|
16
|
+
Set brokers directly:
|
|
17
|
+
<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
|
|
18
|
+
DESC
|
|
19
|
+
config_param :client_id, :string, default: 'fluentd'
|
|
20
|
+
config_param :metrics_topic, :string, default: 'metrics'
|
|
21
|
+
config_param :discard_kafka_delivery_failed, :bool, default: false
|
|
22
|
+
|
|
23
|
+
# ruby-kafka producer options
|
|
24
|
+
config_param :max_send_retries, :integer, default: 2,
|
|
25
|
+
desc: 'Number of times to retry '\
|
|
26
|
+
'sending of messages to a leader.'
|
|
27
|
+
config_param :required_acks, :integer, default: 1,
|
|
28
|
+
desc: 'The number of acks required per request.'
|
|
29
|
+
config_param :ack_timeout, :time, default: nil,
|
|
30
|
+
desc: 'How long the producer waits for acks.'
|
|
31
|
+
config_param :compression_codec, :string, default: nil,
|
|
32
|
+
desc: <<~DESC
|
|
33
|
+
The codec the producer uses to compress messages.
|
|
34
|
+
Supported codecs: (gzip|snappy)
|
|
35
|
+
DESC
|
|
36
|
+
|
|
37
|
+
config_param :max_send_limit_bytes, :size, default: nil
|
|
38
|
+
config_param :kafka_agg_max_bytes, :size, default: 4 * 1024 # 4k
|
|
39
|
+
config_param :kafka_agg_max_messages, :integer, default: nil
|
|
40
|
+
|
|
41
|
+
define_method('log') { $log } unless method_defined?(:log) # rubocop:disable GlobalVars
|
|
42
|
+
|
|
43
|
+
def initialize
|
|
44
|
+
super
|
|
45
|
+
require 'kafka'
|
|
46
|
+
require 'fluent/plugin/kafka_producer_ext'
|
|
47
|
+
|
|
48
|
+
@kafka = nil
|
|
49
|
+
@producers = {}
|
|
50
|
+
@producers_mutex = Mutex.new
|
|
51
|
+
end
|
|
62
52
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
end
|
|
53
|
+
def start
|
|
54
|
+
super
|
|
55
|
+
refresh_client
|
|
56
|
+
end
|
|
68
57
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
58
|
+
def shutdown
|
|
59
|
+
super
|
|
60
|
+
shutdown_producers
|
|
61
|
+
@kafka = nil
|
|
62
|
+
end
|
|
72
63
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
64
|
+
def shutdown_producers
|
|
65
|
+
@producers_mutex.synchronize do
|
|
66
|
+
@producers.each_value(&:shutdown)
|
|
67
|
+
@producers = {}
|
|
68
|
+
end
|
|
77
69
|
end
|
|
78
|
-
end
|
|
79
70
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
71
|
+
def get_producer # rubocop:disable AccessorMethodName
|
|
72
|
+
@producers_mutex.synchronize do
|
|
73
|
+
producer = @producers[Thread.current.object_id]
|
|
74
|
+
unless producer
|
|
75
|
+
producer = @kafka.producer(@producer_opts)
|
|
76
|
+
@producers[Thread.current.object_id] = producer
|
|
77
|
+
end
|
|
78
|
+
producer
|
|
86
79
|
end
|
|
87
|
-
producer
|
|
88
80
|
end
|
|
89
|
-
end
|
|
90
81
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
82
|
+
def deliver_messages(producer, tag)
|
|
83
|
+
if @discard_kafka_delivery_failed
|
|
84
|
+
begin
|
|
85
|
+
producer.deliver_messages
|
|
86
|
+
rescue Kafka::DeliveryFailed => e
|
|
87
|
+
log.warn 'DeliveryFailed occurred. Discard broken event:',
|
|
88
|
+
error: e.to_s, error_class: e.class.to_s, tag: tag
|
|
89
|
+
producer.clear_buffer
|
|
90
|
+
end
|
|
91
|
+
else
|
|
94
92
|
producer.deliver_messages
|
|
95
|
-
rescue Kafka::DeliveryFailed => e
|
|
96
|
-
log.warn 'DeliveryFailed occurred. Discard broken event:',
|
|
97
|
-
error: e.to_s, error_class: e.class.to_s, tag: tag
|
|
98
|
-
producer.clear_buffer
|
|
99
93
|
end
|
|
100
|
-
else
|
|
101
|
-
producer.deliver_messages
|
|
102
94
|
end
|
|
103
|
-
end
|
|
104
95
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
96
|
+
def refresh_client(raise_error = true)
|
|
97
|
+
@kafka = Kafka.new(seed_brokers: @brokers.split(','), client_id: @client_id)
|
|
98
|
+
log.info "initialized kafka producer: #{@client_id}"
|
|
99
|
+
rescue Exception => e # rubocop:disable RescueException
|
|
100
|
+
raise e if raise_error
|
|
101
|
+
log.error e
|
|
102
|
+
end
|
|
112
103
|
|
|
113
|
-
|
|
114
|
-
|
|
104
|
+
def configure(conf)
|
|
105
|
+
super
|
|
115
106
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
107
|
+
@producer_opts = { max_retries: @max_send_retries, required_acks: @required_acks }
|
|
108
|
+
@producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
|
|
109
|
+
@producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
|
|
119
110
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
111
|
+
return unless @discard_kafka_delivery_failed
|
|
112
|
+
log.warn "'discard_kafka_delivery_failed' option discards events which "\
|
|
113
|
+
'cause delivery failure, e.g. invalid topic or something.'
|
|
114
|
+
log.warn 'If this is unexpected, you need to check your configuration or data.'
|
|
115
|
+
end
|
|
125
116
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
117
|
+
# def emit(tag, es, chain)
|
|
118
|
+
# super(tag, es, chain, tag)
|
|
119
|
+
# end
|
|
120
|
+
|
|
121
|
+
def filter_record(record)
|
|
122
|
+
return unless from_router?(record)
|
|
123
|
+
data = build_series(record)
|
|
124
|
+
return unless data
|
|
125
|
+
return data.map do |point|
|
|
126
|
+
InfluxDB::PointValue.new(point).dump
|
|
127
|
+
end.join("\n")
|
|
128
|
+
rescue Exception => e # rubocop:disable RescueException
|
|
129
|
+
puts "Error:#{e.backtrace}"
|
|
130
|
+
end
|
|
140
131
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
132
|
+
def write(chunk)
|
|
133
|
+
tag = chunk.metadata.tag
|
|
134
|
+
producer = get_producer
|
|
135
|
+
|
|
136
|
+
records_by_topic = {}
|
|
137
|
+
bytes_by_topic = {}
|
|
138
|
+
messages = 0
|
|
139
|
+
messages_bytes = 0
|
|
140
|
+
record_buf = nil
|
|
141
|
+
record_buf_bytes = nil
|
|
142
|
+
begin
|
|
143
|
+
Fluent::Engine.msgpack_factory.unpacker(chunk.open).each do |time, record|
|
|
144
|
+
begin
|
|
145
|
+
topic = @metrics_topic
|
|
146
|
+
records_by_topic[topic] ||= 0
|
|
147
|
+
bytes_by_topic[topic] ||= 0
|
|
148
|
+
line = filter_record(record)
|
|
149
|
+
|
|
150
|
+
next unless line
|
|
151
|
+
record_buf_bytes = line.bytesize
|
|
152
|
+
if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
|
|
153
|
+
log.warn 'record size exceeds max_send_limit_bytes. Skip event:',
|
|
154
|
+
time: time, record: record
|
|
155
|
+
next
|
|
156
|
+
end
|
|
157
|
+
rescue StandardError => e
|
|
158
|
+
log.warn 'unexpected error during format record. Skip broken event:',
|
|
159
|
+
error: e.to_s, error_class: e.class.to_s, time: time, record: record
|
|
164
160
|
next
|
|
165
161
|
end
|
|
166
|
-
rescue StandardError => e
|
|
167
|
-
log.warn 'unexpected error during format record. Skip broken event:',
|
|
168
|
-
error: e.to_s, error_class: e.class.to_s, time: time, record: record
|
|
169
|
-
next
|
|
170
|
-
end
|
|
171
162
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
163
|
+
if messages.positive? &&
|
|
164
|
+
(messages_bytes + record_buf_bytes > @kafka_agg_max_bytes) ||
|
|
165
|
+
(@kafka_agg_max_messages && messages >= @kafka_agg_max_messages)
|
|
166
|
+
log.debug do
|
|
167
|
+
"#{messages} messages send because reaches the limit of batch transmission."
|
|
168
|
+
end
|
|
169
|
+
deliver_messages(producer, tag)
|
|
170
|
+
messages = 0
|
|
171
|
+
messages_bytes = 0
|
|
177
172
|
end
|
|
178
|
-
deliver_messages(producer, tag)
|
|
179
|
-
messages = 0
|
|
180
|
-
messages_bytes = 0
|
|
181
|
-
end
|
|
182
173
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
174
|
+
log.trace do
|
|
175
|
+
"message will send to #{topic} with partition_key: #{partition_key},"\
|
|
176
|
+
"partition: #{partition}, message_key: #{message_key} and value: #{record_buf}."
|
|
177
|
+
end
|
|
187
178
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
179
|
+
messages += 1
|
|
180
|
+
producer.produce2(
|
|
181
|
+
line,
|
|
182
|
+
topic: topic
|
|
183
|
+
)
|
|
184
|
+
messages_bytes += record_buf_bytes
|
|
194
185
|
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
186
|
+
records_by_topic[topic] += 1
|
|
187
|
+
bytes_by_topic[topic] += record_buf_bytes
|
|
188
|
+
end
|
|
189
|
+
if messages.positive?
|
|
190
|
+
log.debug { "#{messages} messages send." }
|
|
191
|
+
deliver_messages(producer, tag)
|
|
192
|
+
end
|
|
193
|
+
log.debug { "(records|bytes) (#{records_by_topic}|#{bytes_by_topic})" }
|
|
201
194
|
end
|
|
202
|
-
|
|
195
|
+
rescue Exception => e # rubocop:disable RescueException
|
|
196
|
+
log.warn "Send exception occurred: #{e}"
|
|
197
|
+
log.warn "Exception Backtrace : #{e.backtrace.join("\n")}"
|
|
198
|
+
# For safety, refresh client and its producers
|
|
199
|
+
shutdown_producers
|
|
200
|
+
refresh_client(false)
|
|
201
|
+
# Raise exception to retry sendind messages
|
|
202
|
+
raise e
|
|
203
203
|
end
|
|
204
|
-
rescue Exception => e # rubocop:disable RescueException
|
|
205
|
-
log.warn "Send exception occurred: #{e}"
|
|
206
|
-
log.warn "Exception Backtrace : #{e.backtrace.join("\n")}"
|
|
207
|
-
# For safety, refresh client and its producers
|
|
208
|
-
shutdown_producers
|
|
209
|
-
refresh_client(false)
|
|
210
|
-
# Raise exception to retry sendind messages
|
|
211
|
-
raise e
|
|
212
204
|
end
|
|
213
205
|
end
|
|
214
206
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-deis-kafka
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Guilherme Souza
|
|
@@ -24,48 +24,6 @@ dependencies:
|
|
|
24
24
|
- - "~>"
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: '0.14'
|
|
27
|
-
- !ruby/object:Gem::Dependency
|
|
28
|
-
name: fluent-mixin-plaintextformatter
|
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
|
30
|
-
requirements:
|
|
31
|
-
- - ">="
|
|
32
|
-
- !ruby/object:Gem::Version
|
|
33
|
-
version: '0'
|
|
34
|
-
type: :runtime
|
|
35
|
-
prerelease: false
|
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
-
requirements:
|
|
38
|
-
- - ">="
|
|
39
|
-
- !ruby/object:Gem::Version
|
|
40
|
-
version: '0'
|
|
41
|
-
- !ruby/object:Gem::Dependency
|
|
42
|
-
name: fluent-mixin-config-placeholders
|
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
|
44
|
-
requirements:
|
|
45
|
-
- - ">="
|
|
46
|
-
- !ruby/object:Gem::Version
|
|
47
|
-
version: '0'
|
|
48
|
-
type: :runtime
|
|
49
|
-
prerelease: false
|
|
50
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
-
requirements:
|
|
52
|
-
- - ">="
|
|
53
|
-
- !ruby/object:Gem::Version
|
|
54
|
-
version: '0'
|
|
55
|
-
- !ruby/object:Gem::Dependency
|
|
56
|
-
name: fluent-mixin-rewrite-tag-name
|
|
57
|
-
requirement: !ruby/object:Gem::Requirement
|
|
58
|
-
requirements:
|
|
59
|
-
- - ">="
|
|
60
|
-
- !ruby/object:Gem::Version
|
|
61
|
-
version: '0'
|
|
62
|
-
type: :runtime
|
|
63
|
-
prerelease: false
|
|
64
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
-
requirements:
|
|
66
|
-
- - ">="
|
|
67
|
-
- !ruby/object:Gem::Version
|
|
68
|
-
version: '0'
|
|
69
27
|
- !ruby/object:Gem::Dependency
|
|
70
28
|
name: influxdb
|
|
71
29
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -114,28 +72,28 @@ dependencies:
|
|
|
114
72
|
requirements:
|
|
115
73
|
- - "~>"
|
|
116
74
|
- !ruby/object:Gem::Version
|
|
117
|
-
version: '
|
|
75
|
+
version: '12.0'
|
|
118
76
|
type: :development
|
|
119
77
|
prerelease: false
|
|
120
78
|
version_requirements: !ruby/object:Gem::Requirement
|
|
121
79
|
requirements:
|
|
122
80
|
- - "~>"
|
|
123
81
|
- !ruby/object:Gem::Version
|
|
124
|
-
version: '
|
|
82
|
+
version: '12.0'
|
|
125
83
|
- !ruby/object:Gem::Dependency
|
|
126
84
|
name: test-unit
|
|
127
85
|
requirement: !ruby/object:Gem::Requirement
|
|
128
86
|
requirements:
|
|
129
87
|
- - "~>"
|
|
130
88
|
- !ruby/object:Gem::Version
|
|
131
|
-
version: 3.
|
|
89
|
+
version: '3.2'
|
|
132
90
|
type: :development
|
|
133
91
|
prerelease: false
|
|
134
92
|
version_requirements: !ruby/object:Gem::Requirement
|
|
135
93
|
requirements:
|
|
136
94
|
- - "~>"
|
|
137
95
|
- !ruby/object:Gem::Version
|
|
138
|
-
version: 3.
|
|
96
|
+
version: '3.2'
|
|
139
97
|
- !ruby/object:Gem::Dependency
|
|
140
98
|
name: rubocop
|
|
141
99
|
requirement: !ruby/object:Gem::Requirement
|