fluent-plugin-kafka 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/fluent-plugin-kafka.gemspec +2 -1
- data/lib/fluent/plugin/out_kafka_buffered.rb +82 -26
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dfe217eaf0f6bd1b83a27e0ab49bbc959b8aa311
|
4
|
+
data.tar.gz: d45ade2033759a9c41f2ec98354f792db1302e65
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 04e02ee2614432dc8e0a07e99ef2cb25cf86e764b4e1c37e2b2b719e0e4592d890539b74d5b4ca2def0b3dace18770c28b7eb08bb797fead911b7e6188cf0083
|
7
|
+
data.tar.gz: 0192de37ab4a8408a8fe22effa2a71d7bf0864f553fcc9ea139d2a9146f5b8b26f3e7702d5579bff2b2e4f12b4a66f4b12bec66f7c0c9e45459d44c2e5571eaf
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
|
|
12
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
13
|
gem.name = "fluent-plugin-kafka"
|
14
14
|
gem.require_paths = ["lib"]
|
15
|
-
gem.version = '0.
|
15
|
+
gem.version = '0.2.0'
|
16
16
|
gem.add_development_dependency 'test-unit'
|
17
17
|
gem.add_dependency 'fluentd'
|
18
18
|
gem.add_dependency 'poseidon_cluster'
|
@@ -20,4 +20,5 @@ Gem::Specification.new do |gem|
|
|
20
20
|
gem.add_dependency 'yajl-ruby'
|
21
21
|
gem.add_dependency 'msgpack'
|
22
22
|
gem.add_dependency 'zookeeper'
|
23
|
+
gem.add_dependency 'ruby-kafka', '~> 0.3.2'
|
23
24
|
end
|
@@ -1,10 +1,17 @@
|
|
1
1
|
# encode: utf-8
|
2
|
+
require 'thread'
|
3
|
+
|
2
4
|
class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
|
3
5
|
Fluent::Plugin.register_output('kafka_buffered', self)
|
4
6
|
|
5
7
|
def initialize
|
6
8
|
super
|
7
|
-
|
9
|
+
|
10
|
+
require 'kafka'
|
11
|
+
|
12
|
+
@kafka = nil
|
13
|
+
@producers = {}
|
14
|
+
@producers_mutex = Mutex.new
|
8
15
|
end
|
9
16
|
|
10
17
|
config_param :brokers, :string, :default => 'localhost:9092',
|
@@ -32,17 +39,25 @@ DESC
|
|
32
39
|
config_param :output_include_time, :bool, :default => false
|
33
40
|
config_param :kafka_agg_max_bytes, :size, :default => 4*1024 #4k
|
34
41
|
|
42
|
+
# https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
|
43
|
+
config_param :ssl_ca_cert, :string, :default => nil,
|
44
|
+
:desc => "a PEM encoded CA cert to use with and SSL connection."
|
45
|
+
config_param :ssl_client_cert, :string, :default => nil,
|
46
|
+
:desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
|
47
|
+
config_param :ssl_client_cert_key, :string, :default => nil,
|
48
|
+
:desc => "a PEM encoded client cert key to use with and SSL connection. Must be used in combination with ssl_client_cert."
|
49
|
+
|
35
50
|
# poseidon producer options
|
36
|
-
config_param :max_send_retries, :integer, :default =>
|
51
|
+
config_param :max_send_retries, :integer, :default => 1,
|
37
52
|
:desc => "Number of times to retry sending of messages to a leader."
|
38
53
|
config_param :required_acks, :integer, :default => 0,
|
39
54
|
:desc => "The number of acks required per request."
|
40
|
-
config_param :
|
55
|
+
config_param :ack_timeout, :time, :default => nil,
|
41
56
|
:desc => "How long the producer waits for acks."
|
42
|
-
config_param :compression_codec, :string, :default =>
|
57
|
+
config_param :compression_codec, :string, :default => nil,
|
43
58
|
:desc => <<-DESC
|
44
59
|
The codec the producer uses to compress messages.
|
45
|
-
Supported codecs: (
|
60
|
+
Supported codecs: (gzip|snappy)
|
46
61
|
DESC
|
47
62
|
|
48
63
|
config_param :time_format, :string, :default => nil
|
@@ -54,9 +69,7 @@ DESC
|
|
54
69
|
define_method("log") { $log }
|
55
70
|
end
|
56
71
|
|
57
|
-
|
58
|
-
|
59
|
-
def refresh_producer()
|
72
|
+
def refresh_client(raise_error = true)
|
60
73
|
if @zookeeper
|
61
74
|
@seed_brokers = []
|
62
75
|
z = Zookeeper.new(@zookeeper)
|
@@ -69,27 +82,39 @@ DESC
|
|
69
82
|
end
|
70
83
|
begin
|
71
84
|
if @seed_brokers.length > 0
|
72
|
-
@
|
73
|
-
|
85
|
+
@kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
|
86
|
+
ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
|
87
|
+
log.info "initialized kafka producer: #{@client_id}"
|
74
88
|
else
|
75
89
|
log.warn "No brokers found on Zookeeper"
|
76
90
|
end
|
77
91
|
rescue Exception => e
|
78
|
-
|
92
|
+
if raise_error # During startup, error should be reported to engine and stop its phase for safety.
|
93
|
+
raise e
|
94
|
+
else
|
95
|
+
log.error e
|
96
|
+
end
|
79
97
|
end
|
80
98
|
end
|
81
99
|
|
100
|
+
def read_ssl_file(path)
|
101
|
+
return nil if path.nil?
|
102
|
+
File.read(path)
|
103
|
+
end
|
104
|
+
|
82
105
|
def configure(conf)
|
83
106
|
super
|
107
|
+
|
84
108
|
if @zookeeper
|
85
109
|
require 'zookeeper'
|
86
|
-
require 'yajl'
|
87
110
|
else
|
88
111
|
@seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
|
89
112
|
log.info "brokers has been set directly: #{@seed_brokers}"
|
90
113
|
end
|
91
|
-
|
92
|
-
|
114
|
+
|
115
|
+
if conf['ack_timeout_ms']
|
116
|
+
log.warn "'ack_timeout_ms' parameter is deprecated. Use second unit 'ack_timeout' instead"
|
117
|
+
@ack_timeout = conf['ack_timeout_ms'].to_i / 1000
|
93
118
|
end
|
94
119
|
|
95
120
|
@f_separator = case @field_separator
|
@@ -100,21 +125,48 @@ DESC
|
|
100
125
|
end
|
101
126
|
|
102
127
|
@formatter_proc = setup_formatter(conf)
|
128
|
+
|
129
|
+
@producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks,
|
130
|
+
max_buffer_size: @buffer.buffer_chunk_limit / 10, max_buffer_bytesize: @buffer.buffer_chunk_limit * 2}
|
131
|
+
@producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
|
132
|
+
@producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
|
103
133
|
end
|
104
134
|
|
105
135
|
def start
|
106
136
|
super
|
107
|
-
|
137
|
+
refresh_client
|
108
138
|
end
|
109
139
|
|
110
140
|
def shutdown
|
111
141
|
super
|
142
|
+
shutdown_producers
|
143
|
+
@kafka = nil
|
112
144
|
end
|
113
145
|
|
114
146
|
def format(tag, time, record)
|
115
147
|
[tag, time, record].to_msgpack
|
116
148
|
end
|
117
149
|
|
150
|
+
def shutdown_producers
|
151
|
+
@producers_mutex.synchronize {
|
152
|
+
@producers.each { |key, producer|
|
153
|
+
producer.shutdown
|
154
|
+
}
|
155
|
+
@producers = {}
|
156
|
+
}
|
157
|
+
end
|
158
|
+
|
159
|
+
def get_producer
|
160
|
+
@producers_mutex.synchronize {
|
161
|
+
producer = @producers[Thread.current.object_id]
|
162
|
+
unless producer
|
163
|
+
producer = @kafka.producer(@producer_opts)
|
164
|
+
@producers[Thread.current.object_id] = producer
|
165
|
+
end
|
166
|
+
producer
|
167
|
+
}
|
168
|
+
end
|
169
|
+
|
118
170
|
def setup_formatter(conf)
|
119
171
|
if @output_data_type == 'json'
|
120
172
|
require 'yajl'
|
@@ -144,9 +196,11 @@ DESC
|
|
144
196
|
end
|
145
197
|
|
146
198
|
def write(chunk)
|
199
|
+
producer = get_producer
|
200
|
+
|
147
201
|
records_by_topic = {}
|
148
202
|
bytes_by_topic = {}
|
149
|
-
messages =
|
203
|
+
messages = 0
|
150
204
|
messages_bytes = 0
|
151
205
|
begin
|
152
206
|
chunk.msgpack_each { |tag, time, record|
|
@@ -167,29 +221,31 @@ DESC
|
|
167
221
|
|
168
222
|
record_buf = @formatter_proc.call(tag, time, record)
|
169
223
|
record_buf_bytes = record_buf.bytesize
|
170
|
-
if messages
|
171
|
-
log.on_trace { log.trace("#{messages
|
172
|
-
|
173
|
-
messages =
|
224
|
+
if (messages > 0) and (messages_bytes + record_buf_bytes > @kafka_agg_max_bytes)
|
225
|
+
log.on_trace { log.trace("#{messages} messages send.") }
|
226
|
+
producer.deliver_messages
|
227
|
+
messages = 0
|
174
228
|
messages_bytes = 0
|
175
229
|
end
|
176
230
|
log.on_trace { log.trace("message will send to #{topic} with key: #{partition_key} and value: #{record_buf}.") }
|
177
|
-
messages
|
231
|
+
messages += 1
|
232
|
+
producer.produce(record_buf, topic: topic, partition_key: partition_key)
|
178
233
|
messages_bytes += record_buf_bytes
|
179
234
|
|
180
235
|
records_by_topic[topic] += 1
|
181
236
|
bytes_by_topic[topic] += record_buf_bytes
|
182
237
|
}
|
183
|
-
if messages
|
184
|
-
log.trace("#{messages
|
185
|
-
|
238
|
+
if messages > 0
|
239
|
+
log.trace("#{messages} messages send.")
|
240
|
+
producer.deliver_messages
|
186
241
|
end
|
187
242
|
log.debug "(records|bytes) (#{records_by_topic}|#{bytes_by_topic})"
|
188
243
|
end
|
189
244
|
rescue Exception => e
|
190
245
|
log.warn "Send exception occurred: #{e}"
|
191
|
-
|
192
|
-
|
246
|
+
# For safety, refresh client and its producers
|
247
|
+
shutdown_producers
|
248
|
+
refresh_client(false)
|
193
249
|
# Raise exception to retry sendind messages
|
194
250
|
raise e
|
195
251
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-06-
|
11
|
+
date: 2016-06-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|
@@ -108,6 +108,20 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: ruby-kafka
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 0.3.2
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: 0.3.2
|
111
125
|
description: Fluentd plugin for Apache Kafka > 0.8
|
112
126
|
email:
|
113
127
|
- togachiro@gmail.com
|