fluent-plugin-kafka 0.11.1 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ChangeLog +4 -0
- data/README.md +14 -3
- data/fluent-plugin-kafka.gemspec +1 -1
- data/lib/fluent/plugin/out_rdkafka2.rb +49 -23
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a32cfbf1850b74e02b8b698db2b3676f89239e647debf62ae7d71d0b3afc229c
|
4
|
+
data.tar.gz: c076287c2ea443fd645193432e93ab6ed0c1263043dac7c171943ac4a268da73
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5bb0d0d1628ad5c3193c1fd9c47af0531330dd6f6359b2cdd6d6e13821b2c023fee9d82b8758a022e6502bf09dc7c594583da2815d87d11dde1e7c0601517bf9
|
7
|
+
data.tar.gz: 05e1f5c8e66ef96a938ee79eacbdd95ecccea3f7fdce76e4888fc38c9b06c5166b57b9df5e7fdc4aeb09bae4099f1fd7907251a6d99cc3a384cb0568939739f6
|
data/ChangeLog
CHANGED
data/README.md
CHANGED
@@ -276,7 +276,7 @@ The configuration format is jsonpath. It is descibed in https://docs.fluentd.org
|
|
276
276
|
|
277
277
|
### Buffered output plugin
|
278
278
|
|
279
|
-
This plugin uses ruby-kafka producer for writing data. This plugin
|
279
|
+
This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12. If you use v1, see `kafka2`.
|
280
280
|
|
281
281
|
<match app.**>
|
282
282
|
@type kafka_buffered
|
@@ -373,8 +373,10 @@ You need to install rdkafka gem.
|
|
373
373
|
default_message_key (string) :default => nil
|
374
374
|
exclude_topic_key (bool) :default => false
|
375
375
|
exclude_partition_key (bool) :default => false
|
376
|
-
|
377
|
-
|
376
|
+
|
377
|
+
# same with kafka2
|
378
|
+
headers (hash) :default => {}
|
379
|
+
headers_from_record (hash) :default => {}
|
378
380
|
|
379
381
|
<format>
|
380
382
|
@type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
|
@@ -396,6 +398,15 @@ You need to install rdkafka gem.
|
|
396
398
|
rdkafka_options {
|
397
399
|
"log_level" : 7
|
398
400
|
}
|
401
|
+
|
402
|
+
# rdkafka2 specific parameters
|
403
|
+
|
404
|
+
# share kafka producer between flush threads. This is mainly for reducing kafka operations like kerberos
|
405
|
+
share_producer (bool) :default => false
|
406
|
+
# Timeout for polling message wait. If 0, no wait.
|
407
|
+
rdkafka_delivery_handle_poll_timeout (integer) :default => 30
|
408
|
+
# If the record size is larger than this value, such records are ignored. Default is no limit
|
409
|
+
max_send_limit_bytes (integer) :default => nil
|
399
410
|
</match>
|
400
411
|
|
401
412
|
If you use v0.12, use `rdkafka` instead.
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
|
|
13
13
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
14
|
gem.name = "fluent-plugin-kafka"
|
15
15
|
gem.require_paths = ["lib"]
|
16
|
-
gem.version = '0.11.
|
16
|
+
gem.version = '0.11.2'
|
17
17
|
gem.required_ruby_version = ">= 2.1.0"
|
18
18
|
|
19
19
|
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
@@ -77,6 +77,7 @@ DESC
|
|
77
77
|
config_param :rdkafka_message_max_num, :integer, :default => nil, :desc => 'Used for batch.num.messages'
|
78
78
|
config_param :rdkafka_delivery_handle_poll_timeout, :integer, :default => 30, :desc => 'Timeout for polling message wait'
|
79
79
|
config_param :rdkafka_options, :hash, :default => {}, :desc => 'Set any rdkafka configuration. See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md'
|
80
|
+
config_param :share_producer, :bool, :default => false, :desc => 'share kafka producer between flush threads'
|
80
81
|
|
81
82
|
config_param :max_enqueue_retries, :integer, :default => 3
|
82
83
|
config_param :enqueue_retry_backoff, :integer, :default => 3
|
@@ -97,8 +98,10 @@ DESC
|
|
97
98
|
|
98
99
|
def initialize
|
99
100
|
super
|
100
|
-
|
101
|
-
@
|
101
|
+
|
102
|
+
@producers = nil
|
103
|
+
@producers_mutex = nil
|
104
|
+
@shared_producer = nil
|
102
105
|
end
|
103
106
|
|
104
107
|
def configure(conf)
|
@@ -187,6 +190,13 @@ DESC
|
|
187
190
|
end
|
188
191
|
|
189
192
|
def start
|
193
|
+
if @share_producer
|
194
|
+
@shared_producer = @rdkafka.producer
|
195
|
+
else
|
196
|
+
@producers = {}
|
197
|
+
@producers_mutex = Mutex.new
|
198
|
+
end
|
199
|
+
|
190
200
|
super
|
191
201
|
end
|
192
202
|
|
@@ -200,30 +210,43 @@ DESC
|
|
200
210
|
end
|
201
211
|
|
202
212
|
def shutdown_producers
|
203
|
-
@
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
213
|
+
if @share_producer
|
214
|
+
close_producer(@shared_producer)
|
215
|
+
@shared_producer = nil
|
216
|
+
else
|
217
|
+
@producers_mutex.synchronize {
|
218
|
+
shutdown_threads = @producers.map { |key, producer|
|
219
|
+
th = Thread.new {
|
220
|
+
close_producer(producer)
|
221
|
+
}
|
222
|
+
th.abort_on_exception = true
|
223
|
+
th
|
209
224
|
}
|
210
|
-
th.
|
211
|
-
|
225
|
+
shutdown_threads.each { |th| th.join }
|
226
|
+
@producers = {}
|
212
227
|
}
|
213
|
-
|
214
|
-
|
215
|
-
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
def close_producer(producer)
|
232
|
+
unless producer.close(10)
|
233
|
+
log.warn("Queue is forcefully closed after 10 seconds wait")
|
234
|
+
end
|
216
235
|
end
|
217
236
|
|
218
237
|
def get_producer
|
219
|
-
@
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
@producers[Thread.current.object_id]
|
224
|
-
|
225
|
-
|
226
|
-
|
238
|
+
if @share_producer
|
239
|
+
@shared_producer
|
240
|
+
else
|
241
|
+
@producers_mutex.synchronize {
|
242
|
+
producer = @producers[Thread.current.object_id]
|
243
|
+
unless producer
|
244
|
+
producer = @rdkafka.producer
|
245
|
+
@producers[Thread.current.object_id] = producer
|
246
|
+
end
|
247
|
+
producer
|
248
|
+
}
|
249
|
+
end
|
227
250
|
end
|
228
251
|
|
229
252
|
def setup_formatter(conf)
|
@@ -272,10 +295,13 @@ DESC
|
|
272
295
|
next
|
273
296
|
end
|
274
297
|
|
275
|
-
|
298
|
+
handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers)
|
299
|
+
if @rdkafka_delivery_handle_poll_timeout != 0
|
300
|
+
handlers << handler
|
301
|
+
end
|
276
302
|
}
|
277
303
|
handlers.each { |handler|
|
278
|
-
handler.wait(@rdkafka_delivery_handle_poll_timeout)
|
304
|
+
handler.wait(@rdkafka_delivery_handle_poll_timeout)
|
279
305
|
}
|
280
306
|
end
|
281
307
|
rescue Exception => e
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.11.
|
4
|
+
version: 0.11.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-
|
12
|
+
date: 2019-09-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|