fluent-plugin-kafka 0.7.9 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ChangeLog +6 -0
- data/README.md +23 -3
- data/fluent-plugin-kafka.gemspec +2 -2
- data/lib/fluent/plugin/kafka_producer_ext.rb +104 -19
- data/lib/fluent/plugin/out_kafka.rb +5 -1
- data/lib/fluent/plugin/out_kafka2.rb +6 -2
- data/lib/fluent/plugin/out_kafka_buffered.rb +6 -2
- data/lib/fluent/plugin/out_rdkafka.rb +301 -0
- metadata +7 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: faa29645a4c13d7400459f16cbf7720e3bf8a3cc
|
4
|
+
data.tar.gz: 56a4b88bbfc9f42efe9f87bc845552bc17b1f019
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 13253507831d6f1e07e0937f3b4d01e01ffa2ea60605ad3cb1c326f3a4606c3a1173d07d3688250b8866ae3a20a69662c05e688ba746cc677c448a643082e528
|
7
|
+
data.tar.gz: 4fb3eb6ed6ffd61b12efa4e8da930a573b5420a4681885a66b54c9052a5907bc39ad6c17d6d65d7aefaa9f8546fe8453cee199ad52a48a1d4a799636b8f0165f
|
data/ChangeLog
CHANGED
data/README.md
CHANGED
@@ -18,7 +18,7 @@ And then execute:
|
|
18
18
|
|
19
19
|
Or install it yourself as:
|
20
20
|
|
21
|
-
$ gem install fluent-plugin-kafka
|
21
|
+
$ gem install fluent-plugin-kafka --no-document
|
22
22
|
|
23
23
|
If you want to use zookeeper related parameters, you also need to install zookeeper gem. zookeeper gem includes native extension, so development tools are needed, e.g. gcc, make and etc.
|
24
24
|
|
@@ -192,7 +192,7 @@ See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) f
|
|
192
192
|
This plugin supports compression codec "snappy" also.
|
193
193
|
Install snappy module before you use snappy compression.
|
194
194
|
|
195
|
-
$ gem install snappy
|
195
|
+
$ gem install snappy --no-document
|
196
196
|
|
197
197
|
snappy gem uses native extension, so you need to install several packages before.
|
198
198
|
On Ubuntu, need development packages and snappy library.
|
@@ -219,7 +219,7 @@ If key name `message_key` exists in a message, this plugin publishes the value o
|
|
219
219
|
|
220
220
|
### Output plugin
|
221
221
|
|
222
|
-
This plugin is for v1.0. This will be `out_kafka` plugin in the future.
|
222
|
+
This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
|
223
223
|
|
224
224
|
<match app.**>
|
225
225
|
@type kafka2
|
@@ -290,6 +290,26 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
|
|
290
290
|
|
291
291
|
This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.
|
292
292
|
|
293
|
+
### rdkafka based output plugin
|
294
|
+
|
295
|
+
This plugin uses `rdkafka` instead of `ruby-kafka` for ruby client.
|
296
|
+
You need to install rdkafka gem.
|
297
|
+
|
298
|
+
# rdkafka is C extension library so need development tools like ruby-devel, gcc and etc
|
299
|
+
$ gem install rdkafka --no-document
|
300
|
+
|
301
|
+
<match kafka.**>
|
302
|
+
@type rdkafka
|
303
|
+
|
304
|
+
default_topic kafka
|
305
|
+
flush_interval 1s
|
306
|
+
output_data_type json
|
307
|
+
|
308
|
+
rdkafka_options {
|
309
|
+
"log_level" : 7
|
310
|
+
}
|
311
|
+
</match>
|
312
|
+
|
293
313
|
## Contributing
|
294
314
|
|
295
315
|
1. Fork it
|
data/fluent-plugin-kafka.gemspec
CHANGED
@@ -13,12 +13,12 @@ Gem::Specification.new do |gem|
|
|
13
13
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
14
|
gem.name = "fluent-plugin-kafka"
|
15
15
|
gem.require_paths = ["lib"]
|
16
|
-
gem.version = '0.
|
16
|
+
gem.version = '0.8.0'
|
17
17
|
gem.required_ruby_version = ">= 2.1.0"
|
18
18
|
|
19
19
|
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
20
20
|
gem.add_dependency 'ltsv'
|
21
|
-
gem.add_dependency 'ruby-kafka', '>= 0.
|
21
|
+
gem.add_dependency 'ruby-kafka', '>= 0.7.1', '< 0.8.0'
|
22
22
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
23
23
|
gem.add_development_dependency "test-unit", ">= 3.0.8"
|
24
24
|
end
|
@@ -9,19 +9,28 @@ require 'kafka/producer'
|
|
9
9
|
|
10
10
|
# for out_kafka_buffered
|
11
11
|
module Kafka
|
12
|
+
EMPTY_HEADER = {}
|
13
|
+
|
12
14
|
class Producer
|
13
|
-
def
|
15
|
+
def produce_for_buffered(value, key: nil, topic:, partition: nil, partition_key: nil)
|
14
16
|
create_time = Time.now
|
15
17
|
|
16
18
|
message = PendingMessage.new(
|
17
|
-
value,
|
18
|
-
key,
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
19
|
+
value: value,
|
20
|
+
key: key,
|
21
|
+
headers: EMPTY_HEADER,
|
22
|
+
topic: topic,
|
23
|
+
partition: partition,
|
24
|
+
partition_key: partition_key,
|
25
|
+
create_time: create_time
|
23
26
|
)
|
24
27
|
|
28
|
+
# If the producer is in transactional mode, all the message production
|
29
|
+
# must be used when the producer is currently in transaction
|
30
|
+
if @transaction_manager.transactional? && !@transaction_manager.in_transaction?
|
31
|
+
raise 'You must trigger begin_transaction before producing messages'
|
32
|
+
end
|
33
|
+
|
25
34
|
@target_topics.add(topic)
|
26
35
|
@pending_message_queue.write(message)
|
27
36
|
|
@@ -33,15 +42,26 @@ end
|
|
33
42
|
# for out_kafka2
|
34
43
|
module Kafka
|
35
44
|
class Client
|
36
|
-
def topic_producer(topic, compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000)
|
45
|
+
def topic_producer(topic, compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000, idempotent: false, transactional: false, transactional_id: nil, transactional_timeout: 60)
|
46
|
+
cluster = initialize_cluster
|
37
47
|
compressor = Compressor.new(
|
38
48
|
codec_name: compression_codec,
|
39
49
|
threshold: compression_threshold,
|
40
50
|
instrumenter: @instrumenter,
|
41
51
|
)
|
42
52
|
|
53
|
+
transaction_manager = TransactionManager.new(
|
54
|
+
cluster: cluster,
|
55
|
+
logger: @logger,
|
56
|
+
idempotent: idempotent,
|
57
|
+
transactional: transactional,
|
58
|
+
transactional_id: transactional_id,
|
59
|
+
transactional_timeout: transactional_timeout,
|
60
|
+
)
|
61
|
+
|
43
62
|
TopicProducer.new(topic,
|
44
|
-
cluster:
|
63
|
+
cluster: cluster,
|
64
|
+
transaction_manager: transaction_manager,
|
45
65
|
logger: @logger,
|
46
66
|
instrumenter: @instrumenter,
|
47
67
|
compressor: compressor,
|
@@ -56,8 +76,9 @@ module Kafka
|
|
56
76
|
end
|
57
77
|
|
58
78
|
class TopicProducer
|
59
|
-
def initialize(topic, cluster:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
|
79
|
+
def initialize(topic, cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
|
60
80
|
@cluster = cluster
|
81
|
+
@transaction_manager = transaction_manager
|
61
82
|
@logger = logger
|
62
83
|
@instrumenter = instrumenter
|
63
84
|
@required_acks = required_acks == :all ? -1 : required_acks
|
@@ -78,18 +99,25 @@ module Kafka
|
|
78
99
|
@pending_message_queue = PendingMessageQueue.new
|
79
100
|
end
|
80
101
|
|
81
|
-
def produce(value, key, partition, partition_key)
|
102
|
+
def produce(value, key: nil, partition: nil, partition_key: nil)
|
82
103
|
create_time = Time.now
|
83
104
|
|
84
105
|
message = PendingMessage.new(
|
85
|
-
value,
|
86
|
-
key,
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
106
|
+
value: value,
|
107
|
+
key: key,
|
108
|
+
headers: EMPTY_HEADER,
|
109
|
+
topic: @topic,
|
110
|
+
partition: partition,
|
111
|
+
partition_key: partition_key,
|
112
|
+
create_time: create_time
|
91
113
|
)
|
92
114
|
|
115
|
+
# If the producer is in transactional mode, all the message production
|
116
|
+
# must be used when the producer is currently in transaction
|
117
|
+
if @transaction_manager.transactional? && !@transaction_manager.in_transaction?
|
118
|
+
raise 'You must trigger begin_transaction before producing messages'
|
119
|
+
end
|
120
|
+
|
93
121
|
@pending_message_queue.write(message)
|
94
122
|
|
95
123
|
nil
|
@@ -125,10 +153,37 @@ module Kafka
|
|
125
153
|
#
|
126
154
|
# @return [nil]
|
127
155
|
def shutdown
|
156
|
+
@transaction_manager.close
|
128
157
|
@cluster.disconnect
|
129
158
|
end
|
130
159
|
|
131
|
-
|
160
|
+
def init_transactions
|
161
|
+
@transaction_manager.init_transactions
|
162
|
+
end
|
163
|
+
|
164
|
+
def begin_transaction
|
165
|
+
@transaction_manager.begin_transaction
|
166
|
+
end
|
167
|
+
|
168
|
+
def commit_transaction
|
169
|
+
@transaction_manager.commit_transaction
|
170
|
+
end
|
171
|
+
|
172
|
+
def abort_transaction
|
173
|
+
@transaction_manager.abort_transaction
|
174
|
+
end
|
175
|
+
|
176
|
+
def transaction
|
177
|
+
raise 'This method requires a block' unless block_given?
|
178
|
+
begin_transaction
|
179
|
+
yield
|
180
|
+
commit_transaction
|
181
|
+
rescue Kafka::Producer::AbortTransaction
|
182
|
+
abort_transaction
|
183
|
+
rescue
|
184
|
+
abort_transaction
|
185
|
+
raise
|
186
|
+
end
|
132
187
|
|
133
188
|
def deliver_messages_with_retries
|
134
189
|
attempt = 0
|
@@ -137,6 +192,7 @@ module Kafka
|
|
137
192
|
|
138
193
|
operation = ProduceOperation.new(
|
139
194
|
cluster: @cluster,
|
195
|
+
transaction_manager: @transaction_manager,
|
140
196
|
buffer: @buffer,
|
141
197
|
required_acks: @required_acks,
|
142
198
|
ack_timeout: @ack_timeout,
|
@@ -148,7 +204,11 @@ module Kafka
|
|
148
204
|
loop do
|
149
205
|
attempt += 1
|
150
206
|
|
151
|
-
|
207
|
+
begin
|
208
|
+
@cluster.refresh_metadata_if_necessary!
|
209
|
+
rescue ConnectionError => e
|
210
|
+
raise DeliveryFailed.new(e, buffer_messages)
|
211
|
+
end
|
152
212
|
|
153
213
|
assign_partitions!
|
154
214
|
operation.execute
|
@@ -200,6 +260,7 @@ module Kafka
|
|
200
260
|
@buffer.write(
|
201
261
|
value: message.value,
|
202
262
|
key: message.key,
|
263
|
+
headers: message.headers,
|
203
264
|
topic: message.topic,
|
204
265
|
partition: partition,
|
205
266
|
create_time: message.create_time,
|
@@ -219,5 +280,29 @@ module Kafka
|
|
219
280
|
|
220
281
|
@pending_message_queue.replace(failed_messages)
|
221
282
|
end
|
283
|
+
|
284
|
+
def buffer_messages
|
285
|
+
messages = []
|
286
|
+
|
287
|
+
@pending_message_queue.each do |message|
|
288
|
+
messages << message
|
289
|
+
end
|
290
|
+
|
291
|
+
@buffer.each do |topic, partition, messages_for_partition|
|
292
|
+
messages_for_partition.each do |message|
|
293
|
+
messages << PendingMessage.new(
|
294
|
+
value: message.value,
|
295
|
+
key: message.key,
|
296
|
+
headers: message.headers,
|
297
|
+
topic: topic,
|
298
|
+
partition: partition,
|
299
|
+
partition_key: nil,
|
300
|
+
create_time: message.create_time
|
301
|
+
)
|
302
|
+
end
|
303
|
+
end
|
304
|
+
|
305
|
+
messages
|
306
|
+
end
|
222
307
|
end
|
223
308
|
end
|
@@ -20,6 +20,10 @@ DESC
|
|
20
20
|
config_param :default_partition_key, :string, :default => nil
|
21
21
|
config_param :default_partition, :integer, :default => nil
|
22
22
|
config_param :client_id, :string, :default => 'kafka'
|
23
|
+
config_param :sasl_over_ssl, :bool, :default => true,
|
24
|
+
:desc => <<-DESC
|
25
|
+
Set to false to prevent SSL strict mode when using SASL authentication
|
26
|
+
DESC
|
23
27
|
config_param :output_data_type, :string, :default => 'json',
|
24
28
|
:desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
|
25
29
|
config_param :output_include_tag, :bool, :default => false
|
@@ -104,7 +108,7 @@ DESC
|
|
104
108
|
if @scram_mechanism != nil && @username != nil && @password != nil
|
105
109
|
@kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
|
106
110
|
ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
|
107
|
-
sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
|
111
|
+
sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
|
108
112
|
elsif @username != nil && @password != nil
|
109
113
|
@kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
|
110
114
|
ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
|
@@ -25,6 +25,10 @@ DESC
|
|
25
25
|
config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
|
26
26
|
config_param :default_partition, :integer, :default => nil
|
27
27
|
config_param :client_id, :string, :default => 'fluentd'
|
28
|
+
config_param :sasl_over_ssl, :bool, :default => true,
|
29
|
+
:desc => <<-DESC
|
30
|
+
Set to false to prevent SSL strict mode when using SASL authentication
|
31
|
+
DESC
|
28
32
|
config_param :exclude_partition_key, :bool, :default => false,
|
29
33
|
:desc => 'Set true to remove partition key from data'
|
30
34
|
config_param :exclude_partition, :bool, :default => false,
|
@@ -80,7 +84,7 @@ DESC
|
|
80
84
|
if @scram_mechanism != nil && @username != nil && @password != nil
|
81
85
|
@kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
|
82
86
|
ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
|
83
|
-
sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
|
87
|
+
sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
|
84
88
|
elsif @username != nil && @password != nil
|
85
89
|
@kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
|
86
90
|
ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
|
@@ -212,7 +216,7 @@ DESC
|
|
212
216
|
log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }
|
213
217
|
messages += 1
|
214
218
|
|
215
|
-
producer.produce(record_buf, message_key,
|
219
|
+
producer.produce(record_buf, key: message_key, partition_key: partition_key, partition: partition)
|
216
220
|
}
|
217
221
|
|
218
222
|
if messages > 0
|
@@ -27,6 +27,10 @@ DESC
|
|
27
27
|
config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
|
28
28
|
config_param :default_partition, :integer, :default => nil
|
29
29
|
config_param :client_id, :string, :default => 'kafka'
|
30
|
+
config_param :sasl_over_ssl, :bool, :default => true,
|
31
|
+
:desc => <<-DESC
|
32
|
+
Set to false to prevent SSL strict mode when using SASL authentication
|
33
|
+
DESC
|
30
34
|
config_param :output_data_type, :string, :default => 'json',
|
31
35
|
:desc => <<-DESC
|
32
36
|
Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)
|
@@ -126,7 +130,7 @@ DESC
|
|
126
130
|
if @scram_mechanism != nil && @username != nil && @password != nil
|
127
131
|
@kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
|
128
132
|
ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
|
129
|
-
sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
|
133
|
+
sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
|
130
134
|
elsif @username != nil && @password != nil
|
131
135
|
@kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
|
132
136
|
ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
|
@@ -333,7 +337,7 @@ DESC
|
|
333
337
|
end
|
334
338
|
log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }
|
335
339
|
messages += 1
|
336
|
-
producer.
|
340
|
+
producer.produce_for_buffered(record_buf, topic: topic, key: message_key, partition_key: partition_key, partition: partition)
|
337
341
|
messages_bytes += record_buf_bytes
|
338
342
|
|
339
343
|
records_by_topic[topic] += 1
|
@@ -0,0 +1,301 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require 'fluent/output'
|
3
|
+
require 'fluent/plugin/kafka_plugin_util'
|
4
|
+
|
5
|
+
require 'rdkafka'
|
6
|
+
require 'fluent/plugin/kafka_producer_ext'
|
7
|
+
|
8
|
+
class Rdkafka::Producer
|
9
|
+
# return false if producer is forcefully closed, otherwise return true
|
10
|
+
def close(timeout = nil)
|
11
|
+
@closing = true
|
12
|
+
# Wait for the polling thread to finish up
|
13
|
+
# If the broker isn't alive, the thread doesn't exit
|
14
|
+
if timeout
|
15
|
+
thr = @polling_thread.join(timeout)
|
16
|
+
return !!thr
|
17
|
+
else
|
18
|
+
@polling_thread.join
|
19
|
+
return true
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class Fluent::KafkaOutputBuffered2 < Fluent::BufferedOutput
|
25
|
+
Fluent::Plugin.register_output('rdkafka', self)
|
26
|
+
|
27
|
+
config_param :brokers, :string, :default => 'localhost:9092',
|
28
|
+
:desc => <<-DESC
|
29
|
+
Set brokers directly:
|
30
|
+
<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
|
31
|
+
Brokers: you can choose to use either brokers or zookeeper.
|
32
|
+
DESC
|
33
|
+
config_param :default_topic, :string, :default => nil,
|
34
|
+
:desc => "Output topic"
|
35
|
+
config_param :default_message_key, :string, :default => nil
|
36
|
+
config_param :default_partition, :integer, :default => nil
|
37
|
+
config_param :client_id, :string, :default => 'kafka'
|
38
|
+
config_param :output_data_type, :string, :default => 'json',
|
39
|
+
:desc => <<-DESC
|
40
|
+
Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)
|
41
|
+
DESC
|
42
|
+
config_param :output_include_tag, :bool, :default => false
|
43
|
+
config_param :output_include_time, :bool, :default => false
|
44
|
+
config_param :exclude_partition, :bool, :default => false,
|
45
|
+
:desc => <<-DESC
|
46
|
+
Set true to remove partition from data
|
47
|
+
DESC
|
48
|
+
config_param :exclude_message_key, :bool, :default => false,
|
49
|
+
:desc => <<-DESC
|
50
|
+
Set true to remove partition key from data
|
51
|
+
DESC
|
52
|
+
config_param :exclude_topic_key, :bool, :default => false,
|
53
|
+
:desc => <<-DESC
|
54
|
+
Set true to remove topic name key from data
|
55
|
+
DESC
|
56
|
+
config_param :max_send_retries, :integer, :default => 2,
|
57
|
+
:desc => "Number of times to retry sending of messages to a leader."
|
58
|
+
config_param :required_acks, :integer, :default => -1,
|
59
|
+
:desc => "The number of acks required per request."
|
60
|
+
config_param :ack_timeout, :time, :default => nil,
|
61
|
+
:desc => "How long the producer waits for acks."
|
62
|
+
config_param :compression_codec, :string, :default => nil,
|
63
|
+
:desc => <<-DESC
|
64
|
+
The codec the producer uses to compress messages.
|
65
|
+
Supported codecs: (gzip|snappy)
|
66
|
+
DESC
|
67
|
+
|
68
|
+
config_param :rdkafka_buffering_max_ms, :integer, :default => nil
|
69
|
+
config_param :rdkafka_buffering_max_messages, :integer, :default => nil
|
70
|
+
config_param :rdkafka_message_max_bytes, :integer, :default => nil
|
71
|
+
config_param :rdkafka_message_max_num, :integer, :default => nil
|
72
|
+
config_param :rdkafka_delivery_handle_poll_timeout, :integer, :default => 30
|
73
|
+
config_param :rdkafka_options, :hash, :default => {}
|
74
|
+
|
75
|
+
config_param :max_enqueue_retries, :integer, :default => 3
|
76
|
+
config_param :enqueue_retry_backoff, :integer, :default => 3
|
77
|
+
|
78
|
+
config_param :service_name, :string, :default => nil
|
79
|
+
config_param :ssl_client_cert_key_password, :string, :default => nil
|
80
|
+
|
81
|
+
include Fluent::KafkaPluginUtil::SSLSettings
|
82
|
+
include Fluent::KafkaPluginUtil::SaslSettings
|
83
|
+
|
84
|
+
def initialize
|
85
|
+
super
|
86
|
+
@producers = {}
|
87
|
+
@producers_mutex = Mutex.new
|
88
|
+
end
|
89
|
+
|
90
|
+
def configure(conf)
|
91
|
+
super
|
92
|
+
log.instance_eval {
|
93
|
+
def add(level, &block)
|
94
|
+
if block
|
95
|
+
self.info(block.call)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
}
|
99
|
+
Rdkafka::Config.logger = log
|
100
|
+
config = build_config
|
101
|
+
@rdkafka = Rdkafka::Config.new(config)
|
102
|
+
@formatter_proc = setup_formatter(conf)
|
103
|
+
end
|
104
|
+
|
105
|
+
def build_config
|
106
|
+
config = {
|
107
|
+
:"bootstrap.servers" => @brokers,
|
108
|
+
}
|
109
|
+
|
110
|
+
if @ssl_ca_cert && @ssl_ca_cert[0]
|
111
|
+
ssl = true
|
112
|
+
config[:"ssl.ca.location"] = @ssl_ca_cert[0]
|
113
|
+
config[:"ssl.certificate.location"] = @ssl_client_cert if @ssl_client_cert
|
114
|
+
config[:"ssl.key.location"] = @ssl_client_cert_key if @ssl_client_cert_key
|
115
|
+
config[:"ssl.key.password"] = @ssl_client_cert_key_password if @ssl_client_cert_key_password
|
116
|
+
end
|
117
|
+
|
118
|
+
if @principal
|
119
|
+
sasl = true
|
120
|
+
config[:"sasl.mechanisms"] = "GSSAPI"
|
121
|
+
config[:"sasl.kerberos.principal"] = @principal
|
122
|
+
config[:"sasl.kerberos.service.name"] = @service_name if @service_name
|
123
|
+
config[:"sasl.kerberos.keytab"] = @keytab if @keytab
|
124
|
+
end
|
125
|
+
|
126
|
+
if ssl && sasl
|
127
|
+
security_protocol = "SASL_SSL"
|
128
|
+
elsif ssl && !sasl
|
129
|
+
security_protocol = "SSL"
|
130
|
+
elsif !ssl && sasl
|
131
|
+
security_protocol = "SASL_PLAINTEXT"
|
132
|
+
else
|
133
|
+
security_protocol = "PLAINTEXT"
|
134
|
+
end
|
135
|
+
config[:"security.protocol"] = security_protocol
|
136
|
+
|
137
|
+
config[:"compression.codec"] = @compression_codec if @compression_codec
|
138
|
+
config[:"message.send.max.retries"] = @max_send_retries if @max_send_retries
|
139
|
+
config[:"request.required.acks"] = @required_acks if @required_acks
|
140
|
+
config[:"request.timeout.ms"] = @ack_timeout * 1000 if @ack_timeout
|
141
|
+
config[:"queue.buffering.max.ms"] = @rdkafka_buffering_max_ms if @rdkafka_buffering_max_ms
|
142
|
+
config[:"queue.buffering.max.messages"] = @rdkafka_buffering_max_messages if @rdkafka_buffering_max_messages
|
143
|
+
config[:"message.max.bytes"] = @rdkafka_message_max_bytes if @rdkafka_message_max_bytes
|
144
|
+
config[:"batch.num.messages"] = @rdkafka_message_max_num if @rdkafka_message_max_num
|
145
|
+
|
146
|
+
@rdkafka_options.each { |k, v|
|
147
|
+
config[k.to_sym] = v
|
148
|
+
}
|
149
|
+
|
150
|
+
config
|
151
|
+
end
|
152
|
+
|
153
|
+
def start
|
154
|
+
super
|
155
|
+
end
|
156
|
+
|
157
|
+
def multi_workers_ready?
|
158
|
+
true
|
159
|
+
end
|
160
|
+
|
161
|
+
def shutdown
|
162
|
+
super
|
163
|
+
shutdown_producers
|
164
|
+
end
|
165
|
+
|
166
|
+
def shutdown_producers
|
167
|
+
@producers_mutex.synchronize {
|
168
|
+
shutdown_threads = @producers.map { |key, producer|
|
169
|
+
th = Thread.new {
|
170
|
+
unless producer.close(10)
|
171
|
+
log.warn("Queue is forcefully closed after 10 seconds wait")
|
172
|
+
end
|
173
|
+
}
|
174
|
+
th.abort_on_exception = true
|
175
|
+
th
|
176
|
+
}
|
177
|
+
shutdown_threads.each { |th| th.join }
|
178
|
+
@producers = {}
|
179
|
+
}
|
180
|
+
end
|
181
|
+
|
182
|
+
def get_producer
|
183
|
+
@producers_mutex.synchronize {
|
184
|
+
producer = @producers[Thread.current.object_id]
|
185
|
+
unless producer
|
186
|
+
producer = @rdkafka.producer
|
187
|
+
@producers[Thread.current.object_id] = producer
|
188
|
+
end
|
189
|
+
producer
|
190
|
+
}
|
191
|
+
end
|
192
|
+
|
193
|
+
def emit(tag, es, chain)
|
194
|
+
super(tag, es, chain, tag)
|
195
|
+
end
|
196
|
+
|
197
|
+
def format_stream(tag, es)
|
198
|
+
es.to_msgpack_stream
|
199
|
+
end
|
200
|
+
|
201
|
+
def setup_formatter(conf)
|
202
|
+
if @output_data_type == 'json'
|
203
|
+
begin
|
204
|
+
require 'oj'
|
205
|
+
Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
|
206
|
+
Proc.new { |tag, time, record| Oj.dump(record) }
|
207
|
+
rescue LoadError
|
208
|
+
require 'yajl'
|
209
|
+
Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
|
210
|
+
end
|
211
|
+
elsif @output_data_type == 'ltsv'
|
212
|
+
require 'ltsv'
|
213
|
+
Proc.new { |tag, time, record| LTSV.dump(record) }
|
214
|
+
elsif @output_data_type == 'msgpack'
|
215
|
+
require 'msgpack'
|
216
|
+
Proc.new { |tag, time, record| record.to_msgpack }
|
217
|
+
elsif @output_data_type =~ /^attr:(.*)$/
|
218
|
+
@custom_attributes = $1.split(',').map(&:strip).reject(&:empty?)
|
219
|
+
@custom_attributes.unshift('time') if @output_include_time
|
220
|
+
@custom_attributes.unshift('tag') if @output_include_tag
|
221
|
+
Proc.new { |tag, time, record|
|
222
|
+
@custom_attributes.map { |attr|
|
223
|
+
record[attr].nil? ? '' : record[attr].to_s
|
224
|
+
}.join(@f_separator)
|
225
|
+
}
|
226
|
+
else
|
227
|
+
@formatter = Fluent::Plugin.new_formatter(@output_data_type)
|
228
|
+
@formatter.configure(conf)
|
229
|
+
@formatter.method(:format)
|
230
|
+
end
|
231
|
+
end
|
232
|
+
|
233
|
+
def write(chunk)
|
234
|
+
tag = chunk.key
|
235
|
+
def_topic = @default_topic || tag
|
236
|
+
|
237
|
+
record_buf = nil
|
238
|
+
record_buf_bytes = nil
|
239
|
+
|
240
|
+
begin
|
241
|
+
chunk.msgpack_each.map { |time, record|
|
242
|
+
begin
|
243
|
+
if @output_include_time
|
244
|
+
if @time_format
|
245
|
+
record['time'.freeze] = Time.at(time).strftime(@time_format)
|
246
|
+
else
|
247
|
+
record['time'.freeze] = time
|
248
|
+
end
|
249
|
+
end
|
250
|
+
|
251
|
+
record['tag'] = tag if @output_include_tag
|
252
|
+
topic = (@exclude_topic_key ? record.delete('topic'.freeze) : record['topic'.freeze]) || def_topic
|
253
|
+
partition = (@exclude_partition ? record.delete('partition'.freeze) : record['partition'.freeze]) || @default_partition
|
254
|
+
message_key = (@exclude_message_key ? record.delete('message_key'.freeze) : record['message_key'.freeze]) || @default_message_key
|
255
|
+
|
256
|
+
record_buf = @formatter_proc.call(tag, time, record)
|
257
|
+
record_buf_bytes = record_buf.bytesize
|
258
|
+
if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
|
259
|
+
log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
|
260
|
+
next
|
261
|
+
end
|
262
|
+
rescue StandardError => e
|
263
|
+
log.warn "unexpected error during format record. Skip broken event:", :error => e.to_s, :error_class => e.class.to_s, :time => time, :record => record
|
264
|
+
next
|
265
|
+
end
|
266
|
+
|
267
|
+
producer = get_producer
|
268
|
+
handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition)
|
269
|
+
handler
|
270
|
+
}.each { |handler|
|
271
|
+
handler.wait(@rdkafka_delivery_handle_poll_timeout) if @rdkafka_delivery_handle_poll_timeout != 0
|
272
|
+
}
|
273
|
+
end
|
274
|
+
rescue Exception => e
|
275
|
+
log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
|
276
|
+
# Raise exception to retry sendind messages
|
277
|
+
raise e
|
278
|
+
end
|
279
|
+
|
280
|
+
def enqueue_with_retry(producer, topic, record_buf, message_key, partition)
|
281
|
+
attempt = 0
|
282
|
+
loop do
|
283
|
+
begin
|
284
|
+
handler = producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition)
|
285
|
+
return handler
|
286
|
+
rescue Exception => e
|
287
|
+
if e.code == :queue_full
|
288
|
+
if attempt <= @max_enqueue_retries
|
289
|
+
log.warn "Failed to enqueue message; attempting retry #{attempt} of #{@max_enqueue_retries} after #{@enqueue_retry_backoff}s"
|
290
|
+
sleep @enqueue_retry_backoff
|
291
|
+
attempt += 1
|
292
|
+
else
|
293
|
+
raise "Failed to enqueue message although tried retry #{@max_enqueue_retries} times"
|
294
|
+
end
|
295
|
+
else
|
296
|
+
raise e
|
297
|
+
end
|
298
|
+
end
|
299
|
+
end
|
300
|
+
end
|
301
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hidemasa Togashi
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-
|
12
|
+
date: 2018-10-19 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -51,20 +51,20 @@ dependencies:
|
|
51
51
|
requirements:
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.
|
54
|
+
version: 0.7.1
|
55
55
|
- - "<"
|
56
56
|
- !ruby/object:Gem::Version
|
57
|
-
version: 0.
|
57
|
+
version: 0.8.0
|
58
58
|
type: :runtime
|
59
59
|
prerelease: false
|
60
60
|
version_requirements: !ruby/object:Gem::Requirement
|
61
61
|
requirements:
|
62
62
|
- - ">="
|
63
63
|
- !ruby/object:Gem::Version
|
64
|
-
version: 0.
|
64
|
+
version: 0.7.1
|
65
65
|
- - "<"
|
66
66
|
- !ruby/object:Gem::Version
|
67
|
-
version: 0.
|
67
|
+
version: 0.8.0
|
68
68
|
- !ruby/object:Gem::Dependency
|
69
69
|
name: rake
|
70
70
|
requirement: !ruby/object:Gem::Requirement
|
@@ -116,6 +116,7 @@ files:
|
|
116
116
|
- lib/fluent/plugin/out_kafka.rb
|
117
117
|
- lib/fluent/plugin/out_kafka2.rb
|
118
118
|
- lib/fluent/plugin/out_kafka_buffered.rb
|
119
|
+
- lib/fluent/plugin/out_rdkafka.rb
|
119
120
|
- test/helper.rb
|
120
121
|
- test/plugin/test_out_kafka.rb
|
121
122
|
homepage: https://github.com/fluent/fluent-plugin-kafka
|