karafka-rdkafka 0.12.4 → 0.13.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +1 -1
- data/CHANGELOG.md +21 -2
- data/Gemfile +2 -0
- data/README.md +26 -0
- data/Rakefile +2 -0
- data/certs/cert_chain.pem +21 -21
- data/certs/karafka-pro.pem +11 -0
- data/ext/Rakefile +26 -53
- data/karafka-rdkafka.gemspec +2 -0
- data/lib/rdkafka/abstract_handle.rb +2 -0
- data/lib/rdkafka/admin/create_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/create_topic_report.rb +2 -0
- data/lib/rdkafka/admin/delete_topic_handle.rb +2 -0
- data/lib/rdkafka/admin/delete_topic_report.rb +2 -0
- data/lib/rdkafka/admin.rb +95 -73
- data/lib/rdkafka/bindings.rb +52 -37
- data/lib/rdkafka/callbacks.rb +2 -0
- data/lib/rdkafka/config.rb +13 -10
- data/lib/rdkafka/consumer/headers.rb +24 -7
- data/lib/rdkafka/consumer/message.rb +3 -1
- data/lib/rdkafka/consumer/partition.rb +2 -0
- data/lib/rdkafka/consumer/topic_partition_list.rb +2 -0
- data/lib/rdkafka/consumer.rb +100 -44
- data/lib/rdkafka/error.rb +9 -0
- data/lib/rdkafka/metadata.rb +25 -2
- data/lib/rdkafka/native_kafka.rb +83 -0
- data/lib/rdkafka/producer/delivery_handle.rb +2 -0
- data/lib/rdkafka/producer/delivery_report.rb +3 -1
- data/lib/rdkafka/producer.rb +75 -12
- data/lib/rdkafka/version.rb +3 -1
- data/lib/rdkafka.rb +3 -1
- data/spec/rdkafka/abstract_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/create_topic_report_spec.rb +2 -0
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +2 -0
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +2 -0
- data/spec/rdkafka/admin_spec.rb +4 -3
- data/spec/rdkafka/bindings_spec.rb +2 -0
- data/spec/rdkafka/callbacks_spec.rb +2 -0
- data/spec/rdkafka/config_spec.rb +17 -2
- data/spec/rdkafka/consumer/headers_spec.rb +62 -0
- data/spec/rdkafka/consumer/message_spec.rb +2 -0
- data/spec/rdkafka/consumer/partition_spec.rb +2 -0
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +2 -0
- data/spec/rdkafka/consumer_spec.rb +124 -22
- data/spec/rdkafka/error_spec.rb +2 -0
- data/spec/rdkafka/metadata_spec.rb +2 -0
- data/spec/rdkafka/{producer/client_spec.rb → native_kafka_spec.rb} +13 -34
- data/spec/rdkafka/producer/delivery_handle_spec.rb +2 -0
- data/spec/rdkafka/producer/delivery_report_spec.rb +4 -2
- data/spec/rdkafka/producer_spec.rb +118 -17
- data/spec/spec_helper.rb +17 -1
- data.tar.gz.sig +0 -0
- metadata +33 -33
- metadata.gz.sig +0 -0
- data/bin/console +0 -11
- data/dist/librdkafka_2.0.2.tar.gz +0 -0
- data/lib/rdkafka/producer/client.rb +0 -47
data/lib/rdkafka/bindings.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "ffi"
|
2
4
|
require "json"
|
3
5
|
require "logger"
|
@@ -15,7 +17,7 @@ module Rdkafka
|
|
15
17
|
end
|
16
18
|
end
|
17
19
|
|
18
|
-
ffi_lib File.join(
|
20
|
+
ffi_lib File.join(__dir__, "../../ext/librdkafka.#{lib_extension}")
|
19
21
|
|
20
22
|
RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
|
21
23
|
RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
|
@@ -33,15 +35,17 @@ module Rdkafka
|
|
33
35
|
|
34
36
|
# Polling
|
35
37
|
|
38
|
+
attach_function :rd_kafka_flush, [:pointer, :int], :void, blocking: true
|
36
39
|
attach_function :rd_kafka_poll, [:pointer, :int], :void, blocking: true
|
37
40
|
attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
|
38
41
|
|
39
42
|
# Metadata
|
40
43
|
|
41
|
-
attach_function :
|
42
|
-
attach_function :
|
43
|
-
attach_function :
|
44
|
-
attach_function :
|
44
|
+
attach_function :rd_kafka_name, [:pointer], :string, blocking: true
|
45
|
+
attach_function :rd_kafka_memberid, [:pointer], :string, blocking: true
|
46
|
+
attach_function :rd_kafka_clusterid, [:pointer], :string, blocking: true
|
47
|
+
attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
|
48
|
+
attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
|
45
49
|
|
46
50
|
# Message struct
|
47
51
|
|
@@ -110,6 +114,7 @@ module Rdkafka
|
|
110
114
|
attach_function :rd_kafka_conf_set_stats_cb, [:pointer, :stats_cb], :void
|
111
115
|
callback :error_cb, [:pointer, :int, :string, :pointer], :void
|
112
116
|
attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
|
117
|
+
attach_function :rd_kafka_rebalance_protocol, [:pointer], :string
|
113
118
|
|
114
119
|
# Log queue
|
115
120
|
attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
|
@@ -166,24 +171,28 @@ module Rdkafka
|
|
166
171
|
]
|
167
172
|
|
168
173
|
attach_function :rd_kafka_new, [:kafka_type, :pointer, :pointer, :int], :pointer
|
174
|
+
|
169
175
|
attach_function :rd_kafka_destroy, [:pointer], :void
|
170
176
|
|
171
177
|
# Consumer
|
172
178
|
|
173
|
-
attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int
|
174
|
-
attach_function :rd_kafka_unsubscribe, [:pointer], :int
|
175
|
-
attach_function :rd_kafka_subscription, [:pointer, :pointer], :int
|
176
|
-
attach_function :rd_kafka_assign, [:pointer, :pointer], :int
|
177
|
-
attach_function :
|
178
|
-
attach_function :
|
179
|
+
attach_function :rd_kafka_subscribe, [:pointer, :pointer], :int, blocking: true
|
180
|
+
attach_function :rd_kafka_unsubscribe, [:pointer], :int, blocking: true
|
181
|
+
attach_function :rd_kafka_subscription, [:pointer, :pointer], :int, blocking: true
|
182
|
+
attach_function :rd_kafka_assign, [:pointer, :pointer], :int, blocking: true
|
183
|
+
attach_function :rd_kafka_incremental_assign, [:pointer, :pointer], :int, blocking: true
|
184
|
+
attach_function :rd_kafka_incremental_unassign, [:pointer, :pointer], :int, blocking: true
|
185
|
+
attach_function :rd_kafka_assignment, [:pointer, :pointer], :int, blocking: true
|
186
|
+
attach_function :rd_kafka_assignment_lost, [:pointer], :int, blocking: true
|
187
|
+
attach_function :rd_kafka_committed, [:pointer, :pointer, :int], :int, blocking: true
|
179
188
|
attach_function :rd_kafka_commit, [:pointer, :pointer, :bool], :int, blocking: true
|
180
|
-
attach_function :rd_kafka_poll_set_consumer, [:pointer], :void
|
189
|
+
attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
|
181
190
|
attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
|
182
191
|
attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
|
183
|
-
attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int
|
184
|
-
attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int
|
185
|
-
attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int
|
186
|
-
attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int
|
192
|
+
attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int, blocking: true
|
193
|
+
attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
|
194
|
+
attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
|
195
|
+
attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
|
187
196
|
|
188
197
|
# Headers
|
189
198
|
attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
|
@@ -192,30 +201,36 @@ module Rdkafka
|
|
192
201
|
# Rebalance
|
193
202
|
|
194
203
|
callback :rebalance_cb_function, [:pointer, :int, :pointer, :pointer], :void
|
195
|
-
attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void
|
204
|
+
attach_function :rd_kafka_conf_set_rebalance_cb, [:pointer, :rebalance_cb_function], :void, blocking: true
|
196
205
|
|
197
206
|
RebalanceCallback = FFI::Function.new(
|
198
207
|
:void, [:pointer, :int, :pointer, :pointer]
|
199
208
|
) do |client_ptr, code, partitions_ptr, opaque_ptr|
|
200
209
|
case code
|
201
210
|
when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
|
202
|
-
Rdkafka::Bindings.
|
211
|
+
if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
|
212
|
+
Rdkafka::Bindings.rd_kafka_incremental_assign(client_ptr, partitions_ptr)
|
213
|
+
else
|
214
|
+
Rdkafka::Bindings.rd_kafka_assign(client_ptr, partitions_ptr)
|
215
|
+
end
|
203
216
|
else # RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS or errors
|
204
|
-
Rdkafka::Bindings.
|
217
|
+
if Rdkafka::Bindings.rd_kafka_rebalance_protocol(client_ptr) == "COOPERATIVE"
|
218
|
+
Rdkafka::Bindings.rd_kafka_incremental_unassign(client_ptr, partitions_ptr)
|
219
|
+
else
|
220
|
+
Rdkafka::Bindings.rd_kafka_assign(client_ptr, FFI::Pointer::NULL)
|
221
|
+
end
|
205
222
|
end
|
206
223
|
|
207
224
|
opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
|
208
225
|
return unless opaque
|
209
226
|
|
210
227
|
tpl = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(partitions_ptr).freeze
|
211
|
-
consumer = Rdkafka::Consumer.new(client_ptr)
|
212
|
-
|
213
228
|
begin
|
214
229
|
case code
|
215
230
|
when RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS
|
216
|
-
opaque.call_on_partitions_assigned(
|
231
|
+
opaque.call_on_partitions_assigned(tpl)
|
217
232
|
when RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS
|
218
|
-
opaque.call_on_partitions_revoked(
|
233
|
+
opaque.call_on_partitions_revoked(tpl)
|
219
234
|
end
|
220
235
|
rescue Exception => err
|
221
236
|
Rdkafka::Config.logger.error("Unhandled exception: #{err.class} - #{err.message}")
|
@@ -242,7 +257,7 @@ module Rdkafka
|
|
242
257
|
|
243
258
|
RD_KAFKA_MSG_F_COPY = 0x2
|
244
259
|
|
245
|
-
attach_function :rd_kafka_producev, [:pointer, :varargs], :int
|
260
|
+
attach_function :rd_kafka_producev, [:pointer, :varargs], :int, blocking: true
|
246
261
|
callback :delivery_cb, [:pointer, :pointer, :pointer], :void
|
247
262
|
attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
|
248
263
|
|
@@ -257,11 +272,11 @@ module Rdkafka
|
|
257
272
|
# Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
|
258
273
|
return -1 unless partition_count&.nonzero?
|
259
274
|
|
260
|
-
str_ptr = FFI::MemoryPointer.from_string(str)
|
275
|
+
str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
|
261
276
|
method_name = PARTITIONERS.fetch(partitioner_name) do
|
262
277
|
raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner_name}")
|
263
278
|
end
|
264
|
-
public_send(method_name, nil, str_ptr, str.size, partition_count, nil, nil)
|
279
|
+
public_send(method_name, nil, str_ptr, str.size > 0 ? str.size : 1, partition_count, nil, nil)
|
265
280
|
end
|
266
281
|
|
267
282
|
# Create Topics
|
@@ -269,23 +284,23 @@ module Rdkafka
|
|
269
284
|
RD_KAFKA_ADMIN_OP_CREATETOPICS = 1 # rd_kafka_admin_op_t
|
270
285
|
RD_KAFKA_EVENT_CREATETOPICS_RESULT = 100 # rd_kafka_event_type_t
|
271
286
|
|
272
|
-
attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
|
273
|
-
attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer
|
274
|
-
attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32
|
275
|
-
attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void
|
276
|
-
attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer
|
277
|
-
attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer
|
287
|
+
attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
|
288
|
+
attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer, blocking: true
|
289
|
+
attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32, blocking: true
|
290
|
+
attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void, blocking: true
|
291
|
+
attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer, blocking: true
|
292
|
+
attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
|
278
293
|
|
279
294
|
# Delete Topics
|
280
295
|
|
281
296
|
RD_KAFKA_ADMIN_OP_DELETETOPICS = 2 # rd_kafka_admin_op_t
|
282
297
|
RD_KAFKA_EVENT_DELETETOPICS_RESULT = 101 # rd_kafka_event_type_t
|
283
298
|
|
284
|
-
attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32
|
285
|
-
attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer
|
286
|
-
attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void
|
287
|
-
attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer
|
288
|
-
attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer
|
299
|
+
attach_function :rd_kafka_DeleteTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :int32, blocking: true
|
300
|
+
attach_function :rd_kafka_DeleteTopic_new, [:pointer], :pointer, blocking: true
|
301
|
+
attach_function :rd_kafka_DeleteTopic_destroy, [:pointer], :void, blocking: true
|
302
|
+
attach_function :rd_kafka_event_DeleteTopics_result, [:pointer], :pointer, blocking: true
|
303
|
+
attach_function :rd_kafka_DeleteTopics_result_topics, [:pointer, :pointer], :pointer, blocking: true
|
289
304
|
|
290
305
|
# Create partitions
|
291
306
|
RD_KAFKA_ADMIN_OP_CREATEPARTITIONS = 3
|
data/lib/rdkafka/callbacks.rb
CHANGED
data/lib/rdkafka/config.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "logger"
|
2
4
|
|
3
5
|
module Rdkafka
|
@@ -30,7 +32,6 @@ module Rdkafka
|
|
30
32
|
@@logger
|
31
33
|
end
|
32
34
|
|
33
|
-
|
34
35
|
# Returns a queue whose contents will be passed to the configured logger. Each entry
|
35
36
|
# should follow the format [Logger::Severity, String]. The benefit over calling the
|
36
37
|
# logger directly is that this is safe to use from trap contexts.
|
@@ -47,7 +48,7 @@ module Rdkafka
|
|
47
48
|
# @return [nil]
|
48
49
|
def self.logger=(logger)
|
49
50
|
raise NoLoggerError if logger.nil?
|
50
|
-
@@logger=logger
|
51
|
+
@@logger = logger
|
51
52
|
end
|
52
53
|
|
53
54
|
# Set a callback that will be called every time the underlying client emits statistics.
|
@@ -156,13 +157,14 @@ module Rdkafka
|
|
156
157
|
Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
|
157
158
|
end
|
158
159
|
|
160
|
+
# Create native client
|
159
161
|
kafka = native_kafka(config, :rd_kafka_consumer)
|
160
162
|
|
161
163
|
# Redirect the main queue to the consumer
|
162
164
|
Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
|
163
165
|
|
164
166
|
# Return consumer with Kafka client
|
165
|
-
Rdkafka::Consumer.new(kafka)
|
167
|
+
Rdkafka::Consumer.new(Rdkafka::NativeKafka.new(kafka, run_polling_thread: false))
|
166
168
|
end
|
167
169
|
|
168
170
|
# Create a producer with this configuration.
|
@@ -179,7 +181,8 @@ module Rdkafka
|
|
179
181
|
# Set callback to receive delivery reports on config
|
180
182
|
Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
|
181
183
|
# Return producer with Kafka client
|
182
|
-
|
184
|
+
partitioner_name = self[:partitioner] || self["partitioner"]
|
185
|
+
Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true), partitioner_name).tap do |producer|
|
183
186
|
opaque.producer = producer
|
184
187
|
end
|
185
188
|
end
|
@@ -194,7 +197,7 @@ module Rdkafka
|
|
194
197
|
opaque = Opaque.new
|
195
198
|
config = native_config(opaque)
|
196
199
|
Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
|
197
|
-
Rdkafka::Admin.new(native_kafka(config, :rd_kafka_producer))
|
200
|
+
Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true))
|
198
201
|
end
|
199
202
|
|
200
203
|
# Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
|
@@ -210,7 +213,7 @@ module Rdkafka
|
|
210
213
|
|
211
214
|
# This method is only intended to be used to create a client,
|
212
215
|
# using it in another way will leak memory.
|
213
|
-
def native_config(opaque=nil)
|
216
|
+
def native_config(opaque = nil)
|
214
217
|
Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
|
215
218
|
# Create config
|
216
219
|
@config_hash.merge(REQUIRED_CONFIG).each do |key, value|
|
@@ -282,18 +285,18 @@ module Rdkafka
|
|
282
285
|
producer.call_delivery_callback(delivery_report, delivery_handle) if producer
|
283
286
|
end
|
284
287
|
|
285
|
-
def call_on_partitions_assigned(
|
288
|
+
def call_on_partitions_assigned(list)
|
286
289
|
return unless consumer_rebalance_listener
|
287
290
|
return unless consumer_rebalance_listener.respond_to?(:on_partitions_assigned)
|
288
291
|
|
289
|
-
consumer_rebalance_listener.on_partitions_assigned(
|
292
|
+
consumer_rebalance_listener.on_partitions_assigned(list)
|
290
293
|
end
|
291
294
|
|
292
|
-
def call_on_partitions_revoked(
|
295
|
+
def call_on_partitions_revoked(list)
|
293
296
|
return unless consumer_rebalance_listener
|
294
297
|
return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoked)
|
295
298
|
|
296
|
-
consumer_rebalance_listener.on_partitions_revoked(
|
299
|
+
consumer_rebalance_listener.on_partitions_revoked(list)
|
297
300
|
end
|
298
301
|
end
|
299
302
|
end
|
@@ -1,10 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
class Consumer
|
3
|
-
#
|
4
|
-
|
5
|
-
|
5
|
+
# Interface to return headers for a consumer message
|
6
|
+
module Headers
|
7
|
+
class HashWithSymbolKeysTreatedLikeStrings < Hash
|
8
|
+
def [](key)
|
9
|
+
if key.is_a?(Symbol)
|
10
|
+
Kernel.warn("rdkafka deprecation warning: header access with Symbol key #{key.inspect} treated as a String. " \
|
11
|
+
"Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
|
12
|
+
super(key.to_s)
|
13
|
+
else
|
14
|
+
super
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
# Reads a librdkafka native message's headers and returns them as a Ruby Hash
|
20
|
+
#
|
21
|
+
# @param [librdkakfa message] native_message
|
6
22
|
#
|
7
|
-
# @return [Hash<String, String>]
|
23
|
+
# @return [Hash<String, String>] headers Hash for the native_message
|
8
24
|
#
|
9
25
|
# @raise [Rdkafka::RdkafkaError] when fail to read headers
|
10
26
|
#
|
@@ -24,7 +40,8 @@ module Rdkafka
|
|
24
40
|
name_ptrptr = FFI::MemoryPointer.new(:pointer)
|
25
41
|
value_ptrptr = FFI::MemoryPointer.new(:pointer)
|
26
42
|
size_ptr = Rdkafka::Bindings::SizePtr.new
|
27
|
-
|
43
|
+
|
44
|
+
headers = HashWithSymbolKeysTreatedLikeStrings.new
|
28
45
|
|
29
46
|
idx = 0
|
30
47
|
loop do
|
@@ -51,12 +68,12 @@ module Rdkafka
|
|
51
68
|
|
52
69
|
value = value_ptr.read_string(size)
|
53
70
|
|
54
|
-
headers[name
|
71
|
+
headers[name] = value
|
55
72
|
|
56
73
|
idx += 1
|
57
74
|
end
|
58
75
|
|
59
|
-
headers
|
76
|
+
headers.freeze
|
60
77
|
end
|
61
78
|
end
|
62
79
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
class Consumer
|
3
5
|
# A message that was consumed from a topic.
|
@@ -18,7 +20,7 @@ module Rdkafka
|
|
18
20
|
# @return [String, nil]
|
19
21
|
attr_reader :key
|
20
22
|
|
21
|
-
# This message's offset in
|
23
|
+
# This message's offset in its partition
|
22
24
|
# @return [Integer]
|
23
25
|
attr_reader :offset
|
24
26
|
|
data/lib/rdkafka/consumer.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Rdkafka
|
2
4
|
# A consumer of Kafka messages. It uses the high-level consumer approach where the Kafka
|
3
5
|
# brokers automatically assign partitions and load balance partitions over consumers that
|
@@ -14,18 +16,33 @@ module Rdkafka
|
|
14
16
|
# @private
|
15
17
|
def initialize(native_kafka)
|
16
18
|
@native_kafka = native_kafka
|
17
|
-
|
19
|
+
end
|
20
|
+
|
21
|
+
# @return [String] consumer name
|
22
|
+
def name
|
23
|
+
@name ||= @native_kafka.with_inner do |inner|
|
24
|
+
::Rdkafka::Bindings.rd_kafka_name(inner)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def finalizer
|
29
|
+
->(_) { close }
|
18
30
|
end
|
19
31
|
|
20
32
|
# Close this consumer
|
21
33
|
# @return [nil]
|
22
34
|
def close
|
23
|
-
return
|
35
|
+
return if closed?
|
36
|
+
ObjectSpace.undefine_finalizer(self)
|
37
|
+
@native_kafka.with_inner do |inner|
|
38
|
+
Rdkafka::Bindings.rd_kafka_consumer_close(inner)
|
39
|
+
end
|
40
|
+
@native_kafka.close
|
41
|
+
end
|
24
42
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
@native_kafka = nil
|
43
|
+
# Whether this consumer has closed
|
44
|
+
def closed?
|
45
|
+
@native_kafka.closed?
|
29
46
|
end
|
30
47
|
|
31
48
|
# Subscribe to one or more topics letting Kafka handle partition assignments.
|
@@ -46,7 +63,9 @@ module Rdkafka
|
|
46
63
|
end
|
47
64
|
|
48
65
|
# Subscribe to topic partition list and check this was successful
|
49
|
-
response =
|
66
|
+
response = @native_kafka.with_inner do |inner|
|
67
|
+
Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
|
68
|
+
end
|
50
69
|
if response != 0
|
51
70
|
raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
|
52
71
|
end
|
@@ -62,7 +81,9 @@ module Rdkafka
|
|
62
81
|
def unsubscribe
|
63
82
|
closed_consumer_check(__method__)
|
64
83
|
|
65
|
-
response =
|
84
|
+
response = @native_kafka.with_inner do |inner|
|
85
|
+
Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
|
86
|
+
end
|
66
87
|
if response != 0
|
67
88
|
raise Rdkafka::RdkafkaError.new(response)
|
68
89
|
end
|
@@ -85,7 +106,9 @@ module Rdkafka
|
|
85
106
|
tpl = list.to_native_tpl
|
86
107
|
|
87
108
|
begin
|
88
|
-
response =
|
109
|
+
response = @native_kafka.with_inner do |inner|
|
110
|
+
Rdkafka::Bindings.rd_kafka_pause_partitions(inner, tpl)
|
111
|
+
end
|
89
112
|
|
90
113
|
if response != 0
|
91
114
|
list = TopicPartitionList.from_native_tpl(tpl)
|
@@ -113,7 +136,9 @@ module Rdkafka
|
|
113
136
|
tpl = list.to_native_tpl
|
114
137
|
|
115
138
|
begin
|
116
|
-
response =
|
139
|
+
response = @native_kafka.with_inner do |inner|
|
140
|
+
Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
|
141
|
+
end
|
117
142
|
if response != 0
|
118
143
|
raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
|
119
144
|
end
|
@@ -131,7 +156,9 @@ module Rdkafka
|
|
131
156
|
closed_consumer_check(__method__)
|
132
157
|
|
133
158
|
ptr = FFI::MemoryPointer.new(:pointer)
|
134
|
-
response =
|
159
|
+
response = @native_kafka.with_inner do |inner|
|
160
|
+
Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
|
161
|
+
end
|
135
162
|
|
136
163
|
if response != 0
|
137
164
|
raise Rdkafka::RdkafkaError.new(response)
|
@@ -161,7 +188,9 @@ module Rdkafka
|
|
161
188
|
tpl = list.to_native_tpl
|
162
189
|
|
163
190
|
begin
|
164
|
-
response =
|
191
|
+
response = @native_kafka.with_inner do |inner|
|
192
|
+
Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
|
193
|
+
end
|
165
194
|
if response != 0
|
166
195
|
raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
|
167
196
|
end
|
@@ -179,7 +208,9 @@ module Rdkafka
|
|
179
208
|
closed_consumer_check(__method__)
|
180
209
|
|
181
210
|
ptr = FFI::MemoryPointer.new(:pointer)
|
182
|
-
response =
|
211
|
+
response = @native_kafka.with_inner do |inner|
|
212
|
+
Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
|
213
|
+
end
|
183
214
|
if response != 0
|
184
215
|
raise Rdkafka::RdkafkaError.new(response)
|
185
216
|
end
|
@@ -197,6 +228,15 @@ module Rdkafka
|
|
197
228
|
ptr.free unless ptr.nil?
|
198
229
|
end
|
199
230
|
|
231
|
+
# @return [Boolean] true if our current assignment has been lost involuntarily.
|
232
|
+
def assignment_lost?
|
233
|
+
closed_consumer_check(__method__)
|
234
|
+
|
235
|
+
@native_kafka.with_inner do |inner|
|
236
|
+
!Rdkafka::Bindings.rd_kafka_assignment_lost(inner).zero?
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
200
240
|
# Return the current committed offset per partition for this consumer group.
|
201
241
|
# The offset field of each requested partition will either be set to stored offset or to -1001 in case there was no stored offset for that partition.
|
202
242
|
#
|
@@ -218,7 +258,9 @@ module Rdkafka
|
|
218
258
|
tpl = list.to_native_tpl
|
219
259
|
|
220
260
|
begin
|
221
|
-
response =
|
261
|
+
response = @native_kafka.with_inner do |inner|
|
262
|
+
Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
|
263
|
+
end
|
222
264
|
if response != 0
|
223
265
|
raise Rdkafka::RdkafkaError.new(response)
|
224
266
|
end
|
@@ -243,14 +285,16 @@ module Rdkafka
|
|
243
285
|
low = FFI::MemoryPointer.new(:int64, 1)
|
244
286
|
high = FFI::MemoryPointer.new(:int64, 1)
|
245
287
|
|
246
|
-
response =
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
288
|
+
response = @native_kafka.with_inner do |inner|
|
289
|
+
Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
|
290
|
+
inner,
|
291
|
+
topic,
|
292
|
+
partition,
|
293
|
+
low,
|
294
|
+
high,
|
295
|
+
timeout_ms,
|
296
|
+
)
|
297
|
+
end
|
254
298
|
if response != 0
|
255
299
|
raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
|
256
300
|
end
|
@@ -298,7 +342,9 @@ module Rdkafka
|
|
298
342
|
# @return [String, nil]
|
299
343
|
def cluster_id
|
300
344
|
closed_consumer_check(__method__)
|
301
|
-
|
345
|
+
@native_kafka.with_inner do |inner|
|
346
|
+
Rdkafka::Bindings.rd_kafka_clusterid(inner)
|
347
|
+
end
|
302
348
|
end
|
303
349
|
|
304
350
|
# Returns this client's broker-assigned group member id
|
@@ -308,7 +354,9 @@ module Rdkafka
|
|
308
354
|
# @return [String, nil]
|
309
355
|
def member_id
|
310
356
|
closed_consumer_check(__method__)
|
311
|
-
|
357
|
+
@native_kafka.with_inner do |inner|
|
358
|
+
Rdkafka::Bindings.rd_kafka_memberid(inner)
|
359
|
+
end
|
312
360
|
end
|
313
361
|
|
314
362
|
# Store offset of a message to be used in the next commit of this consumer
|
@@ -325,11 +373,13 @@ module Rdkafka
|
|
325
373
|
|
326
374
|
# rd_kafka_offset_store is one of the few calls that does not support
|
327
375
|
# a string as the topic, so create a native topic for it.
|
328
|
-
native_topic =
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
376
|
+
native_topic = @native_kafka.with_inner do |inner|
|
377
|
+
Rdkafka::Bindings.rd_kafka_topic_new(
|
378
|
+
inner,
|
379
|
+
message.topic,
|
380
|
+
nil
|
381
|
+
)
|
382
|
+
end
|
333
383
|
response = Rdkafka::Bindings.rd_kafka_offset_store(
|
334
384
|
native_topic,
|
335
385
|
message.partition,
|
@@ -357,11 +407,13 @@ module Rdkafka
|
|
357
407
|
|
358
408
|
# rd_kafka_offset_store is one of the few calls that does not support
|
359
409
|
# a string as the topic, so create a native topic for it.
|
360
|
-
native_topic =
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
410
|
+
native_topic = @native_kafka.with_inner do |inner|
|
411
|
+
Rdkafka::Bindings.rd_kafka_topic_new(
|
412
|
+
inner,
|
413
|
+
message.topic,
|
414
|
+
nil
|
415
|
+
)
|
416
|
+
end
|
365
417
|
response = Rdkafka::Bindings.rd_kafka_seek(
|
366
418
|
native_topic,
|
367
419
|
message.partition,
|
@@ -402,7 +454,9 @@ module Rdkafka
|
|
402
454
|
tpl = list ? list.to_native_tpl : nil
|
403
455
|
|
404
456
|
begin
|
405
|
-
response =
|
457
|
+
response = @native_kafka.with_inner do |inner|
|
458
|
+
Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
|
459
|
+
end
|
406
460
|
if response != 0
|
407
461
|
raise Rdkafka::RdkafkaError.new(response)
|
408
462
|
end
|
@@ -421,7 +475,9 @@ module Rdkafka
|
|
421
475
|
def poll(timeout_ms)
|
422
476
|
closed_consumer_check(__method__)
|
423
477
|
|
424
|
-
message_ptr =
|
478
|
+
message_ptr = @native_kafka.with_inner do |inner|
|
479
|
+
Rdkafka::Bindings.rd_kafka_consumer_poll(inner, timeout_ms)
|
480
|
+
end
|
425
481
|
if message_ptr.null?
|
426
482
|
nil
|
427
483
|
else
|
@@ -436,7 +492,7 @@ module Rdkafka
|
|
436
492
|
end
|
437
493
|
ensure
|
438
494
|
# Clean up rdkafka message if there is one
|
439
|
-
if
|
495
|
+
if message_ptr && !message_ptr.null?
|
440
496
|
Rdkafka::Bindings.rd_kafka_message_destroy(message_ptr)
|
441
497
|
end
|
442
498
|
end
|
@@ -459,7 +515,7 @@ module Rdkafka
|
|
459
515
|
if message
|
460
516
|
yield(message)
|
461
517
|
else
|
462
|
-
if
|
518
|
+
if closed?
|
463
519
|
break
|
464
520
|
else
|
465
521
|
next
|
@@ -468,10 +524,6 @@ module Rdkafka
|
|
468
524
|
end
|
469
525
|
end
|
470
526
|
|
471
|
-
def closed_consumer_check(method)
|
472
|
-
raise Rdkafka::ClosedConsumerError.new(method) if @native_kafka.nil?
|
473
|
-
end
|
474
|
-
|
475
527
|
# Poll for new messages and yield them in batches that may contain
|
476
528
|
# messages from more than one partition.
|
477
529
|
#
|
@@ -527,7 +579,7 @@ module Rdkafka
|
|
527
579
|
bytes = 0
|
528
580
|
end_time = monotonic_now + timeout_ms / 1000.0
|
529
581
|
loop do
|
530
|
-
break if
|
582
|
+
break if closed?
|
531
583
|
max_wait = end_time - monotonic_now
|
532
584
|
max_wait_ms = if max_wait <= 0
|
533
585
|
0 # should not block, but may retrieve a message
|
@@ -545,7 +597,7 @@ module Rdkafka
|
|
545
597
|
end
|
546
598
|
if message
|
547
599
|
slice << message
|
548
|
-
bytes += message.payload.bytesize
|
600
|
+
bytes += message.payload.bytesize if message.payload
|
549
601
|
end
|
550
602
|
if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
|
551
603
|
yield slice.dup, nil
|
@@ -561,5 +613,9 @@ module Rdkafka
|
|
561
613
|
# needed because Time.now can go backwards
|
562
614
|
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
563
615
|
end
|
616
|
+
|
617
|
+
def closed_consumer_check(method)
|
618
|
+
raise Rdkafka::ClosedConsumerError.new(method) if closed?
|
619
|
+
end
|
564
620
|
end
|
565
621
|
end
|