karafka-rdkafka 0.30.1-aarch64-linux-gnu → 0.31.0-aarch64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -27
- data/README.md +1 -0
- data/docker-compose-ssl.yml +1 -0
- data/docker-compose.yml +1 -0
- data/ext/librdkafka.so +0 -0
- data/karafka-rdkafka.gemspec +1 -1
- data/lib/rdkafka/admin/describe_cluster_handle.rb +31 -0
- data/lib/rdkafka/admin/describe_cluster_report.rb +48 -0
- data/lib/rdkafka/admin/describe_consumer_groups_handle.rb +31 -0
- data/lib/rdkafka/admin/describe_consumer_groups_report.rb +151 -0
- data/lib/rdkafka/admin/describe_result_parsing.rb +47 -0
- data/lib/rdkafka/admin/describe_topics_handle.rb +31 -0
- data/lib/rdkafka/admin/describe_topics_report.rb +118 -0
- data/lib/rdkafka/admin.rb +245 -0
- data/lib/rdkafka/bindings/admin/acls.rb +85 -0
- data/lib/rdkafka/bindings/admin/configs.rb +37 -0
- data/lib/rdkafka/bindings/admin/list_offsets.rb +26 -0
- data/lib/rdkafka/bindings/admin.rb +197 -0
- data/lib/rdkafka/bindings/callbacks.rb +148 -0
- data/lib/rdkafka/bindings/configuration.rb +50 -0
- data/lib/rdkafka/bindings/consumer.rb +83 -0
- data/lib/rdkafka/bindings/errors.rb +17 -0
- data/lib/rdkafka/bindings/handle.rb +16 -0
- data/lib/rdkafka/bindings/messages.rb +28 -0
- data/lib/rdkafka/bindings/metadata.rb +17 -0
- data/lib/rdkafka/bindings/polling.rb +17 -0
- data/lib/rdkafka/bindings/producer.rb +82 -0
- data/lib/rdkafka/bindings/share_consumer.rb +67 -0
- data/lib/rdkafka/bindings/structs.rb +62 -0
- data/lib/rdkafka/bindings.rb +13 -775
- data/lib/rdkafka/callbacks/describe_cluster_handler.rb +36 -0
- data/lib/rdkafka/callbacks/describe_consumer_groups_handler.rb +36 -0
- data/lib/rdkafka/callbacks/describe_topics_handler.rb +36 -0
- data/lib/rdkafka/callbacks.rb +10 -1
- data/lib/rdkafka/clients.rb +13 -46
- data/lib/rdkafka/consumer/message.rb +6 -9
- data/lib/rdkafka/defaults.rb +4 -0
- data/lib/rdkafka/producer/delivery_handle.rb +18 -16
- data/lib/rdkafka/producer/delivery_report.rb +51 -1
- data/lib/rdkafka/share_consumer.rb +54 -3
- data/lib/rdkafka/version.rb +3 -3
- data/lib/rdkafka.rb +10 -0
- data/package-lock.json +9 -9
- data/renovate.json +3 -0
- metadata +27 -2
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_EVENT_DESCRIBECLUSTER_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class DescribeClusterHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the describe cluster handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeCluster_result(event_ptr)
|
|
14
|
+
handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
15
|
+
|
|
16
|
+
return unless (handle = Rdkafka::Admin::DescribeClusterHandle.remove(handle_ptr.address))
|
|
17
|
+
|
|
18
|
+
return if resolve_operation_error(event_ptr, handle)
|
|
19
|
+
|
|
20
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
21
|
+
|
|
22
|
+
# Parsing must copy everything out of event-owned memory before the event is destroyed.
|
|
23
|
+
# An exception here is captured and re-raised on the waiting thread, since it cannot
|
|
24
|
+
# unwind through librdkafka native frames.
|
|
25
|
+
handle.result = begin
|
|
26
|
+
Rdkafka::Admin::DescribeClusterReport.new(result_ptr)
|
|
27
|
+
rescue => e
|
|
28
|
+
e
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
handle.unlock
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_EVENT_DESCRIBECONSUMERGROUPS_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class DescribeConsumerGroupsHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the describe-consumer-groups handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeConsumerGroups_result(event_ptr)
|
|
14
|
+
handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
15
|
+
|
|
16
|
+
return unless (handle = Rdkafka::Admin::DescribeConsumerGroupsHandle.remove(handle_ptr.address))
|
|
17
|
+
|
|
18
|
+
return if resolve_operation_error(event_ptr, handle)
|
|
19
|
+
|
|
20
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
21
|
+
|
|
22
|
+
# Parsing must copy everything out of event-owned memory before the event is destroyed.
|
|
23
|
+
# An exception here is captured and re-raised on the waiting thread, since it cannot
|
|
24
|
+
# unwind through librdkafka native frames.
|
|
25
|
+
handle.result = begin
|
|
26
|
+
Rdkafka::Admin::DescribeConsumerGroupsReport.new(result_ptr)
|
|
27
|
+
rescue => e
|
|
28
|
+
e
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
handle.unlock
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_EVENT_DESCRIBETOPICS_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class DescribeTopicsHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the describe topics handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
result_ptr = Rdkafka::Bindings.rd_kafka_event_DescribeTopics_result(event_ptr)
|
|
14
|
+
handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
15
|
+
|
|
16
|
+
return unless (handle = Rdkafka::Admin::DescribeTopicsHandle.remove(handle_ptr.address))
|
|
17
|
+
|
|
18
|
+
return if resolve_operation_error(event_ptr, handle)
|
|
19
|
+
|
|
20
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
21
|
+
|
|
22
|
+
# Parsing must copy everything out of event-owned memory before the event is destroyed.
|
|
23
|
+
# An exception here is captured and re-raised on the waiting thread, since it cannot
|
|
24
|
+
# unwind through librdkafka native frames.
|
|
25
|
+
handle.result = begin
|
|
26
|
+
Rdkafka::Admin::DescribeTopicsReport.new(result_ptr)
|
|
27
|
+
rescue => e
|
|
28
|
+
e
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
handle.unlock
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
data/lib/rdkafka/callbacks.rb
CHANGED
|
@@ -225,6 +225,9 @@ module Rdkafka
|
|
|
225
225
|
when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT then IncrementalAlterConfigsHandler
|
|
226
226
|
when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT then ListOffsetsHandler
|
|
227
227
|
when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTCONSUMERGROUPS_RESULT then ListConsumerGroupsHandler
|
|
228
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONSUMERGROUPS_RESULT then DescribeConsumerGroupsHandler
|
|
229
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECLUSTER_RESULT then DescribeClusterHandler
|
|
230
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBETOPICS_RESULT then DescribeTopicsHandler
|
|
228
231
|
end
|
|
229
232
|
|
|
230
233
|
handler&.call(event_ptr)
|
|
@@ -253,6 +256,9 @@ module Rdkafka
|
|
|
253
256
|
delivery_handle[:response] = message[:err]
|
|
254
257
|
delivery_handle[:partition] = message[:partition]
|
|
255
258
|
delivery_handle[:offset] = message[:offset]
|
|
259
|
+
delivery_handle[:status] = Rdkafka::Bindings.rd_kafka_message_status(message_ptr)
|
|
260
|
+
delivery_handle[:latency] = Rdkafka::Bindings.rd_kafka_message_latency(message_ptr)
|
|
261
|
+
delivery_handle[:broker_id] = Rdkafka::Bindings.rd_kafka_message_broker_id(message_ptr)
|
|
256
262
|
# The topic is not stored on the handle here: it is already set as the handle's
|
|
257
263
|
# `topic` Ruby attribute during produce, which spares a native string copy that
|
|
258
264
|
# would otherwise be allocated and retained for every message.
|
|
@@ -266,7 +272,10 @@ module Rdkafka
|
|
|
266
272
|
message[:offset],
|
|
267
273
|
topic_name,
|
|
268
274
|
message[:err],
|
|
269
|
-
delivery_handle.label
|
|
275
|
+
delivery_handle.label,
|
|
276
|
+
status: delivery_handle[:status],
|
|
277
|
+
latency: delivery_handle[:latency],
|
|
278
|
+
broker_id: delivery_handle[:broker_id]
|
|
270
279
|
),
|
|
271
280
|
delivery_handle
|
|
272
281
|
)
|
data/lib/rdkafka/clients.rb
CHANGED
|
@@ -4,22 +4,10 @@ module Rdkafka
|
|
|
4
4
|
# Tracks live public clients so they can be closed before Ruby begins forced finalization.
|
|
5
5
|
# @private
|
|
6
6
|
module Clients
|
|
7
|
-
# Ruby rewrote `ObjectSpace::WeakMap` in 3.3. On 3.2 and earlier its `[]` can hand back a
|
|
8
|
-
# foreign or already-freed object for a key, which raises `NoMethodError` or segfaults the VM
|
|
9
|
-
# the moment the value is touched. Track weakly only where that is safe and fall back to a
|
|
10
|
-
# strong registry - pruned as clients close - on older Rubies. The strong registry holds an
|
|
11
|
-
# open, un-closed client until shutdown, which is exactly when this cleanup needs it anyway.
|
|
12
|
-
WEAK_TRACKING = Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3")
|
|
13
|
-
|
|
14
7
|
@pid = Process.pid
|
|
15
8
|
@mutex = Mutex.new
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
@clients = ObjectSpace::WeakMap.new
|
|
19
|
-
@tokens = [].freeze
|
|
20
|
-
else
|
|
21
|
-
@clients = {}.freeze
|
|
22
|
-
end
|
|
9
|
+
@clients = ObjectSpace::WeakMap.new
|
|
10
|
+
@tokens = [].freeze
|
|
23
11
|
|
|
24
12
|
# Registers a successfully constructed client so it can be closed during process shutdown.
|
|
25
13
|
# @param client [Consumer, Producer, Admin] client to close during process shutdown
|
|
@@ -28,21 +16,14 @@ module Rdkafka
|
|
|
28
16
|
reset_after_fork
|
|
29
17
|
|
|
30
18
|
@mutex.synchronize do
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
@tokens = (@tokens + [token]).freeze
|
|
40
|
-
else
|
|
41
|
-
# Strong references, so drop clients that have already been closed to bound growth.
|
|
42
|
-
live = @clients.reject { |_id, existing| existing.closed? }
|
|
43
|
-
live[client.object_id] = client
|
|
44
|
-
@clients = live.freeze
|
|
45
|
-
end
|
|
19
|
+
live_clients
|
|
20
|
+
|
|
21
|
+
# The strongly-held token is the key and the client is the weak value, so registering a
|
|
22
|
+
# client never keeps it alive. Enumerating weak keys can expose reclaimed slots, so we
|
|
23
|
+
# look the value up by token instead.
|
|
24
|
+
token = Object.new
|
|
25
|
+
@clients[token] = client
|
|
26
|
+
@tokens = (@tokens + [token]).freeze
|
|
46
27
|
end
|
|
47
28
|
|
|
48
29
|
client
|
|
@@ -54,7 +35,7 @@ module Rdkafka
|
|
|
54
35
|
def self.close_all
|
|
55
36
|
return unless @pid == Process.pid
|
|
56
37
|
|
|
57
|
-
clients = @mutex.synchronize {
|
|
38
|
+
clients = @mutex.synchronize { live_clients }
|
|
58
39
|
first_error = nil
|
|
59
40
|
|
|
60
41
|
clients.each do |client|
|
|
@@ -66,16 +47,7 @@ module Rdkafka
|
|
|
66
47
|
raise first_error if first_error
|
|
67
48
|
end
|
|
68
49
|
|
|
69
|
-
# Returns the currently registered live clients.
|
|
70
|
-
# @return [Array<Consumer, Producer, Admin>] live clients
|
|
71
|
-
# @private
|
|
72
|
-
def self.current_clients
|
|
73
|
-
WEAK_TRACKING ? live_clients : @clients.values
|
|
74
|
-
end
|
|
75
|
-
private_class_method :current_clients
|
|
76
|
-
|
|
77
50
|
# Returns the live clients and releases tokens whose clients have been collected.
|
|
78
|
-
# Weak-tracking Rubies only.
|
|
79
51
|
# @return [Array<Consumer, Producer, Admin>] live clients
|
|
80
52
|
# @private
|
|
81
53
|
def self.live_clients
|
|
@@ -99,13 +71,8 @@ module Rdkafka
|
|
|
99
71
|
def self.reset_after_fork
|
|
100
72
|
return if @pid == Process.pid
|
|
101
73
|
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
@tokens = [].freeze
|
|
105
|
-
else
|
|
106
|
-
@clients = {}.freeze
|
|
107
|
-
end
|
|
108
|
-
|
|
74
|
+
@clients = ObjectSpace::WeakMap.new
|
|
75
|
+
@tokens = [].freeze
|
|
109
76
|
@pid = Process.pid
|
|
110
77
|
@mutex = Mutex.new
|
|
111
78
|
end
|
|
@@ -35,19 +35,16 @@ module Rdkafka
|
|
|
35
35
|
# @param native_message [Rdkafka::Bindings::Message] native message struct from librdkafka
|
|
36
36
|
def initialize(native_message)
|
|
37
37
|
# Set topic
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
end
|
|
38
|
+
rkt = native_message[:rkt]
|
|
39
|
+
@topic = Rdkafka::Bindings.rd_kafka_topic_name(rkt) unless rkt.null?
|
|
41
40
|
# Set partition
|
|
42
41
|
@partition = native_message[:partition]
|
|
43
42
|
# Set payload
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
end
|
|
43
|
+
payload = native_message[:payload]
|
|
44
|
+
@payload = payload.read_string(native_message[:len]) unless payload.null?
|
|
47
45
|
# Set key
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
end
|
|
46
|
+
key = native_message[:key]
|
|
47
|
+
@key = key.read_string(native_message[:key_len]) unless key.null?
|
|
51
48
|
# Set offset
|
|
52
49
|
@offset = native_message[:offset]
|
|
53
50
|
# Set timestamp
|
data/lib/rdkafka/defaults.rb
CHANGED
|
@@ -51,6 +51,10 @@ module Rdkafka
|
|
|
51
51
|
# @see ShareConsumer#poll
|
|
52
52
|
SHARE_CONSUMER_POLL_TIMEOUT_MS = 250
|
|
53
53
|
|
|
54
|
+
# Share consumer timeout for events_poll (0 = non-blocking async)
|
|
55
|
+
# @see ShareConsumer#events_poll
|
|
56
|
+
SHARE_CONSUMER_EVENTS_POLL_TIMEOUT_MS = 0
|
|
57
|
+
|
|
54
58
|
# Share consumer timeout for synchronous acknowledgement commits
|
|
55
59
|
# @see ShareConsumer#commit_sync
|
|
56
60
|
SHARE_CONSUMER_COMMIT_SYNC_TIMEOUT_MS = 5_000
|
|
@@ -7,7 +7,10 @@ module Rdkafka
|
|
|
7
7
|
layout :pending, :bool,
|
|
8
8
|
:response, :int,
|
|
9
9
|
:partition, :int,
|
|
10
|
-
:offset, :int64
|
|
10
|
+
:offset, :int64,
|
|
11
|
+
:status, :int,
|
|
12
|
+
:latency, :int64,
|
|
13
|
+
:broker_id, :int32
|
|
11
14
|
|
|
12
15
|
# @return [Object, nil] label set during message production or nil by default
|
|
13
16
|
attr_accessor :label
|
|
@@ -24,23 +27,22 @@ module Rdkafka
|
|
|
24
27
|
|
|
25
28
|
# @return [DeliveryReport] a report on the delivery of the message
|
|
26
29
|
def create_result
|
|
27
|
-
if self[:response] == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
28
|
-
|
|
29
|
-
self[:partition],
|
|
30
|
-
self[:offset],
|
|
31
|
-
topic,
|
|
32
|
-
nil,
|
|
33
|
-
label
|
|
34
|
-
)
|
|
30
|
+
error = if self[:response] == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
31
|
+
nil
|
|
35
32
|
else
|
|
36
|
-
|
|
37
|
-
self[:partition],
|
|
38
|
-
self[:offset],
|
|
39
|
-
topic,
|
|
40
|
-
Rdkafka::RdkafkaError.build(self[:response]),
|
|
41
|
-
label
|
|
42
|
-
)
|
|
33
|
+
Rdkafka::RdkafkaError.build(self[:response])
|
|
43
34
|
end
|
|
35
|
+
|
|
36
|
+
DeliveryReport.new(
|
|
37
|
+
self[:partition],
|
|
38
|
+
self[:offset],
|
|
39
|
+
topic,
|
|
40
|
+
error,
|
|
41
|
+
label,
|
|
42
|
+
status: self[:status],
|
|
43
|
+
latency: self[:latency],
|
|
44
|
+
broker_id: self[:broker_id]
|
|
45
|
+
)
|
|
44
46
|
end
|
|
45
47
|
end
|
|
46
48
|
end
|
|
@@ -25,6 +25,26 @@ module Rdkafka
|
|
|
25
25
|
# @return [Object, nil] label set during message production or nil by default
|
|
26
26
|
attr_reader :label
|
|
27
27
|
|
|
28
|
+
# Persistence status of the message, one of `Bindings::RD_KAFKA_MSG_STATUS_*`:
|
|
29
|
+
# - `RD_KAFKA_MSG_STATUS_NOT_PERSISTED` - the message was not written to the broker. A retry
|
|
30
|
+
# risks reordering, but not a duplicate.
|
|
31
|
+
# - `RD_KAFKA_MSG_STATUS_POSSIBLY_PERSISTED` - the message may have been written (e.g. the
|
|
32
|
+
# request timed out before an ack). A retry risks reordering and a duplicate.
|
|
33
|
+
# - `RD_KAFKA_MSG_STATUS_PERSISTED` - the broker acknowledged the message. Trust this only
|
|
34
|
+
# with `acks=all`.
|
|
35
|
+
# `nil` when the report was not built from a delivery callback.
|
|
36
|
+
# @return [Integer, nil]
|
|
37
|
+
attr_reader :status
|
|
38
|
+
|
|
39
|
+
# Time from the `produce` call until the delivery report, in microseconds, or `nil` when
|
|
40
|
+
# not available
|
|
41
|
+
# @return [Integer, nil]
|
|
42
|
+
attr_reader :latency
|
|
43
|
+
|
|
44
|
+
# Id of the broker the message was produced to, or `nil` when not known
|
|
45
|
+
# @return [Integer, nil]
|
|
46
|
+
attr_reader :broker_id
|
|
47
|
+
|
|
28
48
|
# We alias the `#topic_name` under `#topic` to make this consistent with `Consumer::Message`
|
|
29
49
|
# where the topic name is under `#topic` method. That way we have a consistent name that
|
|
30
50
|
# is present in both places
|
|
@@ -38,12 +58,42 @@ module Rdkafka
|
|
|
38
58
|
# @param topic_name [String, nil] topic name
|
|
39
59
|
# @param error [Integer, nil] error code if any
|
|
40
60
|
# @param label [Object, nil] user-defined label
|
|
41
|
-
|
|
61
|
+
# @param status [Integer, nil] `Bindings::RD_KAFKA_MSG_STATUS_*` persistence status
|
|
62
|
+
# @param latency [Integer, nil] produce latency in microseconds, negative when not available
|
|
63
|
+
# @param broker_id [Integer, nil] broker id, negative when not known
|
|
64
|
+
def initialize(
|
|
65
|
+
partition,
|
|
66
|
+
offset,
|
|
67
|
+
topic_name = nil,
|
|
68
|
+
error = nil,
|
|
69
|
+
label = nil,
|
|
70
|
+
status: nil,
|
|
71
|
+
latency: nil,
|
|
72
|
+
broker_id: nil
|
|
73
|
+
)
|
|
42
74
|
@partition = partition
|
|
43
75
|
@offset = offset
|
|
44
76
|
@topic_name = topic_name
|
|
45
77
|
@error = error
|
|
46
78
|
@label = label
|
|
79
|
+
@status = status
|
|
80
|
+
@latency = (latency.nil? || latency.negative?) ? nil : latency
|
|
81
|
+
@broker_id = (broker_id.nil? || broker_id.negative?) ? nil : broker_id
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# @return [Boolean] true when the broker acknowledged the message
|
|
85
|
+
def persisted?
|
|
86
|
+
status == Bindings::RD_KAFKA_MSG_STATUS_PERSISTED
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# @return [Boolean] true when the message may have been written, so a retry may duplicate it
|
|
90
|
+
def possibly_persisted?
|
|
91
|
+
status == Bindings::RD_KAFKA_MSG_STATUS_POSSIBLY_PERSISTED
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# @return [Boolean] true when the message was not written, so a retry does not duplicate it
|
|
95
|
+
def not_persisted?
|
|
96
|
+
status == Bindings::RD_KAFKA_MSG_STATUS_NOT_PERSISTED
|
|
47
97
|
end
|
|
48
98
|
end
|
|
49
99
|
end
|
|
@@ -22,9 +22,10 @@ module Rdkafka
|
|
|
22
22
|
# share consumer per thread. librdkafka enforces this and calls from a second thread raise
|
|
23
23
|
# an `RdkafkaError` with code `conflict`. This class is deliberately not built on
|
|
24
24
|
# {NativeKafka}: there is no background polling thread (statistics, error and log callbacks
|
|
25
|
-
# are all serviced from within {#poll}
|
|
26
|
-
#
|
|
27
|
-
#
|
|
25
|
+
# are all serviced from within {#poll}, or from {#events_poll} when the caller is not polling
|
|
26
|
+
# for records) and the native handle must never be used from a polling thread, so only the
|
|
27
|
+
# {#close} interaction is synchronized here. {#close} may be called from any thread and waits
|
|
28
|
+
# for in-flight calls to finish before tearing down.
|
|
28
29
|
#
|
|
29
30
|
# Preview limitations inherited from librdkafka worth knowing at this layer:
|
|
30
31
|
# - `max.poll.records` (default 500) is a soft bound
|
|
@@ -166,6 +167,12 @@ module Rdkafka
|
|
|
166
167
|
|
|
167
168
|
# Unsubscribes from all subscribed topics and leaves the share group assignment.
|
|
168
169
|
#
|
|
170
|
+
# Closes the share sessions like {#close} does: acknowledgements made so far (every delivered
|
|
171
|
+
# record in implicit mode) are committed, and records that were delivered but not
|
|
172
|
+
# acknowledged, or acquired by a fetch still in flight, are released to the other members
|
|
173
|
+
# right away. Blocks until the sessions are closed, which takes up to `fetch.wait.max.ms`
|
|
174
|
+
# when a fetch is in flight.
|
|
175
|
+
#
|
|
169
176
|
# @return [nil]
|
|
170
177
|
# @raise [RdkafkaError] When unsubscribing fails
|
|
171
178
|
def unsubscribe
|
|
@@ -264,6 +271,50 @@ module Rdkafka
|
|
|
264
271
|
end
|
|
265
272
|
end
|
|
266
273
|
|
|
274
|
+
# Polls the main rdkafka queue (not the share one), serving queued global callbacks without
|
|
275
|
+
# acquiring any records.
|
|
276
|
+
#
|
|
277
|
+
# Events will cause application-provided callbacks to be called:
|
|
278
|
+
# - error callbacks
|
|
279
|
+
# - stats callbacks
|
|
280
|
+
# - log and OAuthBearer token refresh callbacks
|
|
281
|
+
#
|
|
282
|
+
# Lets a caller keep those callbacks flowing while it is not polling for records (waiting on
|
|
283
|
+
# in-flight work, quieting, or draining on shutdown). Does **NOT** replace {#poll}, which
|
|
284
|
+
# drains the same queue while acquiring records; the "one share consumer per thread" rule
|
|
285
|
+
# means the two never run concurrently.
|
|
286
|
+
#
|
|
287
|
+
# @param timeout_ms [Integer] poll timeout. If set to 0 will run async, when set to -1 will
|
|
288
|
+
# block until any events available.
|
|
289
|
+
# @return [Integer] number of events served
|
|
290
|
+
# @raise [ClosedConsumerError] when the consumer is closed or closing
|
|
291
|
+
def events_poll(timeout_ms = Defaults::SHARE_CONSUMER_EVENTS_POLL_TIMEOUT_MS)
|
|
292
|
+
with_native(__method__) do |native|
|
|
293
|
+
# The main queue lives on the rd_kafka_t the share handle wraps (rkshare_rk), which is the
|
|
294
|
+
# handle librdkafka hands the callbacks; rd_kafka_poll has to run against it (see #name).
|
|
295
|
+
inner = Rdkafka::Bindings::NativeShareConsumer.new(native)[:rkshare_rk]
|
|
296
|
+
Rdkafka::Bindings.rd_kafka_poll(inner, timeout_ms)
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
# Polls the main rdkafka queue without releasing the GVL (Global VM Lock).
|
|
301
|
+
#
|
|
302
|
+
# This is more efficient than {#events_poll} for non-blocking `poll(0)` calls, particularly
|
|
303
|
+
# useful in fiber scheduler contexts where GVL release/reacquire overhead is wasteful since we
|
|
304
|
+
# don't expect to wait.
|
|
305
|
+
#
|
|
306
|
+
# @param timeout_ms [Integer] poll timeout (default: 0 for non-blocking)
|
|
307
|
+
# @return [Integer] number of events served
|
|
308
|
+
# @raise [ClosedConsumerError] when the consumer is closed or closing
|
|
309
|
+
#
|
|
310
|
+
# @see #events_poll for more details on when to use this method
|
|
311
|
+
def events_poll_nb(timeout_ms = 0)
|
|
312
|
+
with_native(__method__) do |native|
|
|
313
|
+
inner = Rdkafka::Bindings::NativeShareConsumer.new(native)[:rkshare_rk]
|
|
314
|
+
Rdkafka::Bindings.rd_kafka_poll_nb(inner, timeout_ms)
|
|
315
|
+
end
|
|
316
|
+
end
|
|
317
|
+
|
|
267
318
|
# Acknowledges a message delivered by {#poll} in explicit acknowledgement mode
|
|
268
319
|
# (`share.acknowledgement.mode` set to `explicit`).
|
|
269
320
|
#
|
data/lib/rdkafka/version.rb
CHANGED
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
module Rdkafka
|
|
4
4
|
# Current rdkafka-ruby gem version
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.31.0"
|
|
6
6
|
# Target librdkafka version to be used
|
|
7
|
-
LIBRDKAFKA_VERSION = "2.
|
|
7
|
+
LIBRDKAFKA_VERSION = "2.16.0"
|
|
8
8
|
# SHA256 hash of the librdkafka source tarball for verification
|
|
9
|
-
LIBRDKAFKA_SOURCE_SHA256 = "
|
|
9
|
+
LIBRDKAFKA_SOURCE_SHA256 = "e6b61de61d3282879a88e4ee3d9f634a8b05bf78a9198dfc25c4820c0c9ed231"
|
|
10
10
|
end
|
data/lib/rdkafka.rb
CHANGED
|
@@ -41,6 +41,13 @@ require "rdkafka/admin/list_offsets_handle"
|
|
|
41
41
|
require "rdkafka/admin/list_offsets_report"
|
|
42
42
|
require "rdkafka/admin/list_consumer_groups_handle"
|
|
43
43
|
require "rdkafka/admin/list_consumer_groups_report"
|
|
44
|
+
require "rdkafka/admin/describe_result_parsing"
|
|
45
|
+
require "rdkafka/admin/describe_cluster_handle"
|
|
46
|
+
require "rdkafka/admin/describe_cluster_report"
|
|
47
|
+
require "rdkafka/admin/describe_topics_handle"
|
|
48
|
+
require "rdkafka/admin/describe_topics_report"
|
|
49
|
+
require "rdkafka/admin/describe_consumer_groups_handle"
|
|
50
|
+
require "rdkafka/admin/describe_consumer_groups_report"
|
|
44
51
|
require "rdkafka/admin/acl_binding_result"
|
|
45
52
|
require "rdkafka/admin/config_binding_result"
|
|
46
53
|
require "rdkafka/admin/config_resource_binding_result"
|
|
@@ -61,6 +68,9 @@ require "rdkafka/callbacks/describe_configs_handler"
|
|
|
61
68
|
require "rdkafka/callbacks/incremental_alter_configs_handler"
|
|
62
69
|
require "rdkafka/callbacks/list_offsets_handler"
|
|
63
70
|
require "rdkafka/callbacks/list_consumer_groups_handler"
|
|
71
|
+
require "rdkafka/callbacks/describe_cluster_handler"
|
|
72
|
+
require "rdkafka/callbacks/describe_topics_handler"
|
|
73
|
+
require "rdkafka/callbacks/describe_consumer_groups_handler"
|
|
64
74
|
require "rdkafka/config"
|
|
65
75
|
require "rdkafka/consumer"
|
|
66
76
|
require "rdkafka/consumer/headers"
|
data/package-lock.json
CHANGED
|
@@ -103,9 +103,9 @@
|
|
|
103
103
|
}
|
|
104
104
|
},
|
|
105
105
|
"node_modules/fastq": {
|
|
106
|
-
"version": "1.20.
|
|
107
|
-
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.
|
|
108
|
-
"integrity": "sha512-
|
|
106
|
+
"version": "1.20.3",
|
|
107
|
+
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz",
|
|
108
|
+
"integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==",
|
|
109
109
|
"dev": true,
|
|
110
110
|
"license": "ISC",
|
|
111
111
|
"dependencies": {
|
|
@@ -286,9 +286,9 @@
|
|
|
286
286
|
}
|
|
287
287
|
},
|
|
288
288
|
"node_modules/smol-toml": {
|
|
289
|
-
"version": "1.
|
|
290
|
-
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.
|
|
291
|
-
"integrity": "sha512-
|
|
289
|
+
"version": "1.9.0",
|
|
290
|
+
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.9.0.tgz",
|
|
291
|
+
"integrity": "sha512-hpd+HLON7HdZXqYchMM/+LaTTbdK0AU3NngIJ4KVyWbY9bfQqdL9cD+4yf6dUoU2Ap4VsU0JkQi6FxAI1B2mXQ==",
|
|
292
292
|
"dev": true,
|
|
293
293
|
"license": "BSD-3-Clause",
|
|
294
294
|
"engines": {
|
|
@@ -312,9 +312,9 @@
|
|
|
312
312
|
}
|
|
313
313
|
},
|
|
314
314
|
"node_modules/yaml": {
|
|
315
|
-
"version": "2.9.
|
|
316
|
-
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.
|
|
317
|
-
"integrity": "sha512-
|
|
315
|
+
"version": "2.9.1",
|
|
316
|
+
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz",
|
|
317
|
+
"integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==",
|
|
318
318
|
"dev": true,
|
|
319
319
|
"license": "ISC",
|
|
320
320
|
"bin": {
|
data/renovate.json
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: karafka-rdkafka
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.31.0
|
|
5
5
|
platform: aarch64-linux-gnu
|
|
6
6
|
authors:
|
|
7
7
|
- Thijs Cadier
|
|
@@ -122,8 +122,15 @@ files:
|
|
|
122
122
|
- lib/rdkafka/admin/delete_topic_report.rb
|
|
123
123
|
- lib/rdkafka/admin/describe_acl_handle.rb
|
|
124
124
|
- lib/rdkafka/admin/describe_acl_report.rb
|
|
125
|
+
- lib/rdkafka/admin/describe_cluster_handle.rb
|
|
126
|
+
- lib/rdkafka/admin/describe_cluster_report.rb
|
|
125
127
|
- lib/rdkafka/admin/describe_configs_handle.rb
|
|
126
128
|
- lib/rdkafka/admin/describe_configs_report.rb
|
|
129
|
+
- lib/rdkafka/admin/describe_consumer_groups_handle.rb
|
|
130
|
+
- lib/rdkafka/admin/describe_consumer_groups_report.rb
|
|
131
|
+
- lib/rdkafka/admin/describe_result_parsing.rb
|
|
132
|
+
- lib/rdkafka/admin/describe_topics_handle.rb
|
|
133
|
+
- lib/rdkafka/admin/describe_topics_report.rb
|
|
127
134
|
- lib/rdkafka/admin/incremental_alter_configs_handle.rb
|
|
128
135
|
- lib/rdkafka/admin/incremental_alter_configs_report.rb
|
|
129
136
|
- lib/rdkafka/admin/list_consumer_groups_handle.rb
|
|
@@ -131,6 +138,21 @@ files:
|
|
|
131
138
|
- lib/rdkafka/admin/list_offsets_handle.rb
|
|
132
139
|
- lib/rdkafka/admin/list_offsets_report.rb
|
|
133
140
|
- lib/rdkafka/bindings.rb
|
|
141
|
+
- lib/rdkafka/bindings/admin.rb
|
|
142
|
+
- lib/rdkafka/bindings/admin/acls.rb
|
|
143
|
+
- lib/rdkafka/bindings/admin/configs.rb
|
|
144
|
+
- lib/rdkafka/bindings/admin/list_offsets.rb
|
|
145
|
+
- lib/rdkafka/bindings/callbacks.rb
|
|
146
|
+
- lib/rdkafka/bindings/configuration.rb
|
|
147
|
+
- lib/rdkafka/bindings/consumer.rb
|
|
148
|
+
- lib/rdkafka/bindings/errors.rb
|
|
149
|
+
- lib/rdkafka/bindings/handle.rb
|
|
150
|
+
- lib/rdkafka/bindings/messages.rb
|
|
151
|
+
- lib/rdkafka/bindings/metadata.rb
|
|
152
|
+
- lib/rdkafka/bindings/polling.rb
|
|
153
|
+
- lib/rdkafka/bindings/producer.rb
|
|
154
|
+
- lib/rdkafka/bindings/share_consumer.rb
|
|
155
|
+
- lib/rdkafka/bindings/structs.rb
|
|
134
156
|
- lib/rdkafka/callbacks.rb
|
|
135
157
|
- lib/rdkafka/callbacks/alter_consumer_group_offsets_handler.rb
|
|
136
158
|
- lib/rdkafka/callbacks/base_handler.rb
|
|
@@ -143,7 +165,10 @@ files:
|
|
|
143
165
|
- lib/rdkafka/callbacks/delete_records_handler.rb
|
|
144
166
|
- lib/rdkafka/callbacks/delete_topic_handler.rb
|
|
145
167
|
- lib/rdkafka/callbacks/describe_acl_handler.rb
|
|
168
|
+
- lib/rdkafka/callbacks/describe_cluster_handler.rb
|
|
146
169
|
- lib/rdkafka/callbacks/describe_configs_handler.rb
|
|
170
|
+
- lib/rdkafka/callbacks/describe_consumer_groups_handler.rb
|
|
171
|
+
- lib/rdkafka/callbacks/describe_topics_handler.rb
|
|
147
172
|
- lib/rdkafka/callbacks/incremental_alter_configs_handler.rb
|
|
148
173
|
- lib/rdkafka/callbacks/list_consumer_groups_handler.rb
|
|
149
174
|
- lib/rdkafka/callbacks/list_offsets_handler.rb
|
|
@@ -191,7 +216,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
191
216
|
requirements:
|
|
192
217
|
- - ">="
|
|
193
218
|
- !ruby/object:Gem::Version
|
|
194
|
-
version: '3.
|
|
219
|
+
version: '3.3'
|
|
195
220
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
196
221
|
requirements:
|
|
197
222
|
- - ">="
|