karafka-rdkafka 0.24.0.rc4-aarch64-linux-gnu → 0.25.0-aarch64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -1
- data/README.md +2 -1
- data/bin/verify_kafka_warnings +2 -0
- data/docker-compose-ssl.yml +2 -2
- data/docker-compose.yml +2 -2
- data/ext/librdkafka.so +0 -0
- data/lib/rdkafka/admin/list_offsets_handle.rb +36 -0
- data/lib/rdkafka/admin/list_offsets_report.rb +51 -0
- data/lib/rdkafka/admin.rb +105 -6
- data/lib/rdkafka/bindings.rb +26 -2
- data/lib/rdkafka/callbacks.rb +43 -0
- data/lib/rdkafka/config.rb +2 -2
- data/lib/rdkafka/consumer.rb +3 -6
- data/lib/rdkafka/error.rb +24 -7
- data/lib/rdkafka/version.rb +3 -3
- data/lib/rdkafka.rb +2 -0
- data/package-lock.json +3 -3
- data/renovate.json +5 -1
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1fc68ea9e1153d48f0385af08a7e3f181eecc58ec8fdcb1799d6df91b62794be
|
|
4
|
+
data.tar.gz: 780de63d72bce253081b6c8dbea38c59ba47749fa28b190e21e6b93833179757
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3186a7ffd882a2ca7cce415b0211ec8204d2800eba1fc8768fb2985b7cf42c5c8b6c49e7e11bee3637dfc56efdc9511cfa94820631539dc98a782b44c33b0afe
|
|
7
|
+
data.tar.gz: 648ee9517e2d8377dd0adccc9a94f058734c76e71c79716d6c7c681d45abb437fccb46f56c5c48a86cf544d9756805ab099e07482d8727fd46a84c35f54c1e10
|
data/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,19 @@
|
|
|
1
1
|
# Rdkafka Changelog
|
|
2
2
|
|
|
3
|
-
## 0.
|
|
3
|
+
## 0.25.0 (2026-04-02)
|
|
4
|
+
- **[Feature]** Support `rd_kafka_ListOffsets` admin API for querying partition offsets by specification (earliest, latest, max_timestamp, or by timestamp) without requiring a consumer group (from upstream).
|
|
5
|
+
- **[Feature]** Extend `Rdkafka::RdkafkaError` with `instance_name` attribute containing the `rd_kafka_name` for tying errors back to specific native Kafka instances (from upstream).
|
|
6
|
+
- [Enhancement] Bump librdkafka to `2.13.2`
|
|
7
|
+
- [Enhancement] Update `confluentinc/cp-kafka` Docker image to `8.2.0` (from upstream).
|
|
8
|
+
- [Enhancement] Disable broker-side auto topic creation to prevent race condition warnings with Kafka 8.2.0 (from upstream).
|
|
9
|
+
- [Enhancement] Embed a per-file SPEC_HASH in test topic and consumer group names for tracing Kafka warnings back to specific spec files (from upstream).
|
|
10
|
+
- [Fix] Fix test topic auto-creation race conditions causing `TOPIC_ALREADY_EXISTS` warnings (from upstream).
|
|
11
|
+
- [Fix] Fix `describe_configs` specs to not depend on config ordering (from upstream).
|
|
12
|
+
- [Fix] Register `ObjectSpace.define_finalizer` in `Rdkafka::Consumer` to prevent segfaults when a consumer is GC'd without being explicitly closed (from upstream).
|
|
13
|
+
- [Fix] Remove dead `#finalizer` instance methods from `Consumer` and `Admin` that could never work as GC finalizers (from upstream).
|
|
14
|
+
- [Fix] Prevent cascading test failures in admin specs when a single handle leaks into the registry (from upstream).
|
|
15
|
+
|
|
16
|
+
## 0.24.0 (2026-02-25)
|
|
4
17
|
- **[Feature]** Add `Producer#queue_size` (and `#queue_length` alias) to report the number of messages waiting in the librdkafka output queue. Useful for monitoring producer backpressure, implementing custom flow control, debugging message delivery issues, and graceful shutdown logic.
|
|
5
18
|
- **[Feature]** Add fiber scheduler API for integration with Ruby fiber schedulers (Falcon, Async) and custom event loops (from upstream). Expose `enable_queue_io_events` and `enable_background_queue_io_events` methods on `Consumer`, `Producer`, and `Admin`.
|
|
6
19
|
- **[Deprecation]** `AbstractHandle#wait` parameter `max_wait_timeout` (seconds) is deprecated in favor of `max_wait_timeout_ms` (milliseconds). The old parameter still works with backwards compatibility but will be removed in v1.0.0.
|
data/README.md
CHANGED
|
@@ -63,7 +63,8 @@ Contributions should generally be made to the upstream [rdkafka-ruby repository]
|
|
|
63
63
|
|
|
64
64
|
| rdkafka-ruby | librdkafka | patches |
|
|
65
65
|
|-|-|-|
|
|
66
|
-
| 0.
|
|
66
|
+
| 0.25.x (2026-04-02) | 2.13.2 (2026-03-30) | yes |
|
|
67
|
+
| 0.24.x (2026-02-25) | 2.13.0 (2026-01-05) | yes |
|
|
67
68
|
| 0.23.x (2025-11-01) | 2.12.1 (2025-10-16) | yes |
|
|
68
69
|
| 0.22.x (2025-09-26) | 2.11.1 (2025-08-18) | yes |
|
|
69
70
|
| 0.21.x (2025-08-18) | 2.11.0 (2025-07-03) | yes |
|
data/bin/verify_kafka_warnings
CHANGED
data/docker-compose-ssl.yml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
services:
|
|
2
2
|
kafka:
|
|
3
3
|
container_name: kafka
|
|
4
|
-
image: confluentinc/cp-kafka:8.
|
|
4
|
+
image: confluentinc/cp-kafka:8.2.0
|
|
5
5
|
ports:
|
|
6
6
|
- 9092:9092 # Support PLAINTEXT so we can run one docker setup for SSL and PLAINTEXT
|
|
7
7
|
- 9093:9093
|
|
@@ -19,7 +19,7 @@ services:
|
|
|
19
19
|
KAFKA_BROKER_ID: 1
|
|
20
20
|
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9094
|
|
21
21
|
ALLOW_PLAINTEXT_LISTENER: 'yes'
|
|
22
|
-
KAFKA_AUTO_CREATE_TOPICS_ENABLE: '
|
|
22
|
+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
|
|
23
23
|
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
|
|
24
24
|
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
|
|
25
25
|
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
|
data/docker-compose.yml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
services:
|
|
2
2
|
kafka:
|
|
3
3
|
container_name: kafka
|
|
4
|
-
image: confluentinc/cp-kafka:8.
|
|
4
|
+
image: confluentinc/cp-kafka:8.2.0
|
|
5
5
|
|
|
6
6
|
ports:
|
|
7
7
|
- 9092:9092
|
|
@@ -18,7 +18,7 @@ services:
|
|
|
18
18
|
KAFKA_BROKER_ID: 1
|
|
19
19
|
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
|
|
20
20
|
ALLOW_PLAINTEXT_LISTENER: 'yes'
|
|
21
|
-
KAFKA_AUTO_CREATE_TOPICS_ENABLE: '
|
|
21
|
+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'
|
|
22
22
|
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
|
|
23
23
|
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
|
|
24
24
|
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
|
data/ext/librdkafka.so
CHANGED
|
Binary file
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
class Admin
|
|
5
|
+
# Handle for list offsets operation
|
|
6
|
+
class ListOffsetsHandle < AbstractHandle
|
|
7
|
+
layout :pending, :bool,
|
|
8
|
+
:response, :int,
|
|
9
|
+
:response_string, :pointer,
|
|
10
|
+
:result_infos, :pointer,
|
|
11
|
+
:result_count, :int
|
|
12
|
+
|
|
13
|
+
# @return [String] the name of the operation.
|
|
14
|
+
def operation_name
|
|
15
|
+
"list offsets"
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# @return [ListOffsetsReport] instance with partition offset information.
|
|
19
|
+
def create_result
|
|
20
|
+
ListOffsetsReport.new(
|
|
21
|
+
result_infos: self[:result_infos],
|
|
22
|
+
result_count: self[:result_count]
|
|
23
|
+
)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Raises an error if the operation failed
|
|
27
|
+
# @raise [RdkafkaError]
|
|
28
|
+
def raise_error
|
|
29
|
+
raise RdkafkaError.new(
|
|
30
|
+
self[:response],
|
|
31
|
+
broker_message: self[:response_string].read_string
|
|
32
|
+
)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
class Admin
|
|
5
|
+
# Report for list offsets operation result
|
|
6
|
+
class ListOffsetsReport
|
|
7
|
+
attr_reader :offsets
|
|
8
|
+
|
|
9
|
+
# @param result_infos [FFI::Pointer] pointer to result info array
|
|
10
|
+
# @param result_count [Integer] number of result info entries
|
|
11
|
+
def initialize(result_infos:, result_count:)
|
|
12
|
+
@offsets = []
|
|
13
|
+
|
|
14
|
+
return if result_infos.null?
|
|
15
|
+
|
|
16
|
+
result_infos
|
|
17
|
+
.read_array_of_pointer(result_count)
|
|
18
|
+
.each { |result_info_ptr| validate!(result_info_ptr) }
|
|
19
|
+
.each do |result_info_ptr|
|
|
20
|
+
tp_ptr = Bindings.rd_kafka_ListOffsetsResultInfo_topic_partition(result_info_ptr)
|
|
21
|
+
tp = Bindings::TopicPartition.new(tp_ptr)
|
|
22
|
+
timestamp = Bindings.rd_kafka_ListOffsetsResultInfo_timestamp(result_info_ptr)
|
|
23
|
+
leader_epoch = Bindings.rd_kafka_topic_partition_get_leader_epoch(tp_ptr)
|
|
24
|
+
|
|
25
|
+
@offsets << {
|
|
26
|
+
topic: tp[:topic],
|
|
27
|
+
partition: tp[:partition],
|
|
28
|
+
offset: tp[:offset],
|
|
29
|
+
timestamp: timestamp,
|
|
30
|
+
leader_epoch: (leader_epoch == -1) ? nil : leader_epoch
|
|
31
|
+
}
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
# Validates the partition result and raises an error if invalid
|
|
38
|
+
# @param result_info_ptr [FFI::Pointer] pointer to the result info
|
|
39
|
+
# @raise [RdkafkaError] when the partition has an error
|
|
40
|
+
def validate!(result_info_ptr)
|
|
41
|
+
tp_ptr = Bindings.rd_kafka_ListOffsetsResultInfo_topic_partition(result_info_ptr)
|
|
42
|
+
tp = Bindings::TopicPartition.new(tp_ptr)
|
|
43
|
+
code = tp[:err]
|
|
44
|
+
|
|
45
|
+
return if code.zero?
|
|
46
|
+
|
|
47
|
+
raise RdkafkaError.new(code)
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
data/lib/rdkafka/admin.rb
CHANGED
|
@@ -130,12 +130,6 @@ module Rdkafka
|
|
|
130
130
|
end
|
|
131
131
|
end
|
|
132
132
|
|
|
133
|
-
# @return [Proc] finalizer proc for closing the admin
|
|
134
|
-
# @private
|
|
135
|
-
def finalizer
|
|
136
|
-
->(_) { close }
|
|
137
|
-
end
|
|
138
|
-
|
|
139
133
|
# Performs the metadata request using admin
|
|
140
134
|
#
|
|
141
135
|
# @param topic_name [String, nil] metadat about particular topic or all if nil
|
|
@@ -882,6 +876,111 @@ module Rdkafka
|
|
|
882
876
|
handle
|
|
883
877
|
end
|
|
884
878
|
|
|
879
|
+
# Queries partition offsets by specification (earliest, latest, max_timestamp, or by
|
|
880
|
+
# timestamp) without requiring a consumer group.
|
|
881
|
+
#
|
|
882
|
+
# @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
|
|
883
|
+
# arrays of partition offset specifications. Each specification is a hash with:
|
|
884
|
+
# - `:partition` [Integer] partition number
|
|
885
|
+
# - `:offset` [Symbol, Integer] offset specification - `:earliest`, `:latest`,
|
|
886
|
+
# `:max_timestamp`, or an integer timestamp in milliseconds
|
|
887
|
+
# @param isolation_level [Integer, nil] optional isolation level:
|
|
888
|
+
# - `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED` (0) - default
|
|
889
|
+
# - `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` (1)
|
|
890
|
+
#
|
|
891
|
+
# @return [ListOffsetsHandle] handle that can be used to wait for the result
|
|
892
|
+
#
|
|
893
|
+
# @raise [ClosedAdminError] when the admin is closed
|
|
894
|
+
# @raise [ConfigError] when the background queue is unavailable
|
|
895
|
+
#
|
|
896
|
+
# @example Query earliest and latest offsets
|
|
897
|
+
# handle = admin.list_offsets(
|
|
898
|
+
# { "my_topic" => [
|
|
899
|
+
# { partition: 0, offset: :earliest },
|
|
900
|
+
# { partition: 1, offset: :latest }
|
|
901
|
+
# ] }
|
|
902
|
+
# )
|
|
903
|
+
# report = handle.wait(max_wait_timeout_ms: 15_000)
|
|
904
|
+
# report.offsets
|
|
905
|
+
# # => [{ topic: "my_topic", partition: 0, offset: 0, ... }, ...]
|
|
906
|
+
def list_offsets(topic_partition_offsets, isolation_level: nil)
|
|
907
|
+
closed_admin_check(__method__)
|
|
908
|
+
|
|
909
|
+
# Count total partitions for pre-allocation
|
|
910
|
+
total = topic_partition_offsets.sum { |_, partitions| partitions.size }
|
|
911
|
+
|
|
912
|
+
# Build native topic partition list
|
|
913
|
+
tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(total)
|
|
914
|
+
|
|
915
|
+
topic_partition_offsets.each do |topic, partitions|
|
|
916
|
+
partitions.each do |spec|
|
|
917
|
+
partition = spec.fetch(:partition)
|
|
918
|
+
offset = spec.fetch(:offset)
|
|
919
|
+
|
|
920
|
+
native_offset = case offset
|
|
921
|
+
when :earliest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_EARLIEST
|
|
922
|
+
when :latest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_LATEST
|
|
923
|
+
when :max_timestamp then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP
|
|
924
|
+
when Integer then offset
|
|
925
|
+
else
|
|
926
|
+
raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
|
|
927
|
+
end
|
|
928
|
+
|
|
929
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
|
|
930
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
|
|
931
|
+
end
|
|
932
|
+
end
|
|
933
|
+
|
|
934
|
+
# Get a pointer to the queue that our request will be enqueued on
|
|
935
|
+
queue_ptr = @native_kafka.with_inner do |inner|
|
|
936
|
+
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
937
|
+
end
|
|
938
|
+
|
|
939
|
+
if queue_ptr.null?
|
|
940
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
941
|
+
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
942
|
+
end
|
|
943
|
+
|
|
944
|
+
# Create and register the handle we will return to the caller
|
|
945
|
+
handle = ListOffsetsHandle.new
|
|
946
|
+
handle[:pending] = true
|
|
947
|
+
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
948
|
+
|
|
949
|
+
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
950
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
|
951
|
+
inner,
|
|
952
|
+
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTOFFSETS
|
|
953
|
+
)
|
|
954
|
+
end
|
|
955
|
+
|
|
956
|
+
if isolation_level
|
|
957
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_isolation_level(admin_options_ptr, isolation_level)
|
|
958
|
+
end
|
|
959
|
+
|
|
960
|
+
ListOffsetsHandle.register(handle)
|
|
961
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
962
|
+
|
|
963
|
+
begin
|
|
964
|
+
@native_kafka.with_inner do |inner|
|
|
965
|
+
Rdkafka::Bindings.rd_kafka_ListOffsets(
|
|
966
|
+
inner,
|
|
967
|
+
tpl,
|
|
968
|
+
admin_options_ptr,
|
|
969
|
+
queue_ptr
|
|
970
|
+
)
|
|
971
|
+
end
|
|
972
|
+
rescue Exception
|
|
973
|
+
ListOffsetsHandle.remove(handle.to_ptr.address)
|
|
974
|
+
raise
|
|
975
|
+
ensure
|
|
976
|
+
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
977
|
+
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
978
|
+
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
979
|
+
end
|
|
980
|
+
|
|
981
|
+
handle
|
|
982
|
+
end
|
|
983
|
+
|
|
885
984
|
private
|
|
886
985
|
|
|
887
986
|
# Checks if the admin is closed and raises an error if so
|
data/lib/rdkafka/bindings.rb
CHANGED
|
@@ -194,6 +194,27 @@ module Rdkafka
|
|
|
194
194
|
RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
|
|
195
195
|
RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
|
|
196
196
|
|
|
197
|
+
# List Offsets
|
|
198
|
+
RD_KAFKA_ADMIN_OP_LISTOFFSETS = 20
|
|
199
|
+
RD_KAFKA_EVENT_LISTOFFSETS_RESULT = 0x400000
|
|
200
|
+
|
|
201
|
+
# rd_kafka_IsolationLevel_t
|
|
202
|
+
RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED = 0
|
|
203
|
+
RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED = 1
|
|
204
|
+
|
|
205
|
+
# rd_kafka_OffsetSpec_t
|
|
206
|
+
RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP = -3
|
|
207
|
+
RD_KAFKA_OFFSET_SPEC_EARLIEST = -2
|
|
208
|
+
RD_KAFKA_OFFSET_SPEC_LATEST = -1
|
|
209
|
+
|
|
210
|
+
attach_function :rd_kafka_ListOffsets, [:pointer, :pointer, :pointer, :pointer], :void, blocking: true
|
|
211
|
+
attach_function :rd_kafka_event_ListOffsets_result, [:pointer], :pointer
|
|
212
|
+
attach_function :rd_kafka_ListOffsets_result_infos, [:pointer, :pointer], :pointer
|
|
213
|
+
attach_function :rd_kafka_ListOffsetsResultInfo_topic_partition, [:pointer], :pointer
|
|
214
|
+
attach_function :rd_kafka_ListOffsetsResultInfo_timestamp, [:pointer], :int64
|
|
215
|
+
attach_function :rd_kafka_AdminOptions_set_isolation_level, [:pointer, :int], :pointer
|
|
216
|
+
attach_function :rd_kafka_topic_partition_get_leader_epoch, [:pointer], :int32
|
|
217
|
+
|
|
197
218
|
# FFI struct for error description (rd_kafka_err_desc)
|
|
198
219
|
class NativeErrorDesc < FFI::Struct
|
|
199
220
|
layout :code, :int,
|
|
@@ -335,6 +356,8 @@ module Rdkafka
|
|
|
335
356
|
:void, [:pointer, :int, :string, :pointer]
|
|
336
357
|
) do |client_ptr, err_code, reason, _opaque|
|
|
337
358
|
if Rdkafka::Config.error_callback
|
|
359
|
+
instance_name = client_ptr.null? ? nil : Rdkafka::Bindings.rd_kafka_name(client_ptr)
|
|
360
|
+
|
|
338
361
|
# Handle fatal errors according to librdkafka documentation:
|
|
339
362
|
# When ERR__FATAL is received, we must call rd_kafka_fatal_error()
|
|
340
363
|
# to get the actual underlying fatal error code and description.
|
|
@@ -342,10 +365,11 @@ module Rdkafka
|
|
|
342
365
|
Rdkafka::RdkafkaError.build_fatal(
|
|
343
366
|
client_ptr,
|
|
344
367
|
fallback_error_code: err_code,
|
|
345
|
-
fallback_message: reason
|
|
368
|
+
fallback_message: reason,
|
|
369
|
+
instance_name: instance_name
|
|
346
370
|
)
|
|
347
371
|
else
|
|
348
|
-
Rdkafka::RdkafkaError.build(err_code, broker_message: reason)
|
|
372
|
+
Rdkafka::RdkafkaError.build(err_code, broker_message: reason, instance_name: instance_name)
|
|
349
373
|
end
|
|
350
374
|
|
|
351
375
|
error.set_backtrace(caller)
|
data/lib/rdkafka/callbacks.rb
CHANGED
|
@@ -180,6 +180,27 @@ module Rdkafka
|
|
|
180
180
|
end
|
|
181
181
|
end
|
|
182
182
|
|
|
183
|
+
# Extracts attributes of rd_kafka_ListOffsets_result_t
|
|
184
|
+
#
|
|
185
|
+
# @private
|
|
186
|
+
class ListOffsetsResult
|
|
187
|
+
attr_reader :result_error, :error_string, :result_infos, :result_count
|
|
188
|
+
|
|
189
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
190
|
+
def initialize(event_ptr)
|
|
191
|
+
@result_infos = FFI::Pointer::NULL
|
|
192
|
+
@result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
|
193
|
+
@error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
|
|
194
|
+
|
|
195
|
+
if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
196
|
+
list_offsets_result = Rdkafka::Bindings.rd_kafka_event_ListOffsets_result(event_ptr)
|
|
197
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
198
|
+
@result_infos = Rdkafka::Bindings.rd_kafka_ListOffsets_result_infos(list_offsets_result, pointer_to_size_t)
|
|
199
|
+
@result_count = pointer_to_size_t.read(:size_t)
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
183
204
|
# @private
|
|
184
205
|
class BackgroundEventCallback
|
|
185
206
|
# Handles background events from librdkafka
|
|
@@ -206,6 +227,8 @@ module Rdkafka
|
|
|
206
227
|
process_describe_acl(event_ptr)
|
|
207
228
|
when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
|
|
208
229
|
process_delete_groups(event_ptr)
|
|
230
|
+
when Rdkafka::Bindings::RD_KAFKA_EVENT_LISTOFFSETS_RESULT
|
|
231
|
+
process_list_offsets(event_ptr)
|
|
209
232
|
end
|
|
210
233
|
end
|
|
211
234
|
|
|
@@ -392,6 +415,26 @@ module Rdkafka
|
|
|
392
415
|
describe_acl_handle.unlock
|
|
393
416
|
end
|
|
394
417
|
end
|
|
418
|
+
|
|
419
|
+
# Processes list offsets result event
|
|
420
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
421
|
+
def self.process_list_offsets(event_ptr)
|
|
422
|
+
list_offsets = ListOffsetsResult.new(event_ptr)
|
|
423
|
+
list_offsets_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
424
|
+
|
|
425
|
+
if list_offsets_handle = Rdkafka::Admin::ListOffsetsHandle.remove(list_offsets_handle_ptr.address)
|
|
426
|
+
list_offsets_handle[:response] = list_offsets.result_error
|
|
427
|
+
list_offsets_handle[:response_string] = list_offsets.error_string
|
|
428
|
+
list_offsets_handle[:pending] = false
|
|
429
|
+
|
|
430
|
+
if list_offsets.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
431
|
+
list_offsets_handle[:result_infos] = list_offsets.result_infos
|
|
432
|
+
list_offsets_handle[:result_count] = list_offsets.result_count
|
|
433
|
+
end
|
|
434
|
+
|
|
435
|
+
list_offsets_handle.unlock
|
|
436
|
+
end
|
|
437
|
+
end
|
|
395
438
|
end
|
|
396
439
|
|
|
397
440
|
# @private
|
data/lib/rdkafka/config.rb
CHANGED
|
@@ -89,10 +89,10 @@ module Rdkafka
|
|
|
89
89
|
# If this callback is not set, global errors such as brokers becoming unavailable will only be sent to the logger, as defined by librdkafka.
|
|
90
90
|
# The callback is called with an instance of RdKafka::Error.
|
|
91
91
|
#
|
|
92
|
-
# @param callback [Proc, #call] callable object to handle errors
|
|
92
|
+
# @param callback [Proc, #call, nil] callable object to handle errors or nil to clear
|
|
93
93
|
# @return [nil]
|
|
94
94
|
def self.error_callback=(callback)
|
|
95
|
-
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
|
|
95
|
+
raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call) || callback.nil?
|
|
96
96
|
@@error_callback = callback
|
|
97
97
|
end
|
|
98
98
|
|
data/lib/rdkafka/consumer.rb
CHANGED
|
@@ -19,6 +19,9 @@ module Rdkafka
|
|
|
19
19
|
# @param native_kafka [NativeKafka] wrapper around the native Kafka consumer handle
|
|
20
20
|
def initialize(native_kafka)
|
|
21
21
|
@native_kafka = native_kafka
|
|
22
|
+
|
|
23
|
+
# Makes sure, that native kafka gets closed before it gets GCed by Ruby
|
|
24
|
+
ObjectSpace.define_finalizer(self, native_kafka.finalizer)
|
|
22
25
|
end
|
|
23
26
|
|
|
24
27
|
# Starts the native Kafka polling thread and kicks off the init polling
|
|
@@ -172,12 +175,6 @@ module Rdkafka
|
|
|
172
175
|
end
|
|
173
176
|
end
|
|
174
177
|
|
|
175
|
-
# @return [Proc] finalizer proc for closing the consumer
|
|
176
|
-
# @private
|
|
177
|
-
def finalizer
|
|
178
|
-
->(_) { close }
|
|
179
|
-
end
|
|
180
|
-
|
|
181
178
|
# Close this consumer
|
|
182
179
|
# @return [nil]
|
|
183
180
|
def close
|
data/lib/rdkafka/error.rb
CHANGED
|
@@ -25,6 +25,10 @@ module Rdkafka
|
|
|
25
25
|
# @return [Hash]
|
|
26
26
|
attr_reader :details
|
|
27
27
|
|
|
28
|
+
# The name of the rdkafka instance that generated this error
|
|
29
|
+
# @return [String, nil]
|
|
30
|
+
attr_reader :instance_name
|
|
31
|
+
|
|
28
32
|
class << self
|
|
29
33
|
# Build an error instance from a rd_kafka_error_t pointer
|
|
30
34
|
#
|
|
@@ -60,13 +64,14 @@ module Rdkafka
|
|
|
60
64
|
# or message struct
|
|
61
65
|
# @param message_prefix [String, nil] Optional prefix for the error message
|
|
62
66
|
# @param broker_message [String, nil] Optional broker error message
|
|
67
|
+
# @param instance_name [String, nil] Optional name of the rdkafka instance
|
|
63
68
|
# @return [RdkafkaError, false] Error instance or false if no error
|
|
64
|
-
def build(response_ptr_or_code, message_prefix = nil, broker_message: nil)
|
|
69
|
+
def build(response_ptr_or_code, message_prefix = nil, broker_message: nil, instance_name: nil)
|
|
65
70
|
case response_ptr_or_code
|
|
66
71
|
when Integer
|
|
67
72
|
return false if response_ptr_or_code == Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
68
73
|
|
|
69
|
-
new(response_ptr_or_code, message_prefix, broker_message: broker_message)
|
|
74
|
+
new(response_ptr_or_code, message_prefix, broker_message: broker_message, instance_name: instance_name)
|
|
70
75
|
when Bindings::Message
|
|
71
76
|
return false if response_ptr_or_code[:err] == Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
72
77
|
|
|
@@ -128,15 +133,17 @@ module Rdkafka
|
|
|
128
133
|
# @param client_ptr [FFI::Pointer] Pointer to rd_kafka_t client
|
|
129
134
|
# @param fallback_error_code [Integer] Error code to use if no fatal error found (default: -150)
|
|
130
135
|
# @param fallback_message [String, nil] Message to use if no fatal error found
|
|
136
|
+
# @param instance_name [String, nil] Optional name of the rdkafka instance
|
|
131
137
|
# @return [RdkafkaError] Error object with fatal flag set to true
|
|
132
|
-
def build_fatal(client_ptr, fallback_error_code: -150, fallback_message: nil)
|
|
138
|
+
def build_fatal(client_ptr, fallback_error_code: -150, fallback_message: nil, instance_name: nil)
|
|
133
139
|
fatal_error_details = Rdkafka::Bindings.extract_fatal_error(client_ptr)
|
|
134
140
|
|
|
135
141
|
if fatal_error_details
|
|
136
142
|
new(
|
|
137
143
|
fatal_error_details[:error_code],
|
|
138
144
|
broker_message: fatal_error_details[:error_string],
|
|
139
|
-
fatal: true
|
|
145
|
+
fatal: true,
|
|
146
|
+
instance_name: instance_name
|
|
140
147
|
)
|
|
141
148
|
else
|
|
142
149
|
# Fallback: if extract_fatal_error returns nil (shouldn't happen in practice),
|
|
@@ -144,7 +151,8 @@ module Rdkafka
|
|
|
144
151
|
new(
|
|
145
152
|
fallback_error_code,
|
|
146
153
|
broker_message: fallback_message,
|
|
147
|
-
fatal: true
|
|
154
|
+
fatal: true,
|
|
155
|
+
instance_name: instance_name
|
|
148
156
|
)
|
|
149
157
|
end
|
|
150
158
|
end
|
|
@@ -158,6 +166,7 @@ module Rdkafka
|
|
|
158
166
|
# @param retryable [Boolean] whether this error is retryable
|
|
159
167
|
# @param abortable [Boolean] whether this error requires transaction abort
|
|
160
168
|
# @param details [Hash] additional error details
|
|
169
|
+
# @param instance_name [String, nil] optional name of the rdkafka instance
|
|
161
170
|
def initialize(
|
|
162
171
|
response,
|
|
163
172
|
message_prefix = nil,
|
|
@@ -165,7 +174,8 @@ module Rdkafka
|
|
|
165
174
|
fatal: false,
|
|
166
175
|
retryable: false,
|
|
167
176
|
abortable: false,
|
|
168
|
-
details: EMPTY_HASH
|
|
177
|
+
details: EMPTY_HASH,
|
|
178
|
+
instance_name: nil
|
|
169
179
|
)
|
|
170
180
|
raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
|
|
171
181
|
@rdkafka_response = response
|
|
@@ -175,6 +185,7 @@ module Rdkafka
|
|
|
175
185
|
@retryable = retryable
|
|
176
186
|
@abortable = abortable
|
|
177
187
|
@details = details
|
|
188
|
+
@instance_name = instance_name
|
|
178
189
|
end
|
|
179
190
|
|
|
180
191
|
# This error's code, for example `:partition_eof`, `:msg_size_too_large`.
|
|
@@ -197,8 +208,14 @@ module Rdkafka
|
|
|
197
208
|
""
|
|
198
209
|
end
|
|
199
210
|
|
|
211
|
+
instance_name_part = if instance_name
|
|
212
|
+
" [#{instance_name}]"
|
|
213
|
+
else
|
|
214
|
+
""
|
|
215
|
+
end
|
|
216
|
+
|
|
200
217
|
err_str = Rdkafka::Bindings.rd_kafka_err2str(@rdkafka_response)
|
|
201
|
-
base = "#{message_prefix_part}#{err_str} (#{code})"
|
|
218
|
+
base = "#{message_prefix_part}#{err_str} (#{code})#{instance_name_part}"
|
|
202
219
|
|
|
203
220
|
return base if broker_message.nil?
|
|
204
221
|
return base if broker_message.empty?
|
data/lib/rdkafka/version.rb
CHANGED
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
module Rdkafka
|
|
4
4
|
# Current rdkafka-ruby gem version
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.25.0"
|
|
6
6
|
# Target librdkafka version to be used
|
|
7
|
-
LIBRDKAFKA_VERSION = "2.13.
|
|
7
|
+
LIBRDKAFKA_VERSION = "2.13.2"
|
|
8
8
|
# SHA256 hash of the librdkafka source tarball for verification
|
|
9
|
-
LIBRDKAFKA_SOURCE_SHA256 = "
|
|
9
|
+
LIBRDKAFKA_SOURCE_SHA256 = "14972092e4115f6e99f798a7cb420cbf6daa0c73502b3c52ae42fb5b418eea8f"
|
|
10
10
|
end
|
data/lib/rdkafka.rb
CHANGED
|
@@ -29,6 +29,8 @@ require "rdkafka/admin/describe_configs_handle"
|
|
|
29
29
|
require "rdkafka/admin/describe_configs_report"
|
|
30
30
|
require "rdkafka/admin/incremental_alter_configs_handle"
|
|
31
31
|
require "rdkafka/admin/incremental_alter_configs_report"
|
|
32
|
+
require "rdkafka/admin/list_offsets_handle"
|
|
33
|
+
require "rdkafka/admin/list_offsets_report"
|
|
32
34
|
require "rdkafka/admin/acl_binding_result"
|
|
33
35
|
require "rdkafka/admin/config_binding_result"
|
|
34
36
|
require "rdkafka/admin/config_resource_binding_result"
|
data/package-lock.json
CHANGED
|
@@ -217,9 +217,9 @@
|
|
|
217
217
|
}
|
|
218
218
|
},
|
|
219
219
|
"node_modules/picomatch": {
|
|
220
|
-
"version": "2.3.
|
|
221
|
-
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.
|
|
222
|
-
"integrity": "sha512-
|
|
220
|
+
"version": "2.3.2",
|
|
221
|
+
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz",
|
|
222
|
+
"integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==",
|
|
223
223
|
"dev": true,
|
|
224
224
|
"license": "MIT",
|
|
225
225
|
"engines": {
|
data/renovate.json
CHANGED
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
"Gemfile",
|
|
10
10
|
"Gemfile.lint",
|
|
11
11
|
"package.json",
|
|
12
|
-
".github/workflows/**"
|
|
12
|
+
".github/workflows/**",
|
|
13
|
+
"docker-compose*.yml"
|
|
13
14
|
],
|
|
14
15
|
"github-actions": {
|
|
15
16
|
"enabled": true,
|
|
@@ -86,5 +87,8 @@
|
|
|
86
87
|
],
|
|
87
88
|
"groupName": "ruby setup"
|
|
88
89
|
}
|
|
90
|
+
],
|
|
91
|
+
"labels": [
|
|
92
|
+
"dependencies"
|
|
89
93
|
]
|
|
90
94
|
}
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: karafka-rdkafka
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.25.0
|
|
5
5
|
platform: aarch64-linux-gnu
|
|
6
6
|
authors:
|
|
7
7
|
- Thijs Cadier
|
|
@@ -123,6 +123,8 @@ files:
|
|
|
123
123
|
- lib/rdkafka/admin/describe_configs_report.rb
|
|
124
124
|
- lib/rdkafka/admin/incremental_alter_configs_handle.rb
|
|
125
125
|
- lib/rdkafka/admin/incremental_alter_configs_report.rb
|
|
126
|
+
- lib/rdkafka/admin/list_offsets_handle.rb
|
|
127
|
+
- lib/rdkafka/admin/list_offsets_report.rb
|
|
126
128
|
- lib/rdkafka/bindings.rb
|
|
127
129
|
- lib/rdkafka/callbacks.rb
|
|
128
130
|
- lib/rdkafka/config.rb
|
|
@@ -170,7 +172,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
170
172
|
- !ruby/object:Gem::Version
|
|
171
173
|
version: '0'
|
|
172
174
|
requirements: []
|
|
173
|
-
rubygems_version: 4.0.
|
|
175
|
+
rubygems_version: 4.0.6
|
|
174
176
|
specification_version: 4
|
|
175
177
|
summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
|
|
176
178
|
It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
|