karafka-rdkafka 0.27.2-aarch64-linux-gnu → 0.28.0-aarch64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +33 -0
- data/README.md +1 -0
- data/docker-compose-ssl.yml +1 -1
- data/docker-compose.yml +1 -1
- data/ext/librdkafka.so +0 -0
- data/karafka-rdkafka.gemspec +1 -0
- data/lib/rdkafka/abstract_handle.rb +37 -4
- data/lib/rdkafka/admin/config_binding_result.rb +1 -1
- data/lib/rdkafka/admin/create_acl_handle.rb +2 -8
- data/lib/rdkafka/admin/create_partitions_handle.rb +1 -20
- data/lib/rdkafka/admin/create_topic_handle.rb +1 -20
- data/lib/rdkafka/admin/delete_acl_handle.rb +2 -10
- data/lib/rdkafka/admin/delete_groups_handle.rb +3 -11
- data/lib/rdkafka/admin/delete_topic_handle.rb +1 -20
- data/lib/rdkafka/admin/describe_acl_handle.rb +2 -10
- data/lib/rdkafka/admin/describe_configs_handle.rb +2 -13
- data/lib/rdkafka/admin/describe_configs_report.rb +5 -6
- data/lib/rdkafka/admin/incremental_alter_configs_handle.rb +2 -13
- data/lib/rdkafka/admin/incremental_alter_configs_report.rb +5 -6
- data/lib/rdkafka/admin/list_offsets_handle.rb +2 -13
- data/lib/rdkafka/admin/list_offsets_report.rb +5 -2
- data/lib/rdkafka/admin.rb +56 -137
- data/lib/rdkafka/bindings.rb +12 -3
- data/lib/rdkafka/callbacks/base_handler.rb +62 -0
- data/lib/rdkafka/callbacks/create_acl_handler.rb +37 -0
- data/lib/rdkafka/callbacks/create_partitions_handler.rb +37 -0
- data/lib/rdkafka/callbacks/create_topic_handler.rb +37 -0
- data/lib/rdkafka/callbacks/delete_acl_handler.rb +42 -0
- data/lib/rdkafka/callbacks/delete_groups_handler.rb +37 -0
- data/lib/rdkafka/callbacks/delete_topic_handler.rb +37 -0
- data/lib/rdkafka/callbacks/describe_acl_handler.rb +35 -0
- data/lib/rdkafka/callbacks/describe_configs_handler.rb +42 -0
- data/lib/rdkafka/callbacks/incremental_alter_configs_handler.rb +42 -0
- data/lib/rdkafka/callbacks/list_offsets_handler.rb +42 -0
- data/lib/rdkafka/callbacks.rb +56 -244
- data/lib/rdkafka/config.rb +50 -33
- data/lib/rdkafka/consumer/headers.rb +19 -5
- data/lib/rdkafka/consumer/topic_partition_list.rb +43 -33
- data/lib/rdkafka/consumer.rb +164 -23
- data/lib/rdkafka/defaults.rb +21 -1
- data/lib/rdkafka/error.rb +12 -4
- data/lib/rdkafka/helpers/list_offsets.rb +127 -0
- data/lib/rdkafka/helpers/metadata.rb +29 -0
- data/lib/rdkafka/metadata.rb +84 -19
- data/lib/rdkafka/producer/delivery_handle.rb +3 -3
- data/lib/rdkafka/producer/partitions_count_cache.rb +24 -38
- data/lib/rdkafka/producer.rb +72 -51
- data/lib/rdkafka/version.rb +3 -3
- data/lib/rdkafka.rb +13 -0
- data/package-lock.json +6 -6
- data/renovate.json +11 -1
- metadata +14 -4
- data/Gemfile +0 -13
- data/Gemfile.lint +0 -14
- data/Gemfile.lint.lock +0 -123
data/lib/rdkafka/admin.rb
CHANGED
|
@@ -4,6 +4,8 @@ module Rdkafka
|
|
|
4
4
|
# Admin client for Kafka administrative operations
|
|
5
5
|
class Admin
|
|
6
6
|
include Helpers::OAuth
|
|
7
|
+
include Helpers::Metadata
|
|
8
|
+
include Helpers::ListOffsets
|
|
7
9
|
|
|
8
10
|
class << self
|
|
9
11
|
# Allows us to retrieve librdkafka errors with descriptions
|
|
@@ -130,19 +132,6 @@ module Rdkafka
|
|
|
130
132
|
end
|
|
131
133
|
end
|
|
132
134
|
|
|
133
|
-
# Performs the metadata request using admin
|
|
134
|
-
#
|
|
135
|
-
# @param topic_name [String, nil] metadat about particular topic or all if nil
|
|
136
|
-
# @param timeout_ms [Integer] metadata request timeout
|
|
137
|
-
# @return [Metadata] requested metadata
|
|
138
|
-
def metadata(topic_name = nil, timeout_ms = Defaults::METADATA_TIMEOUT_MS)
|
|
139
|
-
closed_admin_check(__method__)
|
|
140
|
-
|
|
141
|
-
@native_kafka.with_inner do |inner|
|
|
142
|
-
Metadata.new(inner, topic_name, timeout_ms)
|
|
143
|
-
end
|
|
144
|
-
end
|
|
145
|
-
|
|
146
135
|
# Close this admin instance
|
|
147
136
|
def close
|
|
148
137
|
return if closed?
|
|
@@ -258,7 +247,7 @@ module Rdkafka
|
|
|
258
247
|
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
259
248
|
end
|
|
260
249
|
if queue_ptr.null?
|
|
261
|
-
Rdkafka::Bindings.
|
|
250
|
+
Rdkafka::Bindings.rd_kafka_DeleteGroup_destroy(delete_groups_ptr)
|
|
262
251
|
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
263
252
|
end
|
|
264
253
|
|
|
@@ -581,7 +570,7 @@ module Rdkafka
|
|
|
581
570
|
end
|
|
582
571
|
|
|
583
572
|
if queue_ptr.null?
|
|
584
|
-
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(
|
|
573
|
+
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(delete_acl_ptr)
|
|
585
574
|
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
586
575
|
end
|
|
587
576
|
|
|
@@ -677,7 +666,7 @@ module Rdkafka
|
|
|
677
666
|
end
|
|
678
667
|
|
|
679
668
|
if queue_ptr.null?
|
|
680
|
-
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(
|
|
669
|
+
Rdkafka::Bindings.rd_kafka_AclBinding_destroy(describe_acl_ptr)
|
|
681
670
|
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
682
671
|
end
|
|
683
672
|
|
|
@@ -728,6 +717,14 @@ module Rdkafka
|
|
|
728
717
|
def describe_configs(resources)
|
|
729
718
|
closed_admin_check(__method__)
|
|
730
719
|
|
|
720
|
+
# Parse user input before allocating any native resources or registering the handle, so a
|
|
721
|
+
# missing :resource_type / :resource_name key raises with nothing to clean up. Previously the
|
|
722
|
+
# KeyError leaked the background queue, the AdminOptions, the registered handle and any
|
|
723
|
+
# ConfigResources already built.
|
|
724
|
+
parsed_resources = resources.map do |resource_details|
|
|
725
|
+
[resource_details.fetch(:resource_type), resource_details.fetch(:resource_name)]
|
|
726
|
+
end
|
|
727
|
+
|
|
731
728
|
handle = DescribeConfigsHandle.new
|
|
732
729
|
handle[:pending] = true
|
|
733
730
|
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
@@ -750,12 +747,10 @@ module Rdkafka
|
|
|
750
747
|
DescribeConfigsHandle.register(handle)
|
|
751
748
|
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
752
749
|
|
|
753
|
-
pointer_array =
|
|
750
|
+
pointer_array = parsed_resources.map do |resource_type, resource_name|
|
|
754
751
|
Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
|
755
|
-
|
|
756
|
-
FFI::MemoryPointer.from_string(
|
|
757
|
-
resource_details.fetch(:resource_name)
|
|
758
|
-
)
|
|
752
|
+
resource_type,
|
|
753
|
+
FFI::MemoryPointer.from_string(resource_name)
|
|
759
754
|
)
|
|
760
755
|
end
|
|
761
756
|
|
|
@@ -806,6 +801,19 @@ module Rdkafka
|
|
|
806
801
|
def incremental_alter_configs(resources_with_configs)
|
|
807
802
|
closed_admin_check(__method__)
|
|
808
803
|
|
|
804
|
+
# Parse user input before allocating native resources or registering the handle, so a missing
|
|
805
|
+
# key raises with nothing to clean up. Previously the KeyError leaked the background queue,
|
|
806
|
+
# the AdminOptions, the registered handle and any ConfigResources already built.
|
|
807
|
+
parsed_resources = resources_with_configs.map do |resource_details|
|
|
808
|
+
[
|
|
809
|
+
resource_details.fetch(:resource_type),
|
|
810
|
+
resource_details.fetch(:resource_name),
|
|
811
|
+
resource_details.fetch(:configs).map do |config|
|
|
812
|
+
[config.fetch(:name), config.fetch(:op_type), config.fetch(:value)]
|
|
813
|
+
end
|
|
814
|
+
]
|
|
815
|
+
end
|
|
816
|
+
|
|
809
817
|
handle = IncrementalAlterConfigsHandle.new
|
|
810
818
|
handle[:pending] = true
|
|
811
819
|
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
@@ -828,23 +836,34 @@ module Rdkafka
|
|
|
828
836
|
IncrementalAlterConfigsHandle.register(handle)
|
|
829
837
|
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
830
838
|
|
|
831
|
-
|
|
832
|
-
|
|
839
|
+
add_error = nil
|
|
840
|
+
|
|
841
|
+
pointer_array = parsed_resources.map do |resource_type, resource_name, configs|
|
|
833
842
|
# First build the appropriate resource representation
|
|
834
843
|
resource_ptr = Rdkafka::Bindings.rd_kafka_ConfigResource_new(
|
|
835
|
-
|
|
836
|
-
FFI::MemoryPointer.from_string(
|
|
837
|
-
resource_details.fetch(:resource_name)
|
|
838
|
-
)
|
|
844
|
+
resource_type,
|
|
845
|
+
FFI::MemoryPointer.from_string(resource_name)
|
|
839
846
|
)
|
|
840
847
|
|
|
841
|
-
|
|
842
|
-
|
|
848
|
+
configs.each do |name, op_type, value|
|
|
849
|
+
# rd_kafka_ConfigResource_add_incremental_config returns a non-NULL rd_kafka_error_t for
|
|
850
|
+
# an invalid op_type, an empty/nil name, or a nil value on a non-delete op. The result
|
|
851
|
+
# used to be ignored: the entry was silently dropped, the alter request still reported
|
|
852
|
+
# success, and the error object leaked. Capture the first error (build_from_c also
|
|
853
|
+
# destroys the native error object) and raise it below, before the request is sent.
|
|
854
|
+
error_ptr = Bindings.rd_kafka_ConfigResource_add_incremental_config(
|
|
843
855
|
resource_ptr,
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
856
|
+
name,
|
|
857
|
+
op_type,
|
|
858
|
+
value
|
|
847
859
|
)
|
|
860
|
+
|
|
861
|
+
error = Rdkafka::RdkafkaError.build_from_c(
|
|
862
|
+
error_ptr,
|
|
863
|
+
"rd_kafka_ConfigResource_add_incremental_config"
|
|
864
|
+
)
|
|
865
|
+
|
|
866
|
+
add_error ||= error if error
|
|
848
867
|
end
|
|
849
868
|
|
|
850
869
|
resource_ptr
|
|
@@ -854,6 +873,10 @@ module Rdkafka
|
|
|
854
873
|
configs_array_ptr.write_array_of_pointer(pointer_array)
|
|
855
874
|
|
|
856
875
|
begin
|
|
876
|
+
# Raise only after the full array is built so the ensure below frees every ConfigResource
|
|
877
|
+
# we created, and before the request is sent so a rejected entry can't report success.
|
|
878
|
+
raise add_error if add_error
|
|
879
|
+
|
|
857
880
|
@native_kafka.with_inner do |inner|
|
|
858
881
|
Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs(
|
|
859
882
|
inner,
|
|
@@ -882,111 +905,6 @@ module Rdkafka
|
|
|
882
905
|
handle
|
|
883
906
|
end
|
|
884
907
|
|
|
885
|
-
# Queries partition offsets by specification (earliest, latest, max_timestamp, or by
|
|
886
|
-
# timestamp) without requiring a consumer group.
|
|
887
|
-
#
|
|
888
|
-
# @param topic_partition_offsets [Hash{String => Array<Hash>}] hash mapping topic names to
|
|
889
|
-
# arrays of partition offset specifications. Each specification is a hash with:
|
|
890
|
-
# - `:partition` [Integer] partition number
|
|
891
|
-
# - `:offset` [Symbol, Integer] offset specification - `:earliest`, `:latest`,
|
|
892
|
-
# `:max_timestamp`, or an integer timestamp in milliseconds
|
|
893
|
-
# @param isolation_level [Integer, nil] optional isolation level:
|
|
894
|
-
# - `RD_KAFKA_ISOLATION_LEVEL_READ_UNCOMMITTED` (0) - default
|
|
895
|
-
# - `RD_KAFKA_ISOLATION_LEVEL_READ_COMMITTED` (1)
|
|
896
|
-
#
|
|
897
|
-
# @return [ListOffsetsHandle] handle that can be used to wait for the result
|
|
898
|
-
#
|
|
899
|
-
# @raise [ClosedAdminError] when the admin is closed
|
|
900
|
-
# @raise [ConfigError] when the background queue is unavailable
|
|
901
|
-
#
|
|
902
|
-
# @example Query earliest and latest offsets
|
|
903
|
-
# handle = admin.list_offsets(
|
|
904
|
-
# { "my_topic" => [
|
|
905
|
-
# { partition: 0, offset: :earliest },
|
|
906
|
-
# { partition: 1, offset: :latest }
|
|
907
|
-
# ] }
|
|
908
|
-
# )
|
|
909
|
-
# report = handle.wait(max_wait_timeout_ms: 15_000)
|
|
910
|
-
# report.offsets
|
|
911
|
-
# # => [{ topic: "my_topic", partition: 0, offset: 0, ... }, ...]
|
|
912
|
-
def list_offsets(topic_partition_offsets, isolation_level: nil)
|
|
913
|
-
closed_admin_check(__method__)
|
|
914
|
-
|
|
915
|
-
# Count total partitions for pre-allocation
|
|
916
|
-
total = topic_partition_offsets.sum { |_, partitions| partitions.size }
|
|
917
|
-
|
|
918
|
-
# Build native topic partition list
|
|
919
|
-
tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(total)
|
|
920
|
-
|
|
921
|
-
topic_partition_offsets.each do |topic, partitions|
|
|
922
|
-
partitions.each do |spec|
|
|
923
|
-
partition = spec.fetch(:partition)
|
|
924
|
-
offset = spec.fetch(:offset)
|
|
925
|
-
|
|
926
|
-
native_offset = case offset
|
|
927
|
-
when :earliest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_EARLIEST
|
|
928
|
-
when :latest then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_LATEST
|
|
929
|
-
when :max_timestamp then Rdkafka::Bindings::RD_KAFKA_OFFSET_SPEC_MAX_TIMESTAMP
|
|
930
|
-
when Integer then offset
|
|
931
|
-
else
|
|
932
|
-
raise ArgumentError, "Unknown offset specification: #{offset.inspect}"
|
|
933
|
-
end
|
|
934
|
-
|
|
935
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, partition)
|
|
936
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(tpl, topic, partition, native_offset)
|
|
937
|
-
end
|
|
938
|
-
end
|
|
939
|
-
|
|
940
|
-
# Get a pointer to the queue that our request will be enqueued on
|
|
941
|
-
queue_ptr = @native_kafka.with_inner do |inner|
|
|
942
|
-
Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
|
|
943
|
-
end
|
|
944
|
-
|
|
945
|
-
if queue_ptr.null?
|
|
946
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
947
|
-
raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
|
|
948
|
-
end
|
|
949
|
-
|
|
950
|
-
# Create and register the handle we will return to the caller
|
|
951
|
-
handle = ListOffsetsHandle.new
|
|
952
|
-
handle[:pending] = true
|
|
953
|
-
handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
|
|
954
|
-
|
|
955
|
-
admin_options_ptr = @native_kafka.with_inner do |inner|
|
|
956
|
-
Rdkafka::Bindings.rd_kafka_AdminOptions_new(
|
|
957
|
-
inner,
|
|
958
|
-
Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTOFFSETS
|
|
959
|
-
)
|
|
960
|
-
end
|
|
961
|
-
|
|
962
|
-
if isolation_level
|
|
963
|
-
Rdkafka::Bindings.rd_kafka_AdminOptions_set_isolation_level(admin_options_ptr, isolation_level)
|
|
964
|
-
end
|
|
965
|
-
|
|
966
|
-
ListOffsetsHandle.register(handle)
|
|
967
|
-
Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
|
|
968
|
-
|
|
969
|
-
begin
|
|
970
|
-
@native_kafka.with_inner do |inner|
|
|
971
|
-
Rdkafka::Bindings.rd_kafka_ListOffsets(
|
|
972
|
-
inner,
|
|
973
|
-
tpl,
|
|
974
|
-
admin_options_ptr,
|
|
975
|
-
queue_ptr
|
|
976
|
-
)
|
|
977
|
-
end
|
|
978
|
-
rescue Exception
|
|
979
|
-
ListOffsetsHandle.remove(handle.to_ptr.address)
|
|
980
|
-
raise
|
|
981
|
-
ensure
|
|
982
|
-
Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
|
|
983
|
-
Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
|
|
984
|
-
Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
|
|
985
|
-
end
|
|
986
|
-
|
|
987
|
-
handle
|
|
988
|
-
end
|
|
989
|
-
|
|
990
908
|
private
|
|
991
909
|
|
|
992
910
|
# Checks if the admin is closed and raises an error if so
|
|
@@ -995,5 +913,6 @@ module Rdkafka
|
|
|
995
913
|
def closed_admin_check(method)
|
|
996
914
|
raise Rdkafka::ClosedAdminError.new(method) if closed?
|
|
997
915
|
end
|
|
916
|
+
alias_method :closed_check, :closed_admin_check
|
|
998
917
|
end
|
|
999
918
|
end
|
data/lib/rdkafka/bindings.rb
CHANGED
|
@@ -58,6 +58,7 @@ module Rdkafka
|
|
|
58
58
|
end
|
|
59
59
|
end
|
|
60
60
|
|
|
61
|
+
RD_KAFKA_RESP_ERR__TIMED_OUT = -185
|
|
61
62
|
RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS = -175
|
|
62
63
|
RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
|
|
63
64
|
RD_KAFKA_RESP_ERR__STATE = -172
|
|
@@ -105,8 +106,12 @@ module Rdkafka
|
|
|
105
106
|
# Metadata
|
|
106
107
|
|
|
107
108
|
attach_function :rd_kafka_name, [:pointer], :string
|
|
108
|
-
|
|
109
|
-
|
|
109
|
+
# Both return a newly allocated string the caller must release with rd_kafka_mem_free; they
|
|
110
|
+
# are attached as :pointer (not :string) so we can free them instead of leaking. clusterid
|
|
111
|
+
# also takes a timeout_ms (the previous arity was wrong, leaving that argument as garbage).
|
|
112
|
+
attach_function :rd_kafka_memberid, [:pointer], :pointer, blocking: true
|
|
113
|
+
attach_function :rd_kafka_clusterid, [:pointer, :int], :pointer, blocking: true
|
|
114
|
+
attach_function :rd_kafka_mem_free, [:pointer, :pointer], :void
|
|
110
115
|
attach_function :rd_kafka_metadata, [:pointer, :int, :pointer, :pointer, :int], :int, blocking: true
|
|
111
116
|
attach_function :rd_kafka_metadata_destroy, [:pointer], :void, blocking: true
|
|
112
117
|
|
|
@@ -242,6 +247,8 @@ module Rdkafka
|
|
|
242
247
|
|
|
243
248
|
attach_function :rd_kafka_conf_new, [], :pointer
|
|
244
249
|
attach_function :rd_kafka_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
|
|
250
|
+
attach_function :rd_kafka_conf_get, [:pointer, :string, :pointer, :pointer], :kafka_config_response
|
|
251
|
+
attach_function :rd_kafka_conf, [:pointer], :pointer
|
|
245
252
|
attach_function :rd_kafka_conf_dump, [:pointer, :pointer], :pointer
|
|
246
253
|
attach_function :rd_kafka_conf_dump_free, [:pointer, :size_t], :void
|
|
247
254
|
attach_function :rd_kafka_conf_destroy, [:pointer], :void
|
|
@@ -271,6 +278,7 @@ module Rdkafka
|
|
|
271
278
|
attach_function :rd_kafka_queue_io_event_enable, [:pointer, :int, :pointer, :size_t], :void
|
|
272
279
|
# Per topic configs
|
|
273
280
|
attach_function :rd_kafka_topic_conf_new, [], :pointer
|
|
281
|
+
attach_function :rd_kafka_topic_conf_destroy, [:pointer], :void
|
|
274
282
|
attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
|
|
275
283
|
|
|
276
284
|
LogCallback = FFI::Function.new(
|
|
@@ -493,7 +501,7 @@ module Rdkafka
|
|
|
493
501
|
|
|
494
502
|
# Stats
|
|
495
503
|
|
|
496
|
-
attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int
|
|
504
|
+
attach_function :rd_kafka_query_watermark_offsets, [:pointer, :string, :int, :pointer, :pointer, :int], :int, blocking: true
|
|
497
505
|
|
|
498
506
|
# Producer
|
|
499
507
|
RD_KAFKA_VTYPE_END = 0
|
|
@@ -606,6 +614,7 @@ module Rdkafka
|
|
|
606
614
|
|
|
607
615
|
attach_function :rd_kafka_event_type, [:pointer], :int32
|
|
608
616
|
attach_function :rd_kafka_event_opaque, [:pointer], :pointer
|
|
617
|
+
attach_function :rd_kafka_event_destroy, [:pointer], :void
|
|
609
618
|
|
|
610
619
|
# Extracting data from topic results
|
|
611
620
|
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Base class for admin background-event result handlers.
|
|
6
|
+
#
|
|
7
|
+
# Each subclass handles one librdkafka admin result event type: it extracts the result from
|
|
8
|
+
# the event, looks up the matching {AbstractHandle} in the registry and resolves it (sets the
|
|
9
|
+
# response, builds the report, copies the broker message, then unlocks it).
|
|
10
|
+
#
|
|
11
|
+
# The event is owned by the application and destroyed by {BackgroundEventCallback} right after
|
|
12
|
+
# dispatch, so handlers must copy everything they need out of event-owned memory into Ruby
|
|
13
|
+
# objects before returning.
|
|
14
|
+
#
|
|
15
|
+
# @private
|
|
16
|
+
class BaseHandler
|
|
17
|
+
class << self
|
|
18
|
+
# Processes a background event for the operation this handler is responsible for.
|
|
19
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
20
|
+
# @return [void]
|
|
21
|
+
def call(event_ptr)
|
|
22
|
+
raise NotImplementedError, "#{name} must implement .call"
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
# Copies an event-owned C string into a Ruby string
|
|
28
|
+
# @param ptr [FFI::Pointer] pointer to the C string
|
|
29
|
+
# @return [String, nil] copied string or nil when NULL
|
|
30
|
+
def read_event_string(ptr)
|
|
31
|
+
ptr.null? ? nil : ptr.read_string
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Resolves a handle directly from an operation-level event error and unlocks it.
|
|
35
|
+
#
|
|
36
|
+
# librdkafka signals a failure of the whole admin operation (e.g. `_TIMED_OUT` when the
|
|
37
|
+
# brokers are unreachable, or `_DESTROY` when the client is closed with the request in
|
|
38
|
+
# flight) by setting the event error and delivering an *empty* results array. The
|
|
39
|
+
# per-element result handlers must not index into that empty array; they call this first
|
|
40
|
+
# and skip results parsing when it returns true. Without this, indexing `results[0]`
|
|
41
|
+
# raises inside the FFI callback, the handle is never unlocked, and the caller's `wait`
|
|
42
|
+
# blocks until its own timeout and raises `WaitTimeoutError`, losing the real error code.
|
|
43
|
+
#
|
|
44
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
45
|
+
# @param handle [Rdkafka::AbstractHandle] the handle to resolve
|
|
46
|
+
# @return [Boolean] true when the event carried an operation-level error and the handle
|
|
47
|
+
# was resolved here
|
|
48
|
+
def resolve_operation_error(event_ptr, handle)
|
|
49
|
+
error_code = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
|
|
50
|
+
|
|
51
|
+
return false if error_code == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
52
|
+
|
|
53
|
+
handle[:response] = error_code
|
|
54
|
+
handle.broker_message = read_event_string(Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr))
|
|
55
|
+
handle.unlock
|
|
56
|
+
|
|
57
|
+
true
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_EVENT_CREATEACLS_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class CreateAclHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the create-ACL handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
create_acls_result = Rdkafka::Bindings.rd_kafka_event_CreateAcls_result(event_ptr)
|
|
14
|
+
|
|
15
|
+
# Get the number of acl results
|
|
16
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
17
|
+
create_acl_result_array = Rdkafka::Bindings.rd_kafka_CreateAcls_result_acls(create_acls_result, pointer_to_size_t)
|
|
18
|
+
create_acl_results = CreateAclResult.create_acl_results_from_array(pointer_to_size_t.read_int, create_acl_result_array)
|
|
19
|
+
create_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
20
|
+
|
|
21
|
+
if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
|
|
22
|
+
unless resolve_operation_error(event_ptr, create_acl_handle)
|
|
23
|
+
create_acl_handle[:response] = create_acl_results[0].result_error
|
|
24
|
+
create_acl_handle.result = Rdkafka::Admin::CreateAclReport.new(
|
|
25
|
+
rdkafka_response: create_acl_results[0].result_error,
|
|
26
|
+
rdkafka_response_string: create_acl_results[0].error_string
|
|
27
|
+
)
|
|
28
|
+
create_acl_handle.broker_message = create_acl_handle.result.rdkafka_response_string
|
|
29
|
+
|
|
30
|
+
create_acl_handle.unlock
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class CreatePartitionsHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the create-partitions handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
create_partitionss_result = Rdkafka::Bindings.rd_kafka_event_CreatePartitions_result(event_ptr)
|
|
14
|
+
|
|
15
|
+
# Get the number of create topic results
|
|
16
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
17
|
+
create_partitions_result_array = Rdkafka::Bindings.rd_kafka_CreatePartitions_result_topics(create_partitionss_result, pointer_to_size_t)
|
|
18
|
+
create_partitions_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_partitions_result_array)
|
|
19
|
+
create_partitions_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
20
|
+
|
|
21
|
+
if create_partitions_handle = Rdkafka::Admin::CreatePartitionsHandle.remove(create_partitions_handle_ptr.address)
|
|
22
|
+
unless resolve_operation_error(event_ptr, create_partitions_handle)
|
|
23
|
+
create_partitions_handle[:response] = create_partitions_results[0].result_error
|
|
24
|
+
create_partitions_handle.result = Rdkafka::Admin::CreatePartitionsReport.new(
|
|
25
|
+
create_partitions_results[0].error_string,
|
|
26
|
+
create_partitions_results[0].result_name
|
|
27
|
+
)
|
|
28
|
+
create_partitions_handle.broker_message = create_partitions_handle.result.error_string
|
|
29
|
+
|
|
30
|
+
create_partitions_handle.unlock
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_EVENT_CREATETOPICS_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class CreateTopicHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the create-topic handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
create_topics_result = Rdkafka::Bindings.rd_kafka_event_CreateTopics_result(event_ptr)
|
|
14
|
+
|
|
15
|
+
# Get the number of create topic results
|
|
16
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
17
|
+
create_topic_result_array = Rdkafka::Bindings.rd_kafka_CreateTopics_result_topics(create_topics_result, pointer_to_size_t)
|
|
18
|
+
create_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, create_topic_result_array)
|
|
19
|
+
create_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
20
|
+
|
|
21
|
+
if create_topic_handle = Rdkafka::Admin::CreateTopicHandle.remove(create_topic_handle_ptr.address)
|
|
22
|
+
unless resolve_operation_error(event_ptr, create_topic_handle)
|
|
23
|
+
create_topic_handle[:response] = create_topic_results[0].result_error
|
|
24
|
+
create_topic_handle.result = Rdkafka::Admin::CreateTopicReport.new(
|
|
25
|
+
create_topic_results[0].error_string,
|
|
26
|
+
create_topic_results[0].result_name
|
|
27
|
+
)
|
|
28
|
+
create_topic_handle.broker_message = create_topic_handle.result.error_string
|
|
29
|
+
|
|
30
|
+
create_topic_handle.unlock
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_EVENT_DELETEACLS_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class DeleteAclHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the delete-ACL handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
delete_acls_result = Rdkafka::Bindings.rd_kafka_event_DeleteAcls_result(event_ptr)
|
|
14
|
+
|
|
15
|
+
# Get the number of acl results
|
|
16
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
17
|
+
delete_acl_result_responses = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_responses(delete_acls_result, pointer_to_size_t)
|
|
18
|
+
delete_acl_results = DeleteAclResult.delete_acl_results_from_array(pointer_to_size_t.read_int, delete_acl_result_responses)
|
|
19
|
+
delete_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
20
|
+
|
|
21
|
+
if delete_acl_handle = Rdkafka::Admin::DeleteAclHandle.remove(delete_acl_handle_ptr.address)
|
|
22
|
+
unless resolve_operation_error(event_ptr, delete_acl_handle)
|
|
23
|
+
delete_acl_handle[:response] = delete_acl_results[0].result_error
|
|
24
|
+
delete_acl_handle.broker_message = read_event_string(delete_acl_results[0].error_string)
|
|
25
|
+
|
|
26
|
+
delete_acl_handle.result = if delete_acl_results[0].result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
27
|
+
Rdkafka::Admin::DeleteAclReport.new(
|
|
28
|
+
matching_acls: delete_acl_results[0].matching_acls,
|
|
29
|
+
matching_acls_count: delete_acl_results[0].matching_acls_count
|
|
30
|
+
)
|
|
31
|
+
else
|
|
32
|
+
Rdkafka::Admin::DeleteAclReport.new(matching_acls: FFI::Pointer::NULL, matching_acls_count: 0)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
delete_acl_handle.unlock
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_EVENT_DELETEGROUPS_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class DeleteGroupsHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the delete-groups handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
delete_groups_result = Rdkafka::Bindings.rd_kafka_event_DeleteGroups_result(event_ptr)
|
|
14
|
+
|
|
15
|
+
# Get the number of delete group results
|
|
16
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
17
|
+
delete_group_result_array = Rdkafka::Bindings.rd_kafka_DeleteGroups_result_groups(delete_groups_result, pointer_to_size_t)
|
|
18
|
+
delete_group_results = GroupResult.create_group_results_from_array(pointer_to_size_t.read_int, delete_group_result_array)
|
|
19
|
+
delete_group_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
20
|
+
|
|
21
|
+
if (delete_group_handle = Rdkafka::Admin::DeleteGroupsHandle.remove(delete_group_handle_ptr.address))
|
|
22
|
+
unless resolve_operation_error(event_ptr, delete_group_handle)
|
|
23
|
+
delete_group_handle[:response] = delete_group_results[0].result_error
|
|
24
|
+
delete_group_handle.result = Rdkafka::Admin::DeleteGroupsReport.new(
|
|
25
|
+
delete_group_results[0].error_string,
|
|
26
|
+
delete_group_results[0].result_name
|
|
27
|
+
)
|
|
28
|
+
delete_group_handle.broker_message = delete_group_handle.result.error_string
|
|
29
|
+
|
|
30
|
+
delete_group_handle.unlock
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_EVENT_DELETETOPICS_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class DeleteTopicHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the delete-topic handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
delete_topics_result = Rdkafka::Bindings.rd_kafka_event_DeleteTopics_result(event_ptr)
|
|
14
|
+
|
|
15
|
+
# Get the number of topic results
|
|
16
|
+
pointer_to_size_t = FFI::MemoryPointer.new(:size_t)
|
|
17
|
+
delete_topic_result_array = Rdkafka::Bindings.rd_kafka_DeleteTopics_result_topics(delete_topics_result, pointer_to_size_t)
|
|
18
|
+
delete_topic_results = TopicResult.create_topic_results_from_array(pointer_to_size_t.read_int, delete_topic_result_array)
|
|
19
|
+
delete_topic_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
20
|
+
|
|
21
|
+
if delete_topic_handle = Rdkafka::Admin::DeleteTopicHandle.remove(delete_topic_handle_ptr.address)
|
|
22
|
+
unless resolve_operation_error(event_ptr, delete_topic_handle)
|
|
23
|
+
delete_topic_handle[:response] = delete_topic_results[0].result_error
|
|
24
|
+
delete_topic_handle.result = Rdkafka::Admin::DeleteTopicReport.new(
|
|
25
|
+
delete_topic_results[0].error_string,
|
|
26
|
+
delete_topic_results[0].result_name
|
|
27
|
+
)
|
|
28
|
+
delete_topic_handle.broker_message = delete_topic_handle.result.error_string
|
|
29
|
+
|
|
30
|
+
delete_topic_handle.unlock
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Rdkafka
|
|
4
|
+
module Callbacks
|
|
5
|
+
# Handles `RD_KAFKA_EVENT_DESCRIBEACLS_RESULT` events
|
|
6
|
+
# @private
|
|
7
|
+
class DescribeAclHandler < BaseHandler
|
|
8
|
+
class << self
|
|
9
|
+
# Resolves the describe-ACL handle from its result event
|
|
10
|
+
# @param event_ptr [FFI::Pointer] pointer to the event
|
|
11
|
+
# @return [void]
|
|
12
|
+
def call(event_ptr)
|
|
13
|
+
describe_acl = DescribeAclResult.new(event_ptr)
|
|
14
|
+
describe_acl_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
|
|
15
|
+
|
|
16
|
+
if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
|
|
17
|
+
describe_acl_handle[:response] = describe_acl.result_error
|
|
18
|
+
describe_acl_handle.broker_message = read_event_string(describe_acl.error_string)
|
|
19
|
+
|
|
20
|
+
describe_acl_handle.result = if describe_acl.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
|
|
21
|
+
Rdkafka::Admin::DescribeAclReport.new(
|
|
22
|
+
acls: describe_acl.matching_acls,
|
|
23
|
+
acls_count: describe_acl.matching_acls_count
|
|
24
|
+
)
|
|
25
|
+
else
|
|
26
|
+
Rdkafka::Admin::DescribeAclReport.new(acls: FFI::Pointer::NULL, acls_count: 0)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
describe_acl_handle.unlock
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|