karafka-rdkafka 0.30.2-aarch64-linux-gnu → 0.32.0-aarch64-linux-gnu

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +100 -36
  3. data/README.md +3 -1
  4. data/docker-compose-ssl.yml +1 -0
  5. data/docker-compose.yml +1 -0
  6. data/ext/librdkafka.so +0 -0
  7. data/karafka-rdkafka.gemspec +5 -4
  8. data/lib/rdkafka/admin/acl_binding_result.rb +4 -16
  9. data/lib/rdkafka/admin/alter_consumer_group_offsets_report.rb +2 -2
  10. data/lib/rdkafka/admin/create_acl_report.rb +1 -3
  11. data/lib/rdkafka/admin/create_topic_report.rb +2 -6
  12. data/lib/rdkafka/admin/delete_consumer_group_offsets_report.rb +2 -2
  13. data/lib/rdkafka/admin/delete_groups_report.rb +2 -6
  14. data/lib/rdkafka/admin/delete_topic_report.rb +2 -6
  15. data/lib/rdkafka/admin/describe_cluster_handle.rb +31 -0
  16. data/lib/rdkafka/admin/describe_cluster_report.rb +51 -0
  17. data/lib/rdkafka/admin/describe_consumer_groups_handle.rb +31 -0
  18. data/lib/rdkafka/admin/describe_consumer_groups_report.rb +151 -0
  19. data/lib/rdkafka/admin/describe_result_parsing.rb +47 -0
  20. data/lib/rdkafka/admin/describe_topics_handle.rb +31 -0
  21. data/lib/rdkafka/admin/describe_topics_report.rb +118 -0
  22. data/lib/rdkafka/admin/list_consumer_group_offsets_handle.rb +31 -0
  23. data/lib/rdkafka/admin/list_consumer_group_offsets_report.rb +52 -0
  24. data/lib/rdkafka/admin.rb +424 -9
  25. data/lib/rdkafka/bindings/admin/acls.rb +85 -0
  26. data/lib/rdkafka/bindings/admin/configs.rb +37 -0
  27. data/lib/rdkafka/bindings/admin/list_offsets.rb +26 -0
  28. data/lib/rdkafka/bindings/admin.rb +211 -0
  29. data/lib/rdkafka/bindings/callbacks.rb +155 -0
  30. data/lib/rdkafka/bindings/configuration.rb +50 -0
  31. data/lib/rdkafka/bindings/consumer.rb +125 -0
  32. data/lib/rdkafka/bindings/errors.rb +17 -0
  33. data/lib/rdkafka/bindings/handle.rb +16 -0
  34. data/lib/rdkafka/bindings/messages.rb +28 -0
  35. data/lib/rdkafka/bindings/metadata.rb +17 -0
  36. data/lib/rdkafka/bindings/polling.rb +17 -0
  37. data/lib/rdkafka/bindings/producer.rb +82 -0
  38. data/lib/rdkafka/bindings/share_consumer.rb +67 -0
  39. data/lib/rdkafka/bindings/structs.rb +62 -0
  40. data/lib/rdkafka/bindings.rb +16 -775
  41. data/lib/rdkafka/callbacks/describe_cluster_handler.rb +36 -0
  42. data/lib/rdkafka/callbacks/describe_consumer_groups_handler.rb +36 -0
  43. data/lib/rdkafka/callbacks/describe_topics_handler.rb +36 -0
  44. data/lib/rdkafka/callbacks/list_consumer_group_offsets_handler.rb +62 -0
  45. data/lib/rdkafka/callbacks.rb +15 -7
  46. data/lib/rdkafka/clients.rb +13 -46
  47. data/lib/rdkafka/config.rb +70 -0
  48. data/lib/rdkafka/consumer/allocator.rb +0 -2
  49. data/lib/rdkafka/consumer/message.rb +6 -9
  50. data/lib/rdkafka/consumer/topic_partition_list.rb +7 -7
  51. data/lib/rdkafka/consumer.rb +76 -36
  52. data/lib/rdkafka/defaults.rb +6 -4
  53. data/lib/rdkafka/native_kafka.rb +45 -31
  54. data/lib/rdkafka/producer/delivery_handle.rb +18 -16
  55. data/lib/rdkafka/producer/delivery_report.rb +51 -1
  56. data/lib/rdkafka/producer/partitions_count_cache.rb +109 -26
  57. data/lib/rdkafka/producer.rb +141 -71
  58. data/lib/rdkafka/share_consumer.rb +70 -34
  59. data/lib/rdkafka/version.rb +3 -3
  60. data/lib/rdkafka.rb +14 -1
  61. data/package-lock.json +9 -9
  62. data/renovate.json +7 -0
  63. metadata +31 -31
data/lib/rdkafka/admin.rb CHANGED
@@ -95,7 +95,7 @@ module Rdkafka
95
95
 
96
96
  # Polls for events in a non-blocking loop, yielding the count after each iteration.
97
97
  #
98
- # This method processes events (stats, errors, etc.) in a single GVL/mutex session,
98
+ # This method processes events (stats, errors, etc.) in a single GVL session,
99
99
  # which is more efficient than repeated individual polls. It uses non-blocking polls
100
100
  # internally (no GVL release between polls).
101
101
  #
@@ -116,8 +116,9 @@ module Rdkafka
116
116
  # admin.events_poll_nb_each do |_count|
117
117
  # :stop if monotonic_now >= deadline
118
118
  # end
119
- # @note This method holds the inner lock until the queue is empty or `:stop` is returned.
120
- # Other admin operations will wait until this method returns.
119
+ # @note This method runs inside `with_inner` and does not hold a lock while it loops.
120
+ # Other operations on the same client run concurrently. `close` waits until this
121
+ # method returns.
121
122
  # @note This method is thread-safe as it uses @native_kafka.with_inner synchronization
122
123
  def events_poll_nb_each
123
124
  closed_admin_check(__method__)
@@ -149,14 +150,35 @@ module Rdkafka
149
150
  # @param partition_count [Integer] number of partitions for the topic
150
151
  # @param replication_factor [Integer] replication factor for the topic
151
152
  # @param topic_config [Hash] optional topic configuration settings
153
+ # @param validate_only [Boolean] when true, the broker only validates the request and does not
154
+ # create the topic
155
+ # @param operation_timeout_ms [Integer, nil] how long the broker waits for the topic creation
156
+ # to complete before it responds (librdkafka default: 60 seconds)
157
+ # @param request_timeout_ms [Integer, nil] overall request timeout (librdkafka default:
158
+ # `socket.timeout.ms`)
159
+ # @param braceless_topic_config [Hash] topic configuration given as a hash without braces,
160
+ # e.g. `create_topic("t", 1, 1, "retention.ms" => "1000")`, merged into `topic_config`
161
+ # @option braceless_topic_config [String] any topic configuration name, e.g. `"retention.ms"`
152
162
  # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of
153
163
  # creating the topic
154
164
  # @raise [ConfigError] When the partition count or replication factor are out of valid range
155
165
  # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
156
166
  # @raise [RdkafkaError] When the topic configuration is invalid
157
- def create_topic(topic_name, partition_count, replication_factor, topic_config = {})
167
+ # @raise [RdkafkaError] When librdkafka rejects a timeout value
168
+ def create_topic(
169
+ topic_name,
170
+ partition_count,
171
+ replication_factor,
172
+ topic_config = {},
173
+ validate_only: false,
174
+ operation_timeout_ms: nil,
175
+ request_timeout_ms: nil,
176
+ **braceless_topic_config
177
+ )
158
178
  closed_admin_check(__method__)
159
179
 
180
+ topic_config = (topic_config || {}).merge(braceless_topic_config) unless braceless_topic_config.empty?
181
+
160
182
  # Create a rd_kafka_NewTopic_t representing the new topic
161
183
  error_buffer = FFI::MemoryPointer.from_string(" " * 256)
162
184
  new_topic_ptr = Rdkafka::Bindings.rd_kafka_NewTopic_new(
@@ -203,6 +225,13 @@ module Rdkafka
203
225
  Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_topic_handle.to_ptr)
204
226
 
205
227
  begin
228
+ set_admin_options(
229
+ admin_options_ptr,
230
+ validate_only: validate_only,
231
+ operation_timeout_ms: operation_timeout_ms,
232
+ request_timeout_ms: request_timeout_ms
233
+ )
234
+
206
235
  @native_kafka.with_inner do |inner|
207
236
  Rdkafka::Bindings.rd_kafka_CreateTopics(
208
237
  inner,
@@ -224,6 +253,51 @@ module Rdkafka
224
253
  create_topic_handle
225
254
  end
226
255
 
256
+ # Reads the committed offsets of a consumer group from outside the group.
257
+ #
258
+ # Unlike {Consumer#committed}, this works for any group and does not join it, so it does not
259
+ # trigger a rebalance of the group it inspects.
260
+ #
261
+ # While the group coordinator is loading, not known yet or has moved (e.g. on a fresh cluster
262
+ # or after a broker restart), the handle raises `coordinator_load_in_progress`,
263
+ # `not_coordinator` or `coordinator_not_available`. Retry the call on these codes;
264
+ # `RdkafkaError#retryable?` is false for them.
265
+ #
266
+ # @param group_id [String] the group whose committed offsets we want to read
267
+ # @param topic_partition_list [TopicPartitionList, nil] partitions to read the offsets for, or
268
+ # nil to read every partition the group has committed offsets for. List the partitions
269
+ # explicitly: a topic without partitions is not expanded to its partitions
270
+ # @param require_stable_offsets [Boolean] pending offsets of an open transaction are never
271
+ # returned. Without this option the last committed offset is returned for such a partition.
272
+ # With it, the call fails with `unstable_offset_commit` until that transaction is committed
273
+ # or aborted; retry on this code
274
+ # @return [ListConsumerGroupOffsetsHandle] handle that can be used to wait for the result
275
+ # @raise [RdkafkaError] when listing the offsets fails
276
+ def list_consumer_group_offsets(group_id, topic_partition_list = nil, require_stable_offsets: false)
277
+ closed_admin_check(__method__)
278
+
279
+ native_tpl = topic_partition_list&.to_native_tpl
280
+
281
+ begin
282
+ list_ptr = Rdkafka::Bindings.rd_kafka_ListConsumerGroupOffsets_new(
283
+ FFI::MemoryPointer.from_string(group_id),
284
+ native_tpl || FFI::Pointer::NULL
285
+ )
286
+ ensure
287
+ # librdkafka copies the list into the request object, so ours is no longer needed
288
+ Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native_tpl) if native_tpl
289
+ end
290
+
291
+ enqueue_group_offsets_op(
292
+ request_ptr: list_ptr,
293
+ handle: ListConsumerGroupOffsetsHandle.new,
294
+ admin_op: Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_LISTCONSUMERGROUPOFFSETS,
295
+ destroy: Rdkafka::Bindings.method(:rd_kafka_ListConsumerGroupOffsets_destroy),
296
+ enqueue: Rdkafka::Bindings.method(:rd_kafka_ListConsumerGroupOffsets),
297
+ configure: ->(admin_options_ptr) { set_require_stable_offsets(admin_options_ptr) if require_stable_offsets }
298
+ )
299
+ end
300
+
227
301
  # Sets the committed offsets of a consumer group from outside the group.
228
302
  #
229
303
  # Kafka requires the group to be empty (no active members) for this to be accepted; when it
@@ -520,13 +594,243 @@ module Rdkafka
520
594
  handle
521
595
  end
522
596
 
597
+ # Describes the cluster: its id, the current controller and the broker nodes (with racks).
598
+ #
599
+ # Unlike {#metadata}, this does not fetch any topic metadata.
600
+ #
601
+ # @param include_authorized_operations [Boolean] when true, also return the operations this
602
+ # client is authorized to perform on the cluster
603
+ # @return [DescribeClusterHandle] handle that can be used to wait for the result
604
+ # @raise [RdkafkaError] when describing the cluster fails
605
+ #
606
+ # @example Print the cluster id, the controller and every broker
607
+ # report = admin.describe_cluster.wait(max_wait_timeout_ms: 15_000)
608
+ #
609
+ # puts "cluster: #{report.cluster_id}, controller: #{report.controller[:id]}"
610
+ # report.nodes.each do |node|
611
+ # puts "#{node[:id]} #{node[:host]}:#{node[:port]} #{node[:rack]}"
612
+ # end
613
+ def describe_cluster(include_authorized_operations: false)
614
+ closed_admin_check(__method__)
615
+
616
+ queue_ptr = @native_kafka.with_inner do |inner|
617
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
618
+ end
619
+
620
+ if queue_ptr.null?
621
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
622
+ end
623
+
624
+ handle = DescribeClusterHandle.new
625
+ handle[:pending] = true
626
+ handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
627
+ DescribeClusterHandle.register(handle)
628
+
629
+ admin_options_ptr = @native_kafka.with_inner do |inner|
630
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(
631
+ inner,
632
+ Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECLUSTER
633
+ )
634
+ end
635
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
636
+
637
+ begin
638
+ set_include_authorized_operations(admin_options_ptr) if include_authorized_operations
639
+
640
+ @native_kafka.with_inner do |inner|
641
+ Rdkafka::Bindings.rd_kafka_DescribeCluster(
642
+ inner,
643
+ admin_options_ptr,
644
+ queue_ptr
645
+ )
646
+ end
647
+ rescue Exception
648
+ DescribeClusterHandle.remove(handle.to_ptr.address)
649
+ raise
650
+ ensure
651
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
652
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
653
+ end
654
+
655
+ handle
656
+ end
657
+
658
+ # Describes the given topics: their topic ids, partitions (leader, replicas and in-sync
659
+ # replicas, with racks) and, optionally, the operations this client is authorized to perform.
660
+ #
661
+ # A topic that cannot be described (e.g. it does not exist) does not fail the whole request;
662
+ # its entry in the report carries an `:error` instead.
663
+ #
664
+ # @param topic_names [Array<String>] names of the topics to describe
665
+ # @param include_authorized_operations [Boolean] when true, also return the operations this
666
+ # client is authorized to perform on each topic
667
+ # @return [DescribeTopicsHandle] handle that can be used to wait for the result
668
+ # @raise [ArgumentError] when topic_names is not an Array of Strings
669
+ # @raise [RdkafkaError] when describing the topics fails
670
+ #
671
+ # @example Print the topic id and partition leaders of a topic
672
+ # report = admin.describe_topics(["events"]).wait(max_wait_timeout_ms: 15_000)
673
+ #
674
+ # report.topics.each do |topic|
675
+ # next warn("#{topic[:name]}: #{topic[:error].message}") if topic[:error]
676
+ #
677
+ # puts "#{topic[:name]} (#{topic[:topic_id]})"
678
+ # topic[:partitions].each do |partition|
679
+ # puts " #{partition[:partition]} -> #{partition[:leader]&.fetch(:id)}"
680
+ # end
681
+ # end
682
+ def describe_topics(topic_names, include_authorized_operations: false)
683
+ closed_admin_check(__method__)
684
+
685
+ unless topic_names.is_a?(Array) && topic_names.all?(String)
686
+ raise ArgumentError, "topic_names must be an Array of Strings"
687
+ end
688
+
689
+ queue_ptr = @native_kafka.with_inner do |inner|
690
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
691
+ end
692
+
693
+ if queue_ptr.null?
694
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
695
+ end
696
+
697
+ handle = DescribeTopicsHandle.new
698
+ handle[:pending] = true
699
+ handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
700
+ DescribeTopicsHandle.register(handle)
701
+
702
+ admin_options_ptr = @native_kafka.with_inner do |inner|
703
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(
704
+ inner,
705
+ Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBETOPICS
706
+ )
707
+ end
708
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
709
+
710
+ topic_collection_ptr = nil
711
+
712
+ begin
713
+ set_include_authorized_operations(admin_options_ptr) if include_authorized_operations
714
+
715
+ # librdkafka copies the names, so the Ruby-owned strings only need to outlive this call
716
+ name_ptrs = topic_names.map { |name| FFI::MemoryPointer.from_string(name) }
717
+ names_array_ptr = FFI::MemoryPointer.new(:pointer, [name_ptrs.size, 1].max)
718
+ names_array_ptr.write_array_of_pointer(name_ptrs)
719
+
720
+ topic_collection_ptr = Rdkafka::Bindings.rd_kafka_TopicCollection_of_topic_names(
721
+ names_array_ptr,
722
+ name_ptrs.size
723
+ )
724
+
725
+ @native_kafka.with_inner do |inner|
726
+ Rdkafka::Bindings.rd_kafka_DescribeTopics(
727
+ inner,
728
+ topic_collection_ptr,
729
+ admin_options_ptr,
730
+ queue_ptr
731
+ )
732
+ end
733
+ rescue Exception
734
+ DescribeTopicsHandle.remove(handle.to_ptr.address)
735
+ raise
736
+ ensure
737
+ Rdkafka::Bindings.rd_kafka_TopicCollection_destroy(topic_collection_ptr) if topic_collection_ptr
738
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
739
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
740
+ end
741
+
742
+ handle
743
+ end
744
+
745
+ # Describes the given consumer groups: their state, partition assignor, coordinator and
746
+ # members with their partition assignments.
747
+ #
748
+ # Each requested group is reported separately, with its own `:error`. A group id the cluster
749
+ # does not know is not an error: it is reported in the `Dead` state with no members.
750
+ #
751
+ # @param group_ids [Array<String>] ids of the consumer groups to describe
752
+ # @param include_authorized_operations [Boolean] whether to also fetch the ACL operations the
753
+ # client is authorized to perform on each group (reported as `:authorized_operations`)
754
+ # @return [DescribeConsumerGroupsHandle] handle that can be used to wait for the result
755
+ # @raise [RdkafkaError] when describing the consumer groups fails as a whole
756
+ #
757
+ # @example Describe a consumer group and print its members and their assignments
758
+ # report = admin.describe_consumer_groups(["example-group"]).wait(max_wait_timeout_ms: 15_000)
759
+ #
760
+ # report.groups.each do |group|
761
+ # next warn "#{group[:group_id]}: #{group[:error].message}" if group[:error]
762
+ #
763
+ # puts "#{group[:group_id]} - #{group[:state_name]} (#{group[:members].size} members)"
764
+ #
765
+ # group[:members].each do |member|
766
+ # puts " #{member[:client_id]}@#{member[:host]}: #{member[:assignment]}"
767
+ # end
768
+ # end
769
+ def describe_consumer_groups(group_ids, include_authorized_operations: false)
770
+ closed_admin_check(__method__)
771
+
772
+ group_id_ptrs = group_ids.map { |group_id| FFI::MemoryPointer.from_string(group_id) }
773
+ group_ids_ptr = FFI::MemoryPointer.new(:pointer, group_id_ptrs.size)
774
+ group_ids_ptr.write_array_of_pointer(group_id_ptrs)
775
+
776
+ # Get a pointer to the queue that our request will be enqueued on
777
+ queue_ptr = @native_kafka.with_inner do |inner|
778
+ Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
779
+ end
780
+
781
+ if queue_ptr.null?
782
+ raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
783
+ end
784
+
785
+ # Create and register the handle we will return to the caller
786
+ handle = DescribeConsumerGroupsHandle.new
787
+ handle[:pending] = true
788
+ handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
789
+ DescribeConsumerGroupsHandle.register(handle)
790
+
791
+ admin_options_ptr = @native_kafka.with_inner do |inner|
792
+ Rdkafka::Bindings.rd_kafka_AdminOptions_new(
793
+ inner,
794
+ Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DESCRIBECONSUMERGROUPS
795
+ )
796
+ end
797
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
798
+
799
+ begin
800
+ set_include_authorized_operations(admin_options_ptr) if include_authorized_operations
801
+
802
+ @native_kafka.with_inner do |inner|
803
+ Rdkafka::Bindings.rd_kafka_DescribeConsumerGroups(
804
+ inner,
805
+ group_ids_ptr,
806
+ group_id_ptrs.size,
807
+ admin_options_ptr,
808
+ queue_ptr
809
+ )
810
+ end
811
+ rescue Exception
812
+ DescribeConsumerGroupsHandle.remove(handle.to_ptr.address)
813
+ raise
814
+ ensure
815
+ Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
816
+ Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
817
+ end
818
+
819
+ handle
820
+ end
821
+
523
822
  # Deletes the named topic
524
823
  #
525
824
  # @param topic_name [String] name of the topic to delete
825
+ # @param operation_timeout_ms [Integer, nil] how long the broker waits for the topic deletion
826
+ # to complete before it responds (librdkafka default: 60 seconds)
827
+ # @param request_timeout_ms [Integer, nil] overall request timeout (librdkafka default:
828
+ # `socket.timeout.ms`)
526
829
  # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of
527
830
  # deleting the topic
528
831
  # @raise [RdkafkaError] When the topic name is invalid or the topic does not exist
529
- def delete_topic(topic_name)
832
+ # @raise [RdkafkaError] When librdkafka rejects a timeout value
833
+ def delete_topic(topic_name, operation_timeout_ms: nil, request_timeout_ms: nil)
530
834
  closed_admin_check(__method__)
531
835
 
532
836
  # Create a rd_kafka_DeleteTopic_t representing the topic to be deleted
@@ -557,6 +861,12 @@ module Rdkafka
557
861
  Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_topic_handle.to_ptr)
558
862
 
559
863
  begin
864
+ set_admin_options(
865
+ admin_options_ptr,
866
+ operation_timeout_ms: operation_timeout_ms,
867
+ request_timeout_ms: request_timeout_ms
868
+ )
869
+
560
870
  @native_kafka.with_inner do |inner|
561
871
  Rdkafka::Bindings.rd_kafka_DeleteTopics(
562
872
  inner,
@@ -582,12 +892,25 @@ module Rdkafka
582
892
  #
583
893
  # @param topic_name [String] name of the topic
584
894
  # @param partition_count [Integer] how many partitions we want to end up with for given topic
895
+ # @param validate_only [Boolean] when true, the broker only validates the request and does not
896
+ # add the partitions
897
+ # @param operation_timeout_ms [Integer, nil] how long the broker waits for the partitions to be
898
+ # created before it responds (librdkafka default: 60 seconds)
899
+ # @param request_timeout_ms [Integer, nil] overall request timeout (librdkafka default:
900
+ # `socket.timeout.ms`)
585
901
  # @return [CreatePartitionsHandle] Create partitions handle that can be used to wait for the
586
902
  # result
587
903
  # @raise [ConfigError] When the partition count or replication factor are out of valid range
588
904
  # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
589
905
  # @raise [RdkafkaError] When the topic configuration is invalid
590
- def create_partitions(topic_name, partition_count)
906
+ # @raise [RdkafkaError] When librdkafka rejects a timeout value
907
+ def create_partitions(
908
+ topic_name,
909
+ partition_count,
910
+ validate_only: false,
911
+ operation_timeout_ms: nil,
912
+ request_timeout_ms: nil
913
+ )
591
914
  closed_admin_check(__method__)
592
915
 
593
916
  @native_kafka.with_inner do |inner|
@@ -622,6 +945,13 @@ module Rdkafka
622
945
  Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_partitions_handle.to_ptr)
623
946
 
624
947
  begin
948
+ set_admin_options(
949
+ admin_options_ptr,
950
+ validate_only: validate_only,
951
+ operation_timeout_ms: operation_timeout_ms,
952
+ request_timeout_ms: request_timeout_ms
953
+ )
954
+
625
955
  Rdkafka::Bindings.rd_kafka_CreatePartitions(
626
956
  inner,
627
957
  topics_array_ptr,
@@ -1206,8 +1536,8 @@ module Rdkafka
1206
1536
 
1207
1537
  private
1208
1538
 
1209
- # Shared enqueue path for the two consumer group offset admin operations. They differ only
1210
- # in their C entry points, so keeping one body avoids two near identical copies of the
1539
+ # Shared enqueue path for the consumer group offset admin operations. They differ only in
1540
+ # their C entry points, so keeping one body avoids near identical copies of the
1211
1541
  # queue/handle/options dance and its cleanup.
1212
1542
  #
1213
1543
  # @param request_ptr [FFI::Pointer] the op specific request object
@@ -1215,8 +1545,13 @@ module Rdkafka
1215
1545
  # @param admin_op [Integer] the RD_KAFKA_ADMIN_OP_* constant for this operation
1216
1546
  # @param destroy [Method] bound function used to free the request object
1217
1547
  # @param enqueue [Method] bound function used to enqueue the request
1548
+ # @param configure [Proc, nil] called with the `rd_kafka_AdminOptions_t` pointer before the
1549
+ # request is enqueued, to set an option only this operation supports. For example
1550
+ # `#list_consumer_group_offsets` always passes a lambda that takes `admin_options_ptr` and
1551
+ # calls `set_require_stable_offsets(admin_options_ptr)` only when
1552
+ # `require_stable_offsets: true`. nil (alter and delete) sets no extra option
1218
1553
  # @return [AbstractHandle] the registered handle
1219
- def enqueue_group_offsets_op(request_ptr:, handle:, admin_op:, destroy:, enqueue:)
1554
+ def enqueue_group_offsets_op(request_ptr:, handle:, admin_op:, destroy:, enqueue:, configure: nil)
1220
1555
  array_ptr = FFI::MemoryPointer.new(:pointer)
1221
1556
  array_ptr.write_array_of_pointer([request_ptr])
1222
1557
 
@@ -1239,6 +1574,8 @@ module Rdkafka
1239
1574
  Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, handle.to_ptr)
1240
1575
 
1241
1576
  begin
1577
+ configure&.call(admin_options_ptr)
1578
+
1242
1579
  @native_kafka.with_inner do |inner|
1243
1580
  enqueue.call(inner, array_ptr, 1, admin_options_ptr, queue_ptr)
1244
1581
  end
@@ -1254,6 +1591,84 @@ module Rdkafka
1254
1591
  handle
1255
1592
  end
1256
1593
 
1594
+ # Asks librdkafka to include authorized operations in a describe result
1595
+ # @param admin_options_ptr [FFI::Pointer] the admin options to update
1596
+ # @raise [RdkafkaError] when librdkafka rejects the option
1597
+ def set_include_authorized_operations(admin_options_ptr)
1598
+ raise_admin_option_error(
1599
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_include_authorized_operations(admin_options_ptr, 1)
1600
+ )
1601
+ end
1602
+
1603
+ # Makes the broker fail a partition with `unstable_offset_commit` while a transaction with
1604
+ # pending offsets for it is open, instead of returning its last committed offset
1605
+ # @param admin_options_ptr [FFI::Pointer] the admin options to update
1606
+ # @raise [RdkafkaError] when librdkafka rejects the option
1607
+ def set_require_stable_offsets(admin_options_ptr)
1608
+ raise_admin_option_error(
1609
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_require_stable_offsets(admin_options_ptr, 1)
1610
+ )
1611
+ end
1612
+
1613
+ # Raises and releases the error librdkafka returned from setting an admin option
1614
+ # @param error_ptr [FFI::Pointer] the returned `rd_kafka_error_t*`, NULL on success
1615
+ # @raise [RdkafkaError] when the error pointer is not NULL
1616
+ def raise_admin_option_error(error_ptr)
1617
+ return if error_ptr.null?
1618
+
1619
+ begin
1620
+ string_ptr = Rdkafka::Bindings.rd_kafka_error_string(error_ptr)
1621
+
1622
+ raise RdkafkaError.new(
1623
+ Rdkafka::Bindings.rd_kafka_error_code(error_ptr),
1624
+ broker_message: string_ptr.null? ? nil : string_ptr.read_string
1625
+ )
1626
+ ensure
1627
+ Rdkafka::Bindings.rd_kafka_error_destroy(error_ptr)
1628
+ end
1629
+ end
1630
+
1631
+ # Applies the optional request settings to admin options
1632
+ # @param admin_options_ptr [FFI::Pointer] the admin options to update
1633
+ # @param validate_only [Boolean] whether the broker should only validate the request
1634
+ # @param operation_timeout_ms [Integer, nil] broker operation timeout, librdkafka default if nil
1635
+ # @param request_timeout_ms [Integer, nil] request timeout, librdkafka default if nil
1636
+ # @raise [RdkafkaError] when librdkafka rejects a value
1637
+ def set_admin_options(admin_options_ptr, validate_only: false, operation_timeout_ms: nil, request_timeout_ms: nil)
1638
+ error_buffer = FFI::MemoryPointer.new(:char, 256)
1639
+
1640
+ if validate_only
1641
+ check_admin_option(
1642
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_validate_only(admin_options_ptr, 1, error_buffer, 256),
1643
+ error_buffer
1644
+ )
1645
+ end
1646
+
1647
+ if operation_timeout_ms
1648
+ check_admin_option(
1649
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_operation_timeout(admin_options_ptr, operation_timeout_ms, error_buffer, 256),
1650
+ error_buffer
1651
+ )
1652
+ end
1653
+
1654
+ if request_timeout_ms
1655
+ check_admin_option(
1656
+ Rdkafka::Bindings.rd_kafka_AdminOptions_set_request_timeout(admin_options_ptr, request_timeout_ms, error_buffer, 256),
1657
+ error_buffer
1658
+ )
1659
+ end
1660
+ end
1661
+
1662
+ # Raises when an admin options setter failed
1663
+ # @param response [Integer] result code of an admin options setter
1664
+ # @param error_buffer [FFI::MemoryPointer] buffer the setter wrote its error into
1665
+ # @raise [RdkafkaError] when the setter failed
1666
+ def check_admin_option(response, error_buffer)
1667
+ return if response == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
1668
+
1669
+ raise RdkafkaError.new(response, broker_message: error_buffer.read_string)
1670
+ end
1671
+
1257
1672
  # Checks if the admin is closed and raises an error if so
1258
1673
  # @param method [Symbol] name of the calling method for error context
1259
1674
  # @raise [ClosedAdminError] when the admin is closed
@@ -0,0 +1,85 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ module Bindings
5
+ # Create Acls
6
+ RD_KAFKA_ADMIN_OP_CREATEACLS = 9
7
+ RD_KAFKA_EVENT_CREATEACLS_RESULT = 1024
8
+
9
+ attach_function :rd_kafka_CreateAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
10
+ attach_function :rd_kafka_event_CreateAcls_result, [:pointer], :pointer
11
+ attach_function :rd_kafka_CreateAcls_result_acls, [:pointer, :pointer], :pointer
12
+
13
+ # Delete Acls
14
+ RD_KAFKA_ADMIN_OP_DELETEACLS = 11
15
+ RD_KAFKA_EVENT_DELETEACLS_RESULT = 4096
16
+
17
+ attach_function :rd_kafka_DeleteAcls, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
18
+ attach_function :rd_kafka_event_DeleteAcls_result, [:pointer], :pointer
19
+ attach_function :rd_kafka_DeleteAcls_result_responses, [:pointer, :pointer], :pointer
20
+ attach_function :rd_kafka_DeleteAcls_result_response_error, [:pointer], :pointer
21
+ attach_function :rd_kafka_DeleteAcls_result_response_matching_acls, [:pointer, :pointer], :pointer
22
+
23
+ # Describe Acls
24
+ RD_KAFKA_ADMIN_OP_DESCRIBEACLS = 10
25
+ RD_KAFKA_EVENT_DESCRIBEACLS_RESULT = 2048
26
+
27
+ attach_function :rd_kafka_DescribeAcls, [:pointer, :pointer, :pointer, :pointer], :void
28
+ attach_function :rd_kafka_event_DescribeAcls_result, [:pointer], :pointer
29
+ attach_function :rd_kafka_DescribeAcls_result_acls, [:pointer, :pointer], :pointer
30
+
31
+ # Acl Bindings
32
+
33
+ attach_function :rd_kafka_AclBinding_restype, [:pointer], :int32
34
+ attach_function :rd_kafka_AclBinding_name, [:pointer], :pointer
35
+ attach_function :rd_kafka_AclBinding_resource_pattern_type, [:pointer], :int32
36
+ attach_function :rd_kafka_AclBinding_principal, [:pointer], :pointer
37
+ attach_function :rd_kafka_AclBinding_host, [:pointer], :pointer
38
+ attach_function :rd_kafka_AclBinding_operation, [:pointer], :int32
39
+ attach_function :rd_kafka_AclBinding_permission_type, [:pointer], :int32
40
+ attach_function :rd_kafka_AclBinding_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t], :pointer
41
+ attach_function :rd_kafka_AclBindingFilter_new, [:int32, :pointer, :int32, :pointer, :pointer, :int32, :int32, :pointer, :size_t], :pointer
42
+ attach_function :rd_kafka_AclBinding_destroy, [:pointer], :void
43
+
44
+ # rd_kafka_ResourceType_t -
45
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b992/src/rdkafka.h#L7307
46
+ RD_KAFKA_RESOURCE_ANY = 1
47
+ RD_KAFKA_RESOURCE_TOPIC = 2
48
+ RD_KAFKA_RESOURCE_GROUP = 3
49
+ RD_KAFKA_RESOURCE_BROKER = 4
50
+ RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5
51
+
52
+ # rd_kafka_ResourcePatternType_t -
53
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b992/src/rdkafka.h#L7320
54
+ RD_KAFKA_RESOURCE_PATTERN_UNKNOWN = 0
55
+ RD_KAFKA_RESOURCE_PATTERN_ANY = 1
56
+ RD_KAFKA_RESOURCE_PATTERN_MATCH = 2
57
+ RD_KAFKA_RESOURCE_PATTERN_LITERAL = 3
58
+ RD_KAFKA_RESOURCE_PATTERN_PREFIXED = 4
59
+
60
+ # rd_kafka_AclOperation_t -
61
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b992/src/rdkafka.h#L8403
62
+ RD_KAFKA_ACL_OPERATION_ANY = 1
63
+ RD_KAFKA_ACL_OPERATION_ALL = 2
64
+ RD_KAFKA_ACL_OPERATION_READ = 3
65
+ RD_KAFKA_ACL_OPERATION_WRITE = 4
66
+ RD_KAFKA_ACL_OPERATION_CREATE = 5
67
+ RD_KAFKA_ACL_OPERATION_DELETE = 6
68
+ RD_KAFKA_ACL_OPERATION_ALTER = 7
69
+ RD_KAFKA_ACL_OPERATION_DESCRIBE = 8
70
+ RD_KAFKA_ACL_OPERATION_CLUSTER_ACTION = 9
71
+ RD_KAFKA_ACL_OPERATION_DESCRIBE_CONFIGS = 10
72
+ RD_KAFKA_ACL_OPERATION_ALTER_CONFIGS = 11
73
+ RD_KAFKA_ACL_OPERATION_IDEMPOTENT_WRITE = 12
74
+
75
+ # rd_kafka_AclPermissionType_t -
76
+ # https://github.com/confluentinc/librdkafka/blob/292d2a66b992/src/rdkafka.h#L8435
77
+ RD_KAFKA_ACL_PERMISSION_TYPE_ANY = 1
78
+ RD_KAFKA_ACL_PERMISSION_TYPE_DENY = 2
79
+ RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW = 3
80
+
81
+ # Extracting error details from Acl results
82
+ attach_function :rd_kafka_acl_result_error, [:pointer], :pointer
83
+ attach_function :rd_kafka_AclBinding_error, [:pointer], :pointer
84
+ end
85
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rdkafka
4
+ module Bindings
5
+ attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
6
+ attach_function :rd_kafka_ConfigResource_new, [:int32, :pointer], :pointer
7
+ attach_function :rd_kafka_ConfigResource_destroy, [:pointer], :void
8
+ attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :size_t], :void
9
+ attach_function :rd_kafka_event_DescribeConfigs_result, [:pointer], :pointer
10
+ attach_function :rd_kafka_DescribeConfigs_result_resources, [:pointer, :pointer], :pointer
11
+ attach_function :rd_kafka_ConfigResource_configs, [:pointer, :pointer], :pointer
12
+ attach_function :rd_kafka_ConfigEntry_name, [:pointer], :string
13
+ attach_function :rd_kafka_ConfigEntry_value, [:pointer], :string
14
+ attach_function :rd_kafka_ConfigEntry_is_read_only, [:pointer], :int
15
+ attach_function :rd_kafka_ConfigEntry_is_default, [:pointer], :int
16
+ attach_function :rd_kafka_ConfigEntry_is_sensitive, [:pointer], :int
17
+ attach_function :rd_kafka_ConfigEntry_is_synonym, [:pointer], :int
18
+ attach_function :rd_kafka_ConfigEntry_synonyms, [:pointer, :pointer], :pointer
19
+ attach_function :rd_kafka_ConfigResource_error, [:pointer], :int
20
+ attach_function :rd_kafka_ConfigResource_error_string, [:pointer], :string
21
+ attach_function :rd_kafka_IncrementalAlterConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
22
+ attach_function :rd_kafka_IncrementalAlterConfigs_result_resources, [:pointer, :pointer], :pointer
23
+ attach_function :rd_kafka_ConfigResource_add_incremental_config, [:pointer, :string, :int32, :string], :pointer
24
+ attach_function :rd_kafka_event_IncrementalAlterConfigs_result, [:pointer], :pointer
25
+
26
+ RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS = 5
27
+ RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT = 104
28
+
29
+ RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS = 16
30
+ RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT = 131072
31
+
32
+ RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET = 0
33
+ RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE = 1
34
+ RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND = 2
35
+ RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
36
+ end
37
+ end