karafka-rdkafka 0.30.2-aarch64-linux-gnu → 0.32.0-aarch64-linux-gnu

data/lib/rdkafka/admin/describe_consumer_groups_report.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # Report for describe consumer groups operation result
+    class DescribeConsumerGroupsReport
+      include DescribeResultParsing
+
+      # Described consumer groups, one per requested group id. Each entry is a hash with:
+      #   - `:group_id` [String] the consumer group id
+      #   - `:error` [RdkafkaError, nil] error describing this group, or `nil` on success. The
+      #     other fields are not meaningful when it is set.
+      #   - `:is_simple_consumer_group` [Boolean] `true` for a group that assigns partitions
+      #     manually and uses Kafka only for offset storage
+      #   - `:state` [Integer] the group state as a `Bindings::RD_KAFKA_CONSUMER_GROUP_STATE_*`
+      #     code. A group id the cluster does not know is reported as
+      #     `RD_KAFKA_CONSUMER_GROUP_STATE_DEAD` with no members.
+      #   - `:state_name` [String] human-readable name of that state (e.g. `"Stable"`, `"Empty"`)
+      #   - `:type` [Integer] the group protocol type as a
+      #     `Bindings::RD_KAFKA_CONSUMER_GROUP_TYPE_*` code, one of:
+      #     `RD_KAFKA_CONSUMER_GROUP_TYPE_CLASSIC`,
+      #     `RD_KAFKA_CONSUMER_GROUP_TYPE_CONSUMER` (KIP-848),
+      #     `RD_KAFKA_CONSUMER_GROUP_TYPE_UNKNOWN`
+      #   - `:type_name` [String] human-readable name of that type (e.g. `"Classic"`, `"Consumer"`)
+      #   - `:partition_assignor` [String, nil] partition assignor in use (e.g. `"range"`)
+      #   - `:coordinator` [Hash, nil] the group coordinator broker as
+      #     `{ id:, host:, port:, rack: }`
+      #   - `:authorized_operations` [Array<Integer>, nil] ACL operations the client may perform on
+      #     the group, as `Bindings::RD_KAFKA_ACL_OPERATION_*` codes. `nil` unless requested with
+      #     `include_authorized_operations: true`.
+      #   - `:members` [Array<Hash>] current group members, each with:
+      #     - `:member_id` [String] the member (consumer) id
+      #     - `:client_id` [String] the member's `client.id`
+      #     - `:group_instance_id` [String, nil] the static membership instance id, if any
+      #     - `:host` [String] the host the member connects from
+      #     - `:assignment` [Hash{String => Array<Integer>}] assigned partitions, by topic
+      #     - `:target_assignment` [Hash{String => Array<Integer>}, nil] partitions the member is
+      #       moving to during a rebalance, by topic. Only `consumer` (KIP-848) groups have it;
+      #       `nil` for `classic` groups.
+      # @return [Array<Hash>]
+      attr_reader :groups
+
+      # @param result_ptr [FFI::Pointer] pointer to the `rd_kafka_DescribeConsumerGroups_result_t`
+      def initialize(result_ptr)
+        @groups = []
+
+        return if result_ptr.null?
+
+        count_ptr = FFI::MemoryPointer.new(:size_t)
+        array_ptr = Bindings.rd_kafka_DescribeConsumerGroups_result_groups(result_ptr, count_ptr)
+
+        return if array_ptr.null?
+
+        array_ptr.read_array_of_pointer(count_ptr.read(:size_t)).each do |group_ptr|
+          @groups << build_group(group_ptr)
+        end
+      end
+
+      private
+
+      # @param group_ptr [FFI::Pointer] pointer to the `rd_kafka_ConsumerGroupDescription_t`
+      # @return [Hash]
+      def build_group(group_ptr)
+        state = Bindings.rd_kafka_ConsumerGroupDescription_state(group_ptr)
+        type = Bindings.rd_kafka_ConsumerGroupDescription_type(group_ptr)
+
+        {
+          group_id: read_string(Bindings.rd_kafka_ConsumerGroupDescription_group_id(group_ptr)),
+          error: build_error(Bindings.rd_kafka_ConsumerGroupDescription_error(group_ptr)),
+          is_simple_consumer_group:
+            Bindings.rd_kafka_ConsumerGroupDescription_is_simple_consumer_group(group_ptr) != 0,
+          state: state,
+          state_name: read_string(Bindings.rd_kafka_consumer_group_state_name(state)),
+          type: type,
+          type_name: read_string(Bindings.rd_kafka_consumer_group_type_name(type)),
+          partition_assignor:
+            read_string(Bindings.rd_kafka_ConsumerGroupDescription_partition_assignor(group_ptr)),
+          coordinator: extract_node(Bindings.rd_kafka_ConsumerGroupDescription_coordinator(group_ptr)),
+          authorized_operations: build_authorized_operations(group_ptr),
+          members: Array.new(Bindings.rd_kafka_ConsumerGroupDescription_member_count(group_ptr)) do |index|
+            build_member(Bindings.rd_kafka_ConsumerGroupDescription_member(group_ptr, index))
+          end
+        }
+      end
+
+      # @param member_ptr [FFI::Pointer] pointer to the `rd_kafka_MemberDescription_t`
+      # @return [Hash]
+      def build_member(member_ptr)
+        {
+          member_id: read_string(Bindings.rd_kafka_MemberDescription_consumer_id(member_ptr)),
+          client_id: read_string(Bindings.rd_kafka_MemberDescription_client_id(member_ptr)),
+          group_instance_id: read_string(Bindings.rd_kafka_MemberDescription_group_instance_id(member_ptr)),
+          host: read_string(Bindings.rd_kafka_MemberDescription_host(member_ptr)),
+          assignment: build_assignment(Bindings.rd_kafka_MemberDescription_assignment(member_ptr)),
+          target_assignment: build_target_assignment(member_ptr)
+        }
+      end
+
+      # @param assignment_ptr [FFI::Pointer] pointer to the `rd_kafka_MemberAssignment_t`
+      # @return [Hash{String => Array<Integer>}]
+      def build_assignment(assignment_ptr)
+        return {} if assignment_ptr.null?
+
+        tpl_ptr = Bindings.rd_kafka_MemberAssignment_partitions(assignment_ptr)
+
+        return {} if tpl_ptr.null?
+
+        Consumer::TopicPartitionList
+          .from_native_tpl(tpl_ptr)
+          .to_h
+          .transform_values { |partitions| (partitions || []).map(&:partition) }
+      end
+
+      # @param member_ptr [FFI::Pointer] pointer to the `rd_kafka_MemberDescription_t`
+      # @return [Hash{String => Array<Integer>}, nil]
+      def build_target_assignment(member_ptr)
+        assignment_ptr = Bindings.rd_kafka_MemberDescription_target_assignment(member_ptr)
+
+        assignment_ptr.null? ? nil : build_assignment(assignment_ptr)
+      end
+
+      # @param group_ptr [FFI::Pointer] pointer to the `rd_kafka_ConsumerGroupDescription_t`
+      # @return [Array<Integer>, nil]
+      def build_authorized_operations(group_ptr)
+        count_ptr = FFI::MemoryPointer.new(:size_t)
+
+        extract_authorized_operations(
+          Bindings.rd_kafka_ConsumerGroupDescription_authorized_operations(group_ptr, count_ptr),
+          count_ptr
+        )
+      end
+
+      # @param error_ptr [FFI::Pointer] pointer to the `rd_kafka_error_t`
+      # @return [RdkafkaError, nil]
+      def build_error(error_ptr)
+        return nil if error_ptr.null?
+
+        RdkafkaError.new(
+          Bindings.rd_kafka_error_code(error_ptr),
+          broker_message: read_string(Bindings.rd_kafka_error_string(error_ptr))
+        )
+      end
+
+      # @param ptr [FFI::Pointer] pointer to a C string
+      # @return [String, nil]
+      def read_string(ptr)
+        ptr.null? ? nil : ptr.read_string
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/describe_result_parsing.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # Helpers shared by the describe-topics and describe-cluster reports to copy nodes and
+    # authorized operations out of event-owned memory.
+    # @private
+    module DescribeResultParsing
+      private
+
+      # @param node_ptr [FFI::Pointer] pointer to a `rd_kafka_Node_t`
+      # @return [Hash, nil] node with `:id`, `:host`, `:port` and `:rack` or nil when NULL
+      def extract_node(node_ptr)
+        return nil if node_ptr.null?
+
+        host_ptr = Bindings.rd_kafka_Node_host(node_ptr)
+        rack_ptr = Bindings.rd_kafka_Node_rack(node_ptr)
+
+        {
+          id: Bindings.rd_kafka_Node_id(node_ptr),
+          host: host_ptr.null? ? nil : host_ptr.read_string,
+          port: Bindings.rd_kafka_Node_port(node_ptr),
+          rack: rack_ptr.null? ? nil : rack_ptr.read_string
+        }
+      end
+
+      # @param array_ptr [FFI::Pointer] pointer to a `rd_kafka_Node_t*` array
+      # @param count_ptr [FFI::MemoryPointer] size_t holding the array length
+      # @return [Array<Hash>] nodes as returned by {#extract_node}
+      def extract_nodes(array_ptr, count_ptr)
+        return [] if array_ptr.null?
+
+        array_ptr.read_array_of_pointer(count_ptr.read(:size_t)).map { |node_ptr| extract_node(node_ptr) }
+      end
+
+      # @param array_ptr [FFI::Pointer] pointer to a `rd_kafka_AclOperation_t` array
+      # @param count_ptr [FFI::MemoryPointer] size_t holding the array length
+      # @return [Array<Integer>, nil] `Bindings::RD_KAFKA_ACL_OPERATION_*` codes or nil when the
+      #   authorized operations were not requested
+      def extract_authorized_operations(array_ptr, count_ptr)
+        return nil if array_ptr.null?
+
+        array_ptr.read_array_of_int(count_ptr.read(:size_t))
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/describe_topics_handle.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # Handle for describe topics operation
+    class DescribeTopicsHandle < AbstractHandle
+      layout :pending, :bool,
+        :response, :int
+
+      # @return [String] the name of the operation
+      def operation_name
+        "describe topics"
+      end
+
+      # @return [DescribeTopicsReport] report prepared by the background event callback, with
+      #   the described topics.
+      def create_result
+        prepared_result
+      end
+
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
+      def raise_error
+        raise RdkafkaError.new(
+          self[:response],
+          broker_message: broker_message
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/describe_topics_report.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # Report for describe topics operation result
+    class DescribeTopicsReport
+      include DescribeResultParsing
+
+      # Described topics, one per requested topic name. Each entry is a hash with:
+      #   - `:name` [String] the topic name
+      #   - `:topic_id` [String, nil] the topic id (KIP-516), base64 encoded, or `nil` when the
+      #     topic could not be described
+      #   - `:is_internal` [Boolean] whether the topic is internal (e.g. `__consumer_offsets`)
+      #   - `:partitions` [Array<Hash>] each with `:partition` [Integer], `:leader` [Hash, nil] and
+      #     `:replicas` / `:isr` [Array<Hash>]; nodes have `:id`, `:host`, `:port` and `:rack`
+      #   - `:authorized_operations` [Array<Integer>, nil] `Bindings::RD_KAFKA_ACL_OPERATION_*`
+      #     codes, `nil` unless requested with `include_authorized_operations: true`
+      #   - `:error` [RdkafkaError, nil] per-topic error (e.g. `unknown_topic_or_part`), `nil`
+      #     on success
+      # @return [Array<Hash>]
+      attr_reader :topics
+
+      # @param result_ptr [FFI::Pointer] pointer to the `rd_kafka_DescribeTopics_result_t`
+      def initialize(result_ptr)
+        @topics = []
+
+        return if result_ptr.null?
+
+        count_ptr = FFI::MemoryPointer.new(:size_t)
+        array_ptr = Bindings.rd_kafka_DescribeTopics_result_topics(result_ptr, count_ptr)
+
+        return if array_ptr.null?
+
+        array_ptr.read_array_of_pointer(count_ptr.read(:size_t)).each do |topic_ptr|
+          @topics << extract_topic(topic_ptr)
+        end
+      end
+
+      private
+
+      # @param topic_ptr [FFI::Pointer] pointer to a `rd_kafka_TopicDescription_t`
+      # @return [Hash]
+      def extract_topic(topic_ptr)
+        name_ptr = Bindings.rd_kafka_TopicDescription_name(topic_ptr)
+        operations_count_ptr = FFI::MemoryPointer.new(:size_t)
+
+        {
+          name: name_ptr.null? ? nil : name_ptr.read_string,
+          topic_id: extract_topic_id(topic_ptr),
+          is_internal: Bindings.rd_kafka_TopicDescription_is_internal(topic_ptr) != 0,
+          partitions: extract_partitions(topic_ptr),
+          authorized_operations: extract_authorized_operations(
+            Bindings.rd_kafka_TopicDescription_authorized_operations(topic_ptr, operations_count_ptr),
+            operations_count_ptr
+          ),
+          error: extract_error(topic_ptr)
+        }
+      end
+
+      # librdkafka reports the zero UUID for a topic it could not describe
+      # @param topic_ptr [FFI::Pointer] pointer to a `rd_kafka_TopicDescription_t`
+      # @return [String, nil]
+      def extract_topic_id(topic_ptr)
+        uuid_ptr = Bindings.rd_kafka_TopicDescription_topic_id(topic_ptr)
+
+        return nil if uuid_ptr.null?
+        return nil if Bindings.rd_kafka_Uuid_most_significant_bits(uuid_ptr).zero? &&
+          Bindings.rd_kafka_Uuid_least_significant_bits(uuid_ptr).zero?
+
+        string_ptr = Bindings.rd_kafka_Uuid_base64str(uuid_ptr)
+        string_ptr.null? ? nil : string_ptr.read_string
+      end
+
+      # @param topic_ptr [FFI::Pointer] pointer to a `rd_kafka_TopicDescription_t`
+      # @return [Array<Hash>]
+      def extract_partitions(topic_ptr)
+        count_ptr = FFI::MemoryPointer.new(:size_t)
+        array_ptr = Bindings.rd_kafka_TopicDescription_partitions(topic_ptr, count_ptr)
+
+        return [] if array_ptr.null?
+
+        array_ptr.read_array_of_pointer(count_ptr.read(:size_t)).map do |partition_ptr|
+          replicas_count_ptr = FFI::MemoryPointer.new(:size_t)
+          isr_count_ptr = FFI::MemoryPointer.new(:size_t)
+
+          {
+            partition: Bindings.rd_kafka_TopicPartitionInfo_partition(partition_ptr),
+            leader: extract_node(Bindings.rd_kafka_TopicPartitionInfo_leader(partition_ptr)),
+            replicas: extract_nodes(
+              Bindings.rd_kafka_TopicPartitionInfo_replicas(partition_ptr, replicas_count_ptr),
+              replicas_count_ptr
+            ),
+            isr: extract_nodes(
+              Bindings.rd_kafka_TopicPartitionInfo_isr(partition_ptr, isr_count_ptr),
+              isr_count_ptr
+            )
+          }
+        end
+      end
+
+      # The error is owned by the result, so it is copied and not destroyed here.
+      # @param topic_ptr [FFI::Pointer] pointer to a `rd_kafka_TopicDescription_t`
+      # @return [RdkafkaError, nil]
+      def extract_error(topic_ptr)
+        error_ptr = Bindings.rd_kafka_TopicDescription_error(topic_ptr)
+
+        return nil if error_ptr.null?
+
+        code = Bindings.rd_kafka_error_code(error_ptr)
+
+        return nil if code == Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+        string_ptr = Bindings.rd_kafka_error_string(error_ptr)
+        RdkafkaError.new(code, broker_message: string_ptr.null? ? nil : string_ptr.read_string)
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/list_consumer_group_offsets_handle.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # Handle for list consumer group offsets operation
+    class ListConsumerGroupOffsetsHandle < AbstractHandle
+      layout :pending, :bool,
+        :response, :int
+
+      # @return [String] the name of the operation
+      def operation_name
+        "list consumer group offsets"
+      end
+
+      # Creates the result report
+      # @return [ListConsumerGroupOffsetsReport]
+      def create_result
+        prepared_result
+      end
+
+      # Raises an error if the operation failed
+      # @raise [RdkafkaError]
+      def raise_error
+        raise RdkafkaError.new(
+          self[:response],
+          broker_message: broker_message
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/list_consumer_group_offsets_report.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # Report for list consumer group offsets operation result
+    class ListConsumerGroupOffsetsReport
+      # @return [String, nil] the group the operation ran against
+      attr_reader :group_name
+
+      # @return [String, nil] group level error message, if any
+      attr_reader :error_string
+
+      # Committed offsets per partition, each with `:topic`, `:partition`, `:offset` (nil when
+      # the group has no committed offset for it or the partition has an `:error`), `:metadata`
+      # (nil when none was committed) and `:error` (an `RdkafkaError` or nil).
+      # @return [Array<Hash>]
+      attr_reader :partitions
+
+      # @param error_string [FFI::Pointer] pointer to the group level error string
+      # @param result_name [FFI::Pointer] pointer to the group name
+      # @param partitions_ptr [FFI::Pointer] pointer to the per partition result list
+      def initialize(error_string, result_name, partitions_ptr = FFI::Pointer::NULL)
+        @error_string = error_string.null? ? nil : error_string.read_string
+        @group_name = result_name.null? ? nil : result_name.read_string
+        @partitions = extract_partitions(partitions_ptr)
+      end
+
+      private
+
+      # @param partitions_ptr [FFI::Pointer] pointer to a native topic partition list
+      # @return [Array<Hash>]
+      def extract_partitions(partitions_ptr)
+        return [] if partitions_ptr.nil? || partitions_ptr.null?
+
+        native = Bindings::TopicPartitionList.new(partitions_ptr)
+
+        Array.new(native[:cnt]) do |i|
+          tp = Bindings::TopicPartition.new(native[:elems] + (i * Bindings::TopicPartition.size))
+          offset = tp[:offset]
+
+          {
+            topic: tp[:topic],
+            partition: tp[:partition],
+            offset: (offset == Bindings::RD_KAFKA_OFFSET_INVALID) ? nil : offset,
+            metadata: tp[:metadata].null? ? nil : tp[:metadata].read_string(tp[:metadata_size]),
+            error: tp[:err].zero? ? nil : RdkafkaError.new(tp[:err])
+          }
+        end
+      end
+    end
+  end
+end