karafka-rdkafka 0.30.2-aarch64-linux-gnu → 0.32.0-aarch64-linux-gnu

data/lib/rdkafka/bindings/polling.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    # Polling
+
+    attach_function :rd_kafka_flush, [:pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_poll, [:pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
+
+    # Non-blocking poll variants (do not release GVL)
+    # These are more efficient for poll(0) calls in fiber schedulers where GVL
+    # release/reacquire overhead is wasteful since we don't expect to wait.
+    # Uses the same underlying C function but with blocking: false to skip GVL release.
+    attach_function :rd_kafka_poll_nb, :rd_kafka_poll, [:pointer, :int], :int, blocking: false
+  end
+end

data/lib/rdkafka/bindings/producer.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    # Producer
+    RD_KAFKA_VTYPE_END = 0
+    RD_KAFKA_VTYPE_TOPIC = 1
+    RD_KAFKA_VTYPE_RKT = 2
+    RD_KAFKA_VTYPE_PARTITION = 3
+    RD_KAFKA_VTYPE_VALUE = 4
+    RD_KAFKA_VTYPE_KEY = 5
+    RD_KAFKA_VTYPE_OPAQUE = 6
+    RD_KAFKA_VTYPE_MSGFLAGS = 7
+    RD_KAFKA_VTYPE_TIMESTAMP = 8
+    RD_KAFKA_VTYPE_HEADER = 9
+    RD_KAFKA_VTYPE_HEADERS = 10
+    RD_KAFKA_PURGE_F_QUEUE = 1
+    RD_KAFKA_PURGE_F_INFLIGHT = 2
+
+    RD_KAFKA_MSG_F_COPY = 0x2
+
+    attach_function :rd_kafka_producev, [:pointer, :varargs], :int, blocking: true
+    attach_function :rd_kafka_purge, [:pointer, :int], :int, blocking: true
+    callback :delivery_cb, [:pointer, :pointer, :pointer], :void
+    attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
+    attach_function :rd_kafka_init_transactions, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_send_offsets_to_transaction, [:pointer, :pointer, :pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_begin_transaction, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_abort_transaction, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_commit_transaction, [:pointer, :int], :pointer, blocking: true
+
+    # Hash mapping partitioner names to their FFI function symbols
+    # @return [Hash{String => Symbol}]
+    PARTITIONERS = %w[random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random].each_with_object({}) do |name, hsh|
+      method_name = :"rd_kafka_msg_partitioner_#{name}"
+      attach_function method_name, [:pointer, :pointer, :size_t, :int32, :pointer, :pointer], :int32
+      hsh[name] = method_name
+    end
+
+    # Calculates the partition for a message based on the partitioner
+    #
+    # @param topic_ptr [FFI::Pointer] pointer to the topic handle
+    # @param str [String] the partition key string
+    # @param partition_count [Integer, nil] number of partitions
+    # @param partitioner [String] name of the partitioner to use
+    # @return [Integer] partition number or RD_KAFKA_PARTITION_UA if unassigned
+    # @raise [Rdkafka::Config::ConfigError] when an unknown partitioner is specified
+    def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
+      # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
+      return RD_KAFKA_PARTITION_UA unless partition_count&.nonzero?
+
+      str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
+      method_name = PARTITIONERS.fetch(partitioner) do
+        raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner}")
+      end
+
+      public_send(method_name, topic_ptr, str_ptr, partition_key_length(str), partition_count, nil, nil)
+    end
+
+    # Partition key length as the character count (legacy default).
+    #
+    # @param str [String] the partition key string
+    # @return [Integer]
+    def self.partition_key_size(str)
+      str.size
+    end
+
+    # Partition key length as the byte count, which is what librdkafka hashes (UTF-8 bytes copied
+    # into the key pointer) and what other Kafka clients use.
+    #
+    # @param str [String] the partition key string
+    # @return [Integer]
+    def self.partition_key_bytesize(str)
+      str.bytesize
+    end
+
+    # Length of the partition key passed to librdkafka. Aliased to one of the methods above by
+    # `Rdkafka::Config.partitioner_key_uses_bytesize=`, so the per-message path does not check the
+    # setting on every call.
+    singleton_class.alias_method :partition_key_length, :partition_key_size
+  end
+end

data/lib/rdkafka/bindings/share_consumer.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    # Share consumer (KIP-932, preview)
+    #
+    # The share consumer is a distinct librdkafka handle type (rd_kafka_share_t) created with
+    # rd_kafka_share_consumer_new instead of rd_kafka_new. It is single-threaded by design and
+    # librdkafka itself rejects concurrent access with RD_KAFKA_RESP_ERR__CONFLICT.
+
+    # Minimal mirror of the head of librdkafka's `struct rd_kafka_share_s`, whose first member is
+    # the wrapped `rd_kafka_t` (`rkshare_rk`). `rd_kafka_share_t` has no public accessors of its
+    # own - not even for the client name - but the handle it wraps is a normal `rd_kafka_t` and is
+    # the very handle librdkafka passes to the statistics, error and OAuthBearer callbacks. Reading
+    # it lets a share consumer report a name that matches its statistics `name` field, which
+    # `rd_kafka_name` cannot do when handed the `rd_kafka_share_t` directly (it would read an
+    # unrelated field and return garbage). This mirrors how the gem already maps other librdkafka
+    # structs by layout for the pinned librdkafka version.
+    class NativeShareConsumer < FFI::Struct
+      layout :rkshare_rk, :pointer
+    end
+
+    # Acknowledge types for share consumer records
+    RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_ACCEPT = 1
+    RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_RELEASE = 2
+    RD_KAFKA_SHARE_ACKNOWLEDGE_TYPE_REJECT = 3
+
+    attach_function :rd_kafka_share_consumer_new, [:pointer, :pointer, :size_t], :pointer
+    attach_function :rd_kafka_share_consumer_close, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_share_consumer_closed, [:pointer], :int
+    attach_function :rd_kafka_share_destroy, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_share_set_log_queue, [:pointer, :pointer], :pointer
+
+    attach_function :rd_kafka_share_subscribe, [:pointer, :pointer], :int, blocking: true
+    attach_function :rd_kafka_share_unsubscribe, [:pointer], :int, blocking: true
+    attach_function :rd_kafka_share_subscription, [:pointer, :pointer], :int, blocking: true
+
+    attach_function :rd_kafka_share_poll, [:pointer, :int, :pointer], :pointer, blocking: true
+    attach_function :rd_kafka_messages_count, [:pointer], :size_t
+    attach_function :rd_kafka_messages_get, [:pointer, :size_t], :pointer
+    attach_function :rd_kafka_messages_destroy, [:pointer], :void
+    attach_function :rd_kafka_message_delivery_count, [:pointer], :int16
+
+    # The by-coordinates acknowledge variant is the only one we bind: Ruby messages are copies
+    # that do not retain the native rd_kafka_message_t pointer, so the message-pointer variants
+    # (rd_kafka_share_acknowledge / rd_kafka_share_acknowledge_type) cannot be used safely here.
+    attach_function :rd_kafka_share_acknowledge_offset, [:pointer, :string, :int32, :int64, :int], :int
+
+    attach_function :rd_kafka_share_commit_sync, [:pointer, :int, :pointer], :pointer, blocking: true
+    attach_function :rd_kafka_share_commit_async, [:pointer], :pointer
+
+    callback :share_acknowledgement_commit_cb, [:pointer, :pointer, :int, :pointer], :void
+    attach_function :rd_kafka_share_set_acknowledgement_commit_cb, [:pointer, :share_acknowledgement_commit_cb, :pointer], :pointer
+
+    # Accessors for the rd_kafka_share_partition_offsets_list_t handed to the acknowledgement
+    # commit callback. The list is owned by librdkafka for the duration of the callback: read it
+    # inside the callback only, never destroy or retain it.
+    attach_function :rd_kafka_share_partition_offsets_list_count, [:pointer], :size_t
+    attach_function :rd_kafka_share_partition_offsets_list_get, [:pointer, :size_t], :pointer
+    attach_function :rd_kafka_share_partition_offsets_partition, [:pointer], :pointer
+    attach_function :rd_kafka_share_partition_offsets_offsets, [:pointer], :pointer
+    attach_function :rd_kafka_share_partition_offsets_offsets_cnt, [:pointer], :size_t
+
+    attach_function :rd_kafka_share_oauthbearer_set_token, [:pointer, :string, :int64, :string, :pointer, :size_t, :pointer, :size_t], :int
+    attach_function :rd_kafka_share_oauthbearer_set_token_failure, [:pointer, :string], :int
+  end
+end

data/lib/rdkafka/bindings/structs.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  module Bindings
+    # FFI struct representing a Kafka message (rd_kafka_message_t)
+    class Message < FFI::Struct
+      layout :err, :int,
+        :rkt, :pointer,
+        :partition, :int32,
+        :payload, :pointer,
+        :len, :size_t,
+        :key, :pointer,
+        :key_len, :size_t,
+        :offset, :int64,
+        :_private, :pointer
+    end
+
+    # FFI struct representing a topic partition (rd_kafka_topic_partition_t)
+    class TopicPartition < FFI::Struct
+      layout :topic, :string,
+        :partition, :int32,
+        :offset, :int64,
+        :metadata, :pointer,
+        :metadata_size, :size_t,
+        :opaque, :pointer,
+        :err, :int,
+        :_private, :pointer
+    end
+
+    # FFI struct representing a topic partition list (rd_kafka_topic_partition_list_t)
+    class TopicPartitionList < FFI::Struct
+      layout :cnt, :int,
+        :size, :int,
+        :elems, :pointer
+    end
+
+    # FFI struct representing a config resource (rd_kafka_ConfigResource_t)
+    # Structs for management of configurations. Each configuration is attached to a resource
+    # and one resource can have many configuration details. Each resource will also have
+    # separate errors results if obtaining configuration was not possible for any reason
+    class ConfigResource < FFI::Struct
+      layout :type, :int,
+        :name, :string
+    end
+
+    # FFI struct for error description (rd_kafka_err_desc)
+    class NativeErrorDesc < FFI::Struct
+      layout :code, :int,
+        :name, :pointer,
+        :desc, :pointer
+    end
+
+    # FFI struct for native error (rd_kafka_error_t)
+    class NativeError < FFI::Struct
+      layout :code, :int32,
+        :errstr, :pointer,
+        :fatal, :u_int8_t,
+        :retriable, :u_int8_t,
+        :txn_requires_abort, :u_int8_t
+    end
+  end
+end