kafka 0.5.0

data/lib/kafka/ffi/group_member_info.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "ffi"
+
+module Kafka::FFI
+  class GroupMemberInfo < ::FFI::Struct
+    layout(
+      :member_id,              :string,
+      :client_id,              :string,
+      :client_host,            :string,
+      :member_metadata,        :pointer,
+      :member_metadata_size,   :int,
+      :member_assignment,      :pointer,
+      :member_assignment_size, :int
+    )
+
+    # Returns the broker generated member id for the consumer.
+    #
+    # @return [String] Member ID
+    def member_id
+      self[:member_id]
+    end
+
+    # Returns the consumer's client.id config setting
+    #
+    # @return [String] Client ID
+    def client_id
+      self[:client_id]
+    end
+
+    # Returns the hostname of the consumer
+    #
+    # @return [String] Consumer's hostname
+    def client_host
+      self[:client_host]
+    end
+
+    # Returns the binary metadata for the consumer
+    #
+    # @return [String] Consumer metadata
+    def member_metadata
+      self[:member_metadata].read_string(self[:member_metadata_size])
+    end
+
+    # Returns the binary assignment data for the consumer
+    #
+    # @return [String] Assignments
+    def member_assignment
+      self[:member_assignment].read_string(self[:member_assignment_size])
+    end
+  end
+end

data/lib/kafka/ffi/message/header.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+require "ffi"
+require "kafka/ffi/opaque_pointer"
+
+module Kafka::FFI
+  class Message::Header < OpaquePointer
+    def self.new(count = 0)
+      if count.is_a?(::FFI::Pointer)
+        return super(count)
+      end
+
+      ::Kafka::FFI.rd_kafka_headers_new(count)
+    end
+
+    # Count returns the number of headers in the set.
+    #
+    # @return [Integer] Number of headers
+    def count
+      ::Kafka::FFI.rd_kafka_header_cnt(self)
+    end
+    alias size count
+    alias length count
+
+    # Add header with name and value.
+    #
+    # @param name [String] Header key name
+    # @param value [#to_s, nil] Header key value
+    #
+    # @raise [Kafka::ResponseError] Error that occurred adding the header
+    def add(name, value)
+      name = name.to_s
+
+      value_size = 0
+      if value
+        value = value.to_s
+        value_size = value.bytesize
+      end
+
+      err = ::Kafka::FFI.rd_kafka_header_add(self, name, name.length, value, value_size)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      nil
+    end
+
+    # Make a copy of the headers list
+    #
+    # @return [Header] Copy of the headers
+    def copy
+      ::Kafka::FFI.rd_kafka_headers_copy(self)
+    end
+
+    # Remove all headers with the given name.
+    #
+    # @param name [String] Header key name to remove
+    #
+    # @raise [Kafka::ResponseError] Error that occurred removing the header
+    # @raise [Kafka::ResponseError<RD_KAFKA_RESP_ERR__READ_ONLY>] Header is
+    #   read only.
+    def remove(name)
+      name = name.to_s
+
+      err = ::Kafka::FFI.rd_kafka_header_remove(self, name)
+      case err
+      when :ok
+        nil
+      when ::Kafka::FFI::RD_KAFKA_RESP_ERR__NOENT
+        # Header field does not exist. Just return nil since the effect (key
+        # doesn't exist) is the same.
+        nil
+      else
+        raise ::Kafka::ResponseError, err
+      end
+    end
+
+    # Retrieve all headers that match the given name
+    #
+    # @param name [String] Header key name
+    #
+    # @raise [Kafka::ResponseError] Error that occurred retrieving the header
+    #   values
+    #
+    # @return [Array<String, nil>] List of values for the header
+    def get(name)
+      name = name.to_s
+
+      idx = 0
+      values = []
+
+      value = ::FFI::MemoryPointer.new(:pointer)
+      size  = ::FFI::MemoryPointer.new(:pointer)
+
+      loop do
+        err = ::Kafka::FFI.rd_kafka_header_get(self, idx, name, value, size)
+
+        case err
+        when :ok
+          # Read the returned value and add it to the result list.
+          idx += 1
+          ptr = value.read_pointer
+
+          values << (ptr.null? ? nil : ptr.read_string(size.read(:size_t)))
+        when ::Kafka::FFI::RD_KAFKA_RESP_ERR__NOENT
+          # Reached the end of the list of values so break and return the set
+          # of found values.
+          break
+        else
+          raise ::Kafka::ResponseError, err
+        end
+      end
+
+      values
+    ensure
+      value.free if value
+      size.free if size
+    end
+
+    # rubocop:disable Naming/AccessorMethodName
+
+    # Retrieve all of the headers and their values
+    #
+    # @raise [Kafka::ResponseError] Error if occurred retrieving the headers.
+    #
+    # @return [Hash<String, Array<String>>] Set of header keys and their values
+    # @return [Hash{}] Header is empty
+    def get_all
+      name  = ::FFI::MemoryPointer.new(:pointer)
+      value = ::FFI::MemoryPointer.new(:pointer)
+      size  = ::FFI::MemoryPointer.new(:pointer)
+
+      idx = 0
+      result = {}
+
+      loop do
+        err = ::Kafka::FFI.rd_kafka_header_get_all(self, idx, name, value, size)
+
+        case err
+        when :ok
+          # Read the returned value and add it to the result list.
+          idx += 1
+
+          key = name.read_pointer.read_string
+          val = value.read_pointer
+
+          result[key] ||= []
+          result[key] << (val.null? ? nil : val.read_string(size.read(:size_t)))
+        when ::Kafka::FFI::RD_KAFKA_RESP_ERR__NOENT
+          # Reached the end of the list of values so break and return the set
+          # of found values.
+          break
+        else
+          raise ::Kafka::ResponseError, err
+        end
+      end
+
+      result
+    ensure
+      name.free if name
+      value.free if value
+      size.free if size
+    end
+    alias to_hash get_all
+    alias to_h get_all
+    # rubocop:enable Naming/AccessorMethodName
+
+    # Find the last header in the list that matches the given name.
+    #
+    # @param name [String] Header key name
+    #
+    # @raise [Kafka::ResponseError] Error that occurred retrieving the header
+    #   value
+    #
+    # @return [String] Value of the last matching header with name
+    # @return [nil] No header with that name exists
+    def get_last(name)
+      name  = name.to_s
+      value = ::FFI::MemoryPointer.new(:pointer)
+      size  = ::FFI::MemoryPointer.new(:pointer)
+
+      err = ::Kafka::FFI.rd_kafka_header_get_last(self, name, value, size)
+      if err != :ok
+        # No header with that name exists so just return nil
+        if err == ::Kafka::FFI::RD_KAFKA_RESP_ERR__NOENT
+          return nil
+        end
+
+        raise ::Kafka::ResponseError, err
+      end
+
+      ptr = value.read_pointer
+      ptr.null? ? nil : ptr.read_string(size.read(:size_t))
+    ensure
+      value.free
+      size.free
+    end
+
+    def destroy
+      if !pointer.null?
+        ::Kafka::FFI.rd_kafka_headers_destroy(self)
+      end
+    end
+  end
+end

data/lib/kafka/ffi/message.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+module Kafka::FFI
+  class Message < ::FFI::Struct
+    require "kafka/ffi/message/header"
+
+    layout(
+      :err,       :error_code,
+      :rkt,       Topic,
+      :partition, :int32,
+      :payload,   :pointer,
+      :len,       :size_t,
+      :key,       :pointer,
+      :key_len,   :size_t,
+      :offset,    :int64,
+      :private,   Opaque
+    )
+
+    # Retrieve the error associated with this message. For consumers this is
+    # used to report per-topic+partition consumer errors. For producers this is
+    # set when received in the dr_msg_cb callback to signify a fatal error
+    # publishing the message.
+    #
+    # @return [nil] Message does not have an error
+    # @return [Kafka::ResponseError] RD_KAFKA_RESP_ERR__* error code
+    def error
+      if self[:err] != :ok
+        ::Kafka::ResponseError.new(self[:err])
+      end
+    end
+
+    # Returns the name of the Topic the Message was published to.
+    #
+    # @return [nil] Topic information was not available
+    # @return [String] Name of the Topic the message was published to.
+    def topic
+      if self[:rkt].nil?
+        return nil
+      end
+
+      self[:rkt].name
+    end
+
+    # Returns the optional message key used to publish the message. This key is
+    # used for partition assignment based on the `partitioner` or
+    # `partitioner_cb` config options.
+    #
+    # @return [nil] No partitioning key was provided
+    # @return [String] The partitioning key
+    def key
+      if self[:key].null?
+        return nil
+      end
+
+      self[:key].read_string(self[:key_len])
+    end
+
+    # Returns the partition the message was published to.
+    #
+    # @return [Integer] Partition
+    def partition
+      self[:partition]
+    end
+
+    # Returns the message's offset as published in the topic's partition. When
+    # error != nil, offset the error occurred at.
+    #
+    # @return [Integer] Message offset
+    # @return [RD_KAFKA_OFFSET_INVALID] Message was retried and idempotence is
+    #   enabled.
+    def offset
+      self[:offset]
+    end
+
+    # Returns the per message opaque pointer that was given to produce. This is
+    # a pointer to a Ruby object owned by the application.
+    #
+    # @note Using the opaque is dangerous and requires that the application
+    #   maintain a reference to the object passed to produce. Failing to do so
+    #   will cause segfaults due to the object having been garbage collected.
+    #
+    # @example Retrieve object from opaque
+    #   require "fiddle"
+    #   obj = Fiddle::Pointer.new(msg.opaque.to_i).to_value
+    #
+    # @return [nil] Opaque was not set
+    # @return [FFI::Pointer] Pointer to opaque address
+    def opaque
+      self[:private]
+    end
+
+    # Returns the message's payload. When error != nil, will contain a string
+    # describing the error.
+    #
+    # @return [String] Message payload or error string.
+    def payload
+      if self[:payload].null?
+        return nil
+      end
+
+      self[:payload].read_string(self[:len])
+    end
+
+    # Get the message header list
+    #
+    # @raise [Kafka::ResponseError] Error occurred parsing headers
+    #
+    # @return [nil] Message does not have any headers
+    # @return [Message::Headers] Set of headers
+    def headers
+      ptr = ::FFI::MemoryPointer.new(:pointer)
+
+      err = ::Kafka::FFI.rd_kafka_message_headers(self, ptr)
+      case err
+      when :ok
+        if ptr.null?
+          nil
+        else
+          Message::Headers.new(ptr)
+        end
+      when RD_KAFKA_RESP_ERR__NOENT
+        # Messages does not have headers
+        nil
+      else
+        raise ::Kafka::ResponseError, err
+      end
+    ensure
+      ptr.free
+    end
+
+    # Get the Message's headers and detach them from the Message (setting its
+    # headers to nil). Calling detach_headers means the applicaiton is now the
+    # owner of the returned Message::Header and must eventually call #destroy
+    # when the application is done with them.
+    #
+    # @raise [Kafka::ResponseError] Error occurred parsing headers
+    #
+    # @return [nil] Message does not have any headers
+    # @return [Message::Headers] Set of headers
+    def detach_headers
+      ptr = ::FFI::MemoryPointer.new(:pointer)
+
+      err = ::Kafka::FFI.rd_kafka_message_detach_headers(self, ptr)
+      case err
+      when :ok
+        if ptr.null?
+          nil
+        else
+          Message::Headers.new(ptr)
+        end
+      when RD_KAFKA_RESP_ERR__NOENT
+        # Messages does not have headers
+        nil
+      else
+        raise ::Kafka::ResponseError, err
+      end
+    ensure
+      ptr.free
+    end
+
+    # rubocop:disable Naming/AccessorMethodName
+
+    # Replace the Message's headers with a new set.
+    #
+    # @note The Message takes ownership of the headers and they will be
+    #   destroyed automatically with the Message.
+    def set_headers(headers)
+      ::Kafka::FFI.rd_kafka_set_headers(self, headers)
+    end
+    alias headers= set_headers
+    # rubocop:enable Naming/AccessorMethodName
+
+    # Retrieve the timestamp for a consumed message.
+    #
+    # @example Convert timestamp to a Time
+    #   ts = message.timestamp
+    #   ts = ts && Time.at(0, ts, :millisecond).utc
+    #
+    # @return [Integer] Message timestamp as milliseconds since unix epoch
+    # @return [nil] Timestamp is not available
+    def timestamp
+      # @todo: Type (second param) [rd_kafka_timestamp_type_t enum]
+      ts = ::Kafka::FFI.rd_kafka_message_timestamp(self, nil)
+      ts == -1 ? nil : ts
+    end
+
+    # Retrieve the latency since the Message was published to Kafka.
+    #
+    # @return [Integer] Latency since produce() call in microseconds
+    # @return [nil] Latency not available
+    def latency
+      latency = ::Kafka::FFI.rd_kafka_message_latency(self)
+      latency == -1 ? nil : latency
+    end
+
+    # Frees resources used by the messages and hands ownership by to
+    # librdkafka. The application should call destroy when done processing the
+    # message.
+    def destroy
+      if !null?
+        ::Kafka::FFI.rd_kafka_message_destroy(self)
+      end
+    end
+  end
+end

data/lib/kafka/ffi/metadata.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Kafka::FFI
+  class Metadata < ::FFI::Struct
+    layout(
+      :broker_cnt,       :int,
+      :brokers,          :pointer, # *rd_kafka_metadata_broker
+      :topic_cnt,        :int,
+      :topics,           :pointer, # *rd_kafka_metadata_topic
+      :orig_broker_id,   :int32,
+      :orig_broker_name, :string
+    )
+
+    # Returns detailed metadata for the Brokers in the cluster.
+    #
+    # @return [Array<BrokerMetadata>] Metadata about known Brokers.
+    def brokers
+      ptr = self[:brokers]
+
+      self[:broker_cnt].times.map do |i|
+        BrokerMetadata.new(ptr + (i * BrokerMetadata.size))
+      end
+    end
+
+    # Returns detailed metadata about the topics and their partitions.
+    #
+    # @return [Array<TopicMetadata>] Metadata about known Topics.
+    def topics
+      ptr = self[:topics]
+
+      self[:topic_cnt].times.map do |i|
+        TopicMetadata.new(ptr + (i * TopicMetadata.size))
+      end
+    end
+
+    # Returns the ID of the Broker that the metadata request was served by.
+    #
+    # @return [Integer] Broker ID
+    def broker_id
+      self[:orig_broker_id]
+    end
+
+    # Returns the name of the Broker that the metadata request was served by.
+    #
+    # @return [String] Broker name
+    def broker_name
+      self[:orig_broker_name]
+    end
+
+    # Destroy the Metadata response, returning it's resources back to the
+    # system.
+    def destroy
+      if !null?
+        ::Kafka::FFI.rd_kafka_metadata_destroy(self)
+      end
+    end
+  end
+end

data/lib/kafka/ffi/opaque.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Kafka::FFI
+  # Opaque provides a safe mechanism for providing Ruby objects as opaque
+  # pointers.
+  #
+  # Opaque pointers are used heavily in librdkafka to allow for passing
+  # references to application state into callbacks, configs, and other
+  # contexts. Ruby's garbage collector cannot check for references held in
+  # external FFI memory and will garbage collect objects that are otherwise not
+  # referenced leading to a segmentation fault.
+  #
+  # Opaque solves this by allocated a memory address which is used as a hash
+  # key to look up the Ruby object when needed. This keeps a reference to the
+  # object in Ruby so it is not garbage collected. This method allows for Ruby
+  # objects to be moved in memory during compaction (in Ruby 2.7+) compared to
+  # storing a referece to a Ruby object.
+  class Opaque
+    extend ::FFI::DataConverter
+    native_type :pointer
+
+    # Registry holds references to all known Opaque instances in the system
+    # keyed by the pointer address associated with it.
+    @registry = {}
+
+    class << self
+      # Register the Opaque the registry, keeping a reference to it to avoid it
+      # being garbage collected. This will replace any existing Opaque with the
+      # same address.
+      #
+      # @param opaque [Opaque]
+      def register(opaque)
+        @registry[opaque.pointer.address] = opaque
+      end
+
+      # Remove the Opaque from the registry, putting it back in contention for
+      # garbage collection.
+      #
+      # @param opaque [Opaque]
+      def remove(opaque)
+        @registry.delete(opaque.pointer.address)
+      end
+
+      # @param value [Opaque]
+      def to_native(value, _ctx)
+        value.pointer
+      end
+
+      # @param value [FFI::Pointer]
+      def from_native(value, _ctx)
+        if value.null?
+          return nil
+        end
+
+        @registry.fetch(value.address, nil)
+      end
+    end
+
+    attr_reader :value
+    attr_reader :pointer
+
+    def initialize(value)
+      @value = value
+      @pointer = ::FFI::MemoryPointer.new(:int8)
+
+      Opaque.register(self)
+    end
+
+    # Free releases the pointer back to the system and removes the Opaque from
+    # the registry. free should only be called when the Opaque is no longer
+    # stored in librdkafka as it frees the backing pointer which could cause a
+    # segfault if still referenced.
+    def free
+      Opaque.remove(self)
+      @pointer.free
+
+      @value = nil
+      @pointer = nil
+    end
+  end
+end

data/lib/kafka/ffi/opaque_pointer.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "ffi"
+
+module Kafka::FFI
+  # OpaquePointer pointer provides a common pattern where we receive a pointer
+  # to a struct but don't care about the structure and need to pass it to
+  # functions. OpaquePointer gives type safety by checking types before
+  # converting.
+  #
+  # @note Kafka has several options for `opaque` pointers that get passed to
+  #   callbacks. Those opaque pointers are not related to this opaque pointer.
+  class OpaquePointer
+    extend ::FFI::DataConverter
+
+    # @attr pointer [FFI::Pointer] Pointer to the implementing class
+    attr_reader :pointer
+
+    class << self
+      # Convert from a FFI::Pointer to the implementing class.
+      #
+      # @param value [FFI::Pointer]
+      #
+      # @return [nil] Passed ::FFI::Pointer::NULL
+      # @return Instance of the class backed by the pointer.
+      def from_native(value, _ctx)
+        if !value.is_a?(::FFI::Pointer)
+          raise TypeError, "from_native can only convert from a ::FFI::Pointer to #{self}"
+        end
+
+        # The equivalent of a native NULL pointer is nil.
+        if value.null?
+          return nil
+        end
+
+        obj = allocate
+        obj.send(:initialize, value)
+        obj
+      end
+
+      # Convert from a Kafka::FFI type to a native FFI type.
+      #
+      # @param value [Object] Instance to retrieve a pointer for.
+      #
+      # @return [FFI::Pointer] Pointer to the opaque struct
+      def to_native(value, _ctx)
+        if value.nil?
+          return ::FFI::Pointer::NULL
+        end
+
+        if !value.is_a?(self)
+          raise TypeError, "expected a kind of #{self}, was #{value.class}"
+        end
+
+        value.pointer
+      end
+
+      # Provide ::FFI::Struct API compatility for consistency with
+      # attach_function is called with an OpaquePointer.
+      def by_ref
+        self
+      end
+
+      def inherited(subclass)
+        subclass.native_type :pointer
+      end
+    end
+
+    def initialize(pointer)
+      @pointer = pointer
+    end
+  end
+end