kafka 0.5.0

data/lib/kafka/ffi/partition_metadata.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Kafka::FFI
+  class PartitionMetadata < ::FFI::Struct
+    layout(
+      :id,          :int32,
+      :err,         :error_code,
+      :leader,      :int32,
+      :replica_cnt, :int,
+      :replicas,    :pointer, # *int32_t
+      :isr_cnt,     :int,
+      :isrs,        :pointer  # *int32_t
+    )
+
+    # Returns the Partition's ID
+    #
+    # @return [Integer] Partition ID
+    def id
+      self[:id]
+    end
+
+    # Returns the error for the Partition as reported by the Broker.
+    #
+    # @return [nil] No error reported by Broker
+    # @return [Kafka::ResponseError] Error reported by Broker
+    def error
+      if self[:err] != :ok
+        return ::Kafka::ResponseError.new(self[:err])
+      end
+    end
+
+    # ID of the Leader Broker for this Partition
+    #
+    # @return [Integer] Leader Broker ID
+    def leader
+      self[:leader]
+    end
+
+    # Returns the Broker IDs of the Brokers with replicas of this Partition.
+    #
+    # @return [Array<Integer>] IDs for Brokers with replicas
+    def replicas
+      if self[:replica_cnt] == 0 || self[:replicas].null?
+        return []
+      end
+
+      self[:replicas].read_array_of_int32(self[:replica_cnt])
+    end
+
+    # Returns the Broker IDs of the in-sync replicas for this Partition.
+    #
+    # @return [Array<Integer>] IDs of Brokers that have in-sync replicas.
+    def in_sync_replicas
+      if self[:isr_cnt] == 0 || self[:isrs].null?
+        return []
+      end
+
+      self[:isrs].read_array_of_int32(self[:isr_cnt])
+    end
+  end
+end

data/lib/kafka/ffi/producer.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "ffi"
+require "kafka/ffi/client"
+
+module Kafka::FFI
+  class Producer < Kafka::FFI::Client
+    def self.new(config = nil)
+      super(:producer, config)
+    end
+
+    # Produce and send a single message to the Kafka cluster.
+    #
+    # @param topic [Topic, String] Topic (or name of topic) to receive the
+    #   message.
+    #
+    # @param payload [String, nil] Content of the message.
+    #
+    # @param key [String] Message partitioning key
+    #
+    # @param partition [nil, -1] Use the configured partitioner to determine
+    #   which partition to publish the message to.
+    # @param partition [Integer] Partition of the topic that should receive the
+    #   message.
+    #
+    # @param headers [Kafka::FFI::Message::Header]
+    #
+    # @param timestamp [Time] Timestamp as Time
+    # @param timestamp [Integer] Timestamp as milliseconds since unix epoch
+    # @param timestamp [nil] Timestamp is assigned by librdkafka
+    #
+    # @param opaque [Opaque] Reference to an object owned by the application
+    #   which will be available as Message#opaque in callbacks. The application
+    #   MUST call #free on the Opaque once the final callback has been
+    #   triggered to avoid leaking memory.
+    def produce(topic, payload, key: nil, partition: nil, headers: nil, timestamp: nil, opaque: nil)
+      args = [
+        # Ensure librdkafka copies the payload into its own memory since the
+        # string backing it could be garbage collected.
+        :vtype, :msgflags, :int, Kafka::FFI::RD_KAFKA_MSG_F_COPY,
+      ]
+
+      if payload
+        args.append(:vtype, :value, :buffer_in, payload, :size_t, payload.bytesize)
+      end
+
+      # The partitioning key is optional
+      if key
+        args.append(:vtype, :key, :buffer_in, key, :size_t, key.bytesize)
+      end
+
+      # Partition will default to being auto assigned by the configured
+      # partitioning strategy.
+      if partition
+        args.append(:vtype, :partition, :int32, partition)
+      end
+
+      # Headers are optional and can be passed as either a reference to a
+      # Header object or individual key/value pairs. This only supports the
+      # Header object because supporting key + valu
+      if headers
+        args.append(:vtype, :headers, :pointer, headers.pointer)
+      end
+
+      case topic
+      when Topic
+        args.append(:vtype, :rkt, :pointer, topic.pointer)
+      when String
+        args.append(:vtype, :topic, :string, topic)
+      else
+        raise ArgumentError, "topic must be either a Topic or String"
+      end
+
+      if opaque
+        args.append(:vtype, :opaque, :pointer, opaque.pointer)
+      end
+
+      if timestamp
+        ts =
+          case timestamp
+          when Time    then ((timestamp.to_i * 1000) + (timestamp.nsec / 1000))
+          when Integer then timestamp
+          else
+            raise ArgumentError, "timestamp must be nil, a Time, or an Integer"
+          end
+
+        args.append(:vtype, :timestamp, :int64, ts)
+      end
+
+      # Add the sentinel value to denote the end of the argument list.
+      args.append(:vtype, :end)
+
+      err = ::Kafka::FFI.rd_kafka_producev(self, *args)
+      if err != :ok
+        # The only documented error is RD_KAFKA_RESP_ERR__CONFLICT should both
+        # HEADER and HEADERS keys be passed in. There is no way for HEADER to
+        # be passed to producev based on the above implementation.
+        raise ::Kafka::ResponseError, err
+      end
+
+      nil
+    end
+
+    # Wait until all outstanding produce requests are completed. This should
+    # typically be done prior to destroying a producer to ensure all queued and
+    # in-flight requests are completed before terminating.
+    #
+    # @raise [Kafka::ResponseError] Timeout was reached before all
+    #   outstanding requests were completed.
+    def flush(timeout: 1000)
+      err = ::Kafka::FFI.rd_kafka_flush(self, timeout)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      nil
+    end
+
+    # Purge messages currently handled by the Producer. By default this will
+    # purge all queued and inflight messages asyncronously.
+    #
+    # @param queued [Boolean] Purge any queued messages
+    # @param inflight [Boolean] Purge messages that are inflight
+    # @param blocking [Boolean] When true don't wait for background thread
+    #   queue purging to finish.
+    #
+    # @raise [Kafka::ResponseError] Error occurred purging state. This is
+    #   unlikely as the documented error are not possible with this
+    #   implementation.
+    def purge(queued: true, inflight: true, blocking: false)
+      mask = 0
+      mask |= RD_KAFKA_PURGE_F_QUEUE if queued
+      mask |= RD_KAFKA_PURGE_F_INFLIGHT if inflight
+      mask |= RD_KAFKA_PURGE_F_NON_BLOCKING if blocking
+
+      err = ::Kafka::FFI.rd_kafka_purge(self, mask)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      nil
+    end
+  end
+end

data/lib/kafka/ffi/queue.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "ffi"
+require "kafka/ffi/opaque_pointer"
+
+module Kafka::FFI
+  class Queue < OpaquePointer
+    def self.new(client)
+      ::Kafka::FFI.rd_kafka_queue_new(client)
+    end
+
+    # Poll a queue for an event, waiting up to timeout milliseconds. Takes an
+    # optional block which will handle destroying the event at the completion
+    # of the block.
+    #
+    # @param timeout [Integer] Max time to wait in millseconds for an Event.
+    #
+    # @yield [event]
+    # @yieldparam event [Event] Polled event
+    #
+    # @return [nil] No event was available within the timeout
+    # @return [Event] Event polled from the queue, application is responsible
+    #   for calling #destroy on the Event when finished with it.
+    # @return When passed a block, the result returned by the block
+    def poll(timeout: 1000)
+      event = ::Kafka::FFI.rd_kafka_queue_poll(self, timeout)
+      if event.nil?
+        return nil
+      end
+
+      if block_given?
+        begin
+          yield(event)
+        ensure
+          event.destroy
+        end
+      else
+        event
+      end
+    end
+
+    # Forward events meant for this Queue to the destination Queue instead.
+    #
+    # @param dest [Queue] Destination queue to forward
+    # @param dest [nil] Remove forwarding for this queue.
+    def forward(dest)
+      ::Kafka::FFI.rd_kafka_queue_forward(self, dest)
+    end
+
+    # Retrieve the current number of elemens in the queue.
+    #
+    # @return [Integer] Number of elements in the queue
+    def length
+      ::Kafka::FFI.rd_kafka_queue_length(self)
+    end
+
+    # Release the applications reference on the queue, possibly destroying it
+    # and releasing it's resources.
+    def destroy
+      if !pointer.null?
+        ::Kafka::FFI.rd_kafka_queue_destroy(self)
+      end
+    end
+  end
+end

data/lib/kafka/ffi/topic.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "kafka/ffi/opaque_pointer"
+
+module Kafka::FFI
+  class Topic < OpaquePointer
+    # Retrieve the name of the topic
+    #
+    # @return [String] Name of the topic
+    def name
+      ::Kafka::FFI.rd_kafka_topic_name(self)
+    end
+
+    # Seek consumer for topic_partition to offset.
+    #
+    # @param partition [Integer] Partition to set offset for
+    # @param offset [Integer] Absolute or logical offset
+    # @param timeout [Integer] Maximum time to wait in milliseconds
+    #
+    # @return [Boolean] True when the consumer's offset was changed
+    def seek(partition, offset, timeout: 1000)
+      ::Kafka::FFI.rd_kafka_seek(self, partition, offset, timeout)
+    end
+
+    # Release the application's hold on the backing topic in librdkafka.
+    def destroy
+      if !pointer.null?
+        ::Kafka::FFI.rd_kafka_topic_destroy(self)
+      end
+    end
+  end
+end

data/lib/kafka/ffi/topic_config.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "ffi"
+require "kafka/ffi/opaque_pointer"
+
+module Kafka::FFI
+  # TopicConfig can be passed to Topic.new to configure how the client
+  # interacts with the Topic.
+  class TopicConfig < OpaquePointer
+    def self.new
+      Kafka::FFI.rd_kafka_topic_conf_new
+    end
+
+    # Set the config option at `key` to `value`. The configuration options
+    # match those used by librdkafka (and the Java client).
+    #
+    # @see https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+    #
+    # @param key [String] Configuration key
+    # @param value [String] Value to set
+    #
+    # @raise [Kafka::FFI::UnknownConfigKey]
+    # @raise [Kafka::FFI::InvalidConfigValue]
+    def set(key, value)
+      key = key.to_s
+      value = value.to_s
+
+      error = ::FFI::MemoryPointer.new(:char, 512)
+      result = ::Kafka::FFI.rd_kafka_topic_conf_set(self, key, value, error, error.size)
+
+      # See config_result enum in ffi.rb
+      case result
+      when :ok
+        nil
+      when :unknown
+        raise Kafka::FFI::UnknownConfigKey.new(key, value, error.read_string)
+      when :invalid
+        raise Kafka::FFI::InvalidConfigValue.new(key, value, error.read_string)
+      end
+    ensure
+      error.free if error
+    end
+
+    # Get the current config value for the given key.
+    #
+    # @param key [String] Config key to fetch the setting for.
+    #
+    # @return [String, :unknown] Value for the key or :unknown if not already
+    #   set.
+    def get(key)
+      key = key.to_s
+
+      # Will contain the size of the value at key
+      size = ::FFI::MemoryPointer.new(:size_t)
+
+      # Make an initial request for the size of buffer we need to allocate.
+      # When trying to make a guess at the potential size the code would often
+      # segfault due to rd_kafka_conf_get reallocating the buffer.
+      err = ::Kafka::FFI.rd_kafka_topic_conf_get(self, key, ::FFI::Pointer::NULL, size)
+      if err != :ok
+        return err
+      end
+
+      # Allocate a string long enough to contain the whole value.
+      value = ::FFI::MemoryPointer.new(:char, size.read(:size_t))
+      err = ::Kafka::FFI.rd_kafka_topic_conf_get(self, key, value, size)
+      if err != :ok
+        return err
+      end
+
+      value.read_string
+    ensure
+      size.free if size
+      value.free if value
+    end
+
+    # Duplicate the current config
+    #
+    # @return [TopicConfig] Duplicated config
+    def dup
+      ::Kafka::FFI.rd_kafka_topic_conf_dup(self)
+    end
+
+    # Sets a custom partitioner callback that is called for each message to
+    # determine which partition to publish the message to.
+    #
+    # @example Random partitioner
+    #   set_partitioner_cb do |_topic, _key, parts|
+    #     rand(parts)
+    #   end
+    #
+    # @see "partitioner" config option for predefined strategies
+    #
+    # @yield [topic, key, partition_count]
+    # @yieldparam topic [Topic] Topic the message is being published to
+    # @yieldparam key [String] Partitioning key provided when publishing
+    # @yieldparam partition_count [Integer] Number of partitions the topic has
+    # @yieldreturn [Integer] The partition to publish the message to
+    def set_partitioner_cb
+      if !block_given?
+        raise ArgumentError, "set_partitioner_cb must be called with a block"
+      end
+
+      # @todo How do we guarantee the block does not get garbage collected?
+      # @todo Support opaque pointers?
+      cb = ::FFI::Function.new(:int, [:pointer, :string, :size_t, :int32, :pointer, :pointer]) do |topic, key, _, partitions, _, _|
+        topic = Topic.new(topic)
+
+        yield(topic, key, partitions)
+      end
+
+      ::Kafka::FFI.rd_kafka_topic_conf_set_partitioner_cb(self, cb)
+    end
+
+    # Free all resources used by the topic config.
+    #
+    # @note Never call #destroy on a Config that has been passed to
+    #   Kafka::FFI.rd_kafka_topic_new since the handle will take ownership of
+    #   the config.
+    def destroy
+      if !pointer.null?
+        ::Kafka::FFI.rd_kafka_topic_conf_destroy(self)
+      end
+    end
+  end
+end

data/lib/kafka/ffi/topic_metadata.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Kafka::FFI
+  class TopicMetadata < ::FFI::Struct
+    layout(
+      :topic,         :string,
+      :partition_cnt, :int,
+      :partitions,    :pointer, # *rd_kafka_metadata_partition
+      :err,           :error_code
+    )
+
+    # Returns the name of the topic
+    #
+    # @return [String] Name of the topic
+    def topic
+      self[:topic]
+    end
+    alias name topic
+
+    # Returns any Broker reported errors.
+    #
+    # @return [nil] Broker reported no errors for the topic
+    # @return [Kafka::ResponseError] Error reported by Broker
+    def error
+      if self[:err] != :ok
+        return ::Kafka::ResponseError.new(self[:err])
+      end
+    end
+
+    # Returns the set of PartitionMetadata for the Topic
+    #
+    # @return [Array<PartitionMetadata>] Details about individual Topic
+    #   Partitions.
+    def partitions
+      ptr = self[:partitions]
+
+      self[:partition_cnt].times.map do |i|
+        PartitionMetadata.new(ptr + (i * PartitionMetadata.size))
+      end
+    end
+  end
+end

data/lib/kafka/ffi/topic_partition.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "ffi"
+
+module Kafka::FFI
+  class TopicPartition < ::FFI::Struct
+    layout(
+      :topic,         :string,
+      :partition,     :int32,
+      :offset,        :int64,
+      :metadata,      :pointer,
+      :metadata_size, :size_t,
+      :opaque,        :pointer,
+      :err,           :error_code,
+      :_private,      :pointer # DO NOT TOUCH. Internal to librdkafka
+    )
+
+    # @return [nil] The TopicPartition does not have an error set
+    # @return [Kafka::ResponseError] Error for this topic occurred related to
+    #   the action the TopicPartition (or TopicPartitionList) was passed to.
+    def error
+      if self[:err] != :ok
+        ::Kafka::ResponseError.new(self[:err])
+      end
+    end
+
+    # @return [String] Name of the topic
+    def topic
+      self[:topic]
+    end
+
+    # @return [Integer] Partition number
+    def partition
+      self[:partition]
+    end
+
+    # @return [Integer] Known offset for the consumer group for topic +
+    #   partition.
+    def offset
+      self[:offset]
+    end
+  end
+end

data/lib/kafka/ffi/topic_partition_list.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "ffi"
+require "kafka/ffi/topic_partition"
+
+module Kafka::FFI
+  class TopicPartitionList < ::FFI::Struct
+    layout(
+      :cnt,   :int,
+      :size,  :int,
+      :elems, :pointer
+    )
+
+    # New initializes a new TopicPartitionList with an initial capacity to hold
+    # `count` items.
+    #
+    # @param count [Integer] Initial capacity
+    #
+    # @return [TopicPartitionList] An empty TopicPartitionList
+    def self.new(count = 0)
+      # Handle initialization through FFI. This will be called by
+      # rd_kafka_topic_partition_list_new.
+      if count.is_a?(::FFI::Pointer)
+        return super(count)
+      end
+
+      ::Kafka::FFI.rd_kafka_topic_partition_list_new(count)
+    end
+
+    # Returns the number of elements in the TopicPartitionList.
+    #
+    # @return [Integer] Number of elements
+    def size
+      self[:cnt]
+    end
+
+    # Returns true when the TopicPartitionList is empty
+    #
+    # @return [Boolean] True when the list is empty
+    def empty?
+      size == 0
+    end
+
+    # Add a topic + partition combination to the list
+    #
+    # @param topic [String] Name of the topic to add
+    # @param partition [Integer] Partition of the topic to add to the list.
+    # @param partition [-1] Add all partitions of the topic to the list.
+    #
+    # @return [TopicPartition] TopicPartition for the combination
+    def add(topic, partition = -1)
+      ::Kafka::FFI.rd_kafka_topic_partition_list_add(self, topic.to_s, partition)
+    end
+
+    # Add a range of TopicPartitions to the list.
+    #
+    # @param topic [String] Name of the topic to add
+    # @param range_or_lower [Range, Integer] Either a Range specifying the
+    #   Range of partitions or the lower bound for the range. When providing a
+    #   Range any value for upper is ignored.
+    # @param upper [Integer, nil] The upper bound of the set of partitions
+    #   (inclusive). Required unless range_or_lower is a Range.
+    def add_range(topic, range_or_lower, upper = nil)
+      lower = range_or_lower
+
+      # Allows passing a Range for convenience.
+      if range_or_lower.is_a?(Range)
+        lower = range_or_lower.min
+        upper = range_or_lower.max
+      elsif upper.nil?
+        raise ArgumentError, "upper was nil but must be provided when lower is not a Range"
+      end
+
+      ::Kafka::FFI.rd_kafka_topic_partition_list_add_range(self, topic.to_s, lower.to_i, upper.to_i)
+    end
+
+    # Remove a TopicPartition by partition
+    #
+    # @param topic [String] Name of the topic to remove
+    # @param partition [Integer] Partition to remove
+    #
+    # @return [Boolean] True when the partition was found and removed
+    def del(topic, partition)
+      ::Kafka::FFI.rd_kafka_topic_partition_list_del(self, topic.to_s, partition) == 1
+    end
+
+    # Remove a TopicPartition by index
+    #
+    # @param idx [Integer] Index in elements to remove
+    #
+    # @return [Boolean] True when the TopicPartition was found and removed
+    def del_by_idx(idx)
+      ::Kafka::FFI.rd_kafka_topic_partition_list_del_by_idx(self, idx) == 1
+    end
+
+    alias delete del
+    alias delete_by_index del_by_idx
+
+    # Duplicate the TopicPartitionList as a new TopicPartitionList that is
+    # identical to the current one.
+    #
+    # @return [TopicPartitionList] New clone of this TopicPartitionList
+    def copy
+      ::Kafka::FFI.rd_kafka_topic_partition_list_copy(self)
+    end
+
+    # Set the consumed offset for topic and partition
+    #
+    # @param topic [String] Name of the topic to set the offset for
+    # @param partition [Integer] Partition to set the offset for
+    # @param offset [Integer] Offset of the topic+partition to set
+    #
+    # @return [Integer] 0 for success otherwise rd_kafka_resp_err_t code
+    def set_offset(topic, partition, offset)
+      ::Kafka::FFI.rd_kafka_topic_partition_list_set_offset(self, topic, partition, offset)
+    end
+
+    # Sort the TopicPartitionList. Sort can take a block that should implement
+    # a standard comparison function that returns -1, 0, or 1 depending on if
+    # left is less than, equal to, or greater than the right argument.
+    #
+    # @example Custom sorting function
+    #   sort do |left, right|
+    #     left.partition <=> right.partition
+    #   end
+    def sort(&block)
+      ::Kafka::FFI.rd_kafka_topic_partition_list_sort(self, block, nil)
+    end
+
+    # Find the TopicPartition in the set for the given topic + partition. Will
+    # return nil if the list does not include the combination.
+    #
+    # @param topic [String] Name of the topic
+    # @param partition [Integer] Topic partition
+    #
+    # @return [TopicPartition, nil] The TopicPartion for the topic + partition
+    #   combination.
+    def find(topic, partition)
+      result = ::Kafka::FFI.rd_kafka_topic_partition_list_find(self, topic, partition)
+
+      if result.null?
+        return nil
+      end
+
+      result
+    end
+
+    # Retrieves the set of TopicPartitions for the list.
+    #
+    # @return [Array<TopicPartition>]
+    def elements
+      self[:cnt].times.map do |i|
+        TopicPartition.new(self[:elems] + (i * TopicPartition.size))
+      end
+    end
+
+    # Free all resources used by the list and the list itself. Usage it
+    # dependent on the semantics of librdkafka, so make sure to only call on
+    # TopicPartitionLists that are not owned by objects. Generally, if you
+    # constructed the object it should be safe to destroy.
+    def destroy
+      if !null?
+        ::Kafka::FFI.rd_kafka_topic_partition_list_destroy(self)
+      end
+    end
+  end
+end