kafka 0.5.0

data/lib/kafka/ffi/consumer.rb

@@ -0,0 +1,342 @@
+# frozen_string_literal: true
+
+require "ffi"
+require "kafka/ffi/client"
+
+module Kafka::FFI
+  class Consumer < Kafka::FFI::Client
+    native_type :pointer
+
+    def self.new(config = nil)
+      super(:consumer, config)
+    end
+
+    # Retrieve the Consumer's broker assigned group Member ID.
+    #
+    # @return [String] broker assigned group Member ID
+    def member_id
+      id = ::Kafka::FFI.rd_kafka_memberid(self)
+
+      if id.null?
+        return nil
+      end
+
+      id.read_string
+    ensure
+      ::Kafka::FFI.rd_kafka_mem_free(self, id)
+    end
+
+    # Get the last known (cached) low and high offsets for the partition. This
+    # differs from query_watermark_offsets in that does not query the brokers.
+    #
+    # @see query_watermark_offsets
+    #
+    # @param topic [String] Name of the topic
+    # @param partition [Integer] Topic partition
+    #
+    # @raise [Kafka::ResponseError] Error that occurred retrieving offsets
+    #
+    # @return [Array<(Integer, Integer)>] low and high offsets. If either is
+    #   unknown the RD_KAFKA_OFFSET_INVALID is returned for that value
+    def get_watermark_offsets(topic, partition)
+      low  = ::FFI::MemoryPointer.new(:int64)
+      high = ::FFI::MemoryPointer.new(:int64)
+
+      err = ::Kafka::FFI.rd_kafka_get_watermark_offsets(self, topic, partition, low, high)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      [low.read_int64, high.read_int64]
+    end
+
+    # rubocop:disable Naming/AccessorMethodName
+
+    # Returns a reference to the consume queue. This is the queue served by
+    # consumer_poll.
+    #
+    # @note Caller must call #destroy when done with the Queue.
+    #
+    # @return [Queue] Consumer queue
+    def get_consumer_queue
+      ::Kafka::FFI.rd_kafka_queue_get_consumer(self)
+    end
+    # rubocop:enable Naming/AccessorMethodName
+
+    # Returns a reference to the partition's queue.
+    #
+    # @note Caller must call #destroy when done with the Queue.
+    #
+    # @return [Queue] Partition Queue
+    def get_partition_queue(topic, partition)
+      ::Kafka::FFI.rd_kafka_queue_get_partition(self, topic, partition)
+    end
+
+    # Redirect the main event queue to the Consumer's queue so the consumer
+    # doesn't need to poll from it separately for event callbacks to fire.
+    #
+    # @note It is not permitted to call #poll after redirecting the main queue
+    #   with poll_set_consumer.
+    #
+    # @raise [Kafka::ResponseError] Error occurred redirecting the main
+    #   queue.
+    def poll_set_consumer
+      err = ::Kafka::FFI.rd_kafka_poll_set_consumer(self)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      nil
+    end
+
+    # Subscribe the consumer to receive Messages for a set of topics. The
+    # current set of subscriptions will be replaced.
+    #
+    # @example Subscribe to multiple topics
+    #   client.subscribe("signals", "events", "changes")
+    #
+    # @param [String, Array<String>] Topic name or list of topics to subscribe
+    #   to.
+    #
+    # @raise [Kafka::ResponseError] Error occurred subscribing to the topics.
+    def subscribe(topic, *rest)
+      topics = [topic, rest].flatten
+
+      tpl = TopicPartitionList.new(topics.length)
+      topics.each do |t|
+        tpl.add(t)
+      end
+
+      err = ::Kafka::FFI.rd_kafka_subscribe(self, tpl)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      nil
+    ensure
+      tpl.destroy
+    end
+
+    # Unsubscribe from the current subscription set (e.g. all current
+    # subscriptions).
+    #
+    # @raise [Kafka::ResponseError] Error unsubscribing from topics
+    def unsubscribe
+      err = ::Kafka::FFI.rd_kafka_unsubscribe(self)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      nil
+    end
+
+    # List the current topic subscriptions for the consumer.
+    #
+    # @raise [Kafka::ResponseError] Error that occurred retrieving the
+    #   subscriptions
+    #
+    # @return [Array<String>] List of current subscribed topics
+    def subscription
+      ptr = ::FFI::MemoryPointer.new(:pointer)
+
+      err = ::Kafka::FFI.rd_kafka_subscription(self, ptr)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      begin
+        tpl = ::Kafka::FFI::TopicPartitionList.new(ptr.read_pointer)
+
+        # Map the topic partition list to topic names.
+        tpl.elements.map(&:topic)
+      ensure
+        tpl.destroy
+      end
+    ensure
+      ptr.free
+    end
+
+    # Alias to subscriptions since it does return a list of topics
+    alias subscriptions subscription
+
+    # Atomically assign the set of partitions to consume. This will replace the
+    # existing assignment.
+    #
+    # @see rdkafka.h rd_kafka_assign for semantics on use from callbacks and
+    #   how empty vs NULL lists affect internal state.
+    #
+    # @param list [TopicPartitionList] List of topic+partition assignments
+    #
+    # @raise [Kafka::ResponseError] Error processing assignments
+    def assign(list)
+      err = ::Kafka::FFI.rd_kafka_assign(self, list)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      nil
+    end
+
+    # List the current partition assignment(s) for the consumer.
+    #
+    # @raise [Kafka::ResponseError] Error that occurred retrieving the
+    #   assignments.
+    #
+    # @return [Hash{String => Array<Integer>}] Current assignments for the
+    #   consumer. Hash keys are topic names and values are the list of assigned
+    #   partitions.
+    def assignment
+      ptr = ::FFI::MemoryPointer.new(:pointer)
+
+      err = ::Kafka::FFI.rd_kafka_assignment(self, ptr)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      begin
+        tpl = ::Kafka::FFI::TopicPartitionList.new(ptr.read_pointer)
+
+        # { "topic" => [1, 2, 3] }
+        tpl.elements.inject({}) do |memo, tp|
+          (memo[tp.topic] ||= []) << tp.partition
+          memo
+        end
+      ensure
+        tpl.destroy
+      end
+    ensure
+      ptr.free
+    end
+
+    # Alias assignment since it returns a set
+    alias assignments assignment
+
+    # Retrieve committed offsets for topics + partitions. The offset field for
+    # each TopicPartition in list will be set to the stored offset or
+    # RD_KAFKA_OFFSET_INVALID in case there was no stored offset for that
+    # partition. The error field is set if there was an error with the
+    # TopicPartition.
+    #
+    # @param list [TopicPartitionList] List of topic+partitions to fetch
+    #   current offsets. The list will be updated to set the committed offset
+    #   or error as appropriate.
+    # @param timeout [Integer] Maximum time to wait in milliseconds
+    #
+    # @raise [Kafka::ResponseError] Error with the request (likely a
+    #   timeout). Errors with individual topic+partition combinations are set
+    #   in the returned TopicPartitionList
+    #
+    # @return [TopicPartitionList] the updated list
+    def committed(list, timeout: 1000)
+      if list.nil?
+        raise ArgumentError, "list cannot be nil"
+      end
+
+      err = ::Kafka::FFI.rd_kafka_committed(self, list, timeout)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      # Return the list that was passed in as it should now be augmented with
+      # the committed offsets and any errors fetching said offsets.
+      list
+    end
+
+    # Poll the consumer's queue for a waiting Message and yields that message.
+    # The yielded message must not be cached in the application as it becomes
+    # unusable once the block completes.
+    #
+    # @see max.poll.interal.ms configuration option.
+    #
+    # @param timeout [Integer] How long to wait for a message in milliseconds.
+    #
+    # @raise [ArgumentError] consumer_poll was called without a block.
+    # @raise [Kafka::ResponseError] Error occurred while polling.
+    #
+    # @yield [message]
+    # @yieldparam message [Message] Message received from Kafka. Application
+    #   must not call #destroy as it is owned by the Consumer.
+    #
+    # @return Either nil or the result of the block
+    def consumer_poll(timeout)
+      if !block_given?
+        raise ArgumentError, "consumer_poll must be passed a block"
+      end
+
+      msg = ::Kafka::FFI.rd_kafka_consumer_poll(self, timeout.to_i)
+
+      # No message was available
+      if msg.null?
+        return nil
+      end
+
+      begin
+        if msg.error
+          raise msg.error
+        end
+
+        yield(msg)
+      ensure
+        msg.destroy
+      end
+    end
+
+    # Commit the set of offsets from the given TopicPartitionList.
+    #
+    # @param offsets [TopicPartitionList] Set of topic+partition with offset
+    #   (and maybe metadata) to be committed. If offsets is nil the current
+    #   partition assignment set will be used instead.
+    # @param async [Boolean] If async is false this operation will block until
+    #   the broker offset commit is done.
+    #
+    # @raise [Kafka::ResponseError] Error committing offsets. Only raise if
+    #   async is false.
+    def commit(offsets, async)
+      err = ::Kafka::FFI.rd_kafka_commit(self, offsets, async)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      nil
+    end
+
+    # Commit the message's offset on the broker for the message's partition.
+    #
+    # @param message [Message] The message to commit as processed
+    # @param async [Boolean] True to allow commit to happen in the background.
+    #
+    # @raise [Kafka::ResponseError] Error that occurred commiting the message
+    def commit_message(message, async)
+      if message.nil? || message.null?
+        raise ArgumentError, "message cannot but nil/null"
+      end
+
+      err = ::Kafka::FFI.rd_kafka_commit_message(message, async)
+      if err
+        raise ::Kafka::ResponseError, err
+      end
+
+      nil
+    end
+
+    # Close down the consumer. This will block until the consumer has revoked
+    # its assignment(s), committed offsets, and left the consumer group. The
+    # maximum blocking time is roughly limited to the `session.timeout.ms`
+    # config option.
+    #
+    # Ensure that `destroy` is called after the consumer is closed to free up
+    # resources.
+    #
+    # @note Maximum blocking time is roughly limited to `session.timeout.ms`
+    #
+    # @raise [Kafka::ResponseError] Error occurred closing the consumer
+    def close
+      err = ::Kafka::FFI.rd_kafka_consumer_close(self)
+      if err != :ok
+        raise ::Kafka::ResponseError, err
+      end
+
+      nil
+    end
+  end
+end

data/lib/kafka/ffi/error.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Kafka::FFI
+  class Error < Kafka::Error; end
+
+  # TopicAlreadyConfigured is raised by Client#topic when passing a config to a
+  # topic that has already been initialized for the Client.
+  class TopicAlreadyConfiguredError < Error; end
+
+  # ConfigError is raised when making changes to the global config.
+  class ConfigError < Error
+    attr_reader :key
+    attr_reader :value
+
+    def initialize(key, value, message)
+      super(message)
+
+      @key = key
+      @value = value
+    end
+  end
+
+  class UnknownConfigKey < ConfigError; end
+  class InvalidConfigValue < ConfigError; end
+end

data/lib/kafka/ffi/event.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require "kafka/ffi/opaque_pointer"
+
+module Kafka::FFI
+  class Event < OpaquePointer
+    # LogMessage is attached to RD_KAFKA_EVENT_LOG events.
+    LogMessage = Struct.new(:facility, :message, :level) do
+      # @attr facility [String] Log facility
+      # @attr message [String] Log message
+      # @attr level [Integer] Verbosity level of the message
+
+      def to_s
+        message
+      end
+    end
+
+    # Returns the event's type
+    #
+    # @see RD_KAFKA_EVENT_*
+    #
+    # @return [Symbol] Type of the event
+    def type
+      ::Kafka::FFI.rd_kafka_event_type(self)
+    end
+
+    # Returns the name of the event's type.
+    #
+    # @return [String] Name of the type of event
+    def name
+      ::Kafka::FFI.rd_kafka_event_name(self)
+    end
+
+    # Retrieve the set of messages. Can take a block to iterate over the set of
+    # Messages rather than return them.
+    #
+    # Events:
+    #   - RD_KAFKA_EVENT_FETCH
+    #   - RD_KAFKA_EVENT_DR
+    #
+    # @note Do not call #destroy on the Messages
+    #
+    # @yield [message] Iterate over available messages
+    # @yieldparam [Message]
+    #
+    # @return [Array<Message>] Messages attached to the Event
+    # @return [nil] Event does not have any Messages or a block was given.
+    def messages
+      # This departs from the librdkafka API due to having a collection of
+      # methods that have funky semantics for Ruby.
+
+      # @todo Messages are only on RD_KAFKA_EVENT_FETCH and RD_KAFKA_EVENT_DR.
+      #   Need to test what happens with other event types.
+
+      # No block so fetch all of the messages and return them as an array.
+      if !block_given?
+        count = ::Kafka::FFI.rd_kafka_event_message_count(self)
+        if count == 0
+          return []
+        end
+
+        begin
+          # Allocates enough memory for the full set but only converts as many
+          # as were returned.
+          # @todo Retrieve all until sum(ret) == count?
+          ptr = ::FFI::MemoryPointer.new(:pointer, count)
+          ret = ::Kafka::FFI.rd_kafka_event_message_array(self, ptr, count)
+
+          # Map the return pointers to Messages
+          return ptr.read_array_of_pointer(ret).map! { |p| Message.from_native(p) }
+        ensure
+          ptr.free
+        end
+      end
+
+      # Block was passed so use rd_kafka_event_message_next
+      begin
+        ptr = ::FFI::MemoryPointer.new(:pointer)
+
+        loop do
+          msg = ::Kafka::FFI.rd_kafka_event_message_next(self)
+          if msg.null?
+            break
+          end
+
+          yield(msg)
+        end
+      ensure
+        ptr.free
+      end
+    end
+
+    # Returns the configuration for the event or nil if the configuration
+    # property is not set or not applicable for the event type.
+    #
+    # Events:
+    #   - RD_KAFKA_EVENT_OAUTHBEARER_TOKEN_REFRESH
+    #
+    # @return [String] Configuration string for the event
+    # @return [nil] Property not set or not applicable.
+    def config_string
+      ::Kafka::FFI.rd_kafka_event_string(self)
+    end
+
+    # Returns the error code for the event or nil if there was no error.
+    #
+    # @see error_is_fatal to detect if it is a fatal error.
+    #
+    # @return [nil] No error for the Event
+    # @return [Kafka::ResponseError] Error code for the event.
+    def error
+      err = ::Kafka::FFI.rd_kafka_event_error(self)
+      if err != :ok
+        ::Kafka::ResponseError.new(err, error_string)
+      end
+    end
+
+    # Returns a description of the error or nil when there is no error.
+    #
+    # @return [nil] No error for the Event
+    # @return [String] Description of the error
+    def error_string
+      ::Kafka::FFI.rd_kafka_event_error_string(self)
+    end
+
+    # Returns true or false if the Event represents a fatal error.
+    #
+    # @return [Boolean] There is an error for the Event and it is fatal.
+    def error_is_fatal
+      error && ::Kafka::FFI.rd_kafka_event_error_is_fatal(self)
+    end
+    alias error_is_fatal? error_is_fatal
+
+    # Returns the log message attached to the event.
+    #
+    # Events:
+    #   - RD_KAFKA_EVENT_LOG
+    #
+    # @return [Event::LogMessage] Attach log entry
+    def log
+      facility = ::FFI::MemoryPointer.new(:pointer)
+      message  = ::FFI::MemoryPointer.new(:pointer)
+      level    = ::FFI::MemoryPointer.new(:pointer)
+
+      exists = ::Kafka::FFI.rd_kafka_event_log(self, facility, message, level)
+      if exists != 0
+        # Event type does not support log messages.
+        return nil
+      end
+
+      LogMessage.new(
+        facility.read_pointer.read_string,
+        message.read_pointer.read_string,
+        level.read_int,
+      )
+    ensure
+      facility.free
+      message.free
+      level.free
+    end
+
+    # Extracts stats from the event
+    #
+    # Events:
+    #   - RD_KAFKA_EVENT_STATS
+    #
+    # @return [nil] Event type does not support stats
+    # @return [String] JSON encoded stats information.
+    def stats
+      # Calling stats on an unsupported type causes a segfault with librdkafka
+      # 1.3.0.
+      if type != :stats
+        return nil
+      end
+
+      ::Kafka::FFI.rd_kafka_event_stats(self)
+    end
+
+    # Returns the topic partition list from the Event.
+    #
+    # @note Application MUST NOT call #destroy on the list
+    #
+    # Events:
+    #   - RD_KAFKA_EVENT_REBALANCE
+    #   - RD_KAFKA_EVENT_OFFSET_COMMIT
+    #
+    # @return [TopicPartitionList]
+    def topic_partition_list
+      ::Kafka::FFI.rd_kafka_event_topic_partition_list(self)
+    end
+
+    # Returns the topic partition from the Event.
+    #
+    # @note The application MUST call #destroy on the TopicPartition when done.
+    #
+    # Events:
+    #   - RD_KAFKA_EVENT_ERROR
+    #
+    # @return [TopicPartition]
+    def topic_partition
+      ::Kafka::FFI.rd_kafka_event_topic_partition(self)
+    end
+
+    # Destroy the event, releasing it's resources back to the system.
+    #
+    # @todo Is the applicaiton responsible for calling #destroy?
+    def destroy
+      # It is safe to call destroy even if the Event's pointer is NULL but it
+      # doesn't do anything so might as well guard against it just in case.
+      if !pointer.null?
+        ::Kafka::FFI.rd_kafka_event_destroy(self)
+      end
+    end
+  end
+end

data/lib/kafka/ffi/group_info.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "ffi"
+require "kafka/ffi/broker_metadata"
+require "kafka/ffi/group_member_info"
+
+module Kafka::FFI
+  class GroupInfo < ::FFI::Struct
+    layout(
+      :broker,        BrokerMetadata.by_value,
+      :group,         :string,
+      :err,           :error_code,
+      :state,         :string,
+      :protocol_type, :string,
+      :protocol,      :string,
+      :members,       :pointer,
+      :member_cnt,    :int
+    )
+
+    # Returns information about the broker that originated the group info.
+    #
+    # @return [Kafka::FFI::BrokerMetadata] Broker info
+    def broker
+      self[:broker]
+    end
+
+    # Returns the name of the group
+    #
+    # @return [String] Group name
+    def group
+      self[:group]
+    end
+    alias name group
+
+    # Returns any broker orignated error for the consumer group.
+    #
+    # @return [nil] No error
+    # @return [Kafka::ResponseError] Broker originated error
+    def error
+      if self[:err] != :ok
+        ::Kafka::ResponseError.new(self[:err])
+      end
+    end
+
+    # Returns the current state of the group
+    #
+    # @return [String] Group state
+    def state
+      self[:state]
+    end
+
+    # Returns the group protocol type
+    #
+    # @return [String] Group protocol type
+    def protocol_type
+      self[:protocol_type]
+    end
+
+    # Returns the group protocol
+    #
+    # @return [String] Group protocol
+    def protocol
+      self[:protocol]
+    end
+
+    # Returns information about the members of the consumer group
+    #
+    # @return [Array<GroupMemberInfo>] Member information
+    def members
+      self[:member_cnt].times.map do |i|
+        GroupMemberInfo.new(self[:members] + (i * GroupMemberInfo.size))
+      end
+    end
+  end
+end

data/lib/kafka/ffi/group_list.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "ffi"
+require "kafka/ffi/group_info"
+
+module Kafka::FFI
+  class GroupList < ::FFI::Struct
+    layout(
+      :groups,    :pointer,
+      :group_cnt, :int
+    )
+
+    # Returns information about the consumer groups in the cluster.
+    #
+    # @return [Array<GroupInfo>] Group metadata
+    def groups
+      self[:group_cnt].times.map do |i|
+        GroupInfo.new(self[:groups] + (i * GroupInfo.size))
+      end
+    end
+
+    # Release the resources used by the group list back to the system
+    def destroy
+      ::Kafka::FFI.rd_kafka_group_list_destroy(self)
+    end
+  end
+end