karafka-rdkafka 0.12.0

data/lib/rdkafka/config.rb

@@ -0,0 +1,299 @@
+require "logger"
+
+module Rdkafka
+  # Configuration for a Kafka consumer or producer. You can create an instance and use
+  # the consumer and producer methods to create a client. Documentation of the available
+  # configuration options is available on https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md.
+  class Config
+    # @private
+    @@logger = Logger.new(STDOUT)
+    # @private
+    @@statistics_callback = nil
+    # @private
+    @@error_callback = nil
+    # @private
+    @@opaques = {}
+    # @private
+    @@log_queue = Queue.new
+
+    Thread.start do
+      loop do
+        severity, msg = @@log_queue.pop
+        @@logger.add(severity, msg)
+      end
+    end
+
+    # Returns the current logger, by default this is a logger to stdout.
+    #
+    # @return [Logger]
+    def self.logger
+      @@logger
+    end
+
+
+    # Returns a queue whose contents will be passed to the configured logger. Each entry
+    # should follow the format [Logger::Severity, String]. The benefit over calling the
+    # logger directly is that this is safe to use from trap contexts.
+    #
+    # @return [Queue]
+    def self.log_queue
+      @@log_queue
+    end
+
+    # Set the logger that will be used for all logging output by this library.
+    #
+    # @param logger [Logger] The logger to be used
+    #
+    # @return [nil]
+    def self.logger=(logger)
+      raise NoLoggerError if logger.nil?
+      @@logger=logger
+    end
+
+    # Set a callback that will be called every time the underlying client emits statistics.
+    # You can configure if and how often this happens using `statistics.interval.ms`.
+    # The callback is called with a hash that's documented here: https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md
+    #
+    # @param callback [Proc, #call] The callback
+    #
+    # @return [nil]
+    def self.statistics_callback=(callback)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
+      @@statistics_callback = callback
+    end
+
+    # Returns the current statistics callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    def self.statistics_callback
+      @@statistics_callback
+    end
+
+    # Set a callback that will be called every time the underlying client emits an error.
+    # If this callback is not set, global errors such as brokers becoming unavailable will only be sent to the logger, as defined by librdkafka.
+    # The callback is called with an instance of RdKafka::Error.
+    #
+    # @param callback [Proc, #call] The callback
+    #
+    # @return [nil]
+    def self.error_callback=(callback)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
+      @@error_callback = callback
+    end
+
+    # Returns the current error callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    def self.error_callback
+      @@error_callback
+    end
+
+    # @private
+    def self.opaques
+      @@opaques
+    end
+
+    # Default config that can be overwritten.
+    DEFAULT_CONFIG = {
+      # Request api version so advanced features work
+      :"api.version.request" => true
+    }.freeze
+
+    # Required config that cannot be overwritten.
+    REQUIRED_CONFIG = {
+      # Enable log queues so we get callbacks in our own Ruby threads
+      :"log.queue" => true
+    }.freeze
+
+    # Returns a new config with the provided options which are merged with {DEFAULT_CONFIG}.
+    #
+    # @param config_hash [Hash{String,Symbol => String}] The config options for rdkafka
+    #
+    # @return [Config]
+    def initialize(config_hash = {})
+      @config_hash = DEFAULT_CONFIG.merge(config_hash)
+      @consumer_rebalance_listener = nil
+    end
+
+    # Set a config option.
+    #
+    # @param key [String] The config option's key
+    # @param value [String] The config option's value
+    #
+    # @return [nil]
+    def []=(key, value)
+      @config_hash[key] = value
+    end
+
+    # Get a config option with the specified key
+    #
+    # @param key [String] The config option's key
+    #
+    # @return [String, nil] The config option or `nil` if it is not present
+    def [](key)
+      @config_hash[key]
+    end
+
+    # Get notifications on partition assignment/revocation for the subscribed topics
+    #
+    # @param listener [Object, #on_partitions_assigned, #on_partitions_revoked] listener instance
+    def consumer_rebalance_listener=(listener)
+      @consumer_rebalance_listener = listener
+    end
+
+    # Create a consumer with this configuration.
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    # @raise [ClientCreationError] When the native client cannot be created
+    #
+    # @return [Consumer] The created consumer
+    def consumer
+      opaque = Opaque.new
+      config = native_config(opaque)
+
+      if @consumer_rebalance_listener
+        opaque.consumer_rebalance_listener = @consumer_rebalance_listener
+        Rdkafka::Bindings.rd_kafka_conf_set_rebalance_cb(config, Rdkafka::Bindings::RebalanceCallback)
+      end
+
+      kafka = native_kafka(config, :rd_kafka_consumer)
+
+      # Redirect the main queue to the consumer
+      Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
+
+      # Return consumer with Kafka client
+      Rdkafka::Consumer.new(kafka)
+    end
+
+    # Create a producer with this configuration.
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    # @raise [ClientCreationError] When the native client cannot be created
+    #
+    # @return [Producer] The created producer
+    def producer
+      # Create opaque
+      opaque = Opaque.new
+      # Create Kafka config
+      config = native_config(opaque)
+      # Set callback to receive delivery reports on config
+      Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
+      # Return producer with Kafka client
+      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer)), self[:partitioner]).tap do |producer|
+        opaque.producer = producer
+      end
+    end
+
+    # Create an admin instance with this configuration.
+    #
+    # @raise [ConfigError] When the configuration contains invalid options
+    # @raise [ClientCreationError] When the native client cannot be created
+    #
+    # @return [Admin] The created admin instance
+    def admin
+      opaque = Opaque.new
+      config = native_config(opaque)
+      Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
+      Rdkafka::Admin.new(native_kafka(config, :rd_kafka_producer))
+    end
+
+    # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.
+    class ConfigError < RuntimeError; end
+
+    # Error that is returned by the underlying rdkafka library if the client cannot be created.
+    class ClientCreationError < RuntimeError; end
+
+    # Error that is raised when trying to set a nil logger
+    class NoLoggerError < RuntimeError; end
+
+    private
+
+    # This method is only intended to be used to create a client,
+    # using it in another way will leak memory.
+    def native_config(opaque=nil)
+      Rdkafka::Bindings.rd_kafka_conf_new.tap do |config|
+        # Create config
+        @config_hash.merge(REQUIRED_CONFIG).each do |key, value|
+          error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+          result = Rdkafka::Bindings.rd_kafka_conf_set(
+            config,
+            key.to_s,
+            value.to_s,
+            error_buffer,
+            256
+          )
+          unless result == :config_ok
+            raise ConfigError.new(error_buffer.read_string)
+          end
+        end
+
+        # Set opaque pointer that's used as a proxy for callbacks
+        if opaque
+          pointer = ::FFI::Pointer.new(:pointer, opaque.object_id)
+          Rdkafka::Bindings.rd_kafka_conf_set_opaque(config, pointer)
+
+          # Store opaque with the pointer as key. We use this approach instead
+          # of trying to convert the pointer to a Ruby object because there is
+          # no risk of a segfault this way.
+          Rdkafka::Config.opaques[pointer.to_i] = opaque
+        end
+
+        # Set log callback
+        Rdkafka::Bindings.rd_kafka_conf_set_log_cb(config, Rdkafka::Bindings::LogCallback)
+
+        # Set stats callback
+        Rdkafka::Bindings.rd_kafka_conf_set_stats_cb(config, Rdkafka::Bindings::StatsCallback)
+
+        # Set error callback
+        Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
+      end
+    end
+
+    def native_kafka(config, type)
+      error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+      handle = Rdkafka::Bindings.rd_kafka_new(
+        type,
+        config,
+        error_buffer,
+        256
+      )
+
+      if handle.null?
+        raise ClientCreationError.new(error_buffer.read_string)
+      end
+
+      # Redirect log to handle's queue
+      Rdkafka::Bindings.rd_kafka_set_log_queue(
+        handle,
+        Rdkafka::Bindings.rd_kafka_queue_get_main(handle)
+      )
+
+      # Return handle which should be closed using rd_kafka_destroy after usage.
+      handle
+    end
+  end
+
+  # @private
+  class Opaque
+    attr_accessor :producer
+    attr_accessor :consumer_rebalance_listener
+
+    def call_delivery_callback(delivery_report, delivery_handle)
+      producer.call_delivery_callback(delivery_report, delivery_handle) if producer
+    end
+
+    def call_on_partitions_assigned(consumer, list)
+      return unless consumer_rebalance_listener
+      return unless consumer_rebalance_listener.respond_to?(:on_partitions_assigned)
+
+      consumer_rebalance_listener.on_partitions_assigned(consumer, list)
+    end
+
+    def call_on_partitions_revoked(consumer, list)
+      return unless consumer_rebalance_listener
+      return unless consumer_rebalance_listener.respond_to?(:on_partitions_revoked)
+
+      consumer_rebalance_listener.on_partitions_revoked(consumer, list)
+    end
+  end
+end

data/lib/rdkafka/consumer/headers.rb

@@ -0,0 +1,63 @@
+module Rdkafka
+  class Consumer
+    # A message headers
+    class Headers
+      # Reads a native kafka's message header into ruby's hash
+      #
+      # @return [Hash<String, String>] a message headers
+      #
+      # @raise [Rdkafka::RdkafkaError] when fail to read headers
+      #
+      # @private
+      def self.from_native(native_message)
+        headers_ptrptr = FFI::MemoryPointer.new(:pointer)
+        err = Rdkafka::Bindings.rd_kafka_message_headers(native_message, headers_ptrptr)
+
+        if err == Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
+          return {}
+        elsif err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+          raise Rdkafka::RdkafkaError.new(err, "Error reading message headers")
+        end
+
+        headers_ptr = headers_ptrptr.read_pointer
+
+        name_ptrptr = FFI::MemoryPointer.new(:pointer)
+        value_ptrptr = FFI::MemoryPointer.new(:pointer)
+        size_ptr = Rdkafka::Bindings::SizePtr.new
+        headers = {}
+
+        idx = 0
+        loop do
+          err = Rdkafka::Bindings.rd_kafka_header_get_all(
+            headers_ptr,
+            idx,
+            name_ptrptr,
+            value_ptrptr,
+            size_ptr
+          )
+
+          if err == Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
+            break
+          elsif err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            raise Rdkafka::RdkafkaError.new(err, "Error reading a message header at index #{idx}")
+          end
+
+          name_ptr = name_ptrptr.read_pointer
+          name = name_ptr.respond_to?(:read_string_to_null) ? name_ptr.read_string_to_null : name_ptr.read_string
+
+          size = size_ptr[:value]
+
+          value_ptr = value_ptrptr.read_pointer
+
+          value = value_ptr.read_string(size)
+
+          headers[name.to_sym] = value
+
+          idx += 1
+        end
+
+        headers
+      end
+    end
+  end
+end

data/lib/rdkafka/consumer/message.rb

@@ -0,0 +1,84 @@
+module Rdkafka
+  class Consumer
+    # A message that was consumed from a topic.
+    class Message
+      # The topic this message was consumed from
+      # @return [String]
+      attr_reader :topic
+
+      # The partition this message was consumed from
+      # @return [Integer]
+      attr_reader :partition
+
+      # This message's payload
+      # @return [String, nil]
+      attr_reader :payload
+
+      # This message's key
+      # @return [String, nil]
+      attr_reader :key
+
+      # This message's offset in it's partition
+      # @return [Integer]
+      attr_reader :offset
+
+      # This message's timestamp, if provided by the broker
+      # @return [Time, nil]
+      attr_reader :timestamp
+
+      # @return [Hash<String, String>] a message headers
+      attr_reader :headers
+
+      # @private
+      def initialize(native_message)
+        # Set topic
+        unless native_message[:rkt].null?
+          @topic = Rdkafka::Bindings.rd_kafka_topic_name(native_message[:rkt])
+        end
+        # Set partition
+        @partition = native_message[:partition]
+        # Set payload
+        unless native_message[:payload].null?
+          @payload = native_message[:payload].read_string(native_message[:len])
+        end
+        # Set key
+        unless native_message[:key].null?
+          @key = native_message[:key].read_string(native_message[:key_len])
+        end
+        # Set offset
+        @offset = native_message[:offset]
+        # Set timestamp
+        raw_timestamp = Rdkafka::Bindings.rd_kafka_message_timestamp(native_message, nil)
+        @timestamp = if raw_timestamp && raw_timestamp > -1
+                       # Calculate seconds and microseconds
+                       seconds = raw_timestamp / 1000
+                       milliseconds = (raw_timestamp - seconds * 1000) * 1000
+                       Time.at(seconds, milliseconds)
+                     else
+                       nil
+                     end
+
+        @headers = Headers.from_native(native_message)
+      end
+
+      # Human readable representation of this message.
+      # @return [String]
+      def to_s
+        is_headers = @headers.empty? ? "" : ", headers #{headers.size}"
+
+        "<Message in '#{topic}' with key '#{truncate(key)}', payload '#{truncate(payload)}', partition #{partition}, offset #{offset}, timestamp #{timestamp}#{is_headers}>"
+      end
+
+      def truncate(string)
+        if string && string.length > 40
+          "#{string[0..39]}..."
+        else
+          string
+        end
+      end
+
+      private
+
+    end
+  end
+end

data/lib/rdkafka/consumer/partition.rb

@@ -0,0 +1,49 @@
+module Rdkafka
+  class Consumer
+    # Information about a partition, used in {TopicPartitionList}.
+    class Partition
+      # Partition number
+      # @return [Integer]
+      attr_reader :partition
+
+      # Partition's offset
+      # @return [Integer, nil]
+      attr_reader :offset
+
+      # Partition's error code
+      # @return [Integer]
+      attr_reader :err
+
+      # @private
+      def initialize(partition, offset, err = 0)
+        @partition = partition
+        @offset = offset
+        @err = err
+      end
+
+      # Human readable representation of this partition.
+      # @return [String]
+      def to_s
+        message = "<Partition #{partition}"
+        message += " offset=#{offset}" if offset
+        message += " err=#{err}" if err != 0
+        message += ">"
+        message
+      end
+
+      # Human readable representation of this partition.
+      # @return [String]
+      def inspect
+        to_s
+      end
+
+      # Whether another partition is equal to this
+      # @return [Boolean]
+      def ==(other)
+        self.class == other.class &&
+          self.partition == other.partition &&
+          self.offset == other.offset
+      end
+    end
+  end
+end

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -0,0 +1,164 @@
+module Rdkafka
+  class Consumer
+    # A list of topics with their partition information
+    class TopicPartitionList
+      # Create a topic partition list.
+      #
+      # @param data [Hash{String => nil,Partition}] The topic and partition data or nil to create an empty list
+      #
+      # @return [TopicPartitionList]
+      def initialize(data=nil)
+        @data = data || {}
+      end
+
+      # Number of items in the list
+      # @return [Integer]
+      def count
+        i = 0
+        @data.each do |_topic, partitions|
+          if partitions
+            i += partitions.count
+          else
+            i+= 1
+          end
+        end
+        i
+      end
+
+      # Whether this list is empty
+      # @return [Boolean]
+      def empty?
+        @data.empty?
+      end
+
+      # Add a topic with optionally partitions to the list.
+      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
+      #
+      # @example Add a topic with unassigned partitions
+      #   tpl.add_topic("topic")
+      #
+      # @example Add a topic with assigned partitions
+      #   tpl.add_topic("topic", (0..8))
+      #
+      # @example Add a topic with all topics up to a count
+      #   tpl.add_topic("topic", 9)
+      #
+      # @param topic [String] The topic's name
+      # @param partitions [Array<Integer>, Range<Integer>, Integer] The topic's partitions or partition count
+      #
+      # @return [nil]
+      def add_topic(topic, partitions=nil)
+        if partitions.nil?
+          @data[topic.to_s] = nil
+        else
+          if partitions.is_a? Integer
+            partitions = (0..partitions - 1)
+          end
+          @data[topic.to_s] = partitions.map { |p| Partition.new(p, nil, 0) }
+        end
+      end
+
+      # Add a topic with partitions and offsets set to the list
+      # Calling this method multiple times for the same topic will overwrite the previous configuraton.
+      #
+      # @param topic [String] The topic's name
+      # @param partitions_with_offsets [Hash<Integer, Integer>] The topic's partitions and offsets
+      #
+      # @return [nil]
+      def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
+        @data[topic.to_s] = partitions_with_offsets.map { |p, o| Partition.new(p, o) }
+      end
+
+      # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
+      #
+      # @return [Hash{String => Array<Partition>,nil}]
+      def to_h
+        @data
+      end
+
+      # Human readable representation of this list.
+      # @return [String]
+      def to_s
+        "<TopicPartitionList: #{to_h}>"
+      end
+
+      def ==(other)
+        self.to_h == other.to_h
+      end
+
+      # Create a new topic partition list based of a native one.
+      #
+      # @param pointer [FFI::Pointer] Optional pointer to an existing native list. Its contents will be copied.
+      #
+      # @return [TopicPartitionList]
+      #
+      # @private
+      def self.from_native_tpl(pointer)
+        # Data to be moved into the tpl
+        data = {}
+
+        # Create struct and copy its contents
+        native_tpl = Rdkafka::Bindings::TopicPartitionList.new(pointer)
+        native_tpl[:cnt].times do |i|
+          ptr = native_tpl[:elems] + (i * Rdkafka::Bindings::TopicPartition.size)
+          elem = Rdkafka::Bindings::TopicPartition.new(ptr)
+          if elem[:partition] == -1
+            data[elem[:topic]] = nil
+          else
+            partitions = data[elem[:topic]] || []
+            offset = if elem[:offset] == Rdkafka::Bindings::RD_KAFKA_OFFSET_INVALID
+                       nil
+                     else
+                       elem[:offset]
+                     end
+            partition = Partition.new(elem[:partition], offset, elem[:err])
+            partitions.push(partition)
+            data[elem[:topic]] = partitions
+          end
+        end
+
+        # Return the created object
+        TopicPartitionList.new(data)
+      end
+
+      # Create a native tpl with the contents of this object added.
+      #
+      # The pointer will be cleaned by `rd_kafka_topic_partition_list_destroy` when GC releases it.
+      #
+      # @return [FFI::Pointer]
+      # @private
+      def to_native_tpl
+        tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(count)
+
+        @data.each do |topic, partitions|
+          if partitions
+            partitions.each do |p|
+              Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+                tpl,
+                topic,
+                p.partition
+              )
+
+              if p.offset
+                Rdkafka::Bindings.rd_kafka_topic_partition_list_set_offset(
+                  tpl,
+                  topic,
+                  p.partition,
+                  p.offset
+                )
+              end
+            end
+          else
+            Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+              tpl,
+              topic,
+              -1
+            )
+          end
+        end
+
+        tpl
+      end
+    end
+  end
+end