karafka-rdkafka 0.12.0

data/lib/rdkafka/metadata.rb

@@ -0,0 +1,92 @@
+module Rdkafka
+  class Metadata
+    attr_reader :brokers, :topics
+
+    def initialize(native_client, topic_name = nil)
+      native_topic = if topic_name
+        Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
+      end
+
+      ptr = FFI::MemoryPointer.new(:pointer)
+
+      # If topic_flag is 1, we request info about *all* topics in the cluster.  If topic_flag is 0,
+      # we only request info about locally known topics (or a single topic if one is passed in).
+      topic_flag = topic_name.nil? ? 1 : 0
+
+      # Retrieve the Metadata
+      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 250)
+
+      # Error Handling
+      raise Rdkafka::RdkafkaError.new(result) unless result.zero?
+
+      metadata_from_native(ptr.read_pointer)
+    ensure
+      Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
+      Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)
+    end
+
+    private
+
+    def metadata_from_native(ptr)
+      metadata = Metadata.new(ptr)
+      @brokers = Array.new(metadata[:brokers_count]) do |i|
+        BrokerMetadata.new(metadata[:brokers_metadata] + (i * BrokerMetadata.size)).to_h
+      end
+
+      @topics = Array.new(metadata[:topics_count]) do |i|
+        topic = TopicMetadata.new(metadata[:topics_metadata] + (i * TopicMetadata.size))
+        raise Rdkafka::RdkafkaError.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
+
+        partitions = Array.new(topic[:partition_count]) do |j|
+          partition = PartitionMetadata.new(topic[:partitions_metadata] + (j * PartitionMetadata.size))
+          raise Rdkafka::RdkafkaError.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
+          partition.to_h
+        end
+        topic.to_h.merge!(partitions: partitions)
+      end
+    end
+
+    class CustomFFIStruct < FFI::Struct
+      def to_h
+        members.each_with_object({}) do |mem, hsh|
+          val = self.[](mem)
+          next if val.is_a?(FFI::Pointer) || mem == :rd_kafka_resp_err
+
+          hsh[mem] = self.[](mem)
+        end
+      end
+    end
+
+    class Metadata < CustomFFIStruct
+      layout :brokers_count, :int,
+             :brokers_metadata, :pointer,
+             :topics_count, :int,
+             :topics_metadata, :pointer,
+             :broker_id, :int32,
+             :broker_name, :string
+    end
+
+    class BrokerMetadata < CustomFFIStruct
+      layout :broker_id, :int32,
+             :broker_name, :string,
+             :broker_port, :int
+    end
+
+    class TopicMetadata < CustomFFIStruct
+      layout :topic_name, :string,
+             :partition_count, :int,
+             :partitions_metadata, :pointer,
+             :rd_kafka_resp_err, :int
+    end
+
+    class PartitionMetadata < CustomFFIStruct
+      layout :partition_id, :int32,
+             :rd_kafka_resp_err, :int,
+             :leader, :int32,
+             :replica_count, :int,
+             :replicas, :pointer,
+             :in_sync_replica_brokers, :int,
+             :isrs, :pointer
+    end
+  end
+end

data/lib/rdkafka/producer/client.rb

@@ -0,0 +1,47 @@
+module Rdkafka
+  class Producer
+    class Client
+      def initialize(native)
+        @native = native
+
+        # Start thread to poll client for delivery callbacks
+        @polling_thread = Thread.new do
+          loop do
+            Rdkafka::Bindings.rd_kafka_poll(native, 250)
+            # Exit thread if closing and the poll queue is empty
+            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(native) == 0
+              break
+            end
+          end
+        end
+        @polling_thread.abort_on_exception = true
+        @polling_thread[:closing] = false
+      end
+
+      def native
+        @native
+      end
+
+      def finalizer
+        ->(_) { close }
+      end
+
+      def closed?
+        @native.nil?
+      end
+
+      def close(object_id=nil)
+        return unless @native
+
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+        # Wait for the polling thread to finish up
+        @polling_thread.join
+
+        Rdkafka::Bindings.rd_kafka_destroy(@native)
+
+        @native = nil
+      end
+    end
+  end
+end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -0,0 +1,22 @@
+module Rdkafka
+  class Producer
+    # Handle to wait for a delivery report which is returned when
+    # producing a message.
+    class DeliveryHandle < Rdkafka::AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :partition, :int,
+             :offset, :int64
+
+      # @return [String] the name of the operation (e.g. "delivery")
+      def operation_name
+        "delivery"
+      end
+
+      # @return [DeliveryReport] a report on the delivery of the message
+      def create_result
+        DeliveryReport.new(self[:partition], self[:offset])
+      end
+    end
+  end
+end

data/lib/rdkafka/producer/delivery_report.rb

@@ -0,0 +1,26 @@
+module Rdkafka
+  class Producer
+    # Delivery report for a successfully produced message.
+    class DeliveryReport
+      # The partition this message was produced to.
+      # @return [Integer]
+      attr_reader :partition
+
+      # The offset of the produced message.
+      # @return [Integer]
+      attr_reader :offset
+
+      # Error in case happen during produce.
+      # @return [String]
+      attr_reader :error
+
+      private
+
+      def initialize(partition, offset, error = nil)
+        @partition = partition
+        @offset = offset
+        @error = error
+      end
+    end
+  end
+end

data/lib/rdkafka/producer.rb

@@ -0,0 +1,178 @@
+require "objspace"
+
+module Rdkafka
+  # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
+  class Producer
+    # @private
+    # Returns the current delivery callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    attr_reader :delivery_callback
+
+    # @private
+    # Returns the number of arguments accepted by the callback, by default this is nil.
+    #
+    # @return [Integer, nil]
+    attr_reader :delivery_callback_arity
+
+    # @private
+    def initialize(client, partitioner_name)
+      @client = client
+      @partitioner_name = partitioner_name || "consistent_random"
+
+      # Makes sure, that the producer gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(self, client.finalizer)
+    end
+
+    # Set a callback that will be called every time a message is successfully produced.
+    # The callback is called with a {DeliveryReport} and {DeliveryHandle}
+    #
+    # @param callback [Proc, #call] The callback
+    #
+    # @return [nil]
+    def delivery_callback=(callback)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
+      @delivery_callback = callback
+      @delivery_callback_arity = arity(callback)
+    end
+
+    # Close this producer and wait for the internal poll queue to empty.
+    def close
+      ObjectSpace.undefine_finalizer(self)
+
+      @client.close
+    end
+
+    # Partition count for a given topic.
+    # NOTE: If 'allow.auto.create.topics' is set to true in the broker, the topic will be auto-created after returning nil.
+    #
+    # @param topic [String] The topic name.
+    #
+    # @return partition count [Integer,nil]
+    #
+    def partition_count(topic)
+      closed_producer_check(__method__)
+      Rdkafka::Metadata.new(@client.native, topic).topics&.first[:partition_count]
+    end
+
+    # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
+    #
+    # When no partition is specified the underlying Kafka library picks a partition based on the key. If no key is specified, a random partition will be used.
+    # When a timestamp is provided this is used instead of the auto-generated timestamp.
+    #
+    # @param topic [String] The topic to produce to
+    # @param payload [String,nil] The message's payload
+    # @param key [String, nil] The message's key
+    # @param partition [Integer,nil] Optional partition to produce to
+    # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
+    # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
+    # @param headers [Hash<String,String>] Optional message headers
+    #
+    # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
+    #
+    # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
+    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
+      closed_producer_check(__method__)
+
+      # Start by checking and converting the input
+
+      # Get payload length
+      payload_size = if payload.nil?
+                       0
+                     else
+                       payload.bytesize
+                     end
+
+      # Get key length
+      key_size = if key.nil?
+                   0
+                 else
+                   key.bytesize
+                 end
+
+      if partition_key
+        partition_count = partition_count(topic)
+        # If the topic is not present, set to -1
+        partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count
+      end
+
+      # If partition is nil, use -1 to let librdafka set the partition randomly or
+      # based on the key when present.
+      partition ||= -1
+
+      # If timestamp is nil use 0 and let Kafka set one. If an integer or time
+      # use it.
+      raw_timestamp = if timestamp.nil?
+                        0
+                      elsif timestamp.is_a?(Integer)
+                        timestamp
+                      elsif timestamp.is_a?(Time)
+                        (timestamp.to_i * 1000) + (timestamp.usec / 1000)
+                      else
+                        raise TypeError.new("Timestamp has to be nil, an Integer or a Time")
+                      end
+
+      delivery_handle = DeliveryHandle.new
+      delivery_handle[:pending] = true
+      delivery_handle[:response] = -1
+      delivery_handle[:partition] = -1
+      delivery_handle[:offset] = -1
+      DeliveryHandle.register(delivery_handle)
+
+      args = [
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_PARTITION, :int32, partition,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TIMESTAMP, :int64, raw_timestamp,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_OPAQUE, :pointer, delivery_handle,
+      ]
+
+      if headers
+        headers.each do |key0, value0|
+          key = key0.to_s
+          value = value0.to_s
+          args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+          args << :string << key
+          args << :pointer << value
+          args << :size_t << value.bytes.size
+        end
+      end
+
+      args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
+
+      # Produce the message
+      response = Rdkafka::Bindings.rd_kafka_producev(
+        @client.native,
+        *args
+      )
+
+      # Raise error if the produce call was not successful
+      if response != 0
+        DeliveryHandle.remove(delivery_handle.to_ptr.address)
+        raise RdkafkaError.new(response)
+      end
+
+      delivery_handle
+    end
+
+    # @private
+    def call_delivery_callback(delivery_report, delivery_handle)
+      return unless @delivery_callback
+
+      args = [delivery_report, delivery_handle].take(@delivery_callback_arity)
+      @delivery_callback.call(*args)
+    end
+
+    def arity(callback)
+      return callback.arity if callback.respond_to?(:arity)
+
+      callback.method(:call).arity
+    end
+
+    def closed_producer_check(method)
+      raise Rdkafka::ClosedProducerError.new(method) if @client.closed?
+    end
+  end
+end

data/lib/rdkafka/version.rb

@@ -0,0 +1,5 @@
+module Rdkafka
+  VERSION = "0.12.0"
+  LIBRDKAFKA_VERSION = "1.9.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "59b6088b69ca6cf278c3f9de5cd6b7f3fd604212cd1c59870bc531c54147e889"
+end

data/lib/rdkafka.rb

@@ -0,0 +1,22 @@
+require "rdkafka/version"
+
+require "rdkafka/abstract_handle"
+require "rdkafka/admin"
+require "rdkafka/admin/create_topic_handle"
+require "rdkafka/admin/create_topic_report"
+require "rdkafka/admin/delete_topic_handle"
+require "rdkafka/admin/delete_topic_report"
+require "rdkafka/bindings"
+require "rdkafka/callbacks"
+require "rdkafka/config"
+require "rdkafka/consumer"
+require "rdkafka/consumer/headers"
+require "rdkafka/consumer/message"
+require "rdkafka/consumer/partition"
+require "rdkafka/consumer/topic_partition_list"
+require "rdkafka/error"
+require "rdkafka/metadata"
+require "rdkafka/producer"
+require "rdkafka/producer/client"
+require "rdkafka/producer/delivery_handle"
+require "rdkafka/producer/delivery_report"

data/rdkafka.gemspec

@@ -0,0 +1,36 @@
+require File.expand_path('lib/rdkafka/version', __dir__)
+
+Gem::Specification.new do |gem|
+  gem.authors = ['Thijs Cadier']
+  gem.email = ["thijs@appsignal.com"]
+  gem.description = "Modern Kafka client library for Ruby based on librdkafka"
+  gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.4+."
+  gem.license = 'MIT'
+  gem.homepage = 'https://github.com/thijsc/rdkafka-ruby'
+
+  gem.files = `git ls-files`.split($\)
+  gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name = 'karafka-rdkafka'
+  gem.require_paths = ['lib']
+  gem.version = Rdkafka::VERSION
+  gem.required_ruby_version = '>= 2.6'
+  gem.extensions = %w(ext/Rakefile)
+
+  if $PROGRAM_NAME.end_with?('gem')
+    gem.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
+  end
+
+  gem.cert_chain    = %w[certs/cert_chain.pem]
+
+  gem.add_dependency 'ffi', '~> 1.15'
+  gem.add_dependency 'mini_portile2', '~> 2.6'
+  gem.add_dependency 'rake', '> 12'
+
+  gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'rspec', '~> 3.5'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'guard'
+  gem.add_development_dependency 'guard-rspec'
+end

data/spec/rdkafka/abstract_handle_spec.rb

@@ -0,0 +1,113 @@
+require "spec_helper"
+
+describe Rdkafka::AbstractHandle do
+  let(:response) { 0 }
+  let(:result) { -1 }
+
+  context "A subclass that does not implement the required methods" do
+
+    class BadTestHandle < Rdkafka::AbstractHandle
+      layout :pending, :bool,
+             :response, :int
+    end
+
+    it "raises an exception if operation_name is called" do
+      expect {
+        BadTestHandle.new.operation_name
+      }.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
+    end
+
+    it "raises an exception if create_result is called" do
+      expect {
+        BadTestHandle.new.create_result
+      }.to raise_exception(RuntimeError, /Must be implemented by subclass!/)
+    end
+  end
+
+  class TestHandle < Rdkafka::AbstractHandle
+    layout :pending, :bool,
+           :response, :int,
+           :result, :int
+
+    def operation_name
+      "test_operation"
+    end
+
+    def create_result
+      self[:result]
+    end
+  end
+
+  subject do
+    TestHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:result] = result
+    end
+  end
+
+  describe ".register and .remove" do
+    let(:pending_handle) { true }
+
+    it "should register and remove a delivery handle" do
+      Rdkafka::AbstractHandle.register(subject)
+      removed = Rdkafka::AbstractHandle.remove(subject.to_ptr.address)
+      expect(removed).to eq subject
+      expect(Rdkafka::AbstractHandle::REGISTRY).to be_empty
+    end
+  end
+
+  describe "#pending?" do
+    context "when true" do
+      let(:pending_handle) { true }
+
+      it "should be true" do
+        expect(subject.pending?).to be true
+      end
+    end
+
+    context "when not true" do
+      let(:pending_handle) { false }
+
+      it "should be false" do
+        expect(subject.pending?).to be false
+      end
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+      let(:result) { 1 }
+
+      it "should return a result" do
+        wait_result = subject.wait
+        expect(wait_result).to eq(result)
+      end
+
+      it "should wait without a timeout" do
+        wait_result = subject.wait(max_wait_timeout: nil)
+        expect(wait_result).to eq(result)
+      end
+    end
+
+    context "when not pending anymore and there was an error" do
+      let(:pending_handle) { false }
+      let(:response) { 20 }
+
+      it "should raise an rdkafka error" do
+        expect {
+          subject.wait
+        }.to raise_error Rdkafka::RdkafkaError
+      end
+    end
+  end
+end

data/spec/rdkafka/admin/create_topic_handle_spec.rb

@@ -0,0 +1,52 @@
+require "spec_helper"
+
+describe Rdkafka::Admin::CreateTopicHandle do
+  let(:response) { 0 }
+
+  subject do
+    Rdkafka::Admin::CreateTopicHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:error_string] = FFI::Pointer::NULL
+      handle[:result_name] = FFI::MemoryPointer.from_string("my-test-topic")
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::CreateTopicHandle::WaitTimeoutError, /create topic/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a create topic report" do
+        report = subject.wait
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    it "should raise the appropriate error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+    end
+  end
+end

data/spec/rdkafka/admin/create_topic_report_spec.rb

@@ -0,0 +1,16 @@
+require "spec_helper"
+
+describe Rdkafka::Admin::CreateTopicReport do
+  subject { Rdkafka::Admin::CreateTopicReport.new(
+      FFI::MemoryPointer.from_string("error string"),
+      FFI::MemoryPointer.from_string("result name")
+  )}
+
+  it "should get the error string" do
+    expect(subject.error_string).to eq("error string")
+  end
+
+  it "should get the result name" do
+    expect(subject.result_name).to eq("result name")
+  end
+end

data/spec/rdkafka/admin/delete_topic_handle_spec.rb

@@ -0,0 +1,52 @@
+require "spec_helper"
+
+describe Rdkafka::Admin::DeleteTopicHandle do
+  let(:response) { 0 }
+
+  subject do
+    Rdkafka::Admin::DeleteTopicHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:error_string] = FFI::Pointer::NULL
+      handle[:result_name] = FFI::MemoryPointer.from_string("my-test-topic")
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::DeleteTopicHandle::WaitTimeoutError, /delete topic/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a delete topic report" do
+        report = subject.wait
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.error_string).to eq(nil)
+        expect(report.result_name).to eq("my-test-topic")
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    it "should raise the appropriate error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+    end
+  end
+end

data/spec/rdkafka/admin/delete_topic_report_spec.rb

@@ -0,0 +1,16 @@
+require "spec_helper"
+
+describe Rdkafka::Admin::DeleteTopicReport do
+  subject { Rdkafka::Admin::DeleteTopicReport.new(
+      FFI::MemoryPointer.from_string("error string"),
+      FFI::MemoryPointer.from_string("result name")
+  )}
+
+  it "should get the error string" do
+    expect(subject.error_string).to eq("error string")
+  end
+
+  it "should get the result name" do
+    expect(subject.result_name).to eq("result name")
+  end
+end