karafka-rdkafka 0.12.4 → 0.13.0.beta1

data/lib/rdkafka/error.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   # Base error class.
   class BaseError < RuntimeError; end
@@ -83,4 +85,11 @@ module Rdkafka
       super("Illegal call to #{method.to_s} on a closed producer")
     end
   end
+
+  # Error class for public consumer method calls on a closed admin.
+  class ClosedAdminError < BaseError
+    def initialize(method)
+      super("Illegal call to #{method.to_s} on a closed admin")
+    end
+  end
 end

data/lib/rdkafka/metadata.rb

@@ -1,8 +1,21 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Metadata
     attr_reader :brokers, :topics
 
-    def initialize(native_client, topic_name = nil)
+    # Errors upon which we retry the metadata fetch
+    RETRIED_ERRORS = %i[
+      timed_out
+      leader_not_available
+    ].freeze
+
+    private_constant :RETRIED_ERRORS
+
+    def initialize(native_client, topic_name = nil, timeout_ms = 2_000)
+      attempt ||= 0
+      attempt += 1
+
       native_topic = if topic_name
         Rdkafka::Bindings.rd_kafka_topic_new(native_client, topic_name, nil)
       end
@@ -14,12 +27,22 @@ module Rdkafka
       topic_flag = topic_name.nil? ? 1 : 0
 
       # Retrieve the Metadata
-      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, 2_000)
+      result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, timeout_ms)
 
       # Error Handling
       raise Rdkafka::RdkafkaError.new(result) unless result.zero?
 
       metadata_from_native(ptr.read_pointer)
+    rescue ::Rdkafka::RdkafkaError => e
+      raise unless RETRIED_ERRORS.include?(e.code)
+      raise if attempt > 10
+
+      backoff_factor = 2**attempt
+      timeout = backoff_factor * 0.1
+
+      sleep(timeout)
+
+      retry
     ensure
       Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic) if topic_name
       Rdkafka::Bindings.rd_kafka_metadata_destroy(ptr.read_pointer)

data/lib/rdkafka/native_kafka.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # @private
+  # A wrapper around a native kafka that polls and cleanly exits
+  class NativeKafka
+    def initialize(inner, run_polling_thread:)
+      @inner = inner
+      # Lock around external access
+      @access_mutex = Mutex.new
+      # Lock around internal polling
+      @poll_mutex = Mutex.new
+
+      if run_polling_thread
+        # Start thread to poll client for delivery callbacks,
+        # not used in consumer.
+        @polling_thread = Thread.new do
+          loop do
+            @poll_mutex.synchronize do
+              Rdkafka::Bindings.rd_kafka_poll(inner, 100)
+            end
+
+            # Exit thread if closing and the poll queue is empty
+            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
+              break
+            end
+          end
+        end
+
+        @polling_thread.abort_on_exception = true
+        @polling_thread[:closing] = false
+      end
+
+      @closing = false
+    end
+
+    def with_inner
+      return if @inner.nil?
+
+      @access_mutex.synchronize do
+        yield @inner
+      end
+    end
+
+    def finalizer
+      ->(_) { close }
+    end
+
+    def closed?
+      @closing || @inner.nil?
+    end
+
+    def close(object_id=nil)
+      return if closed?
+
+      @access_mutex.lock
+
+      # Indicate to the outside world that we are closing
+      @closing = true
+
+      if @polling_thread
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+
+        # Wait for the polling thread to finish up,
+        # this can be aborted in practice if this
+        # code runs from a finalizer.
+        @polling_thread.join
+      end
+
+      # Destroy the client after locking both mutexes
+      @poll_mutex.lock
+
+      # This check prevents a race condition, where we would enter the close in two threads
+      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+      # and would continue to run, trying to destroy inner twice
+      return unless @inner
+
+      Rdkafka::Bindings.rd_kafka_destroy(@inner)
+      @inner = nil
+    end
+  end
+end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Producer
     # Handle to wait for a delivery report which is returned when

data/lib/rdkafka/producer/delivery_report.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Producer
     # Delivery report for a successfully produced message.
@@ -15,7 +17,7 @@ module Rdkafka
       attr_reader :topic_name
 
       # Error in case happen during produce.
-      # @return [String]
+      # @return [Integer]
       attr_reader :error
 
       private

data/lib/rdkafka/producer.rb

@@ -1,8 +1,15 @@
+# frozen_string_literal: true
+
 require "objspace"
 
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
+    # Cache partitions count for 30 seconds
+    PARTITIONS_COUNT_TTL = 30
+
+    private_constant :PARTITIONS_COUNT_TTL
+
     # @private
     # Returns the current delivery callback, by default this is nil.
     #
@@ -16,12 +23,32 @@ module Rdkafka
     attr_reader :delivery_callback_arity
 
     # @private
-    def initialize(client, partitioner_name)
-      @client = client
+    def initialize(native_kafka, partitioner_name)
+      @native_kafka = native_kafka
       @partitioner_name = partitioner_name || "consistent_random"
 
-      # Makes sure, that the producer gets closed before it gets GCed by Ruby
-      ObjectSpace.define_finalizer(self, client.finalizer)
+      # Makes sure, that native kafka gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(self, native_kafka.finalizer)
+
+      @_partitions_count_cache = Hash.new do |cache, topic|
+        topic_metadata = nil
+
+        @native_kafka.with_inner do |inner|
+          topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
+        end
+
+        cache[topic] = [
+          monotonic_now,
+          topic_metadata ? topic_metadata[:partition_count] : nil
+        ]
+      end
+    end
+
+    # @return [String] producer name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
     end
 
     # Set a callback that will be called every time a message is successfully produced.
@@ -38,9 +65,26 @@ module Rdkafka
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
+      return if closed?
       ObjectSpace.undefine_finalizer(self)
+      @native_kafka.close
+    end
 
-      @client.close
+    # Whether this producer has closed
+    def closed?
+      @native_kafka.closed?
+    end
+
+    # Wait until all outstanding producer requests are completed, with the given timeout
+    # in seconds. Call this before closing a producer to ensure delivery of all messages.
+    #
+    # @param timeout_ms [Integer] how long should we wait for flush of all messages
+    def flush(timeout_ms=5_000)
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
+      end
     end
 
     # Partition count for a given topic.
@@ -50,9 +94,20 @@ module Rdkafka
     #
     # @return partition count [Integer,nil]
     #
+    # We cache the partition count for a given topic for given time
+    # This prevents us in case someone uses `partition_key` from querying for the count with
+    # each message. Instead we query once every 30 seconds at most
+    #
+    # @param topic [String] topic name
+    # @return [Integer] partition count for a given topic
     def partition_count(topic)
       closed_producer_check(__method__)
-      Rdkafka::Metadata.new(@client.native, topic).topics&.first[:partition_count]
+
+      @_partitions_count_cache.delete_if do |_, cached|
+        monotonic_now - cached.first > PARTITIONS_COUNT_TTL
+      end
+
+      @_partitions_count_cache[topic].last
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -143,10 +198,12 @@ module Rdkafka
       args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
 
       # Produce the message
-      response = Rdkafka::Bindings.rd_kafka_producev(
-        @client.native,
-        *args
-      )
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_producev(
+          inner,
+          *args
+        )
+      end
 
       # Raise error if the produce call was not successful
       if response != 0
@@ -157,7 +214,6 @@ module Rdkafka
       delivery_handle
     end
 
-    # @private
     def call_delivery_callback(delivery_report, delivery_handle)
       return unless @delivery_callback
 
@@ -171,8 +227,15 @@ module Rdkafka
       callback.method(:call).arity
     end
 
+    private
+
+    def monotonic_now
+      # needed because Time.now can go backwards
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
     def closed_producer_check(method)
-      raise Rdkafka::ClosedProducerError.new(method) if @client.closed?
+      raise Rdkafka::ClosedProducerError.new(method) if closed?
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module Rdkafka
-  VERSION = "0.12.4"
+  VERSION = "0.13.0.beta1"
   LIBRDKAFKA_VERSION = "2.0.2"
   LIBRDKAFKA_SOURCE_SHA256 = "f321bcb1e015a34114c83cf1aa7b99ee260236aab096b85c003170c90a47ca9d"
 end

data/lib/rdkafka.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "rdkafka/version"
 
 require "rdkafka/abstract_handle"
@@ -18,7 +20,7 @@ require "rdkafka/consumer/partition"
 require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
 require "rdkafka/metadata"
+require "rdkafka/native_kafka"
 require "rdkafka/producer"
-require "rdkafka/producer/client"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/spec/rdkafka/abstract_handle_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::AbstractHandle do

data/spec/rdkafka/admin/create_topic_handle_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Admin::CreateTopicHandle do

data/spec/rdkafka/admin/create_topic_report_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Admin::CreateTopicReport do

data/spec/rdkafka/admin/delete_topic_handle_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Admin::DeleteTopicHandle do

data/spec/rdkafka/admin/delete_topic_report_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Admin::DeleteTopicReport do

data/spec/rdkafka/admin_spec.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 require "ostruct"
 
 describe Rdkafka::Admin do
-  let(:config)   { rdkafka_config }
-  let(:admin)    { config.admin }
+  let(:config) { rdkafka_config }
+  let(:admin)  { config.admin }
 
   after do
     # Registry should always end up being empty
@@ -174,7 +176,6 @@ describe Rdkafka::Admin do
       end
     end
 
-
     it "deletes a topic that was newly created" do
       create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
       create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)

data/spec/rdkafka/bindings_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 require 'zlib'
 

data/spec/rdkafka/callbacks_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Callbacks do

data/spec/rdkafka/config_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Config do
@@ -148,11 +150,24 @@ describe Rdkafka::Config do
       }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
     end
 
+    it "allows string partitioner key" do
+      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2").and_call_original
+      config = Rdkafka::Config.new("partitioner" => "murmur2")
+      config.producer.close
+    end
+
+    it "allows symbol partitioner key" do
+      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2").and_call_original
+      config = Rdkafka::Config.new(:partitioner => "murmur2")
+      config.producer.close
+    end
+
     it "should allow configuring zstd compression" do
       config = Rdkafka::Config.new('compression.codec' => 'zstd')
       begin
-        expect(config.producer).to be_a Rdkafka::Producer
-        config.producer.close
+        producer = config.producer
+        expect(producer).to be_a Rdkafka::Producer
+        producer.close
       rescue Rdkafka::Config::ConfigError => ex
         pending "Zstd compression not supported on this machine"
         raise ex

data/spec/rdkafka/consumer/headers_spec.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Consumer::Headers do
+  let(:headers) do
+    { # Note String keys!
+      "version" => "2.1.3",
+      "type" => "String"
+    }
+  end
+  let(:native_message) { double('native message') }
+  let(:headers_ptr) { double('headers pointer') }
+
+  describe '.from_native' do
+    before do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(native_message, anything) do |_, headers_ptrptr|
+        expect(headers_ptrptr).to receive(:read_pointer).and_return(headers_ptr)
+        Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+      end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 0, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: headers.keys[0]))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[0].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers.values[0]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 1, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: headers.keys[1]))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers.keys[1].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers.values[1]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 2, anything, anything, anything)
+          .and_return(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT)
+    end
+
+    subject { described_class.from_native(native_message) }
+
+    it { is_expected.to eq(headers) }
+    it { is_expected.to be_frozen }
+
+    it 'allows String key' do
+      expect(subject['version']).to eq("2.1.3")
+    end
+
+    it 'allows Symbol key, but warns' do
+      expect(Kernel).to \
+        receive(:warn).with("rdkafka deprecation warning: header access with Symbol key :version treated as a String. " \
+                            "Please change your code to use String keys to avoid this warning. Symbol keys will break in version 1.")
+      expect(subject[:version]).to eq("2.1.3")
+    end
+  end
+end

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Consumer::Message do

data/spec/rdkafka/consumer/partition_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Consumer::Partition do

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Consumer::TopicPartitionList do