rdkafka 0.12.0 → 0.15.0

data/lib/rdkafka/native_kafka.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # @private
+  # A wrapper around a native kafka that polls and cleanly exits
+  class NativeKafka
+    def initialize(inner, run_polling_thread:, opaque:)
+      @inner = inner
+      @opaque = opaque
+      # Lock around external access
+      @access_mutex = Mutex.new
+      # Lock around internal polling
+      @poll_mutex = Mutex.new
+      # Lock around decrementing the operations in progress counter
+      # We have two mutexes - one for increment (`@access_mutex`) and one for decrement mutex
+      # because they serve different purposes:
+      #
+      #   - `@access_mutex` allows us to lock the execution and make sure that any operation within
+      #      the `#synchronize` is the only one running and that there are no other running
+      #      operations.
+      #   - `@decrement_mutex` ensures, that our decrement operation is thread-safe for any Ruby
+      #     implementation.
+      #
+      # We do not use the same mutex, because it could create a deadlock when an already
+      # incremented operation cannot decrement because `@access_lock` is now owned by a different
+      # thread in a synchronized mode and the synchronized mode is waiting on the decrement.
+      @decrement_mutex = Mutex.new
+      # counter for operations in progress using inner
+      @operations_in_progress = 0
+
+      # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
+      Rdkafka::Bindings.rd_kafka_poll(inner, 0)
+
+      if run_polling_thread
+        # Start thread to poll client for delivery callbacks,
+        # not used in consumer.
+        @polling_thread = Thread.new do
+          loop do
+            @poll_mutex.synchronize do
+              Rdkafka::Bindings.rd_kafka_poll(inner, 100)
+            end
+
+            # Exit thread if closing and the poll queue is empty
+            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
+              break
+            end
+          end
+        end
+
+        @polling_thread.abort_on_exception = true
+        @polling_thread[:closing] = false
+      end
+
+      @closing = false
+    end
+
+    def with_inner
+      if @access_mutex.owned?
+        @operations_in_progress += 1
+      else
+        @access_mutex.synchronize { @operations_in_progress += 1 }
+      end
+
+      @inner.nil? ? raise(ClosedInnerError) : yield(@inner)
+    ensure
+      @decrement_mutex.synchronize { @operations_in_progress -= 1 }
+    end
+
+    def synchronize(&block)
+      @access_mutex.synchronize do
+        # Wait for any commands using the inner to finish
+        # This can take a while on blocking operations like polling but is essential not to proceed
+        # with certain types of operations like resources destruction as it can cause the process
+        # to hang or crash
+        sleep(0.01) until @operations_in_progress.zero?
+
+        with_inner(&block)
+      end
+    end
+
+    def finalizer
+      ->(_) { close }
+    end
+
+    def closed?
+      @closing || @inner.nil?
+    end
+
+    def close(object_id=nil)
+      return if closed?
+
+      synchronize do
+        # Indicate to the outside world that we are closing
+        @closing = true
+
+        if @polling_thread
+          # Indicate to polling thread that we're closing
+          @polling_thread[:closing] = true
+
+          # Wait for the polling thread to finish up,
+          # this can be aborted in practice if this
+          # code runs from a finalizer.
+          @polling_thread.join
+        end
+
+        # Destroy the client after locking both mutexes
+        @poll_mutex.lock
+
+        # This check prevents a race condition, where we would enter the close in two threads
+        # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+        # and would continue to run, trying to destroy inner twice
+        return unless @inner
+
+        Rdkafka::Bindings.rd_kafka_destroy(@inner)
+        @inner = nil
+        @opaque = nil
+      end
+    end
+  end
+end

data/lib/rdkafka/producer/delivery_handle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Producer
     # Handle to wait for a delivery report which is returned when
@@ -6,7 +8,8 @@ module Rdkafka
       layout :pending, :bool,
              :response, :int,
              :partition, :int,
-             :offset, :int64
+             :offset, :int64,
+             :topic_name, :pointer
 
       # @return [String] the name of the operation (e.g. "delivery")
       def operation_name
@@ -15,7 +18,7 @@ module Rdkafka
 
       # @return [DeliveryReport] a report on the delivery of the message
       def create_result
-        DeliveryReport.new(self[:partition], self[:offset])
+        DeliveryReport.new(self[:partition], self[:offset], self[:topic_name].read_string)
       end
     end
   end

data/lib/rdkafka/producer/delivery_report.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Producer
     # Delivery report for a successfully produced message.
@@ -10,15 +12,20 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
-      # Error in case happen during produce.
+      # The name of the topic this message was produced to.
       # @return [String]
+      attr_reader :topic_name
+
+      # Error in case happen during produce.
+      # @return [Integer]
       attr_reader :error
 
       private
 
-      def initialize(partition, offset, error = nil)
+      def initialize(partition, offset, topic_name = nil, error = nil)
         @partition = partition
         @offset = offset
+        @topic_name = topic_name
         @error = error
       end
     end

data/lib/rdkafka/producer.rb

@@ -1,8 +1,15 @@
-require "objspace"
+# frozen_string_literal: true
 
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
+    include Helpers::Time
+
+    # Cache partitions count for 30 seconds
+    PARTITIONS_COUNT_TTL = 30
+
+    private_constant :PARTITIONS_COUNT_TTL
+
     # @private
     # Returns the current delivery callback, by default this is nil.
     #
@@ -16,12 +23,32 @@ module Rdkafka
     attr_reader :delivery_callback_arity
 
     # @private
-    def initialize(client, partitioner_name)
-      @client = client
+    def initialize(native_kafka, partitioner_name)
+      @native_kafka = native_kafka
       @partitioner_name = partitioner_name || "consistent_random"
 
-      # Makes sure, that the producer gets closed before it gets GCed by Ruby
-      ObjectSpace.define_finalizer(self, client.finalizer)
+      # Makes sure, that native kafka gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(self, native_kafka.finalizer)
+
+      @_partitions_count_cache = Hash.new do |cache, topic|
+        topic_metadata = nil
+
+        @native_kafka.with_inner do |inner|
+          topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
+        end
+
+        cache[topic] = [
+          monotonic_now,
+          topic_metadata ? topic_metadata[:partition_count] : nil
+        ]
+      end
+    end
+
+    # @return [String] producer name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
     end
 
     # Set a callback that will be called every time a message is successfully produced.
@@ -38,21 +65,91 @@ module Rdkafka
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
+      return if closed?
       ObjectSpace.undefine_finalizer(self)
+      @native_kafka.close
+    end
 
-      @client.close
+    # Whether this producer has closed
+    def closed?
+      @native_kafka.closed?
+    end
+
+    # Wait until all outstanding producer requests are completed, with the given timeout
+    # in seconds. Call this before closing a producer to ensure delivery of all messages.
+    #
+    # @param timeout_ms [Integer] how long should we wait for flush of all messages
+    # @return [Boolean] true if no more data and all was flushed, false in case there are still
+    #   outgoing messages after the timeout
+    #
+    # @note We raise an exception for other errors because based on the librdkafka docs, there
+    #   should be no other errors.
+    #
+    # @note For `timed_out` we do not raise an error to keep it backwards compatible
+    def flush(timeout_ms=5_000)
+      closed_producer_check(__method__)
+
+      code = nil
+
+      @native_kafka.with_inner do |inner|
+        code = Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
+      end
+
+      # Early skip not to build the error message
+      return true if code.zero?
+
+      error = Rdkafka::RdkafkaError.new(code)
+
+      return false if error.code == :timed_out
+
+      raise(error)
+    end
+
+    # Purges the outgoing queue and releases all resources.
+    #
+    # Useful when closing the producer with outgoing messages to unstable clusters or when for
+    # any other reasons waiting cannot go on anymore. This purges both the queue and all the
+    # inflight requests + updates the delivery handles statuses so they can be materialized into
+    # `purge_queue` errors.
+    def purge
+      closed_producer_check(__method__)
+
+      code = nil
+
+      @native_kafka.with_inner do |inner|
+        code = Bindings.rd_kafka_purge(
+          inner,
+          Bindings::RD_KAFKA_PURGE_F_QUEUE | Bindings::RD_KAFKA_PURGE_F_INFLIGHT
+        )
+      end
+
+      code.zero? || raise(Rdkafka::RdkafkaError.new(code))
+
+      # Wait for the purge to affect everything
+      sleep(0.001) until flush(100)
+
+      true
     end
 
     # Partition count for a given topic.
-    # NOTE: If 'allow.auto.create.topics' is set to true in the broker, the topic will be auto-created after returning nil.
     #
     # @param topic [String] The topic name.
+    # @return [Integer] partition count for a given topic
     #
-    # @return partition count [Integer,nil]
+    # @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
+    #   auto-created after returning nil.
     #
+    # @note We cache the partition count for a given topic for given time.
+    #   This prevents us in case someone uses `partition_key` from querying for the count with
+    #   each message. Instead we query once every 30 seconds at most
     def partition_count(topic)
       closed_producer_check(__method__)
-      Rdkafka::Metadata.new(@client.native, topic).topics&.first[:partition_count]
+
+      @_partitions_count_cache.delete_if do |_, cached|
+        monotonic_now - cached.first > PARTITIONS_COUNT_TTL
+      end
+
+      @_partitions_count_cache[topic].last
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -68,9 +165,9 @@ module Rdkafka
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     # @param headers [Hash<String,String>] Optional message headers
     #
-    # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
-    #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
+    #
+    # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
     def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
       closed_producer_check(__method__)
 
@@ -143,10 +240,12 @@ module Rdkafka
       args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
 
       # Produce the message
-      response = Rdkafka::Bindings.rd_kafka_producev(
-        @client.native,
-        *args
-      )
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_producev(
+          inner,
+          *args
+        )
+      end
 
       # Raise error if the produce call was not successful
       if response != 0
@@ -157,7 +256,6 @@ module Rdkafka
       delivery_handle
     end
 
-    # @private
     def call_delivery_callback(delivery_report, delivery_handle)
       return unless @delivery_callback
 
@@ -171,8 +269,10 @@ module Rdkafka
       callback.method(:call).arity
     end
 
+    private
+
     def closed_producer_check(method)
-      raise Rdkafka::ClosedProducerError.new(method) if @client.closed?
+      raise Rdkafka::ClosedProducerError.new(method) if closed?
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,5 +1,7 @@
+# frozen_string_literal: true
+
 module Rdkafka
-  VERSION = "0.12.0"
-  LIBRDKAFKA_VERSION = "1.9.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "59b6088b69ca6cf278c3f9de5cd6b7f3fd604212cd1c59870bc531c54147e889"
+  VERSION = "0.15.0"
+  LIBRDKAFKA_VERSION = "2.3.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/lib/rdkafka.rb

@@ -1,11 +1,29 @@
-require "rdkafka/version"
+# frozen_string_literal: true
+
+require "logger"
+require "objspace"
+require "ffi"
+require "json"
 
+require "rdkafka/version"
+require "rdkafka/helpers/time"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
 require "rdkafka/admin/create_topic_handle"
 require "rdkafka/admin/create_topic_report"
+require "rdkafka/admin/delete_groups_handle"
+require "rdkafka/admin/delete_groups_report"
 require "rdkafka/admin/delete_topic_handle"
 require "rdkafka/admin/delete_topic_report"
+require "rdkafka/admin/create_partitions_handle"
+require "rdkafka/admin/create_partitions_report"
+require "rdkafka/admin/create_acl_handle"
+require "rdkafka/admin/create_acl_report"
+require "rdkafka/admin/delete_acl_handle"
+require "rdkafka/admin/delete_acl_report"
+require "rdkafka/admin/describe_acl_handle"
+require "rdkafka/admin/describe_acl_report"
+require "rdkafka/admin/acl_binding_result"
 require "rdkafka/bindings"
 require "rdkafka/callbacks"
 require "rdkafka/config"
@@ -16,7 +34,11 @@ require "rdkafka/consumer/partition"
 require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
 require "rdkafka/metadata"
+require "rdkafka/native_kafka"
 require "rdkafka/producer"
-require "rdkafka/producer/client"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"
+
+# Main Rdkafka namespace of this gem
+module Rdkafka
+end

data/rdkafka.gemspec

@@ -1,12 +1,13 @@
+# frozen_string_literal: true
+
 require File.expand_path('lib/rdkafka/version', __dir__)
 
 Gem::Specification.new do |gem|
   gem.authors = ['Thijs Cadier']
-  gem.email = ["thijs@appsignal.com"]
+  gem.email = ["contact@karafka.io"]
   gem.description = "Modern Kafka client library for Ruby based on librdkafka"
   gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.4+."
   gem.license = 'MIT'
-  gem.homepage = 'https://github.com/thijsc/rdkafka-ruby'
 
   gem.files = `git ls-files`.split($\)
   gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -14,8 +15,13 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.6'
+  gem.required_ruby_version = '>= 2.7'
   gem.extensions = %w(ext/Rakefile)
+  gem.cert_chain = %w[certs/cert_chain.pem]
+
+  if $PROGRAM_NAME.end_with?('gem')
+    gem.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
+  end
 
   gem.add_dependency 'ffi', '~> 1.15'
   gem.add_dependency 'mini_portile2', '~> 2.6'
@@ -27,4 +33,14 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'simplecov'
   gem.add_development_dependency 'guard'
   gem.add_development_dependency 'guard-rspec'
+
+  gem.metadata = {
+    'funding_uri' => 'https://karafka.io/#become-pro',
+    'homepage_uri' => 'https://karafka.io',
+    'changelog_uri' => 'https://github.com/karafka/rdkafka-ruby/blob/main/CHANGELOG.md',
+    'bug_tracker_uri' => 'https://github.com/karafka/rdkafka-ruby/issues',
+    'source_code_uri' => 'https://github.com/karafka/rdkafka-ruby',
+    'documentation_uri' => 'https://github.com/karafka/rdkafka-ruby/blob/main/README.md',
+    'rubygems_mfa_required' => 'true'
+  }
 end

data/renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

data/spec/rdkafka/abstract_handle_spec.rb

@@ -1,4 +1,4 @@
-require "spec_helper"
+# frozen_string_literal: true
 
 describe Rdkafka::AbstractHandle do
   let(:response) { 0 }

data/spec/rdkafka/admin/create_acl_handle_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Admin::CreateAclHandle do
+  # If create acl was successful there is no error object
+  # the error code is set to RD_KAFKA_RESP_ERR_NO_ERRORa
+  # https://github.com/confluentinc/librdkafka/blob/1f9f245ac409f50f724695c628c7a0d54a763b9a/src/rdkafka_error.c#L169
+  let(:response) { Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR }
+
+  subject do
+    Rdkafka::Admin::CreateAclHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      # If create acl was successful there is no error object and the error_string is set to ""
+      # https://github.com/confluentinc/librdkafka/blob/1f9f245ac409f50f724695c628c7a0d54a763b9a/src/rdkafka_error.c#L178
+      handle[:response_string] = FFI::MemoryPointer.from_string("")
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::CreateAclHandle::WaitTimeoutError, /create acl/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a create acl report" do
+        report = subject.wait
+
+        expect(report.rdkafka_response_string).to eq("")
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.rdkafka_response_string).to eq("")
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    it "should raise the appropriate error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+    end
+  end
+end

data/spec/rdkafka/admin/create_acl_report_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Admin::CreateAclReport do
+  subject { Rdkafka::Admin::CreateAclReport.new(
+      rdkafka_response: Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR,
+      rdkafka_response_string: FFI::MemoryPointer.from_string("")
+  )}
+
+  it "should get RD_KAFKA_RESP_ERR_NO_ERROR " do
+    expect(subject.rdkafka_response).to eq(0)
+  end
+
+  it "should get empty string" do
+    expect(subject.rdkafka_response_string).to eq("")
+  end
+end

data/spec/rdkafka/admin/create_topic_handle_spec.rb

@@ -1,4 +1,4 @@
-require "spec_helper"
+# frozen_string_literal: true
 
 describe Rdkafka::Admin::CreateTopicHandle do
   let(:response) { 0 }

data/spec/rdkafka/admin/create_topic_report_spec.rb

@@ -1,4 +1,4 @@
-require "spec_helper"
+# frozen_string_literal: true
 
 describe Rdkafka::Admin::CreateTopicReport do
   subject { Rdkafka::Admin::CreateTopicReport.new(

data/spec/rdkafka/admin/delete_acl_handle_spec.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Admin::DeleteAclHandle do
+  let(:response) { Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR }
+  let(:resource_name)         {"acl-test-topic"}
+  let(:resource_type)         {Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC}
+  let(:resource_pattern_type) {Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL}
+  let(:principal)             {"User:anonymous"}
+  let(:host)                  {"*"}
+  let(:operation)             {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
+  let(:permission_type)       {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
+  let(:delete_acl_ptr)        {FFI::Pointer::NULL}
+
+  subject do
+    error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+    delete_acl_ptr = Rdkafka::Bindings.rd_kafka_AclBinding_new(
+      resource_type,
+      FFI::MemoryPointer.from_string(resource_name),
+      resource_pattern_type,
+      FFI::MemoryPointer.from_string(principal),
+      FFI::MemoryPointer.from_string(host),
+      operation,
+      permission_type,
+      error_buffer,
+      256
+    )
+    if delete_acl_ptr.null?
+      raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+    end
+    pointer_array = [delete_acl_ptr]
+    delete_acls_array_ptr = FFI::MemoryPointer.new(:pointer)
+    delete_acls_array_ptr.write_array_of_pointer(pointer_array)
+    Rdkafka::Admin::DeleteAclHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:response_string] = FFI::MemoryPointer.from_string("")
+      handle[:matching_acls] = delete_acls_array_ptr
+      handle[:matching_acls_count] = 1
+    end
+  end
+
+  after do
+    if delete_acl_ptr != FFI::Pointer::NULL
+      Rdkafka::Bindings.rd_kafka_AclBinding_destroy(delete_acl_ptr)
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::DeleteAclHandle::WaitTimeoutError, /delete acl/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a delete acl report" do
+        report = subject.wait
+
+        expect(report.deleted_acls.length).to eq(1)
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.deleted_acls[0].matching_acl_resource_name).to eq(resource_name)
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    it "should raise the appropriate error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+    end
+  end
+end