rdkafka 0.10.0 → 0.12.0.beta.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f954490d858591f12e5a7fc1fed82e1b5c6b42049c5ed66d2d76bb352e03a7f
-  data.tar.gz: 0ae8cbac09963d52a3075cece9e0bf6c5895e31f87995d7997ee7f04e52a10c8
+  metadata.gz: a58b7e8b03dec32027bf3d9ec513a87e6bb0a8abf2f6598cd34a965bebce2dbb
+  data.tar.gz: 603c985bd19a2213671b600680a748279e21d7b4aeb3f23a7f1efa16a5e1229c
 SHA512:
-  metadata.gz: 33d43862b6b7cee1a3284221f75fae0c6bdfa6df4faa4396378c96e34a8f1a2395202dd01d6a5f2baac5557010505f548e7862bdd14202de6b3f85c77ca95309
-  data.tar.gz: 22d6243a853bfd630497eab13a8b6823f1a1d4a358aee53a98e68e5a0255f5ca68932db6d261cbad34dadeca5854522d8fceb15b43004ce8719a1d5d9aa3e9bd
+  metadata.gz: 9e5a5bd4c911f93572d0f8be2e5f576b08ee0ff86ab773e9e7d9db1182002e62c1c57b03cfe6e7b686c9890c35f6fca811fe10daad3a0d4d57bd67a5340c7fa1
+  data.tar.gz: 5fd3c57d9797f3b1f7c636f8190cee0b8cfc56f14daa03a2be21da6ff11fff1b7e00e233ab10a37b280b06ac2a197c155ebf1fdd6dfc8a97af9ce8bd38a341e1

data/.rspec

@@ -0,0 +1 @@
+--format documentation

data/.semaphore/semaphore.yml

@@ -13,7 +13,7 @@ blocks:
         - name: bundle exec rspec
           matrix:
             - env_var: RUBY_VERSION
-              values: [ "2.5.8", "2.6.6", "2.7.2", "3.0.0", "jruby-9.2.13.0" ]
+              values: [ "2.6.8", "2.7.4", "3.0.2", "jruby-9.3.1.0"]
           commands:
             - sem-version ruby $RUBY_VERSION
             - checkout

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 0.11.0
+* Upgrade librdkafka to 1.8.2
+* Bump supported minimum Ruby version to 2.6
+* Better homebrew path detection
+
 # 0.10.0
 * Upgrade librdkafka to 1.5.0
 * Add error callback config
@@ -45,7 +50,7 @@
 * Use default Homebrew openssl location if present
 * Consumer lag handles empty topics
 * End iteration in consumer when it is closed
-* Add suport for storing message offsets
+* Add support for storing message offsets
 * Add missing runtime dependency to rake
 
 # 0.4.1

data/Guardfile

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+logger level: :error
+
+guard :rspec, cmd: "bundle exec rspec --format #{ENV.fetch("FORMAT", "documentation")}" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+  watch(%r{^lib/(.+)\.rb}) { |m| "spec/#{m[1]}_spec.rb" }
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+end

data/README.md

@@ -7,7 +7,9 @@
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
 It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
-gem and targets Kafka 1.0+ and Ruby 2.4+.
+gem and targets Kafka 1.0+ and Ruby versions that are under security or
+active maintenance. We remove Ruby version from our CI builds if they 
+become EOL.
 
 `rdkafka` was written because we needed a reliable Ruby client for
 Kafka that supports modern Kafka at [AppSignal](https://appsignal.com).

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+ENV["IRBRC"] = File.join(File.dirname(__FILE__), ".irbrc")
+
+require "bundler/setup"
+require "rdkafka"
+
+require "irb"
+IRB.start(__FILE__)

data/docker-compose.yml

@@ -4,13 +4,13 @@ version: '2'
 
 services:
   zookeeper:
-    image: confluentinc/cp-zookeeper:latest
+    image: confluentinc/cp-zookeeper:5.2.6
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
 
   kafka:
-    image: confluentinc/cp-kafka:latest
+    image: confluentinc/cp-kafka:5.2.5-10
     depends_on:
       - zookeeper
     ports:
@@ -18,7 +18,7 @@ services:
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

data/ext/README.md

@@ -1,6 +1,6 @@
 # Ext
 
-This gem dependes on the `librdkafka` C library. It is downloaded when
+This gem depends on the `librdkafka` C library. It is downloaded when
 this gem is installed.
 
 To update the `librdkafka` version follow the following steps:

data/ext/Rakefile

@@ -4,30 +4,14 @@ require "fileutils"
 require "open-uri"
 
 task :default => :clean do
-  # MiniPortile#download_file_http is a monkey patch that removes the download
-  # progress indicator. This indicator relies on the 'Content Length' response
-  # headers, which is not set by GitHub
-  class MiniPortile
-    def download_file_http(url, full_path, _count)
-      filename = File.basename(full_path)
-      with_tempfile(filename, full_path) do |temp_file|
-        params = { 'Accept-Encoding' => 'identity' }
-        OpenURI.open_uri(url, 'rb', params) do |io|
-          temp_file.write(io.read)
-        end
-        output
-      end
-    end
-  end
-
   # Download and compile librdkafka
   recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
   # Use default homebrew openssl if we're on mac and the directory exists
   # and each of flags is not empty
-  if recipe.host&.include?("darwin") && Dir.exist?("/usr/local/opt/openssl")
-    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include" unless ENV["CPPFLAGS"]
-    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib" unless ENV["LDFLAGS"]
+  if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
+    ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
+    ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
   end
 
   recipe.files << {

data/lib/rdkafka/config.rb

@@ -179,7 +179,7 @@ module Rdkafka
       # Set callback to receive delivery reports on config
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
-      Rdkafka::Producer.new(native_kafka(config, :rd_kafka_producer)).tap do |producer|
+      Rdkafka::Producer.new(Rdkafka::Producer::Client.new(native_kafka(config, :rd_kafka_producer))).tap do |producer|
         opaque.producer = producer
       end
     end

data/lib/rdkafka/consumer.rb

@@ -498,11 +498,11 @@ module Rdkafka
     # Exception behavior is more complicated than with `each`, in that if
     # :yield_on_error is true, and an exception is raised during the
     # poll, and messages have already been received, they will be yielded to
-    # the caller before the exception is allowed to propogate.
+    # the caller before the exception is allowed to propagate.
     #
     # If you are setting either auto.commit or auto.offset.store to false in
     # the consumer configuration, then you should let yield_on_error keep its
-    # default value of false because you are gauranteed to see these messages
+    # default value of false because you are guaranteed to see these messages
     # again. However, if both auto.commit and auto.offset.store are set to
     # true, you should set yield_on_error to true so you can process messages
     # that you may or may not see again.
@@ -518,7 +518,7 @@ module Rdkafka
     # @yield [messages, pending_exception]
     # @yieldparam messages [Array] An array of received Message
     # @yieldparam pending_exception [Exception] normally nil, or an exception
-    # which will be propogated after processing of the partial batch is complete.
+    # which will be propagated after processing of the partial batch is complete.
     #
     # @return [nil]
     def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)

data/lib/rdkafka/producer/client.rb

@@ -0,0 +1,47 @@
+module Rdkafka
+  class Producer
+    class Client
+      def initialize(native)
+        @native = native
+
+        # Start thread to poll client for delivery callbacks
+        @polling_thread = Thread.new do
+          loop do
+            Rdkafka::Bindings.rd_kafka_poll(native, 250)
+            # Exit thread if closing and the poll queue is empty
+            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(native) == 0
+              break
+            end
+          end
+        end
+        @polling_thread.abort_on_exception = true
+        @polling_thread[:closing] = false
+      end
+
+      def native
+        @native
+      end
+
+      def finalizer
+        ->(_) { close }
+      end
+
+      def closed?
+        @native.nil?
+      end
+
+      def close(object_id=nil)
+        return unless @native
+
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+        # Wait for the polling thread to finish up
+        @polling_thread.join
+
+        Rdkafka::Bindings.rd_kafka_destroy(@native)
+
+        @native = nil
+      end
+    end
+  end
+end

data/lib/rdkafka/producer/delivery_report.rb

@@ -11,7 +11,7 @@ module Rdkafka
       attr_reader :offset
 
       # Error in case happen during produce.
-      # @return [string]
+      # @return [String]
       attr_reader :error
 
       private

data/lib/rdkafka/producer.rb

@@ -1,4 +1,4 @@
-require "securerandom"
+require "objspace"
 
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
@@ -10,25 +10,11 @@ module Rdkafka
     attr_reader :delivery_callback
 
     # @private
-    def initialize(native_kafka)
-      @id = SecureRandom.uuid
-      @closing = false
-      @native_kafka = native_kafka
+    def initialize(client)
+      @client = client
 
       # Makes sure, that the producer gets closed before it gets GCed by Ruby
-      ObjectSpace.define_finalizer(@id, proc { close })
-
-      # Start thread to poll client for delivery callbacks
-      @polling_thread = Thread.new do
-        loop do
-          Rdkafka::Bindings.rd_kafka_poll(@native_kafka, 250)
-          # Exit thread if closing and the poll queue is empty
-          if @closing && Rdkafka::Bindings.rd_kafka_outq_len(@native_kafka) == 0
-            break
-          end
-        end
-      end
-      @polling_thread.abort_on_exception = true
+      ObjectSpace.define_finalizer(self, client.finalizer)
     end
 
     # Set a callback that will be called every time a message is successfully produced.
@@ -44,16 +30,9 @@ module Rdkafka
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
-      ObjectSpace.undefine_finalizer(@id)
-
-      return unless @native_kafka
+      ObjectSpace.undefine_finalizer(self)
 
-      # Indicate to polling thread that we're closing
-      @closing = true
-      # Wait for the polling thread to finish up
-      @polling_thread.join
-      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
-      @native_kafka = nil
+      @client.close
     end
 
     # Partition count for a given topic.
@@ -65,7 +44,7 @@ module Rdkafka
     #
     def partition_count(topic)
       closed_producer_check(__method__)
-      Rdkafka::Metadata.new(@native_kafka, topic).topics&.first[:partition_count]
+      Rdkafka::Metadata.new(@client.native, topic).topics&.first[:partition_count]
     end
 
     # Produces a message to a Kafka topic. The message is added to rdkafka's queue, call {DeliveryHandle#wait wait} on the returned delivery handle to make sure it is delivered.
@@ -75,8 +54,9 @@ module Rdkafka
     #
     # @param topic [String] The topic to produce to
     # @param payload [String,nil] The message's payload
-    # @param key [String] The message's key
+    # @param key [String, nil] The message's key
     # @param partition [Integer,nil] Optional partition to produce to
+    # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     # @param headers [Hash<String,String>] Optional message headers
     #
@@ -156,7 +136,7 @@ module Rdkafka
 
       # Produce the message
       response = Rdkafka::Bindings.rd_kafka_producev(
-        @native_kafka,
+        @client.native,
         *args
       )
 
@@ -175,7 +155,7 @@ module Rdkafka
     end
 
     def closed_producer_check(method)
-      raise Rdkafka::ClosedProducerError.new(method) if @native_kafka.nil?
+      raise Rdkafka::ClosedProducerError.new(method) if @client.closed?
     end
   end
 end

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.10.0"
-  LIBRDKAFKA_VERSION = "1.5.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "f7fee59fdbf1286ec23ef0b35b2dfb41031c8727c90ced6435b8cf576f23a656"
+  VERSION = "0.12.0.beta.0"
+  LIBRDKAFKA_VERSION = "1.8.2"
+  LIBRDKAFKA_SOURCE_SHA256 = "6a747d293a7a4613bd2897e28e8791476fbe1ae7361f2530a876e0fd483482a6"
 end

data/lib/rdkafka.rb

@@ -17,5 +17,6 @@ require "rdkafka/consumer/topic_partition_list"
 require "rdkafka/error"
 require "rdkafka/metadata"
 require "rdkafka/producer"
+require "rdkafka/producer/client"
 require "rdkafka/producer/delivery_handle"
 require "rdkafka/producer/delivery_report"

data/rdkafka.gemspec

@@ -14,15 +14,17 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.4'
+  gem.required_ruby_version = '>= 2.6'
   gem.extensions = %w(ext/Rakefile)
 
-  gem.add_dependency 'ffi', '~> 1.9'
-  gem.add_dependency 'mini_portile2', '~> 2.1'
-  gem.add_dependency 'rake', '>= 12.3'
+  gem.add_dependency 'ffi', '~> 1.15'
+  gem.add_dependency 'mini_portile2', '~> 2.6'
+  gem.add_dependency 'rake', '> 12'
 
-  gem.add_development_dependency 'pry', '~> 0.10'
+  gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rspec', '~> 3.5'
-  gem.add_development_dependency 'rake', '~> 12.0'
-  gem.add_development_dependency 'simplecov', '~> 0.15'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'guard'
+  gem.add_development_dependency 'guard-rspec'
 end

data/spec/rdkafka/abstract_handle_spec.rb

@@ -111,4 +111,3 @@ describe Rdkafka::AbstractHandle do
     end
   end
 end
-

data/spec/rdkafka/admin_spec.rb

@@ -52,7 +52,7 @@ describe Rdkafka::Admin do
       end
 
       describe "with an invalid partition count" do
-        let(:topic_partition_count) { -1 }
+        let(:topic_partition_count) { -999 }
 
         it "raises an exception" do
           expect {

data/spec/rdkafka/config_spec.rb

@@ -108,7 +108,7 @@ describe Rdkafka::Config do
     end
 
     it "should create a consumer with valid config" do
-      consumer = rdkafka_config.consumer
+      consumer = rdkafka_consumer_config.consumer
       expect(consumer).to be_a Rdkafka::Consumer
       consumer.close
     end
@@ -136,7 +136,7 @@ describe Rdkafka::Config do
     end
 
     it "should create a producer with valid config" do
-      producer = rdkafka_config.producer
+      producer = rdkafka_consumer_config.producer
       expect(producer).to be_a Rdkafka::Producer
       producer.close
     end

data/spec/rdkafka/consumer_spec.rb

@@ -3,9 +3,8 @@ require "ostruct"
 require 'securerandom'
 
 describe Rdkafka::Consumer do
-  let(:config) { rdkafka_config }
-  let(:consumer) { config.consumer }
-  let(:producer) { config.producer }
+  let(:consumer) { rdkafka_consumer_config.consumer }
+  let(:producer) { rdkafka_producer_config.producer }
 
   after { consumer.close }
   after { producer.close }
@@ -328,7 +327,7 @@ describe Rdkafka::Consumer do
       before :all do
         # Make sure there are some messages.
         handles = []
-        producer = rdkafka_config.producer
+        producer = rdkafka_producer_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -404,7 +403,7 @@ describe Rdkafka::Consumer do
           config = {}
           config[:'enable.auto.offset.store'] = false
           config[:'enable.auto.commit'] = false
-          @new_consumer = rdkafka_config(config).consumer
+          @new_consumer = rdkafka_consumer_config(config).consumer
           @new_consumer.subscribe("consume_test_topic")
           wait_for_assignment(@new_consumer)
         end
@@ -459,13 +458,13 @@ describe Rdkafka::Consumer do
   end
 
   describe "#lag" do
-    let(:config) { rdkafka_config(:"enable.partition.eof" => true) }
+    let(:consumer) { rdkafka_consumer_config(:"enable.partition.eof" => true).consumer }
 
     it "should calculate the consumer lag" do
       # Make sure there's a message in every partition and
       # wait for the message to make sure everything is committed.
       (0..2).each do |i|
-        report = producer.produce(
+        producer.produce(
           topic:     "consume_test_topic",
           key:       "key lag #{i}",
           partition: i
@@ -508,7 +507,7 @@ describe Rdkafka::Consumer do
 
       # Produce message on every topic again
       (0..2).each do |i|
-        report = producer.produce(
+        producer.produce(
           topic:     "consume_test_topic",
           key:       "key lag #{i}",
           partition: i
@@ -824,8 +823,12 @@ describe Rdkafka::Consumer do
 
     context "error raised from poll and yield_on_error is true" do
       it "should yield buffered exceptions on rebalance, then break" do
-        config = rdkafka_config({:"enable.auto.commit" => false,
-                                 :"enable.auto.offset.store" => false })
+        config = rdkafka_consumer_config(
+          {
+            :"enable.auto.commit" => false,
+            :"enable.auto.offset.store" => false
+          }
+        )
         consumer = config.consumer
         consumer.subscribe(topic_name)
         loop_count = 0
@@ -864,8 +867,12 @@ describe Rdkafka::Consumer do
 
     context "error raised from poll and yield_on_error is false" do
       it "should yield buffered exceptions on rebalance, then break" do
-        config = rdkafka_config({:"enable.auto.commit" => false,
-                                 :"enable.auto.offset.store" => false })
+        config = rdkafka_consumer_config(
+          {
+            :"enable.auto.commit" => false,
+            :"enable.auto.offset.store" => false
+          }
+        )
         consumer = config.consumer
         consumer.subscribe(topic_name)
         loop_count = 0
@@ -902,51 +909,64 @@ describe Rdkafka::Consumer do
   end
 
   describe "a rebalance listener" do
-    it "should get notifications" do
-      listener = Struct.new(:queue) do
-        def on_partitions_assigned(consumer, list)
-          collect(:assign, list)
-        end
+    let(:consumer) do
+      config = rdkafka_consumer_config
+      config.consumer_rebalance_listener = listener
+      config.consumer
+    end
 
-        def on_partitions_revoked(consumer, list)
-          collect(:revoke, list)
-        end
+    context "with a working listener" do
+      let(:listener) do
+        Struct.new(:queue) do
+          def on_partitions_assigned(consumer, list)
+            collect(:assign, list)
+          end
 
-        def collect(name, list)
-          partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
-          queue << ([name] + partitions)
-        end
-      end.new([])
+          def on_partitions_revoked(consumer, list)
+            collect(:revoke, list)
+          end
+
+          def collect(name, list)
+            partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
+            queue << ([name] + partitions)
+          end
+        end.new([])
+      end
 
-      notify_listener(listener)
+      it "should get notifications" do
+        notify_listener(listener)
 
-      expect(listener.queue).to eq([
-        [:assign, "consume_test_topic", 0, 1, 2],
-        [:revoke, "consume_test_topic", 0, 1, 2]
-      ])
+        expect(listener.queue).to eq([
+          [:assign, "consume_test_topic", 0, 1, 2],
+          [:revoke, "consume_test_topic", 0, 1, 2]
+        ])
+      end
     end
 
-    it 'should handle callback exceptions' do
-      listener = Struct.new(:queue) do
-        def on_partitions_assigned(consumer, list)
-          queue << :assigned
-          raise 'boom'
-        end
+    context "with a broken listener" do
+      let(:listener) do
+        Struct.new(:queue) do
+          def on_partitions_assigned(consumer, list)
+            queue << :assigned
+            raise 'boom'
+          end
 
-        def on_partitions_revoked(consumer, list)
-          queue << :revoked
-          raise 'boom'
-        end
-      end.new([])
+          def on_partitions_revoked(consumer, list)
+            queue << :revoked
+            raise 'boom'
+          end
+        end.new([])
+      end
 
-      notify_listener(listener)
+      it 'should handle callback exceptions' do
+        notify_listener(listener)
 
-      expect(listener.queue).to eq([:assigned, :revoked])
+        expect(listener.queue).to eq([:assigned, :revoked])
+      end
     end
 
     def notify_listener(listener)
       # 1. subscribe and poll
-      config.consumer_rebalance_listener = listener
       consumer.subscribe("consume_test_topic")
       wait_for_assignment(consumer)
       consumer.poll(100)

data/spec/rdkafka/metadata_spec.rb

@@ -2,7 +2,7 @@ require "spec_helper"
 require "securerandom"
 
 describe Rdkafka::Metadata do
-  let(:config)        { rdkafka_config }
+  let(:config)        { rdkafka_consumer_config }
   let(:native_config) { config.send(:native_config) }
   let(:native_kafka)  { config.send(:native_kafka, native_config, :rd_kafka_consumer) }
 

data/spec/rdkafka/producer/client_spec.rb

@@ -0,0 +1,144 @@
+require "spec_helper"
+
+describe Rdkafka::Producer::Client do
+  let(:native) { double }
+  let(:closing) { false }
+  let(:thread) { double(Thread) }
+
+  subject(:client) { described_class.new(native) }
+
+  before do
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(native, 250)
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native).and_return(0)
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_destroy)
+    allow(Thread).to receive(:new).and_return(thread)
+
+    allow(thread).to receive(:[]=).with(:closing, anything)
+    allow(thread).to receive(:join)
+    allow(thread).to receive(:abort_on_exception=).with(anything)
+  end
+
+  context "defaults" do
+    it "sets the thread to abort on exception" do
+      expect(thread).to receive(:abort_on_exception=).with(true)
+
+      client
+    end
+
+    it "sets the thread `closing` flag to false" do
+      expect(thread).to receive(:[]=).with(:closing, false)
+
+      client
+    end
+  end
+
+  context "the polling thread" do
+    it "is created" do
+      expect(Thread).to receive(:new)
+
+      client
+    end
+
+    it "polls the native with default 250ms timeout" do
+      polling_loop_expects do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_poll).with(native, 250)
+      end
+    end
+
+    it "check the out queue of native client" do
+      polling_loop_expects do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_outq_len).with(native)
+      end
+    end
+  end
+
+  def polling_loop_expects(&block)
+    Thread.current[:closing] = true # this forces the loop break with line #12
+
+    allow(Thread).to receive(:new).and_yield do |_|
+      block.call
+    end.and_return(thread)
+
+    client
+  end
+
+  it "exposes `native` client" do
+    expect(client.native).to eq(native)
+  end
+
+  context "when client was not yet closed (`nil`)" do
+    it "is not closed" do
+      expect(client.closed?).to eq(false)
+    end
+
+    context "and attempt to close" do
+      it "calls the `destroy` binding" do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native)
+
+        client.close
+      end
+
+      it "indicates to the polling thread that it is closing" do
+        expect(thread).to receive(:[]=).with(:closing, true)
+
+        client.close
+      end
+
+      it "joins the polling thread" do
+        expect(thread).to receive(:join)
+
+        client.close
+      end
+
+      it "closes and unassign the native client" do
+        client.close
+
+        expect(client.native).to eq(nil)
+        expect(client.closed?).to eq(true)
+      end
+    end
+  end
+
+  context "when client was already closed" do
+    before { client.close }
+
+    it "is closed" do
+      expect(client.closed?).to eq(true)
+    end
+
+    context "and attempt to close again" do
+      it "does not call the `destroy` binding" do
+        expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy)
+
+        client.close
+      end
+
+      it "does not indicate to the polling thread that it is closing" do
+        expect(thread).not_to receive(:[]=).with(:closing, true)
+
+        client.close
+      end
+
+      it "does not join the polling thread" do
+        expect(thread).not_to receive(:join)
+
+        client.close
+      end
+
+      it "does not close and unassign the native client again" do
+        client.close
+
+        expect(client.native).to eq(nil)
+        expect(client.closed?).to eq(true)
+      end
+    end
+  end
+
+  it "provide a finalizer Proc that closes the `native` client" do
+    expect(client.closed?).to eq(false)
+
+    client.finalizer.call("some-ignored-object-id")
+
+    expect(client.closed?).to eq(true)
+  end
+end

data/spec/rdkafka/producer_spec.rb

@@ -1,8 +1,9 @@
 require "spec_helper"
+require "zlib"
 
 describe Rdkafka::Producer do
-  let(:producer) { rdkafka_config.producer }
-  let(:consumer) { rdkafka_config.consumer }
+  let(:producer) { rdkafka_producer_config.producer }
+  let(:consumer) { rdkafka_consumer_config.consumer }
 
   after do
     # Registry should always end up being empty
@@ -388,7 +389,7 @@ describe Rdkafka::Producer do
       reader.close
 
       # Avoids sharing the socket between processes.
-      producer = rdkafka_config.producer
+      producer = rdkafka_producer_config.producer
 
       handle = producer.produce(
         topic:   "produce_test_topic",

data/spec/spec_helper.rb

@@ -8,27 +8,57 @@ end
 require "pry"
 require "rspec"
 require "rdkafka"
+require "timeout"
 
-def rdkafka_config(config_overrides={})
-  config = {
+def rdkafka_base_config
+  {
     :"api.version.request" => false,
     :"broker.version.fallback" => "1.0",
     :"bootstrap.servers" => "localhost:9092",
-    :"group.id" => "ruby-test-#{Random.new.rand(0..1_000_000)}",
-    :"auto.offset.reset" => "earliest",
-    :"enable.partition.eof" => false
   }
+end
+
+def rdkafka_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Merge overrides
+  config.merge!(config_overrides)
+  # Return it
+  Rdkafka::Config.new(config)
+end
+
+def rdkafka_consumer_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Add consumer specific fields to it
+  config[:"auto.offset.reset"] = "earliest"
+  config[:"enable.partition.eof"] = false
+  config[:"group.id"] = "ruby-test-#{Random.new.rand(0..1_000_000)}"
+  # Enable debug mode if required
+  if ENV["DEBUG_CONSUMER"]
+    config[:debug] = "cgrp,topic,fetch"
+  end
+  # Merge overrides
+  config.merge!(config_overrides)
+  # Return it
+  Rdkafka::Config.new(config)
+end
+
+def rdkafka_producer_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Enable debug mode if required
   if ENV["DEBUG_PRODUCER"]
     config[:debug] = "broker,topic,msg"
-  elsif ENV["DEBUG_CONSUMER"]
-    config[:debug] = "cgrp,topic,fetch"
   end
+  # Merge overrides
   config.merge!(config_overrides)
+  # Return it
   Rdkafka::Config.new(config)
 end
 
 def new_native_client
-  config = rdkafka_config
+  config = rdkafka_consumer_config
   config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
 end
 
@@ -42,7 +72,7 @@ end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
   new_consumer = !!consumer
-  consumer ||= rdkafka_config.consumer
+  consumer ||= rdkafka_consumer_config.consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
   loop do
@@ -75,6 +105,9 @@ def wait_for_unassignment(consumer)
 end
 
 RSpec.configure do |config|
+  config.filter_run focus: true
+  config.run_all_when_everything_filtered = true
+
   config.before(:suite) do
     admin = rdkafka_config.admin
     {
@@ -95,4 +128,12 @@ RSpec.configure do |config|
     end
     admin.close
   end
+
+  config.around(:each) do |example|
+    # Timeout specs after a minute. If they take longer
+    # they are probably stuck
+    Timeout::timeout(60) do
+      example.run
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.12.0.beta.0
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-07 00:00:00.000000000 Z
+date: 2022-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -16,56 +16,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.15'
 - !ruby/object:Gem::Dependency
   name: mini_portile2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -84,46 +84,78 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Modern Kafka client library for Ruby based on librdkafka
 email:
 - thijs@appsignal.com
-executables: []
+executables:
+- console
 extensions:
 - ext/Rakefile
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - ".semaphore/semaphore.yml"
 - ".yardopts"
 - CHANGELOG.md
 - Gemfile
+- Guardfile
 - LICENSE
 - README.md
 - Rakefile
+- bin/console
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
@@ -145,6 +177,7 @@ files:
 - lib/rdkafka/error.rb
 - lib/rdkafka/metadata.rb
 - lib/rdkafka/producer.rb
+- lib/rdkafka/producer/client.rb
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
 - lib/rdkafka/version.rb
@@ -164,6 +197,7 @@ files:
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
 - spec/rdkafka/metadata_spec.rb
+- spec/rdkafka/producer/client_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb
@@ -180,12 +214,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.1.4
 signing_key:
@@ -209,6 +243,7 @@ test_files:
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
 - spec/rdkafka/metadata_spec.rb
+- spec/rdkafka/producer/client_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb