rdkafka 0.9.0 → 0.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22bda74a815bebd487179ac5ec3faa4566752f2c87945ca5c461c427633daf1f
-  data.tar.gz: '0802ca4118ed248fa11226a2c1b2140d073d230c02e3696a03e462fe59cb129e'
+  metadata.gz: 9b7042f241be5aad91c403f61aabb8e8ba87afa46674db8ba58c487fe01f88c9
+  data.tar.gz: 5cb00dc3dc8b4069a8e62cda9271e290d1371434332767270426488dedcff641
 SHA512:
-  metadata.gz: 4d3b7042fa2290386d4feaaa01c4289bc858042d66806157066257f282c84db459e80e32a2613efd3452a46c63faaff7fad9f110fb153c3385cef00fb6017f27
-  data.tar.gz: e431a320535df8293e099e1b25bf0e6da68caf9a641621faf847dd206093dd9973b41f03f95412c43a740b50a0ef70e7caad418ff20c07218f19342d1cb3826a
+  metadata.gz: 2b287a0d81aca7909702969778fb64585b6808860971d50e1c5540eb0c43cf73617f9e701605b816e8f52a81e39d9909512e1d113f6a550555e6ca503dc5ff58
+  data.tar.gz: 7924ab06f52646168b52d0478b605484e4aa1a3674ace4f97af5b56588aea6dec8e614d3258b0670f92f18a905509ae6982d2c8999fa719f8ea7451e515ecb3d

data/.rspec

@@ -0,0 +1 @@
+--format documentation

data/.semaphore/semaphore.yml

@@ -13,7 +13,7 @@ blocks:
         - name: bundle exec rspec
           matrix:
             - env_var: RUBY_VERSION
-              values: [ "2.5.8", "2.6.6", "2.7.2", "3.0.0", "jruby-9.2.13.0" ]
+              values: [ "2.6.8", "2.7.4", "3.0.2", "jruby-9.3.1.0"]
           commands:
             - sem-version ruby $RUBY_VERSION
             - checkout

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+# 0.11.1
+* Use mini_portile2 2.6, otherwise you can't run nokogiri and rdkafka at the same time.
+
+# 0.11.0
+* Upgrade librdkafka to 1.8.2
+* Bump supported minimum Ruby version to 2.6
+* Better homebrew path detection
+
+# 0.10.0
+* Upgrade librdkafka to 1.5.0
+* Add error callback config
+
 # 0.9.0
 * Fixes for Ruby 3.0
 * Allow any callable object for callbacks (gremerritt)
@@ -41,7 +53,7 @@
 * Use default Homebrew openssl location if present
 * Consumer lag handles empty topics
 * End iteration in consumer when it is closed
-* Add suport for storing message offsets
+* Add support for storing message offsets
 * Add missing runtime dependency to rake
 
 # 0.4.1

data/Guardfile

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+logger level: :error
+
+guard :rspec, cmd: "bundle exec rspec --format #{ENV.fetch("FORMAT", "documentation")}" do
+  require "guard/rspec/dsl"
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+  watch(%r{^lib/(.+)\.rb}) { |m| "spec/#{m[1]}_spec.rb" }
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+end

data/README.md

@@ -7,7 +7,9 @@
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
 It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
-gem and targets Kafka 1.0+ and Ruby 2.4+.
+gem and targets Kafka 1.0+ and Ruby versions that are under security or
+active maintenance. We remove Ruby version from our CI builds if they 
+become EOL.
 
 `rdkafka` was written because we needed a reliable Ruby client for
 Kafka that supports modern Kafka at [AppSignal](https://appsignal.com).

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+ENV["IRBRC"] = File.join(File.dirname(__FILE__), ".irbrc")
+
+require "bundler/setup"
+require "rdkafka"
+
+require "irb"
+IRB.start(__FILE__)

data/docker-compose.yml

@@ -4,13 +4,13 @@ version: '2'
 
 services:
   zookeeper:
-    image: confluentinc/cp-zookeeper:latest
+    image: confluentinc/cp-zookeeper:5.2.6
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
 
   kafka:
-    image: confluentinc/cp-kafka:latest
+    image: confluentinc/cp-kafka:5.2.5-10
     depends_on:
       - zookeeper
     ports:
@@ -18,7 +18,7 @@ services:
     environment:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

data/ext/README.md

@@ -1,6 +1,6 @@
 # Ext
 
-This gem dependes on the `librdkafka` C library. It is downloaded when
+This gem depends on the `librdkafka` C library. It is downloaded when
 this gem is installed.
 
 To update the `librdkafka` version follow the following steps:

data/ext/Rakefile

@@ -4,30 +4,14 @@ require "fileutils"
 require "open-uri"
 
 task :default => :clean do
-  # MiniPortile#download_file_http is a monkey patch that removes the download
-  # progress indicator. This indicator relies on the 'Content Length' response
-  # headers, which is not set by GitHub
-  class MiniPortile
-    def download_file_http(url, full_path, _count)
-      filename = File.basename(full_path)
-      with_tempfile(filename, full_path) do |temp_file|
-        params = { 'Accept-Encoding' => 'identity' }
-        OpenURI.open_uri(url, 'rb', params) do |io|
-          temp_file.write(io.read)
-        end
-        output
-      end
-    end
-  end
-
   # Download and compile librdkafka
   recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
   # Use default homebrew openssl if we're on mac and the directory exists
   # and each of flags is not empty
-  if recipe.host&.include?("darwin") && Dir.exist?("/usr/local/opt/openssl")
-    ENV["CPPFLAGS"] = "-I/usr/local/opt/openssl/include" unless ENV["CPPFLAGS"]
-    ENV["LDFLAGS"] = "-L/usr/local/opt/openssl/lib" unless ENV["LDFLAGS"]
+  if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
+    ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
+    ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
   end
 
   recipe.files << {

data/lib/rdkafka/bindings.rb

@@ -108,6 +108,8 @@ module Rdkafka
     attach_function :rd_kafka_conf_set_opaque, [:pointer, :pointer], :void
     callback :stats_cb, [:pointer, :string, :int, :pointer], :int
     attach_function :rd_kafka_conf_set_stats_cb, [:pointer, :stats_cb], :void
+    callback :error_cb, [:pointer, :int, :string, :pointer], :void
+    attach_function :rd_kafka_conf_set_error_cb, [:pointer, :error_cb], :void
 
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
@@ -146,6 +148,15 @@ module Rdkafka
       0
     end
 
+    ErrorCallback = FFI::Function.new(
+      :void, [:pointer, :int, :string, :pointer]
+    ) do |_client_prr, err_code, reason, _opaque|
+      if Rdkafka::Config.error_callback
+        error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
+        Rdkafka::Config.error_callback.call(error)
+      end
+    end
+
     # Handle
 
     enum :kafka_type, [

data/lib/rdkafka/config.rb

@@ -10,6 +10,8 @@ module Rdkafka
     # @private
     @@statistics_callback = nil
     # @private
+    @@error_callback = nil
+    # @private
     @@opaques = {}
     # @private
     @@log_queue = Queue.new
@@ -28,6 +30,7 @@ module Rdkafka
       @@logger
     end
 
+
     # Returns a queue whose contents will be passed to the configured logger. Each entry
     # should follow the format [Logger::Severity, String]. The benefit over calling the
     # logger directly is that this is safe to use from trap contexts.
@@ -66,6 +69,25 @@ module Rdkafka
       @@statistics_callback
     end
 
+    # Set a callback that will be called every time the underlying client emits an error.
+    # If this callback is not set, global errors such as brokers becoming unavailable will only be sent to the logger, as defined by librdkafka.
+    # The callback is called with an instance of RdKafka::Error.
+    #
+    # @param callback [Proc, #call] The callback
+    #
+    # @return [nil]
+    def self.error_callback=(callback)
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
+      @@error_callback = callback
+    end
+
+    # Returns the current error callback, by default this is nil.
+    #
+    # @return [Proc, nil]
+    def self.error_callback
+      @@error_callback
+    end
+
     # @private
     def self.opaques
       @@opaques
@@ -221,6 +243,9 @@ module Rdkafka
 
         # Set stats callback
         Rdkafka::Bindings.rd_kafka_conf_set_stats_cb(config, Rdkafka::Bindings::StatsCallback)
+
+        # Set error callback
+        Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
       end
     end
 

data/lib/rdkafka/consumer.rb

@@ -498,11 +498,11 @@ module Rdkafka
     # Exception behavior is more complicated than with `each`, in that if
     # :yield_on_error is true, and an exception is raised during the
     # poll, and messages have already been received, they will be yielded to
-    # the caller before the exception is allowed to propogate.
+    # the caller before the exception is allowed to propagate.
     #
     # If you are setting either auto.commit or auto.offset.store to false in
     # the consumer configuration, then you should let yield_on_error keep its
-    # default value of false because you are gauranteed to see these messages
+    # default value of false because you are guaranteed to see these messages
     # again. However, if both auto.commit and auto.offset.store are set to
     # true, you should set yield_on_error to true so you can process messages
     # that you may or may not see again.
@@ -518,7 +518,7 @@ module Rdkafka
     # @yield [messages, pending_exception]
     # @yieldparam messages [Array] An array of received Message
     # @yieldparam pending_exception [Exception] normally nil, or an exception
-    # which will be propogated after processing of the partial batch is complete.
+    # which will be propagated after processing of the partial batch is complete.
     #
     # @return [nil]
     def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)

data/lib/rdkafka/producer/delivery_report.rb

@@ -11,7 +11,7 @@ module Rdkafka
       attr_reader :offset
 
       # Error in case happen during produce.
-      # @return [string]
+      # @return [String]
       attr_reader :error
 
       private

data/lib/rdkafka/producer.rb

@@ -1,3 +1,5 @@
+require "securerandom"
+
 module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
@@ -9,11 +11,12 @@ module Rdkafka
 
     # @private
     def initialize(native_kafka)
+      @id = SecureRandom.uuid
       @closing = false
       @native_kafka = native_kafka
 
       # Makes sure, that the producer gets closed before it gets GCed by Ruby
-      ObjectSpace.define_finalizer(self, proc { close })
+      ObjectSpace.define_finalizer(@id, proc { close })
 
       # Start thread to poll client for delivery callbacks
       @polling_thread = Thread.new do
@@ -41,6 +44,8 @@ module Rdkafka
 
     # Close this producer and wait for the internal poll queue to empty.
     def close
+      ObjectSpace.undefine_finalizer(@id)
+
       return unless @native_kafka
 
       # Indicate to polling thread that we're closing
@@ -70,8 +75,9 @@ module Rdkafka
     #
     # @param topic [String] The topic to produce to
     # @param payload [String,nil] The message's payload
-    # @param key [String] The message's key
+    # @param key [String, nil] The message's key
     # @param partition [Integer,nil] Optional partition to produce to
+    # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     # @param headers [Hash<String,String>] Optional message headers
     #

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.9.0"
-  LIBRDKAFKA_VERSION = "1.4.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "ae27ea3f3d0d32d29004e7f709efbba2666c5383a107cc45b3a1949486b2eb84"
+  VERSION = "0.11.1"
+  LIBRDKAFKA_VERSION = "1.8.2"
+  LIBRDKAFKA_SOURCE_SHA256 = "6a747d293a7a4613bd2897e28e8791476fbe1ae7361f2530a876e0fd483482a6"
 end

data/rdkafka.gemspec

@@ -14,15 +14,17 @@ Gem::Specification.new do |gem|
   gem.name = 'rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.4'
+  gem.required_ruby_version = '>= 2.6'
   gem.extensions = %w(ext/Rakefile)
 
-  gem.add_dependency 'ffi', '~> 1.9'
-  gem.add_dependency 'mini_portile2', '~> 2.1'
-  gem.add_dependency 'rake', '>= 12.3'
+  gem.add_dependency 'ffi', '~> 1.15'
+  gem.add_dependency 'mini_portile2', '~> 2.6'
+  gem.add_dependency 'rake', '> 12'
 
-  gem.add_development_dependency 'pry', '~> 0.10'
+  gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rspec', '~> 3.5'
-  gem.add_development_dependency 'rake', '~> 12.0'
-  gem.add_development_dependency 'simplecov', '~> 0.15'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'guard'
+  gem.add_development_dependency 'guard-rspec'
 end

data/spec/rdkafka/abstract_handle_spec.rb

@@ -111,4 +111,3 @@ describe Rdkafka::AbstractHandle do
     end
   end
 end
-

data/spec/rdkafka/admin_spec.rb

@@ -52,7 +52,7 @@ describe Rdkafka::Admin do
       end
 
       describe "with an invalid partition count" do
-        let(:topic_partition_count) { -1 }
+        let(:topic_partition_count) { -999 }
 
         it "raises an exception" do
           expect {

data/spec/rdkafka/bindings_spec.rb

@@ -100,4 +100,28 @@ describe Rdkafka::Bindings do
       end
     end
   end
+
+  describe "error callback" do
+    context "without an error callback" do
+      it "should do nothing" do
+        expect {
+          Rdkafka::Bindings::ErrorCallback.call(nil, 1, "error", nil)
+        }.not_to raise_error
+      end
+    end
+
+    context "with an error callback" do
+      before do
+        Rdkafka::Config.error_callback = lambda do |error|
+          $received_error = error
+        end
+      end
+
+      it "should call the error callback with an Rdkafka::Error" do
+        Rdkafka::Bindings::ErrorCallback.call(nil, 8, "Broker not available", nil)
+        expect($received_error.code).to eq(:broker_not_available)
+        expect($received_error.broker_message).to eq("Broker not available")
+      end
+    end
+  end
 end

data/spec/rdkafka/config_spec.rb

@@ -64,6 +64,37 @@ describe Rdkafka::Config do
     end
   end
 
+  context "error callback" do
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.error_callback = lambda do |error|
+            puts error
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.error_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
+        end
+        expect {
+          Rdkafka::Config.error_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.error_callback).to respond_to :call
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        Rdkafka::Config.error_callback = 'a string'
+      }.to raise_error(TypeError)
+    end
+  end
+
   context "configuration" do
     it "should store configuration" do
       config = Rdkafka::Config.new
@@ -77,7 +108,7 @@ describe Rdkafka::Config do
     end
 
     it "should create a consumer with valid config" do
-      consumer = rdkafka_config.consumer
+      consumer = rdkafka_consumer_config.consumer
       expect(consumer).to be_a Rdkafka::Consumer
       consumer.close
     end
@@ -105,7 +136,7 @@ describe Rdkafka::Config do
     end
 
     it "should create a producer with valid config" do
-      producer = rdkafka_config.producer
+      producer = rdkafka_consumer_config.producer
       expect(producer).to be_a Rdkafka::Producer
       producer.close
     end

data/spec/rdkafka/consumer_spec.rb

@@ -3,9 +3,8 @@ require "ostruct"
 require 'securerandom'
 
 describe Rdkafka::Consumer do
-  let(:config) { rdkafka_config }
-  let(:consumer) { config.consumer }
-  let(:producer) { config.producer }
+  let(:consumer) { rdkafka_consumer_config.consumer }
+  let(:producer) { rdkafka_producer_config.producer }
 
   after { consumer.close }
   after { producer.close }
@@ -328,7 +327,7 @@ describe Rdkafka::Consumer do
       before :all do
         # Make sure there are some messages.
         handles = []
-        producer = rdkafka_config.producer
+        producer = rdkafka_producer_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -404,7 +403,7 @@ describe Rdkafka::Consumer do
           config = {}
           config[:'enable.auto.offset.store'] = false
           config[:'enable.auto.commit'] = false
-          @new_consumer = rdkafka_config(config).consumer
+          @new_consumer = rdkafka_consumer_config(config).consumer
           @new_consumer.subscribe("consume_test_topic")
           wait_for_assignment(@new_consumer)
         end
@@ -459,13 +458,13 @@ describe Rdkafka::Consumer do
   end
 
   describe "#lag" do
-    let(:config) { rdkafka_config(:"enable.partition.eof" => true) }
+    let(:consumer) { rdkafka_consumer_config(:"enable.partition.eof" => true).consumer }
 
     it "should calculate the consumer lag" do
       # Make sure there's a message in every partition and
       # wait for the message to make sure everything is committed.
       (0..2).each do |i|
-        report = producer.produce(
+        producer.produce(
           topic:     "consume_test_topic",
           key:       "key lag #{i}",
           partition: i
@@ -508,7 +507,7 @@ describe Rdkafka::Consumer do
 
       # Produce message on every topic again
       (0..2).each do |i|
-        report = producer.produce(
+        producer.produce(
           topic:     "consume_test_topic",
           key:       "key lag #{i}",
           partition: i
@@ -724,6 +723,8 @@ describe Rdkafka::Consumer do
       #
       # This is, in effect, an integration test and the subsequent specs are
       # unit tests.
+      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      create_topic_handle.wait(max_wait_timeout: 15.0)
       consumer.subscribe(topic_name)
       produce_n 42
       all_yields = []
@@ -777,6 +778,8 @@ describe Rdkafka::Consumer do
     end
 
     it "should yield [] if nothing is received before the timeout" do
+      create_topic_handle = rdkafka_config.admin.create_topic(topic_name, 1, 1)
+      create_topic_handle.wait(max_wait_timeout: 15.0)
       consumer.subscribe(topic_name)
       consumer.each_batch do |batch|
         expect(batch).to eq([])
@@ -820,8 +823,12 @@ describe Rdkafka::Consumer do
 
     context "error raised from poll and yield_on_error is true" do
       it "should yield buffered exceptions on rebalance, then break" do
-        config = rdkafka_config({:"enable.auto.commit" => false,
-                                 :"enable.auto.offset.store" => false })
+        config = rdkafka_consumer_config(
+          {
+            :"enable.auto.commit" => false,
+            :"enable.auto.offset.store" => false
+          }
+        )
         consumer = config.consumer
         consumer.subscribe(topic_name)
         loop_count = 0
@@ -860,8 +867,12 @@ describe Rdkafka::Consumer do
 
     context "error raised from poll and yield_on_error is false" do
       it "should yield buffered exceptions on rebalance, then break" do
-        config = rdkafka_config({:"enable.auto.commit" => false,
-                                 :"enable.auto.offset.store" => false })
+        config = rdkafka_consumer_config(
+          {
+            :"enable.auto.commit" => false,
+            :"enable.auto.offset.store" => false
+          }
+        )
         consumer = config.consumer
         consumer.subscribe(topic_name)
         loop_count = 0
@@ -898,51 +909,64 @@ describe Rdkafka::Consumer do
   end
 
   describe "a rebalance listener" do
-    it "should get notifications" do
-      listener = Struct.new(:queue) do
-        def on_partitions_assigned(consumer, list)
-          collect(:assign, list)
-        end
+    let(:consumer) do
+      config = rdkafka_consumer_config
+      config.consumer_rebalance_listener = listener
+      config.consumer
+    end
 
-        def on_partitions_revoked(consumer, list)
-          collect(:revoke, list)
-        end
+    context "with a working listener" do
+      let(:listener) do
+        Struct.new(:queue) do
+          def on_partitions_assigned(consumer, list)
+            collect(:assign, list)
+          end
 
-        def collect(name, list)
-          partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
-          queue << ([name] + partitions)
-        end
-      end.new([])
+          def on_partitions_revoked(consumer, list)
+            collect(:revoke, list)
+          end
+
+          def collect(name, list)
+            partitions = list.to_h.map { |key, values| [key, values.map(&:partition)] }.flatten
+            queue << ([name] + partitions)
+          end
+        end.new([])
+      end
 
-      notify_listener(listener)
+      it "should get notifications" do
+        notify_listener(listener)
 
-      expect(listener.queue).to eq([
-        [:assign, "consume_test_topic", 0, 1, 2],
-        [:revoke, "consume_test_topic", 0, 1, 2]
-      ])
+        expect(listener.queue).to eq([
+          [:assign, "consume_test_topic", 0, 1, 2],
+          [:revoke, "consume_test_topic", 0, 1, 2]
+        ])
+      end
     end
 
-    it 'should handle callback exceptions' do
-      listener = Struct.new(:queue) do
-        def on_partitions_assigned(consumer, list)
-          queue << :assigned
-          raise 'boom'
-        end
+    context "with a broken listener" do
+      let(:listener) do
+        Struct.new(:queue) do
+          def on_partitions_assigned(consumer, list)
+            queue << :assigned
+            raise 'boom'
+          end
 
-        def on_partitions_revoked(consumer, list)
-          queue << :revoked
-          raise 'boom'
-        end
-      end.new([])
+          def on_partitions_revoked(consumer, list)
+            queue << :revoked
+            raise 'boom'
+          end
+        end.new([])
+      end
 
-      notify_listener(listener)
+      it 'should handle callback exceptions' do
+        notify_listener(listener)
 
-      expect(listener.queue).to eq([:assigned, :revoked])
+        expect(listener.queue).to eq([:assigned, :revoked])
+      end
     end
 
     def notify_listener(listener)
       # 1. subscribe and poll
-      config.consumer_rebalance_listener = listener
       consumer.subscribe("consume_test_topic")
       wait_for_assignment(consumer)
       consumer.poll(100)

data/spec/rdkafka/metadata_spec.rb

@@ -2,7 +2,7 @@ require "spec_helper"
 require "securerandom"
 
 describe Rdkafka::Metadata do
-  let(:config)        { rdkafka_config }
+  let(:config)        { rdkafka_consumer_config }
   let(:native_config) { config.send(:native_config) }
   let(:native_kafka)  { config.send(:native_kafka, native_config, :rd_kafka_consumer) }
 
@@ -18,7 +18,7 @@ describe Rdkafka::Metadata do
       it "raises an appropriate exception" do
         expect {
           described_class.new(native_kafka, topic_name)
-        }.to raise_exception(Rdkafka::RdkafkaError, "Broker: Leader not available (leader_not_available)")
+        }.to raise_exception(Rdkafka::RdkafkaError, "Broker: Unknown topic or partition (unknown_topic_or_part)")
       end
     end
 

data/spec/rdkafka/producer_spec.rb

@@ -1,8 +1,9 @@
 require "spec_helper"
+require "zlib"
 
 describe Rdkafka::Producer do
-  let(:producer) { rdkafka_config.producer }
-  let(:consumer) { rdkafka_config.consumer }
+  let(:producer) { rdkafka_producer_config.producer }
+  let(:consumer) { rdkafka_consumer_config.consumer }
 
   after do
     # Registry should always end up being empty
@@ -388,7 +389,7 @@ describe Rdkafka::Producer do
       reader.close
 
       # Avoids sharing the socket between processes.
-      producer = rdkafka_config.producer
+      producer = rdkafka_producer_config.producer
 
       handle = producer.produce(
         topic:   "produce_test_topic",

data/spec/spec_helper.rb

@@ -8,27 +8,57 @@ end
 require "pry"
 require "rspec"
 require "rdkafka"
+require "timeout"
 
-def rdkafka_config(config_overrides={})
-  config = {
+def rdkafka_base_config
+  {
     :"api.version.request" => false,
     :"broker.version.fallback" => "1.0",
     :"bootstrap.servers" => "localhost:9092",
-    :"group.id" => "ruby-test-#{Random.new.rand(0..1_000_000)}",
-    :"auto.offset.reset" => "earliest",
-    :"enable.partition.eof" => false
   }
+end
+
+def rdkafka_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Merge overrides
+  config.merge!(config_overrides)
+  # Return it
+  Rdkafka::Config.new(config)
+end
+
+def rdkafka_consumer_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Add consumer specific fields to it
+  config[:"auto.offset.reset"] = "earliest"
+  config[:"enable.partition.eof"] = false
+  config[:"group.id"] = "ruby-test-#{Random.new.rand(0..1_000_000)}"
+  # Enable debug mode if required
+  if ENV["DEBUG_CONSUMER"]
+    config[:debug] = "cgrp,topic,fetch"
+  end
+  # Merge overrides
+  config.merge!(config_overrides)
+  # Return it
+  Rdkafka::Config.new(config)
+end
+
+def rdkafka_producer_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Enable debug mode if required
   if ENV["DEBUG_PRODUCER"]
     config[:debug] = "broker,topic,msg"
-  elsif ENV["DEBUG_CONSUMER"]
-    config[:debug] = "cgrp,topic,fetch"
   end
+  # Merge overrides
   config.merge!(config_overrides)
+  # Return it
   Rdkafka::Config.new(config)
 end
 
 def new_native_client
-  config = rdkafka_config
+  config = rdkafka_consumer_config
   config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
 end
 
@@ -42,7 +72,7 @@ end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
   new_consumer = !!consumer
-  consumer ||= rdkafka_config.consumer
+  consumer ||= rdkafka_consumer_config.consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
   loop do
@@ -75,6 +105,9 @@ def wait_for_unassignment(consumer)
 end
 
 RSpec.configure do |config|
+  config.filter_run focus: true
+  config.run_all_when_everything_filtered = true
+
   config.before(:suite) do
     admin = rdkafka_config.admin
     {
@@ -95,4 +128,12 @@ RSpec.configure do |config|
     end
     admin.close
   end
+
+  config.around(:each) do |example|
+    # Timeout specs after a minute. If they take longer
+    # they are probably stuck
+    Timeout::timeout(60) do
+      example.run
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.11.1
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-23 00:00:00.000000000 Z
+date: 2021-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -16,56 +16,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.15'
 - !ruby/object:Gem::Dependency
   name: mini_portile2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -84,46 +84,78 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0'
 description: Modern Kafka client library for Ruby based on librdkafka
 email:
 - thijs@appsignal.com
-executables: []
+executables:
+- console
 extensions:
 - ext/Rakefile
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - ".semaphore/semaphore.yml"
 - ".yardopts"
 - CHANGELOG.md
 - Gemfile
+- Guardfile
 - LICENSE
 - README.md
 - Rakefile
+- bin/console
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
@@ -180,14 +212,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.