rdkafka 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d09f2751b883400c550fee28fca85ffdd3e9f8090a9ba8e7bb22eadcba5c05c6
-  data.tar.gz: 107a5b723b40ec43b10f02b1993f42ff4b30a4ba16efc21611efb860de798a3a
+  metadata.gz: 22bda74a815bebd487179ac5ec3faa4566752f2c87945ca5c461c427633daf1f
+  data.tar.gz: '0802ca4118ed248fa11226a2c1b2140d073d230c02e3696a03e462fe59cb129e'
 SHA512:
-  metadata.gz: 373bd71c9e1f3c1fadd4deb9e67bec7aa5821e0a7835c851f1bb64fb1ad142692145e01efa62ac357d333eebb76cef7d676ddfa53ef3a9cd08720060b4bc0937
-  data.tar.gz: ec6d8932a2987e419f6fd731ff038cb88d9d898eab75d7eff82a6e606fcb71c249dd20f6d200bda44629f4bb81550c06ef4e8083ac1b4736c57932ab2486fdbc
+  metadata.gz: 4d3b7042fa2290386d4feaaa01c4289bc858042d66806157066257f282c84db459e80e32a2613efd3452a46c63faaff7fad9f110fb153c3385cef00fb6017f27
+  data.tar.gz: e431a320535df8293e099e1b25bf0e6da68caf9a641621faf847dd206093dd9973b41f03f95412c43a740b50a0ef70e7caad418ff20c07218f19342d1cb3826a

data/.semaphore/semaphore.yml

@@ -3,7 +3,7 @@ name: Rdkafka Ruby
 
 agent:
   machine:
-    type: e1-standard-2
+    type: e1-standard-4
     os_image: ubuntu1804
 
 blocks:
@@ -13,11 +13,11 @@ blocks:
         - name: bundle exec rspec
           matrix:
             - env_var: RUBY_VERSION
-              values: [ "2.5.8", "2.6.6", "2.7.2", "jruby-9.2.13.0" ]
+              values: [ "2.5.8", "2.6.6", "2.7.2", "3.0.0", "jruby-9.2.13.0" ]
           commands:
             - sem-version ruby $RUBY_VERSION
             - checkout
-            - bundle install --path vendor/bundl
+            - bundle install --path vendor/bundle
             - cd ext && bundle exec rake && cd ..
             - docker-compose up -d --no-recreate
             - bundle exec rspec

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+# 0.9.0
+* Fixes for Ruby 3.0
+* Allow any callable object for callbacks (gremerritt)
+* Reduce memory allocations in Rdkafka::Producer#produce (jturkel)
+* Use queue as log callback to avoid unsafe calls from trap context (breunigs)
+* Allow passing in topic configuration on create_topic (dezka)
+* Add each_batch method to consumer (mgrosso)
+
 # 0.8.1
 * Fix topic_flag behaviour and add tests for Metadata (geoff2k)
 * Add topic admin interface (geoff2k)

data/ext/README.md

@@ -9,3 +9,10 @@ To update the `librdkafka` version follow the following steps:
   version number and asset checksum for `tar.gz`.
 * Change the version in `lib/rdkafka/version.rb`
 * Change the `sha256` in `lib/rdkafka/version.rb`
+* Run `bundle exec rake` in the `ext` directory to download and build
+  the new version
+* Run `docker-compose pull` in the main gem directory to ensure the docker
+  images used by the tests and run `docker-compose up`
+* Finally, run `bundle exec rspec` in the main gem directory to execute
+  the test suite to detect any regressions that may have been introduced
+  by the update

data/ext/Rakefile

@@ -1,6 +1,7 @@
 require File.expand_path('../../lib/rdkafka/version', __FILE__)
 require "mini_portile2"
 require "fileutils"
+require "open-uri"
 
 task :default => :clean do
   # MiniPortile#download_file_http is a monkey patch that removes the download

data/lib/rdkafka/admin.rb

@@ -34,9 +34,10 @@ module Rdkafka
     #
     # @raise [ConfigError] When the partition count or replication factor are out of valid range
     # @raise [RdkafkaError] When the topic name is invalid or the topic already exists
+    # @raise [RdkafkaError] When the topic configuration is invalid
     #
     # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
-    def create_topic(topic_name, partition_count, replication_factor)
+    def create_topic(topic_name, partition_count, replication_factor, topic_config={})
 
       # Create a rd_kafka_NewTopic_t representing the new topic
       error_buffer = FFI::MemoryPointer.from_string(" " * 256)
@@ -51,6 +52,16 @@ module Rdkafka
         raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
       end
 
+      unless topic_config.nil?
+        topic_config.each do |key, value|
+          Rdkafka::Bindings.rd_kafka_NewTopic_set_config(
+            new_topic_ptr,
+            key.to_s,
+            value.to_s
+          )
+        end
+      end
+
       # Note that rd_kafka_CreateTopics can create more than one topic at a time
       pointer_array = [new_topic_ptr]
       topics_array_ptr = FFI::MemoryPointer.new(:pointer)

data/lib/rdkafka/bindings.rb

@@ -130,7 +130,7 @@ module Rdkafka
                  else
                    Logger::UNKNOWN
                  end
-      Rdkafka::Config.logger.add(severity) { "rdkafka: #{line}" }
+      Rdkafka::Config.log_queue << [severity, "rdkafka: #{line}"]
     end
 
     StatsCallback = FFI::Function.new(
@@ -252,6 +252,7 @@ module Rdkafka
 
     attach_function :rd_kafka_CreateTopics, [:pointer, :pointer, :size_t, :pointer, :pointer], :void
     attach_function :rd_kafka_NewTopic_new, [:pointer, :size_t, :size_t, :pointer, :size_t], :pointer
+    attach_function :rd_kafka_NewTopic_set_config, [:pointer, :string, :string], :int32
     attach_function :rd_kafka_NewTopic_destroy, [:pointer], :void
     attach_function :rd_kafka_event_CreateTopics_result, [:pointer], :pointer
     attach_function :rd_kafka_CreateTopics_result_topics, [:pointer, :pointer], :pointer

data/lib/rdkafka/config.rb

@@ -11,6 +11,15 @@ module Rdkafka
     @@statistics_callback = nil
     # @private
     @@opaques = {}
+    # @private
+    @@log_queue = Queue.new
+
+    Thread.start do
+      loop do
+        severity, msg = @@log_queue.pop
+        @@logger.add(severity, msg)
+      end
+    end
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -19,6 +28,15 @@ module Rdkafka
       @@logger
     end
 
+    # Returns a queue whose contents will be passed to the configured logger. Each entry
+    # should follow the format [Logger::Severity, String]. The benefit over calling the
+    # logger directly is that this is safe to use from trap contexts.
+    #
+    # @return [Queue]
+    def self.log_queue
+      @@log_queue
+    end
+
     # Set the logger that will be used for all logging output by this library.
     #
     # @param logger [Logger] The logger to be used
@@ -33,11 +51,11 @@ module Rdkafka
     # You can configure if and how often this happens using `statistics.interval.ms`.
     # The callback is called with a hash that's documented here: https://github.com/edenhill/librdkafka/blob/master/STATISTICS.md
     #
-    # @param callback [Proc] The callback
+    # @param callback [Proc, #call] The callback
     #
     # @return [nil]
     def self.statistics_callback=(callback)
-      raise TypeError.new("Callback has to be a proc or lambda") unless callback.is_a? Proc
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
       @@statistics_callback = callback
     end
 

data/lib/rdkafka/consumer.rb

@@ -471,5 +471,95 @@ module Rdkafka
     def closed_consumer_check(method)
       raise Rdkafka::ClosedConsumerError.new(method) if @native_kafka.nil?
     end
+
+    # Poll for new messages and yield them in batches that may contain
+    # messages from more than one partition.
+    #
+    # Rather than yield each message immediately as soon as it is received,
+    # each_batch will attempt to wait for as long as `timeout_ms` in order
+    # to create a batch of up to but no more than `max_items` in size.
+    #
+    # Said differently, if more than `max_items` are available within
+    # `timeout_ms`, then `each_batch` will yield early with `max_items` in the
+    # array, but if `timeout_ms` passes by with fewer messages arriving, it
+    # will yield an array of fewer messages, quite possibly zero.
+    #
+    # In order to prevent wrongly auto committing many messages at once across
+    # possibly many partitions, callers must explicitly indicate which messages
+    # have been successfully processed as some consumed messages may not have
+    # been yielded yet. To do this, the caller should set
+    # `enable.auto.offset.store` to false and pass processed messages to
+    # {store_offset}. It is also possible, though more complex, to set
+    # 'enable.auto.commit' to false and then pass a manually assembled
+    # TopicPartitionList to {commit}.
+    #
+    # As with `each`, iteration will end when the consumer is closed.
+    #
+    # Exception behavior is more complicated than with `each`, in that if
+    # :yield_on_error is true, and an exception is raised during the
+    # poll, and messages have already been received, they will be yielded to
+    # the caller before the exception is allowed to propogate.
+    #
+    # If you are setting either auto.commit or auto.offset.store to false in
+    # the consumer configuration, then you should let yield_on_error keep its
+    # default value of false because you are gauranteed to see these messages
+    # again. However, if both auto.commit and auto.offset.store are set to
+    # true, you should set yield_on_error to true so you can process messages
+    # that you may or may not see again.
+    #
+    # @param max_items [Integer] Maximum size of the yielded array of messages
+    #
+    # @param bytes_threshold [Integer] Threshold number of total message bytes in the yielded array of messages
+    #
+    # @param timeout_ms [Integer] max time to wait for up to max_items
+    #
+    # @raise [RdkafkaError] When polling fails
+    #
+    # @yield [messages, pending_exception]
+    # @yieldparam messages [Array] An array of received Message
+    # @yieldparam pending_exception [Exception] normally nil, or an exception
+    # which will be propogated after processing of the partial batch is complete.
+    #
+    # @return [nil]
+    def each_batch(max_items: 100, bytes_threshold: Float::INFINITY, timeout_ms: 250, yield_on_error: false, &block)
+      closed_consumer_check(__method__)
+      slice = []
+      bytes = 0
+      end_time = monotonic_now + timeout_ms / 1000.0
+      loop do
+        break if @closing
+        max_wait = end_time - monotonic_now
+        max_wait_ms = if max_wait <= 0
+                        0  # should not block, but may retrieve a message
+                      else
+                        (max_wait * 1000).floor
+                      end
+        message = nil
+        begin
+          message = poll max_wait_ms
+        rescue Rdkafka::RdkafkaError => error
+          raise unless yield_on_error
+          raise if slice.empty?
+          yield slice.dup, error
+          raise
+        end
+        if message
+          slice << message
+          bytes += message.payload.bytesize
+        end
+        if slice.size == max_items || bytes >= bytes_threshold || monotonic_now >= end_time - 0.001
+          yield slice.dup, nil
+          slice.clear
+          bytes = 0
+          end_time = monotonic_now + timeout_ms / 1000.0
+        end
+      end
+    end
+
+    private
+    def monotonic_now
+      # needed because Time.now can go backwards
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
   end
 end

data/lib/rdkafka/producer.rb

@@ -31,11 +31,11 @@ module Rdkafka
     # Set a callback that will be called every time a message is successfully produced.
     # The callback is called with a {DeliveryReport}
     #
-    # @param callback [Proc] The callback
+    # @param callback [Proc, #call] The callback
     #
     # @return [nil]
     def delivery_callback=(callback)
-      raise TypeError.new("Callback has to be a proc or lambda") unless callback.is_a? Proc
+      raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
       @delivery_callback = callback
     end
 
@@ -140,16 +140,14 @@ module Rdkafka
         headers.each do |key0, value0|
           key = key0.to_s
           value = value0.to_s
-          args += [
-            :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER,
-            :string, key,
-            :pointer, value,
-            :size_t, value.bytes.size
-          ]
+          args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_HEADER
+          args << :string << key
+          args << :pointer << value
+          args << :size_t << value.bytes.size
         end
       end
 
-      args += [:int, Rdkafka::Bindings::RD_KAFKA_VTYPE_END]
+      args << :int << Rdkafka::Bindings::RD_KAFKA_VTYPE_END
 
       # Produce the message
       response = Rdkafka::Bindings.rd_kafka_producev(

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.8.1"
+  VERSION = "0.9.0"
   LIBRDKAFKA_VERSION = "1.4.0"
   LIBRDKAFKA_SOURCE_SHA256 = "ae27ea3f3d0d32d29004e7f709efbba2666c5383a107cc45b3a1949486b2eb84"
 end

data/spec/rdkafka/admin_spec.rb

@@ -14,6 +14,8 @@ describe Rdkafka::Admin do
   let(:topic_name)               { "test-topic-#{Random.new.rand(0..1_000_000)}" }
   let(:topic_partition_count)    { 3 }
   let(:topic_replication_factor) { 1 }
+  let(:topic_config)             { {"cleanup.policy" => "compact", "min.cleanable.dirty.ratio" => 0.8} }
+  let(:invalid_topic_config)     { {"cleeeeenup.policee" => "campact"} }
 
   describe "#create_topic" do
     describe "called with invalid input" do
@@ -68,6 +70,15 @@ describe Rdkafka::Admin do
           }.to raise_error Rdkafka::Config::ConfigError, /replication_factor out of expected range/
         end
       end
+
+      describe "with an invalid topic configuration" do
+        it "doesn't create the topic" do
+          create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor, invalid_topic_config)
+          expect {
+            create_topic_handle.wait(max_wait_timeout: 15.0)
+          }.to raise_error Rdkafka::RdkafkaError, /Broker: Configuration is invalid \(invalid_config\)/
+        end
+      end
     end
 
     context "edge case" do
@@ -97,7 +108,7 @@ describe Rdkafka::Admin do
     end
 
     it "creates a topic" do
-      create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+      create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor, topic_config)
       create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)
       expect(create_topic_report.error_string).to be_nil
       expect(create_topic_report.result_name).to eq(topic_name)

data/spec/rdkafka/bindings_spec.rb

@@ -25,39 +25,39 @@ describe Rdkafka::Bindings do
   end
 
   describe "log callback" do
-    let(:log) { StringIO.new }
+    let(:log_queue) { Rdkafka::Config.log_queue }
     before do
-      Rdkafka::Config.logger = Logger.new(log)
+      allow(log_queue).to receive(:<<)
     end
 
     it "should log fatal messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 0, nil, "log line")
-      expect(log.string).to include "FATAL -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
     end
 
     it "should log error messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 3, nil, "log line")
-      expect(log.string).to include "ERROR -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::ERROR, "rdkafka: log line"])
     end
 
     it "should log warning messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 4, nil, "log line")
-      expect(log.string).to include "WARN -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::WARN, "rdkafka: log line"])
     end
 
     it "should log info messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 5, nil, "log line")
-      expect(log.string).to include "INFO -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::INFO, "rdkafka: log line"])
     end
 
     it "should log debug messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 7, nil, "log line")
-      expect(log.string).to include "DEBUG -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::DEBUG, "rdkafka: log line"])
     end
 
     it "should log unknown messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 100, nil, "log line")
-      expect(log.string).to include "ANY -- : rdkafka: log line"
+      expect(log_queue).to have_received(:<<).with([Logger::UNKNOWN, "rdkafka: log line"])
     end
   end
 

data/spec/rdkafka/config_spec.rb

@@ -18,19 +18,46 @@ describe Rdkafka::Config do
         Rdkafka::Config.logger = nil
       }.to raise_error(Rdkafka::Config::NoLoggerError)
     end
+
+    it "supports logging queue" do
+      log = StringIO.new
+      Rdkafka::Config.logger = Logger.new(log)
+
+      Rdkafka::Config.log_queue << [Logger::FATAL, "I love testing"]
+      20.times do
+        break if log.string != ""
+        sleep 0.05
+      end
+
+      expect(log.string).to include "FATAL -- : I love testing"
+    end
   end
 
   context "statistics callback" do
-    it "should set the callback" do
-      expect {
-        Rdkafka::Config.statistics_callback = lambda do |stats|
-          puts stats
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.statistics_callback = lambda do |stats|
+            puts stats
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.statistics_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
         end
-      }.not_to raise_error
-      expect(Rdkafka::Config.statistics_callback).to be_a Proc
+        expect {
+          Rdkafka::Config.statistics_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.statistics_callback).to respond_to :call
+      end
     end
 
-    it "should not accept a callback that's not a proc" do
+    it "should not accept a callback that's not callable" do
       expect {
         Rdkafka::Config.statistics_callback = 'a string'
       }.to raise_error(TypeError)

data/spec/rdkafka/consumer_spec.rb

@@ -1,5 +1,6 @@
 require "spec_helper"
 require "ostruct"
+require 'securerandom'
 
 describe Rdkafka::Consumer do
   let(:config) { rdkafka_config }
@@ -271,6 +272,14 @@ describe Rdkafka::Consumer do
   describe "#close" do
     it "should close a consumer" do
       consumer.subscribe("consume_test_topic")
+      100.times do |i|
+        report = producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload #{i}",
+          key:       "key #{i}",
+          partition: 0
+        ).wait
+      end
       consumer.close
       expect {
         consumer.poll(100)
@@ -670,6 +679,224 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#each_batch" do
+    let(:message_payload) { 'a' * 10 }
+
+    before do
+      @topic = SecureRandom.base64(10).tr('+=/', '')
+    end
+
+    after do
+      @topic = nil
+    end
+
+    def topic_name
+      @topic
+    end
+
+    def produce_n(n)
+      handles = []
+      n.times do |i|
+        handles << producer.produce(
+          topic:     topic_name,
+          payload:   Time.new.to_f.to_s,
+          key:       i.to_s,
+          partition: 0
+        )
+      end
+      handles.each(&:wait)
+    end
+
+    def new_message
+      instance_double("Rdkafka::Consumer::Message").tap do |message|
+        allow(message).to receive(:payload).and_return(message_payload)
+      end
+    end
+
+    it "retrieves messages produced into a topic" do
+      # This is the only each_batch test that actually produces real messages
+      # into a topic in the real kafka of the container.
+      #
+      # The other tests stub 'poll' which makes them faster and more reliable,
+      # but it makes sense to keep a single test with a fully integrated flow.
+      # This will help to catch breaking changes in the behavior of 'poll',
+      # libdrkafka, or Kafka.
+      #
+      # This is, in effect, an integration test and the subsequent specs are
+      # unit tests.
+      consumer.subscribe(topic_name)
+      produce_n 42
+      all_yields = []
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 42
+      end
+      expect(all_yields.flatten.first).to be_a Rdkafka::Consumer::Message
+      expect(all_yields.flatten.size).to eq 42
+      expect(all_yields.size).to be > 4
+      expect(all_yields.flatten.map(&:key)).to eq (0..41).map { |x| x.to_s }
+    end
+
+    it "should batch poll results and yield arrays of messages" do
+      consumer.subscribe(topic_name)
+      all_yields = []
+      expect(consumer)
+        .to receive(:poll)
+        .exactly(10).times
+        .and_return(new_message)
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 10
+      end
+      expect(all_yields.first).to be_instance_of(Array)
+      expect(all_yields.flatten.size).to eq 10
+      non_empty_yields = all_yields.reject { |batch| batch.empty? }
+      expect(non_empty_yields.size).to be < 10
+    end
+
+    it "should yield a partial batch if the timeout is hit with some messages" do
+      consumer.subscribe(topic_name)
+      poll_count = 0
+      expect(consumer)
+        .to receive(:poll)
+        .at_least(3).times do
+        poll_count = poll_count + 1
+        if poll_count > 2
+          sleep 0.1
+          nil
+        else
+          new_message
+        end
+      end
+      all_yields = []
+      consumer.each_batch(max_items: 10) do |batch|
+        all_yields << batch
+        break if all_yields.flatten.size >= 2
+      end
+      expect(all_yields.flatten.size).to eq 2
+    end
+
+    it "should yield [] if nothing is received before the timeout" do
+      consumer.subscribe(topic_name)
+      consumer.each_batch do |batch|
+        expect(batch).to eq([])
+        break
+      end
+    end
+
+    it "should yield batchs of max_items in size if messages are already fetched" do
+      yielded_batches = []
+      expect(consumer)
+        .to receive(:poll)
+        .with(anything)
+        .exactly(20).times
+        .and_return(new_message)
+
+      consumer.each_batch(max_items: 10, timeout_ms: 500) do |batch|
+        yielded_batches << batch
+        break if yielded_batches.flatten.size >= 20
+        break if yielded_batches.size >= 20 # so failure doesn't hang
+      end
+      expect(yielded_batches.size).to eq 2
+      expect(yielded_batches.map(&:size)).to eq 2.times.map { 10 }
+    end
+
+    it "should yield batchs as soon as bytes_threshold is hit" do
+      yielded_batches = []
+      expect(consumer)
+        .to receive(:poll)
+        .with(anything)
+        .exactly(20).times
+        .and_return(new_message)
+
+      consumer.each_batch(bytes_threshold: message_payload.size * 4, timeout_ms: 500) do |batch|
+        yielded_batches << batch
+        break if yielded_batches.flatten.size >= 20
+        break if yielded_batches.size >= 20 # so failure doesn't hang
+      end
+      expect(yielded_batches.size).to eq 5
+      expect(yielded_batches.map(&:size)).to eq 5.times.map { 4 }
+    end
+
+    context "error raised from poll and yield_on_error is true" do
+      it "should yield buffered exceptions on rebalance, then break" do
+        config = rdkafka_config({:"enable.auto.commit" => false,
+                                 :"enable.auto.offset.store" => false })
+        consumer = config.consumer
+        consumer.subscribe(topic_name)
+        loop_count = 0
+        batches_yielded = []
+        exceptions_yielded = []
+        each_batch_iterations = 0
+        poll_count = 0
+        expect(consumer)
+          .to receive(:poll)
+          .with(anything)
+          .exactly(3).times
+          .and_wrap_original do |method, *args|
+              poll_count = poll_count + 1
+              if poll_count == 3
+                raise Rdkafka::RdkafkaError.new(27,
+                    "partitions ... too ... heavy ... must ... rebalance")
+              else
+                new_message
+              end
+          end
+        expect {
+          consumer.each_batch(max_items: 30, yield_on_error: true) do |batch, pending_error|
+            batches_yielded << batch
+            exceptions_yielded << pending_error
+            each_batch_iterations = each_batch_iterations + 1
+          end
+        }.to raise_error(Rdkafka::RdkafkaError)
+        expect(poll_count).to eq 3
+        expect(each_batch_iterations).to eq 1
+        expect(batches_yielded.size).to eq 1
+        expect(batches_yielded.first.size).to eq 2
+        expect(exceptions_yielded.flatten.size).to eq 1
+        expect(exceptions_yielded.flatten.first).to be_instance_of(Rdkafka::RdkafkaError)
+      end
+    end
+
+    context "error raised from poll and yield_on_error is false" do
+      it "should yield buffered exceptions on rebalance, then break" do
+        config = rdkafka_config({:"enable.auto.commit" => false,
+                                 :"enable.auto.offset.store" => false })
+        consumer = config.consumer
+        consumer.subscribe(topic_name)
+        loop_count = 0
+        batches_yielded = []
+        exceptions_yielded = []
+        each_batch_iterations = 0
+        poll_count = 0
+        expect(consumer)
+          .to receive(:poll)
+          .with(anything)
+          .exactly(3).times
+          .and_wrap_original do |method, *args|
+              poll_count = poll_count + 1
+              if poll_count == 3
+                raise Rdkafka::RdkafkaError.new(27,
+                    "partitions ... too ... heavy ... must ... rebalance")
+              else
+                new_message
+              end
+          end
+        expect {
+          consumer.each_batch(max_items: 30, yield_on_error: false) do |batch, pending_error|
+            batches_yielded << batch
+            exceptions_yielded << pending_error
+            each_batch_iterations = each_batch_iterations + 1
+          end
+        }.to raise_error(Rdkafka::RdkafkaError)
+        expect(poll_count).to eq 3
+        expect(each_batch_iterations).to eq 0
+        expect(batches_yielded.size).to eq 0
+        expect(exceptions_yielded.size).to eq 0
+      end
+    end
+  end
+
   describe "a rebalance listener" do
     it "should get notifications" do
       listener = Struct.new(:queue) do
@@ -736,6 +963,7 @@ describe Rdkafka::Consumer do
     {
         :subscribe               => [ nil ],
         :unsubscribe             => nil,
+        :each_batch              => nil,
         :pause                   => [ nil ],
         :resume                  => [ nil ],
         :subscription            => nil,

data/spec/rdkafka/producer_spec.rb

@@ -12,47 +12,92 @@ describe Rdkafka::Producer do
   end
 
   context "delivery callback" do
-    it "should set the callback" do
-      expect {
-        producer.delivery_callback = lambda do |delivery_handle|
-          puts stats
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          producer.delivery_callback = lambda do |delivery_handle|
+            puts delivery_handle
+          end
+        }.not_to raise_error
+        expect(producer.delivery_callback).to respond_to :call
+      end
+
+      it "should call the callback when a message is delivered" do
+        @callback_called = false
+
+        producer.delivery_callback = lambda do |report|
+          expect(report).not_to be_nil
+          expect(report.partition).to eq 1
+          expect(report.offset).to be >= 0
+          @callback_called = true
         end
-      }.not_to raise_error
-      expect(producer.delivery_callback).to be_a Proc
-    end
 
-    it "should not accept a callback that's not a proc" do
-      expect {
-        producer.delivery_callback = 'a string'
-      }.to raise_error(TypeError)
-    end
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
 
-    it "should call the callback when a message is delivered" do
-      @callback_called = false
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
 
+        # Join the producer thread.
+        producer.close
 
-      producer.delivery_callback = lambda do |report|
-        expect(report).not_to be_nil
-        expect(report.partition).to eq 1
-        expect(report.offset).to be >= 0
-        @callback_called = true
+        # Callback should have been called
+        expect(@callback_called).to be true
       end
+    end
 
-      # Produce a message
-      handle = producer.produce(
-        topic:   "produce_test_topic",
-        payload: "payload",
-        key:     "key"
-      )
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
+        end
+        expect {
+          producer.delivery_callback = callback.new
+        }.not_to raise_error
+        expect(producer.delivery_callback).to respond_to :call
+      end
 
-      # Wait for it to be delivered
-      handle.wait(max_wait_timeout: 15)
+      it "should call the callback when a message is delivered" do
+        called_report = []
+        callback = Class.new do
+          def initialize(called_report)
+            @called_report = called_report
+          end
 
-      # Join the producer thread.
-      producer.close
+          def call(report)
+            @called_report << report
+          end
+        end
+        producer.delivery_callback = callback.new(called_report)
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
+
+        # Join the producer thread.
+        producer.close
 
-      # Callback should have been called
-      expect(@callback_called).to be true
+        # Callback should have been called
+        expect(called_report.first).not_to be_nil
+        expect(called_report.first.partition).to eq 1
+        expect(called_report.first.offset).to be >= 0
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        producer.delivery_callback = 'a string'
+      }.to raise_error(TypeError)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Thijs Cadier
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-07 00:00:00.000000000 Z
+date: 2021-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -172,7 +172,7 @@ homepage: https://github.com/thijsc/rdkafka-ruby
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -187,8 +187,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+