rdkafka 0.11.1 → 0.13.1

data/spec/rdkafka/error_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::RdkafkaError do

data/spec/rdkafka/metadata_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 require "securerandom"
 

data/spec/rdkafka/native_kafka_spec.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::NativeKafka do
+  let(:config) { rdkafka_producer_config }
+  let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
+  let(:closing) { false }
+  let(:thread) { double(Thread) }
+
+  subject(:client) { described_class.new(native, run_polling_thread: true) }
+
+  before do
+    allow(Thread).to receive(:new).and_return(thread)
+
+    allow(thread).to receive(:[]=).with(:closing, anything)
+    allow(thread).to receive(:join)
+    allow(thread).to receive(:abort_on_exception=).with(anything)
+  end
+
+  after { client.close }
+
+  context "defaults" do
+    it "sets the thread to abort on exception" do
+      expect(thread).to receive(:abort_on_exception=).with(true)
+
+      client
+    end
+
+    it "sets the thread `closing` flag to false" do
+      expect(thread).to receive(:[]=).with(:closing, false)
+
+      client
+    end
+  end
+
+  context "the polling thread" do
+    it "is created" do
+      expect(Thread).to receive(:new)
+
+      client
+    end
+  end
+
+  it "exposes the inner client" do
+    client.with_inner do |inner|
+      expect(inner).to eq(native)
+    end
+  end
+
+  context "when client was not yet closed (`nil`)" do
+    it "is not closed" do
+      expect(client.closed?).to eq(false)
+    end
+
+    context "and attempt to close" do
+      it "calls the `destroy` binding" do
+        expect(Rdkafka::Bindings).to receive(:rd_kafka_destroy).with(native).and_call_original
+
+        client.close
+      end
+
+      it "indicates to the polling thread that it is closing" do
+        expect(thread).to receive(:[]=).with(:closing, true)
+
+        client.close
+      end
+
+      it "joins the polling thread" do
+        expect(thread).to receive(:join)
+
+        client.close
+      end
+
+      it "closes and unassign the native client" do
+        client.close
+
+        expect(client.closed?).to eq(true)
+      end
+    end
+  end
+
+  context "when client was already closed" do
+    before { client.close }
+
+    it "is closed" do
+      expect(client.closed?).to eq(true)
+    end
+
+    context "and attempt to close again" do
+      it "does not call the `destroy` binding" do
+        expect(Rdkafka::Bindings).not_to receive(:rd_kafka_destroy_flags)
+
+        client.close
+      end
+
+      it "does not indicate to the polling thread that it is closing" do
+        expect(thread).not_to receive(:[]=).with(:closing, true)
+
+        client.close
+      end
+
+      it "does not join the polling thread" do
+        expect(thread).not_to receive(:join)
+
+        client.close
+      end
+
+      it "does not close and unassign the native client again" do
+        client.close
+
+        expect(client.closed?).to eq(true)
+      end
+    end
+  end
+
+  it "provides a finalizer that closes the native kafka client" do
+    expect(client.closed?).to eq(false)
+
+    client.finalizer.call("some-ignored-object-id")
+
+    expect(client.closed?).to eq(true)
+  end
+end

data/spec/rdkafka/producer/delivery_handle_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Producer::DeliveryHandle do
@@ -9,6 +11,7 @@ describe Rdkafka::Producer::DeliveryHandle do
       handle[:response] = response
       handle[:partition] = 2
       handle[:offset] = 100
+      handle[:topic_name] = FFI::MemoryPointer.from_string("produce_test_topic")
     end
   end
 
@@ -29,6 +32,7 @@ describe Rdkafka::Producer::DeliveryHandle do
 
         expect(report.partition).to eq(2)
         expect(report.offset).to eq(100)
+        expect(report.topic_name).to eq("produce_test_topic")
       end
 
       it "should wait without a timeout" do
@@ -36,6 +40,7 @@ describe Rdkafka::Producer::DeliveryHandle do
 
         expect(report.partition).to eq(2)
         expect(report.offset).to eq(100)
+        expect(report.topic_name).to eq("produce_test_topic")
       end
     end
   end

data/spec/rdkafka/producer/delivery_report_spec.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 
 describe Rdkafka::Producer::DeliveryReport do
-  subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "error") }
+  subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "topic", -1) }
 
   it "should get the partition" do
     expect(subject.partition).to eq 2
@@ -11,7 +13,11 @@ describe Rdkafka::Producer::DeliveryReport do
     expect(subject.offset).to eq 100
   end
 
+  it "should get the topic_name" do
+    expect(subject.topic_name).to eq "topic"
+  end
+
   it "should get the error" do
-    expect(subject.error).to eq "error"
+    expect(subject.error).to eq -1
   end
 end

data/spec/rdkafka/producer_spec.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "spec_helper"
 require "zlib"
 
@@ -7,7 +9,8 @@ describe Rdkafka::Producer do
 
   after do
     # Registry should always end up being empty
-    expect(Rdkafka::Producer::DeliveryHandle::REGISTRY).to be_empty
+    registry = Rdkafka::Producer::DeliveryHandle::REGISTRY
+    expect(registry).to be_empty, registry.inspect
     producer.close
     consumer.close
   end
@@ -30,6 +33,7 @@ describe Rdkafka::Producer do
           expect(report).not_to be_nil
           expect(report.partition).to eq 1
           expect(report.offset).to be >= 0
+          expect(report.topic_name).to eq "produce_test_topic"
           @callback_called = true
         end
 
@@ -49,6 +53,27 @@ describe Rdkafka::Producer do
         # Callback should have been called
         expect(@callback_called).to be true
       end
+
+      it "should provide handle" do
+        @callback_handle = nil
+
+        producer.delivery_callback = lambda { |_, handle| @callback_handle = handle }
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
+
+        # Join the producer thread.
+        producer.close
+
+        expect(handle).to be @callback_handle
+      end
     end
 
     context "with a callable object" do
@@ -92,6 +117,37 @@ describe Rdkafka::Producer do
         expect(called_report.first).not_to be_nil
         expect(called_report.first.partition).to eq 1
         expect(called_report.first.offset).to be >= 0
+        expect(called_report.first.topic_name).to eq "produce_test_topic"
+      end
+
+      it "should provide handle" do
+        callback_handles = []
+        callback = Class.new do
+          def initialize(callback_handles)
+            @callback_handles = callback_handles
+          end
+
+          def call(_, handle)
+            @callback_handles << handle
+          end
+        end
+        producer.delivery_callback = callback.new(callback_handles)
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
+
+        # Join the producer thread.
+        producer.close
+
+        # Callback should have been called
+        expect(handle).to be callback_handles.first
       end
     end
 
@@ -129,10 +185,11 @@ describe Rdkafka::Producer do
     expect(report.partition).to eq 1
     expect(report.offset).to be >= 0
 
-    # Close producer
+    # Flush and close producer
+    producer.flush
     producer.close
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -156,7 +213,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -200,6 +257,28 @@ describe Rdkafka::Producer do
     expect(messages[2].key).to eq key
   end
 
+  it "should produce a message with empty string without crashing" do
+    messages = [{key: 'a', partition_key: ''}]
+
+    messages = messages.map do |m|
+      handle = producer.produce(
+        topic:     "partitioner_test_topic",
+        payload:   "payload partition",
+        key:       m[:key],
+        partition_key: m[:partition_key]
+      )
+      report = handle.wait(max_wait_timeout: 5)
+
+      wait_for_message(
+        topic: "partitioner_test_topic",
+        delivery_report: report,
+      )
+    end
+
+    expect(messages[0].partition).to eq 0
+    expect(messages[0].key).to eq 'a'
+  end
+
   it "should produce a message with utf-8 encoding" do
     handle = producer.produce(
       topic:   "produce_test_topic",
@@ -208,7 +287,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -241,7 +320,7 @@ describe Rdkafka::Producer do
       )
       report = handle.wait(max_wait_timeout: 5)
 
-      # Consume message and verify it's content
+      # Consume message and verify its content
       message = wait_for_message(
         topic: "produce_test_topic",
         delivery_report: report,
@@ -262,7 +341,7 @@ describe Rdkafka::Producer do
       )
       report = handle.wait(max_wait_timeout: 5)
 
-      # Consume message and verify it's content
+      # Consume message and verify its content
       message = wait_for_message(
         topic: "produce_test_topic",
         delivery_report: report,
@@ -282,7 +361,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -300,7 +379,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -320,7 +399,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -329,9 +408,9 @@ describe Rdkafka::Producer do
 
     expect(message.payload).to eq "payload headers"
     expect(message.key).to eq "key headers"
-    expect(message.headers[:foo]).to eq "bar"
-    expect(message.headers[:baz]).to eq "foobar"
-    expect(message.headers[:foobar]).to be_nil
+    expect(message.headers["foo"]).to eq "bar"
+    expect(message.headers["baz"]).to eq "foobar"
+    expect(message.headers["foobar"]).to be_nil
   end
 
   it "should produce a message with empty headers" do
@@ -343,7 +422,7 @@ describe Rdkafka::Producer do
     )
     report = handle.wait(max_wait_timeout: 5)
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,
@@ -376,19 +455,15 @@ describe Rdkafka::Producer do
     end
   end
 
-  it "should produce a message in a forked process" do
+  it "should produce a message in a forked process", skip: defined?(JRUBY_VERSION) && "Kernel#fork is not available" do
     # Fork, produce a message, send the report over a pipe and
     # wait for and check the message in the main process.
-
-    # Kernel#fork is not available in JRuby
-    skip if defined?(JRUBY_VERSION)
-
     reader, writer = IO.pipe
 
-    fork do
+    pid = fork do
       reader.close
 
-      # Avoids sharing the socket between processes.
+      # Avoid sharing the client between processes.
       producer = rdkafka_producer_config.producer
 
       handle = producer.produce(
@@ -401,24 +476,28 @@ describe Rdkafka::Producer do
 
       report_json = JSON.generate(
         "partition" => report.partition,
-        "offset" => report.offset
+        "offset" => report.offset,
+        "topic_name" => report.topic_name
       )
 
       writer.write(report_json)
       writer.close
+      producer.flush
       producer.close
     end
+    Process.wait(pid)
 
     writer.close
     report_hash = JSON.parse(reader.read)
     report = Rdkafka::Producer::DeliveryReport.new(
       report_hash["partition"],
-      report_hash["offset"]
+      report_hash["offset"],
+      report_hash["topic_name"]
     )
 
     reader.close
 
-    # Consume message and verify it's content
+    # Consume message and verify its content
     message = wait_for_message(
       topic: "produce_test_topic",
       delivery_report: report,

data/spec/spec_helper.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 unless ENV["CI"] == "true"
   require "simplecov"
   SimpleCov.start do
@@ -71,7 +73,7 @@ def new_native_topic(topic_name="topic_name", native_client: )
 end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
-  new_consumer = !!consumer
+  new_consumer = consumer.nil?
   consumer ||= rdkafka_consumer_config.consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
@@ -104,6 +106,20 @@ def wait_for_unassignment(consumer)
   end
 end
 
+def notify_listener(listener, &block)
+  # 1. subscribe and poll
+  consumer.subscribe("consume_test_topic")
+  wait_for_assignment(consumer)
+  consumer.poll(100)
+
+  block.call if block
+
+  # 2. unsubscribe
+  consumer.unsubscribe
+  wait_for_unassignment(consumer)
+  consumer.close
+end
+
 RSpec.configure do |config|
   config.filter_run focus: true
   config.run_all_when_everything_filtered = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.13.1
 platform: ruby
 authors:
 - Thijs Cadier
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-23 00:00:00.000000000 Z
+date: 2024-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -139,8 +139,7 @@ dependencies:
 description: Modern Kafka client library for Ruby based on librdkafka
 email:
 - thijs@appsignal.com
-executables:
-- console
+executables: []
 extensions:
 - ext/Rakefile
 extra_rdoc_files: []
@@ -155,7 +154,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- bin/console
+- dist/librdkafka_2.0.2.tar.gz
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
@@ -176,6 +175,7 @@ files:
 - lib/rdkafka/consumer/topic_partition_list.rb
 - lib/rdkafka/error.rb
 - lib/rdkafka/metadata.rb
+- lib/rdkafka/native_kafka.rb
 - lib/rdkafka/producer.rb
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
@@ -190,12 +190,14 @@ files:
 - spec/rdkafka/bindings_spec.rb
 - spec/rdkafka/callbacks_spec.rb
 - spec/rdkafka/config_spec.rb
+- spec/rdkafka/consumer/headers_spec.rb
 - spec/rdkafka/consumer/message_spec.rb
 - spec/rdkafka/consumer/partition_spec.rb
 - spec/rdkafka/consumer/topic_partition_list_spec.rb
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
 - spec/rdkafka/metadata_spec.rb
+- spec/rdkafka/native_kafka_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb
@@ -204,7 +206,7 @@ homepage: https://github.com/thijsc/rdkafka-ruby
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -219,8 +221,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.5.14
+signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
@@ -235,12 +237,14 @@ test_files:
 - spec/rdkafka/bindings_spec.rb
 - spec/rdkafka/callbacks_spec.rb
 - spec/rdkafka/config_spec.rb
+- spec/rdkafka/consumer/headers_spec.rb
 - spec/rdkafka/consumer/message_spec.rb
 - spec/rdkafka/consumer/partition_spec.rb
 - spec/rdkafka/consumer/topic_partition_list_spec.rb
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
 - spec/rdkafka/metadata_spec.rb
+- spec/rdkafka/native_kafka_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb

data/bin/console

@@ -1,11 +0,0 @@
-#!/usr/bin/env ruby
-
-# frozen_string_literal: true
-
-ENV["IRBRC"] = File.join(File.dirname(__FILE__), ".irbrc")
-
-require "bundler/setup"
-require "rdkafka"
-
-require "irb"
-IRB.start(__FILE__)