rdkafka 0.8.0 → 0.11.0

data/spec/rdkafka/metadata_spec.rb

@@ -0,0 +1,78 @@
+require "spec_helper"
+require "securerandom"
+
+describe Rdkafka::Metadata do
+  let(:config)        { rdkafka_consumer_config }
+  let(:native_config) { config.send(:native_config) }
+  let(:native_kafka)  { config.send(:native_kafka, native_config, :rd_kafka_consumer) }
+
+  after do
+    Rdkafka::Bindings.rd_kafka_consumer_close(native_kafka)
+    Rdkafka::Bindings.rd_kafka_destroy(native_kafka)
+  end
+
+  context "passing in a topic name" do
+    context "that is non-existent topic" do
+      let(:topic_name) { SecureRandom.uuid.to_s }
+
+      it "raises an appropriate exception" do
+        expect {
+          described_class.new(native_kafka, topic_name)
+        }.to raise_exception(Rdkafka::RdkafkaError, "Broker: Unknown topic or partition (unknown_topic_or_part)")
+      end
+    end
+
+    context "that is one of our test topics" do
+      subject { described_class.new(native_kafka, topic_name) }
+      let(:topic_name) { "partitioner_test_topic" }
+
+      it "#brokers returns our single broker" do
+        expect(subject.brokers.length).to eq(1)
+        expect(subject.brokers[0][:broker_id]).to eq(1)
+        expect(subject.brokers[0][:broker_name]).to eq("localhost")
+        expect(subject.brokers[0][:broker_port]).to eq(9092)
+      end
+
+      it "#topics returns data on our test topic" do
+        expect(subject.topics.length).to eq(1)
+        expect(subject.topics[0][:partition_count]).to eq(25)
+        expect(subject.topics[0][:partitions].length).to eq(25)
+        expect(subject.topics[0][:topic_name]).to eq(topic_name)
+      end
+    end
+  end
+
+  context "not passing in a topic name" do
+    subject { described_class.new(native_kafka, topic_name) }
+    let(:topic_name) { nil }
+    let(:test_topics) {
+      %w(consume_test_topic empty_test_topic load_test_topic produce_test_topic rake_test_topic watermarks_test_topic partitioner_test_topic)
+    } # Test topics crated in spec_helper.rb
+
+    it "#brokers returns our single broker" do
+      expect(subject.brokers.length).to eq(1)
+      expect(subject.brokers[0][:broker_id]).to eq(1)
+      expect(subject.brokers[0][:broker_name]).to eq("localhost")
+      expect(subject.brokers[0][:broker_port]).to eq(9092)
+    end
+
+    it "#topics returns data about all of our test topics" do
+      result = subject.topics.map { |topic| topic[:topic_name] }
+      expect(result).to include(*test_topics)
+    end
+  end
+
+  context "when a non-zero error code is returned" do
+    let(:topic_name) { SecureRandom.uuid.to_s }
+
+    before do
+      allow(Rdkafka::Bindings).to receive(:rd_kafka_metadata).and_return(-165)
+    end
+
+    it "creating the instance raises an exception" do
+      expect {
+        described_class.new(native_kafka, topic_name)
+      }.to raise_error(Rdkafka::RdkafkaError, /Local: Required feature not supported by broker \(unsupported_feature\)/)
+    end
+  end
+end

data/spec/rdkafka/producer/delivery_handle_spec.rb

@@ -12,42 +12,13 @@ describe Rdkafka::Producer::DeliveryHandle do
     end
   end
 
-  describe ".register and .remove" do
-    let(:pending_handle) { true }
-
-    it "should register and remove a delivery handle" do
-      Rdkafka::Producer::DeliveryHandle.register(subject.to_ptr.address, subject)
-      removed = Rdkafka::Producer::DeliveryHandle.remove(subject.to_ptr.address)
-      expect(removed).to eq subject
-      expect(Rdkafka::Producer::DeliveryHandle::REGISTRY).to be_empty
-    end
-  end
-
-  describe "#pending?" do
-    context "when true" do
-      let(:pending_handle) { true }
-
-      it "should be true" do
-        expect(subject.pending?).to be true
-      end
-    end
-
-    context "when not true" do
-      let(:pending_handle) { false }
-
-      it "should be false" do
-        expect(subject.pending?).to be false
-      end
-    end
-  end
-
   describe "#wait" do
     let(:pending_handle) { true }
 
     it "should wait until the timeout and then raise an error" do
       expect {
         subject.wait(max_wait_timeout: 0.1)
-      }.to raise_error Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
+      }.to raise_error Rdkafka::Producer::DeliveryHandle::WaitTimeoutError, /delivery/
     end
 
     context "when not pending anymore and no error" do
@@ -67,16 +38,5 @@ describe Rdkafka::Producer::DeliveryHandle do
         expect(report.offset).to eq(100)
       end
     end
-
-    context "when not pending anymore and there was an error" do
-      let(:pending_handle) { false }
-      let(:response) { 20 }
-
-      it "should raise an rdkafka error" do
-        expect {
-          subject.wait
-        }.to raise_error Rdkafka::RdkafkaError
-      end
-    end
   end
 end

data/spec/rdkafka/producer_spec.rb

@@ -1,8 +1,9 @@
 require "spec_helper"
+require "zlib"
 
 describe Rdkafka::Producer do
-  let(:producer) { rdkafka_config.producer }
-  let(:consumer) { rdkafka_config.consumer }
+  let(:producer) { rdkafka_producer_config.producer }
+  let(:consumer) { rdkafka_consumer_config.consumer }
 
   after do
     # Registry should always end up being empty
@@ -12,47 +13,92 @@ describe Rdkafka::Producer do
   end
 
   context "delivery callback" do
-    it "should set the callback" do
-      expect {
-        producer.delivery_callback = lambda do |delivery_handle|
-          puts stats
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          producer.delivery_callback = lambda do |delivery_handle|
+            puts delivery_handle
+          end
+        }.not_to raise_error
+        expect(producer.delivery_callback).to respond_to :call
+      end
+
+      it "should call the callback when a message is delivered" do
+        @callback_called = false
+
+        producer.delivery_callback = lambda do |report|
+          expect(report).not_to be_nil
+          expect(report.partition).to eq 1
+          expect(report.offset).to be >= 0
+          @callback_called = true
         end
-      }.not_to raise_error
-      expect(producer.delivery_callback).to be_a Proc
-    end
 
-    it "should not accept a callback that's not a proc" do
-      expect {
-        producer.delivery_callback = 'a string'
-      }.to raise_error(TypeError)
-    end
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
 
-    it "should call the callback when a message is delivered" do
-      @callback_called = false
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
 
+        # Join the producer thread.
+        producer.close
 
-      producer.delivery_callback = lambda do |report|
-        expect(report).not_to be_nil
-        expect(report.partition).to eq 1
-        expect(report.offset).to be >= 0
-        @callback_called = true
+        # Callback should have been called
+        expect(@callback_called).to be true
       end
+    end
 
-      # Produce a message
-      handle = producer.produce(
-        topic:   "produce_test_topic",
-        payload: "payload",
-        key:     "key"
-      )
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
+        end
+        expect {
+          producer.delivery_callback = callback.new
+        }.not_to raise_error
+        expect(producer.delivery_callback).to respond_to :call
+      end
 
-      # Wait for it to be delivered
-      handle.wait(max_wait_timeout: 15)
+      it "should call the callback when a message is delivered" do
+        called_report = []
+        callback = Class.new do
+          def initialize(called_report)
+            @called_report = called_report
+          end
 
-      # Join the producer thread.
-      producer.close
+          def call(report)
+            @called_report << report
+          end
+        end
+        producer.delivery_callback = callback.new(called_report)
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
 
-      # Callback should have been called
-      expect(@callback_called).to be true
+        # Join the producer thread.
+        producer.close
+
+        # Callback should have been called
+        expect(called_report.first).not_to be_nil
+        expect(called_report.first.partition).to eq 1
+        expect(called_report.first.offset).to be >= 0
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        producer.delivery_callback = 'a string'
+      }.to raise_error(TypeError)
     end
   end
 
@@ -343,7 +389,7 @@ describe Rdkafka::Producer do
       reader.close
 
       # Avoids sharing the socket between processes.
-      producer = rdkafka_config.producer
+      producer = rdkafka_producer_config.producer
 
       handle = producer.produce(
         topic:   "produce_test_topic",
@@ -407,4 +453,26 @@ describe Rdkafka::Producer do
     # Waiting a second time should work
     handle.wait(max_wait_timeout: 5)
   end
+
+  context "methods that should not be called after a producer has been closed" do
+    before do
+      producer.close
+    end
+
+    # Affected methods and a non-invalid set of parameters for the method
+    {
+        :produce         => { topic: nil },
+        :partition_count => nil,
+    }.each do |method, args|
+      it "raises an exception if #{method} is called" do
+        expect {
+          if args.is_a?(Hash)
+            producer.public_send(method, **args)
+          else
+            producer.public_send(method, args)
+          end
+        }.to raise_exception(Rdkafka::ClosedProducerError, /#{method.to_s}/)
+      end
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -1,40 +1,64 @@
-require "simplecov"
-SimpleCov.start do
-  add_filter "/spec/"
+unless ENV["CI"] == "true"
+  require "simplecov"
+  SimpleCov.start do
+    add_filter "/spec/"
+  end
 end
 
 require "pry"
 require "rspec"
 require "rdkafka"
+require "timeout"
 
-`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic consume_test_topic`
-`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic empty_test_topic`
-`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic load_test_topic`
-`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic produce_test_topic`
-`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic rake_test_topic`
-`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic watermarks_test_topic`
-`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 25 --if-not-exists --topic partitioner_test_topic`
-
-def rdkafka_config(config_overrides={})
-  config = {
+def rdkafka_base_config
+  {
     :"api.version.request" => false,
     :"broker.version.fallback" => "1.0",
     :"bootstrap.servers" => "localhost:9092",
-    :"group.id" => "ruby-test-#{Random.new.rand(0..1_000_000)}",
-    :"auto.offset.reset" => "earliest",
-    :"enable.partition.eof" => false
   }
+end
+
+def rdkafka_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Merge overrides
+  config.merge!(config_overrides)
+  # Return it
+  Rdkafka::Config.new(config)
+end
+
+def rdkafka_consumer_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Add consumer specific fields to it
+  config[:"auto.offset.reset"] = "earliest"
+  config[:"enable.partition.eof"] = false
+  config[:"group.id"] = "ruby-test-#{Random.new.rand(0..1_000_000)}"
+  # Enable debug mode if required
+  if ENV["DEBUG_CONSUMER"]
+    config[:debug] = "cgrp,topic,fetch"
+  end
+  # Merge overrides
+  config.merge!(config_overrides)
+  # Return it
+  Rdkafka::Config.new(config)
+end
+
+def rdkafka_producer_config(config_overrides={})
+  # Generate the base config
+  config = rdkafka_base_config
+  # Enable debug mode if required
   if ENV["DEBUG_PRODUCER"]
     config[:debug] = "broker,topic,msg"
-  elsif ENV["DEBUG_CONSUMER"]
-    config[:debug] = "cgrp,topic,fetch"
   end
+  # Merge overrides
   config.merge!(config_overrides)
+  # Return it
   Rdkafka::Config.new(config)
 end
 
 def new_native_client
-  config = rdkafka_config
+  config = rdkafka_consumer_config
   config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
 end
 
@@ -48,7 +72,7 @@ end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
   new_consumer = !!consumer
-  consumer ||= rdkafka_config.consumer
+  consumer ||= rdkafka_consumer_config.consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
   loop do
@@ -79,3 +103,37 @@ def wait_for_unassignment(consumer)
     sleep 1
   end
 end
+
+RSpec.configure do |config|
+  config.filter_run focus: true
+  config.run_all_when_everything_filtered = true
+
+  config.before(:suite) do
+    admin = rdkafka_config.admin
+    {
+        consume_test_topic:      3,
+        empty_test_topic:        3,
+        load_test_topic:         3,
+        produce_test_topic:      3,
+        rake_test_topic:         3,
+        watermarks_test_topic:   3,
+        partitioner_test_topic: 25,
+    }.each do |topic, partitions|
+      create_topic_handle = admin.create_topic(topic.to_s, partitions, 1)
+      begin
+        create_topic_handle.wait(max_wait_timeout: 15)
+      rescue Rdkafka::RdkafkaError => ex
+        raise unless ex.message.match?(/topic_already_exists/)
+      end
+    end
+    admin.close
+  end
+
+  config.around(:each) do |example|
+    # Timeout specs after a minute. If they take longer
+    # they are probably stuck
+    Timeout::timeout(60) do
+      example.run
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Thijs Cadier
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-02 00:00:00.000000000 Z
+date: 2021-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -16,56 +16,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.15'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '1.15'
 - !ruby/object:Gem::Dependency
   name: mini_portile2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '12.3'
+        version: '12'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.10'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -84,51 +84,90 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0'
 description: Modern Kafka client library for Ruby based on librdkafka
 email:
 - thijs@appsignal.com
-executables: []
+executables:
+- console
 extensions:
 - ext/Rakefile
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".travis.yml"
+- ".rspec"
+- ".semaphore/semaphore.yml"
 - ".yardopts"
 - CHANGELOG.md
 - Gemfile
+- Guardfile
 - LICENSE
 - README.md
 - Rakefile
+- bin/console
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
 - lib/rdkafka.rb
+- lib/rdkafka/abstract_handle.rb
+- lib/rdkafka/admin.rb
+- lib/rdkafka/admin/create_topic_handle.rb
+- lib/rdkafka/admin/create_topic_report.rb
+- lib/rdkafka/admin/delete_topic_handle.rb
+- lib/rdkafka/admin/delete_topic_report.rb
 - lib/rdkafka/bindings.rb
+- lib/rdkafka/callbacks.rb
 - lib/rdkafka/config.rb
 - lib/rdkafka/consumer.rb
 - lib/rdkafka/consumer/headers.rb
@@ -142,13 +181,21 @@ files:
 - lib/rdkafka/producer/delivery_report.rb
 - lib/rdkafka/version.rb
 - rdkafka.gemspec
+- spec/rdkafka/abstract_handle_spec.rb
+- spec/rdkafka/admin/create_topic_handle_spec.rb
+- spec/rdkafka/admin/create_topic_report_spec.rb
+- spec/rdkafka/admin/delete_topic_handle_spec.rb
+- spec/rdkafka/admin/delete_topic_report_spec.rb
+- spec/rdkafka/admin_spec.rb
 - spec/rdkafka/bindings_spec.rb
+- spec/rdkafka/callbacks_spec.rb
 - spec/rdkafka/config_spec.rb
 - spec/rdkafka/consumer/message_spec.rb
 - spec/rdkafka/consumer/partition_spec.rb
 - spec/rdkafka/consumer/topic_partition_list_spec.rb
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
+- spec/rdkafka/metadata_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb
@@ -157,7 +204,7 @@ homepage: https://github.com/thijsc/rdkafka-ruby
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -165,27 +212,35 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.
   It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+
   and Ruby 2.4+.
 test_files:
+- spec/rdkafka/abstract_handle_spec.rb
+- spec/rdkafka/admin/create_topic_handle_spec.rb
+- spec/rdkafka/admin/create_topic_report_spec.rb
+- spec/rdkafka/admin/delete_topic_handle_spec.rb
+- spec/rdkafka/admin/delete_topic_report_spec.rb
+- spec/rdkafka/admin_spec.rb
 - spec/rdkafka/bindings_spec.rb
+- spec/rdkafka/callbacks_spec.rb
 - spec/rdkafka/config_spec.rb
 - spec/rdkafka/consumer/message_spec.rb
 - spec/rdkafka/consumer/partition_spec.rb
 - spec/rdkafka/consumer/topic_partition_list_spec.rb
 - spec/rdkafka/consumer_spec.rb
 - spec/rdkafka/error_spec.rb
+- spec/rdkafka/metadata_spec.rb
 - spec/rdkafka/producer/delivery_handle_spec.rb
 - spec/rdkafka/producer/delivery_report_spec.rb
 - spec/rdkafka/producer_spec.rb