rdkafka 0.7.0 → 0.8.0.beta.1

data/spec/rdkafka/error_spec.rb

@@ -71,15 +71,15 @@ describe Rdkafka::RdkafkaError do
     end
 
     it "should not equal another error with a different error code" do
-      expect(subject).to eq Rdkafka::RdkafkaError.new(20, "Error explanation")
+      expect(subject).not_to eq Rdkafka::RdkafkaError.new(20, "Error explanation")
     end
 
     it "should not equal another error with a different message" do
-      expect(subject).to eq Rdkafka::RdkafkaError.new(10, "Different error explanation")
+      expect(subject).not_to eq Rdkafka::RdkafkaError.new(10, "Different error explanation")
     end
 
     it "should not equal another error with no message" do
-      expect(subject).to eq Rdkafka::RdkafkaError.new(10)
+      expect(subject).not_to eq Rdkafka::RdkafkaError.new(10)
     end
   end
 end

data/spec/rdkafka/producer/delivery_report_spec.rb

@@ -1,7 +1,7 @@
 require "spec_helper"
 
 describe Rdkafka::Producer::DeliveryReport do
-  subject { Rdkafka::Producer::DeliveryReport.new(2, 100) }
+  subject { Rdkafka::Producer::DeliveryReport.new(2, 100, "error") }
 
   it "should get the partition" do
     expect(subject.partition).to eq 2
@@ -10,4 +10,8 @@ describe Rdkafka::Producer::DeliveryReport do
   it "should get the offset" do
     expect(subject.offset).to eq 100
   end
+
+  it "should get the error" do
+    expect(subject.error).to eq "error"
+  end
 end

data/spec/rdkafka/producer_spec.rb

@@ -2,10 +2,13 @@ require "spec_helper"
 
 describe Rdkafka::Producer do
   let(:producer) { rdkafka_config.producer }
+  let(:consumer) { rdkafka_config.consumer }
 
   after do
     # Registry should always end up being empty
     expect(Rdkafka::Producer::DeliveryHandle::REGISTRY).to be_empty
+    producer.close
+    consumer.close
   end
 
   context "delivery callback" do
@@ -27,6 +30,7 @@ describe Rdkafka::Producer do
     it "should call the callback when a message is delivered" do
       @callback_called = false
 
+
       producer.delivery_callback = lambda do |report|
         expect(report).not_to be_nil
         expect(report.partition).to eq 1
@@ -42,7 +46,10 @@ describe Rdkafka::Producer do
       )
 
       # Wait for it to be delivered
-      handle.wait(max_wait_timeout: 5)
+      handle.wait(max_wait_timeout: 15)
+
+      # Join the producer thread.
+      producer.close
 
       # Callback should have been called
       expect(@callback_called).to be true
@@ -55,7 +62,7 @@ describe Rdkafka::Producer do
         payload: "payload",
         key:     "key"
      )
-    }.to raise_error ArgumentError, "missing keyword: topic"
+    }.to raise_error ArgumentError, /missing keyword: [\:]?topic/
   end
 
   it "should produce a message" do
@@ -82,14 +89,15 @@ describe Rdkafka::Producer do
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
     expect(message.partition).to eq 1
     expect(message.payload).to eq "payload"
     expect(message.key).to eq "key"
     # Since api.version.request is on by default we will get
     # the message creation timestamp if it's not set.
-    expect(message.timestamp).to be_within(5).of(Time.now)
+    expect(message.timestamp).to be_within(10).of(Time.now)
   end
 
   it "should produce a message with a specified partition" do
@@ -105,12 +113,47 @@ describe Rdkafka::Producer do
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
     expect(message.partition).to eq 1
     expect(message.key).to eq "key partition"
   end
 
+  it "should produce a message to the same partition with a similar partition key" do
+    # Avoid partitioner collisions.
+    while true
+      key = ('a'..'z').to_a.shuffle.take(10).join('')
+      partition_key = ('a'..'z').to_a.shuffle.take(10).join('')
+      partition_count = producer.partition_count('partitioner_test_topic')
+      break if (Zlib.crc32(key) % partition_count) != (Zlib.crc32(partition_key) % partition_count)
+    end
+
+    # Produce a message with key, partition_key and key + partition_key
+    messages = [{key: key}, {partition_key: partition_key}, {key: key, partition_key: partition_key}]
+
+    messages = messages.map do |m|
+      handle = producer.produce(
+        topic:     "partitioner_test_topic",
+        payload:   "payload partition",
+        key:       m[:key],
+        partition_key: m[:partition_key]
+      )
+      report = handle.wait(max_wait_timeout: 5)
+
+      wait_for_message(
+        topic: "partitioner_test_topic",
+        delivery_report: report,
+      )
+    end
+
+    expect(messages[0].partition).not_to eq(messages[2].partition)
+    expect(messages[1].partition).to eq(messages[2].partition)
+    expect(messages[0].key).to eq key
+    expect(messages[1].key).to be_nil
+    expect(messages[2].key).to eq key
+  end
+
   it "should produce a message with utf-8 encoding" do
     handle = producer.produce(
       topic:   "produce_test_topic",
@@ -122,7 +165,8 @@ describe Rdkafka::Producer do
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
 
     expect(message.partition).to eq 1
@@ -154,7 +198,8 @@ describe Rdkafka::Producer do
       # Consume message and verify it's content
       message = wait_for_message(
         topic: "produce_test_topic",
-        delivery_report: report
+        delivery_report: report,
+        consumer: consumer
       )
 
       expect(message.partition).to eq 2
@@ -174,7 +219,8 @@ describe Rdkafka::Producer do
       # Consume message and verify it's content
       message = wait_for_message(
         topic: "produce_test_topic",
-        delivery_report: report
+        delivery_report: report,
+        consumer: consumer
       )
 
       expect(message.partition).to eq 2
@@ -193,7 +239,8 @@ describe Rdkafka::Producer do
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
 
     expect(message.key).to be_nil
@@ -210,7 +257,8 @@ describe Rdkafka::Producer do
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
 
     expect(message.key).to eq "key no payload"
@@ -229,7 +277,8 @@ describe Rdkafka::Producer do
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
 
     expect(message.payload).to eq "payload headers"
@@ -251,7 +300,8 @@ describe Rdkafka::Producer do
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
 
     expect(message.payload).to eq "payload headers"
@@ -284,11 +334,17 @@ describe Rdkafka::Producer do
     # Fork, produce a message, send the report over a pipe and
     # wait for and check the message in the main process.
 
+    # Kernel#fork is not available in JRuby
+    skip if defined?(JRUBY_VERSION)
+
     reader, writer = IO.pipe
 
     fork do
       reader.close
 
+      # Avoids sharing the socket between processes.
+      producer = rdkafka_config.producer
+
       handle = producer.produce(
         topic:   "produce_test_topic",
         payload: "payload-forked",
@@ -296,7 +352,6 @@ describe Rdkafka::Producer do
       )
 
       report = handle.wait(max_wait_timeout: 5)
-      producer.close
 
       report_json = JSON.generate(
         "partition" => report.partition,
@@ -304,20 +359,24 @@ describe Rdkafka::Producer do
       )
 
       writer.write(report_json)
+      writer.close
+      producer.close
     end
 
     writer.close
-
     report_hash = JSON.parse(reader.read)
     report = Rdkafka::Producer::DeliveryReport.new(
       report_hash["partition"],
       report_hash["offset"]
     )
 
+    reader.close
+
     # Consume message and verify it's content
     message = wait_for_message(
       topic: "produce_test_topic",
-      delivery_report: report
+      delivery_report: report,
+      consumer: consumer
     )
     expect(message.partition).to eq 0
     expect(message.payload).to eq "payload-forked"

data/spec/spec_helper.rb

@@ -7,6 +7,14 @@ require "pry"
 require "rspec"
 require "rdkafka"
 
+`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic consume_test_topic`
+`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic empty_test_topic`
+`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic load_test_topic`
+`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic produce_test_topic`
+`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic rake_test_topic`
+`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 3 --if-not-exists --topic watermarks_test_topic`
+`docker-compose exec kafka kafka-topics --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 25 --if-not-exists --topic partitioner_test_topic`
+
 def rdkafka_config(config_overrides={})
   config = {
     :"api.version.request" => false,
@@ -25,12 +33,12 @@ def rdkafka_config(config_overrides={})
   Rdkafka::Config.new(config)
 end
 
-def native_client
+def new_native_client
   config = rdkafka_config
   config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer)
 end
 
-def new_native_topic(topic_name="topic_name")
+def new_native_topic(topic_name="topic_name", native_client: )
   Rdkafka::Bindings.rd_kafka_topic_new(
     native_client,
     topic_name,
@@ -39,7 +47,8 @@ def new_native_topic(topic_name="topic_name")
 end
 
 def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer: nil)
-  consumer = rdkafka_config.consumer if consumer.nil?
+  new_consumer = !!consumer
+  consumer ||= rdkafka_config.consumer
   consumer.subscribe(topic)
   timeout = Time.now.to_i + timeout_in_seconds
   loop do
@@ -53,6 +62,8 @@ def wait_for_message(topic:, delivery_report:, timeout_in_seconds: 30, consumer:
       return message
     end
   end
+ensure
+  consumer.close if new_consumer
 end
 
 def wait_for_assignment(consumer)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0.beta.1
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-21 00:00:00.000000000 Z
+date: 2020-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -42,14 +42,14 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '12.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '12.3'
 - !ruby/object:Gem::Dependency
@@ -136,6 +136,7 @@ files:
 - lib/rdkafka/consumer/partition.rb
 - lib/rdkafka/consumer/topic_partition_list.rb
 - lib/rdkafka/error.rb
+- lib/rdkafka/metadata.rb
 - lib/rdkafka/producer.rb
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
@@ -167,12 +168,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.