rdkafka 0.3.5 → 0.4.0

data/spec/rdkafka/consumer_spec.rb

@@ -47,7 +47,64 @@ describe Rdkafka::Consumer do
     end
   end
 
-  context "#close" do
+  describe "#assign and #assignment" do
+    it "should return an empty assignment if nothing is assigned" do
+      expect(consumer.assignment).to be_empty
+    end
+
+    it "should only accept a topic partition list in assign" do
+      expect {
+        consumer.assign("list")
+      }.to raise_error TypeError
+    end
+
+    it "should raise an error when assigning fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_assign).and_return(20)
+      expect {
+        consumer.assign(Rdkafka::Consumer::TopicPartitionList.new)
+      }.to raise_error Rdkafka::RdkafkaError
+    end
+
+    it "should assign specific topic/partitions and return that assignment" do
+      tpl = Rdkafka::Consumer::TopicPartitionList.new
+      tpl.add_topic("consume_test_topic", (0..2))
+      consumer.assign(tpl)
+
+      assignment = consumer.assignment
+      expect(assignment).not_to be_empty
+      expect(assignment.to_h["consume_test_topic"].length).to eq 3
+    end
+
+    it "should return the assignment when subscribed" do
+      # Make sure there's a message
+      report = producer.produce(
+        topic:     "consume_test_topic",
+        payload:   "payload 1",
+        key:       "key 1",
+        partition: 0
+      ).wait
+
+      # Subscribe and poll until partitions are assigned
+      consumer.subscribe("consume_test_topic")
+      100.times do
+        consumer.poll(100)
+        break unless consumer.assignment.empty?
+      end
+
+      assignment = consumer.assignment
+      expect(assignment).not_to be_empty
+      expect(assignment.to_h["consume_test_topic"].length).to eq 3
+    end
+
+    it "should raise an error when getting assignment fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_assignment).and_return(20)
+      expect {
+        consumer.assignment
+      }.to raise_error Rdkafka::RdkafkaError
+    end
+  end
+
+  describe "#close" do
     it "should close a consumer" do
       consumer.subscribe("consume_test_topic")
       consumer.close
@@ -73,12 +130,46 @@ describe Rdkafka::Consumer do
       )
     end
 
-    it "should only accept a topic partition list" do
+    it "should only accept a topic partition list in committed" do
       expect {
         consumer.committed("list")
       }.to raise_error TypeError
     end
 
+    it "should commit in sync mode" do
+      expect {
+        consumer.commit(nil, true)
+      }.not_to raise_error
+    end
+
+    it "should only accept a topic partition list in commit if not nil" do
+      expect {
+        consumer.commit("list")
+      }.to raise_error TypeError
+    end
+
+    it "should commit a specific topic partion list" do
+      # Make sure there are some message
+      3.times do |i|
+        producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload 1",
+          key:       "key 1",
+          partition: i
+        ).wait
+      end
+
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic_and_partitions_with_offsets("consume_test_topic", {0 => 1, 1 => 1, 2 => 1})
+      end
+      consumer.commit(list)
+
+      partitions = consumer.committed(list).to_h["consume_test_topic"]
+      expect(partitions[0].offset).to eq 1
+      expect(partitions[1].offset).to eq 1
+      expect(partitions[2].offset).to eq 1
+    end
+
     it "should raise an error when committing fails" do
       expect(Rdkafka::Bindings).to receive(:rd_kafka_commit).and_return(20)
 
@@ -87,14 +178,30 @@ describe Rdkafka::Consumer do
       }.to raise_error(Rdkafka::RdkafkaError)
     end
 
+    it "should fetch the committed offsets for the current assignment" do
+      consumer.subscribe("consume_test_topic")
+      # Wait for the assignment to be made
+      10.times do
+        break if !consumer.assignment.empty?
+        sleep 1
+      end
+
+      partitions = consumer.committed.to_h["consume_test_topic"]
+      expect(partitions).not_to be_nil
+      expect(partitions[0].offset).to be > 0
+      expect(partitions[1].offset).to be nil
+      expect(partitions[2].offset).to be nil
+    end
+
     it "should fetch the committed offsets for a specified topic partition list" do
       list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
         list.add_topic("consume_test_topic", [0, 1, 2])
       end
       partitions = consumer.committed(list).to_h["consume_test_topic"]
+      expect(partitions).not_to be_nil
       expect(partitions[0].offset).to be > 0
-      expect(partitions[1].offset).to eq -1001
-      expect(partitions[2].offset).to eq -1001
+      expect(partitions[1].offset).to be nil
+      expect(partitions[2].offset).to be nil
     end
 
     it "should raise an error when getting committed fails" do

data/spec/rdkafka/producer_spec.rb

@@ -48,7 +48,7 @@ describe Rdkafka::Producer do
     expect(message.key).to eq "key"
     # Since api.version.request is on by default we will get
     # the message creation timestamp if it's not set.
-    expect(message.timestamp).to be > 1505069891557
+    expect(message.timestamp).to be_within(5).of(Time.now)
   end
 
   it "should produce a message with a specified partition" do
@@ -89,24 +89,57 @@ describe Rdkafka::Producer do
     expect(message.key).to eq "key utf8"
   end
 
-  it "should produce a message with a timestamp" do
-    handle = producer.produce(
-      topic:     "produce_test_topic",
-      payload:   "payload timestamp",
-      key:       "key timestamp",
-      timestamp: 1505069646000
-    )
-    report = handle.wait(5)
+  context "timestamp" do
+    it "should raise a type error if not nil, integer or time" do
+      expect {
+        producer.produce(
+          topic:     "produce_test_topic",
+          payload:   "payload timestamp",
+          key:       "key timestamp",
+          timestamp: "10101010"
+        )
+      }.to raise_error TypeError
+    end
 
-    # Consume message and verify it's content
-    message = wait_for_message(
-      topic: "produce_test_topic",
-      delivery_report: report
-    )
+    it "should produce a message with an integer timestamp" do
+      handle = producer.produce(
+        topic:     "produce_test_topic",
+        payload:   "payload timestamp",
+        key:       "key timestamp",
+        timestamp: 1505069646252
+      )
+      report = handle.wait(5)
+
+      # Consume message and verify it's content
+      message = wait_for_message(
+        topic: "produce_test_topic",
+        delivery_report: report
+      )
+
+      expect(message.partition).to eq 2
+      expect(message.key).to eq "key timestamp"
+      expect(message.timestamp).to eq Time.at(1505069646, 252_000)
+    end
+
+    it "should produce a message with a time timestamp" do
+      handle = producer.produce(
+        topic:     "produce_test_topic",
+        payload:   "payload timestamp",
+        key:       "key timestamp",
+        timestamp: Time.at(1505069646, 353_000)
+      )
+      report = handle.wait(5)
+
+      # Consume message and verify it's content
+      message = wait_for_message(
+        topic: "produce_test_topic",
+        delivery_report: report
+      )
 
-    expect(message.partition).to eq 2
-    expect(message.key).to eq "key timestamp"
-    expect(message.timestamp).to eq 1505069646000
+      expect(message.partition).to eq 2
+      expect(message.key).to eq "key timestamp"
+      expect(message.timestamp).to eq Time.at(1505069646, 353_000)
+    end
   end
 
   it "should produce a message with nil key" do
@@ -164,6 +197,56 @@ describe Rdkafka::Producer do
     end
   end
 
+# TODO this spec crashes if you create and use the producer before
+# forking like so:
+#
+# @producer = producer
+#
+# This will be added as part of https://github.com/appsignal/rdkafka-ruby/issues/19
+  #it "should produce a message in a forked process" do
+  #  # Fork, produce a message, send the report of a pipe and
+  #  # wait for it in the main process.
+
+  #  reader, writer = IO.pipe
+
+  #  fork do
+  #    reader.close
+
+  #    handle = producer.produce(
+  #      topic:   "produce_test_topic",
+  #      payload: "payload",
+  #      key:     "key"
+  #    )
+
+  #    report = handle.wait(5)
+  #    producer.close
+
+  #    report_json = JSON.generate(
+  #      "partition" => report.partition,
+  #      "offset" => report.offset
+  #    )
+
+  #    writer.write(report_json)
+  #  end
+
+  #  writer.close
+
+  #  report_hash = JSON.parse(reader.read)
+  #  report = Rdkafka::Producer::DeliveryReport.new(
+  #    report_hash["partition"],
+  #    report_hash["offset"]
+  #  )
+
+  #  # Consume message and verify it's content
+  #  message = wait_for_message(
+  #    topic: "produce_test_topic",
+  #    delivery_report: report
+  #  )
+  #  expect(message.partition).to eq 1
+  #  expect(message.payload).to eq "payload"
+  #  expect(message.key).to eq "key"
+  #end
+
   it "should raise an error when producing fails" do
     expect(Rdkafka::Bindings).to receive(:rd_kafka_producev).and_return(20)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.4.0
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-17 00:00:00.000000000 Z
+date: 2018-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -110,6 +110,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- docker-compose.yml
 - ext/Rakefile
 - lib/rdkafka.rb
 - lib/rdkafka/bindings.rb
@@ -155,7 +156,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Kafka client library wrapping librdkafka using the ffi gem and futures from