rdkafka 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c01eae203c0d0a12be82edacae9fb3ecde7c73dc96e6a95d456434a2d0a2e5d9
-  data.tar.gz: 50bced86691e02ca0f1af3df2140751f530b667aa5ddf6f6822110991e3f4b43
+  metadata.gz: 0b248aee9a260de3d8bb56efba9255f21d3bfcb6c269a1ab15beac7ae2521b6c
+  data.tar.gz: 2d1199d5717a832674f659ac1d5e69d2a659845e9c8e5509428270a4a8390ae1
 SHA512:
-  metadata.gz: af1930db8fea0a9ebec2ab63c404be88a805349badbc575e351e90d694d989a962dcfd4db32a5e344fddc83e8f97ca7600188ccc6d8e46ccd9a73afa99affbed
-  data.tar.gz: c8a629fe6ea2a9e1a0d6c38e6d2eb46f1ea8d6791e7d2bdd2751c83ab5a6ed6fc90d7783eaeef3e2f324f01d50e3bb3d308c12048001519dea42c3810c399240
+  metadata.gz: 62afc350fa40f5e2117e7791927fe5ad7fcb3cb7ba13cdbba2f450dc6537d41e37668a5997fccbb260abe3fbb09e7a73c0cb5d5059b94837d5097f373c27eb75
+  data.tar.gz: 1723633bcce6569992a2368df38c2d59b217dd94af9e8afab9932ca085e9ae0fbc3661ee8fbc370d31f3985204b38b3bbe15bbdfa11ca2be61c69b4c938a40f3

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.6.0
+* Bump librdkafka to 1.1.0 (by Chris Gaffney)
+* Implement seek (by breunigs)
+
 # 0.5.0
 * Bump librdkafka to 1.0.0 (by breunigs)
 * Add cluster and member information (by dmexe)

data/lib/rdkafka/bindings.rb

@@ -164,6 +164,7 @@ module Rdkafka
     attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int
+    attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int

data/lib/rdkafka/consumer.rb

@@ -282,6 +282,37 @@ module Rdkafka
       end
     end
 
+    # Seek to a particular message. The next poll on the topic/partition will return the
+    # message at the given offset.
+    #
+    # @param message [Rdkafka::Consumer::Message] The message to which to seek
+    #
+    # @raise [RdkafkaError] When seeking fails
+    #
+    # @return [nil]
+    def seek(message)
+      # rd_kafka_offset_store is one of the few calls that does not support
+      # a string as the topic, so create a native topic for it.
+      native_topic = Rdkafka::Bindings.rd_kafka_topic_new(
+        @native_kafka,
+        message.topic,
+        nil
+      )
+      response = Rdkafka::Bindings.rd_kafka_seek(
+        native_topic,
+        message.partition,
+        message.offset,
+        0 # timeout
+      )
+      if response != 0
+        raise Rdkafka::RdkafkaError.new(response)
+      end
+    ensure
+      if native_topic && !native_topic.null?
+        Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
+      end
+    end
+
     # Commit the current offsets of this consumer
     #
     # @param list [TopicPartitionList,nil] The topic with partitions to commit

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.5.0"
-  LIBRDKAFKA_VERSION = "1.0.0"
-  LIBRDKAFKA_SOURCE_SHA256 = 'b00a0d9f0e8c7ceb67b93b4ee67f3c68279a843a15bf4a6742eb64897519aa09'
+  VERSION = "0.6.0"
+  LIBRDKAFKA_VERSION = "1.1.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "123b47404c16bcde194b4bd1221c21fdce832ad12912bd8074f88f64b2b86f2b"
 end

data/spec/rdkafka/consumer_spec.rb

@@ -1,4 +1,5 @@
 require "spec_helper"
+require "ostruct"
 
 describe Rdkafka::Consumer do
   let(:config) { rdkafka_config }
@@ -75,7 +76,7 @@ describe Rdkafka::Consumer do
         tpl.add_topic("consume_test_topic", (0..2))
         consumer.pause(tpl)
 
-        # 6. unsure that messages are not available
+        # 6. ensure that messages are not available
         records = consumer.poll(timeout)
         expect(records).to be_nil
 
@@ -124,6 +125,92 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#seek" do
+    it "should raise an error when seeking fails" do
+      fake_msg = OpenStruct.new(topic: "consume_test_topic", partition: 0, offset: 0)
+
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
+      expect {
+        consumer.seek(fake_msg)
+      }.to raise_error Rdkafka::RdkafkaError
+    end
+
+    context "subscription" do
+      let(:timeout) { 1000 }
+
+      before do
+        consumer.subscribe("consume_test_topic")
+
+        # 1. partitions are assigned
+        wait_for_assignment(consumer)
+        expect(consumer.assignment).not_to be_empty
+
+        # 2. eat unrelated messages
+        while(consumer.poll(timeout)) do; end
+      end
+      after { consumer.unsubscribe }
+
+      def send_one_message(val)
+        producer.produce(
+          topic:     "consume_test_topic",
+          payload:   "payload #{val}",
+          key:       "key 1",
+          partition: 0
+        ).wait
+      end
+
+      it "works when a partition is paused" do
+        # 3. get reference message
+        send_one_message(:a)
+        message1 = consumer.poll(timeout)
+        expect(message1&.payload).to eq "payload a"
+
+        # 4. pause the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic("consume_test_topic", 1)
+        consumer.pause(tpl)
+
+        # 5. seek to previous message
+        consumer.seek(message1)
+
+        # 6. resume the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic("consume_test_topic", 1)
+        consumer.resume(tpl)
+
+        # 7. ensure same message is read again
+        message2 = consumer.poll(timeout)
+        consumer.commit
+        expect(message1.offset).to eq message2.offset
+        expect(message1.payload).to eq message2.payload
+      end
+
+      it "allows skipping messages" do
+        # 3. send messages
+        send_one_message(:a)
+        send_one_message(:b)
+        send_one_message(:c)
+
+        # 4. get reference message
+        message = consumer.poll(timeout)
+        expect(message&.payload).to eq "payload a"
+
+        # 5. seek over one message
+        fake_msg = message.dup
+        fake_msg.instance_variable_set(:@offset, fake_msg.offset + 2)
+        consumer.seek(fake_msg)
+
+        # 6. ensure that only one message is available
+        records = consumer.poll(timeout)
+        expect(records&.payload).to eq "payload c"
+        records = consumer.poll(timeout)
+        expect(records).to be_nil
+
+        consumer.commit
+      end
+    end
+  end
+
   describe "#assign and #assignment" do
     it "should return an empty assignment if nothing is assigned" do
       expect(consumer.assignment).to be_empty

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-11 00:00:00.000000000 Z
+date: 2019-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi