karafka-rdkafka 0.16.0 → 0.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78fe68d4a566b63ffa94633d2752cc95c535c020236749aa0452192d10cd2c43
-  data.tar.gz: 61a9fe426703153762c94508a535a780b9ee1917cacf93ccbe37d1c4eb974d6c
+  metadata.gz: f8685868c1ad3f222639845933ba5aec660ec3923465c4cf91c3235ece03eec8
+  data.tar.gz: 96d7c33291d166418e696f05cce117b89861dd7fbbf53b587dbf1a90a3f4d2b2
 SHA512:
-  metadata.gz: 46e2862b087c375042b93682d10378901d23b238f696cfd17048a880c3b28170243250e5cceecfd42c74c98791273df9577ed1371b300629a60727df8df2f9c4
-  data.tar.gz: c76d0666e617aa0879ffa99c1888b7fad0ebbd1f1195f14285b130b82ba34c339c3f92c9718f0e8180d2d1cccd1e99f23ea765ea314fb0570713ce4ba23b8071
+  metadata.gz: 30de272e1cac622957ecc061426bf7314791528769dc6afc3f445f5fe6b3241a5b51a88931c97619fcdf1447bd0d6b7913c959ccb3b5d384e32d5ce89ef5d915
+  data.tar.gz: f912ec91dd3983a89807879b272f8bbcf777f705bd305aba77148998afcf2ba4372ae070d2510a19110a424ad9231177c64a25742b5de47b22c426769da5c858

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Rdkafka Changelog
 
+## 0.16.1 (2024-07-10)
+- [Feature] Add `#seek_by` to be able to seek for a message by topic, partition and offset (zinahia)
+- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
+
 ## 0.16.0 (2024-06-17)
 - **[Breaking]** Messages without headers returned by `#poll` contain frozen empty hash.
 - **[Breaking]** `HashWithSymbolKeysTreatedLikeStrings` has been removed so headers are regular hashes with string keys.

data/README.md

@@ -163,6 +163,7 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka |
 |-|-|
+| 0.17.0 (Unreleased) | 2.4.0 (2024-05-07) |
 | 0.16.0 (2024-06-13) | 2.3.0 (2023-10-25) |
 | 0.15.0 (2023-12-03) | 2.3.0 (2023-10-25) |
 | 0.14.0 (2023-11-21) | 2.2.0 (2023-07-12) |

data/ext/Rakefile

@@ -22,8 +22,10 @@ task :default => :clean do
       ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
     end
 
+    releases = File.expand_path(File.join(File.dirname(__FILE__), '../dist'))
+
     recipe.files << {
-      :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
+      :url => "file://#{releases}/librdkafka_#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz",
       :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
     }
     recipe.configure_options = ["--host=#{recipe.host}"]

data/lib/rdkafka/consumer.rb

@@ -435,6 +435,19 @@ module Rdkafka
     # @return [nil]
     # @raise [RdkafkaError] When seeking fails
     def seek(message)
+      seek_by(message.topic, message.partition, message.offset)
+    end
+
+    # Seek to a particular message by providing the topic, partition and offset.
+    # The next poll on the topic/partition will return the
+    # message at the given offset.
+    #
+    # @param topic [String] The topic in which to seek
+    # @param partition [Integer] The partition number to seek
+    # @param offset [Integer] The partition offset to seek
+    # @return [nil]
+    # @raise [RdkafkaError] When seeking fails
+    def seek_by(topic, partition, offset)
       closed_consumer_check(__method__)
 
       # rd_kafka_offset_store is one of the few calls that does not support
@@ -442,14 +455,14 @@ module Rdkafka
       native_topic = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_topic_new(
           inner,
-          message.topic,
+          topic,
           nil
         )
       end
       response = Rdkafka::Bindings.rd_kafka_seek(
         native_topic,
-        message.partition,
-        message.offset,
+        partition,
+        offset,
         0 # timeout
       )
       Rdkafka::RdkafkaError.validate!(response)

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.16.0"
+  VERSION = "0.16.1"
   LIBRDKAFKA_VERSION = "2.4.0"
   LIBRDKAFKA_SOURCE_SHA256 = "d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a"
 end

data/spec/rdkafka/consumer_spec.rb

@@ -258,6 +258,95 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe "#seek_by" do
+    let(:topic) { "consume_test_topic" }
+    let(:partition) { 0 }
+    let(:offset) { 0 }
+
+    it "should raise an error when seeking fails" do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
+      expect {
+        consumer.seek_by(topic, partition, offset)
+      }.to raise_error Rdkafka::RdkafkaError
+    end
+
+    context "subscription" do
+      let(:timeout) { 1000 }
+
+      before do
+        consumer.subscribe(topic)
+
+        # 1. partitions are assigned
+        wait_for_assignment(consumer)
+        expect(consumer.assignment).not_to be_empty
+
+        # 2. eat unrelated messages
+        while(consumer.poll(timeout)) do; end
+      end
+      after { consumer.unsubscribe }
+
+      def send_one_message(val)
+        producer.produce(
+          topic:     topic,
+          payload:   "payload #{val}",
+          key:       "key 1",
+          partition: 0
+        ).wait
+      end
+
+      it "works when a partition is paused" do
+        # 3. get reference message
+        send_one_message(:a)
+        message1 = consumer.poll(timeout)
+        expect(message1&.payload).to eq "payload a"
+
+        # 4. pause the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic(topic, 1)
+        consumer.pause(tpl)
+
+        # 5. seek by the previous message fields
+        consumer.seek_by(message1.topic, message1.partition, message1.offset)
+
+        # 6. resume the subscription
+        tpl = Rdkafka::Consumer::TopicPartitionList.new
+        tpl.add_topic(topic, 1)
+        consumer.resume(tpl)
+
+        # 7. ensure same message is read again
+        message2 = consumer.poll(timeout)
+
+        # This is needed because `enable.auto.offset.store` is true but when running in CI that
+        # is overloaded, offset store lags
+        sleep(2)
+
+        consumer.commit
+        expect(message1.offset).to eq message2.offset
+        expect(message1.payload).to eq message2.payload
+      end
+
+      it "allows skipping messages" do
+        # 3. send messages
+        send_one_message(:a)
+        send_one_message(:b)
+        send_one_message(:c)
+
+        # 4. get reference message
+        message = consumer.poll(timeout)
+        expect(message&.payload).to eq "payload a"
+
+        # 5. seek over one message
+        consumer.seek_by(message.topic, message.partition, message.offset + 2)
+
+        # 6. ensure that only one message is available
+        records = consumer.poll(timeout)
+        expect(records&.payload).to eq "payload c"
+        records = consumer.poll(timeout)
+        expect(records).to be_nil
+      end
+    end
+  end
+
   describe "#assign and #assignment" do
     it "should return an empty assignment if nothing is assigned" do
       expect(consumer.assignment).to be_empty

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.16.1
 platform: ruby
 authors:
 - Thijs Cadier
@@ -36,7 +36,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-06-17 00:00:00.000000000 Z
+date: 2024-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -186,6 +186,7 @@ files:
 - README.md
 - Rakefile
 - certs/cert_chain.pem
+- dist/librdkafka_2.4.0.tar.gz
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile