karafka-rdkafka 0.13.7 → 0.13.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4fe6b47d334265ef793c32b215dca1d97adfa42a51215b9492f1d36b58a84403
-  data.tar.gz: 4fc125147d796f981314640779adf5234376b0d0357c0321683002820fcb3301
+  metadata.gz: 4082c381a9d131273cc005b61b06f0e4c73b27044213d730fa5c7faeec606e07
+  data.tar.gz: 06754bdba16fc3feaf648670e766cdbbc60342b1a43902ef08e42a5acfd4b2ac
 SHA512:
-  metadata.gz: f7fc5d7dc99653f117b4bef3baa0d6e772537f4a7df6375a6a33692d4a9728f81308e11c3cd873fbb5686fb83cf2021a5866eb252e10661b3fe512639e3e3349
-  data.tar.gz: 9d23217c18093187f1fecc5afbbdbe05d2a6be3cb99adb0dc20ed7458da19c7035cc51eda664e09c3acaa5ac3dd3539b0c58c91cd6d6faed3c1d21edc9662dde
+  metadata.gz: 89c0b5078f97c31d0e209ebbf13279a2d079aef35f7b4c2a4bf92cac17a69ccfed1769f84d1d06ca734c9a95174b0b5fa4fa9b6933f63e30fe71c8493bda4ed1
+  data.tar.gz: 490a160689be7c4e261b2b8ff1ffdbcb5aec53cfe1dd0acabd0da33eff708cb7a1d09afe3c3a6752fc9a6345c005f1f0919604035a97240c5f9ed120870e802c

data/CHANGELOG.md

@@ -1,4 +1,7 @@
-# 0.13.7 (Unreleased)
+# 0.13.8 (2023-10-31)
+- [Enhancement] Get consumer position (thijsc & mensfeld)
+
+# 0.13.7 (2023-10-31)
 - [Change] Drop support for Ruby 2.6 due to incompatibilities in usage of `ObjectSpace::WeakMap`
 - [Fix] Fix dangling Opaque references.
 

data/lib/rdkafka/bindings.rb

@@ -208,6 +208,7 @@ module Rdkafka
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
     attach_function :rd_kafka_offsets_for_times, [:pointer, :pointer, :int], :int, blocking: true
+    attach_function :rd_kafka_position, [:pointer, :pointer], :int, blocking: true
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int

data/lib/rdkafka/consumer.rb

@@ -29,6 +29,13 @@ module Rdkafka
       ->(_) { close }
     end
 
+    # @return [String] consumer name
+    def name
+      @name ||= @native_kafka.with_inner do |inner|
+        ::Rdkafka::Bindings.rd_kafka_name(inner)
+      end
+    end
+
     # Close this consumer
     # @return [nil]
     def close
@@ -269,6 +276,32 @@ module Rdkafka
       end
     end
 
+    # Return the current positions (offsets) for topics and partitions.
+    # The offset field of each requested partition will be set to the offset of the last consumed message + 1, or nil in case there was no previous message.
+    #
+    # @param list [TopicPartitionList, nil] The topic with partitions to get the offsets for or nil to use the current subscription.
+    #
+    # @raise [RdkafkaError] When getting the positions fails.
+    #
+    # @return [TopicPartitionList]
+    def position(list=nil)
+      if list.nil?
+        list = assignment
+      elsif !list.is_a?(TopicPartitionList)
+        raise TypeError.new("list has to be nil or a TopicPartitionList")
+      end
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_position(inner, tpl)
+      end
+
+      Rdkafka::RdkafkaError.validate!(response)
+
+      TopicPartitionList.from_native_tpl(tpl)
+    end
+
     # Query broker for low (oldest/beginning) and high (newest/end) offsets for a partition.
     #
     # @param topic [String] The topic to query

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.7"
+  VERSION = "0.13.8"
   LIBRDKAFKA_VERSION = "2.2.0"
   LIBRDKAFKA_SOURCE_SHA256 = "af9a820cbecbc64115629471df7c7cecd40403b6c34bfdbb9223152677a47226"
 end

data/spec/rdkafka/consumer_spec.rb

@@ -337,8 +337,9 @@ describe Rdkafka::Consumer do
     end
   end
 
-  describe "#commit, #committed and #store_offset" do
-    # Make sure there's a stored offset
+
+  describe "#position, #commit, #committed and #store_offset" do
+    # Make sure there are messages to work with
     let!(:report) do
       producer.produce(
         topic:     "consume_test_topic",
@@ -356,29 +357,33 @@ describe Rdkafka::Consumer do
       )
     end
 
-    it "should only accept a topic partition list in committed" do
-      expect {
-        consumer.committed("list")
-      }.to raise_error TypeError
+    describe "#position" do
+      it "should only accept a topic partition list in position if not nil" do
+        expect {
+          consumer.position("list")
+        }.to raise_error TypeError
+      end
     end
 
-    it "should commit in sync mode" do
-      expect {
-        consumer.commit(nil, true)
-      }.not_to raise_error
-    end
+    describe "#committed" do
+      it "should only accept a topic partition list in commit if not nil" do
+        expect {
+          consumer.commit("list")
+        }.to raise_error TypeError
+      end
 
-    it "should only accept a topic partition list in commit if not nil" do
-      expect {
-        consumer.commit("list")
-      }.to raise_error TypeError
+      it "should commit in sync mode" do
+        expect {
+          consumer.commit(nil, true)
+        }.not_to raise_error
+      end
     end
 
     context "with a committed consumer" do
       before :all do
         # Make sure there are some messages.
         handles = []
-        producer = rdkafka_producer_config.producer
+        producer = rdkafka_config.producer
         10.times do
           (0..2).each do |i|
             handles << producer.produce(
@@ -422,31 +427,33 @@ describe Rdkafka::Consumer do
         }.to raise_error(Rdkafka::RdkafkaError)
       end
 
-      it "should fetch the committed offsets for the current assignment" do
-        partitions = consumer.committed.to_h["consume_test_topic"]
-        expect(partitions).not_to be_nil
-        expect(partitions[0].offset).to eq 1
-      end
+      describe "#committed" do
+        it "should fetch the committed offsets for the current assignment" do
+          partitions = consumer.committed.to_h["consume_test_topic"]
+          expect(partitions).not_to be_nil
+          expect(partitions[0].offset).to eq 1
+        end
 
-      it "should fetch the committed offsets for a specified topic partition list" do
-        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-          list.add_topic("consume_test_topic", [0, 1, 2])
+        it "should fetch the committed offsets for a specified topic partition list" do
+          list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+            list.add_topic("consume_test_topic", [0, 1, 2])
+          end
+          partitions = consumer.committed(list).to_h["consume_test_topic"]
+          expect(partitions).not_to be_nil
+          expect(partitions[0].offset).to eq 1
+          expect(partitions[1].offset).to eq 1
+          expect(partitions[2].offset).to eq 1
         end
-        partitions = consumer.committed(list).to_h["consume_test_topic"]
-        expect(partitions).not_to be_nil
-        expect(partitions[0].offset).to eq 1
-        expect(partitions[1].offset).to eq 1
-        expect(partitions[2].offset).to eq 1
-      end
 
-      it "should raise an error when getting committed fails" do
-        expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
-        list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
-          list.add_topic("consume_test_topic", [0, 1, 2])
+        it "should raise an error when getting committed fails" do
+          expect(Rdkafka::Bindings).to receive(:rd_kafka_committed).and_return(20)
+          list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+            list.add_topic("consume_test_topic", [0, 1, 2])
+          end
+          expect {
+            consumer.committed(list)
+          }.to raise_error Rdkafka::RdkafkaError
         end
-        expect {
-          consumer.committed(list)
-        }.to raise_error Rdkafka::RdkafkaError
       end
 
       describe "#store_offset" do
@@ -467,6 +474,8 @@ describe Rdkafka::Consumer do
           @new_consumer.store_offset(message)
           @new_consumer.commit
 
+          # TODO use position here, should be at offset
+
           list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
             list.add_topic("consume_test_topic", [0, 1, 2])
           end
@@ -481,6 +490,35 @@ describe Rdkafka::Consumer do
             @new_consumer.store_offset(message)
           }.to raise_error Rdkafka::RdkafkaError
         end
+
+        describe "#position" do
+          it "should fetch the positions for the current assignment" do
+            consumer.store_offset(message)
+
+            partitions = consumer.position.to_h["consume_test_topic"]
+            expect(partitions).not_to be_nil
+            expect(partitions[0].offset).to eq message.offset + 1
+          end
+
+          it "should fetch the positions for a specified assignment" do
+            consumer.store_offset(message)
+
+            list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+              list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => nil, 1 => nil, 2 => nil)
+            end
+            partitions = consumer.position(list).to_h["consume_test_topic"]
+            expect(partitions).not_to be_nil
+            expect(partitions[0].offset).to eq message.offset + 1
+          end
+
+          it "should raise an error when getting the position fails" do
+            expect(Rdkafka::Bindings).to receive(:rd_kafka_position).and_return(20)
+
+            expect {
+              consumer.position
+            }.to raise_error(Rdkafka::RdkafkaError)
+          end
+        end
       end
     end
   end

data/spec/spec_helper.rb

@@ -36,7 +36,7 @@ def rdkafka_consumer_config(config_overrides={})
   # Add consumer specific fields to it
   config[:"auto.offset.reset"] = "earliest"
   config[:"enable.partition.eof"] = false
-  config[:"group.id"] = "ruby-test-#{Random.new.rand(0..1_000_000)}"
+  config[:"group.id"] = "ruby-test-#{SecureRandom.uuid}"
   # Enable debug mode if required
   if ENV["DEBUG_CONSUMER"]
     config[:debug] = "cgrp,topic,fetch"

data.tar.gz.sig

@@ -1,4 +1,2 @@
-�>o���S���Q�Vz���'��f�P���o'��&O션���Obb3k����<-�i��߰M��ϘG���D��Pt��V��[������d[��ʆ>�")�G*:A�(
Q�0٪6���VV�R�5֜P}<o��S۲��I70b\
-T��J���#����f�
-2$��yD
-.�oV0(�O
Eq��-<[tH��@��[׀|V�-����'���+$Y��:Xnl�h)�� ��(���'�:�@�RI�Yq�ٺ�mPX~�R�7P���@�*�\oڎY��r*UdZL�ݻ{iRho?�g�����Y�p�D�z��E��Ęٍ�Ci������
+إ�D�̡}f���ř��w�Oٯ뽊t�#Q�2�Q��K����+a���(�m	��
+���U��	^��5�.
�+��O��+�u����.;�À��q����L�`x�

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.13.7
+  version: 0.13.8
 platform: ruby
 authors:
 - Thijs Cadier