karafka-rdkafka 0.14.3 → 0.14.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5fee43511074dfab207280d7114f70f8fb5cc7b6055fdb348cd9da302ee037d2
-  data.tar.gz: 22656cc0b49a6d3aa7d6fad85c1b3c936b49089802ed07f4f43d8fdd85a16f2d
+  metadata.gz: d22e33d93f77f138063f03111b25386c08e1667012b6ab4c8449fef61754ad44
+  data.tar.gz: b8cc0557cbb945e5d7b7e025198a292a7cf601ce9decef97cb5102fd25af742b
 SHA512:
-  metadata.gz: cdc7873ca1dda1d2afc6193c8e189850d5d3f865f19307cd997cc76c3fa39a919f2757ed610cf18854dc1ff61b270d1a9c85f880257c1c833435d7af9992a27a
-  data.tar.gz: ed37156d9f17e8be5804720b426424260c3e70567cf2e24dc385ed5e8d66763199fb056a2c47e11fec1ff80ddaae8b2926f9420b05d1c9e88dc38735c32b4eb1
+  metadata.gz: cfc9af0e70030f513c4c34b00f9b94b6b3b9c828add2ac6cd742ebe2eb308a1e1a5489bf4c4280c34a98d2f0118d741608918314fa8f267b7cf98b22b3d8782d
+  data.tar.gz: 315b552a77691b62e5a30fd5eaec1bf89efd8ec5789f9f0deeadf3997eeca9dc83bb6466ee8365e40e53d2e09707d3d68a57278b3637b170ad027b5080838519

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Rdkafka Changelog
 
+## 0.14.4 (2023-12-19)
+- [Enhancement] Add ability to store offsets in a transaction (mensfeld)
+
 ## 0.14.3 (2023-12-17)
 - [Enhancement] Replace `rd_kafka_offset_store` with `rd_kafka_offsets_store` (mensfeld)
 - [Fix] Missing ACL `RD_KAFKA_RESOURCE_BROKER` constant reference (mensfeld)

data/lib/rdkafka/bindings.rb

@@ -205,6 +205,9 @@ module Rdkafka
     attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
     attach_function :rd_kafka_offsets_for_times, [:pointer, :pointer, :int], :int, blocking: true
     attach_function :rd_kafka_position, [:pointer, :pointer], :int, blocking: true
+    # those two are used for eos support
+    attach_function :rd_kafka_consumer_group_metadata, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_consumer_group_metadata_destroy, [:pointer], :void, blocking: true
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
@@ -276,6 +279,7 @@ module Rdkafka
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
     attach_function :rd_kafka_init_transactions, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_send_offsets_to_transaction, [:pointer, :pointer, :pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_begin_transaction, [:pointer], :pointer, blocking: true
     attach_function :rd_kafka_abort_transaction, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_commit_transaction, [:pointer, :int], :pointer, blocking: true

data/lib/rdkafka/consumer.rb

@@ -671,6 +671,22 @@ module Rdkafka
       end
     end
 
+    # Returns pointer to the consumer group metadata. It is used only in the context of
+    # exactly-once-semantics in transactions, this is why it is never remapped to Ruby
+    #
+    # This API is **not** usable by itself from Ruby
+    #
+    # @note This pointer **needs** to be removed with `#rd_kafka_consumer_group_metadata_destroy`
+    #
+    # @private
+    def consumer_group_metadata_pointer
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Bindings.rd_kafka_consumer_group_metadata(inner)
+      end
+    end
+
     private
 
     def closed_consumer_check(method)

data/lib/rdkafka/producer.rb

@@ -81,7 +81,7 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_begin_transaction(inner)
 
-        Rdkafka::RdkafkaError.validate!(response_ptr)
+        Rdkafka::RdkafkaError.validate!(response_ptr) || true
       end
     end
 
@@ -91,7 +91,7 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_commit_transaction(inner, timeout_ms)
 
-        Rdkafka::RdkafkaError.validate!(response_ptr)
+        Rdkafka::RdkafkaError.validate!(response_ptr) || true
       end
     end
 
@@ -100,8 +100,35 @@ module Rdkafka
 
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_abort_transaction(inner, timeout_ms)
+        Rdkafka::RdkafkaError.validate!(response_ptr) || true
+      end
+    end
+
+    # Sends provided offsets of a consumer to the transaction for collective commit
+    #
+    # @param consumer [Consumer] consumer that owns the given tpls
+    # @param tpl [Consumer::TopicPartitionList]
+    # @param timeout_ms [Integer] offsets send timeout
+    # @note Use **only** in the context of an active transaction
+    def send_offsets_to_transaction(consumer, tpl, timeout_ms = 5_000)
+      closed_producer_check(__method__)
+
+      return if tpl.empty?
+
+      cgmetadata = consumer.consumer_group_metadata_pointer
+      native_tpl = tpl.to_native_tpl
+
+      @native_kafka.with_inner do |inner|
+        response_ptr = Bindings.rd_kafka_send_offsets_to_transaction(inner, native_tpl, cgmetadata, timeout_ms)
+
         Rdkafka::RdkafkaError.validate!(response_ptr)
       end
+    ensure
+      if cgmetadata && !cgmetadata.null?
+        Bindings.rd_kafka_consumer_group_metadata_destroy(cgmetadata)
+      end
+
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native_tpl) unless native_tpl.nil?
     end
 
     # Close this producer and wait for the internal poll queue to empty.

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.14.3"
+  VERSION = "0.14.4"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/spec/rdkafka/consumer_spec.rb

@@ -360,7 +360,6 @@ describe Rdkafka::Consumer do
     end
   end
 
-
   describe "#position, #commit, #committed and #store_offset" do
     # Make sure there are messages to work with
     let!(:report) do
@@ -1142,6 +1141,14 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe '#consumer_group_metadata_pointer' do
+    it 'expect to yield pointer' do
+      consumer.consumer_group_metadata_pointer do |pointer|
+        expect(pointer).to be_a(FFI::Pointer)
+      end
+    end
+  end
+
   describe "a rebalance listener" do
     let(:consumer) do
       config = rdkafka_consumer_config

data/spec/rdkafka/producer_spec.rb

@@ -883,5 +883,30 @@ describe Rdkafka::Producer do
         expect { producer2.commit_transaction }.not_to raise_error
       end
     end
+
+    context 'when having a consumer with tpls for exactly once semantics' do
+      let(:tpl) do
+        producer.produce(topic: 'consume_test_topic', payload: 'data1', partition: 0).wait
+        result = producer.produce(topic: 'consume_test_topic', payload: 'data1', partition: 0).wait
+
+        Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => result.offset + 1)
+        end
+      end
+
+      before do
+        consumer.subscribe("consume_test_topic")
+        wait_for_assignment(consumer)
+        producer.init_transactions
+        producer.begin_transaction
+      end
+
+      after { consumer.unsubscribe }
+
+      it 'expect to store offsets and not crash' do
+        producer.send_offsets_to_transaction(consumer, tpl)
+        producer.commit_transaction
+      end
+    end
   end
 end

data/spec/spec_helper.rb

@@ -107,10 +107,6 @@ def wait_for_unassignment(consumer)
   end
 end
 
-def objects_of_type_count(type)
-  ObjectSpace.each_object(type).count
-end
-
 def notify_listener(listener, &block)
   # 1. subscribe and poll
   consumer.subscribe("consume_test_topic")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.14.3
+  version: 0.14.4
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-12-17 00:00:00.000000000 Z
+date: 2023-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi