karafka-rdkafka 0.14.3 → 0.14.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5fee43511074dfab207280d7114f70f8fb5cc7b6055fdb348cd9da302ee037d2
-  data.tar.gz: 22656cc0b49a6d3aa7d6fad85c1b3c936b49089802ed07f4f43d8fdd85a16f2d
+  metadata.gz: 5ed56a133dd27e17055cf63b7e68945d9cb97b42690afa6c7eb230f56273754c
+  data.tar.gz: bcb63a1c2c77d599ab403bfdca1df0badd28f1c1cc72eed16169e6d4044fe778
 SHA512:
-  metadata.gz: cdc7873ca1dda1d2afc6193c8e189850d5d3f865f19307cd997cc76c3fa39a919f2757ed610cf18854dc1ff61b270d1a9c85f880257c1c833435d7af9992a27a
-  data.tar.gz: ed37156d9f17e8be5804720b426424260c3e70567cf2e24dc385ed5e8d66763199fb056a2c47e11fec1ff80ddaae8b2926f9420b05d1c9e88dc38735c32b4eb1
+  metadata.gz: '07185f868e86c3aaa917f8be42a14047cfb9dd9901001351f416f14338bf1dcc942b4d4649749832efed5c10cb3c7e9ea3543706fd7e11bb79fc332206569898'
+  data.tar.gz: c7bafd507d2b86c9ae61daab5ed32b142ec6d9e7dbd6d4cf26a704d432b257c00e84fb8872daab978c685941ab71e3201d7fe05635fa0fc8526eac016faf48d9

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Rdkafka Changelog
 
+## 0.14.5 (2023-12-20)
+- [Enhancement] Provide `label` producer handler and report reference for improved traceability.
+
+## 0.14.4 (2023-12-19)
+- [Enhancement] Add ability to store offsets in a transaction (mensfeld)
+
 ## 0.14.3 (2023-12-17)
 - [Enhancement] Replace `rd_kafka_offset_store` with `rd_kafka_offsets_store` (mensfeld)
 - [Fix] Missing ACL `RD_KAFKA_RESOURCE_BROKER` constant reference (mensfeld)

data/lib/rdkafka/bindings.rb

@@ -205,6 +205,9 @@ module Rdkafka
     attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
     attach_function :rd_kafka_offsets_for_times, [:pointer, :pointer, :int], :int, blocking: true
     attach_function :rd_kafka_position, [:pointer, :pointer], :int, blocking: true
+    # those two are used for eos support
+    attach_function :rd_kafka_consumer_group_metadata, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_consumer_group_metadata_destroy, [:pointer], :void, blocking: true
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
@@ -276,6 +279,7 @@ module Rdkafka
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
     attach_function :rd_kafka_init_transactions, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_send_offsets_to_transaction, [:pointer, :pointer, :pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_begin_transaction, [:pointer], :pointer, blocking: true
     attach_function :rd_kafka_abort_transaction, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_commit_transaction, [:pointer, :int], :pointer, blocking: true

data/lib/rdkafka/callbacks.rb

@@ -288,7 +288,16 @@ module Rdkafka
 
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], topic_name, message[:err]), delivery_handle)
+            opaque.call_delivery_callback(
+              Rdkafka::Producer::DeliveryReport.new(
+                message[:partition],
+                message[:offset],
+                topic_name,
+                message[:err],
+                delivery_handle.label
+              ),
+              delivery_handle
+            )
           end
         end
       end

data/lib/rdkafka/consumer.rb

@@ -671,6 +671,22 @@ module Rdkafka
       end
     end
 
+    # Returns pointer to the consumer group metadata. It is used only in the context of
+    # exactly-once-semantics in transactions, this is why it is never remapped to Ruby
+    #
+    # This API is **not** usable by itself from Ruby
+    #
+    # @note This pointer **needs** to be removed with `#rd_kafka_consumer_group_metadata_destroy`
+    #
+    # @private
+    def consumer_group_metadata_pointer
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Bindings.rd_kafka_consumer_group_metadata(inner)
+      end
+    end
+
     private
 
     def closed_consumer_check(method)

data/lib/rdkafka/producer/delivery_handle.rb

@@ -11,6 +11,9 @@ module Rdkafka
              :offset, :int64,
              :topic_name, :pointer
 
+      # @return [Object, nil] label set during message production or nil by default
+      attr_accessor :label
+
       # @return [String] the name of the operation (e.g. "delivery")
       def operation_name
         "delivery"
@@ -22,7 +25,9 @@ module Rdkafka
           DeliveryReport.new(
             self[:partition],
             self[:offset],
-            self[:topic_name].read_string
+            self[:topic_name].read_string,
+            nil,
+            label
           )
         else
           DeliveryReport.new(
@@ -31,7 +36,8 @@ module Rdkafka
             # For part of errors, we will not get a topic name reference and in cases like this
             # we should not return it
             self[:topic_name].null? ? nil : self[:topic_name].read_string,
-            Rdkafka::RdkafkaError.build(self[:response])
+            Rdkafka::RdkafkaError.build(self[:response]),
+            label
           )
         end
       end

data/lib/rdkafka/producer/delivery_report.rb

@@ -21,6 +21,9 @@ module Rdkafka
       # @return [Integer]
       attr_reader :error
 
+      # @return [Object, nil] label set during message production or nil by default
+      attr_reader :label
+
       # We alias the `#topic_name` under `#topic` to make this consistent with `Consumer::Message`
       # where the topic name is under `#topic` method. That way we have a consistent name that
       # is present in both places
@@ -30,11 +33,12 @@ module Rdkafka
 
       private
 
-      def initialize(partition, offset, topic_name = nil, error = nil)
+      def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
         @partition = partition
         @offset = offset
         @topic_name = topic_name
         @error = error
+        @label = label
       end
     end
   end

data/lib/rdkafka/producer.rb

@@ -81,7 +81,7 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_begin_transaction(inner)
 
-        Rdkafka::RdkafkaError.validate!(response_ptr)
+        Rdkafka::RdkafkaError.validate!(response_ptr) || true
       end
     end
 
@@ -91,7 +91,7 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_commit_transaction(inner, timeout_ms)
 
-        Rdkafka::RdkafkaError.validate!(response_ptr)
+        Rdkafka::RdkafkaError.validate!(response_ptr) || true
       end
     end
 
@@ -100,8 +100,35 @@ module Rdkafka
 
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_abort_transaction(inner, timeout_ms)
+        Rdkafka::RdkafkaError.validate!(response_ptr) || true
+      end
+    end
+
+    # Sends provided offsets of a consumer to the transaction for collective commit
+    #
+    # @param consumer [Consumer] consumer that owns the given tpls
+    # @param tpl [Consumer::TopicPartitionList]
+    # @param timeout_ms [Integer] offsets send timeout
+    # @note Use **only** in the context of an active transaction
+    def send_offsets_to_transaction(consumer, tpl, timeout_ms = 5_000)
+      closed_producer_check(__method__)
+
+      return if tpl.empty?
+
+      cgmetadata = consumer.consumer_group_metadata_pointer
+      native_tpl = tpl.to_native_tpl
+
+      @native_kafka.with_inner do |inner|
+        response_ptr = Bindings.rd_kafka_send_offsets_to_transaction(inner, native_tpl, cgmetadata, timeout_ms)
+
         Rdkafka::RdkafkaError.validate!(response_ptr)
       end
+    ensure
+      if cgmetadata && !cgmetadata.null?
+        Bindings.rd_kafka_consumer_group_metadata_destroy(cgmetadata)
+      end
+
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native_tpl) unless native_tpl.nil?
     end
 
     # Close this producer and wait for the internal poll queue to empty.
@@ -199,11 +226,12 @@ module Rdkafka
     # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     # @param headers [Hash<String,String>] Optional message headers
+    # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     #
     # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
-    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
+    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil, label: nil)
       closed_producer_check(__method__)
 
       # Start by checking and converting the input
@@ -245,6 +273,7 @@ module Rdkafka
                       end
 
       delivery_handle = DeliveryHandle.new
+      delivery_handle.label = label
       delivery_handle[:pending] = true
       delivery_handle[:response] = -1
       delivery_handle[:partition] = -1

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.14.3"
+  VERSION = "0.14.5"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/spec/rdkafka/consumer_spec.rb

@@ -360,7 +360,6 @@ describe Rdkafka::Consumer do
     end
   end
 
-
   describe "#position, #commit, #committed and #store_offset" do
     # Make sure there are messages to work with
     let!(:report) do
@@ -1142,6 +1141,14 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe '#consumer_group_metadata_pointer' do
+    it 'expect to yield pointer' do
+      consumer.consumer_group_metadata_pointer do |pointer|
+        expect(pointer).to be_a(FFI::Pointer)
+      end
+    end
+  end
+
   describe "a rebalance listener" do
     let(:consumer) do
       config = rdkafka_consumer_config

data/spec/rdkafka/producer_spec.rb

@@ -34,6 +34,7 @@ describe Rdkafka::Producer do
 
         producer.delivery_callback = lambda do |report|
           expect(report).not_to be_nil
+          expect(report.label).to eq "label"
           expect(report.partition).to eq 1
           expect(report.offset).to be >= 0
           expect(report.topic_name).to eq "produce_test_topic"
@@ -44,9 +45,12 @@ describe Rdkafka::Producer do
         handle = producer.produce(
           topic:   "produce_test_topic",
           payload: "payload",
-          key:     "key"
+          key:     "key",
+          label:   "label"
         )
 
+        expect(handle.label).to eq "label"
+
         # Wait for it to be delivered
         handle.wait(max_wait_timeout: 15)
 
@@ -175,11 +179,13 @@ describe Rdkafka::Producer do
     handle = producer.produce(
       topic:   "produce_test_topic",
       payload: "payload",
-      key:     "key"
+      key:     "key",
+      label:   "label"
     )
 
     # Should be pending at first
     expect(handle.pending?).to be true
+    expect(handle.label).to eq "label"
 
     # Check delivery handle and report
     report = handle.wait(max_wait_timeout: 5)
@@ -187,6 +193,7 @@ describe Rdkafka::Producer do
     expect(report).not_to be_nil
     expect(report.partition).to eq 1
     expect(report.offset).to be >= 0
+    expect(report.label).to eq "label"
 
     # Flush and close producer
     producer.flush
@@ -567,10 +574,11 @@ describe Rdkafka::Producer do
     end
 
     it "should contain the error in the response when not deliverable" do
-      handler = producer.produce(topic: 'produce_test_topic', payload: nil)
+      handler = producer.produce(topic: 'produce_test_topic', payload: nil, label: 'na')
       # Wait for the async callbacks and delivery registry to update
       sleep(2)
       expect(handler.create_result.error).to be_a(Rdkafka::RdkafkaError)
+      expect(handler.create_result.label).to eq('na')
     end
   end
 
@@ -883,5 +891,30 @@ describe Rdkafka::Producer do
         expect { producer2.commit_transaction }.not_to raise_error
       end
     end
+
+    context 'when having a consumer with tpls for exactly once semantics' do
+      let(:tpl) do
+        producer.produce(topic: 'consume_test_topic', payload: 'data1', partition: 0).wait
+        result = producer.produce(topic: 'consume_test_topic', payload: 'data1', partition: 0).wait
+
+        Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => result.offset + 1)
+        end
+      end
+
+      before do
+        consumer.subscribe("consume_test_topic")
+        wait_for_assignment(consumer)
+        producer.init_transactions
+        producer.begin_transaction
+      end
+
+      after { consumer.unsubscribe }
+
+      it 'expect to store offsets and not crash' do
+        producer.send_offsets_to_transaction(consumer, tpl)
+        producer.commit_transaction
+      end
+    end
   end
 end

data/spec/spec_helper.rb

@@ -107,10 +107,6 @@ def wait_for_unassignment(consumer)
   end
 end
 
-def objects_of_type_count(type)
-  ObjectSpace.each_object(type).count
-end
-
 def notify_listener(listener, &block)
   # 1. subscribe and poll
   consumer.subscribe("consume_test_topic")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.14.3
+  version: 0.14.5
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-12-17 00:00:00.000000000 Z
+date: 2023-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi