karafka-rdkafka 0.14.4 → 0.14.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d22e33d93f77f138063f03111b25386c08e1667012b6ab4c8449fef61754ad44
-  data.tar.gz: b8cc0557cbb945e5d7b7e025198a292a7cf601ce9decef97cb5102fd25af742b
+  metadata.gz: c7376c9030672ba0205c3973317a626b7ececd3b100c312e5ef310d08de397cc
+  data.tar.gz: b843498f05cdadfd034c9c20338fd698b37df792c0bd54919c19632ae6ac8d29
 SHA512:
-  metadata.gz: cfc9af0e70030f513c4c34b00f9b94b6b3b9c828add2ac6cd742ebe2eb308a1e1a5489bf4c4280c34a98d2f0118d741608918314fa8f267b7cf98b22b3d8782d
-  data.tar.gz: 315b552a77691b62e5a30fd5eaec1bf89efd8ec5789f9f0deeadf3997eeca9dc83bb6466ee8365e40e53d2e09707d3d68a57278b3637b170ad027b5080838519
+  metadata.gz: ca8944dae92ea927a09748524a04df8079e9a1bef1c84034cdf2d5c372ca122e652a1b728c5a497ac26cefa3ae6b2eeb52da3df988332f06ef1a153ce8773028
+  data.tar.gz: 5772911202fae3ccd51d54300a0dd74159188d17395e4ee6895132abcc30a916a9adcf9643f1626dc9884f8a17c88e6d3d546bb0191218a4e263a4cbb37b3009

data/.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.3.0-rc1'
+          - '3.3'
           - '3.2'
           - '3.1'
           - '3.1.0'
@@ -31,7 +31,7 @@ jobs:
           - '2.7'
           - '2.7.0'
         include:
-          - ruby: '3.2'
+          - ruby: '3.3'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v4

data/.ruby-version

@@ -1 +1 @@
-3.2.2
+3.3.0

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Rdkafka Changelog
 
+## 0.14.6 (2023-12-29)
+- **[Feature]** Support storing metadata alongside offsets via `rd_kafka_offsets_store` in `#store_offset` (mensfeld)
+- [Enhancement] Increase the `#committed` default timeout from 1_200ms to 2000ms. This will compensate for network glitches and remote clusters operations and will align with metadata query timeout.
+
+## 0.14.5 (2023-12-20)
+- [Enhancement] Provide `label` producer handler and report reference for improved traceability.
+
 ## 0.14.4 (2023-12-19)
 - [Enhancement] Add ability to store offsets in a transaction (mensfeld)
 

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.2
+    image: confluentinc/cp-kafka:7.5.3
 
     ports:
       - 9092:9092

data/lib/rdkafka/bindings.rb

@@ -92,7 +92,7 @@ module Rdkafka
     end
 
     attach_function :rd_kafka_topic_partition_list_new, [:int32], :pointer
-    attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :void
+    attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :pointer
     attach_function :rd_kafka_topic_partition_list_set_offset, [:pointer, :string, :int32, :int64], :void
     attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
     attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer

data/lib/rdkafka/callbacks.rb

@@ -288,7 +288,16 @@ module Rdkafka
 
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], topic_name, message[:err]), delivery_handle)
+            opaque.call_delivery_callback(
+              Rdkafka::Producer::DeliveryReport.new(
+                message[:partition],
+                message[:offset],
+                topic_name,
+                message[:err],
+                delivery_handle.label
+              ),
+              delivery_handle
+            )
           end
         end
       end

data/lib/rdkafka/consumer/partition.rb

@@ -16,11 +16,16 @@ module Rdkafka
       # @return [Integer]
       attr_reader :err
 
+      # Partition metadata in the context of a consumer
+      # @return [String, nil]
+      attr_reader :metadata
+
       # @private
-      def initialize(partition, offset, err = 0)
+      def initialize(partition, offset, err = 0, metadata = nil)
         @partition = partition
         @offset = offset
         @err = err
+        @metadata = metadata
       end
 
       # Human readable representation of this partition.
@@ -29,6 +34,7 @@ module Rdkafka
         message = "<Partition #{partition}"
         message += " offset=#{offset}" if offset
         message += " err=#{err}" if err != 0
+        message += " metadata=#{metadata}" if metadata != nil
         message += ">"
         message
       end

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -66,10 +66,14 @@ module Rdkafka
       #
       # @param topic [String] The topic's name
       # @param partitions_with_offsets [Hash<Integer, Integer>] The topic's partitions and offsets
+      # @param partitions_with_offsets [Array<Consumer::Partition>] The topic's partitions with offsets
+      #   and metadata (if any)
       #
       # @return [nil]
       def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
-        @data[topic.to_s] = partitions_with_offsets.map { |p, o| Partition.new(p, o) }
+        @data[topic.to_s] = partitions_with_offsets.map do |p, o|
+          p.is_a?(Partition) ? p : Partition.new(p, o)
+        end
       end
 
       # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
@@ -114,7 +118,13 @@ module Rdkafka
                      else
                        elem[:offset]
                      end
-            partition = Partition.new(elem[:partition], offset, elem[:err])
+
+            partition = Partition.new(
+              elem[:partition],
+              offset,
+              elem[:err],
+              elem[:metadata].null? ? nil : elem[:metadata].read_string(elem[:metadata_size])
+            )
             partitions.push(partition)
             data[elem[:topic]] = partitions
           end
@@ -136,12 +146,22 @@ module Rdkafka
         @data.each do |topic, partitions|
           if partitions
             partitions.each do |p|
-              Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+              ref = Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
                 tpl,
                 topic,
                 p.partition
               )
 
+              if p.metadata
+                part = Rdkafka::Bindings::TopicPartition.new(ref)
+                str_ptr = FFI::MemoryPointer.from_string(p.metadata)
+                # released here:
+                # https://github.com/confluentinc/librdkafka/blob/e03d3bb91ed92a38f38d9806b8d8deffe78a1de5/src/rdkafka_partition.c#L2682C18-L2682C18
+                str_ptr.autorelease = false
+                part[:metadata] = str_ptr
+                part[:metadata_size] = p.metadata.bytesize
+              end
+
               if p.offset
                 offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
 

data/lib/rdkafka/consumer.rb

@@ -243,7 +243,7 @@ module Rdkafka
     # @param timeout_ms [Integer] The timeout for fetching this information.
     # @return [TopicPartitionList]
     # @raise [RdkafkaError] When getting the committed positions fails.
-    def committed(list=nil, timeout_ms=1200)
+    def committed(list=nil, timeout_ms=2_000)
       closed_consumer_check(__method__)
 
       if list.nil?
@@ -383,16 +383,34 @@ module Rdkafka
     # When using this `enable.auto.offset.store` should be set to `false` in the config.
     #
     # @param message [Rdkafka::Consumer::Message] The message which offset will be stored
+    # @param metadata [String, nil] commit metadata string or nil if none
     # @return [nil]
     # @raise [RdkafkaError] When storing the offset fails
-    def store_offset(message)
+    def store_offset(message, metadata = nil)
       closed_consumer_check(__method__)
 
       list = TopicPartitionList.new
-      list.add_topic_and_partitions_with_offsets(
-        message.topic,
-        message.partition => message.offset + 1
-      )
+
+      # For metadata aware commits we build the partition reference directly to save on
+      # objects allocations
+      if metadata
+        list.add_topic_and_partitions_with_offsets(
+          message.topic,
+          [
+            Consumer::Partition.new(
+              message.partition,
+              message.offset + 1,
+              0,
+              metadata
+            )
+          ]
+        )
+      else
+        list.add_topic_and_partitions_with_offsets(
+          message.topic,
+          message.partition => message.offset + 1
+        )
+      end
 
       tpl = list.to_native_tpl
 

data/lib/rdkafka/producer/delivery_handle.rb

@@ -11,6 +11,9 @@ module Rdkafka
              :offset, :int64,
              :topic_name, :pointer
 
+      # @return [Object, nil] label set during message production or nil by default
+      attr_accessor :label
+
       # @return [String] the name of the operation (e.g. "delivery")
       def operation_name
         "delivery"
@@ -22,7 +25,9 @@ module Rdkafka
           DeliveryReport.new(
             self[:partition],
             self[:offset],
-            self[:topic_name].read_string
+            self[:topic_name].read_string,
+            nil,
+            label
           )
         else
           DeliveryReport.new(
@@ -31,7 +36,8 @@ module Rdkafka
             # For part of errors, we will not get a topic name reference and in cases like this
             # we should not return it
             self[:topic_name].null? ? nil : self[:topic_name].read_string,
-            Rdkafka::RdkafkaError.build(self[:response])
+            Rdkafka::RdkafkaError.build(self[:response]),
+            label
           )
         end
       end

data/lib/rdkafka/producer/delivery_report.rb

@@ -21,6 +21,9 @@ module Rdkafka
       # @return [Integer]
       attr_reader :error
 
+      # @return [Object, nil] label set during message production or nil by default
+      attr_reader :label
+
       # We alias the `#topic_name` under `#topic` to make this consistent with `Consumer::Message`
       # where the topic name is under `#topic` method. That way we have a consistent name that
       # is present in both places
@@ -30,11 +33,12 @@ module Rdkafka
 
       private
 
-      def initialize(partition, offset, topic_name = nil, error = nil)
+      def initialize(partition, offset, topic_name = nil, error = nil, label = nil)
         @partition = partition
         @offset = offset
         @topic_name = topic_name
         @error = error
+        @label = label
       end
     end
   end

data/lib/rdkafka/producer.rb

@@ -226,11 +226,12 @@ module Rdkafka
     # @param partition_key [String, nil] Optional partition key based on which partition assignment can happen
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     # @param headers [Hash<String,String>] Optional message headers
+    # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     #
     # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
-    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil)
+    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil, label: nil)
       closed_producer_check(__method__)
 
       # Start by checking and converting the input
@@ -272,6 +273,7 @@ module Rdkafka
                       end
 
       delivery_handle = DeliveryHandle.new
+      delivery_handle.label = label
       delivery_handle[:pending] = true
       delivery_handle[:response] = -1
       delivery_handle[:partition] = -1

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.14.4"
+  VERSION = "0.14.6"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/spec/rdkafka/consumer_spec.rb

@@ -480,12 +480,18 @@ describe Rdkafka::Consumer do
 
       describe "#store_offset" do
         let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': false).consumer }
+        let(:metadata) { SecureRandom.uuid }
+        let(:group_id) { SecureRandom.uuid }
+        let(:base_config) do
+          {
+            'group.id': group_id,
+            'enable.auto.offset.store': false,
+            'enable.auto.commit': false
+          }
+        end
 
         before do
-          config = {}
-          config[:'enable.auto.offset.store'] = false
-          config[:'enable.auto.commit'] = false
-          @new_consumer = rdkafka_consumer_config(config).consumer
+          @new_consumer = rdkafka_consumer_config(base_config).consumer
           @new_consumer.subscribe("consume_test_topic")
           wait_for_assignment(@new_consumer)
         end
@@ -508,6 +514,19 @@ describe Rdkafka::Consumer do
           expect(partitions[message.partition].offset).to eq(message.offset + 1)
         end
 
+        it "should store the offset for a message with metadata" do
+          @new_consumer.store_offset(message, metadata)
+          @new_consumer.commit
+          @new_consumer.close
+
+          meta_consumer = rdkafka_consumer_config(base_config).consumer
+          meta_consumer.subscribe("consume_test_topic")
+          wait_for_assignment(meta_consumer)
+          meta_consumer.poll(1_000)
+          expect(meta_consumer.committed.to_h[message.topic][message.partition].metadata).to eq(metadata)
+          meta_consumer.close
+        end
+
         it "should raise an error with invalid input" do
           allow(message).to receive(:partition).and_return(9999)
           expect {

data/spec/rdkafka/producer_spec.rb

@@ -34,6 +34,7 @@ describe Rdkafka::Producer do
 
         producer.delivery_callback = lambda do |report|
           expect(report).not_to be_nil
+          expect(report.label).to eq "label"
           expect(report.partition).to eq 1
           expect(report.offset).to be >= 0
           expect(report.topic_name).to eq "produce_test_topic"
@@ -44,9 +45,12 @@ describe Rdkafka::Producer do
         handle = producer.produce(
           topic:   "produce_test_topic",
           payload: "payload",
-          key:     "key"
+          key:     "key",
+          label:   "label"
         )
 
+        expect(handle.label).to eq "label"
+
         # Wait for it to be delivered
         handle.wait(max_wait_timeout: 15)
 
@@ -175,11 +179,13 @@ describe Rdkafka::Producer do
     handle = producer.produce(
       topic:   "produce_test_topic",
       payload: "payload",
-      key:     "key"
+      key:     "key",
+      label:   "label"
     )
 
     # Should be pending at first
     expect(handle.pending?).to be true
+    expect(handle.label).to eq "label"
 
     # Check delivery handle and report
     report = handle.wait(max_wait_timeout: 5)
@@ -187,6 +193,7 @@ describe Rdkafka::Producer do
     expect(report).not_to be_nil
     expect(report.partition).to eq 1
     expect(report.offset).to be >= 0
+    expect(report.label).to eq "label"
 
     # Flush and close producer
     producer.flush
@@ -567,10 +574,11 @@ describe Rdkafka::Producer do
     end
 
     it "should contain the error in the response when not deliverable" do
-      handler = producer.produce(topic: 'produce_test_topic', payload: nil)
+      handler = producer.produce(topic: 'produce_test_topic', payload: nil, label: 'na')
       # Wait for the async callbacks and delivery registry to update
       sleep(2)
       expect(handler.create_result.error).to be_a(Rdkafka::RdkafkaError)
+      expect(handler.create_result.label).to eq('na')
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.14.4
+  version: 0.14.6
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-12-19 00:00:00.000000000 Z
+date: 2023-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -277,7 +277,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.