karafka-rdkafka 0.14.5 → 0.14.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ed56a133dd27e17055cf63b7e68945d9cb97b42690afa6c7eb230f56273754c
-  data.tar.gz: bcb63a1c2c77d599ab403bfdca1df0badd28f1c1cc72eed16169e6d4044fe778
+  metadata.gz: c7376c9030672ba0205c3973317a626b7ececd3b100c312e5ef310d08de397cc
+  data.tar.gz: b843498f05cdadfd034c9c20338fd698b37df792c0bd54919c19632ae6ac8d29
 SHA512:
-  metadata.gz: '07185f868e86c3aaa917f8be42a14047cfb9dd9901001351f416f14338bf1dcc942b4d4649749832efed5c10cb3c7e9ea3543706fd7e11bb79fc332206569898'
-  data.tar.gz: c7bafd507d2b86c9ae61daab5ed32b142ec6d9e7dbd6d4cf26a704d432b257c00e84fb8872daab978c685941ab71e3201d7fe05635fa0fc8526eac016faf48d9
+  metadata.gz: ca8944dae92ea927a09748524a04df8079e9a1bef1c84034cdf2d5c372ca122e652a1b728c5a497ac26cefa3ae6b2eeb52da3df988332f06ef1a153ce8773028
+  data.tar.gz: 5772911202fae3ccd51d54300a0dd74159188d17395e4ee6895132abcc30a916a9adcf9643f1626dc9884f8a17c88e6d3d546bb0191218a4e263a4cbb37b3009

data/.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.3.0-rc1'
+          - '3.3'
           - '3.2'
           - '3.1'
           - '3.1.0'
@@ -31,7 +31,7 @@ jobs:
           - '2.7'
           - '2.7.0'
         include:
-          - ruby: '3.2'
+          - ruby: '3.3'
             coverage: 'true'
     steps:
       - uses: actions/checkout@v4

data/.ruby-version

@@ -1 +1 @@
-3.2.2
+3.3.0

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Rdkafka Changelog
 
+## 0.14.6 (2023-12-29)
+- **[Feature]** Support storing metadata alongside offsets via `rd_kafka_offsets_store` in `#store_offset` (mensfeld)
+- [Enhancement] Increase the `#committed` default timeout from 1_200ms to 2000ms. This will compensate for network glitches and remote clusters operations and will align with metadata query timeout.
+
 ## 0.14.5 (2023-12-20)
 - [Enhancement] Provide `label` producer handler and report reference for improved traceability.
 

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.2
+    image: confluentinc/cp-kafka:7.5.3
 
     ports:
       - 9092:9092

data/lib/rdkafka/bindings.rb

@@ -92,7 +92,7 @@ module Rdkafka
     end
 
     attach_function :rd_kafka_topic_partition_list_new, [:int32], :pointer
-    attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :void
+    attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :pointer
     attach_function :rd_kafka_topic_partition_list_set_offset, [:pointer, :string, :int32, :int64], :void
     attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
     attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer

data/lib/rdkafka/consumer/partition.rb

@@ -16,11 +16,16 @@ module Rdkafka
       # @return [Integer]
       attr_reader :err
 
+      # Partition metadata in the context of a consumer
+      # @return [String, nil]
+      attr_reader :metadata
+
       # @private
-      def initialize(partition, offset, err = 0)
+      def initialize(partition, offset, err = 0, metadata = nil)
         @partition = partition
         @offset = offset
         @err = err
+        @metadata = metadata
       end
 
       # Human readable representation of this partition.
@@ -29,6 +34,7 @@ module Rdkafka
         message = "<Partition #{partition}"
         message += " offset=#{offset}" if offset
         message += " err=#{err}" if err != 0
+        message += " metadata=#{metadata}" if metadata != nil
         message += ">"
         message
       end

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -66,10 +66,14 @@ module Rdkafka
       #
       # @param topic [String] The topic's name
       # @param partitions_with_offsets [Hash<Integer, Integer>] The topic's partitions and offsets
+      # @param partitions_with_offsets [Array<Consumer::Partition>] The topic's partitions with offsets
+      #   and metadata (if any)
       #
       # @return [nil]
       def add_topic_and_partitions_with_offsets(topic, partitions_with_offsets)
-        @data[topic.to_s] = partitions_with_offsets.map { |p, o| Partition.new(p, o) }
+        @data[topic.to_s] = partitions_with_offsets.map do |p, o|
+          p.is_a?(Partition) ? p : Partition.new(p, o)
+        end
       end
 
       # Return a `Hash` with the topics as keys and and an array of partition information as the value if present.
@@ -114,7 +118,13 @@ module Rdkafka
                      else
                        elem[:offset]
                      end
-            partition = Partition.new(elem[:partition], offset, elem[:err])
+
+            partition = Partition.new(
+              elem[:partition],
+              offset,
+              elem[:err],
+              elem[:metadata].null? ? nil : elem[:metadata].read_string(elem[:metadata_size])
+            )
             partitions.push(partition)
             data[elem[:topic]] = partitions
           end
@@ -136,12 +146,22 @@ module Rdkafka
         @data.each do |topic, partitions|
           if partitions
             partitions.each do |p|
-              Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
+              ref = Rdkafka::Bindings.rd_kafka_topic_partition_list_add(
                 tpl,
                 topic,
                 p.partition
               )
 
+              if p.metadata
+                part = Rdkafka::Bindings::TopicPartition.new(ref)
+                str_ptr = FFI::MemoryPointer.from_string(p.metadata)
+                # released here:
+                # https://github.com/confluentinc/librdkafka/blob/e03d3bb91ed92a38f38d9806b8d8deffe78a1de5/src/rdkafka_partition.c#L2682C18-L2682C18
+                str_ptr.autorelease = false
+                part[:metadata] = str_ptr
+                part[:metadata_size] = p.metadata.bytesize
+              end
+
               if p.offset
                 offset = p.offset.is_a?(Time) ? p.offset.to_f * 1_000 : p.offset
 

data/lib/rdkafka/consumer.rb

@@ -243,7 +243,7 @@ module Rdkafka
     # @param timeout_ms [Integer] The timeout for fetching this information.
     # @return [TopicPartitionList]
     # @raise [RdkafkaError] When getting the committed positions fails.
-    def committed(list=nil, timeout_ms=1200)
+    def committed(list=nil, timeout_ms=2_000)
       closed_consumer_check(__method__)
 
       if list.nil?
@@ -383,16 +383,34 @@ module Rdkafka
     # When using this `enable.auto.offset.store` should be set to `false` in the config.
     #
     # @param message [Rdkafka::Consumer::Message] The message which offset will be stored
+    # @param metadata [String, nil] commit metadata string or nil if none
     # @return [nil]
     # @raise [RdkafkaError] When storing the offset fails
-    def store_offset(message)
+    def store_offset(message, metadata = nil)
       closed_consumer_check(__method__)
 
       list = TopicPartitionList.new
-      list.add_topic_and_partitions_with_offsets(
-        message.topic,
-        message.partition => message.offset + 1
-      )
+
+      # For metadata aware commits we build the partition reference directly to save on
+      # objects allocations
+      if metadata
+        list.add_topic_and_partitions_with_offsets(
+          message.topic,
+          [
+            Consumer::Partition.new(
+              message.partition,
+              message.offset + 1,
+              0,
+              metadata
+            )
+          ]
+        )
+      else
+        list.add_topic_and_partitions_with_offsets(
+          message.topic,
+          message.partition => message.offset + 1
+        )
+      end
 
       tpl = list.to_native_tpl
 

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.14.5"
+  VERSION = "0.14.6"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/spec/rdkafka/consumer_spec.rb

@@ -480,12 +480,18 @@ describe Rdkafka::Consumer do
 
       describe "#store_offset" do
         let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': false).consumer }
+        let(:metadata) { SecureRandom.uuid }
+        let(:group_id) { SecureRandom.uuid }
+        let(:base_config) do
+          {
+            'group.id': group_id,
+            'enable.auto.offset.store': false,
+            'enable.auto.commit': false
+          }
+        end
 
         before do
-          config = {}
-          config[:'enable.auto.offset.store'] = false
-          config[:'enable.auto.commit'] = false
-          @new_consumer = rdkafka_consumer_config(config).consumer
+          @new_consumer = rdkafka_consumer_config(base_config).consumer
           @new_consumer.subscribe("consume_test_topic")
           wait_for_assignment(@new_consumer)
         end
@@ -508,6 +514,19 @@ describe Rdkafka::Consumer do
           expect(partitions[message.partition].offset).to eq(message.offset + 1)
         end
 
+        it "should store the offset for a message with metadata" do
+          @new_consumer.store_offset(message, metadata)
+          @new_consumer.commit
+          @new_consumer.close
+
+          meta_consumer = rdkafka_consumer_config(base_config).consumer
+          meta_consumer.subscribe("consume_test_topic")
+          wait_for_assignment(meta_consumer)
+          meta_consumer.poll(1_000)
+          expect(meta_consumer.committed.to_h[message.topic][message.partition].metadata).to eq(metadata)
+          meta_consumer.close
+        end
+
         it "should raise an error with invalid input" do
           allow(message).to receive(:partition).and_return(9999)
           expect {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.14.5
+  version: 0.14.6
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-12-20 00:00:00.000000000 Z
+date: 2023-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -277,7 +277,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.