karafka-rdkafka 0.14.2 → 0.14.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88083ab7dfa50e2dc5b29ea72d2d846f1e2db77871b86e76057b9b49cbf0440a
-  data.tar.gz: 39f2b241c4786caa4829f73270446adcd9ef744e84c583dfbefaa09b0dc669eb
+  metadata.gz: d22e33d93f77f138063f03111b25386c08e1667012b6ab4c8449fef61754ad44
+  data.tar.gz: b8cc0557cbb945e5d7b7e025198a292a7cf601ce9decef97cb5102fd25af742b
 SHA512:
-  metadata.gz: 83165b051816f6145eb89f20481ae15eb849aff39d69431c2382b00a101b58117a558e13ac5065970868a1e4d64ac829787b02a7918fc12b6f77f01e82c8f19c
-  data.tar.gz: 5546e37c51c0da66d3226805534f16e8f6211f4dc748869f41afa8abcbf0c09e367ef9279c838aee71684606eaebca2347b7f20c763a5116d2143b762eedfbc9
+  metadata.gz: cfc9af0e70030f513c4c34b00f9b94b6b3b9c828add2ac6cd742ebe2eb308a1e1a5489bf4c4280c34a98d2f0118d741608918314fa8f267b7cf98b22b3d8782d
+  data.tar.gz: 315b552a77691b62e5a30fd5eaec1bf89efd8ec5789f9f0deeadf3997eeca9dc83bb6466ee8365e40e53d2e09707d3d68a57278b3637b170ad027b5080838519

data/.github/workflows/ci.yml

@@ -22,7 +22,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.3.0-preview2'
+          - '3.3.0-rc1'
           - '3.2'
           - '3.1'
           - '3.1.0'

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Rdkafka Changelog
 
+## 0.14.4 (2023-12-19)
+- [Enhancement] Add ability to store offsets in a transaction (mensfeld)
+
+## 0.14.3 (2023-12-17)
+- [Enhancement] Replace `rd_kafka_offset_store` with `rd_kafka_offsets_store` (mensfeld)
+- [Fix] Missing ACL `RD_KAFKA_RESOURCE_BROKER` constant reference (mensfeld)
+- [Change] Rename `matching_acl_pattern_type` to `matching_acl_resource_pattern_type` to align the whole API (mensfeld)
+
 ## 0.14.2 (2023-12-11)
 - [Enhancement] Alias `topic_name` as `topic` in the delivery report (mensfeld)
 - [Fix] Fix return type on `#rd_kafka_poll` (mensfeld)

data/lib/rdkafka/admin/acl_binding_result.rb

@@ -2,36 +2,50 @@
 
 module Rdkafka
   class Admin
-
     # Extracts attributes of rd_kafka_AclBinding_t
     #
     class AclBindingResult
-      attr_reader :result_error, :error_string, :matching_acl_resource_type, :matching_acl_resource_name, :matching_acl_pattern_type, :matching_acl_principal, :matching_acl_host, :matching_acl_operation, :matching_acl_permission_type
+      attr_reader :result_error, :error_string, :matching_acl_resource_type,
+                  :matching_acl_resource_name, :matching_acl_resource_pattern_type,
+                  :matching_acl_principal, :matching_acl_host, :matching_acl_operation,
+                  :matching_acl_permission_type
+
+      # This attribute was initially released under the name that is now an alias
+      # We keep it for backwards compatibility but it was changed for the consistency
+      alias matching_acl_pattern_type matching_acl_resource_pattern_type
 
       def initialize(matching_acl)
-          rd_kafka_error_pointer        = Rdkafka::Bindings.rd_kafka_AclBinding_error(matching_acl)
-          @result_error                 = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
-          error_string                 = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
-          if error_string != FFI::Pointer::NULL
-            @error_string = error_string.read_string
-          end
-          @matching_acl_resource_type   = Rdkafka::Bindings.rd_kafka_AclBinding_restype(matching_acl)
-          matching_acl_resource_name    = Rdkafka::Bindings.rd_kafka_AclBinding_name(matching_acl)
-          if matching_acl_resource_name != FFI::Pointer::NULL
-            @matching_acl_resource_name = matching_acl_resource_name.read_string
-          end
-          @matching_acl_pattern_type    = Rdkafka::Bindings.rd_kafka_AclBinding_resource_pattern_type(matching_acl)
-          matching_acl_principal        = Rdkafka::Bindings.rd_kafka_AclBinding_principal(matching_acl)
-          if matching_acl_principal != FFI::Pointer::NULL
-            @matching_acl_principal = matching_acl_principal.read_string
-          end
-          matching_acl_host             = Rdkafka::Bindings.rd_kafka_AclBinding_host(matching_acl)
-          if matching_acl_host != FFI::Pointer::NULL
-            @matching_acl_host = matching_acl_host.read_string
-          end
-          @matching_acl_operation       = Rdkafka::Bindings.rd_kafka_AclBinding_operation(matching_acl)
-          @matching_acl_permission_type = Rdkafka::Bindings.rd_kafka_AclBinding_permission_type(matching_acl)
+        rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_AclBinding_error(matching_acl)
+        @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
+        error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
+
+        if error_string != FFI::Pointer::NULL
+          @error_string = error_string.read_string
         end
+
+        @matching_acl_resource_type = Rdkafka::Bindings.rd_kafka_AclBinding_restype(matching_acl)
+        matching_acl_resource_name = Rdkafka::Bindings.rd_kafka_AclBinding_name(matching_acl)
+
+        if matching_acl_resource_name != FFI::Pointer::NULL
+          @matching_acl_resource_name = matching_acl_resource_name.read_string
+        end
+
+        @matching_acl_resource_pattern_type = Rdkafka::Bindings.rd_kafka_AclBinding_resource_pattern_type(matching_acl)
+        matching_acl_principal = Rdkafka::Bindings.rd_kafka_AclBinding_principal(matching_acl)
+
+        if matching_acl_principal != FFI::Pointer::NULL
+          @matching_acl_principal = matching_acl_principal.read_string
+        end
+
+        matching_acl_host = Rdkafka::Bindings.rd_kafka_AclBinding_host(matching_acl)
+
+        if matching_acl_host != FFI::Pointer::NULL
+          @matching_acl_host = matching_acl_host.read_string
+        end
+
+        @matching_acl_operation = Rdkafka::Bindings.rd_kafka_AclBinding_operation(matching_acl)
+        @matching_acl_permission_type = Rdkafka::Bindings.rd_kafka_AclBinding_permission_type(matching_acl)
       end
+    end
   end
 end

data/lib/rdkafka/bindings.rb

@@ -199,12 +199,15 @@ module Rdkafka
     attach_function :rd_kafka_poll_set_consumer, [:pointer], :void, blocking: true
     attach_function :rd_kafka_consumer_poll, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_consumer_close, [:pointer], :void, blocking: true
-    attach_function :rd_kafka_offset_store, [:pointer, :int32, :int64], :int, blocking: true
+    attach_function :rd_kafka_offsets_store, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_pause_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_resume_partitions, [:pointer, :pointer], :int, blocking: true
     attach_function :rd_kafka_seek, [:pointer, :int32, :int64, :int], :int, blocking: true
     attach_function :rd_kafka_offsets_for_times, [:pointer, :pointer, :int], :int, blocking: true
     attach_function :rd_kafka_position, [:pointer, :pointer], :int, blocking: true
+    # those two are used for eos support
+    attach_function :rd_kafka_consumer_group_metadata, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_consumer_group_metadata_destroy, [:pointer], :void, blocking: true
 
     # Headers
     attach_function :rd_kafka_header_get_all, [:pointer, :size_t, :pointer, :pointer, SizePtr], :int
@@ -276,6 +279,7 @@ module Rdkafka
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
     attach_function :rd_kafka_init_transactions, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_send_offsets_to_transaction, [:pointer, :pointer, :pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_begin_transaction, [:pointer], :pointer, blocking: true
     attach_function :rd_kafka_abort_transaction, [:pointer, :int], :pointer, blocking: true
     attach_function :rd_kafka_commit_transaction, [:pointer, :int], :pointer, blocking: true
@@ -412,6 +416,7 @@ module Rdkafka
     RD_KAFKA_RESOURCE_ANY   = 1
     RD_KAFKA_RESOURCE_TOPIC = 2
     RD_KAFKA_RESOURCE_GROUP = 3
+    RD_KAFKA_RESOURCE_BROKER = 4
 
     # rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
 

data/lib/rdkafka/consumer.rb

@@ -388,28 +388,26 @@ module Rdkafka
     def store_offset(message)
       closed_consumer_check(__method__)
 
-      # rd_kafka_offset_store is one of the few calls that does not support
-      # a string as the topic, so create a native topic for it.
-      native_topic = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_topic_new(
+      list = TopicPartitionList.new
+      list.add_topic_and_partitions_with_offsets(
+        message.topic,
+        message.partition => message.offset + 1
+      )
+
+      tpl = list.to_native_tpl
+
+      response = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_offsets_store(
           inner,
-          message.topic,
-          nil
+          tpl
         )
       end
-      response = Rdkafka::Bindings.rd_kafka_offset_store(
-        native_topic,
-        message.partition,
-        message.offset
-      )
 
       Rdkafka::RdkafkaError.validate!(response)
 
       nil
     ensure
-      if native_topic && !native_topic.null?
-        Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
-      end
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
     end
 
     # Seek to a particular message. The next poll on the topic/partition will return the
@@ -673,6 +671,22 @@ module Rdkafka
       end
     end
 
+    # Returns pointer to the consumer group metadata. It is used only in the context of
+    # exactly-once-semantics in transactions, this is why it is never remapped to Ruby
+    #
+    # This API is **not** usable by itself from Ruby
+    #
+    # @note This pointer **needs** to be removed with `#rd_kafka_consumer_group_metadata_destroy`
+    #
+    # @private
+    def consumer_group_metadata_pointer
+      closed_consumer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        Bindings.rd_kafka_consumer_group_metadata(inner)
+      end
+    end
+
     private
 
     def closed_consumer_check(method)

data/lib/rdkafka/producer.rb

@@ -81,7 +81,7 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_begin_transaction(inner)
 
-        Rdkafka::RdkafkaError.validate!(response_ptr)
+        Rdkafka::RdkafkaError.validate!(response_ptr) || true
       end
     end
 
@@ -91,7 +91,7 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_commit_transaction(inner, timeout_ms)
 
-        Rdkafka::RdkafkaError.validate!(response_ptr)
+        Rdkafka::RdkafkaError.validate!(response_ptr) || true
       end
     end
 
@@ -100,8 +100,35 @@ module Rdkafka
 
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_abort_transaction(inner, timeout_ms)
+        Rdkafka::RdkafkaError.validate!(response_ptr) || true
+      end
+    end
+
+    # Sends provided offsets of a consumer to the transaction for collective commit
+    #
+    # @param consumer [Consumer] consumer that owns the given tpls
+    # @param tpl [Consumer::TopicPartitionList]
+    # @param timeout_ms [Integer] offsets send timeout
+    # @note Use **only** in the context of an active transaction
+    def send_offsets_to_transaction(consumer, tpl, timeout_ms = 5_000)
+      closed_producer_check(__method__)
+
+      return if tpl.empty?
+
+      cgmetadata = consumer.consumer_group_metadata_pointer
+      native_tpl = tpl.to_native_tpl
+
+      @native_kafka.with_inner do |inner|
+        response_ptr = Bindings.rd_kafka_send_offsets_to_transaction(inner, native_tpl, cgmetadata, timeout_ms)
+
         Rdkafka::RdkafkaError.validate!(response_ptr)
       end
+    ensure
+      if cgmetadata && !cgmetadata.null?
+        Bindings.rd_kafka_consumer_group_metadata_destroy(cgmetadata)
+      end
+
+      Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(native_tpl) unless native_tpl.nil?
     end
 
     # Close this producer and wait for the internal poll queue to empty.

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.14.2"
+  VERSION = "0.14.4"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/spec/rdkafka/admin/delete_acl_report_spec.rb

@@ -50,6 +50,7 @@ describe Rdkafka::Admin::DeleteAclReport do
   end
 
   it "should get deleted acl resource pattern type as Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL" do
+    expect(subject.deleted_acls[0].matching_acl_resource_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
     expect(subject.deleted_acls[0].matching_acl_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
   end
 

data/spec/rdkafka/admin/describe_acl_report_spec.rb

@@ -51,6 +51,7 @@ describe Rdkafka::Admin::DescribeAclReport do
   end
 
   it "should get matching acl resource pattern type as Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL" do
+    expect(subject.acls[0].matching_acl_resource_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
     expect(subject.acls[0].matching_acl_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
   end
 

data/spec/rdkafka/consumer_spec.rb

@@ -360,7 +360,6 @@ describe Rdkafka::Consumer do
     end
   end
 
-
   describe "#position, #commit, #committed and #store_offset" do
     # Make sure there are messages to work with
     let!(:report) do
@@ -480,6 +479,8 @@ describe Rdkafka::Consumer do
       end
 
       describe "#store_offset" do
+        let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': false).consumer }
+
         before do
           config = {}
           config[:'enable.auto.offset.store'] = false
@@ -542,6 +543,14 @@ describe Rdkafka::Consumer do
             }.to raise_error(Rdkafka::RdkafkaError)
           end
         end
+
+        context "when trying to use with enable.auto.offset.store set to true" do
+          let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': true).consumer }
+
+          it "expect to raise invalid configuration error" do
+            expect { consumer.store_offset(message) }.to raise_error(Rdkafka::RdkafkaError, /invalid_arg/)
+          end
+        end
       end
     end
   end
@@ -1132,6 +1141,14 @@ describe Rdkafka::Consumer do
     end
   end
 
+  describe '#consumer_group_metadata_pointer' do
+    it 'expect to yield pointer' do
+      consumer.consumer_group_metadata_pointer do |pointer|
+        expect(pointer).to be_a(FFI::Pointer)
+      end
+    end
+  end
+
   describe "a rebalance listener" do
     let(:consumer) do
       config = rdkafka_consumer_config

data/spec/rdkafka/producer_spec.rb

@@ -883,5 +883,30 @@ describe Rdkafka::Producer do
         expect { producer2.commit_transaction }.not_to raise_error
       end
     end
+
+    context 'when having a consumer with tpls for exactly once semantics' do
+      let(:tpl) do
+        producer.produce(topic: 'consume_test_topic', payload: 'data1', partition: 0).wait
+        result = producer.produce(topic: 'consume_test_topic', payload: 'data1', partition: 0).wait
+
+        Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+          list.add_topic_and_partitions_with_offsets("consume_test_topic", 0 => result.offset + 1)
+        end
+      end
+
+      before do
+        consumer.subscribe("consume_test_topic")
+        wait_for_assignment(consumer)
+        producer.init_transactions
+        producer.begin_transaction
+      end
+
+      after { consumer.unsubscribe }
+
+      it 'expect to store offsets and not crash' do
+        producer.send_offsets_to_transaction(consumer, tpl)
+        producer.commit_transaction
+      end
+    end
   end
 end

data/spec/spec_helper.rb

@@ -107,10 +107,6 @@ def wait_for_unassignment(consumer)
   end
 end
 
-def objects_of_type_count(type)
-  ObjectSpace.each_object(type).count
-end
-
 def notify_listener(listener, &block)
   # 1. subscribe and poll
   consumer.subscribe("consume_test_topic")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.14.2
+  version: 0.14.4
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-12-11 00:00:00.000000000 Z
+date: 2023-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi