karafka-rdkafka 0.15.0 → 0.15.1.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a8d87787ad144c70fc58e6f70d50581e2ad4e1324089f3b0ebf6c27e42e000ad
-  data.tar.gz: 400929c4f6ae6d617039a716d47088d18a17e6922e93b1e699ebc1755ceeeafa
+  metadata.gz: 6c9fbe378b3d1bdf4f72d9bdb1339757bb52145c5865aa54326c0b93b113d3cd
+  data.tar.gz: f5db0b05352eac209bcf6c4bac756200bdaf822866c96bc1c0a7bd66eebb326e
 SHA512:
-  metadata.gz: 4b97d0a84d0d0e59d0482533b9e6ca93e91c5963910d6f9b3e4a8f5004fc968fc9a5a0042c65c2f32902de7a38b1126fbc31c296bb913df1e2eb876057dad5f3
-  data.tar.gz: da3ce8ebf3d1726d23f87cc28f2f7f4ede47afc1309dd098212fcab7c0d0b89bf33956282e45669b21a5251c93e89cbd73f13ae02a085b89b7695e035092c187
+  metadata.gz: 98cd5043f6a9573aeecb92123ed9c4b522a946aeea6b3d7fe71974092f1f036ffe983af27d1b67234436071e88698f45f0d15b6204189d15dfe1edb6dae34094
+  data.tar.gz: 60306bfb7d9d367ea0bb0804d801e0b34b1e8b887033355d22ecb56170319a418aed4d258ab53b406f28eb26b8f0905f1458bb99ee35b7e48ac32e99f08b95ad

data/CHANGELOG.md

@@ -1,15 +1,14 @@
 # Rdkafka Changelog
 
+## 0.15.1 (Unreleased)
+- **[Feature]** Provide ability to use topic config on a producer for custom behaviors per dispatch.
+- [Enhancement] Use topic config reference cache for messages production to prevent topic objects allocation with each message.
+- [Enhancement] Provide `Rrdkafka::Admin#describe_errors` to get errors descriptions (mensfeld)
+
 ## 0.15.0 (2024-04-26)
 - **[Feature]** Oauthbearer token refresh callback (bruce-szalwinski-he)
 - **[Feature]** Support incremental config describe + alter API (mensfeld)
 - [Enhancement] name polling Thread as `rdkafka.native_kafka#<name>` (nijikon)
-- [Enhancement] Replace time poll based wait engine with an event based to improve response times on blocking operations and wait (nijikon + mensfeld)
-- [Enhancement] Allow for usage of the second regex engine of librdkafka by setting `RDKAFKA_DISABLE_REGEX_EXT` during build (mensfeld)
-- [Enhancement] name polling Thread as `rdkafka.native_kafka#<name>` (nijikon)
-- [Change] Allow for native kafka thread operations deferring and manual start for consumer, producer and admin.
-- [Change] The `wait_timeout` argument in `AbstractHandle.wait` method is deprecated and will be removed in future versions without replacement. We don't rely on it's value anymore (nijikon)
-- [Fix] Fix bogus case/when syntax. Levels 1, 2, and 6 previously defaulted to UNKNOWN (jjowdy)
 
 ## 0.14.10 (2024-02-08)
 - [Fix] Background logger stops working after forking causing memory leaks (mensfeld).

data/lib/rdkafka/admin.rb

@@ -4,6 +4,50 @@ module Rdkafka
   class Admin
     include Helpers::OAuth
 
+    class << self
+      # Allows us to retrieve librdkafka errors with descriptions
+      # Useful for debugging and building UIs, etc.
+      #
+      # @return [Hash<Integer, Hash>] hash with errors mapped by code
+      def describe_errors
+        # Memory pointers for the array of structures and count
+        p_error_descs = FFI::MemoryPointer.new(:pointer)
+        p_count = FFI::MemoryPointer.new(:size_t)
+
+        # Call the attached function
+        Bindings.rd_kafka_get_err_descs(p_error_descs, p_count)
+
+        # Retrieve the number of items in the array
+        count = p_count.read_uint
+
+        # Get the pointer to the array of error descriptions
+        array_of_errors = FFI::Pointer.new(Bindings::NativeErrorDesc, p_error_descs.read_pointer)
+
+        errors = {}
+
+        count.times do |i|
+          # Get the pointer to each struct
+          error_ptr = array_of_errors[i]
+
+          # Create a new instance of NativeErrorDesc for each item
+          error_desc = Bindings::NativeErrorDesc.new(error_ptr)
+
+          # Read values from the struct
+          code = error_desc[:code]
+
+          name = ''
+          desc = ''
+
+          name = error_desc[:name].read_string unless error_desc[:name].null?
+          desc = error_desc[:desc].read_string unless error_desc[:desc].null?
+
+          errors[code] = { code: code, name: name, description: desc }
+        end
+
+        errors
+      end
+    end
+
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka

data/lib/rdkafka/bindings.rb

@@ -141,6 +141,11 @@ module Rdkafka
     RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
 
     # Errors
+    class NativeErrorDesc < FFI::Struct
+      layout :code, :int,
+             :name, :pointer,
+             :desc, :pointer
+    end
 
     attach_function :rd_kafka_err2name, [:int], :string
     attach_function :rd_kafka_err2str, [:int], :string
@@ -149,6 +154,7 @@ module Rdkafka
     attach_function :rd_kafka_error_txn_requires_abort, [:pointer], :int
     attach_function :rd_kafka_error_destroy, [:pointer], :void
     attach_function :rd_kafka_error_code, [:pointer], :int
+    attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
 
     # Configuration
 
@@ -175,6 +181,9 @@ module Rdkafka
     # Log queue
     attach_function :rd_kafka_set_log_queue, [:pointer, :pointer], :void
     attach_function :rd_kafka_queue_get_main, [:pointer], :pointer
+    # Per topic configs
+    attach_function :rd_kafka_topic_conf_new, [], :pointer
+    attach_function :rd_kafka_topic_conf_set, [:pointer, :string, :string, :pointer, :int], :kafka_config_response
 
     LogCallback = FFI::Function.new(
       :void, [:pointer, :int, :string, :string]

data/lib/rdkafka/producer.rb

@@ -9,7 +9,15 @@ module Rdkafka
     # Cache partitions count for 30 seconds
     PARTITIONS_COUNT_TTL = 30
 
-    private_constant :PARTITIONS_COUNT_TTL
+    # Empty hash used as a default
+    EMPTY_HASH = {}.freeze
+
+    private_constant :PARTITIONS_COUNT_TTL, :EMPTY_HASH
+
+    # Raised when there was a critical issue when invoking rd_kafka_topic_new
+    # This is a temporary solution until https://github.com/karafka/rdkafka-ruby/issues/451 is
+    # resolved and this is normalized in all the places
+    class TopicHandleCreationError < RuntimeError; end
 
     # @private
     # Returns the current delivery callback, by default this is nil.
@@ -28,6 +36,8 @@ module Rdkafka
     # @param partitioner_name [String, nil] name of the partitioner we want to use or nil to use
     #   the "consistent_random" default
     def initialize(native_kafka, partitioner_name)
+      @topics_refs_map = {}
+      @topics_configs = {}
       @native_kafka = native_kafka
       @partitioner_name = partitioner_name || "consistent_random"
 
@@ -54,6 +64,52 @@ module Rdkafka
       end
     end
 
+    # Sets alternative set of configuration details that can be set per topic
+    # @note It is not allowed to re-set the same topic config twice because of the underlying
+    #   librdkafka caching
+    # @param topic [String] The topic name
+    # @param config [Hash] config we want to use per topic basis
+    # @param config_hash [Integer] hash of the config. We expect it here instead of computing it,
+    #   because it is already computed during the retrieval attempt in the `#produce` flow.
+    def set_topic_config(topic, config, config_hash)
+      # Ensure lock on topic reference just in case
+      @native_kafka.with_inner do |inner|
+        @topics_refs_map[topic] ||= {}
+        @topics_configs[topic] ||= {}
+
+        return if @topics_configs[topic].key?(config_hash)
+
+        # If config is empty, we create an empty reference that will be used with defaults
+        rd_topic_config = if config.empty?
+                            nil
+                          else
+                            Rdkafka::Bindings.rd_kafka_topic_conf_new.tap do |topic_config|
+                              config.each do |key, value|
+                                error_buffer = FFI::MemoryPointer.new(:char, 256)
+                                result = Rdkafka::Bindings.rd_kafka_topic_conf_set(
+                                  topic_config,
+                                  key.to_s,
+                                  value.to_s,
+                                  error_buffer,
+                                  256
+                                )
+
+                                unless result == :config_ok
+                                  raise Config::ConfigError.new(error_buffer.read_string)
+                                end
+                              end
+                            end
+                          end
+
+        topic_handle = Bindings.rd_kafka_topic_new(inner, topic, rd_topic_config)
+
+        raise TopicHandleCreationError.new("Error creating topic handle for topic #{topic}") if topic_handle.null?
+
+        @topics_configs[topic][config_hash] = config
+        @topics_refs_map[topic][config_hash] = topic_handle
+      end
+    end
+
     # Starts the native Kafka polling thread and kicks off the init polling
     # @note Not needed to run unless explicit start was disabled
     def start
@@ -151,7 +207,18 @@ module Rdkafka
     def close
       return if closed?
       ObjectSpace.undefine_finalizer(self)
-      @native_kafka.close
+
+      @native_kafka.close do
+        # We need to remove the topics references objects before we destroy the producer,
+        # otherwise they would leak out
+        @topics_refs_map.each_value do |refs|
+          refs.each_value do |ref|
+            Rdkafka::Bindings.rd_kafka_topic_destroy(ref)
+          end
+        end
+      end
+
+      @topics_refs_map.clear
     end
 
     # Whether this producer has closed
@@ -244,11 +311,22 @@ module Rdkafka
     # @param timestamp [Time,Integer,nil] Optional timestamp of this message. Integer timestamp is in milliseconds since Jan 1 1970.
     # @param headers [Hash<String,String>] Optional message headers
     # @param label [Object, nil] a label that can be assigned when producing a message that will be part of the delivery handle and the delivery report
+    # @param topic_config [Hash] topic config for given message dispatch. Allows to send messages to topics with different configuration
     #
     # @return [DeliveryHandle] Delivery handle that can be used to wait for the result of producing this message
     #
     # @raise [RdkafkaError] When adding the message to rdkafka's queue failed
-    def produce(topic:, payload: nil, key: nil, partition: nil, partition_key: nil, timestamp: nil, headers: nil, label: nil)
+    def produce(
+      topic:,
+      payload: nil,
+      key: nil,
+      partition: nil,
+      partition_key: nil,
+      timestamp: nil,
+      headers: nil,
+      label: nil,
+      topic_config: EMPTY_HASH
+    )
       closed_producer_check(__method__)
 
       # Start by checking and converting the input
@@ -267,8 +345,20 @@ module Rdkafka
                    key.bytesize
                  end
 
+      topic_config_hash = topic_config.hash
+
+      # Checks if we have the rdkafka topic reference object ready. It saves us on object
+      # allocation and allows to use custom config on demand.
+      set_topic_config(topic, topic_config, topic_config_hash) unless @topics_refs_map.dig(topic, topic_config_hash)
+      topic_ref = @topics_refs_map.dig(topic, topic_config_hash)
+
       if partition_key
         partition_count = partition_count(topic)
+
+        # Check if there are no overrides for the partitioner and use the default one only when
+        # no per-topic is present.
+        partitioner_name = @topics_configs.dig(topic, topic_config_hash, :partitioner) || @partitioner_name
+
         # If the topic is not present, set to -1
         partition = Rdkafka::Bindings.partitioner(partition_key, partition_count, @partitioner_name) if partition_count.positive?
       end
@@ -298,7 +388,7 @@ module Rdkafka
       DeliveryHandle.register(delivery_handle)
 
       args = [
-        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_TOPIC, :string, topic,
+        :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_RKT, :pointer, topic_ref,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_MSGFLAGS, :int, Rdkafka::Bindings::RD_KAFKA_MSG_F_COPY,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_VALUE, :buffer_in, payload, :size_t, payload_size,
         :int, Rdkafka::Bindings::RD_KAFKA_VTYPE_KEY, :buffer_in, key, :size_t, key_size,

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.15.0"
+  VERSION = "0.15.1.rc1"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/spec/rdkafka/admin_spec.rb

@@ -31,6 +31,14 @@ describe Rdkafka::Admin do
   let(:operation)             {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
   let(:permission_type)       {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
 
+  describe '#describe_errors' do
+    let(:errors) { admin.class.describe_errors }
+
+    it { expect(errors.size).to eq(162) }
+    it { expect(errors[-184]).to eq(code: -184, description: 'Local: Queue full', name: '_QUEUE_FULL') }
+    it { expect(errors[21]).to eq(code: 21, description: 'Broker: Invalid required acks value', name: 'INVALID_REQUIRED_ACKS') }
+  end
+
   describe 'admin without auto-start' do
     let(:admin) { config.admin(native_kafka_auto_start: false) }
 

data/spec/rdkafka/producer_spec.rb

@@ -31,6 +31,48 @@ describe Rdkafka::Producer do
     it { expect(producer.name).to include('rdkafka#producer-') }
   end
 
+  describe '#produce with topic config alterations' do
+    context 'when config is not valid' do
+      it 'expect to raise error' do
+        expect do
+          producer.produce(topic: 'test', payload: '', topic_config: { 'invalid': 'invalid' })
+        end.to raise_error(Rdkafka::Config::ConfigError)
+      end
+    end
+
+    context 'when config is valid' do
+      it 'expect to raise error' do
+        expect do
+          producer.produce(topic: 'test', payload: '', topic_config: { 'acks': 1 }).wait
+        end.not_to raise_error
+      end
+
+      context 'when alteration should change behavior' do
+        # This is set incorrectly for a reason
+        # If alteration would not work, this will hang the spec suite
+        let(:producer) do
+          rdkafka_producer_config(
+            'message.timeout.ms': 1_000_000,
+            :"bootstrap.servers" => "localhost:9094",
+          ).producer
+        end
+
+        it 'expect to give up on delivery fast based on alteration config' do
+          expect do
+            producer.produce(
+              topic: 'produce_config_test',
+              payload: 'test',
+              topic_config: {
+                'compression.type': 'gzip',
+                'message.timeout.ms': 1
+              }
+            ).wait
+          end.to raise_error(Rdkafka::RdkafkaError, /msg_timed_out/)
+        end
+      end
+    end
+  end
+
   context "delivery callback" do
     context "with a proc/lambda" do
       it "should set the callback" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.15.1.rc1
 platform: ruby
 authors:
 - Thijs Cadier
@@ -36,7 +36,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2024-04-26 00:00:00.000000000 Z
+date: 2024-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi