karafka-rdkafka 0.23.0.beta2-arm64-darwin → 0.23.1.rc1-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f502d8686bf1b5327ce301418e6fe207318cda982e3b6024f96e4b67564994a
-  data.tar.gz: 257800302b2ea158b27be09ff202998b7938947451df24c4a354bd7e529472eb
+  metadata.gz: 6fa37f05662fa3ac3af06c2efac7b3253a0a0216a6dea7cf67d3541c6b785e2b
+  data.tar.gz: 4e525c0932641e87bad94d96e370d2530a5aba1ff49bf33faf3b76b87aca3653
 SHA512:
-  metadata.gz: 3583ac8e7cdf5cd871a1152c34d12395ccd9e4b405fb994f0c9180b69fc9e2a02ba6792bfcf8d812f5ec4535b5a013f4a390a8e33ba63f91fd6dc1abe302f86c
-  data.tar.gz: f5380cefddaad754e6560ca27ef26cde84fafeecda1fe89c9cfb8ea532e6576435068cfa9043b501afb71de948a6b694851b2945156d357832f96db71bab2bf7
+  metadata.gz: 811cd4e18589fe4836f45c8c707dc5065905c5dd24d7844ecbcb425ffda3ab6f1ab5e4a4ac7578b7420a792224924692a5ca01b370ad7bff7db1ade8e5e697f2
+  data.tar.gz: 4068d9eca642fb548e8e8e85e4839210bc8ce4a5616dbfd346b514c78092e7aab10928cbec103871ee64f2c66ec2bf18f129287313819ac0bb0bfcc979b861f1

data/CHANGELOG.md

@@ -1,6 +1,19 @@
 # Rdkafka Changelog
 
-## 0.23.0 (Unreleased)
+## 0.23.1 (Unreleased)
+- **[Feature]** Add integrated fatal error handling in `RdkafkaError.validate!` - automatically detects and handles fatal errors (-150) with single entrypoint API.
+- [Enhancement] Add optional `client_ptr` parameter to `validate!` for automatic fatal error remapping to actual underlying error codes.
+- [Enhancement] Update all Producer and Consumer `validate!` calls to provide `client_ptr` for comprehensive fatal error handling.
+- [Enhancement] Add `rd_kafka_fatal_error()` FFI binding to retrieve actual fatal error details.
+- [Enhancement] Add `rd_kafka_test_fatal_error()` FFI binding for testing fatal error scenarios.
+- [Enhancement] Add `RdkafkaError.build_fatal` class method for centralized fatal error construction.
+- [Enhancement] Add comprehensive tests for fatal error handling including unit tests and integration tests.
+- [Enhancement] Add `RD_KAFKA_PARTITION_UA` constant for unassigned partition (-1).
+- [Enhancement] Replace magic numbers with named constants: use `RD_KAFKA_RESP_ERR_NO_ERROR` instead of `0` for error code checks (18 instances) and `RD_KAFKA_PARTITION_UA` instead of `-1` for partition values (9 instances) across the codebase for better code clarity and maintainability.
+- [Enhancement] Add `Rdkafka::Testing` module for testing fatal error scenarios on both producers and consumers.
+- [Deprecated] `RdkafkaError.validate_fatal!` - use `validate!` with `client_ptr` parameter instead.
+
+## 0.23.0 (2025-11-01)
 - [Enhancement] Bump librdkafka to 2.12.1.
 - [Enhancement] Force lock FFI to 1.17.1 or higher to include critical bug fixes around GCC, write barriers, and thread restarts for forks.
 - [Fix] Fix for Core dump when providing extensions to oauthbearer_set_token (dssjoblom)

data/README.md

@@ -63,7 +63,7 @@ Contributions should generally be made to the upstream [rdkafka-ruby repository]
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
-| 0.23.x (Unreleased) | 2.12.1 (2025-10-16)  | yes |
+| 0.23.x (2025-11-01) | 2.12.1 (2025-10-16)  | yes |
 | 0.22.x (2025-09-26) | 2.11.1 (2025-08-18)  | yes |
 | 0.21.x (2025-08-18) | 2.11.0 (2025-07-03)  | yes |
 | 0.20.x (2025-07-17) | 2.8.0  (2025-01-07)  | yes |

data/lib/rdkafka/abstract_handle.rb

@@ -77,7 +77,7 @@ module Rdkafka
                 "Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds"
               )
             end
-          elsif self[:response] != 0 && raise_response_error
+          elsif self[:response] != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR && raise_response_error
             raise_error
           else
             return create_result

data/lib/rdkafka/admin.rb

@@ -327,7 +327,7 @@ module Rdkafka
         # Create and register the handle we will return to the caller
         create_partitions_handle = CreatePartitionsHandle.new
         create_partitions_handle[:pending] = true
-        create_partitions_handle[:response] = -1
+        create_partitions_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
         CreatePartitionsHandle.register(create_partitions_handle)
         admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS)
         Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_partitions_handle.to_ptr)

data/lib/rdkafka/bindings.rb

@@ -28,8 +28,17 @@ module Rdkafka
     RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS = -174
     RD_KAFKA_RESP_ERR__STATE = -172
     RD_KAFKA_RESP_ERR__NOENT = -156
+    RD_KAFKA_RESP_ERR__FATAL = -150
     RD_KAFKA_RESP_ERR_NO_ERROR = 0
 
+    # Buffer size for fatal error strings, matches librdkafka expectations
+    FATAL_ERROR_BUFFER_SIZE = 256
+
+    # Unassigned partition
+    RD_KAFKA_PARTITION_UA = -1
+    # String representation of unassigned partition (used in stats hash keys)
+    RD_KAFKA_PARTITION_UA_STR = '-1'.freeze
+
     RD_KAFKA_OFFSET_END       = -1
     RD_KAFKA_OFFSET_BEGINNING = -2
     RD_KAFKA_OFFSET_STORED    = -1000
@@ -161,6 +170,8 @@ module Rdkafka
     attach_function :rd_kafka_error_txn_requires_abort, [:pointer], :int
     attach_function :rd_kafka_error_destroy, [:pointer], :void
     attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
+    attach_function :rd_kafka_fatal_error, [:pointer, :pointer, :int], :int
+    attach_function :rd_kafka_test_fatal_error, [:pointer, :int, :string], :int
 
     # Configuration
 
@@ -234,7 +245,7 @@ module Rdkafka
         # Since this cache is shared, having few consumers and/or producers in one process will
         # automatically improve the querying times even with low refresh times.
         (stats['topics'] || EMPTY_HASH).each do |topic_name, details|
-          partitions_count = details['partitions'].keys.reject { |k| k == '-1' }.size
+          partitions_count = details['partitions'].keys.reject { |k| k == RD_KAFKA_PARTITION_UA_STR }.size
 
           next unless partitions_count.positive?
 
@@ -248,11 +259,48 @@ module Rdkafka
       0
     end
 
+    # Retrieves fatal error details from a kafka client handle.
+    # This is a helper method to extract fatal error information consistently
+    # across different parts of the codebase (callbacks, testing utilities, etc.).
+    #
+    # @param client_ptr [FFI::Pointer] Native kafka client pointer
+    # @return [Hash, nil] Hash with :error_code and :error_string if fatal error occurred, nil otherwise
+    #
+    # @example
+    #   details = Rdkafka::Bindings.extract_fatal_error(client_ptr)
+    #   if details
+    #     puts "Fatal error #{details[:error_code]}: #{details[:error_string]}"
+    #   end
+    def self.extract_fatal_error(client_ptr)
+      error_buffer = FFI::MemoryPointer.new(:char, FATAL_ERROR_BUFFER_SIZE)
+
+      error_code = rd_kafka_fatal_error(client_ptr, error_buffer, FATAL_ERROR_BUFFER_SIZE)
+
+      return nil if error_code == RD_KAFKA_RESP_ERR_NO_ERROR
+
+      {
+        error_code: error_code,
+        error_string: error_buffer.read_string
+      }
+    end
+
     ErrorCallback = FFI::Function.new(
       :void, [:pointer, :int, :string, :pointer]
-    ) do |_client_prr, err_code, reason, _opaque|
+    ) do |client_ptr, err_code, reason, _opaque|
       if Rdkafka::Config.error_callback
-        error = Rdkafka::RdkafkaError.build(err_code, broker_message: reason)
+        # Handle fatal errors according to librdkafka documentation:
+        # When ERR__FATAL is received, we must call rd_kafka_fatal_error()
+        # to get the actual underlying fatal error code and description.
+        if err_code == RD_KAFKA_RESP_ERR__FATAL
+          error = Rdkafka::RdkafkaError.build_fatal(
+            client_ptr,
+            fallback_error_code: err_code,
+            fallback_message: reason
+          )
+        else
+          error = Rdkafka::RdkafkaError.build(err_code, broker_message: reason)
+        end
+
         error.set_backtrace(caller)
         Rdkafka::Config.error_callback.call(error)
       end
@@ -404,7 +452,7 @@ module Rdkafka
 
     def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
       # Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
-      return -1 unless partition_count&.nonzero?
+      return RD_KAFKA_PARTITION_UA unless partition_count&.nonzero?
 
       str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
       method_name = PARTITIONERS.fetch(partitioner) do

data/lib/rdkafka/callbacks.rb

@@ -76,7 +76,7 @@ module Rdkafka
         rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_error(acl_result_pointer)
         @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
         @error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
-        if @result_error == 0
+        if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
            # Get the number of matching acls
            pointer_to_size_t = FFI::MemoryPointer.new(:int32)
            @matching_acls = Rdkafka::Bindings.rd_kafka_DeleteAcls_result_response_matching_acls(acl_result_pointer, pointer_to_size_t)
@@ -102,7 +102,7 @@ module Rdkafka
         @matching_acls=[]
         @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
         @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
-        if @result_error == 0
+        if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           acl_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeAcls_result(event_ptr)
           # Get the number of matching acls
           pointer_to_size_t = FFI::MemoryPointer.new(:int32)
@@ -120,7 +120,7 @@ module Rdkafka
         @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
         @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
 
-        if @result_error == 0
+        if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
           # Get the number of matching acls
           pointer_to_size_t = FFI::MemoryPointer.new(:int32)
@@ -138,7 +138,7 @@ module Rdkafka
         @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
         @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
 
-        if @result_error == 0
+        if @result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
           # Get the number of matching acls
           pointer_to_size_t = FFI::MemoryPointer.new(:int32)
@@ -202,7 +202,7 @@ module Rdkafka
           describe_configs_handle[:response_string] = describe_configs.error_string
           describe_configs_handle[:pending] = false
 
-          if describe_configs.result_error == 0
+          if describe_configs.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             describe_configs_handle[:config_entries] = describe_configs.results
             describe_configs_handle[:entry_count] = describe_configs.results_count
           end
@@ -220,7 +220,7 @@ module Rdkafka
           incremental_alter_handle[:response_string] = incremental_alter.error_string
           incremental_alter_handle[:pending] = false
 
-          if incremental_alter.result_error == 0
+          if incremental_alter.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             incremental_alter_handle[:config_entries] = incremental_alter.results
             incremental_alter_handle[:entry_count] = incremental_alter.results_count
           end
@@ -313,7 +313,7 @@ module Rdkafka
           delete_acl_handle[:response] = delete_acl_results[0].result_error
           delete_acl_handle[:response_string] = delete_acl_results[0].error_string
 
-          if delete_acl_results[0].result_error == 0
+          if delete_acl_results[0].result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
             delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
           end
@@ -330,7 +330,7 @@ module Rdkafka
           describe_acl_handle[:response] = describe_acl.result_error
           describe_acl_handle[:response_string] = describe_acl.error_string
 
-          if describe_acl.result_error == 0
+          if describe_acl.result_error == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
             describe_acl_handle[:acls] = describe_acl.matching_acls
             describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
           end

data/lib/rdkafka/consumer/partition.rb

@@ -33,7 +33,7 @@ module Rdkafka
       def to_s
         message = "<Partition #{partition}"
         message += " offset=#{offset}" if offset
-        message += " err=#{err}" if err != 0
+        message += " err=#{err}" if err != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         message += " metadata=#{metadata}" if metadata != nil
         message += ">"
         message

data/lib/rdkafka/consumer/topic_partition_list.rb

@@ -109,7 +109,7 @@ module Rdkafka
         native_tpl[:cnt].times do |i|
           ptr = native_tpl[:elems] + (i * Rdkafka::Bindings::TopicPartition.size)
           elem = Rdkafka::Bindings::TopicPartition.new(ptr)
-          if elem[:partition] == -1
+          if elem[:partition] == Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
             data[elem[:topic]] = nil
           else
             partitions = data[elem[:topic]] || []

data/lib/rdkafka/consumer.rb

@@ -67,15 +67,14 @@ module Rdkafka
       tpl = Rdkafka::Bindings.rd_kafka_topic_partition_list_new(topics.length)
 
       topics.each do |topic|
-        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, -1)
+        Rdkafka::Bindings.rd_kafka_topic_partition_list_add(tpl, topic, Rdkafka::Bindings::RD_KAFKA_PARTITION_UA)
       end
 
       # Subscribe to topic partition list and check this was successful
-      response = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
+      @native_kafka.with_inner do |inner|
+        response = Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
+        Rdkafka::RdkafkaError.validate!(response, "Error subscribing to '#{topics.join(', ')}'", client_ptr: inner)
       end
-
-      Rdkafka::RdkafkaError.validate!(response, "Error subscribing to '#{topics.join(', ')}'")
     ensure
       Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) unless tpl.nil?
     end
@@ -87,12 +86,11 @@ module Rdkafka
     def unsubscribe
       closed_consumer_check(__method__)
 
-      response = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
+      @native_kafka.with_inner do |inner|
+        response = Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
+        Rdkafka::RdkafkaError.validate!(response, client_ptr: inner)
       end
 
-      Rdkafka::RdkafkaError.validate!(response)
-
       nil
     end
 
@@ -115,7 +113,7 @@ module Rdkafka
           Rdkafka::Bindings.rd_kafka_pause_partitions(inner, tpl)
         end
 
-        if response != 0
+        if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           list = TopicPartitionList.from_native_tpl(tpl)
           raise Rdkafka::RdkafkaTopicPartitionListError.new(response, list, "Error pausing '#{list.to_h}'")
         end
@@ -139,12 +137,11 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = @native_kafka.with_inner do |inner|
-          Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
+        @native_kafka.with_inner do |inner|
+          response = Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
+          Rdkafka::RdkafkaError.validate!(response, "Error resume '#{list.to_h}'", client_ptr: inner)
         end
 
-        Rdkafka::RdkafkaError.validate!(response, "Error resume '#{list.to_h}'")
-
         nil
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
@@ -159,12 +156,11 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
+      @native_kafka.with_inner do |inner|
+        response = Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
+        Rdkafka::RdkafkaError.validate!(response, client_ptr: inner)
       end
 
-      Rdkafka::RdkafkaError.validate!(response)
-
       native = ptr.read_pointer
 
       begin
@@ -188,11 +184,10 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = @native_kafka.with_inner do |inner|
-          Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
+        @native_kafka.with_inner do |inner|
+          response = Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
+          Rdkafka::RdkafkaError.validate!(response, "Error assigning '#{list.to_h}'", client_ptr: inner)
         end
-
-        Rdkafka::RdkafkaError.validate!(response, "Error assigning '#{list.to_h}'")
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
@@ -206,12 +201,11 @@ module Rdkafka
       closed_consumer_check(__method__)
 
       ptr = FFI::MemoryPointer.new(:pointer)
-      response = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
+      @native_kafka.with_inner do |inner|
+        response = Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
+        Rdkafka::RdkafkaError.validate!(response, client_ptr: inner)
       end
 
-      Rdkafka::RdkafkaError.validate!(response)
-
       tpl = ptr.read_pointer
 
       if !tpl.null?
@@ -255,12 +249,11 @@ module Rdkafka
       tpl = list.to_native_tpl
 
       begin
-        response = @native_kafka.with_inner do |inner|
-          Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
+        @native_kafka.with_inner do |inner|
+          response = Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
+          Rdkafka::RdkafkaError.validate!(response, client_ptr: inner)
         end
 
-        Rdkafka::RdkafkaError.validate!(response)
-
         TopicPartitionList.from_native_tpl(tpl)
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
@@ -284,12 +277,11 @@ module Rdkafka
 
       tpl = list.to_native_tpl
 
-      response = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_position(inner, tpl)
+      @native_kafka.with_inner do |inner|
+        response = Rdkafka::Bindings.rd_kafka_position(inner, tpl)
+        Rdkafka::RdkafkaError.validate!(response, client_ptr: inner)
       end
 
-      Rdkafka::RdkafkaError.validate!(response)
-
       TopicPartitionList.from_native_tpl(tpl)
     end
 
@@ -306,8 +298,8 @@ module Rdkafka
       low = FFI::MemoryPointer.new(:int64, 1)
       high = FFI::MemoryPointer.new(:int64, 1)
 
-      response = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
+      @native_kafka.with_inner do |inner|
+        response = Rdkafka::Bindings.rd_kafka_query_watermark_offsets(
           inner,
           topic,
           partition,
@@ -315,10 +307,9 @@ module Rdkafka
           high,
           timeout_ms,
         )
+        Rdkafka::RdkafkaError.validate!(response, "Error querying watermark offsets for partition #{partition} of #{topic}", client_ptr: inner)
       end
 
-      Rdkafka::RdkafkaError.validate!(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
-
       return low.read_array_of_int64(1).first, high.read_array_of_int64(1).first
     ensure
       low.free   unless low.nil?
@@ -414,15 +405,14 @@ module Rdkafka
 
       tpl = list.to_native_tpl
 
-      response = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_offsets_store(
+      @native_kafka.with_inner do |inner|
+        response = Rdkafka::Bindings.rd_kafka_offsets_store(
           inner,
           tpl
         )
+        Rdkafka::RdkafkaError.validate!(response, client_ptr: inner)
       end
 
-      Rdkafka::RdkafkaError.validate!(response)
-
       nil
     ensure
       Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
@@ -459,15 +449,19 @@ module Rdkafka
           nil
         )
       end
+
       response = Rdkafka::Bindings.rd_kafka_seek(
         native_topic,
         partition,
         offset,
         0 # timeout
       )
-      Rdkafka::RdkafkaError.validate!(response)
 
-      nil
+      return nil if response == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+
+      @native_kafka.with_inner do |inner|
+        Rdkafka::RdkafkaError.validate!(response, client_ptr: inner)
+      end
     ensure
       if native_topic && !native_topic.null?
         Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
@@ -490,16 +484,15 @@ module Rdkafka
 
       tpl = list.to_native_tpl
 
-      response = @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_offsets_for_times(
+      @native_kafka.with_inner do |inner|
+        response = Rdkafka::Bindings.rd_kafka_offsets_for_times(
           inner,
           tpl,
           timeout_ms # timeout
         )
+        Rdkafka::RdkafkaError.validate!(response, client_ptr: inner)
       end
 
-      Rdkafka::RdkafkaError.validate!(response)
-
       TopicPartitionList.from_native_tpl(tpl)
     ensure
       Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
@@ -528,12 +521,11 @@ module Rdkafka
       tpl = list ? list.to_native_tpl : nil
 
       begin
-        response = @native_kafka.with_inner do |inner|
-          Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
+        @native_kafka.with_inner do |inner|
+          response = Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
+          Rdkafka::RdkafkaError.validate!(response, client_ptr: inner)
         end
 
-        Rdkafka::RdkafkaError.validate!(response)
-
         nil
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
@@ -558,10 +550,12 @@ module Rdkafka
       native_message = Rdkafka::Bindings::Message.new(message_ptr)
 
       # Create a message to pass out
-      return Rdkafka::Consumer::Message.new(native_message) if native_message[:err].zero?
+      return Rdkafka::Consumer::Message.new(native_message) if native_message[:err] == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
 
-      # Raise error if needed
-      Rdkafka::RdkafkaError.validate!(native_message)
+      # Raise error if needed - wrap just the validate! call with client access
+      @native_kafka.with_inner do |inner|
+        Rdkafka::RdkafkaError.validate!(native_message, client_ptr: inner)
+      end
     ensure
       # Clean up rdkafka message if there is one
       if message_ptr && !message_ptr.null?

data/lib/rdkafka/error.rb

@@ -29,7 +29,7 @@ module Rdkafka
       def build_from_c(response_ptr, message_prefix = nil, broker_message: nil)
         code = Rdkafka::Bindings.rd_kafka_error_code(response_ptr)
 
-        return false if code.zero?
+        return false if code == Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
 
         message = broker_message || Rdkafka::Bindings.rd_kafka_err2str(code)
         fatal = !Rdkafka::Bindings.rd_kafka_error_is_fatal(response_ptr).zero?
@@ -51,11 +51,11 @@ module Rdkafka
       def build(response_ptr_or_code, message_prefix = nil, broker_message: nil)
         case response_ptr_or_code
         when Integer
-          return false if response_ptr_or_code.zero?
+          return false if response_ptr_or_code == Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
 
           new(response_ptr_or_code, message_prefix, broker_message: broker_message)
         when Bindings::Message
-          return false if response_ptr_or_code[:err].zero?
+          return false if response_ptr_or_code[:err] == Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
 
           unless response_ptr_or_code[:payload].null?
             message_prefix ||= response_ptr_or_code[:payload].read_string(response_ptr_or_code[:len])
@@ -81,9 +81,49 @@ module Rdkafka
         end
       end
 
-      def validate!(response_ptr_or_code, message_prefix = nil, broker_message: nil)
+      def validate!(response_ptr_or_code, message_prefix = nil, broker_message: nil, client_ptr: nil)
         error = build(response_ptr_or_code, message_prefix, broker_message: broker_message)
-        error ? raise(error) : false
+
+        return false unless error
+
+        # Auto-detect and handle fatal errors (-150)
+        if error.rdkafka_response == Bindings::RD_KAFKA_RESP_ERR__FATAL && client_ptr
+          # Discover the underlying fatal error from librdkafka
+          error = build_fatal(
+            client_ptr,
+            fallback_error_code: error.rdkafka_response,
+            fallback_message: broker_message
+          )
+        end
+
+        raise error
+      end
+
+      # Build a fatal error from librdkafka's fatal error state.
+      # Calls rd_kafka_fatal_error() to get the actual underlying error code and description.
+      #
+      # @param client_ptr [FFI::Pointer] Pointer to rd_kafka_t client
+      # @param fallback_error_code [Integer] Error code to use if no fatal error found (default: -150)
+      # @param fallback_message [String, nil] Message to use if no fatal error found
+      # @return [RdkafkaError] Error object with fatal flag set to true
+      def build_fatal(client_ptr, fallback_error_code: -150, fallback_message: nil)
+        fatal_error_details = Rdkafka::Bindings.extract_fatal_error(client_ptr)
+
+        if fatal_error_details
+          new(
+            fatal_error_details[:error_code],
+            broker_message: fatal_error_details[:error_string],
+            fatal: true
+          )
+        else
+          # Fallback: if extract_fatal_error returns nil (shouldn't happen in practice),
+          # the error code itself still indicates a fatal condition
+          new(
+            fallback_error_code,
+            broker_message: fallback_message,
+            fatal: true
+          )
+        end
       end
     end
 

data/lib/rdkafka/helpers/oauth.rb

@@ -26,7 +26,7 @@ module Rdkafka
           extensions_ptr&.free
         end
 
-        return response if response.zero?
+        return response if response == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
 
         oauthbearer_set_token_failure("Failed to set token: #{error_buffer.read_string}")
 

data/lib/rdkafka/producer/delivery_handle.rb

@@ -25,7 +25,7 @@ module Rdkafka
 
       # @return [DeliveryReport] a report on the delivery of the message
       def create_result
-        if self[:response] == 0
+        if self[:response] == Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
           DeliveryReport.new(
             self[:partition],
             self[:offset],

data/lib/rdkafka/producer/testing.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  # Testing utilities for Producer instances.
+  # This module is NOT included by default and should only be used in test environments.
+  #
+  # This module provides librdkafka native testing utilities that are needed to trigger certain
+  # behaviours that are hard to reproduce in stable environments, particularly fatal error
+  # scenarios in idempotent and transactional producers.
+  #
+  # To use in tests for producers:
+  #   producer.singleton_class.include(Rdkafka::Testing)
+  #
+  # Or include it for all producers in your test suite:
+  #   Rdkafka::Producer.include(Rdkafka::Testing)
+  #
+  # IMPORTANT: Fatal errors leave the producer client in an unusable state. After triggering
+  # a fatal error, the producer should be closed and discarded. Do not attempt to reuse a
+  # producer that has experienced a fatal error.
+  module Testing
+    # Triggers a test fatal error using rd_kafka_test_fatal_error.
+    # This is useful for testing fatal error handling without needing actual broker issues.
+    #
+    # @param error_code [Integer] The error code to trigger (e.g., 47 for invalid_producer_epoch)
+    # @param reason [String] Descriptive reason for the error
+    # @return [Integer] Result code from rd_kafka_test_fatal_error (0 on success)
+    #
+    # @example
+    #   producer.trigger_test_fatal_error(47, "Test producer fencing")
+    def trigger_test_fatal_error(error_code, reason)
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_test_fatal_error(inner, error_code, reason)
+      end
+    end
+
+    # Checks if a fatal error has occurred and retrieves error details.
+    # Calls rd_kafka_fatal_error to get the actual fatal error code and message.
+    #
+    # @return [Hash, nil] Hash with :error_code and :error_string if fatal error occurred, nil otherwise
+    #
+    # @example
+    #   if fatal_error = producer.fatal_error
+    #     puts "Fatal error #{fatal_error[:error_code]}: #{fatal_error[:error_string]}"
+    #   end
+    def fatal_error
+      @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.extract_fatal_error(inner)
+      end
+    end
+  end
+end

data/lib/rdkafka/producer.rb

@@ -142,7 +142,7 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_init_transactions(inner, -1)
 
-        Rdkafka::RdkafkaError.validate!(response_ptr) || true
+        Rdkafka::RdkafkaError.validate!(response_ptr, client_ptr: inner) || true
       end
     end
 
@@ -152,7 +152,7 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_begin_transaction(inner)
 
-        Rdkafka::RdkafkaError.validate!(response_ptr) || true
+        Rdkafka::RdkafkaError.validate!(response_ptr, client_ptr: inner) || true
       end
     end
 
@@ -162,7 +162,7 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_commit_transaction(inner, timeout_ms)
 
-        Rdkafka::RdkafkaError.validate!(response_ptr) || true
+        Rdkafka::RdkafkaError.validate!(response_ptr, client_ptr: inner) || true
       end
     end
 
@@ -171,7 +171,7 @@ module Rdkafka
 
       @native_kafka.with_inner do |inner|
         response_ptr = Rdkafka::Bindings.rd_kafka_abort_transaction(inner, timeout_ms)
-        Rdkafka::RdkafkaError.validate!(response_ptr) || true
+        Rdkafka::RdkafkaError.validate!(response_ptr, client_ptr: inner) || true
       end
     end
 
@@ -192,7 +192,7 @@ module Rdkafka
       @native_kafka.with_inner do |inner|
         response_ptr = Bindings.rd_kafka_send_offsets_to_transaction(inner, native_tpl, cgmetadata, timeout_ms)
 
-        Rdkafka::RdkafkaError.validate!(response_ptr)
+        Rdkafka::RdkafkaError.validate!(response_ptr, client_ptr: inner)
       end
     ensure
       if cgmetadata && !cgmetadata.null?
@@ -266,7 +266,7 @@ module Rdkafka
           Bindings::RD_KAFKA_PURGE_F_QUEUE | Bindings::RD_KAFKA_PURGE_F_INFLIGHT
         )
 
-        Rdkafka::RdkafkaError.validate!(response)
+        Rdkafka::RdkafkaError.validate!(response, client_ptr: inner)
       end
 
       # Wait for the purge to affect everything
@@ -278,7 +278,7 @@ module Rdkafka
     # Partition count for a given topic.
     #
     # @param topic [String] The topic name.
-    # @return [Integer] partition count for a given topic or `-1` if it could not be obtained.
+    # @return [Integer] partition count for a given topic or `RD_KAFKA_PARTITION_UA (-1)` if it could not be obtained.
     #
     # @note If 'allow.auto.create.topics' is set to true in the broker, the topic will be
     #   auto-created after returning nil.
@@ -299,7 +299,7 @@ module Rdkafka
           topic_metadata = ::Rdkafka::Metadata.new(inner, topic).topics&.first
         end
 
-        topic_metadata ? topic_metadata[:partition_count] : -1
+        topic_metadata ? topic_metadata[:partition_count] : Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
       end
     rescue Rdkafka::RdkafkaError => e
       # If the topic does not exist, it will be created or if not allowed another error will be
@@ -380,9 +380,9 @@ module Rdkafka
           selected_partitioner) if partition_count.positive?
       end
 
-      # If partition is nil, use -1 to let librdafka set the partition randomly or
+      # If partition is nil, use RD_KAFKA_PARTITION_UA to let librdafka set the partition randomly or
       # based on the key when present.
-      partition ||= -1
+      partition ||= Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
 
       # If timestamp is nil use 0 and let Kafka set one. If an integer or time
       # use it.
@@ -400,9 +400,9 @@ module Rdkafka
       delivery_handle.label = label
       delivery_handle.topic = topic
       delivery_handle[:pending] = true
-      delivery_handle[:response] = -1
-      delivery_handle[:partition] = -1
-      delivery_handle[:offset] = -1
+      delivery_handle[:response] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      delivery_handle[:partition] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
+      delivery_handle[:offset] = Rdkafka::Bindings::RD_KAFKA_PARTITION_UA
       DeliveryHandle.register(delivery_handle)
 
       args = [
@@ -454,9 +454,12 @@ module Rdkafka
       end
 
       # Raise error if the produce call was not successful
-      if response != 0
+      if response != Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
         DeliveryHandle.remove(delivery_handle.to_ptr.address)
-        Rdkafka::RdkafkaError.validate!(response)
+
+        @native_kafka.with_inner do |inner|
+          Rdkafka::RdkafkaError.validate!(response, client_ptr: inner)
+        end
       end
 
       delivery_handle

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.23.0.beta2"
+  VERSION = "0.23.1.rc1"
   LIBRDKAFKA_VERSION = "2.12.1"
   LIBRDKAFKA_SOURCE_SHA256 = "ec103fa05cb0f251e375f6ea0b6112cfc9d0acd977dc5b69fdc54242ba38a16f"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.23.0.beta2
+  version: 0.23.1.rc1
 platform: arm64-darwin
 authors:
 - Thijs Cadier
@@ -222,6 +222,7 @@ files:
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
 - lib/rdkafka/producer/partitions_count_cache.rb
+- lib/rdkafka/producer/testing.rb
 - lib/rdkafka/version.rb
 - renovate.json
 licenses: