karafka-rdkafka 0.13.5 → 0.13.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 460d2befd6142375c9e19c1874c2c44e749e48d91119395a0e3a2dcee37ddf67
-  data.tar.gz: ca6a6f51754841d6b8bce3cd63d644242e28231c3244969264df0f00061ceae6
+  metadata.gz: 4da2961103e92a94c36f605359cc5380ffa6d740cdc667329797ce9345caa663
+  data.tar.gz: 419aff211c5bfe35c244549fa14e042e92a39ad8583af2b8844283b7f1dca5b0
 SHA512:
-  metadata.gz: '09f5ee7ad4eeaed20631d12810948defc1812bce1962bdf018fc792db6fb59b82431fdc69bcea76968edd7a40b1c4537d01900d1eb084190b607c9682084517e'
-  data.tar.gz: 5d525cb93564d1bedde6d4bd6f7022bf064610cab1a8cb6ad676c0fe2097946a160a87f6e3227c73b3bbd7804802130c56a17fef28cb3c06db9d796f62369714
+  metadata.gz: c0b0d1069163ece1dcf5734ebc80e952a86c3eccc96885f856dc3f4569a0aaddc12db28fda7927d4c360b6f2ba0078a782f7ee6582554159c0e0b80c975e03bf
+  data.tar.gz: 2e3ba6a6ef850bafa305f2d9f2a9d3b4268fedbafc50f2b5ce2fedffcb4bb018b8a1ed5b1ce1b3a1efbad51621a2ad74d448641774d93879da628c06d4a1260c

data/.github/workflows/ci.yml

@@ -20,6 +20,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
+          - '3.3.0-preview2'
           - '3.2'
           - '3.1'
           - '3.1.0'
@@ -36,6 +37,10 @@ jobs:
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 
+      - name: Start Kafka with docker-compose
+        run: |
+          docker-compose up -d || (sleep 5 && docker-compose up -d)
+
       - name: Set up Ruby
         uses: ruby/setup-ruby@v1
         with:
@@ -47,7 +52,6 @@ jobs:
           GITHUB_COVERAGE: ${{matrix.coverage}}
 
         run: |
-          docker-compose up -d --no-recreate
           bundle install --path vendor/bundle
           cd ext && bundle exec rake && cd ..
           bundle exec rspec

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.13.6 (2023-10-17)
+* **[Feature]** Support transactions API in the producer
+* [Enhancement] Add `raise_response_error` flag to the `Rdkafka::AbstractHandle`.
+* [Enhancement] Provide `#purge` to remove any outstanding requests from the producer.
+* [Enhancement] Fix `#flush` does not handle the timeouts errors by making it return true if all flushed or false if failed. We do **not** raise an exception here to keep it backwards compatible.
+
 # 0.13.5
 * Fix DeliveryReport `create_result#error` being nil despite an error being associated with it
 

data/docker-compose.yml

@@ -1,24 +1,25 @@
----
-
 version: '2'
 
 services:
-  zookeeper:
-    image: confluentinc/cp-zookeeper:5.2.6
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
-
   kafka:
-    image: confluentinc/cp-kafka:5.2.5-10
-    depends_on:
-      - zookeeper
+    container_name: kafka
+    image: confluentinc/cp-kafka:7.5.0
+
     ports:
       - 9092:9092
+
     environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      CLUSTER_ID: kafka-docker-cluster-1
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
+      KAFKA_BROKER_ID: 1
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
+      ALLOW_PLAINTEXT_LISTENER: 'yes'
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1

data/lib/rdkafka/abstract_handle.rb

@@ -37,12 +37,13 @@ module Rdkafka
     #
     # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
     # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the operation has completed
+    # @param raise_response_error [Boolean] should we raise error when waiting finishes
     #
     # @raise [RdkafkaError] When the operation failed
     # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
     #
     # @return [Object] Operation-specific result
-    def wait(max_wait_timeout: 60, wait_timeout: 0.1)
+    def wait(max_wait_timeout: 60, wait_timeout: 0.1, raise_response_error: true)
       timeout = if max_wait_timeout
                   CURRENT_TIME.call + max_wait_timeout
                 else
@@ -54,7 +55,7 @@ module Rdkafka
             raise WaitTimeoutError.new("Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds")
           end
           sleep wait_timeout
-        elsif self[:response] != 0
+        elsif self[:response] != 0 && raise_response_error
           raise_error
         else
           return create_result
@@ -74,7 +75,7 @@ module Rdkafka
 
     # Allow subclasses to override
     def raise_error
-      raise RdkafkaError.new(self[:response])
+      RdkafkaError.validate!(self[:response])
     end
 
     # Error that is raised when waiting for the handle to complete

data/lib/rdkafka/admin/create_partitions_handle.rb

@@ -17,9 +17,12 @@ module Rdkafka
       end
 
       def raise_error
-        raise RdkafkaError.new(
-            self[:response],
-            broker_message: CreatePartitionsReport.new(self[:error_string], self[:result_name]).error_string
+        RdkafkaError.validate!(
+          self[:response],
+          broker_message: CreatePartitionsReport.new(
+            self[:error_string],
+            self[:result_name]
+          ).error_string
         )
       end
     end

data/lib/rdkafka/admin/create_topic_handle.rb

@@ -19,9 +19,12 @@ module Rdkafka
       end
 
       def raise_error
-        raise RdkafkaError.new(
-            self[:response],
-            broker_message: CreateTopicReport.new(self[:error_string], self[:result_name]).error_string
+        RdkafkaError.validate!(
+          self[:response],
+          broker_message: CreateTopicReport.new(
+            self[:error_string],
+            self[:result_name]
+          ).error_string
         )
       end
     end

data/lib/rdkafka/admin/delete_topic_handle.rb

@@ -19,9 +19,12 @@ module Rdkafka
       end
 
       def raise_error
-        raise RdkafkaError.new(
-            self[:response],
-            broker_message: DeleteTopicReport.new(self[:error_string], self[:result_name]).error_string
+        RdkafkaError.validate!(
+          self[:response],
+          broker_message: DeleteTopicReport.new(
+            self[:error_string],
+            self[:result_name]
+          ).error_string
         )
       end
     end

data/lib/rdkafka/bindings.rb

@@ -6,6 +6,15 @@ require "logger"
 
 module Rdkafka
   # @private
+  #
+  # @note
+  #   There are two types of responses related to errors:
+  #     - rd_kafka_error_t - a C object that we need to remap into an error or null when no error
+  #     - rd_kafka_resp_err_t - response error code (numeric) that we can use directly
+  #
+  #   It is critical to ensure, that we handle them correctly. The result type should be:
+  #     - rd_kafka_error_t - :pointer
+  #     - rd_kafka_resp_err_t - :int
   module Bindings
     extend FFI::Library
 
@@ -35,7 +44,7 @@ module Rdkafka
 
     # Polling
 
-    attach_function :rd_kafka_flush, [:pointer, :int], :void, blocking: true
+    attach_function :rd_kafka_flush, [:pointer, :int], :int, blocking: true
     attach_function :rd_kafka_poll, [:pointer, :int], :void, blocking: true
     attach_function :rd_kafka_outq_len, [:pointer], :int, blocking: true
 
@@ -96,6 +105,11 @@ module Rdkafka
 
     attach_function :rd_kafka_err2name, [:int], :string
     attach_function :rd_kafka_err2str, [:int], :string
+    attach_function :rd_kafka_error_is_fatal, [:pointer], :int
+    attach_function :rd_kafka_error_is_retriable, [:pointer], :int
+    attach_function :rd_kafka_error_txn_requires_abort, [:pointer], :int
+    attach_function :rd_kafka_error_destroy, [:pointer], :void
+    attach_function :rd_kafka_error_code, [:pointer], :int
 
     # Configuration
 
@@ -157,7 +171,7 @@ module Rdkafka
       :void, [:pointer, :int, :string, :pointer]
     ) do |_client_prr, err_code, reason, _opaque|
       if Rdkafka::Config.error_callback
-        error = Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
+        error = Rdkafka::RdkafkaError.build(err_code, broker_message: reason)
         error.set_backtrace(caller)
         Rdkafka::Config.error_callback.call(error)
       end
@@ -255,12 +269,19 @@ module Rdkafka
     RD_KAFKA_VTYPE_TIMESTAMP = 8
     RD_KAFKA_VTYPE_HEADER = 9
     RD_KAFKA_VTYPE_HEADERS = 10
+    RD_KAFKA_PURGE_F_QUEUE = 1
+    RD_KAFKA_PURGE_F_INFLIGHT = 2
 
     RD_KAFKA_MSG_F_COPY = 0x2
 
     attach_function :rd_kafka_producev, [:pointer, :varargs], :int, blocking: true
+    attach_function :rd_kafka_purge, [:pointer, :int], :int, blocking: true
     callback :delivery_cb, [:pointer, :pointer, :pointer], :void
     attach_function :rd_kafka_conf_set_dr_msg_cb, [:pointer, :delivery_cb], :void
+    attach_function :rd_kafka_init_transactions, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_begin_transaction, [:pointer], :pointer, blocking: true
+    attach_function :rd_kafka_abort_transaction, [:pointer, :int], :pointer, blocking: true
+    attach_function :rd_kafka_commit_transaction, [:pointer, :int], :pointer, blocking: true
 
     # Partitioner
     PARTITIONERS = %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random).each_with_object({}) do |name, hsh|

data/lib/rdkafka/consumer.rb

@@ -68,9 +68,8 @@ module Rdkafka
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_subscribe(inner, tpl)
       end
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error subscribing to '#{topics.join(', ')}'")
-      end
+
+      Rdkafka::RdkafkaError.validate!(response, "Error subscribing to '#{topics.join(', ')}'")
     ensure
       Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) unless tpl.nil?
     end
@@ -86,9 +85,10 @@ module Rdkafka
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_unsubscribe(inner)
       end
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+
+      Rdkafka::RdkafkaError.validate!(response)
+
+      nil
     end
 
     # Pause producing or consumption for the provided list of partitions
@@ -141,9 +141,10 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_resume_partitions(inner, tpl)
         end
-        if response != 0
-          raise Rdkafka::RdkafkaError.new(response, "Error resume '#{list.to_h}'")
-        end
+
+        Rdkafka::RdkafkaError.validate!(response, "Error resume '#{list.to_h}'")
+
+        nil
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
@@ -162,9 +163,7 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_subscription(inner, ptr)
       end
 
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+      Rdkafka::RdkafkaError.validate!(response)
 
       native = ptr.read_pointer
 
@@ -193,9 +192,8 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_assign(inner, tpl)
         end
-        if response != 0
-          raise Rdkafka::RdkafkaError.new(response, "Error assigning '#{list.to_h}'")
-        end
+
+        Rdkafka::RdkafkaError.validate!(response, "Error assigning '#{list.to_h}'")
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
       end
@@ -213,9 +211,8 @@ module Rdkafka
       response = @native_kafka.with_inner do |inner|
         Rdkafka::Bindings.rd_kafka_assignment(inner, ptr)
       end
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+
+      Rdkafka::RdkafkaError.validate!(response)
 
       tpl = ptr.read_pointer
 
@@ -263,9 +260,9 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_committed(inner, tpl, timeout_ms)
         end
-        if response != 0
-          raise Rdkafka::RdkafkaError.new(response)
-        end
+
+        Rdkafka::RdkafkaError.validate!(response)
+
         TopicPartitionList.from_native_tpl(tpl)
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl)
@@ -297,9 +294,8 @@ module Rdkafka
           timeout_ms,
         )
       end
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
-      end
+
+      Rdkafka::RdkafkaError.validate!(response, "Error querying watermark offsets for partition #{partition} of #{topic}")
 
       return low.read_array_of_int64(1).first, high.read_array_of_int64(1).first
     ensure
@@ -387,9 +383,10 @@ module Rdkafka
         message.partition,
         message.offset
       )
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+
+      Rdkafka::RdkafkaError.validate!(response)
+
+      nil
     ensure
       if native_topic && !native_topic.null?
         Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
@@ -422,9 +419,9 @@ module Rdkafka
         message.offset,
         0 # timeout
       )
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+      Rdkafka::RdkafkaError.validate!(response)
+
+      nil
     ensure
       if native_topic && !native_topic.null?
         Rdkafka::Bindings.rd_kafka_topic_destroy(native_topic)
@@ -455,9 +452,7 @@ module Rdkafka
         )
       end
 
-      if response != 0
-        raise Rdkafka::RdkafkaError.new(response)
-      end
+      Rdkafka::RdkafkaError.validate!(response)
 
       TopicPartitionList.from_native_tpl(tpl)
     ensure
@@ -492,9 +487,10 @@ module Rdkafka
         response = @native_kafka.with_inner do |inner|
           Rdkafka::Bindings.rd_kafka_commit(inner, tpl, async)
         end
-        if response != 0
-          raise Rdkafka::RdkafkaError.new(response)
-        end
+
+        Rdkafka::RdkafkaError.validate!(response)
+
+        nil
       ensure
         Rdkafka::Bindings.rd_kafka_topic_partition_list_destroy(tpl) if tpl
       end
@@ -519,9 +515,9 @@ module Rdkafka
         # Create struct wrapper
         native_message = Rdkafka::Bindings::Message.new(message_ptr)
         # Raise error if needed
-        if native_message[:err] != 0
-          raise Rdkafka::RdkafkaError.new(native_message[:err])
-        end
+
+        Rdkafka::RdkafkaError.validate!(native_message[:err])
+
         # Create a message to pass out
         Rdkafka::Consumer::Message.new(native_message)
       end

data/lib/rdkafka/error.rb

@@ -18,12 +18,59 @@ module Rdkafka
     # @return [String]
     attr_reader :broker_message
 
+    class << self
+      def build_from_c(response_ptr, message_prefix = nil, broker_message: nil)
+        code = Rdkafka::Bindings.rd_kafka_error_code(response_ptr)
+
+        return false if code.zero?
+
+        message = broker_message || Rdkafka::Bindings.rd_kafka_err2str(code)
+        fatal = !Rdkafka::Bindings.rd_kafka_error_is_fatal(response_ptr).zero?
+        retryable = !Rdkafka::Bindings.rd_kafka_error_is_retriable(response_ptr).zero?
+        abortable = !Rdkafka::Bindings.rd_kafka_error_txn_requires_abort(response_ptr).zero?
+
+        Rdkafka::Bindings.rd_kafka_error_destroy(response_ptr)
+
+        new(
+          code,
+          message_prefix,
+          broker_message: message,
+          fatal: fatal,
+          retryable: retryable,
+          abortable: abortable
+        )
+      end
+
+      def build(response_ptr_or_code, message_prefix = nil, broker_message: nil)
+        if response_ptr_or_code.is_a?(Integer)
+          response_ptr_or_code.zero? ? false : new(response_ptr_or_code, message_prefix, broker_message: broker_message)
+        else
+          build_from_c(response_ptr_or_code, message_prefix)
+        end
+      end
+
+      def validate!(response_ptr_or_code, message_prefix = nil, broker_message: nil)
+        error = build(response_ptr_or_code, message_prefix, broker_message: broker_message)
+        error ? raise(error) : false
+      end
+    end
+
     # @private
-    def initialize(response, message_prefix=nil, broker_message: nil)
+    def initialize(
+      response,
+      message_prefix=nil,
+      broker_message: nil,
+      fatal: false,
+      retryable: false,
+      abortable: false
+    )
       raise TypeError.new("Response has to be an integer") unless response.is_a? Integer
       @rdkafka_response = response
       @message_prefix = message_prefix
       @broker_message = broker_message
+      @fatal = fatal
+      @retryable = retryable
+      @abortable = abortable
     end
 
     # This error's code, for example `:partition_eof`, `:msg_size_too_large`.
@@ -58,6 +105,18 @@ module Rdkafka
     def ==(another_error)
        another_error.is_a?(self.class) && (self.to_s == another_error.to_s)
     end
+
+    def fatal?
+      @fatal
+    end
+
+    def retryable?
+      @retryable
+    end
+
+    def abortable?
+      @abortable
+    end
   end
 
   # Error with topic partition list returned by the underlying rdkafka library.

data/lib/rdkafka/metadata.rb

@@ -29,8 +29,7 @@ module Rdkafka
       # Retrieve the Metadata
       result = Rdkafka::Bindings.rd_kafka_metadata(native_client, topic_flag, native_topic, ptr, timeout_ms)
 
-      # Error Handling
-      raise Rdkafka::RdkafkaError.new(result) unless result.zero?
+      Rdkafka::RdkafkaError.validate!(result)
 
       metadata_from_native(ptr.read_pointer)
     rescue ::Rdkafka::RdkafkaError => e
@@ -58,11 +57,12 @@ module Rdkafka
 
       @topics = Array.new(metadata[:topics_count]) do |i|
         topic = TopicMetadata.new(metadata[:topics_metadata] + (i * TopicMetadata.size))
-        raise Rdkafka::RdkafkaError.new(topic[:rd_kafka_resp_err]) unless topic[:rd_kafka_resp_err].zero?
+
+        RdkafkaError.validate!(topic[:rd_kafka_resp_err])
 
         partitions = Array.new(topic[:partition_count]) do |j|
           partition = PartitionMetadata.new(topic[:partitions_metadata] + (j * PartitionMetadata.size))
-          raise Rdkafka::RdkafkaError.new(partition[:rd_kafka_resp_err]) unless partition[:rd_kafka_resp_err].zero?
+          RdkafkaError.validate!(partition[:rd_kafka_resp_err])
           partition.to_h
         end
         topic.to_h.merge!(partitions: partitions)

data/lib/rdkafka/producer/delivery_handle.rb

@@ -31,7 +31,7 @@ module Rdkafka
             # For part of errors, we will not get a topic name reference and in cases like this
             # we should not return it
             self[:topic_name].null? ? nil : self[:topic_name].read_string,
-            RdkafkaError.new(self[:response])
+            Rdkafka::RdkafkaError.build(self[:response])
           )
         end
       end

data/lib/rdkafka/producer.rb

@@ -63,6 +63,47 @@ module Rdkafka
       @delivery_callback_arity = arity(callback)
     end
 
+    # Init transactions
+    # Run once per producer
+    def init_transactions
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        response_ptr = Rdkafka::Bindings.rd_kafka_init_transactions(inner, -1)
+
+        Rdkafka::RdkafkaError.validate!(response_ptr) || true
+      end
+    end
+
+    def begin_transaction
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        response_ptr = Rdkafka::Bindings.rd_kafka_begin_transaction(inner)
+
+        Rdkafka::RdkafkaError.validate!(response_ptr)
+      end
+    end
+
+    def commit_transaction(timeout_ms = -1)
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        response_ptr = Rdkafka::Bindings.rd_kafka_commit_transaction(inner, timeout_ms)
+
+        Rdkafka::RdkafkaError.validate!(response_ptr)
+      end
+    end
+
+    def abort_transaction(timeout_ms = -1)
+      closed_producer_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        response_ptr = Rdkafka::Bindings.rd_kafka_abort_transaction(inner, timeout_ms)
+        Rdkafka::RdkafkaError.validate!(response_ptr)
+      end
+    end
+
     # Close this producer and wait for the internal poll queue to empty.
     def close
       return if closed?
@@ -79,12 +120,50 @@ module Rdkafka
     # in seconds. Call this before closing a producer to ensure delivery of all messages.
     #
     # @param timeout_ms [Integer] how long should we wait for flush of all messages
+    # @return [Boolean] true if no more data and all was flushed, false in case there are still
+    #   outgoing messages after the timeout
+    #
+    # @note We raise an exception for other errors because based on the librdkafka docs, there
+    #   should be no other errors.
+    #
+    # @note For `timed_out` we do not raise an error to keep it backwards compatible
     def flush(timeout_ms=5_000)
       closed_producer_check(__method__)
 
+      error = @native_kafka.with_inner do |inner|
+        response = Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
+        Rdkafka::RdkafkaError.build(response)
+      end
+
+      # Early skip not to build the error message
+      return true unless error
+      return false if error.code == :timed_out
+
+      raise(error)
+    end
+
+    # Purges the outgoing queue and releases all resources.
+    #
+    # Useful when closing the producer with outgoing messages to unstable clusters or when for
+    # any other reasons waiting cannot go on anymore. This purges both the queue and all the
+    # inflight requests + updates the delivery handles statuses so they can be materialized into
+    # `purge_queue` errors.
+    def purge
+      closed_producer_check(__method__)
+
       @native_kafka.with_inner do |inner|
-        Rdkafka::Bindings.rd_kafka_flush(inner, timeout_ms)
+        response = Bindings.rd_kafka_purge(
+          inner,
+          Bindings::RD_KAFKA_PURGE_F_QUEUE | Bindings::RD_KAFKA_PURGE_F_INFLIGHT
+        )
+
+        Rdkafka::RdkafkaError.validate!(response)
       end
+
+      # Wait for the purge to affect everything
+      sleep(0.001) until flush(100)
+
+      true
     end
 
     # Partition count for a given topic.
@@ -208,7 +287,7 @@ module Rdkafka
       # Raise error if the produce call was not successful
       if response != 0
         DeliveryHandle.remove(delivery_handle.to_ptr.address)
-        raise RdkafkaError.new(response)
+        Rdkafka::RdkafkaError.validate!(response)
       end
 
       delivery_handle

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.5"
+  VERSION = "0.13.6"
   LIBRDKAFKA_VERSION = "2.2.0"
   LIBRDKAFKA_SOURCE_SHA256 = "af9a820cbecbc64115629471df7c7cecd40403b6c34bfdbb9223152677a47226"
 end

data/spec/rdkafka/admin/create_topic_handle_spec.rb

@@ -45,10 +45,12 @@ describe Rdkafka::Admin::CreateTopicHandle do
   describe "#raise_error" do
     let(:pending_handle) { false }
 
-    it "should raise the appropriate error" do
+    before { subject[:response] = -1 }
+
+    it "should raise the appropriate error when there is an error" do
       expect {
         subject.raise_error
-      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+      }.to raise_exception(Rdkafka::RdkafkaError, /Unknown broker error \(unknown\)/)
     end
   end
 end

data/spec/rdkafka/admin/delete_topic_handle_spec.rb

@@ -45,10 +45,12 @@ describe Rdkafka::Admin::DeleteTopicHandle do
   describe "#raise_error" do
     let(:pending_handle) { false }
 
+    before { subject[:response] = -1 }
+
     it "should raise the appropriate error" do
       expect {
         subject.raise_error
-      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+      }.to raise_exception(Rdkafka::RdkafkaError, /Unknown broker error \(unknown\)/)
     end
   end
 end

data/spec/rdkafka/admin_spec.rb

@@ -33,7 +33,7 @@ describe Rdkafka::Admin do
           }.to raise_exception { |ex|
             expect(ex).to be_a(Rdkafka::RdkafkaError)
             expect(ex.message).to match(/Broker: Invalid topic \(topic_exception\)/)
-            expect(ex.broker_message).to match(/Topic name.*is illegal, it contains a character other than ASCII alphanumerics/)
+            expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or more characters other than ASCII alphanumerics, '.', '_' and '-'/)
           }
         end
       end

data/spec/rdkafka/metadata_spec.rb

@@ -31,7 +31,7 @@ describe Rdkafka::Metadata do
       it "#brokers returns our single broker" do
         expect(subject.brokers.length).to eq(1)
         expect(subject.brokers[0][:broker_id]).to eq(1)
-        expect(subject.brokers[0][:broker_name]).to eq("localhost")
+        expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
         expect(subject.brokers[0][:broker_port]).to eq(9092)
       end
 
@@ -54,7 +54,7 @@ describe Rdkafka::Metadata do
     it "#brokers returns our single broker" do
       expect(subject.brokers.length).to eq(1)
       expect(subject.brokers[0][:broker_id]).to eq(1)
-      expect(subject.brokers[0][:broker_name]).to eq("localhost")
+      expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
       expect(subject.brokers[0][:broker_port]).to eq(9092)
     end
 

data/spec/rdkafka/producer_spec.rb

@@ -559,6 +559,22 @@ describe Rdkafka::Producer do
     end
   end
 
+  context "when not being able to deliver the message" do
+    let(:producer) do
+      rdkafka_producer_config(
+        "bootstrap.servers": "localhost:9093",
+        "message.timeout.ms": 100
+      ).producer
+    end
+
+    it "should contain the error in the response when not deliverable" do
+      handler = producer.produce(topic: 'produce_test_topic', payload: nil)
+      # Wait for the async callbacks and delivery registry to update
+      sleep(2)
+      expect(handler.create_result.error).to be_a(Rdkafka::RdkafkaError)
+    end
+  end
+
   describe '#partition_count' do
     it { expect(producer.partition_count('example_topic')).to eq(1) }
 
@@ -628,19 +644,274 @@ describe Rdkafka::Producer do
     end
   end
 
-  context "when not being able to deliver the message" do
+  describe '#flush' do
+    it "should return flush when it can flush all outstanding messages or when no messages" do
+      producer.produce(
+        topic:     "produce_test_topic",
+        payload:   "payload headers",
+        key:       "key headers",
+        headers:   {}
+      )
+
+      expect(producer.flush(5_000)).to eq(true)
+    end
+
+    context 'when it cannot flush due to a timeout' do
+      let(:producer) do
+        rdkafka_producer_config(
+          "bootstrap.servers": "localhost:9093",
+          "message.timeout.ms": 2_000
+        ).producer
+      end
+
+      after do
+        # Allow rdkafka to evict message preventing memory-leak
+        sleep(2)
+      end
+
+      it "should return false on flush when cannot deliver and beyond timeout" do
+        producer.produce(
+          topic:     "produce_test_topic",
+          payload:   "payload headers",
+          key:       "key headers",
+          headers:   {}
+        )
+
+        expect(producer.flush(1_000)).to eq(false)
+      end
+    end
+
+    context 'when there is a different error' do
+      before { allow(Rdkafka::Bindings).to receive(:rd_kafka_flush).and_return(-199) }
+
+      it 'should raise it' do
+        expect { producer.flush }.to raise_error(Rdkafka::RdkafkaError)
+      end
+    end
+  end
+
+  describe '#purge' do
+    context 'when no outgoing messages' do
+      it { expect(producer.purge).to eq(true) }
+    end
+
+    context 'when librdkafka purge returns an error' do
+      before { expect(Rdkafka::Bindings).to receive(:rd_kafka_purge).and_return(-153) }
+
+      it 'expect to raise an error' do
+        expect { producer.purge }.to raise_error(Rdkafka::RdkafkaError, /retry/)
+      end
+    end
+
+    context 'when there are outgoing things in the queue' do
+      let(:producer) do
+        rdkafka_producer_config(
+          "bootstrap.servers": "localhost:9093",
+          "message.timeout.ms": 2_000
+        ).producer
+      end
+
+      it "should should purge and move forward" do
+        producer.produce(
+          topic:     "produce_test_topic",
+          payload:   "payload headers"
+        )
+
+        expect(producer.purge).to eq(true)
+        expect(producer.flush(1_000)).to eq(true)
+      end
+
+      it "should materialize the delivery handles" do
+        handle = producer.produce(
+          topic:     "produce_test_topic",
+          payload:   "payload headers"
+        )
+
+        expect(producer.purge).to eq(true)
+
+        expect { handle.wait }.to raise_error(Rdkafka::RdkafkaError, /purge_queue/)
+      end
+
+      context "when using delivery_callback" do
+        let(:delivery_reports) { [] }
+
+        let(:delivery_callback) do
+          ->(delivery_report) { delivery_reports << delivery_report }
+        end
+
+        before { producer.delivery_callback = delivery_callback }
+
+        it "should run the callback" do
+          handle = producer.produce(
+            topic:     "produce_test_topic",
+            payload:   "payload headers"
+          )
+
+          expect(producer.purge).to eq(true)
+          # queue purge
+          expect(delivery_reports[0].error).to eq(-152)
+        end
+      end
+    end
+  end
+
+  context 'when working with transactions' do
     let(:producer) do
       rdkafka_producer_config(
-        "bootstrap.servers": "localhost:9093",
-        "message.timeout.ms": 100
+        'transactional.id': SecureRandom.uuid,
+        'transaction.timeout.ms': 5_000
       ).producer
     end
 
-    it "should contain the error in the response when not deliverable" do
-      handler = producer.produce(topic: 'produce_test_topic', payload: nil)
-      # Wait for the async callbacks and delivery registry to update
-      sleep(2)
-      expect(handler.create_result.error).to be_a(Rdkafka::RdkafkaError)
+    it 'expect not to allow to produce without transaction init' do
+      expect do
+        producer.produce(topic: 'produce_test_topic', payload: 'data')
+      end.to raise_error(Rdkafka::RdkafkaError, /Erroneous state \(state\)/)
+    end
+
+    it 'expect to raise error when transactions are initialized but producing not in one' do
+      producer.init_transactions
+
+      expect do
+        producer.produce(topic: 'produce_test_topic', payload: 'data')
+      end.to raise_error(Rdkafka::RdkafkaError, /Erroneous state \(state\)/)
+    end
+
+    it 'expect to allow to produce within a transaction, finalize and ship data' do
+      producer.init_transactions
+      producer.begin_transaction
+      handle1 = producer.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+      handle2 = producer.produce(topic: 'example_topic', payload: 'data2', partition: 0)
+      producer.commit_transaction
+
+      report1 = handle1.wait(max_wait_timeout: 15)
+      report2 = handle2.wait(max_wait_timeout: 15)
+
+      message1 = wait_for_message(
+        topic: "produce_test_topic",
+        delivery_report: report1,
+        consumer: consumer
+      )
+
+      expect(message1.partition).to eq 1
+      expect(message1.payload).to eq "data1"
+      expect(message1.timestamp).to be_within(10).of(Time.now)
+
+      message2 = wait_for_message(
+        topic: "example_topic",
+        delivery_report: report2,
+        consumer: consumer
+      )
+
+      expect(message2.partition).to eq 0
+      expect(message2.payload).to eq "data2"
+      expect(message2.timestamp).to be_within(10).of(Time.now)
+    end
+
+    it 'expect not to send data and propagate purge queue error on abort' do
+      producer.init_transactions
+      producer.begin_transaction
+      handle1 = producer.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+      handle2 = producer.produce(topic: 'example_topic', payload: 'data2', partition: 0)
+      producer.abort_transaction
+
+      expect { handle1.wait(max_wait_timeout: 15) }
+        .to raise_error(Rdkafka::RdkafkaError, /Purged in queue \(purge_queue\)/)
+      expect { handle2.wait(max_wait_timeout: 15) }
+        .to raise_error(Rdkafka::RdkafkaError, /Purged in queue \(purge_queue\)/)
+    end
+
+    it 'expect to have non retryable, non abortable and not fatal error on abort' do
+      producer.init_transactions
+      producer.begin_transaction
+      handle = producer.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+      producer.abort_transaction
+
+      response = handle.wait(raise_response_error: false)
+
+      expect(response.error).to be_a(Rdkafka::RdkafkaError)
+      expect(response.error.retryable?).to eq(false)
+      expect(response.error.fatal?).to eq(false)
+      expect(response.error.abortable?).to eq(false)
+    end
+
+    # This may not always crash, depends on load but no other way to check it
+    context 'when timeout is too short and error occurs and we can abort' do
+      let(:producer) do
+        rdkafka_producer_config(
+          'transactional.id': SecureRandom.uuid,
+          'transaction.timeout.ms': 1_000
+        ).producer
+      end
+
+      it 'expect to allow to produce within a transaction, finalize and ship data' do
+        producer.init_transactions
+        producer.begin_transaction
+
+        handle1 = producer.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+        handle2 = producer.produce(topic: 'example_topic', payload: 'data2', partition: 0)
+
+        begin
+          producer.commit_transaction
+        rescue Rdkafka::RdkafkaError => e
+          next unless e.abortable?
+
+          producer.abort_transaction
+
+          expect { handle1.wait(max_wait_timeout: 15) }.to raise_error(Rdkafka::RdkafkaError)
+          expect { handle2.wait(max_wait_timeout: 15) }.to raise_error(Rdkafka::RdkafkaError)
+        end
+      end
+    end
+
+    context 'fencing against previous active producer with same transactional id' do
+      let(:transactional_id) { SecureRandom.uuid }
+
+      let(:producer1) do
+        rdkafka_producer_config(
+          'transactional.id': transactional_id,
+          'transaction.timeout.ms': 10_000
+        ).producer
+      end
+
+      let(:producer2) do
+        rdkafka_producer_config(
+          'transactional.id': transactional_id,
+          'transaction.timeout.ms': 10_000
+        ).producer
+      end
+
+      after do
+        producer1.close
+        producer2.close
+      end
+
+      it 'expect older producer not to be able to commit when fanced out' do
+        producer1.init_transactions
+        producer1.begin_transaction
+        producer1.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+
+        producer2.init_transactions
+        producer2.begin_transaction
+        producer2.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
+
+        expect { producer1.commit_transaction }
+          .to raise_error(Rdkafka::RdkafkaError, /This instance has been fenced/)
+
+        error = false
+
+        begin
+          producer1.commit_transaction
+        rescue Rdkafka::RdkafkaError => e
+          error = e
+        end
+
+        expect(error.fatal?).to eq(true)
+        expect(error.abortable?).to eq(false)
+        expect(error.retryable?).to eq(false)
+
+        expect { producer2.commit_transaction }.not_to raise_error
+      end
     end
   end
 end

data/spec/spec_helper.rb

@@ -11,6 +11,7 @@ require "pry"
 require "rspec"
 require "rdkafka"
 require "timeout"
+require 'securerandom'
 
 def rdkafka_base_config
   {
@@ -134,6 +135,7 @@ RSpec.configure do |config|
         rake_test_topic:         3,
         watermarks_test_topic:   3,
         partitioner_test_topic: 25,
+        example_topic:           1
     }.each do |topic, partitions|
       create_topic_handle = admin.create_topic(topic.to_s, partitions, 1)
       begin

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.13.5
+  version: 0.13.6
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-09-16 00:00:00.000000000 Z
+date: 2023-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi