rdkafka 0.23.1 → 0.24.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fcc01c63ae339c6d9a2bda9ac171d022bcbe900e2b358725914a3b1950bb775b
-  data.tar.gz: 2c95e85ed2b2890d2b59e7c171285fcc2050cf50dff27fafb5b8a30282f34fb8
+  metadata.gz: 5b0467e6f670f137f458bb1cb856f812ba2292c45a4c2a4b19477531fd59a3b7
+  data.tar.gz: 9e18e607e5114b7ed31266565c308517729890e643a68b8535d19fb8df4f8256
 SHA512:
-  metadata.gz: b96a9a6362fb86e3c6f6210c4b93b8204801256178747fd1e25dc08bd8132b454c56102ad009fee1ee6c4de89a4a8f1f8835c3262063ef68c04e31a555f5a28e
-  data.tar.gz: 60ab2b193be2e30d22fed7fe6c12bed7041bc2408c05d9702993c69e029321fa49e0c8a4d33d6334e347c21072c113cfcd065f0581b47dcea734e53140b3f03a
+  metadata.gz: d90b9aff5f184ec165c37451773845f66c71cfa47dc4dbf62d4764c009711ac28ab40fe4b3ecc083c2b599043f3906d552b52a1296fcbee089a8024a03c16e9e
+  data.tar.gz: 91db6ebb4e3dde0d98395c1714205bae0a5027b31f4d1a1fa435171873d81998efa56c76bdb2c8a410f3cbb3d4348ae847d564d8c2348884d0a7ee080877e7a4

data/CHANGELOG.md

@@ -1,12 +1,22 @@
 # Rdkafka Changelog
 
+## 0.24.2 (2025-10-31)
+- [Enhancement] Force lock FFI to 1.17.1 or higher to include critical bug fixes around GCC, write barriers, and thread restarts for forks.
+- [Fix] Fix for Core dump when providing extensions to oauthbearer_set_token (dssjoblom)
+
+## 0.24.1 (2025-10-10)
+- [Fix] Fix Github Action Ruby reference preventing non-compiled releases.
+
+## 0.24.0 (2025-10-10)
+- [Enhancement] Bump librdkafka to `2.11.1`
+
 ## 0.23.1 (2025-09-25)
 - [Enhancement] Improve sigstore attestation for precompiled releases.
 - [Fix] Fix incorrectly set default SSL certs dir.
 - [Fix] Disable OpenSSL Heartbeats during compilation.
 
 ## 0.23.0 (2025-09-04)
-- **[Breaking]** Drop support for Ruby 3.1 to move forward with the fiber scheduler work.
+- **[EOL]** Drop support for Ruby 3.1 to move forward with the fiber scheduler work.
 - [Enhancement] Bump librdkafka to `2.11.0`
 - [Enhancement] Support explicit Debian testing due to lib issues.
 - [Enhancement] Support ARM64 Gnu precompilation.
@@ -53,7 +63,7 @@
 - [Patch] Retire no longer needed cooperative-sticky patch.
 
 ## 0.19.0 (2024-10-01)
-- **[Breaking]** Drop Ruby 3.0 support
+- **[EOL]** Drop Ruby 3.0 support
 - [Enhancement] Update `librdkafka` to `2.5.3`
 - [Enhancement] Use default oauth callback if none is passed (bachmanity1)
 - [Fix] Fix incorrectly behaving CI on failures. 
@@ -77,7 +87,7 @@
 - [Fix] Switch to local release of librdkafka to mitigate its unavailability.
 
 ## 0.16.0 (2024-06-13)
-- **[Breaking]** Retire support for Ruby 2.7.
+- **[EOL]** Retire support for Ruby 2.7.
 - **[Breaking]** Messages without headers returned by `#poll` contain frozen empty hash.
 - **[Breaking]** `HashWithSymbolKeysTreatedLikeStrings` has been removed so headers are regular hashes with string keys.
 - **[Feature]** Support incremental config describe + alter API.
@@ -138,7 +148,7 @@
 - [Fix] Reference to Opaque is not released when Admin, Consumer or Producer is closed (mensfeld)
 - [Fix] Trigger `#poll` on native kafka creation to handle oauthbearer cb (mensfeld)
 - [Fix] `#flush` does not handle the timeouts errors by making it return `true` if all flushed or `false` if failed. We do **not** raise an exception here to keep it backwards compatible (mensfeld)
-- [Change] Remove support for Ruby 2.6 due to it being EOL and WeakMap incompatibilities (mensfeld)
+- **[EOL]** Remove support for Ruby 2.6 due to it being EOL and WeakMap incompatibilities (mensfeld)
 - [Change] Update Kafka Docker with Confluent KRaft (mensfeld)
 - [Change] Update librdkafka repo reference from edenhill to confluentinc (mensfeld)
 

data/README.md

@@ -163,6 +163,7 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
+| 0.24.x (Unreleased) | 2.11.1 (2025-08-08) | yes |
 | 0.23.x (2025-09-04) | 2.11.0 (2025-07-03) | yes |
 | 0.22.x (2025-07-17) | 2.8.0 (2025-01-07)  | yes |
 | 0.21.x (2025-02-13) | 2.8.0 (2025-01-07)  | yes |

data/bin/verify_kafka_warnings

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Checks Kafka logs for unsupported warning patterns
+# Only specified warnings are allowed, all others should trigger failure
+
+allowed_patterns=(
+  "Performing controller activation"
+  "registered with feature metadata.version"
+  "Replayed TopicRecord for"
+  "Replayed PartitionRecord for"
+  "Previous leader None and previous leader epoch"
+  "Creating new"
+  "Unloaded transaction metadata"
+  "closing connection"
+)
+
+# Get all warnings
+warnings=$(docker logs --since=0 kafka | grep "] WARN ")
+exit_code=0
+
+while IFS= read -r line; do
+  allowed=0
+  for pattern in "${allowed_patterns[@]}"; do
+    if echo "$line" | grep -q "$pattern"; then
+      allowed=1
+      break
+    fi
+  done
+
+  if [ $allowed -eq 0 ]; then
+    echo "Unexpected warning found:"
+    echo "$line"
+    exit_code=1
+  fi
+done <<< "$warnings"
+
+exit $exit_code

data/docker-compose-ssl.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:8.0.0
+    image: confluentinc/cp-kafka:8.1.0
     ports:
       - 9092:9092 # Support PLAINTEXT so we can run one docker setup for SSL and PLAINTEXT
       - 9093:9093

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:8.0.0
+    image: confluentinc/cp-kafka:8.1.0
 
     ports:
       - 9092:9092

data/lib/rdkafka/bindings.rb

@@ -2,6 +2,15 @@
 
 module Rdkafka
   # @private
+  #
+  # @note
+  #   There are two types of responses related to errors:
+  #     - rd_kafka_error_t - a C object that we need to remap into an error or null when no error
+  #     - rd_kafka_resp_err_t - response error code (numeric) that we can use directly
+  #
+  #   It is critical to ensure, that we handle them correctly. The result type should be:
+  #     - rd_kafka_error_t - :pointer
+  #     - rd_kafka_resp_err_t - :int
   module Bindings
     extend FFI::Library
 
@@ -91,7 +100,7 @@ module Rdkafka
     end
 
     attach_function :rd_kafka_topic_partition_list_new, [:int32], :pointer
-    attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :void
+    attach_function :rd_kafka_topic_partition_list_add, [:pointer, :string, :int32], :pointer
     attach_function :rd_kafka_topic_partition_list_set_offset, [:pointer, :string, :int32, :int64], :void
     attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
     attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
@@ -147,6 +156,10 @@ module Rdkafka
 
     attach_function :rd_kafka_err2name, [:int], :string
     attach_function :rd_kafka_err2str, [:int], :string
+    attach_function :rd_kafka_error_is_fatal, [:pointer], :int
+    attach_function :rd_kafka_error_is_retriable, [:pointer], :int
+    attach_function :rd_kafka_error_txn_requires_abort, [:pointer], :int
+    attach_function :rd_kafka_error_destroy, [:pointer], :void
     attach_function :rd_kafka_get_err_descs, [:pointer, :pointer], :void
 
     # Configuration

data/lib/rdkafka/helpers/oauth.rb

@@ -12,12 +12,18 @@ module Rdkafka
       # @return [Integer] 0 on success
       def oauthbearer_set_token(token:, lifetime_ms:, principal_name:, extensions: nil)
         error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+        extensions_ptr, extensions_str_ptrs = map_extensions(extensions)
 
-        response = @native_kafka.with_inner do |inner|
-          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
-            inner, token, lifetime_ms, principal_name,
-            flatten_extensions(extensions), extension_size(extensions), error_buffer, 256
-          )
+        begin
+          response = @native_kafka.with_inner do |inner|
+            Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
+              inner, token, lifetime_ms, principal_name,
+              extensions_ptr, extension_size(extensions), error_buffer, 256
+            )
+          end
+        ensure
+          extensions_str_ptrs&.each { |ptr| ptr.free }
+          extensions_ptr&.free
         end
 
         return response if response.zero?
@@ -41,10 +47,31 @@ module Rdkafka
 
       private
 
-      # Flatten the extensions hash into a string according to the spec, https://datatracker.ietf.org/doc/html/rfc7628#section-3.1
-      def flatten_extensions(extensions)
-        return nil unless extensions
-        "\x01#{extensions.map { |e| e.join("=") }.join("\x01")}"
+      # Convert extensions hash to FFI::MemoryPointer (const char **).
+      # Note: the returned pointers must be freed manually (autorelease = false).
+      def map_extensions(extensions)
+        return [nil, nil] if extensions.nil? || extensions.empty?
+
+        # https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
+
+        # The method argument is const char **
+        array_ptr = FFI::MemoryPointer.new(:pointer, extension_size(extensions))
+        array_ptr.autorelease = false
+        str_ptrs = []
+
+        # Element i is the key, i + 1 is the value.
+        extensions.each_with_index do |(k, v), i|
+          k_ptr = FFI::MemoryPointer.from_string(k.to_s)
+          k_ptr.autorelease = false
+          str_ptrs << k_ptr
+          v_ptr = FFI::MemoryPointer.from_string(v.to_s)
+          v_ptr.autorelease = false
+          str_ptrs << v_ptr
+          array_ptr[i * 2].put_pointer(0, k_ptr)
+          array_ptr[i * 2 + 1].put_pointer(0, v_ptr)
+        end
+
+        [array_ptr, str_ptrs]
       end
 
       # extension_size is the number of keys + values which should be a non-negative even number

data/lib/rdkafka/producer/delivery_report.rb

@@ -12,8 +12,8 @@ module Rdkafka
       # @return [Integer]
       attr_reader :offset
 
-      # The name of the topic this message was produced to or nil in case of reports with errors
-      # where topic was not reached.
+      # The name of the topic this message was produced to or nil in case delivery failed and we
+      #   we not able to get the topic reference
       #
       # @return [String, nil]
       attr_reader :topic_name

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.23.1"
-  LIBRDKAFKA_VERSION = "2.11.0"
-  LIBRDKAFKA_SOURCE_SHA256 = "592a823dc7c09ad4ded1bc8f700da6d4e0c88ffaf267815c6f25e7450b9395ca"
+  VERSION = "0.24.2"
+  LIBRDKAFKA_VERSION = "2.11.1"
+  LIBRDKAFKA_SOURCE_SHA256 = "a2c87186b081e2705bb7d5338d5a01bc88d43273619b372ccb7bb0d264d0ca9f"
 end

data/rdkafka.gemspec

@@ -67,7 +67,7 @@ Gem::Specification.new do |gem|
     gem.extensions = %w(ext/Rakefile)
   end
 
-  gem.add_dependency 'ffi', '~> 1.15'
+  gem.add_dependency 'ffi', '~> 1.17.1'
   gem.add_dependency 'json', '> 2.0'
   gem.add_dependency 'logger'
   gem.add_dependency 'mini_portile2', '~> 2.6'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.23.1
+  version: 0.24.2
 platform: ruby
 authors:
 - Thijs Cadier
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: 1.17.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: 1.17.1
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -177,7 +177,8 @@ files:
 - MIT-LICENSE
 - README.md
 - Rakefile
-- dist/librdkafka-2.11.0.tar.gz
+- bin/verify_kafka_warnings
+- dist/librdkafka-2.11.1.tar.gz
 - dist/patches/rdkafka_global_init.patch
 - docker-compose-ssl.yml
 - docker-compose.yml