karafka-rdkafka 0.17.5 → 0.18.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f06207b891f56e8feded0ccb4e69302f0eff0418a40603f4493e6b5dbac5d2c5
-  data.tar.gz: f09a9ddb7fa7f32dcfdae11ff6244f631afe2af2ce0cde256c6ec098b1cd8061
+  metadata.gz: 6e328f121d4a410ea1db50fd3661217757f2a772c3060f94abb4f3278ae59558
+  data.tar.gz: cb2aeb8e915c5fde19e73c74f91725bf04810a1afd211b9c418c57c936a2f3a7
 SHA512:
-  metadata.gz: 00f295b856ec4ecc8ab54b805b8a49c73d213d064952b53930747386ab3641f46de1bcb437c1e5a55a32fdadf44bafae8c0a2a67635465b27cf2c3d811d7fa95
-  data.tar.gz: 1faec6ed0b725245aab1077885a73df3658bda891ef0b11f93012c093d93f6fa59e2126c3ef30f23fe5a9e496bd38847c5994d677d45d4877f3cc623baa71829
+  metadata.gz: a8217261e8118bf53d80e026fd30f8e76013c58babc4783e9f40fbfb706b0991a8ebc9fec94e86c67cf378e35cb57fa8ffa0a9163a87ca8e7b1a40326d24f75a
+  data.tar.gz: 20dc090cb6b6934a94497e06041c9fbfdeec1229720bade9f21bfa457bd53d9840a903a968c77d713d40796c048980ae8c6f94bfda1f45f26842681c6a800003

data/.github/workflows/ci.yml

@@ -22,11 +22,10 @@ jobs:
       fail-fast: false
       matrix:
         ruby:
-          - '3.4.0-preview1'
+          - '3.4.0-preview2'
           - '3.3'
           - '3.2'
           - '3.1'
-          - '3.0'
         include:
           - ruby: '3.3'
             coverage: 'true'
@@ -50,6 +49,8 @@ jobs:
           GITHUB_COVERAGE: ${{matrix.coverage}}
 
         run: |
+          set -e
           bundle install --path vendor/bundle
-          cd ext && bundle exec rake && cd ..
+          cd ext && bundle exec rake
+          cd ..
           bundle exec rspec

data/.ruby-version

@@ -1 +1 @@
-3.3.5
+3.3.6

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Rdkafka Changelog
 
+## 0.18.0 (2024-11-26)
+- **[Breaking]** Drop Ruby 3.0 support
+- [Enhancement] Bump librdkafka to 2.6.1
+- [Enhancement] Use default oauth callback if none is passed (bachmanity1)
+- [Enhancement] Expose `rd_kafka_global_init` to mitigate macos forking issues.
+- [Patch] Retire no longer needed cooperative-sticky patch.
+
+## 0.17.6 (2024-09-03)
+- [Fix] Fix incorrectly behaving CI on failures. 
+- [Fix] Fix invalid patches librdkafka references.
+
 ## 0.17.5 (2024-09-03)
 - [Patch] Patch with "Add forward declaration to fix compilation without ssl" fix
 

data/README.md

@@ -163,6 +163,7 @@ bundle exec rake produce_messages
 
 | rdkafka-ruby | librdkafka | patches |
 |-|-|-|
+| 0.18.0 (2024-11-26) | 2.6.1 (2024-11-18) | yes |
 | 0.17.4 (2024-09-02) | 2.5.3 (2024-09-02) | yes |
 | 0.17.0 (2024-08-01) | 2.5.0 (2024-07-10) | yes |
 | 0.16.0 (2024-06-13) | 2.4.0 (2024-05-07) | no  |

data/dist/patches/rdkafka_global_init.patch

@@ -0,0 +1,15 @@
+# This patch is released under the 2-clause BSD license, same as librdkafka
+#
+--- librdkafka_2.5.3/src/rdkafka.h
++++ librdkafka_2.5.3/src/rdkafka.h
+@@ -1101,6 +1101,10 @@ rd_kafka_topic_partition_t *rd_kafka_topic_partition_list_find(
+     const char *topic,
+     int32_t partition);
+ 
++/**
++ * @brief Allows for the global init of rdkafka when required in the Ruby process.
++ */
++RD_EXPORT void rd_kafka_global_init(void);
+ 
+ /**
+  * @brief Sort list using comparator \p cmp.

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.7.0
+    image: confluentinc/cp-kafka:7.7.1
 
     ports:
       - 9092:9092

data/ext/Rakefile

@@ -26,7 +26,7 @@ task :default => :clean do
     releases = File.expand_path(File.join(File.dirname(__FILE__), '../dist'))
 
     recipe.files << {
-      :url => "file://#{releases}/librdkafka_#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz",
+      :url => "file://#{releases}/librdkafka-#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz",
       :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
     }
     recipe.configure_options = ["--host=#{recipe.host}"]

data/karafka-rdkafka.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.name = 'karafka-rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 3.0'
+  gem.required_ruby_version = '>= 3.1'
   gem.extensions = %w(ext/Rakefile)
   gem.cert_chain = %w[certs/cert.pem]
 

data/lib/rdkafka/bindings.rb

@@ -39,6 +39,11 @@ module Rdkafka
       layout :value, :size_t
     end
 
+    # This function comes from our patch on top of librdkafka. It allows os to load all the
+    # librdkafka components without initializing the client
+    # @see https://github.com/confluentinc/librdkafka/issues/4590
+    attach_function :rd_kafka_global_init, [], :void
+
     # Polling
 
     attach_function :rd_kafka_flush, [:pointer, :int], :int, blocking: true

data/lib/rdkafka/config.rb

@@ -338,7 +338,9 @@ module Rdkafka
         Rdkafka::Bindings.rd_kafka_conf_set_error_cb(config, Rdkafka::Bindings::ErrorCallback)
 
         # Set oauth callback
-        Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
+        if Rdkafka::Config.oauthbearer_token_refresh_callback
+          Rdkafka::Bindings.rd_kafka_conf_set_oauthbearer_token_refresh_cb(config, Rdkafka::Bindings::OAuthbearerTokenRefreshCallback)
+        end
       end
     end
 

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.17.5"
-  LIBRDKAFKA_VERSION = "2.5.3"
-  LIBRDKAFKA_SOURCE_SHA256 = "eaa1213fdddf9c43e28834d9a832d9dd732377d35121e42f875966305f52b8ff"
+  VERSION = "0.18.0.rc1"
+  LIBRDKAFKA_VERSION = "2.6.1"
+  LIBRDKAFKA_SOURCE_SHA256 = "0ddf205ad8d36af0bc72a2fec20639ea02e1d583e353163bf7f4683d949e901b"
 end

data/lib/rdkafka.rb

@@ -49,3 +49,5 @@ require "rdkafka/producer/delivery_report"
 # Main Rdkafka namespace of this gem
 module Rdkafka
 end
+
+Rdkafka::Bindings.rd_kafka_global_init

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -114,12 +114,18 @@ describe Rdkafka::Consumer::TopicPartitionList do
   end
 
   describe "#to_s" do
+    let(:expected) do
+      if RUBY_VERSION >= '3.4.0'
+        "<TopicPartitionList: {\"topic1\" => [<Partition 0>, <Partition 1>]}>"
+      else
+        "<TopicPartitionList: {\"topic1\"=>[<Partition 0>, <Partition 1>]}>"
+      end
+    end
+
     it "should return a human readable representation" do
       list = Rdkafka::Consumer::TopicPartitionList.new
       list.add_topic("topic1", [0, 1])
 
-      expected = "<TopicPartitionList: {\"topic1\"=>[<Partition 0>, <Partition 1>]}>"
-
       expect(list.to_s).to eq expected
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.17.5
+  version: 0.18.0.rc1
 platform: ruby
 authors:
 - Thijs Cadier
@@ -36,7 +36,7 @@ cert_chain:
   i9zWxov0mr44TWegTVeypcWGd/0nxu1+QHVNHJrpqlPBRvwQsUm7fwmRInGpcaB8
   ap8wNYvryYzrzvzUxIVFBVM5PacgkFqRmolCa8I7tdKQN+R1
   -----END CERTIFICATE-----
-date: 2024-09-03 00:00:00.000000000 Z
+date: 2024-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -186,9 +186,8 @@ files:
 - README.md
 - Rakefile
 - certs/cert.pem
-- dist/librdkafka_2.5.3.tar.gz
-- dist/patches/rdkafka_op.h.patch
-- dist/patches/rdkafka_sticky_assignor.c.patch
+- dist/librdkafka-2.6.1.tar.gz
+- dist/patches/rdkafka_global_init.patch
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
@@ -281,14 +280,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.0'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.16
+rubygems_version: 3.5.22
 signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.

data/dist/patches/rdkafka_op.h.patch

@@ -1,11 +0,0 @@
-# ref https://github.com/confluentinc/librdkafka/pull/4794/files
---- librdkafka_2.5.0/src/rdkafka_op.h  2024-07-08 09:47:43.000000000 +0200
-+++ librdkafka_2.5.0/src/rdkafka_op.h  2024-07-30 09:44:38.529759640 +0200
-@@ -39,6 +39,7 @@
- typedef struct rd_kafka_q_s rd_kafka_q_t;
- typedef struct rd_kafka_toppar_s rd_kafka_toppar_t;
- typedef struct rd_kafka_op_s rd_kafka_op_t;
-+typedef struct rd_kafka_broker_s rd_kafka_broker_t;
- 
- /* One-off reply queue + reply version.
-  * All APIs that take a rd_kafka_replyq_t makes a copy of the

data/dist/patches/rdkafka_sticky_assignor.c.patch

@@ -1,26 +0,0 @@
-# This patch is released under the 2-clause BSD license, same as librdkafka
-# Fixes: https://github.com/confluentinc/librdkafka/issues/4783
-#
---- librdkafka_2.5.0/src/rdkafka_sticky_assignor.c	2024-07-08 09:47:43.000000000 +0200
-+++ librdkafka_2.5.0/src/rdkafka_sticky_assignor.c	2024-07-30 09:44:38.529759640 +0200
-@@ -769,7 +769,7 @@
-         const rd_kafka_topic_partition_list_t *partitions;
-         const char *consumer;
-         const rd_map_elem_t *elem;
--        int i;
-+        int i, j;
- 
-         /* The assignment is balanced if minimum and maximum numbers of
-          * partitions assigned to consumers differ by at most one. */
-@@ -836,9 +836,9 @@
- 
-                 /* Otherwise make sure it can't get any more partitions */
- 
--                for (i = 0; i < potentialTopicPartitions->cnt; i++) {
-+                for (j = 0; j < potentialTopicPartitions->cnt; j++) {
-                         const rd_kafka_topic_partition_t *partition =
--                            &potentialTopicPartitions->elems[i];
-+                            &potentialTopicPartitions->elems[j];
-                         const char *otherConsumer;
-                         int otherConsumerPartitionCount;
-