karafka-rdkafka 0.13.6 → 0.13.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4da2961103e92a94c36f605359cc5380ffa6d740cdc667329797ce9345caa663
-  data.tar.gz: 419aff211c5bfe35c244549fa14e042e92a39ad8583af2b8844283b7f1dca5b0
+  metadata.gz: 4fe6b47d334265ef793c32b215dca1d97adfa42a51215b9492f1d36b58a84403
+  data.tar.gz: 4fc125147d796f981314640779adf5234376b0d0357c0321683002820fcb3301
 SHA512:
-  metadata.gz: c0b0d1069163ece1dcf5734ebc80e952a86c3eccc96885f856dc3f4569a0aaddc12db28fda7927d4c360b6f2ba0078a782f7ee6582554159c0e0b80c975e03bf
-  data.tar.gz: 2e3ba6a6ef850bafa305f2d9f2a9d3b4268fedbafc50f2b5ce2fedffcb4bb018b8a1ed5b1ce1b3a1efbad51621a2ad74d448641774d93879da628c06d4a1260c
+  metadata.gz: f7fc5d7dc99653f117b4bef3baa0d6e772537f4a7df6375a6a33692d4a9728f81308e11c3cd873fbb5686fb83cf2021a5866eb252e10661b3fe512639e3e3349
+  data.tar.gz: 9d23217c18093187f1fecc5afbbdbe05d2a6be3cb99adb0dc20ed7458da19c7035cc51eda664e09c3acaa5ac3dd3539b0c58c91cd6d6faed3c1d21edc9662dde

data/.github/workflows/ci.yml

@@ -1,6 +1,8 @@
 name: ci
 
-concurrency: ci-${{ github.ref }}
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 on:
   pull_request:
@@ -28,12 +30,11 @@ jobs:
           - '3.0.0'
           - '2.7'
           - '2.7.0'
-          - '2.6.8'
         include:
           - ruby: '3.2'
             coverage: 'true'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
 

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.13.7 (Unreleased)
+- [Change] Drop support for Ruby 2.6 due to incompatibilities in usage of `ObjectSpace::WeakMap`
+- [Fix] Fix dangling Opaque references.
+
 # 0.13.6 (2023-10-17)
 * **[Feature]** Support transactions API in the producer
 * [Enhancement] Add `raise_response_error` flag to the `Rdkafka::AbstractHandle`.

data/README.md

@@ -1,8 +1,13 @@
 # Rdkafka
 
-[![Build Status](https://appsignal.semaphoreci.com/badges/rdkafka-ruby/branches/master.svg?style=shields)](https://appsignal.semaphoreci.com/projects/rdkafka-ruby)
+[![Build Status](https://github.com/karafka/rdkafka-ruby/actions/workflows/ci.yml/badge.svg)](https://github.com/karafka/rdkafka-ruby/actions/workflows/ci.yml)
 [![Gem Version](https://badge.fury.io/rb/rdkafka.svg)](https://badge.fury.io/rb/rdkafka)
-[![Maintainability](https://api.codeclimate.com/v1/badges/ecb1765f81571cccdb0e/maintainability)](https://codeclimate.com/github/appsignal/rdkafka-ruby/maintainability)
+[![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
+
+> [!NOTE]
+> The `rdkafka-ruby` gem was created and developed by [AppSignal](https://www.appsignal.com/). Their impactful contributions have significantly shaped the Ruby Kafka and Karafka ecosystems. For robust monitoring, we highly recommend AppSignal.
+
+---
 
 The `rdkafka` gem is a modern Kafka client library for Ruby based on
 [librdkafka](https://github.com/edenhill/librdkafka/).
@@ -11,13 +16,7 @@ gem and targets Kafka 1.0+ and Ruby versions that are under security or
 active maintenance. We remove Ruby version from our CI builds if they 
 become EOL.
 
-`rdkafka` was written because we needed a reliable Ruby client for
-Kafka that supports modern Kafka at [AppSignal](https://appsignal.com).
-We run it in production on very high traffic systems.
-
-This gem only provides a high-level Kafka consumer. If you are running
-an older version of Kafka and/or need the legacy simple consumer we
-suggest using the [Hermann](https://github.com/reiseburo/hermann) gem.
+`rdkafka` was written because of the need for a reliable Ruby client for Kafka that supports modern Kafka at [AppSignal](https://appsignal.com). AppSignal runs it in production on very high-traffic systems.
 
 The most important pieces of a Kafka client are implemented. We're
 working towards feature completeness, you can track that here:

data/docker-compose.yml

@@ -3,7 +3,7 @@ version: '2'
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.5.0
+    image: confluentinc/cp-kafka:7.5.1
 
     ports:
       - 9092:9092

data/karafka-rdkafka.gemspec

@@ -4,7 +4,7 @@ require File.expand_path('lib/rdkafka/version', __dir__)
 
 Gem::Specification.new do |gem|
   gem.authors = ['Thijs Cadier']
-  gem.email = ["thijs@appsignal.com"]
+  gem.email = ["contact@karafka.io"]
   gem.description = "Modern Kafka client library for Ruby based on librdkafka"
   gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.4+."
   gem.license = 'MIT'
@@ -15,7 +15,7 @@ Gem::Specification.new do |gem|
   gem.name = 'karafka-rdkafka'
   gem.require_paths = ['lib']
   gem.version = Rdkafka::VERSION
-  gem.required_ruby_version = '>= 2.6'
+  gem.required_ruby_version = '>= 2.7'
   gem.extensions = %w(ext/Rakefile)
   gem.cert_chain = %w[certs/cert_chain.pem]
 

data/lib/rdkafka/config.rb

@@ -14,7 +14,7 @@ module Rdkafka
     # @private
     @@error_callback = nil
     # @private
-    @@opaques = {}
+    @@opaques = ObjectSpace::WeakMap.new
     # @private
     @@log_queue = Queue.new
 
@@ -164,7 +164,13 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_poll_set_consumer(kafka)
 
       # Return consumer with Kafka client
-      Rdkafka::Consumer.new(Rdkafka::NativeKafka.new(kafka, run_polling_thread: false))
+      Rdkafka::Consumer.new(
+        Rdkafka::NativeKafka.new(
+          kafka,
+          run_polling_thread: false,
+          opaque: opaque
+        )
+      )
     end
 
     # Create a producer with this configuration.
@@ -182,7 +188,14 @@ module Rdkafka
       Rdkafka::Bindings.rd_kafka_conf_set_dr_msg_cb(config, Rdkafka::Callbacks::DeliveryCallbackFunction)
       # Return producer with Kafka client
       partitioner_name = self[:partitioner] || self["partitioner"]
-      Rdkafka::Producer.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true), partitioner_name).tap do |producer|
+      Rdkafka::Producer.new(
+        Rdkafka::NativeKafka.new(
+          native_kafka(config, :rd_kafka_producer),
+          run_polling_thread: true,
+          opaque: opaque
+        ),
+        partitioner_name
+      ).tap do |producer|
         opaque.producer = producer
       end
     end
@@ -197,7 +210,13 @@ module Rdkafka
       opaque = Opaque.new
       config = native_config(opaque)
       Rdkafka::Bindings.rd_kafka_conf_set_background_event_cb(config, Rdkafka::Callbacks::BackgroundEventCallbackFunction)
-      Rdkafka::Admin.new(Rdkafka::NativeKafka.new(native_kafka(config, :rd_kafka_producer), run_polling_thread: true))
+      Rdkafka::Admin.new(
+        Rdkafka::NativeKafka.new(
+          native_kafka(config, :rd_kafka_producer),
+          run_polling_thread: true,
+          opaque: opaque
+        )
+      )
     end
 
     # Error that is returned by the underlying rdkafka error if an invalid configuration option is present.

data/lib/rdkafka/native_kafka.rb

@@ -4,8 +4,9 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:)
+    def initialize(inner, run_polling_thread:, opaque:)
       @inner = inner
+      @opaque = opaque
       # Lock around external access
       @access_mutex = Mutex.new
       # Lock around internal polling
@@ -112,6 +113,7 @@ module Rdkafka
 
         Rdkafka::Bindings.rd_kafka_destroy(@inner)
         @inner = nil
+        @opaque = nil
       end
     end
   end

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.13.6"
+  VERSION = "0.13.7"
   LIBRDKAFKA_VERSION = "2.2.0"
   LIBRDKAFKA_SOURCE_SHA256 = "af9a820cbecbc64115629471df7c7cecd40403b6c34bfdbb9223152677a47226"
 end

data/renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

data/spec/rdkafka/native_kafka_spec.rb

@@ -7,8 +7,9 @@ describe Rdkafka::NativeKafka do
   let(:native) { config.send(:native_kafka, config.send(:native_config), :rd_kafka_producer) }
   let(:closing) { false }
   let(:thread) { double(Thread) }
+  let(:opaque) { Rdkafka::Opaque.new }
 
-  subject(:client) { described_class.new(native, run_polling_thread: true) }
+  subject(:client) { described_class.new(native, run_polling_thread: true, opaque: opaque) }
 
   before do
     allow(Thread).to receive(:new).and_return(thread)

data/spec/rdkafka/producer_spec.rb

@@ -844,19 +844,25 @@ describe Rdkafka::Producer do
         ).producer
       end
 
-      it 'expect to allow to produce within a transaction, finalize and ship data' do
+      it 'expect to allow to produce within a transaction, finalize and not ship data' do
         producer.init_transactions
         producer.begin_transaction
 
+        sleep(5)
+
         handle1 = producer.produce(topic: 'produce_test_topic', payload: 'data1', partition: 1)
         handle2 = producer.produce(topic: 'example_topic', payload: 'data2', partition: 0)
 
         begin
-          producer.commit_transaction
+          producer.commit_transaction(15_000)
         rescue Rdkafka::RdkafkaError => e
           next unless e.abortable?
 
-          producer.abort_transaction
+          begin
+            producer.abort_transaction(15_000)
+          rescue Rdkafka::RdkafkaError => e
+            nil
+          end
 
           expect { handle1.wait(max_wait_timeout: 15) }.to raise_error(Rdkafka::RdkafkaError)
           expect { handle2.wait(max_wait_timeout: 15) }.to raise_error(Rdkafka::RdkafkaError)

data/spec/spec_helper.rb

@@ -107,6 +107,10 @@ def wait_for_unassignment(consumer)
   end
 end
 
+def objects_of_type_count(type)
+  ObjectSpace.each_object(type).count
+end
+
 def notify_listener(listener, &block)
   # 1. subscribe and poll
   consumer.subscribe("consume_test_topic")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.13.6
+  version: 0.13.7
 platform: ruby
 authors:
 - Thijs Cadier
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-10-17 00:00:00.000000000 Z
+date: 2023-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -165,7 +165,7 @@ dependencies:
         version: '0'
 description: Modern Kafka client library for Ruby based on librdkafka
 email:
-- thijs@appsignal.com
+- contact@karafka.io
 executables: []
 extensions:
 - ext/Rakefile
@@ -182,7 +182,6 @@ files:
 - README.md
 - Rakefile
 - certs/cert_chain.pem
-- certs/karafka-pro.pem
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
@@ -211,6 +210,7 @@ files:
 - lib/rdkafka/producer/delivery_handle.rb
 - lib/rdkafka/producer/delivery_report.rb
 - lib/rdkafka/version.rb
+- renovate.json
 - spec/rdkafka/abstract_handle_spec.rb
 - spec/rdkafka/admin/create_topic_handle_spec.rb
 - spec/rdkafka/admin/create_topic_report_spec.rb
@@ -251,7 +251,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/certs/karafka-pro.pem

@@ -1,11 +0,0 @@
------BEGIN RSA PUBLIC KEY-----
-MIIBigKCAYEApcd6ybskiNs9WUvBGVUE8GdWDehjZ9TyjSj/fDl/UcMYqY0R5YX9
-tnYxEwZZRMdVltKWxr88Qmshh1IQz6CpJVbcfYjt/158pSGPm+AUua6tkLqIvZDM
-ocFOMafmroI+BMuL+Zu5QH7HC2tkT16jclGYfMQkJjXVUQTk2UZr+94+8RlUz/CH
-Y6hPA7xPgIyPfyPCxz1VWzAwXwT++NCJQPBr5MqT84LNSEzUSlR9pFNShf3UCUT+
-8LWOvjFSNGmMMSsbo2T7/+dz9/FM02YG00EO0x04qteggwcaEYLFrigDN6/fM0ih
-BXZILnMUqC/qrfW2YFg4ZqKZJuxaALqqkPxrkBDYqoqcAloqn36jBSke6tc/2I/J
-2Afq3r53UoAbUH7h5I/L8YeaiA4MYjAuq724lHlrOmIr4D6yjYC0a1LGlPjLk869
-2nsVXNgomhVb071E6amR+rJJnfvkdZgCmEBFnqnBV5A1u4qgNsa2rVcD+gJRvb2T
-aQtjlQWKPx5xAgMBAAE=
------END RSA PUBLIC KEY-----