karafka-rdkafka 0.12.4 → 0.13.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4e203bd92d9f622715bce381d382a2e7f503331922d3237cd770e9f386755b6
-  data.tar.gz: 721e511f93bfcc8ded44eeb6e368fb50e0cd42038702e4f3d51c4b022408a665
+  metadata.gz: 419dcef5e71a6578202fde7eacd628054ac77bae0c91c57845fd6dbd6ade6715
+  data.tar.gz: 05e7e1ef8e07c1b2dddf02f1e3f8e8e0da7dfcb75af41ccc2f9dcba2e30bba1f
 SHA512:
-  metadata.gz: 37182da237f5f469f94554e3b07247f102fa3e7b64a8a492975bc1c663687615d4da1ad955a6f894ed229be7ea405bb2575e15571ace94ece563952e36a40874
-  data.tar.gz: adf411222f06d391a95b661e5c3fe9006a5000d01f2be63d1d8cd238814c01d73ab31384617e9f5b484ff422a97b2da6d8eb9babc5eb501fd60f46079a803e21
+  metadata.gz: 4643afd749af09b9bd2c8d16ae25b989eb03feb39b12c966059956cad47966a8a74e6d0c356bfdd4115a3a419aebbab1d8421410e0bef64bd2f1611d66ba2ab6
+  data.tar.gz: 7f526e3bb375126862a84b7824e60872454366cfa7a2b5abb8d6e09f978423dc1e56e1c2a488f36f28ff622f942834b7d576d6fe91e4263cb5864165597c881f

data/.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
           GITHUB_COVERAGE: ${{matrix.coverage}}
 
         run: |
+          docker-compose up -d --no-recreate
           bundle install --path vendor/bundle
           cd ext && bundle exec rake && cd ..
-          docker-compose up -d --no-recreate
           bundle exec rspec

data/CHANGELOG.md

@@ -1,5 +1,22 @@
-## 0.12.4 (2024-07-10)
-- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
+# 0.13.0
+* Support cooperative sticky partition assignment in the rebalance callback (methodmissing)
+* Support both string and symbol header keys (ColinDKelley)
+* Handle tombstone messages properly (kgalieva)
+* Add topic name to delivery report (maeve)
+* Allow string partitioner config (mollyegibson)
+* Fix documented type for DeliveryReport#error (jimmydo)
+* Bump librdkafka to 2.0.2 (lmaia)
+* Use finalizers to cleanly exit producer and admin (thijsc)
+* Lock access to the native kafka client (thijsc)
+* Fix potential race condition in multi-threaded producer (mensfeld)
+* Fix leaking FFI resources in specs (mensfeld)
+* Improve specs stability (mensfeld)
+* Make metadata request timeout configurable (mensfeld)
+* call_on_partitions_assigned and call_on_partitions_revoked only get a tpl passed in (thijsc)
+* Support `#assignment_lost?` on a consumer to check for involuntary assignment revocation (mensfeld)
+* Expose `#name` on the consumer and producer (mensfeld)
+* Introduce producer partitions count metadata cache (mensfeld)
+* Retry metadta fetches on certain errors with a backoff (mensfeld)
 
 # 0.12.3
 - Include backtrace in non-raised binded errors.
@@ -14,6 +31,8 @@
 
 # 0.12.0
 * Bumps librdkafka to 1.9.0
+* Fix crash on empty partition key (mensfeld)
+* Pass the delivery handle to the callback (gvisokinskas)
 
 # 0.11.0
 * Upgrade librdkafka to 1.8.2

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gemspec

data/README.md

@@ -23,6 +23,19 @@ The most important pieces of a Kafka client are implemented. We're
 working towards feature completeness, you can track that here:
 https://github.com/appsignal/rdkafka-ruby/milestone/1
 
+## Table of content
+
+- [Installation](#installation)
+- [Usage](#usage)
+  * [Consuming messages](#consuming-messages)
+  * [Producing messages](#producing-messages)
+- [Higher level libraries](#higher-level-libraries)
+  * [Message processing frameworks](#message-processing-frameworks)
+  * [Message publishing libraries](#message-publishing-libraries)
+- [Development](#development)
+- [Example](#example)
+
+
 ## Installation
 
 This gem downloads and compiles librdkafka when it is installed. If you
@@ -77,6 +90,19 @@ Note that creating a producer consumes some resources that will not be
 released until it `#close` is explicitly called, so be sure to call
 `Config#producer` only as necessary.
 
+## Higher level libraries
+
+Currently, there are two actively developed frameworks based on rdkafka-ruby, that provide higher level API that can be used to work with Kafka messages and one library for publishing messages.
+
+### Message processing frameworks
+
+* [Karafka](https://github.com/karafka/karafka) - Ruby and Rails efficient Kafka processing framework.
+* [Racecar](https://github.com/zendesk/racecar) - A simple framework for Kafka consumers in Ruby 
+
+### Message publishing libraries
+
+* [WaterDrop](https://github.com/karafka/waterdrop) – Standalone Karafka library for producing Kafka messages.
+
 ## Development
 
 A Docker Compose file is included to run Kafka and Zookeeper. To run

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 # Rakefile
 
 require 'bundler/gem_tasks'

data/certs/cert_chain.pem

@@ -1,26 +1,26 @@
 -----BEGIN CERTIFICATE-----
 MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
 YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
-MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
+MB4XDTIyMDgxOTE3MjEzN1oXDTIzMDgxOTE3MjEzN1owPzEQMA4GA1UEAwwHY29u
 dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
-bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
-8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
-sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
-wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
-b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
-ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
-5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
-YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
-kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
-HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
-1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
-bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
-bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
-3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
-kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
-IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
-fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
-qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
-AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
-msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
+bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAODzeO3L6lxdATzMHKNW
+jFA/GGunoPuylO/BMzy8RiQHh7VIvysAKs0tHhTx3g2D0STDpF+hcQcPELFikiT2
+F+1wOHj/SsrK7VKqfA8+gq04hKc5sQoX2Egf9k3V0YJ3eZ6R/koHkQ8A0TVt0w6F
+ZQckoV4MqnEAx0g/FZN3mnHTlJ3VFLSBqJEIe+S6FZMl92mSv+hTrlUG8VaYxSfN
+lTCvnKk284F6QZq5XIENLRmcDd/3aPBLnLwNnyMyhB+6gK8cUO+CFlDO5tjo/aBA
+rUnl++wGG0JooF1ed0v+evOn9KoMBG6rHewcf79qJbVOscbD8qSAmo+sCXtcFryr
+KRMTB8gNbowJkFRJDEe8tfRy11u1fYzFg/qNO82FJd62rKAw2wN0C29yCeQOPRb1
+Cw9Y4ZwK9VFNEcV9L+3pHTHn2XfuZHtDaG198VweiF6raFO4yiEYccodH/USP0L5
+cbcCFtmu/4HDSxL1ByQXO84A0ybJuk3/+aPUSXe9C9U8fwIDAQABo3cwdTAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUSlcEakb7gfn/5E2WY6z73BF/
+iZkwHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
+bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEA1aS+E7RXJ1w9g9mJ
+G0NzFxe64OEuENosNlvYQCbRKGCXAU1qqelYkBQHseRgRKxLICrnypRo9IEobyHa
+vDnJ4r7Tsb34dleqQW2zY/obG+cia3Ym2JsegXWF7dDOzCXJ4FN8MFoT2jHlqLLw
+yrap0YO5zx0GSQ0Dwy8h2n2v2vanMEeCx7iNm3ERgR5WuN5sjzWoz2A/JLEEcK0C
+EnAGKCWAd1fuG8IemDjT1edsd5FyYR4bIX0m+99oDuFZyPiiIbalmyYiSBBp59Yb
+Q0P8zeBi4OfwCZNcxqz0KONmw9JLNv6DgyEAH5xe/4JzhMEgvIRiPj0pHfA7oqQF
+KUNqvD1KlxbEC+bZfE5IZhnqYLdld/Ksqd22FI1RBhiS1Ejfsj99LVIm9cBuZEY2
+Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
+MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
 -----END CERTIFICATE-----

data/certs/karafka-pro.pem

@@ -0,0 +1,11 @@
+-----BEGIN RSA PUBLIC KEY-----
+MIIBigKCAYEApcd6ybskiNs9WUvBGVUE8GdWDehjZ9TyjSj/fDl/UcMYqY0R5YX9
+tnYxEwZZRMdVltKWxr88Qmshh1IQz6CpJVbcfYjt/158pSGPm+AUua6tkLqIvZDM
+ocFOMafmroI+BMuL+Zu5QH7HC2tkT16jclGYfMQkJjXVUQTk2UZr+94+8RlUz/CH
+Y6hPA7xPgIyPfyPCxz1VWzAwXwT++NCJQPBr5MqT84LNSEzUSlR9pFNShf3UCUT+
+8LWOvjFSNGmMMSsbo2T7/+dz9/FM02YG00EO0x04qteggwcaEYLFrigDN6/fM0ih
+BXZILnMUqC/qrfW2YFg4ZqKZJuxaALqqkPxrkBDYqoqcAloqn36jBSke6tc/2I/J
+2Afq3r53UoAbUH7h5I/L8YeaiA4MYjAuq724lHlrOmIr4D6yjYC0a1LGlPjLk869
+2nsVXNgomhVb071E6amR+rJJnfvkdZgCmEBFnqnBV5A1u4qgNsa2rVcD+gJRvb2T
+aQtjlQWKPx5xAgMBAAE=
+-----END RSA PUBLIC KEY-----

data/ext/Rakefile

@@ -1,67 +1,40 @@
 # frozen_string_literal: true
 
 require File.expand_path('../../lib/rdkafka/version', __FILE__)
+require "mini_portile2"
 require "fileutils"
 require "open-uri"
 
 task :default => :clean do
-  # For nix users, nix can't locate the file paths because the packages it's requiring aren't managed by the system but are
-  # managed by nix itself, so using the normal file paths doesn't work for nix users.
-  #
-  # Mini_portile causes an issue because it's dependencies are downloaded on the fly and therefore don't exist/aren't
-  # accessible in the nix environment
-  if ENV.fetch('RDKAFKA_EXT_PATH', '').empty?
-    # Download and compile librdkafka if RDKAFKA_EXT_PATH is not set
-    require "mini_portile2"
-    recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
+  # Download and compile librdkafka
+  recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
-    # Use default homebrew openssl if we're on mac and the directory exists
-    # and each of flags is not empty
-    if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
-      ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
-      ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
-    end
-
-    releases = File.expand_path(File.join(File.dirname(__FILE__), '../dist'))
-
-    recipe.files << {
-      :url => "file://#{releases}/librdkafka_#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz",
-      :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
-    }
-    recipe.configure_options = ["--host=#{recipe.host}"]
-
-    # Disable using libc regex engine in favor of the embedded one
-    # The default regex engine of librdkafka does not always work exactly as most of the users
-    # would expect, hence this flag allows for changing it to the other one
-    if ENV.key?('RDKAFKA_DISABLE_REGEX_EXT')
-      recipe.configure_options << '--disable-regex-ext'
-    end
+  # Use default homebrew openssl if we're on mac and the directory exists
+  # and each of flags is not empty
+  if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
+    ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
+    ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
+  end
 
-    recipe.cook
-    # Move dynamic library we're interested in
-    if recipe.host.include?('darwin')
-      from_extension = '1.dylib'
-      to_extension   = 'dylib'
-    else
-      from_extension = 'so.1'
-      to_extension = 'so'
-    end
-    lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
-    FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
-    # Cleanup files created by miniportile we don't need in the gem
-    FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
-    FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
+  recipe.files << {
+    :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
+    :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
+  }
+  recipe.configure_options = ["--host=#{recipe.host}"]
+  recipe.cook
+  # Move dynamic library we're interested in
+  if recipe.host.include?('darwin')
+    from_extension = '1.dylib'
+    to_extension   = 'dylib'
   else
-    # Otherwise, copy existing libraries to ./ext
-    if ENV['RDKAFKA_EXT_PATH'].nil? || ENV['RDKAFKA_EXT_PATH'].empty?
-      raise "RDKAFKA_EXT_PATH must be set in your nix config when running under nix"
-    end
-    files = [
-      File.join(ENV['RDKAFKA_EXT_PATH'], 'lib', 'librdkafka.dylib'),
-      File.join(ENV['RDKAFKA_EXT_PATH'], 'lib', 'librdkafka.so')
-    ]
-    files.each { |ext| FileUtils.cp(ext, File.dirname(__FILE__)) if File.exist?(ext) }
+    from_extension = 'so.1'
+    to_extension = 'so'
   end
+  lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
+  FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
+  # Cleanup files created by miniportile we don't need in the gem
+  FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
+  FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
 end
 
 task :clean do

data/karafka-rdkafka.gemspec

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require File.expand_path('lib/rdkafka/version', __dir__)
 
 Gem::Specification.new do |gem|

data/lib/rdkafka/abstract_handle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "ffi"
 
 module Rdkafka

data/lib/rdkafka/admin/create_topic_handle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Admin
     class CreateTopicHandle < AbstractHandle

data/lib/rdkafka/admin/create_topic_report.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Admin
     class CreateTopicReport

data/lib/rdkafka/admin/delete_topic_handle.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Admin
     class DeleteTopicHandle < AbstractHandle

data/lib/rdkafka/admin/delete_topic_report.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rdkafka
   class Admin
     class DeleteTopicReport

data/lib/rdkafka/admin.rb

@@ -1,33 +1,31 @@
+# frozen_string_literal: true
+
+require "objspace"
+
 module Rdkafka
   class Admin
     # @private
     def initialize(native_kafka)
       @native_kafka = native_kafka
-      @closing = false
-
-      # Start thread to poll client for callbacks
-      @polling_thread = Thread.new do
-        loop do
-          Rdkafka::Bindings.rd_kafka_poll(@native_kafka, 250)
-          # Exit thread if closing and the poll queue is empty
-          if @closing && Rdkafka::Bindings.rd_kafka_outq_len(@native_kafka) == 0
-            break
-          end
-        end
-      end
-      @polling_thread.abort_on_exception = true
+
+      # Makes sure, that native kafka gets closed before it gets GCed by Ruby
+      ObjectSpace.define_finalizer(self, native_kafka.finalizer)
+    end
+
+    def finalizer
+      ->(_) { close }
     end
 
     # Close this admin instance
     def close
-      return unless @native_kafka
-
-      # Indicate to polling thread that we're closing
-      @closing = true
-      # Wait for the polling thread to finish up
-      @polling_thread.join
-      Rdkafka::Bindings.rd_kafka_destroy(@native_kafka)
-      @native_kafka = nil
+      return if closed?
+      ObjectSpace.undefine_finalizer(self)
+      @native_kafka.close
+    end
+
+    # Whether this admin has closed
+    def closed?
+      @native_kafka.closed?
     end
 
     # Create a topic with the given partition count and replication factor
@@ -38,6 +36,7 @@ module Rdkafka
     #
     # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
     def create_topic(topic_name, partition_count, replication_factor, topic_config={})
+      closed_admin_check(__method__)
 
       # Create a rd_kafka_NewTopic_t representing the new topic
       error_buffer = FFI::MemoryPointer.from_string(" " * 256)
@@ -68,7 +67,9 @@ module Rdkafka
       topics_array_ptr.write_array_of_pointer(pointer_array)
 
       # Get a pointer to the queue that our request will be enqueued on
-      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka)
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
       if queue_ptr.null?
         Rdkafka::Bindings.rd_kafka_NewTopic_destroy(new_topic_ptr)
         raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
@@ -79,17 +80,21 @@ module Rdkafka
       create_topic_handle[:pending] = true
       create_topic_handle[:response] = -1
       CreateTopicHandle.register(create_topic_handle)
-      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATETOPICS)
+      end
       Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_topic_handle.to_ptr)
 
       begin
-        Rdkafka::Bindings.rd_kafka_CreateTopics(
-            @native_kafka,
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_CreateTopics(
+            inner,
             topics_array_ptr,
             1,
             admin_options_ptr,
             queue_ptr
-        )
+          )
+        end
       rescue Exception
         CreateTopicHandle.remove(create_topic_handle.to_ptr.address)
         raise
@@ -108,6 +113,7 @@ module Rdkafka
     #
     # @return [DeleteTopicHandle] Delete topic handle that can be used to wait for the result of deleting the topic
     def delete_topic(topic_name)
+      closed_admin_check(__method__)
 
       # Create a rd_kafka_DeleteTopic_t representing the topic to be deleted
       delete_topic_ptr = Rdkafka::Bindings.rd_kafka_DeleteTopic_new(FFI::MemoryPointer.from_string(topic_name))
@@ -118,7 +124,9 @@ module Rdkafka
       topics_array_ptr.write_array_of_pointer(pointer_array)
 
       # Get a pointer to the queue that our request will be enqueued on
-      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka)
+      queue_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+      end
       if queue_ptr.null?
         Rdkafka::Bindings.rd_kafka_DeleteTopic_destroy(delete_topic_ptr)
         raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
@@ -129,17 +137,21 @@ module Rdkafka
       delete_topic_handle[:pending] = true
       delete_topic_handle[:response] = -1
       DeleteTopicHandle.register(delete_topic_handle)
-      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
+      admin_options_ptr = @native_kafka.with_inner do |inner|
+        Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_DELETETOPICS)
+      end
       Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, delete_topic_handle.to_ptr)
 
       begin
-        Rdkafka::Bindings.rd_kafka_DeleteTopics(
-            @native_kafka,
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_DeleteTopics(
+            inner,
             topics_array_ptr,
             1,
             admin_options_ptr,
             queue_ptr
-        )
+          )
+        end
       rescue Exception
         DeleteTopicHandle.remove(delete_topic_handle.to_ptr.address)
         raise
@@ -163,54 +175,64 @@ module Rdkafka
     #
     # @return [CreateTopicHandle] Create topic handle that can be used to wait for the result of creating the topic
     def create_partitions(topic_name, partition_count)
-      error_buffer = FFI::MemoryPointer.from_string(" " * 256)
-      new_partitions_ptr = Rdkafka::Bindings.rd_kafka_NewPartitions_new(
-        FFI::MemoryPointer.from_string(topic_name),
-        partition_count,
-        error_buffer,
-        256
-      )
-      if new_partitions_ptr.null?
-        raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
-      end
+      closed_admin_check(__method__)
+
+      @native_kafka.with_inner do |inner|
+        error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+        new_partitions_ptr = Rdkafka::Bindings.rd_kafka_NewPartitions_new(
+          FFI::MemoryPointer.from_string(topic_name),
+          partition_count,
+          error_buffer,
+          256
+        )
+        if new_partitions_ptr.null?
+          raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+        end
 
-      pointer_array = [new_partitions_ptr]
-      topics_array_ptr = FFI::MemoryPointer.new(:pointer)
-      topics_array_ptr.write_array_of_pointer(pointer_array)
+        pointer_array = [new_partitions_ptr]
+        topics_array_ptr = FFI::MemoryPointer.new(:pointer)
+        topics_array_ptr.write_array_of_pointer(pointer_array)
 
-      # Get a pointer to the queue that our request will be enqueued on
-      queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(@native_kafka)
-      if queue_ptr.null?
-        Rdkafka::Bindings.rd_kafka_NewPartitions_destroy(new_partitions_ptr)
-        raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
-      end
+        # Get a pointer to the queue that our request will be enqueued on
+        queue_ptr = Rdkafka::Bindings.rd_kafka_queue_get_background(inner)
+        if queue_ptr.null?
+          Rdkafka::Bindings.rd_kafka_NewPartitions_destroy(new_partitions_ptr)
+          raise Rdkafka::Config::ConfigError.new("rd_kafka_queue_get_background was NULL")
+        end
 
-      # Create and register the handle we will return to the caller
-      create_partitions_handle = CreatePartitionsHandle.new
-      create_partitions_handle[:pending] = true
-      create_partitions_handle[:response] = -1
-      CreatePartitionsHandle.register(create_partitions_handle)
-      admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(@native_kafka, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS)
-      Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_partitions_handle.to_ptr)
+        # Create and register the handle we will return to the caller
+        create_partitions_handle = CreatePartitionsHandle.new
+        create_partitions_handle[:pending] = true
+        create_partitions_handle[:response] = -1
+        CreatePartitionsHandle.register(create_partitions_handle)
+        admin_options_ptr = Rdkafka::Bindings.rd_kafka_AdminOptions_new(inner, Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS)
+        Rdkafka::Bindings.rd_kafka_AdminOptions_set_opaque(admin_options_ptr, create_partitions_handle.to_ptr)
+
+        begin
+          Rdkafka::Bindings.rd_kafka_CreatePartitions(
+            inner,
+            topics_array_ptr,
+            1,
+            admin_options_ptr,
+            queue_ptr
+          )
+        rescue Exception
+          CreatePartitionsHandle.remove(create_partitions_handle.to_ptr.address)
+          raise
+        ensure
+          Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
+          Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
+          Rdkafka::Bindings.rd_kafka_NewPartitions_destroy(new_partitions_ptr)
+        end
 
-      begin
-        Rdkafka::Bindings.rd_kafka_CreatePartitions(
-          @native_kafka,
-          topics_array_ptr,
-          1,
-          admin_options_ptr,
-          queue_ptr
-        )
-      rescue Exception
-        CreatePartitionsHandle.remove(create_partitions_handle.to_ptr.address)
-        raise
-      ensure
-        Rdkafka::Bindings.rd_kafka_AdminOptions_destroy(admin_options_ptr)
-        Rdkafka::Bindings.rd_kafka_queue_destroy(queue_ptr)
-        Rdkafka::Bindings.rd_kafka_NewPartitions_destroy(new_partitions_ptr)
+        create_partitions_handle
       end
+    end
+
+    private
 
-      create_partitions_handle
+    def closed_admin_check(method)
+      raise Rdkafka::ClosedAdminError.new(method) if closed?
     end
   end
 end