rdkafka 0.12.0.beta.4 → 0.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e539d59efa2f340f4e746eb56458a2ec7ccc4b3772339c92168b49afa6eefe3
-  data.tar.gz: 88b22c82f5580adf63657384afde3d395ea72ad7d35a8f98aa4f55ba57ed9fd1
+  metadata.gz: a75568ae9eddc2f80921c43835725e535ddc0c72d5ff931018d08dc8737a6c12
+  data.tar.gz: 7f65353380913d15603728f3fb3c4355ac87b46f380b762ad40a534f981df1d2
 SHA512:
-  metadata.gz: 4bfbed165c2df7b716c93014c5f20df64aa3177e7c5799603a4315e823dfea935f27a58875eb2453b8b87163ab32dcae26e2aef86640e8ebf74fe5b1b13adae2
-  data.tar.gz: 25cbc82bbfa3b92070c4d33dd9105ff6b73892345437077127e63027855b78ecba80ca1e4be133175cb1efa720d1547f1991523e7f71b1a863dbf0d28356720a
+  metadata.gz: b802ae224948f5e51c83025cf5a8dd937058c34a336d10f2d4cc3f8dfcebfdb8cdcfc73d92dab169f1f4fac412e1ce424412584f7398af65371bcbb6e271f0e5
+  data.tar.gz: c668c6dfa9f7aadd35614c36cb9ad259b1e8519b949f32116f306e56d61719292a3e49bced751d1abcf7065fab713694756cec8777236b8f61c615dc9745e2e0

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 0.12.1 (2024-07-11)
+- [Fix] Switch to local release of librdkafka to mitigate its unavailability.
+
 # 0.12.0
 * Bumps librdkafka to 1.9.0
 

data/ext/Rakefile

@@ -1,38 +1,67 @@
+# frozen_string_literal: true
+
 require File.expand_path('../../lib/rdkafka/version', __FILE__)
-require "mini_portile2"
 require "fileutils"
 require "open-uri"
 
 task :default => :clean do
-  # Download and compile librdkafka
-  recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
-
-  # Use default homebrew openssl if we're on mac and the directory exists
-  # and each of flags is not empty
-  if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
-    ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
-    ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
-  end
+  # For nix users, nix can't locate the file paths because the packages it's requiring aren't managed by the system but are
+  # managed by nix itself, so using the normal file paths doesn't work for nix users.
+  #
+  # Mini_portile causes an issue because it's dependencies are downloaded on the fly and therefore don't exist/aren't
+  # accessible in the nix environment
+  if ENV.fetch('RDKAFKA_EXT_PATH', '').empty?
+    # Download and compile librdkafka if RDKAFKA_EXT_PATH is not set
+    require "mini_portile2"
+    recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
+
+    # Use default homebrew openssl if we're on mac and the directory exists
+    # and each of flags is not empty
+    if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
+      ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
+      ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
+    end
+
+    releases = File.expand_path(File.join(File.dirname(__FILE__), '../dist'))
+
+    recipe.files << {
+      :url => "file://#{releases}/librdkafka_#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz",
+      :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
+    }
+    recipe.configure_options = ["--host=#{recipe.host}"]
 
-  recipe.files << {
-    :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
-    :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
-  }
-  recipe.configure_options = ["--host=#{recipe.host}"]
-  recipe.cook
-  # Move dynamic library we're interested in
-  if recipe.host.include?('darwin')
-    from_extension = '1.dylib'
-    to_extension   = 'dylib'
+    # Disable using libc regex engine in favor of the embedded one
+    # The default regex engine of librdkafka does not always work exactly as most of the users
+    # would expect, hence this flag allows for changing it to the other one
+    if ENV.key?('RDKAFKA_DISABLE_REGEX_EXT')
+      recipe.configure_options << '--disable-regex-ext'
+    end
+
+    recipe.cook
+    # Move dynamic library we're interested in
+    if recipe.host.include?('darwin')
+      from_extension = '1.dylib'
+      to_extension   = 'dylib'
+    else
+      from_extension = 'so.1'
+      to_extension = 'so'
+    end
+    lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
+    FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
+    # Cleanup files created by miniportile we don't need in the gem
+    FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
+    FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
   else
-    from_extension = 'so.1'
-    to_extension = 'so'
+    # Otherwise, copy existing libraries to ./ext
+    if ENV['RDKAFKA_EXT_PATH'].nil? || ENV['RDKAFKA_EXT_PATH'].empty?
+      raise "RDKAFKA_EXT_PATH must be set in your nix config when running under nix"
+    end
+    files = [
+      File.join(ENV['RDKAFKA_EXT_PATH'], 'lib', 'librdkafka.dylib'),
+      File.join(ENV['RDKAFKA_EXT_PATH'], 'lib', 'librdkafka.so')
+    ]
+    files.each { |ext| FileUtils.cp(ext, File.dirname(__FILE__)) if File.exist?(ext) }
   end
-  lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
-  FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
-  # Cleanup files created by miniportile we don't need in the gem
-  FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
-  FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
 end
 
 task :clean do

data/lib/rdkafka/callbacks.rb

@@ -97,7 +97,7 @@ module Rdkafka
           delivery_handle[:pending] = false
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
-            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]))
+            opaque.call_delivery_callback(Rdkafka::Producer::DeliveryReport.new(message[:partition], message[:offset], message[:err]), delivery_handle)
           end
         end
       end

data/lib/rdkafka/config.rb

@@ -278,8 +278,8 @@ module Rdkafka
     attr_accessor :producer
     attr_accessor :consumer_rebalance_listener
 
-    def call_delivery_callback(delivery_handle)
-      producer.call_delivery_callback(delivery_handle) if producer
+    def call_delivery_callback(delivery_report, delivery_handle)
+      producer.call_delivery_callback(delivery_report, delivery_handle) if producer
     end
 
     def call_on_partitions_assigned(consumer, list)

data/lib/rdkafka/producer.rb

@@ -9,6 +9,12 @@ module Rdkafka
     # @return [Proc, nil]
     attr_reader :delivery_callback
 
+    # @private
+    # Returns the number of arguments accepted by the callback, by default this is nil.
+    #
+    # @return [Integer, nil]
+    attr_reader :delivery_callback_arity
+
     # @private
     def initialize(client, partitioner_name)
       @client = client
@@ -19,7 +25,7 @@ module Rdkafka
     end
 
     # Set a callback that will be called every time a message is successfully produced.
-    # The callback is called with a {DeliveryReport}
+    # The callback is called with a {DeliveryReport} and {DeliveryHandle}
     #
     # @param callback [Proc, #call] The callback
     #
@@ -27,6 +33,7 @@ module Rdkafka
     def delivery_callback=(callback)
       raise TypeError.new("Callback has to be callable") unless callback.respond_to?(:call)
       @delivery_callback = callback
+      @delivery_callback_arity = arity(callback)
     end
 
     # Close this producer and wait for the internal poll queue to empty.
@@ -151,8 +158,17 @@ module Rdkafka
     end
 
     # @private
-    def call_delivery_callback(delivery_handle)
-      @delivery_callback.call(delivery_handle) if @delivery_callback
+    def call_delivery_callback(delivery_report, delivery_handle)
+      return unless @delivery_callback
+
+      args = [delivery_report, delivery_handle].take(@delivery_callback_arity)
+      @delivery_callback.call(*args)
+    end
+
+    def arity(callback)
+      return callback.arity if callback.respond_to?(:arity)
+
+      callback.method(:call).arity
     end
 
     def closed_producer_check(method)

data/lib/rdkafka/version.rb

@@ -1,5 +1,5 @@
 module Rdkafka
-  VERSION = "0.12.0.beta.4"
-  LIBRDKAFKA_VERSION = "1.9.0-RC10"
-  LIBRDKAFKA_SOURCE_SHA256 = "e8d49dcd6aafc169b1a75ed3553a338c4b9cf7821d365bac3c6e31d9a4aa9296"
+  VERSION = "0.12.1"
+  LIBRDKAFKA_VERSION = "1.9.0"
+  LIBRDKAFKA_SOURCE_SHA256 = "59b6088b69ca6cf278c3f9de5cd6b7f3fd604212cd1c59870bc531c54147e889"
 end

data/spec/rdkafka/producer_spec.rb

@@ -49,6 +49,27 @@ describe Rdkafka::Producer do
         # Callback should have been called
         expect(@callback_called).to be true
       end
+
+      it "should provide handle" do
+        @callback_handle = nil
+
+        producer.delivery_callback = lambda { |_, handle| @callback_handle = handle }
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
+
+        # Join the producer thread.
+        producer.close
+
+        expect(handle).to be @callback_handle
+      end
     end
 
     context "with a callable object" do
@@ -93,6 +114,36 @@ describe Rdkafka::Producer do
         expect(called_report.first.partition).to eq 1
         expect(called_report.first.offset).to be >= 0
       end
+
+      it "should provide handle" do
+        callback_handles = []
+        callback = Class.new do
+          def initialize(callback_handles)
+            @callback_handles = callback_handles
+          end
+
+          def call(_, handle)
+            @callback_handles << handle
+          end
+        end
+        producer.delivery_callback = callback.new(callback_handles)
+
+        # Produce a message
+        handle = producer.produce(
+          topic:   "produce_test_topic",
+          payload: "payload",
+          key:     "key"
+        )
+
+        # Wait for it to be delivered
+        handle.wait(max_wait_timeout: 15)
+
+        # Join the producer thread.
+        producer.close
+
+        # Callback should have been called
+        expect(handle).to be callback_handles.first
+      end
     end
 
     it "should not accept a callback that's not callable" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rdkafka
 version: !ruby/object:Gem::Version
-  version: 0.12.0.beta.4
+  version: 0.12.1
 platform: ruby
 authors:
 - Thijs Cadier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-05-17 00:00:00.000000000 Z
+date: 2024-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi
@@ -156,6 +156,7 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- dist/librdkafka_1.9.0.tar.gz
 - docker-compose.yml
 - ext/README.md
 - ext/Rakefile
@@ -217,11 +218,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.5.14
 signing_key: 
 specification_version: 4
 summary: The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka.