rdkafka 0.13.0 → 0.15.1

data/ext/README.md

@@ -5,7 +5,7 @@ this gem is installed.
 
 To update the `librdkafka` version follow the following steps:
 
-* Go to https://github.com/edenhill/librdkafka/releases to get the new
+* Go to https://github.com/confluentinc/librdkafka/releases to get the new
   version number and asset checksum for `tar.gz`.
 * Change the version in `lib/rdkafka/version.rb`
 * Change the `sha256` in `lib/rdkafka/version.rb`

data/ext/Rakefile

@@ -1,40 +1,57 @@
 # frozen_string_literal: true
 
 require File.expand_path('../../lib/rdkafka/version', __FILE__)
-require "mini_portile2"
 require "fileutils"
 require "open-uri"
 
 task :default => :clean do
-  # Download and compile librdkafka
-  recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
+  # For nix users, nix can't locate the file paths because the packages it's requiring aren't managed by the system but are
+  # managed by nix itself, so using the normal file paths doesn't work for nix users.
+  #
+  # Mini_portile causes an issue because it's dependencies are downloaded on the fly and therefore don't exist/aren't
+  # accessible in the nix environment
+  if ENV.fetch('RDKAFKA_EXT_PATH', '').empty?
+    # Download and compile librdkafka if RDKAFKA_EXT_PATH is not set
+    require "mini_portile2"
+    recipe = MiniPortile.new("librdkafka", Rdkafka::LIBRDKAFKA_VERSION)
 
-  # Use default homebrew openssl if we're on mac and the directory exists
-  # and each of flags is not empty
-  if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
-    ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
-    ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
-  end
+    # Use default homebrew openssl if we're on mac and the directory exists
+    # and each of flags is not empty
+    if recipe.host&.include?("darwin") && system("which brew &> /dev/null") && Dir.exist?("#{homebrew_prefix = %x(brew --prefix openssl).strip}")
+      ENV["CPPFLAGS"] = "-I#{homebrew_prefix}/include" unless ENV["CPPFLAGS"]
+      ENV["LDFLAGS"] = "-L#{homebrew_prefix}/lib" unless ENV["LDFLAGS"]
+    end
 
-  recipe.files << {
-    :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
-    :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
-  }
-  recipe.configure_options = ["--host=#{recipe.host}"]
-  recipe.cook
-  # Move dynamic library we're interested in
-  if recipe.host.include?('darwin')
-    from_extension = '1.dylib'
-    to_extension   = 'dylib'
+    recipe.files << {
+      :url => "https://codeload.github.com/edenhill/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
+      :sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
+    }
+    recipe.configure_options = ["--host=#{recipe.host}"]
+    recipe.cook
+    # Move dynamic library we're interested in
+    if recipe.host.include?('darwin')
+      from_extension = '1.dylib'
+      to_extension   = 'dylib'
+    else
+      from_extension = 'so.1'
+      to_extension = 'so'
+    end
+    lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
+    FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
+    # Cleanup files created by miniportile we don't need in the gem
+    FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
+    FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
   else
-    from_extension = 'so.1'
-    to_extension = 'so'
+    # Otherwise, copy existing libraries to ./ext
+    if ENV['RDKAFKA_EXT_PATH'].nil? || ENV['RDKAFKA_EXT_PATH'].empty?
+      raise "RDKAFKA_EXT_PATH must be set in your nix config when running under nix"
+    end
+    files = [
+      File.join(ENV['RDKAFKA_EXT_PATH'], 'lib', 'librdkafka.dylib'),
+      File.join(ENV['RDKAFKA_EXT_PATH'], 'lib', 'librdkafka.so')
+    ]
+    files.each { |ext| FileUtils.cp(ext, File.dirname(__FILE__)) if File.exist?(ext) }
   end
-  lib_path = File.join(File.dirname(__FILE__), "ports/#{recipe.host}/librdkafka/#{Rdkafka::LIBRDKAFKA_VERSION}/lib/librdkafka.#{from_extension}")
-  FileUtils.mv(lib_path, File.join(File.dirname(__FILE__), "librdkafka.#{to_extension}"))
-  # Cleanup files created by miniportile we don't need in the gem
-  FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
-  FileUtils.rm_rf File.join(File.dirname(__FILE__), "ports")
 end
 
 task :clean do

data/lib/rdkafka/abstract_handle.rb

@@ -1,28 +1,37 @@
 # frozen_string_literal: true
 
-require "ffi"
-
 module Rdkafka
+  # This class serves as an abstract base class to represent handles within the Rdkafka module.
+  # As a subclass of `FFI::Struct`, this class provides a blueprint for other specific handle
+  # classes to inherit from, ensuring they adhere to a particular structure and behavior.
+  #
+  # Subclasses must define their own layout, and the layout must start with:
+  #
+  # layout :pending, :bool,
+  #        :response, :int
   class AbstractHandle < FFI::Struct
-    # Subclasses must define their own layout, and the layout must start with:
-    #
-    # layout :pending, :bool,
-    #        :response, :int
+    include Helpers::Time
 
+    # Registry for registering all the handles.
     REGISTRY = {}
 
-    CURRENT_TIME = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }.freeze
-
-    private_constant :CURRENT_TIME
+    class << self
+      # Adds handle to the register
+      #
+      # @param handle [AbstractHandle] any handle we want to register
+      def register(handle)
+        address = handle.to_ptr.address
+        REGISTRY[address] = handle
+      end
 
-    def self.register(handle)
-      address = handle.to_ptr.address
-      REGISTRY[address] = handle
+      # Removes handle from the register based on the handle address
+      #
+      # @param address [Integer] address of the registered handle we want to remove
+      def remove(address)
+        REGISTRY.delete(address)
+      end
     end
 
-    def self.remove(address)
-      REGISTRY.delete(address)
-    end
 
     # Whether the handle is still pending.
     #
@@ -32,29 +41,34 @@ module Rdkafka
     end
 
     # Wait for the operation to complete or raise an error if this takes longer than the timeout.
-    # If there is a timeout this does not mean the operation failed, rdkafka might still be working on the operation.
-    # In this case it is possible to call wait again.
+    # If there is a timeout this does not mean the operation failed, rdkafka might still be working
+    # on the operation. In this case it is possible to call wait again.
+    #
+    # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out.
+    #   If this is nil it does not time out.
+    # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the
+    #   operation has completed
+    # @param raise_response_error [Boolean] should we raise error when waiting finishes
     #
-    # @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out. If this is nil it does not time out.
-    # @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the operation has completed
+    # @return [Object] Operation-specific result
     #
     # @raise [RdkafkaError] When the operation failed
     # @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
-    #
-    # @return [Object] Operation-specific result
-    def wait(max_wait_timeout: 60, wait_timeout: 0.1)
+    def wait(max_wait_timeout: 60, wait_timeout: 0.1, raise_response_error: true)
       timeout = if max_wait_timeout
-                  CURRENT_TIME.call + max_wait_timeout
+                  monotonic_now + max_wait_timeout
                 else
                   nil
                 end
       loop do
         if pending?
-          if timeout && timeout <= CURRENT_TIME.call
-            raise WaitTimeoutError.new("Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds")
+          if timeout && timeout <= monotonic_now
+            raise WaitTimeoutError.new(
+              "Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds"
+            )
           end
           sleep wait_timeout
-        elsif self[:response] != 0
+        elsif self[:response] != 0 && raise_response_error
           raise_error
         else
           return create_result

data/lib/rdkafka/admin/acl_binding_result.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    # Extracts attributes of rd_kafka_AclBinding_t
+    #
+    class AclBindingResult
+      attr_reader :result_error, :error_string, :matching_acl_resource_type,
+                  :matching_acl_resource_name, :matching_acl_resource_pattern_type,
+                  :matching_acl_principal, :matching_acl_host, :matching_acl_operation,
+                  :matching_acl_permission_type
+
+      # This attribute was initially released under the name that is now an alias
+      # We keep it for backwards compatibility but it was changed for the consistency
+      alias matching_acl_pattern_type matching_acl_resource_pattern_type
+
+      def initialize(matching_acl)
+        rd_kafka_error_pointer = Rdkafka::Bindings.rd_kafka_AclBinding_error(matching_acl)
+        @result_error = Rdkafka::Bindings.rd_kafka_error_code(rd_kafka_error_pointer)
+        error_string = Rdkafka::Bindings.rd_kafka_error_string(rd_kafka_error_pointer)
+
+        if error_string != FFI::Pointer::NULL
+          @error_string = error_string.read_string
+        end
+
+        @matching_acl_resource_type = Rdkafka::Bindings.rd_kafka_AclBinding_restype(matching_acl)
+        matching_acl_resource_name = Rdkafka::Bindings.rd_kafka_AclBinding_name(matching_acl)
+
+        if matching_acl_resource_name != FFI::Pointer::NULL
+          @matching_acl_resource_name = matching_acl_resource_name.read_string
+        end
+
+        @matching_acl_resource_pattern_type = Rdkafka::Bindings.rd_kafka_AclBinding_resource_pattern_type(matching_acl)
+        matching_acl_principal = Rdkafka::Bindings.rd_kafka_AclBinding_principal(matching_acl)
+
+        if matching_acl_principal != FFI::Pointer::NULL
+          @matching_acl_principal = matching_acl_principal.read_string
+        end
+
+        matching_acl_host = Rdkafka::Bindings.rd_kafka_AclBinding_host(matching_acl)
+
+        if matching_acl_host != FFI::Pointer::NULL
+          @matching_acl_host = matching_acl_host.read_string
+        end
+
+        @matching_acl_operation = Rdkafka::Bindings.rd_kafka_AclBinding_operation(matching_acl)
+        @matching_acl_permission_type = Rdkafka::Bindings.rd_kafka_AclBinding_permission_type(matching_acl)
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/create_acl_handle.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class CreateAclHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :response_string, :pointer
+
+      # @return [String] the name of the operation
+      def operation_name
+        "create acl"
+      end
+
+      # @return [CreateAclReport] instance with rdkafka_response value as 0 and rdkafka_response_string value as empty string if the acl creation was successful
+      def create_result
+        CreateAclReport.new(rdkafka_response: self[:response], rdkafka_response_string: self[:response_string])
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+            self[:response],
+            broker_message: self[:response_string].read_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/create_acl_report.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class CreateAclReport
+
+      # Upon successful creation of Acl RD_KAFKA_RESP_ERR_NO_ERROR - 0 is returned as rdkafka_response
+      # @return [Integer]
+      attr_reader :rdkafka_response
+
+
+      # Upon successful creation of Acl empty string will be returned as rdkafka_response_string
+      # @return [String]
+      attr_reader :rdkafka_response_string
+
+      def initialize(rdkafka_response:, rdkafka_response_string:)
+        @rdkafka_response = rdkafka_response
+        if rdkafka_response_string != FFI::Pointer::NULL
+          @rdkafka_response_string = rdkafka_response_string.read_string
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/create_partitions_handle.rb

@@ -0,0 +1,27 @@
+module Rdkafka
+  class Admin
+    class CreatePartitionsHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :error_string, :pointer,
+             :result_name, :pointer
+
+      # @return [String] the name of the operation
+      def operation_name
+        "create partitions"
+      end
+
+      # @return [Boolean] whether the create topic was successful
+      def create_result
+        CreatePartitionsReport.new(self[:error_string], self[:result_name])
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+            self[:response],
+            broker_message: CreateTopicReport.new(self[:error_string], self[:result_name]).error_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/create_partitions_report.rb

@@ -0,0 +1,6 @@
+module Rdkafka
+  class Admin
+    class CreatePartitionsReport < CreateTopicReport
+    end
+  end
+end

data/lib/rdkafka/admin/delete_acl_handle.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class DeleteAclHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :response_string, :pointer,
+             :matching_acls, :pointer,
+             :matching_acls_count, :int
+
+      # @return [String] the name of the operation
+      def operation_name
+        "delete acl"
+      end
+
+      # @return [DeleteAclReport] instance with an array of matching_acls
+      def create_result
+        DeleteAclReport.new(matching_acls: self[:matching_acls], matching_acls_count: self[:matching_acls_count])
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+            self[:response],
+            broker_message: self[:response_string].read_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/delete_acl_report.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class DeleteAclReport
+
+      # deleted acls
+      # @return [Rdkafka::Bindings::AclBindingResult]
+      attr_reader :deleted_acls
+
+      def initialize(matching_acls:, matching_acls_count:)
+        @deleted_acls=[]
+        if matching_acls != FFI::Pointer::NULL
+          acl_binding_result_pointers  = matching_acls.read_array_of_pointer(matching_acls_count)
+          (1..matching_acls_count).map do |matching_acl_index|
+            acl_binding_result = AclBindingResult.new(acl_binding_result_pointers[matching_acl_index - 1])
+            @deleted_acls << acl_binding_result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/delete_groups_handle.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class DeleteGroupsHandle < AbstractHandle
+      layout :pending, :bool, # TODO: ???
+             :response, :int,
+             :error_string, :pointer,
+             :result_name, :pointer
+
+      # @return [String] the name of the operation
+      def operation_name
+        "delete groups"
+      end
+
+      def create_result
+        DeleteGroupsReport.new(self[:error_string], self[:result_name])
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+            self[:response],
+            broker_message: create_result.error_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/delete_groups_report.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class DeleteGroupsReport
+      # Any error message generated from the DeleteTopic
+      # @return [String]
+      attr_reader :error_string
+
+      # The name of the topic deleted
+      # @return [String]
+      attr_reader :result_name
+
+      def initialize(error_string, result_name)
+        if error_string != FFI::Pointer::NULL
+          @error_string = error_string.read_string
+        end
+        if result_name != FFI::Pointer::NULL
+          @result_name = @result_name = result_name.read_string
+        end
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/describe_acl_handle.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class DescribeAclHandle < AbstractHandle
+      layout :pending, :bool,
+             :response, :int,
+             :response_string, :pointer,
+             :acls, :pointer,
+             :acls_count, :int
+
+      # @return [String] the name of the operation.
+      def operation_name
+        "describe acl"
+      end
+
+      # @return [DescribeAclReport] instance with an array of acls that matches the request filters.
+      def create_result
+        DescribeAclReport.new(acls: self[:acls], acls_count: self[:acls_count])
+      end
+
+      def raise_error
+        raise RdkafkaError.new(
+            self[:response],
+            broker_message: self[:response_string].read_string
+        )
+      end
+    end
+  end
+end

data/lib/rdkafka/admin/describe_acl_report.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Rdkafka
+  class Admin
+    class DescribeAclReport
+
+      # acls that exists in the cluster for the resource_type, resource_name and pattern_type filters provided in the request.
+      # @return [Rdkafka::Bindings::AclBindingResult] array of matching acls.
+      attr_reader :acls
+
+      def initialize(acls:, acls_count:)
+        @acls=[]
+        if acls != FFI::Pointer::NULL
+          acl_binding_result_pointers  = acls.read_array_of_pointer(acls_count)
+          (1..acls_count).map do |acl_index|
+            acl_binding_result = AclBindingResult.new(acl_binding_result_pointers[acl_index - 1])
+            @acls << acl_binding_result
+          end
+        end
+      end
+    end
+  end
+end