karafka-rdkafka 0.14.9 → 0.15.0.alpha1

data/lib/rdkafka/bindings.rb

@@ -97,6 +97,48 @@ module Rdkafka
     attach_function :rd_kafka_topic_partition_list_destroy, [:pointer], :void
     attach_function :rd_kafka_topic_partition_list_copy, [:pointer], :pointer
 
+    # Configs management
+    #
+    # Structs for management of configurations
+    # Each configuration is attached to a resource and one resource can have many configuration
+    # details. Each resource will also have separate errors results if obtaining configuration
+    # was not possible for any reason
+    class ConfigResource < FFI::Struct
+      layout :type, :int,
+             :name, :string
+    end
+
+    attach_function :rd_kafka_DescribeConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_ConfigResource_new, [:int32, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_destroy_array, [:pointer, :int32], :void
+    attach_function :rd_kafka_event_DescribeConfigs_result, [:pointer], :pointer
+    attach_function :rd_kafka_DescribeConfigs_result_resources, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_configs, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigEntry_name, [:pointer], :string
+    attach_function :rd_kafka_ConfigEntry_value, [:pointer], :string
+    attach_function :rd_kafka_ConfigEntry_is_read_only, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_is_default, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_is_sensitive, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_is_synonym, [:pointer], :int
+    attach_function :rd_kafka_ConfigEntry_synonyms, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_error, [:pointer], :int
+    attach_function :rd_kafka_ConfigResource_error_string, [:pointer], :string
+    attach_function :rd_kafka_IncrementalAlterConfigs, [:pointer, :pointer, :size_t, :pointer, :pointer], :void, blocking: true
+    attach_function :rd_kafka_IncrementalAlterConfigs_result_resources, [:pointer, :pointer], :pointer
+    attach_function :rd_kafka_ConfigResource_add_incremental_config, [:pointer, :string, :int32, :string], :pointer
+    attach_function :rd_kafka_event_IncrementalAlterConfigs_result, [:pointer], :pointer
+
+    RD_KAFKA_ADMIN_OP_DESCRIBECONFIGS = 5
+    RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT = 104
+
+    RD_KAFKA_ADMIN_OP_INCREMENTALALTERCONFIGS = 16
+    RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT = 131072
+
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_SET      = 0
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_DELETE   = 1
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_APPEND   = 2
+    RD_KAFKA_ALTER_CONFIG_OP_TYPE_SUBTRACT = 3
+
     # Errors
 
     attach_function :rd_kafka_err2name, [:int], :string
@@ -147,6 +189,8 @@ module Rdkafka
                  else
                    Logger::UNKNOWN
                  end
+
+      Rdkafka::Config.ensure_log_thread
       Rdkafka::Config.log_queue << [severity, "rdkafka: #{line}"]
     end
 

data/lib/rdkafka/callbacks.rb

@@ -113,6 +113,42 @@ module Rdkafka
       end
     end
 
+    class DescribeConfigsResult
+      attr_reader :result_error, :error_string, :results, :results_count
+
+      def initialize(event_ptr)
+        @results=[]
+        @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
+        @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
+
+        if @result_error == 0
+          configs_describe_result = Rdkafka::Bindings.rd_kafka_event_DescribeConfigs_result(event_ptr)
+          # Get the number of matching acls
+          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          @results = Rdkafka::Bindings.rd_kafka_DescribeConfigs_result_resources(configs_describe_result, pointer_to_size_t)
+          @results_count = pointer_to_size_t.read_int
+        end
+      end
+    end
+
+    class IncrementalAlterConfigsResult
+      attr_reader :result_error, :error_string, :results, :results_count
+
+      def initialize(event_ptr)
+        @results=[]
+        @result_error = Rdkafka::Bindings.rd_kafka_event_error(event_ptr)
+        @error_string = Rdkafka::Bindings.rd_kafka_event_error_string(event_ptr)
+
+        if @result_error == 0
+          incremental_alter_result = Rdkafka::Bindings.rd_kafka_event_IncrementalAlterConfigs_result(event_ptr)
+          # Get the number of matching acls
+          pointer_to_size_t = FFI::MemoryPointer.new(:int32)
+          @results = Rdkafka::Bindings.rd_kafka_IncrementalAlterConfigs_result_resources(incremental_alter_result, pointer_to_size_t)
+          @results_count = pointer_to_size_t.read_int
+        end
+      end
+    end
+
     # FFI Function used for Create Topic and Delete Topic callbacks
     BackgroundEventCallbackFunction = FFI::Function.new(
         :void, [:pointer, :pointer, :pointer]
@@ -123,20 +159,24 @@ module Rdkafka
     # @private
     class BackgroundEventCallback
       def self.call(_, event_ptr, _)
-        event_type = Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
-        if event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
+        case Rdkafka::Bindings.rd_kafka_event_type(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATETOPICS_RESULT
           process_create_topic(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBECONFIGS_RESULT
+          process_describe_configs(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_INCREMENTALALTERCONFIGS_RESULT
+          process_incremental_alter_configs(event_ptr)
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETETOPICS_RESULT
           process_delete_topic(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_ADMIN_OP_CREATEPARTITIONS_RESULT
           process_create_partitions(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_CREATEACLS_RESULT
           process_create_acl(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEACLS_RESULT
           process_delete_acl(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DESCRIBEACLS_RESULT
           process_describe_acl(event_ptr)
-        elsif event_type == Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
+        when Rdkafka::Bindings::RD_KAFKA_EVENT_DELETEGROUPS_RESULT
           process_delete_groups(event_ptr)
         end
       end
@@ -156,7 +196,44 @@ module Rdkafka
           create_topic_handle[:response] = create_topic_results[0].result_error
           create_topic_handle[:error_string] = create_topic_results[0].error_string
           create_topic_handle[:result_name] = create_topic_results[0].result_name
-          create_topic_handle[:pending] = false
+
+          create_topic_handle.unlock
+        end
+      end
+
+      def self.process_describe_configs(event_ptr)
+        describe_configs = DescribeConfigsResult.new(event_ptr)
+        describe_configs_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if describe_configs_handle = Rdkafka::Admin::DescribeConfigsHandle.remove(describe_configs_handle_ptr.address)
+          describe_configs_handle[:response] = describe_configs.result_error
+          describe_configs_handle[:response_string] = describe_configs.error_string
+          describe_configs_handle[:pending] = false
+
+          if describe_configs.result_error == 0
+            describe_configs_handle[:config_entries] = describe_configs.results
+            describe_configs_handle[:entry_count] = describe_configs.results_count
+          end
+
+          describe_configs_handle.unlock
+        end
+      end
+
+      def self.process_incremental_alter_configs(event_ptr)
+        incremental_alter = IncrementalAlterConfigsResult.new(event_ptr)
+        incremental_alter_handle_ptr = Rdkafka::Bindings.rd_kafka_event_opaque(event_ptr)
+
+        if incremental_alter_handle = Rdkafka::Admin::IncrementalAlterConfigsHandle.remove(incremental_alter_handle_ptr.address)
+          incremental_alter_handle[:response] = incremental_alter.result_error
+          incremental_alter_handle[:response_string] = incremental_alter.error_string
+          incremental_alter_handle[:pending] = false
+
+          if incremental_alter.result_error == 0
+            incremental_alter_handle[:config_entries] = incremental_alter.results
+            incremental_alter_handle[:entry_count] = incremental_alter.results_count
+          end
+
+          incremental_alter_handle.unlock
         end
       end
 
@@ -173,7 +250,8 @@ module Rdkafka
           delete_group_handle[:response] = delete_group_results[0].result_error
           delete_group_handle[:error_string] = delete_group_results[0].error_string
           delete_group_handle[:result_name] = delete_group_results[0].result_name
-          delete_group_handle[:pending] = false
+
+          delete_group_handle.unlock
         end
       end
 
@@ -190,7 +268,8 @@ module Rdkafka
           delete_topic_handle[:response] = delete_topic_results[0].result_error
           delete_topic_handle[:error_string] = delete_topic_results[0].error_string
           delete_topic_handle[:result_name] = delete_topic_results[0].result_name
-          delete_topic_handle[:pending] = false
+
+          delete_topic_handle.unlock
         end
       end
 
@@ -207,7 +286,8 @@ module Rdkafka
           create_partitions_handle[:response] = create_partitions_results[0].result_error
           create_partitions_handle[:error_string] = create_partitions_results[0].error_string
           create_partitions_handle[:result_name] = create_partitions_results[0].result_name
-          create_partitions_handle[:pending] = false
+
+          create_partitions_handle.unlock
         end
       end
 
@@ -223,7 +303,8 @@ module Rdkafka
         if create_acl_handle = Rdkafka::Admin::CreateAclHandle.remove(create_acl_handle_ptr.address)
           create_acl_handle[:response] = create_acl_results[0].result_error
           create_acl_handle[:response_string] = create_acl_results[0].error_string
-          create_acl_handle[:pending] = false
+
+          create_acl_handle.unlock
         end
       end
 
@@ -239,11 +320,13 @@ module Rdkafka
         if delete_acl_handle = Rdkafka::Admin::DeleteAclHandle.remove(delete_acl_handle_ptr.address)
           delete_acl_handle[:response] = delete_acl_results[0].result_error
           delete_acl_handle[:response_string] = delete_acl_results[0].error_string
-          delete_acl_handle[:pending] = false
+
           if delete_acl_results[0].result_error == 0
             delete_acl_handle[:matching_acls] = delete_acl_results[0].matching_acls
             delete_acl_handle[:matching_acls_count] = delete_acl_results[0].matching_acls_count
           end
+
+          delete_acl_handle.unlock
         end
       end
 
@@ -254,17 +337,18 @@ module Rdkafka
         if describe_acl_handle = Rdkafka::Admin::DescribeAclHandle.remove(describe_acl_handle_ptr.address)
           describe_acl_handle[:response] = describe_acl.result_error
           describe_acl_handle[:response_string] = describe_acl.error_string
-          describe_acl_handle[:pending] = false
+
           if describe_acl.result_error == 0
-            describe_acl_handle[:acls]       = describe_acl.matching_acls
+            describe_acl_handle[:acls] = describe_acl.matching_acls
             describe_acl_handle[:acls_count] = describe_acl.matching_acls_count
           end
+
+          describe_acl_handle.unlock
         end
       end
     end
 
     # FFI Function used for Message Delivery callbacks
-
     DeliveryCallbackFunction = FFI::Function.new(
         :void, [:pointer, :pointer, :pointer]
     ) do |client_ptr, message_ptr, opaque_ptr|
@@ -284,7 +368,6 @@ module Rdkafka
           delivery_handle[:partition] = message[:partition]
           delivery_handle[:offset] = message[:offset]
           delivery_handle[:topic_name] = FFI::MemoryPointer.from_string(topic_name)
-          delivery_handle[:pending] = false
 
           # Call delivery callback on opaque
           if opaque = Rdkafka::Config.opaques[opaque_ptr.to_i]
@@ -299,9 +382,10 @@ module Rdkafka
               delivery_handle
             )
           end
+
+          delivery_handle.unlock
         end
       end
     end
-
   end
 end

data/lib/rdkafka/config.rb

@@ -15,13 +15,12 @@ module Rdkafka
     @@opaques = ObjectSpace::WeakMap.new
     # @private
     @@log_queue = Queue.new
-
-    Thread.start do
-      loop do
-        severity, msg = @@log_queue.pop
-        @@logger.add(severity, msg)
-      end
-    end
+    # @private
+    # We memoize thread on the first log flush
+    # This allows us also to restart logger thread on forks
+    @@log_thread = nil
+    # @private
+    @@log_mutex = Mutex.new
 
     # Returns the current logger, by default this is a logger to stdout.
     #
@@ -30,6 +29,24 @@ module Rdkafka
       @@logger
     end
 
+    # Makes sure that there is a thread for consuming logs
+    # We do not spawn thread immediately and we need to check if it operates to support forking
+    def self.ensure_log_thread
+      return if @@log_thread && @@log_thread.alive?
+
+      @@log_mutex.synchronize do
+        # Restart if dead (fork, crash)
+        @@log_thread = nil if @@log_thread && !@@log_thread.alive?
+
+        @@log_thread ||= Thread.start do
+          loop do
+            severity, msg = @@log_queue.pop
+            @@logger.add(severity, msg)
+          end
+        end
+      end
+    end
+
     # Returns a queue whose contents will be passed to the configured logger. Each entry
     # should follow the format [Logger::Severity, String]. The benefit over calling the
     # logger directly is that this is safe to use from trap contexts.

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.14.9"
+  VERSION = "0.15.0.alpha1"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/lib/rdkafka.rb

@@ -23,7 +23,13 @@ require "rdkafka/admin/delete_acl_handle"
 require "rdkafka/admin/delete_acl_report"
 require "rdkafka/admin/describe_acl_handle"
 require "rdkafka/admin/describe_acl_report"
+require "rdkafka/admin/describe_configs_handle"
+require "rdkafka/admin/describe_configs_report"
+require "rdkafka/admin/incremental_alter_configs_handle"
+require "rdkafka/admin/incremental_alter_configs_report"
 require "rdkafka/admin/acl_binding_result"
+require "rdkafka/admin/config_binding_result"
+require "rdkafka/admin/config_resource_binding_result"
 require "rdkafka/bindings"
 require "rdkafka/callbacks"
 require "rdkafka/config"

data/spec/rdkafka/abstract_handle_spec.rb

@@ -76,37 +76,50 @@ describe Rdkafka::AbstractHandle do
   end
 
   describe "#wait" do
-    let(:pending_handle) { true }
+    context 'when pending_handle true' do
+      let(:pending_handle) { true }
 
-    it "should wait until the timeout and then raise an error" do
-      expect {
-        subject.wait(max_wait_timeout: 0.1)
-      }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
+      it "should wait until the timeout and then raise an error" do
+        expect(Kernel).not_to receive(:warn)
+        expect {
+          subject.wait(max_wait_timeout: 0.1)
+        }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
+      end
     end
 
-    context "when not pending anymore and no error" do
+    context 'when pending_handle false' do
       let(:pending_handle) { false }
-      let(:result) { 1 }
 
-      it "should return a result" do
-        wait_result = subject.wait
-        expect(wait_result).to eq(result)
+      it 'should show a deprecation warning when wait_timeout is set' do
+        expect(Kernel).to receive(:warn).with(Rdkafka::AbstractHandle::WAIT_TIMEOUT_DEPRECATION_MESSAGE)
+        subject.wait(wait_timeout: 0.1)
       end
 
-      it "should wait without a timeout" do
-        wait_result = subject.wait(max_wait_timeout: nil)
-        expect(wait_result).to eq(result)
+      context "without error" do
+        let(:result) { 1 }
+
+        it "should return a result" do
+          expect(Kernel).not_to receive(:warn)
+          wait_result = subject.wait
+          expect(wait_result).to eq(result)
+        end
+
+        it "should wait without a timeout" do
+          expect(Kernel).not_to receive(:warn)
+          wait_result = subject.wait(max_wait_timeout: nil)
+          expect(wait_result).to eq(result)
+        end
       end
-    end
 
-    context "when not pending anymore and there was an error" do
-      let(:pending_handle) { false }
-      let(:response) { 20 }
+      context "with error" do
+        let(:response) { 20 }
 
-      it "should raise an rdkafka error" do
-        expect {
-          subject.wait
-        }.to raise_error Rdkafka::RdkafkaError
+        it "should raise an rdkafka error" do
+          expect(Kernel).not_to receive(:warn)
+          expect {
+            subject.wait
+          }.to raise_error Rdkafka::RdkafkaError
+        end
       end
     end
   end