karafka-rdkafka 0.14.10 → 0.15.0

data/lib/rdkafka/helpers/oauth.rb

@@ -0,0 +1,58 @@
+module Rdkafka
+  module Helpers
+
+    module OAuth
+
+      # Set the OAuthBearer token
+      #
+      # @param token [String] the mandatory token value to set, often (but not necessarily) a JWS compact serialization as per https://tools.ietf.org/html/rfc7515#section-3.1.
+      # @param lifetime_ms [Integer] when the token expires, in terms of the number of milliseconds since the epoch. See https://currentmillis.com/.
+      # @param principal_name [String] the mandatory Kafka principal name associated with the token.
+      # @param extensions [Hash] optional SASL extensions key-value pairs to be communicated to the broker as additional key-value pairs during the initial client response as per https://tools.ietf.org/html/rfc7628#section-3.1.
+      # @return [Integer] 0 on success
+      def oauthbearer_set_token(token:, lifetime_ms:, principal_name:, extensions: nil)
+        error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+
+        response = @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(
+            inner, token, lifetime_ms, principal_name,
+            flatten_extensions(extensions), extension_size(extensions), error_buffer, 256
+          )
+        end
+
+        return response if response.zero?
+
+        oauthbearer_set_token_failure("Failed to set token: #{error_buffer.read_string}")
+
+        response
+      end
+
+      # Marks failed oauth token acquire in librdkafka
+      #
+      # @param reason [String] human readable error reason for failing to acquire token
+      def oauthbearer_set_token_failure(reason)
+        @native_kafka.with_inner do |inner|
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure(
+            inner,
+            reason
+          )
+        end
+      end
+
+      private
+
+      # Flatten the extensions hash into a string according to the spec, https://datatracker.ietf.org/doc/html/rfc7628#section-3.1
+      def flatten_extensions(extensions)
+        return nil unless extensions
+        "\x01#{extensions.map { |e| e.join("=") }.join("\x01")}"
+      end
+
+      # extension_size is the number of keys + values which should be a non-negative even number
+      # https://github.com/confluentinc/librdkafka/blob/master/src/rdkafka_sasl_oauthbearer.c#L327-L347
+      def extension_size(extensions)
+        return 0 unless extensions
+        extensions.size * 2
+      end
+    end
+  end
+end

data/lib/rdkafka/native_kafka.rb

@@ -4,7 +4,7 @@ module Rdkafka
   # @private
   # A wrapper around a native kafka that polls and cleanly exits
   class NativeKafka
-    def initialize(inner, run_polling_thread:, opaque:)
+    def initialize(inner, run_polling_thread:, opaque:, auto_start: true)
       @inner = inner
       @opaque = opaque
       # Lock around external access
@@ -28,30 +28,43 @@ module Rdkafka
       # counter for operations in progress using inner
       @operations_in_progress = 0
 
-      # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
-      Rdkafka::Bindings.rd_kafka_poll(inner, 0)
+      @run_polling_thread = run_polling_thread
 
-      if run_polling_thread
-        # Start thread to poll client for delivery callbacks,
-        # not used in consumer.
-        @polling_thread = Thread.new do
-          loop do
-            @poll_mutex.synchronize do
-              Rdkafka::Bindings.rd_kafka_poll(inner, 100)
-            end
+      start if auto_start
 
-            # Exit thread if closing and the poll queue is empty
-            if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(inner) == 0
-              break
+      @closing = false
+    end
+
+    def start
+      synchronize do
+        return if @started
+
+        @started = true
+
+        # Trigger initial poll to make sure oauthbearer cb and other initial cb are handled
+        Rdkafka::Bindings.rd_kafka_poll(@inner, 0)
+
+        if @run_polling_thread
+          # Start thread to poll client for delivery callbacks,
+          # not used in consumer.
+          @polling_thread = Thread.new do
+            loop do
+              @poll_mutex.synchronize do
+                Rdkafka::Bindings.rd_kafka_poll(@inner, 100)
+              end
+
+              # Exit thread if closing and the poll queue is empty
+              if Thread.current[:closing] && Rdkafka::Bindings.rd_kafka_outq_len(@inner) == 0
+                break
+              end
             end
           end
-        end
 
-        @polling_thread.abort_on_exception = true
-        @polling_thread[:closing] = false
+          @polling_thread.name = "rdkafka.native_kafka##{Rdkafka::Bindings.rd_kafka_name(@inner).gsub('rdkafka', '')}"
+          @polling_thread.abort_on_exception = true
+          @polling_thread[:closing] = false
+        end
       end
-
-      @closing = false
     end
 
     def with_inner

data/lib/rdkafka/producer.rb

@@ -4,6 +4,7 @@ module Rdkafka
   # A producer for Kafka messages. To create a producer set up a {Config} and call {Config#producer producer} on that.
   class Producer
     include Helpers::Time
+    include Helpers::OAuth
 
     # Cache partitions count for 30 seconds
     PARTITIONS_COUNT_TTL = 30
@@ -53,6 +54,12 @@ module Rdkafka
       end
     end
 
+    # Starts the native Kafka polling thread and kicks off the init polling
+    # @note Not needed to run unless explicit start was disabled
+    def start
+      @native_kafka.start
+    end
+
     # @return [String] producer name
     def name
       @name ||= @native_kafka.with_inner do |inner|

data/lib/rdkafka/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Rdkafka
-  VERSION = "0.14.10"
+  VERSION = "0.15.0"
   LIBRDKAFKA_VERSION = "2.3.0"
   LIBRDKAFKA_SOURCE_SHA256 = "2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12"
 end

data/lib/rdkafka.rb

@@ -7,6 +7,7 @@ require "json"
 
 require "rdkafka/version"
 require "rdkafka/helpers/time"
+require "rdkafka/helpers/oauth"
 require "rdkafka/abstract_handle"
 require "rdkafka/admin"
 require "rdkafka/admin/create_topic_handle"
@@ -23,7 +24,13 @@ require "rdkafka/admin/delete_acl_handle"
 require "rdkafka/admin/delete_acl_report"
 require "rdkafka/admin/describe_acl_handle"
 require "rdkafka/admin/describe_acl_report"
+require "rdkafka/admin/describe_configs_handle"
+require "rdkafka/admin/describe_configs_report"
+require "rdkafka/admin/incremental_alter_configs_handle"
+require "rdkafka/admin/incremental_alter_configs_report"
 require "rdkafka/admin/acl_binding_result"
+require "rdkafka/admin/config_binding_result"
+require "rdkafka/admin/config_resource_binding_result"
 require "rdkafka/bindings"
 require "rdkafka/callbacks"
 require "rdkafka/config"

data/spec/rdkafka/abstract_handle_spec.rb

@@ -76,37 +76,50 @@ describe Rdkafka::AbstractHandle do
   end
 
   describe "#wait" do
-    let(:pending_handle) { true }
+    context 'when pending_handle true' do
+      let(:pending_handle) { true }
 
-    it "should wait until the timeout and then raise an error" do
-      expect {
-        subject.wait(max_wait_timeout: 0.1)
-      }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
+      it "should wait until the timeout and then raise an error" do
+        expect(Kernel).not_to receive(:warn)
+        expect {
+          subject.wait(max_wait_timeout: 0.1)
+        }.to raise_error Rdkafka::AbstractHandle::WaitTimeoutError, /test_operation/
+      end
     end
 
-    context "when not pending anymore and no error" do
+    context 'when pending_handle false' do
       let(:pending_handle) { false }
-      let(:result) { 1 }
 
-      it "should return a result" do
-        wait_result = subject.wait
-        expect(wait_result).to eq(result)
+      it 'should show a deprecation warning when wait_timeout is set' do
+        expect(Kernel).to receive(:warn).with(Rdkafka::AbstractHandle::WAIT_TIMEOUT_DEPRECATION_MESSAGE)
+        subject.wait(wait_timeout: 0.1)
       end
 
-      it "should wait without a timeout" do
-        wait_result = subject.wait(max_wait_timeout: nil)
-        expect(wait_result).to eq(result)
+      context "without error" do
+        let(:result) { 1 }
+
+        it "should return a result" do
+          expect(Kernel).not_to receive(:warn)
+          wait_result = subject.wait
+          expect(wait_result).to eq(result)
+        end
+
+        it "should wait without a timeout" do
+          expect(Kernel).not_to receive(:warn)
+          wait_result = subject.wait(max_wait_timeout: nil)
+          expect(wait_result).to eq(result)
+        end
       end
-    end
 
-    context "when not pending anymore and there was an error" do
-      let(:pending_handle) { false }
-      let(:response) { 20 }
+      context "with error" do
+        let(:response) { 20 }
 
-      it "should raise an rdkafka error" do
-        expect {
-          subject.wait
-        }.to raise_error Rdkafka::RdkafkaError
+        it "should raise an rdkafka error" do
+          expect(Kernel).not_to receive(:warn)
+          expect {
+            subject.wait
+          }.to raise_error Rdkafka::RdkafkaError
+        end
       end
     end
   end

data/spec/rdkafka/admin_spec.rb

@@ -16,12 +16,12 @@ describe Rdkafka::Admin do
     admin.close
   end
 
-  let(:topic_name)               { "test-topic-#{Random.new.rand(0..1_000_000)}" }
+  let(:topic_name)               { "test-topic-#{SecureRandom.uuid}" }
   let(:topic_partition_count)    { 3 }
   let(:topic_replication_factor) { 1 }
   let(:topic_config)             { {"cleanup.policy" => "compact", "min.cleanable.dirty.ratio" => 0.8} }
   let(:invalid_topic_config)     { {"cleeeeenup.policee" => "campact"} }
-  let(:group_name)               { "test-group-#{Random.new.rand(0..1_000_000)}" }
+  let(:group_name)               { "test-group-#{SecureRandom.uuid}" }
 
   let(:resource_name)         {"acl-test-topic"}
   let(:resource_type)         {Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC}
@@ -31,6 +31,19 @@ describe Rdkafka::Admin do
   let(:operation)             {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
   let(:permission_type)       {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
 
+  describe 'admin without auto-start' do
+    let(:admin) { config.admin(native_kafka_auto_start: false) }
+
+    it 'expect to be able to start it later and close' do
+      admin.start
+      admin.close
+    end
+
+    it 'expect to be able to close it without starting' do
+      admin.close
+    end
+  end
+
   describe "#create_topic" do
     describe "called with invalid input" do
       describe "with an invalid topic name" do
@@ -129,6 +142,275 @@ describe Rdkafka::Admin do
     end
   end
 
+  describe "describe_configs" do
+    subject(:resources_results) { admin.describe_configs(resources).wait.resources }
+
+    before do
+      admin.create_topic(topic_name, 2, 1).wait
+      sleep(1)
+    end
+
+    context 'when describing config of an existing topic' do
+      let(:resources) { [{ resource_type: 2, resource_name: topic_name }] }
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq(topic_name)
+        expect(resources_results.first.configs.size).to be > 25
+        expect(resources_results.first.configs.first.name).to eq('compression.type')
+        expect(resources_results.first.configs.first.value).to eq('producer')
+        expect(resources_results.first.configs.map(&:synonyms)).not_to be_empty
+      end
+    end
+
+    context 'when describing config of a non-existing topic' do
+      let(:resources) { [{ resource_type: 2, resource_name: SecureRandom.uuid }] }
+
+      it 'expect to raise error' do
+        expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /unknown_topic_or_part/)
+      end
+    end
+
+    context 'when describing both existing and non-existing topics' do
+      let(:resources) do
+        [
+          { resource_type: 2, resource_name: topic_name },
+          { resource_type: 2, resource_name: SecureRandom.uuid }
+        ]
+      end
+
+      it 'expect to raise error' do
+        expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /unknown_topic_or_part/)
+      end
+    end
+
+    context 'when describing multiple existing topics' do
+      let(:resources) do
+        [
+          { resource_type: 2, resource_name: 'example_topic' },
+          { resource_type: 2, resource_name: topic_name }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(2)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq('example_topic')
+        expect(resources_results.last.type).to eq(2)
+        expect(resources_results.last.name).to eq(topic_name)
+      end
+    end
+
+    context 'when trying to describe invalid resource type' do
+      let(:resources) { [{ resource_type: 0, resource_name: SecureRandom.uuid }] }
+
+      it 'expect to raise error' do
+        expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_request/)
+      end
+    end
+
+    context 'when trying to describe invalid broker' do
+      let(:resources) { [{ resource_type: 4, resource_name: 'non-existing' }] }
+
+      it 'expect to raise error' do
+        expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_arg/)
+      end
+    end
+
+    context 'when trying to describe valid broker' do
+      let(:resources) { [{ resource_type: 4, resource_name: '1' }] }
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(4)
+        expect(resources_results.first.name).to eq('1')
+        expect(resources_results.first.configs.size).to be > 230
+        expect(resources_results.first.configs.first.name).to eq('log.cleaner.min.compaction.lag.ms')
+        expect(resources_results.first.configs.first.value).to eq('0')
+        expect(resources_results.first.configs.map(&:synonyms)).not_to be_empty
+      end
+    end
+
+    context 'when describing valid broker with topics in one request' do
+      let(:resources) do
+        [
+          { resource_type: 4, resource_name: '1' },
+          { resource_type: 2, resource_name: topic_name }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(2)
+        expect(resources_results.first.type).to eq(4)
+        expect(resources_results.first.name).to eq('1')
+        expect(resources_results.first.configs.size).to be > 230
+        expect(resources_results.first.configs.first.name).to eq('log.cleaner.min.compaction.lag.ms')
+        expect(resources_results.first.configs.first.value).to eq('0')
+        expect(resources_results.last.type).to eq(2)
+        expect(resources_results.last.name).to eq(topic_name)
+        expect(resources_results.last.configs.size).to be > 25
+        expect(resources_results.last.configs.first.name).to eq('compression.type')
+        expect(resources_results.last.configs.first.value).to eq('producer')
+      end
+    end
+  end
+
+  describe "incremental_alter_configs" do
+    subject(:resources_results) { admin.incremental_alter_configs(resources_with_configs).wait.resources }
+
+    before do
+      admin.create_topic(topic_name, 2, 1).wait
+      sleep(1)
+    end
+
+    context 'when altering one topic with one valid config via set' do
+      let(:target_retention) { (86400002 + rand(10_000)).to_s }
+      let(:resources_with_configs) do
+        [
+          {
+            resource_type: 2,
+            resource_name: topic_name,
+            configs: [
+              {
+                name: 'delete.retention.ms',
+                value: target_retention,
+                op_type: 0
+              }
+            ]
+          }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq(topic_name)
+
+        ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
+          config.name == 'delete.retention.ms'
+        end
+
+        expect(ret_config.value).to eq(target_retention)
+      end
+    end
+
+    context 'when altering one topic with one valid config via delete' do
+      let(:target_retention) { (8640002 + rand(10_000)).to_s }
+      let(:resources_with_configs) do
+        [
+          {
+            resource_type: 2,
+            resource_name: topic_name,
+            configs: [
+              {
+                name: 'delete.retention.ms',
+                value: target_retention,
+                op_type: 1
+              }
+            ]
+          }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq(topic_name)
+        ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
+          config.name == 'delete.retention.ms'
+        end
+
+        expect(ret_config.value).to eq('86400000')
+      end
+    end
+
+    context 'when altering one topic with one valid config via append' do
+      let(:target_policy) { 'compact' }
+      let(:resources_with_configs) do
+        [
+          {
+            resource_type: 2,
+            resource_name: topic_name,
+            configs: [
+              {
+                name: 'cleanup.policy',
+                value: target_policy,
+                op_type: 2
+              }
+            ]
+          }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq(topic_name)
+
+        ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
+          config.name == 'cleanup.policy'
+        end
+
+        expect(ret_config.value).to eq("delete,#{target_policy}")
+      end
+    end
+
+    context 'when altering one topic with one valid config via subtrack' do
+      let(:target_policy) { 'delete' }
+      let(:resources_with_configs) do
+        [
+          {
+            resource_type: 2,
+            resource_name: topic_name,
+            configs: [
+              {
+                name: 'cleanup.policy',
+                value: target_policy,
+                op_type: 3
+              }
+            ]
+          }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq(topic_name)
+
+        ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
+          config.name == 'cleanup.policy'
+        end
+
+        expect(ret_config.value).to eq('')
+      end
+    end
+
+    context 'when altering one topic with invalid config' do
+      let(:target_retention) { '-10' }
+      let(:resources_with_configs) do
+        [
+          {
+            resource_type: 2,
+            resource_name: topic_name,
+            configs: [
+              {
+                name: 'delete.retention.ms',
+                value: target_retention,
+                op_type: 0
+              }
+            ]
+          }
+        ]
+      end
+
+      it 'expect to raise error' do
+        expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_config/)
+      end
+    end
+  end
+
   describe "#delete_topic" do
     describe "called with invalid input" do
       # https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L29
@@ -275,6 +557,9 @@ describe Rdkafka::Admin do
         expect(create_acl_report.rdkafka_response).to eq(0)
         expect(create_acl_report.rdkafka_response_string).to eq("")
 
+        # Since we create and immediately check, this is slow on loaded CIs, hence we wait
+        sleep(2)
+
         #describe_acl
         describe_acl_handle = admin.describe_acl(resource_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_ANY, resource_name: nil, resource_pattern_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_ANY, principal: nil, host: nil, operation: Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_ANY, permission_type: Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ANY)
         describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
@@ -396,7 +681,10 @@ describe Rdkafka::Admin do
     end
 
     context 'when topic has less then desired number of partitions' do
-      before { admin.create_topic(topic_name, 1, 1).wait }
+      before do
+        admin.create_topic(topic_name, 1, 1).wait
+        sleep(1)
+      end
 
       it 'expect to change number of partitions' do
         admin.create_partitions(topic_name, 10).wait
@@ -404,4 +692,41 @@ describe Rdkafka::Admin do
       end
     end
   end
+
+  describe '#oauthbearer_set_token' do
+    context 'when sasl not configured' do
+      it 'should return RD_KAFKA_RESP_ERR__STATE' do
+        response = admin.oauthbearer_set_token(
+          token: "foo",
+          lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+          principal_name: "kafka-cluster"
+        )
+        expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
+      end
+    end
+
+    context 'when sasl configured' do
+      before do
+        config_sasl = rdkafka_config(
+          "security.protocol": "sasl_ssl",
+          "sasl.mechanisms": 'OAUTHBEARER'
+        )
+        $admin_sasl = config_sasl.admin
+      end
+
+      after do
+        $admin_sasl.close
+      end
+
+      it 'should succeed' do
+
+        response = $admin_sasl.oauthbearer_set_token(
+          token: "foo",
+          lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+          principal_name: "kafka-cluster"
+        )
+        expect(response).to eq(0)
+      end
+    end
+  end
 end