rdkafka 0.15.1 → 0.16.0.rc1

data/spec/rdkafka/admin_spec.rb

@@ -16,12 +16,12 @@ describe Rdkafka::Admin do
     admin.close
   end
 
-  let(:topic_name)               { "test-topic-#{Random.new.rand(0..1_000_000)}" }
+  let(:topic_name)               { "test-topic-#{SecureRandom.uuid}" }
   let(:topic_partition_count)    { 3 }
   let(:topic_replication_factor) { 1 }
   let(:topic_config)             { {"cleanup.policy" => "compact", "min.cleanable.dirty.ratio" => 0.8} }
   let(:invalid_topic_config)     { {"cleeeeenup.policee" => "campact"} }
-  let(:group_name)               { "test-group-#{Random.new.rand(0..1_000_000)}" }
+  let(:group_name)               { "test-group-#{SecureRandom.uuid}" }
 
   let(:resource_name)         {"acl-test-topic"}
   let(:resource_type)         {Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC}
@@ -31,6 +31,27 @@ describe Rdkafka::Admin do
   let(:operation)             {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
   let(:permission_type)       {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
 
+  describe '#describe_errors' do
+    let(:errors) { admin.class.describe_errors }
+
+    it { expect(errors.size).to eq(162) }
+    it { expect(errors[-184]).to eq(code: -184, description: 'Local: Queue full', name: '_QUEUE_FULL') }
+    it { expect(errors[21]).to eq(code: 21, description: 'Broker: Invalid required acks value', name: 'INVALID_REQUIRED_ACKS') }
+  end
+
+  describe 'admin without auto-start' do
+    let(:admin) { config.admin(native_kafka_auto_start: false) }
+
+    it 'expect to be able to start it later and close' do
+      admin.start
+      admin.close
+    end
+
+    it 'expect to be able to close it without starting' do
+      admin.close
+    end
+  end
+
   describe "#create_topic" do
     describe "called with invalid input" do
       describe "with an invalid topic name" do
@@ -129,6 +150,275 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
     end
   end
 
+  describe "describe_configs" do
+    subject(:resources_results) { admin.describe_configs(resources).wait.resources }
+
+    before do
+      admin.create_topic(topic_name, 2, 1).wait
+      sleep(1)
+    end
+
+    context 'when describing config of an existing topic' do
+      let(:resources) { [{ resource_type: 2, resource_name: topic_name }] }
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq(topic_name)
+        expect(resources_results.first.configs.size).to be > 25
+        expect(resources_results.first.configs.first.name).to eq('compression.type')
+        expect(resources_results.first.configs.first.value).to eq('producer')
+        expect(resources_results.first.configs.map(&:synonyms)).not_to be_empty
+      end
+    end
+
+    context 'when describing config of a non-existing topic' do
+      let(:resources) { [{ resource_type: 2, resource_name: SecureRandom.uuid }] }
+
+      it 'expect to raise error' do
+        expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /unknown_topic_or_part/)
+      end
+    end
+
+    context 'when describing both existing and non-existing topics' do
+      let(:resources) do
+        [
+          { resource_type: 2, resource_name: topic_name },
+          { resource_type: 2, resource_name: SecureRandom.uuid }
+        ]
+      end
+
+      it 'expect to raise error' do
+        expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /unknown_topic_or_part/)
+      end
+    end
+
+    context 'when describing multiple existing topics' do
+      let(:resources) do
+        [
+          { resource_type: 2, resource_name: 'example_topic' },
+          { resource_type: 2, resource_name: topic_name }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(2)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq('example_topic')
+        expect(resources_results.last.type).to eq(2)
+        expect(resources_results.last.name).to eq(topic_name)
+      end
+    end
+
+    context 'when trying to describe invalid resource type' do
+      let(:resources) { [{ resource_type: 0, resource_name: SecureRandom.uuid }] }
+
+      it 'expect to raise error' do
+        expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_request/)
+      end
+    end
+
+    context 'when trying to describe invalid broker' do
+      let(:resources) { [{ resource_type: 4, resource_name: 'non-existing' }] }
+
+      it 'expect to raise error' do
+        expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_arg/)
+      end
+    end
+
+    context 'when trying to describe valid broker' do
+      let(:resources) { [{ resource_type: 4, resource_name: '1' }] }
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(4)
+        expect(resources_results.first.name).to eq('1')
+        expect(resources_results.first.configs.size).to be > 230
+        expect(resources_results.first.configs.first.name).to eq('log.cleaner.min.compaction.lag.ms')
+        expect(resources_results.first.configs.first.value).to eq('0')
+        expect(resources_results.first.configs.map(&:synonyms)).not_to be_empty
+      end
+    end
+
+    context 'when describing valid broker with topics in one request' do
+      let(:resources) do
+        [
+          { resource_type: 4, resource_name: '1' },
+          { resource_type: 2, resource_name: topic_name }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(2)
+        expect(resources_results.first.type).to eq(4)
+        expect(resources_results.first.name).to eq('1')
+        expect(resources_results.first.configs.size).to be > 230
+        expect(resources_results.first.configs.first.name).to eq('log.cleaner.min.compaction.lag.ms')
+        expect(resources_results.first.configs.first.value).to eq('0')
+        expect(resources_results.last.type).to eq(2)
+        expect(resources_results.last.name).to eq(topic_name)
+        expect(resources_results.last.configs.size).to be > 25
+        expect(resources_results.last.configs.first.name).to eq('compression.type')
+        expect(resources_results.last.configs.first.value).to eq('producer')
+      end
+    end
+  end
+
+  describe "incremental_alter_configs" do
+    subject(:resources_results) { admin.incremental_alter_configs(resources_with_configs).wait.resources }
+
+    before do
+      admin.create_topic(topic_name, 2, 1).wait
+      sleep(1)
+    end
+
+    context 'when altering one topic with one valid config via set' do
+      let(:target_retention) { (86400002 + rand(10_000)).to_s }
+      let(:resources_with_configs) do
+        [
+          {
+            resource_type: 2,
+            resource_name: topic_name,
+            configs: [
+              {
+                name: 'delete.retention.ms',
+                value: target_retention,
+                op_type: 0
+              }
+            ]
+          }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq(topic_name)
+
+        ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
+          config.name == 'delete.retention.ms'
+        end
+
+        expect(ret_config.value).to eq(target_retention)
+      end
+    end
+
+    context 'when altering one topic with one valid config via delete' do
+      let(:target_retention) { (8640002 + rand(10_000)).to_s }
+      let(:resources_with_configs) do
+        [
+          {
+            resource_type: 2,
+            resource_name: topic_name,
+            configs: [
+              {
+                name: 'delete.retention.ms',
+                value: target_retention,
+                op_type: 1
+              }
+            ]
+          }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq(topic_name)
+        ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
+          config.name == 'delete.retention.ms'
+        end
+
+        expect(ret_config.value).to eq('86400000')
+      end
+    end
+
+    context 'when altering one topic with one valid config via append' do
+      let(:target_policy) { 'compact' }
+      let(:resources_with_configs) do
+        [
+          {
+            resource_type: 2,
+            resource_name: topic_name,
+            configs: [
+              {
+                name: 'cleanup.policy',
+                value: target_policy,
+                op_type: 2
+              }
+            ]
+          }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq(topic_name)
+
+        ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
+          config.name == 'cleanup.policy'
+        end
+
+        expect(ret_config.value).to eq("delete,#{target_policy}")
+      end
+    end
+
+    context 'when altering one topic with one valid config via subtrack' do
+      let(:target_policy) { 'delete' }
+      let(:resources_with_configs) do
+        [
+          {
+            resource_type: 2,
+            resource_name: topic_name,
+            configs: [
+              {
+                name: 'cleanup.policy',
+                value: target_policy,
+                op_type: 3
+              }
+            ]
+          }
+        ]
+      end
+
+      it do
+        expect(resources_results.size).to eq(1)
+        expect(resources_results.first.type).to eq(2)
+        expect(resources_results.first.name).to eq(topic_name)
+
+        ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
+          config.name == 'cleanup.policy'
+        end
+
+        expect(ret_config.value).to eq('')
+      end
+    end
+
+    context 'when altering one topic with invalid config' do
+      let(:target_retention) { '-10' }
+      let(:resources_with_configs) do
+        [
+          {
+            resource_type: 2,
+            resource_name: topic_name,
+            configs: [
+              {
+                name: 'delete.retention.ms',
+                value: target_retention,
+                op_type: 0
+              }
+            ]
+          }
+        ]
+      end
+
+      it 'expect to raise error' do
+        expect { resources_results }.to raise_error(Rdkafka::RdkafkaError, /invalid_config/)
+      end
+    end
+  end
+
   describe "#delete_topic" do
     describe "called with invalid input" do
       # https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L29
@@ -275,6 +565,9 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
         expect(create_acl_report.rdkafka_response).to eq(0)
         expect(create_acl_report.rdkafka_response_string).to eq("")
 
+        # Since we create and immediately check, this is slow on loaded CIs, hence we wait
+        sleep(2)
+
         #describe_acl
         describe_acl_handle = admin.describe_acl(resource_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_ANY, resource_name: nil, resource_pattern_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_ANY, principal: nil, host: nil, operation: Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_ANY, permission_type: Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ANY)
         describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
@@ -396,7 +689,10 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
     end
 
     context 'when topic has less then desired number of partitions' do
-      before { admin.create_topic(topic_name, 1, 1).wait }
+      before do
+        admin.create_topic(topic_name, 1, 1).wait
+        sleep(1)
+      end
 
       it 'expect to change number of partitions' do
         admin.create_partitions(topic_name, 10).wait
@@ -404,4 +700,41 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
       end
     end
   end
+
+  describe '#oauthbearer_set_token' do
+    context 'when sasl not configured' do
+      it 'should return RD_KAFKA_RESP_ERR__STATE' do
+        response = admin.oauthbearer_set_token(
+          token: "foo",
+          lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+          principal_name: "kafka-cluster"
+        )
+        expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
+      end
+    end
+
+    context 'when sasl configured' do
+      before do
+        config_sasl = rdkafka_config(
+          "security.protocol": "sasl_ssl",
+          "sasl.mechanisms": 'OAUTHBEARER'
+        )
+        $admin_sasl = config_sasl.admin
+      end
+
+      after do
+        $admin_sasl.close
+      end
+
+      it 'should succeed' do
+
+        response = $admin_sasl.oauthbearer_set_token(
+          token: "foo",
+          lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+          principal_name: "kafka-cluster"
+        )
+        expect(response).to eq(0)
+      end
+    end
+  end
 end

data/spec/rdkafka/bindings_spec.rb

@@ -36,6 +36,16 @@ describe Rdkafka::Bindings do
       expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
     end
 
+    it "should log fatal messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 1, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
+    end
+
+    it "should log fatal messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 2, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
+    end
+
     it "should log error messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 3, nil, "log line")
       expect(log_queue).to have_received(:<<).with([Logger::ERROR, "rdkafka: log line"])
@@ -51,6 +61,11 @@ describe Rdkafka::Bindings do
       expect(log_queue).to have_received(:<<).with([Logger::INFO, "rdkafka: log line"])
     end
 
+    it "should log info messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 6, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::INFO, "rdkafka: log line"])
+    end
+
     it "should log debug messages" do
       Rdkafka::Bindings::LogCallback.call(nil, 7, nil, "log line")
       expect(log_queue).to have_received(:<<).with([Logger::DEBUG, "rdkafka: log line"])
@@ -132,4 +147,86 @@ describe Rdkafka::Bindings do
       end
     end
   end
+
+  describe "oauthbearer set token" do
+
+    context "without args" do
+      it "should raise argument error" do
+        expect {
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token
+        }.to raise_error(ArgumentError)
+      end
+    end
+
+    context "with args" do
+      before do
+        DEFAULT_TOKEN_EXPIRY_SECONDS = 900
+        $token_value = "token"
+        $md_lifetime_ms = Time.now.to_i*1000 + DEFAULT_TOKEN_EXPIRY_SECONDS * 1000
+        $md_principal_name = "kafka-cluster"
+        $extensions = nil
+        $extension_size = 0
+        $error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+      end
+
+      it "should set token or capture failure" do
+        RdKafkaTestConsumer.with do |consumer_ptr|
+          response = Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(consumer_ptr, $token_value, $md_lifetime_ms, $md_principal_name, $extensions, $extension_size, $error_buffer, 256)
+          expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
+          expect($error_buffer.read_string).to eq("SASL/OAUTHBEARER is not the configured authentication mechanism")
+        end
+      end
+    end
+  end
+
+  describe "oauthbearer set token failure" do
+
+    context "without args" do
+
+      it "should fail" do
+        expect {
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure
+        }.to raise_error(ArgumentError)
+      end
+    end
+
+    context "with args" do
+      it "should succeed" do
+        expect {
+          errstr = "error"
+          RdKafkaTestConsumer.with do |consumer_ptr|
+            Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure(consumer_ptr, errstr)
+          end
+        }.to_not raise_error
+      end
+    end
+  end
+
+  describe "oauthbearer callback" do
+
+    context "without an oauthbearer callback" do
+      it "should do nothing" do
+        expect {
+          Rdkafka::Bindings::OAuthbearerTokenRefreshCallback.call(nil, "", nil)
+        }.not_to raise_error
+      end
+    end
+
+    context "with an oauthbearer callback" do
+      before do
+        Rdkafka::Config.oauthbearer_token_refresh_callback = lambda do |config, client_name|
+          $received_config = config
+          $received_client_name = client_name
+        end
+      end
+
+      it "should call the oauth bearer callback and receive config and client name" do
+        RdKafkaTestConsumer.with do |consumer_ptr|
+          Rdkafka::Bindings::OAuthbearerTokenRefreshCallback.call(consumer_ptr, "{}", nil)
+            expect($received_config).to eq("{}")
+            expect($received_client_name).to match(/consumer/)
+        end
+      end
+    end
+  end
 end

data/spec/rdkafka/config_spec.rb

@@ -22,6 +22,7 @@ describe Rdkafka::Config do
     it "supports logging queue" do
       log = StringIO.new
       Rdkafka::Config.logger = Logger.new(log)
+      Rdkafka::Config.ensure_log_thread
 
       Rdkafka::Config.log_queue << [Logger::FATAL, "I love testing"]
       20.times do
@@ -31,6 +32,25 @@ describe Rdkafka::Config do
 
       expect(log.string).to include "FATAL -- : I love testing"
     end
+
+    it "expect to start new logger thread after fork and work" do
+      reader, writer = IO.pipe
+
+      pid = fork do
+        $stdout.reopen(writer)
+        Rdkafka::Config.logger = Logger.new($stdout)
+        reader.close
+        producer = rdkafka_producer_config(debug: 'all').producer
+        producer.close
+        writer.close
+        sleep(1)
+      end
+
+      writer.close
+      Process.wait(pid)
+      output = reader.read
+      expect(output.split("\n").size).to be >= 20
+    end
   end
 
   context "statistics callback" do
@@ -95,6 +115,39 @@ describe Rdkafka::Config do
     end
   end
 
+  context "oauthbearer calllback" do
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.oauthbearer_token_refresh_callback = lambda do |config, client_name|
+            puts config
+            puts client_name
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.oauthbearer_token_refresh_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(config, client_name); end
+        end
+
+        expect {
+          Rdkafka::Config.oauthbearer_token_refresh_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.oauthbearer_token_refresh_callback).to respond_to :call
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        Rdkafka::Config.oauthbearer_token_refresh_callback = 'not a callback'
+      }.to raise_error(TypeError)
+    end
+  end
+
   context "configuration" do
     it "should store configuration" do
       config = Rdkafka::Config.new

data/spec/rdkafka/consumer_spec.rb

@@ -14,6 +14,19 @@ describe Rdkafka::Consumer do
     it { expect(consumer.name).to include('rdkafka#consumer-') }
   end
 
+  describe 'consumer without auto-start' do
+    let(:consumer) { rdkafka_consumer_config.consumer(native_kafka_auto_start: false) }
+
+    it 'expect to be able to start it later and close' do
+      consumer.start
+      consumer.close
+    end
+
+    it 'expect to be able to close it without starting' do
+      consumer.close
+    end
+  end
+
   describe "#subscribe, #unsubscribe and #subscription" do
     it "should subscribe, unsubscribe and return the subscription" do
       expect(consumer.subscription).to be_empty
@@ -211,6 +224,11 @@ describe Rdkafka::Consumer do
 
         # 7. ensure same message is read again
         message2 = consumer.poll(timeout)
+
+        # This is needed because `enable.auto.offset.store` is true but when running in CI that
+        # is overloaded, offset store lags
+        sleep(2)
+
         consumer.commit
         expect(message1.offset).to eq message2.offset
         expect(message1.payload).to eq message2.payload
@@ -1296,4 +1314,40 @@ describe Rdkafka::Consumer do
       ])
     end
   end
+
+  describe '#oauthbearer_set_token' do
+    context 'when sasl not configured' do
+      it 'should return RD_KAFKA_RESP_ERR__STATE' do
+        response = consumer.oauthbearer_set_token(
+          token: "foo",
+          lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+          principal_name: "kafka-cluster"
+        )
+        expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
+      end
+    end
+
+    context 'when sasl configured' do
+      before do
+        $consumer_sasl = rdkafka_producer_config(
+          "security.protocol": "sasl_ssl",
+          "sasl.mechanisms": 'OAUTHBEARER'
+        ).consumer
+      end
+
+      after do
+        $consumer_sasl.close
+      end
+
+      it 'should succeed' do
+
+        response = $consumer_sasl.oauthbearer_set_token(
+          token: "foo",
+          lifetime_ms: Time.now.to_i*1000 + 900 * 1000,
+          principal_name: "kafka-cluster"
+        )
+        expect(response).to eq(0)
+      end
+    end
+  end
 end

data/spec/rdkafka/native_kafka_spec.rb

@@ -10,8 +10,9 @@ describe Rdkafka::NativeKafka do
   subject(:client) { described_class.new(native, run_polling_thread: true, opaque: opaque) }
 
   before do
+    allow(Rdkafka::Bindings).to receive(:rd_kafka_name).and_return('producer-1')
     allow(Thread).to receive(:new).and_return(thread)
-
+    allow(thread).to receive(:name=).with("rdkafka.native_kafka#producer-1")
     allow(thread).to receive(:[]=).with(:closing, anything)
     allow(thread).to receive(:join)
     allow(thread).to receive(:abort_on_exception=).with(anything)
@@ -20,6 +21,12 @@ describe Rdkafka::NativeKafka do
   after { client.close }
 
   context "defaults" do
+    it "sets the thread name" do
+      expect(thread).to receive(:name=).with("rdkafka.native_kafka#producer-1")
+
+      client
+    end
+
     it "sets the thread to abort on exception" do
       expect(thread).to receive(:abort_on_exception=).with(true)