karafka-rdkafka 0.20.0.rc2 → 0.20.0

data/spec/rdkafka/admin_spec.rb

@@ -34,7 +34,7 @@ describe Rdkafka::Admin do
   describe '#describe_errors' do
     let(:errors) { admin.class.describe_errors }
 
-    it { expect(errors.size).to eq(172) }
+    it { expect(errors.size).to eq(170) }
     it { expect(errors[-184]).to eq(code: -184, description: 'Local: Queue full', name: '_QUEUE_FULL') }
     it { expect(errors[21]).to eq(code: 21, description: 'Broker: Invalid required acks value', name: 'INVALID_REQUIRED_ACKS') }
   end
@@ -295,6 +295,8 @@ describe Rdkafka::Admin do
         expect(resources_results.first.type).to eq(2)
         expect(resources_results.first.name).to eq(topic_name)
 
+        sleep(1)
+
         ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
           config.name == 'delete.retention.ms'
         end
@@ -325,6 +327,9 @@ describe Rdkafka::Admin do
         expect(resources_results.size).to eq(1)
         expect(resources_results.first.type).to eq(2)
         expect(resources_results.first.name).to eq(topic_name)
+
+        sleep(1)
+
         ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
           config.name == 'delete.retention.ms'
         end
@@ -356,6 +361,8 @@ describe Rdkafka::Admin do
         expect(resources_results.first.type).to eq(2)
         expect(resources_results.first.name).to eq(topic_name)
 
+        sleep(1)
+
         ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
           config.name == 'cleanup.policy'
         end
@@ -387,6 +394,8 @@ describe Rdkafka::Admin do
         expect(resources_results.first.type).to eq(2)
         expect(resources_results.first.name).to eq(topic_name)
 
+        sleep(1)
+
         ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
           config.name == 'cleanup.policy'
         end
@@ -504,7 +513,7 @@ describe Rdkafka::Admin do
     end
   end
 
-  describe "#ACL tests" do
+  describe "#ACL tests for topic resource" do
     let(:non_existing_resource_name) {"non-existing-topic"}
     before do
       #create topic for testing acl
@@ -606,6 +615,207 @@ describe Rdkafka::Admin do
     end
   end
 
+  describe "#ACL tests for transactional_id" do
+    let(:transactional_id_resource_name) {"test-transactional-id"}
+    let(:non_existing_transactional_id) {"non-existing-transactional-id"}
+    let(:transactional_id_resource_type) { Rdkafka::Bindings::RD_KAFKA_RESOURCE_TRANSACTIONAL_ID }
+    let(:transactional_id_resource_pattern_type) { Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL }
+    let(:transactional_id_principal) { "User:test-user" }
+    let(:transactional_id_host) { "*" }
+    let(:transactional_id_operation) { Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_WRITE }
+    let(:transactional_id_permission_type) { Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW }
+
+    after do
+      # Clean up any ACLs that might have been created during tests
+      begin
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_handle.wait(max_wait_timeout: 15.0)
+      rescue
+        # Ignore cleanup errors
+      end
+    end
+
+    describe "#create_acl" do
+      it "creates acl for a transactional_id" do
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: transactional_id_resource_name,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+      end
+
+      it "creates acl for a non-existing transactional_id" do
+        # ACL creation for transactional_ids that don't exist will still get created successfully
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Clean up the ACL that was created for the non-existing transactional_id
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.size).to eq(1)
+      end
+    end
+
+    describe "#describe_acl" do
+      it "describes acl of a transactional_id that does not exist" do
+        describe_acl_handle = admin.describe_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(describe_acl_handle[:response]).to eq(0)
+        expect(describe_acl_report.acls.size).to eq(0)
+      end
+
+      it "creates acls and describes the newly created transactional_id acls" do
+        # Create first ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_1",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Create second ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_2",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Since we create and immediately check, this is slow on loaded CIs, hence we wait
+        sleep(2)
+
+        # Describe ACLs - filter by transactional_id resource type
+        describe_acl_handle = admin.describe_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_ANY,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(describe_acl_handle[:response]).to eq(0)
+        expect(describe_acl_report.acls.length).to eq(2)
+      end
+    end
+
+    describe "#delete_acl" do
+      it "deletes acl of a transactional_id that does not exist" do
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.size).to eq(0)
+      end
+
+      it "creates transactional_id acls and deletes the newly created acls" do
+        # Create first ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_1",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Create second ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_2",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Delete ACLs - resource_name nil to delete all ACLs with any resource name and matching all other filters
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.length).to eq(2)
+      end
+    end
+  end
+
   describe('Group tests') do
     describe "#delete_group" do
       describe("with an existing group") do
@@ -622,7 +832,11 @@ describe Rdkafka::Admin do
 
           consumer.subscribe(topic_name)
           wait_for_assignment(consumer)
-          message = consumer.poll(100)
+          message = nil
+
+          10.times do
+            message ||= consumer.poll(100)
+          end
 
           expect(message).to_not be_nil
 

data/spec/rdkafka/bindings_spec.rb

@@ -77,30 +77,6 @@ describe Rdkafka::Bindings do
     end
   end
 
-  describe "partitioner" do
-    let(:partition_key) { ('a'..'z').to_a.shuffle.take(15).join('') }
-    let(:partition_count) { rand(50) + 1 }
-
-    it "should return the same partition for a similar string and the same partition count" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      result_2 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      expect(result_1).to eq(result_2)
-    end
-
-    it "should match the old partitioner" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      result_2 = (Zlib.crc32(partition_key) % partition_count)
-      expect(result_1).to eq(result_2)
-    end
-
-    it "should return the partition calculated by the specified partitioner" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count, "murmur2")
-      ptr = FFI::MemoryPointer.from_string(partition_key)
-      result_2 = Rdkafka::Bindings.rd_kafka_msg_partitioner_murmur2(nil, ptr, partition_key.size, partition_count, nil, nil)
-      expect(result_1).to eq(result_2)
-    end
-  end
-
   describe "stats callback" do
     context "without a stats callback" do
       it "should do nothing" do
@@ -194,7 +170,6 @@ describe Rdkafka::Bindings do
   end
 
   describe "oauthbearer callback" do
-
     context "without an oauthbearer callback" do
       it "should do nothing" do
         expect {

data/spec/rdkafka/config_spec.rb

@@ -159,7 +159,7 @@ describe Rdkafka::Config do
 
     it "should use default configuration" do
       config = Rdkafka::Config.new
-      expect(config[:"api.version.request"]).to eq nil
+      expect(config[:"api.version.request"]).to eq true
     end
 
     it "should create a consumer with valid config" do

data/spec/rdkafka/consumer_spec.rb

@@ -170,8 +170,16 @@ describe Rdkafka::Consumer do
   end
 
   describe "#seek" do
+    let(:topic) { "it-#{SecureRandom.uuid}" }
+
+    before do
+      admin = rdkafka_producer_config.admin
+      admin.create_topic(topic, 1, 1).wait
+      admin.close
+    end
+
     it "should raise an error when seeking fails" do
-      fake_msg = OpenStruct.new(topic: "consume_test_topic", partition: 0, offset: 0)
+      fake_msg = OpenStruct.new(topic: topic, partition: 0, offset: 0)
 
       expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
       expect {
@@ -181,9 +189,12 @@ describe Rdkafka::Consumer do
 
     context "subscription" do
       let(:timeout) { 1000 }
+      # Some specs here test the manual offset commit hence we want to ensure, that we have some
+      # offsets in-memory that we can manually commit
+      let(:consumer) { rdkafka_consumer_config('auto.commit.interval.ms': 60_000).consumer }
 
       before do
-        consumer.subscribe("consume_test_topic")
+        consumer.subscribe(topic)
 
         # 1. partitions are assigned
         wait_for_assignment(consumer)
@@ -196,7 +207,7 @@ describe Rdkafka::Consumer do
 
       def send_one_message(val)
         producer.produce(
-          topic:     "consume_test_topic",
+          topic:     topic,
           payload:   "payload #{val}",
           key:       "key 1",
           partition: 0
@@ -211,7 +222,7 @@ describe Rdkafka::Consumer do
 
         # 4. pause the subscription
         tpl = Rdkafka::Consumer::TopicPartitionList.new
-        tpl.add_topic("consume_test_topic", 1)
+        tpl.add_topic(topic, 1)
         consumer.pause(tpl)
 
         # 5. seek to previous message
@@ -219,7 +230,7 @@ describe Rdkafka::Consumer do
 
         # 6. resume the subscription
         tpl = Rdkafka::Consumer::TopicPartitionList.new
-        tpl.add_topic("consume_test_topic", 1)
+        tpl.add_topic(topic, 1)
         consumer.resume(tpl)
 
         # 7. ensure same message is read again
@@ -227,7 +238,7 @@ describe Rdkafka::Consumer do
 
         # This is needed because `enable.auto.offset.store` is true but when running in CI that
         # is overloaded, offset store lags
-        sleep(2)
+        sleep(1)
 
         consumer.commit
         expect(message1.offset).to eq message2.offset
@@ -259,10 +270,17 @@ describe Rdkafka::Consumer do
   end
 
   describe "#seek_by" do
-    let(:topic) { "consume_test_topic" }
+    let(:consumer) { rdkafka_consumer_config('auto.commit.interval.ms': 60_000).consumer }
+    let(:topic) { "it-#{SecureRandom.uuid}" }
     let(:partition) { 0 }
     let(:offset) { 0 }
 
+    before do
+      admin = rdkafka_producer_config.admin
+      admin.create_topic(topic, 1, 1).wait
+      admin.close
+    end
+
     it "should raise an error when seeking fails" do
       expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
       expect {
@@ -283,6 +301,7 @@ describe Rdkafka::Consumer do
         # 2. eat unrelated messages
         while(consumer.poll(timeout)) do; end
       end
+
       after { consumer.unsubscribe }
 
       def send_one_message(val)
@@ -813,12 +832,14 @@ describe Rdkafka::Consumer do
     end
 
     it "should return a message if there is one" do
+      topic = "it-#{SecureRandom.uuid}"
+
       producer.produce(
-        topic:     "consume_test_topic",
+        topic:     topic,
         payload:   "payload 1",
         key:       "key 1"
       ).wait
-      consumer.subscribe("consume_test_topic")
+      consumer.subscribe(topic)
       message = consumer.each {|m| break m}
 
       expect(message).to be_a Rdkafka::Consumer::Message
@@ -838,13 +859,13 @@ describe Rdkafka::Consumer do
       }.to raise_error Rdkafka::RdkafkaError
     end
 
-    it "expect not to raise error when polling non-existing topic" do
+    it "expect to raise error when polling non-existing topic" do
       missing_topic = SecureRandom.uuid
       consumer.subscribe(missing_topic)
 
-      # @note it used to raise "Subscribed topic not available" in previous librdkafka versions
-      #   but this behaviour has been changed
-      expect { consumer.poll(1_000) }.not_to raise_error
+      expect {
+        consumer.poll(1_000)
+      }.to raise_error Rdkafka::RdkafkaError, /Subscribed topic not available: #{missing_topic}/
     end
   end
 
@@ -1027,7 +1048,7 @@ describe Rdkafka::Consumer do
     after { Rdkafka::Config.statistics_callback = nil }
 
     let(:consumer) do
-      config = rdkafka_consumer_config('statistics.interval.ms': 100)
+      config = rdkafka_consumer_config('statistics.interval.ms': 500)
       config.consumer_poll_set = false
       config.consumer
     end
@@ -1270,9 +1291,6 @@ describe Rdkafka::Consumer do
       end
 
       expect(eof_error.code).to eq(:partition_eof)
-      expect(eof_error.details[:topic]).to eq('consume_test_topic')
-      expect(eof_error.details[:partition]).to be_a(Integer)
-      expect(eof_error.details[:offset]).to be_a(Integer)
     end
   end
 end

data/spec/rdkafka/metadata_spec.rb

@@ -30,7 +30,7 @@ describe Rdkafka::Metadata do
       it "#brokers returns our single broker" do
         expect(subject.brokers.length).to eq(1)
         expect(subject.brokers[0][:broker_id]).to eq(1)
-        expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
+        expect(%w[127.0.0.1 localhost]).to include(subject.brokers[0][:broker_name])
         expect(subject.brokers[0][:broker_port]).to eq(9092)
       end
 
@@ -53,7 +53,7 @@ describe Rdkafka::Metadata do
     it "#brokers returns our single broker" do
       expect(subject.brokers.length).to eq(1)
       expect(subject.brokers[0][:broker_id]).to eq(1)
-      expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
+      expect(%w[127.0.0.1 localhost]).to include(subject.brokers[0][:broker_name])
       expect(subject.brokers[0][:broker_port]).to eq(9092)
     end