rdkafka 0.21.1.alpha2 → 0.22.0

data/renovate.json

@@ -14,5 +14,79 @@
       ],
       "minimumReleaseAge": "7 days"
     }
+  ],
+  "customManagers": [
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly OPENSSL_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "openssl/openssl",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^OpenSSL_(?<version>.*)$"
+    },
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly CYRUS_SASL_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "cyrusimap/cyrus-sasl",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^cyrus-sasl-(?<version>.*)$"
+    },
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly ZLIB_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "madler/zlib",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^v(?<version>.*)$"
+    },
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly ZSTD_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "facebook/zstd",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^v(?<version>.*)$"
+    },
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly KRB5_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "krb5/krb5",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^krb5-(?<version>.*)$"
+    },
+    {
+      "customType": "regex",
+      "managerFilePatterns": [
+        "/^ext/build_common\\.sh$/"
+      ],
+      "matchStrings": [
+        "readonly LIBRDKAFKA_VERSION=\"(?<currentValue>.*)\""
+      ],
+      "depNameTemplate": "confluentinc/librdkafka",
+      "datasourceTemplate": "github-releases",
+      "extractVersionTemplate": "^v(?<version>.*)$"
+    }
   ]
 }

data/spec/rdkafka/admin_spec.rb

@@ -295,6 +295,8 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
         expect(resources_results.first.type).to eq(2)
         expect(resources_results.first.name).to eq(topic_name)
 
+        sleep(1)
+
         ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
           config.name == 'delete.retention.ms'
         end
@@ -325,6 +327,9 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
         expect(resources_results.size).to eq(1)
         expect(resources_results.first.type).to eq(2)
         expect(resources_results.first.name).to eq(topic_name)
+
+        sleep(1)
+
         ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
           config.name == 'delete.retention.ms'
         end
@@ -356,6 +361,8 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
         expect(resources_results.first.type).to eq(2)
         expect(resources_results.first.name).to eq(topic_name)
 
+        sleep(1)
+
         ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
           config.name == 'cleanup.policy'
         end
@@ -387,6 +394,8 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
         expect(resources_results.first.type).to eq(2)
         expect(resources_results.first.name).to eq(topic_name)
 
+        sleep(1)
+
         ret_config = admin.describe_configs(resources_with_configs).wait.resources.first.configs.find do |config|
           config.name == 'cleanup.policy'
         end
@@ -504,7 +513,7 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
     end
   end
 
-  describe "#ACL tests" do
+  describe "#ACL tests for topic resource" do
     let(:non_existing_resource_name) {"non-existing-topic"}
     before do
       #create topic for testing acl
@@ -606,6 +615,207 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
     end
   end
 
+  describe "#ACL tests for transactional_id" do
+    let(:transactional_id_resource_name) {"test-transactional-id"}
+    let(:non_existing_transactional_id) {"non-existing-transactional-id"}
+    let(:transactional_id_resource_type) { Rdkafka::Bindings::RD_KAFKA_RESOURCE_TRANSACTIONAL_ID }
+    let(:transactional_id_resource_pattern_type) { Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL }
+    let(:transactional_id_principal) { "User:test-user" }
+    let(:transactional_id_host) { "*" }
+    let(:transactional_id_operation) { Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_WRITE }
+    let(:transactional_id_permission_type) { Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW }
+
+    after do
+      # Clean up any ACLs that might have been created during tests
+      begin
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_handle.wait(max_wait_timeout: 15.0)
+      rescue
+        # Ignore cleanup errors
+      end
+    end
+
+    describe "#create_acl" do
+      it "creates acl for a transactional_id" do
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: transactional_id_resource_name,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+      end
+
+      it "creates acl for a non-existing transactional_id" do
+        # ACL creation for transactional_ids that don't exist will still get created successfully
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Clean up the ACL that was created for the non-existing transactional_id
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.size).to eq(1)
+      end
+    end
+
+    describe "#describe_acl" do
+      it "describes acl of a transactional_id that does not exist" do
+        describe_acl_handle = admin.describe_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(describe_acl_handle[:response]).to eq(0)
+        expect(describe_acl_report.acls.size).to eq(0)
+      end
+
+      it "creates acls and describes the newly created transactional_id acls" do
+        # Create first ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_1",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Create second ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_2",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Since we create and immediately check, this is slow on loaded CIs, hence we wait
+        sleep(2)
+
+        # Describe ACLs - filter by transactional_id resource type
+        describe_acl_handle = admin.describe_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_ANY,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(describe_acl_handle[:response]).to eq(0)
+        expect(describe_acl_report.acls.length).to eq(2)
+      end
+    end
+
+    describe "#delete_acl" do
+      it "deletes acl of a transactional_id that does not exist" do
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: non_existing_transactional_id,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.size).to eq(0)
+      end
+
+      it "creates transactional_id acls and deletes the newly created acls" do
+        # Create first ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_1",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Create second ACL
+        create_acl_handle = admin.create_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: "test_transactional_id_2",
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # Delete ACLs - resource_name nil to delete all ACLs with any resource name and matching all other filters
+        delete_acl_handle = admin.delete_acl(
+          resource_type: transactional_id_resource_type,
+          resource_name: nil,
+          resource_pattern_type: transactional_id_resource_pattern_type,
+          principal: transactional_id_principal,
+          host: transactional_id_host,
+          operation: transactional_id_operation,
+          permission_type: transactional_id_permission_type
+        )
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.length).to eq(2)
+      end
+    end
+  end
+
   describe('Group tests') do
     describe "#delete_group" do
       describe("with an existing group") do
@@ -622,7 +832,12 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
 
           consumer.subscribe(topic_name)
           wait_for_assignment(consumer)
-          message = consumer.poll(100)
+
+          message = nil
+
+          10.times do
+            message ||= consumer.poll(100)
+          end
 
           expect(message).to_not be_nil
 

data/spec/rdkafka/bindings_spec.rb

@@ -77,30 +77,6 @@ describe Rdkafka::Bindings do
     end
   end
 
-  describe "partitioner" do
-    let(:partition_key) { ('a'..'z').to_a.shuffle.take(15).join('') }
-    let(:partition_count) { rand(50) + 1 }
-
-    it "should return the same partition for a similar string and the same partition count" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      result_2 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      expect(result_1).to eq(result_2)
-    end
-
-    it "should match the old partitioner" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
-      result_2 = (Zlib.crc32(partition_key) % partition_count)
-      expect(result_1).to eq(result_2)
-    end
-
-    it "should return the partition calculated by the specified partitioner" do
-      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count, "murmur2")
-      ptr = FFI::MemoryPointer.from_string(partition_key)
-      result_2 = Rdkafka::Bindings.rd_kafka_msg_partitioner_murmur2(nil, ptr, partition_key.size, partition_count, nil, nil)
-      expect(result_1).to eq(result_2)
-    end
-  end
-
   describe "stats callback" do
     context "without a stats callback" do
       it "should do nothing" do

data/spec/rdkafka/consumer_spec.rb

@@ -170,8 +170,16 @@ describe Rdkafka::Consumer do
   end
 
   describe "#seek" do
+    let(:topic) { "it-#{SecureRandom.uuid}" }
+
+    before do
+      admin = rdkafka_producer_config.admin
+      admin.create_topic(topic, 1, 1).wait
+      admin.close
+    end
+
     it "should raise an error when seeking fails" do
-      fake_msg = OpenStruct.new(topic: "consume_test_topic", partition: 0, offset: 0)
+      fake_msg = OpenStruct.new(topic: topic, partition: 0, offset: 0)
 
       expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
       expect {
@@ -181,9 +189,12 @@ describe Rdkafka::Consumer do
 
     context "subscription" do
       let(:timeout) { 1000 }
+      # Some specs here test the manual offset commit hence we want to ensure, that we have some
+      # offsets in-memory that we can manually commit
+      let(:consumer) { rdkafka_consumer_config('auto.commit.interval.ms': 60_000).consumer }
 
       before do
-        consumer.subscribe("consume_test_topic")
+        consumer.subscribe(topic)
 
         # 1. partitions are assigned
         wait_for_assignment(consumer)
@@ -196,7 +207,7 @@ describe Rdkafka::Consumer do
 
       def send_one_message(val)
         producer.produce(
-          topic:     "consume_test_topic",
+          topic:     topic,
           payload:   "payload #{val}",
           key:       "key 1",
           partition: 0
@@ -211,7 +222,7 @@ describe Rdkafka::Consumer do
 
         # 4. pause the subscription
         tpl = Rdkafka::Consumer::TopicPartitionList.new
-        tpl.add_topic("consume_test_topic", 1)
+        tpl.add_topic(topic, 1)
         consumer.pause(tpl)
 
         # 5. seek to previous message
@@ -219,7 +230,7 @@ describe Rdkafka::Consumer do
 
         # 6. resume the subscription
         tpl = Rdkafka::Consumer::TopicPartitionList.new
-        tpl.add_topic("consume_test_topic", 1)
+        tpl.add_topic(topic, 1)
         consumer.resume(tpl)
 
         # 7. ensure same message is read again
@@ -227,7 +238,7 @@ describe Rdkafka::Consumer do
 
         # This is needed because `enable.auto.offset.store` is true but when running in CI that
         # is overloaded, offset store lags
-        sleep(2)
+        sleep(1)
 
         consumer.commit
         expect(message1.offset).to eq message2.offset
@@ -259,10 +270,17 @@ describe Rdkafka::Consumer do
   end
 
   describe "#seek_by" do
-    let(:topic) { "consume_test_topic" }
+    let(:consumer) { rdkafka_consumer_config('auto.commit.interval.ms': 60_000).consumer }
+    let(:topic) { "it-#{SecureRandom.uuid}" }
     let(:partition) { 0 }
     let(:offset) { 0 }
 
+    before do
+      admin = rdkafka_producer_config.admin
+      admin.create_topic(topic, 1, 1).wait
+      admin.close
+    end
+
     it "should raise an error when seeking fails" do
       expect(Rdkafka::Bindings).to receive(:rd_kafka_seek).and_return(20)
       expect {
@@ -588,12 +606,18 @@ describe Rdkafka::Consumer do
 
       describe "#store_offset" do
         let(:consumer) { rdkafka_consumer_config('enable.auto.offset.store': false).consumer }
+        let(:metadata) { SecureRandom.uuid }
+        let(:group_id) { SecureRandom.uuid }
+        let(:base_config) do
+          {
+            'group.id': group_id,
+            'enable.auto.offset.store': false,
+            'enable.auto.commit': false
+          }
+        end
 
         before do
-          config = {}
-          config[:'enable.auto.offset.store'] = false
-          config[:'enable.auto.commit'] = false
-          @new_consumer = rdkafka_consumer_config(config).consumer
+          @new_consumer = rdkafka_consumer_config(base_config).consumer
           @new_consumer.subscribe("consume_test_topic")
           wait_for_assignment(@new_consumer)
         end
@@ -795,12 +819,14 @@ describe Rdkafka::Consumer do
     end
 
     it "should return a message if there is one" do
+      topic = "it-#{SecureRandom.uuid}"
+
       producer.produce(
-        topic:     "consume_test_topic",
+        topic:     topic,
         payload:   "payload 1",
         key:       "key 1"
       ).wait
-      consumer.subscribe("consume_test_topic")
+      consumer.subscribe(topic)
       message = consumer.each {|m| break m}
 
       expect(message).to be_a Rdkafka::Consumer::Message
@@ -1000,7 +1026,7 @@ describe Rdkafka::Consumer do
     after { Rdkafka::Config.statistics_callback = nil }
 
     let(:consumer) do
-      config = rdkafka_consumer_config('statistics.interval.ms': 100)
+      config = rdkafka_consumer_config('statistics.interval.ms': 500)
       config.consumer_poll_set = false
       config.consumer
     end
@@ -1097,7 +1123,8 @@ describe Rdkafka::Consumer do
         :assign                  => [ nil ],
         :assignment              => nil,
         :committed               => [],
-        :query_watermark_offsets => [ nil, nil ]
+        :query_watermark_offsets => [ nil, nil ],
+        :assignment_lost?        => []
     }.each do |method, args|
       it "raises an exception if #{method} is called" do
         expect {
@@ -1212,4 +1239,36 @@ describe Rdkafka::Consumer do
       end
     end
   end
+
+  describe "when reaching eof on a topic and eof reporting enabled" do
+    let(:consumer) { rdkafka_consumer_config(:"enable.partition.eof" => true).consumer }
+
+    it "should return proper details" do
+      (0..2).each do |i|
+        producer.produce(
+          topic:     "consume_test_topic",
+          key:       "key lag #{i}",
+          partition: i
+        ).wait
+      end
+
+      # Consume to the end
+      consumer.subscribe("consume_test_topic")
+      eof_count = 0
+      eof_error = nil
+
+      loop do
+        begin
+          consumer.poll(100)
+        rescue Rdkafka::RdkafkaError => error
+          if error.is_partition_eof?
+            eof_error = error
+          end
+          break if eof_error
+        end
+      end
+
+      expect(eof_error.code).to eq(:partition_eof)
+    end
+  end
 end

data/spec/rdkafka/metadata_spec.rb

@@ -30,7 +30,7 @@ describe Rdkafka::Metadata do
       it "#brokers returns our single broker" do
         expect(subject.brokers.length).to eq(1)
         expect(subject.brokers[0][:broker_id]).to eq(1)
-        expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
+        expect(%w[127.0.0.1 localhost]).to include(subject.brokers[0][:broker_name])
         expect(subject.brokers[0][:broker_port]).to eq(9092)
       end
 
@@ -53,7 +53,7 @@ describe Rdkafka::Metadata do
     it "#brokers returns our single broker" do
       expect(subject.brokers.length).to eq(1)
       expect(subject.brokers[0][:broker_id]).to eq(1)
-      expect(subject.brokers[0][:broker_name]).to eq("127.0.0.1")
+      expect(%w[127.0.0.1 localhost]).to include(subject.brokers[0][:broker_name])
       expect(subject.brokers[0][:broker_port]).to eq(9092)
     end
 

data/spec/rdkafka/producer/{partitions_count_spec.rb → partitions_count_cache_spec.rb}

@@ -257,7 +257,7 @@ RSpec.describe Rdkafka::Producer::PartitionsCountCache do
       cache.get(topic) { partition_count }
 
       # Wait just under TTL
-      sleep(default_ttl - 0.1)
+      sleep(default_ttl - 0.2)
 
       # Value should still be cached (block should not be called)
       result = cache.get(topic) { fail "Should not be called when cache is valid" }