rdkafka 0.13.0 → 0.15.1

data/spec/rdkafka/admin/describe_acl_handle_spec.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Admin::DescribeAclHandle do
+  let(:response) { Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR }
+  let(:resource_name)         {"acl-test-topic"}
+  let(:resource_type)         {Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC}
+  let(:resource_pattern_type) {Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL}
+  let(:principal)             {"User:anonymous"}
+  let(:host)                  {"*"}
+  let(:operation)             {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
+  let(:permission_type)       {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
+  let(:describe_acl_ptr)      {FFI::Pointer::NULL}
+
+  subject do
+    error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+    describe_acl_ptr = Rdkafka::Bindings.rd_kafka_AclBinding_new(
+      resource_type,
+      FFI::MemoryPointer.from_string(resource_name),
+      resource_pattern_type,
+      FFI::MemoryPointer.from_string(principal),
+      FFI::MemoryPointer.from_string(host),
+      operation,
+      permission_type,
+      error_buffer,
+      256
+    )
+    if describe_acl_ptr.null?
+       raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+    end
+    pointer_array = [describe_acl_ptr]
+    describe_acls_array_ptr = FFI::MemoryPointer.new(:pointer)
+    describe_acls_array_ptr.write_array_of_pointer(pointer_array)
+    Rdkafka::Admin::DescribeAclHandle.new.tap do |handle|
+      handle[:pending] = pending_handle
+      handle[:response] = response
+      handle[:response_string] = FFI::MemoryPointer.from_string("")
+      handle[:acls] = describe_acls_array_ptr
+      handle[:acls_count] = 1
+    end
+  end
+
+  after do
+    if describe_acl_ptr != FFI::Pointer::NULL
+      Rdkafka::Bindings.rd_kafka_AclBinding_destroy(describe_acl_ptr)
+    end
+  end
+
+  describe "#wait" do
+    let(:pending_handle) { true }
+
+    it "should wait until the timeout and then raise an error" do
+      expect {
+        subject.wait(max_wait_timeout: 0.1)
+      }.to raise_error Rdkafka::Admin::DescribeAclHandle::WaitTimeoutError, /describe acl/
+    end
+
+    context "when not pending anymore and no error" do
+      let(:pending_handle) { false }
+
+      it "should return a describe acl report" do
+        report = subject.wait
+
+        expect(report.acls.length).to eq(1)
+      end
+
+      it "should wait without a timeout" do
+        report = subject.wait(max_wait_timeout: nil)
+
+        expect(report.acls[0].matching_acl_resource_name).to eq("acl-test-topic")
+      end
+    end
+  end
+
+  describe "#raise_error" do
+    let(:pending_handle) { false }
+
+    it "should raise the appropriate error" do
+      expect {
+        subject.raise_error
+      }.to raise_exception(Rdkafka::RdkafkaError, /Success \(no_error\)/)
+    end
+  end
+end

data/spec/rdkafka/admin/describe_acl_report_spec.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+describe Rdkafka::Admin::DescribeAclReport do
+
+  let(:resource_name)         {"acl-test-topic"}
+  let(:resource_type)         {Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC}
+  let(:resource_pattern_type) {Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL}
+  let(:principal)             {"User:anonymous"}
+  let(:host)                  {"*"}
+  let(:operation)             {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
+  let(:permission_type)       {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
+  let(:describe_acl_ptr)      {FFI::Pointer::NULL}
+
+  subject do
+    error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+    describe_acl_ptr = Rdkafka::Bindings.rd_kafka_AclBinding_new(
+      resource_type,
+      FFI::MemoryPointer.from_string(resource_name),
+      resource_pattern_type,
+      FFI::MemoryPointer.from_string(principal),
+      FFI::MemoryPointer.from_string(host),
+      operation,
+      permission_type,
+      error_buffer,
+      256
+    )
+    if describe_acl_ptr.null?
+       raise Rdkafka::Config::ConfigError.new(error_buffer.read_string)
+    end
+    pointer_array = [describe_acl_ptr]
+    describe_acls_array_ptr = FFI::MemoryPointer.new(:pointer)
+    describe_acls_array_ptr.write_array_of_pointer(pointer_array)
+    Rdkafka::Admin::DescribeAclReport.new(acls: describe_acls_array_ptr, acls_count: 1)
+  end
+
+  after do
+    if describe_acl_ptr != FFI::Pointer::NULL
+      Rdkafka::Bindings.rd_kafka_AclBinding_destroy(describe_acl_ptr)
+    end
+  end
+
+
+  it "should get matching acl resource type as Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC" do
+    expect(subject.acls[0].matching_acl_resource_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC)
+  end
+
+  it "should get matching acl resource name as acl-test-topic" do
+    expect(subject.acls[0].matching_acl_resource_name).to eq(resource_name)
+  end
+
+  it "should get matching acl resource pattern type as Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL" do
+    expect(subject.acls[0].matching_acl_resource_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
+    expect(subject.acls[0].matching_acl_pattern_type).to eq(Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL)
+  end
+
+  it "should get matching acl principal as User:anonymous" do
+    expect(subject.acls[0].matching_acl_principal).to eq("User:anonymous")
+  end
+
+  it "should get matching acl host as * " do
+    expect(subject.acls[0].matching_acl_host).to eq("*")
+  end
+
+  it "should get matching acl operation as Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ" do
+    expect(subject.acls[0].matching_acl_operation).to eq(Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ)
+  end
+
+  it "should get matching acl permission_type as Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW" do
+    expect(subject.acls[0].matching_acl_permission_type).to eq(Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW)
+  end
+end

data/spec/rdkafka/admin_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require "ostruct"
 
 describe Rdkafka::Admin do
@@ -10,6 +9,10 @@ describe Rdkafka::Admin do
   after do
     # Registry should always end up being empty
     expect(Rdkafka::Admin::CreateTopicHandle::REGISTRY).to be_empty
+    expect(Rdkafka::Admin::CreatePartitionsHandle::REGISTRY).to be_empty
+    expect(Rdkafka::Admin::DescribeAclHandle::REGISTRY).to be_empty
+    expect(Rdkafka::Admin::CreateAclHandle::REGISTRY).to be_empty
+    expect(Rdkafka::Admin::DeleteAclHandle::REGISTRY).to be_empty
     admin.close
   end
 
@@ -18,6 +21,15 @@ describe Rdkafka::Admin do
   let(:topic_replication_factor) { 1 }
   let(:topic_config)             { {"cleanup.policy" => "compact", "min.cleanable.dirty.ratio" => 0.8} }
   let(:invalid_topic_config)     { {"cleeeeenup.policee" => "campact"} }
+  let(:group_name)               { "test-group-#{Random.new.rand(0..1_000_000)}" }
+
+  let(:resource_name)         {"acl-test-topic"}
+  let(:resource_type)         {Rdkafka::Bindings::RD_KAFKA_RESOURCE_TOPIC}
+  let(:resource_pattern_type) {Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL}
+  let(:principal)             {"User:anonymous"}
+  let(:host)                  {"*"}
+  let(:operation)             {Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_READ}
+  let(:permission_type)       {Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW}
 
   describe "#create_topic" do
     describe "called with invalid input" do
@@ -33,7 +45,7 @@ describe Rdkafka::Admin do
           }.to raise_exception { |ex|
             expect(ex).to be_a(Rdkafka::RdkafkaError)
             expect(ex.message).to match(/Broker: Invalid topic \(topic_exception\)/)
-            expect(ex.broker_message).to match(/Topic name.*is illegal, it contains a character other than ASCII alphanumerics/)
+expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or more characters other than ASCII alphanumerics, '.', '_' and '-'/)
           }
         end
       end
@@ -201,4 +213,195 @@ describe Rdkafka::Admin do
       expect(delete_topic_report.result_name).to eq(topic_name)
     end
   end
+
+  describe "#ACL tests" do
+    let(:non_existing_resource_name) {"non-existing-topic"}
+    before do
+      #create topic for testing acl
+      create_topic_handle = admin.create_topic(resource_name, topic_partition_count, topic_replication_factor)
+      create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)
+    end
+
+    after do
+      #delete acl
+      delete_acl_handle = admin.delete_acl(resource_type: resource_type, resource_name: resource_name, resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
+      delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+
+      #delete topic that was created for testing acl
+      delete_topic_handle = admin.delete_topic(resource_name)
+      delete_topic_report = delete_topic_handle.wait(max_wait_timeout: 15.0)
+    end
+
+    describe "#create_acl" do
+      it "create acl for a topic that does not exist" do
+        # acl creation for resources that does not exist will still get created successfully.
+        create_acl_handle = admin.create_acl(resource_type: resource_type, resource_name: non_existing_resource_name, resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        # delete the acl that was created for a non existing topic"
+        delete_acl_handle = admin.delete_acl(resource_type: resource_type, resource_name: non_existing_resource_name, resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.size).to eq(1)
+      end
+
+      it "creates a acl for topic that was newly created" do
+        create_acl_handle = admin.create_acl(resource_type: resource_type, resource_name: resource_name, resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+      end
+    end
+
+    describe "#describe_acl" do
+      it "describe acl of a topic that does not exist" do
+        describe_acl_handle = admin.describe_acl(resource_type: resource_type, resource_name: non_existing_resource_name, resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
+        describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(describe_acl_handle[:response]).to eq(0)
+        expect(describe_acl_report.acls.size).to eq(0)
+      end
+
+      it "create acls and describe the newly created acls" do
+        #create_acl
+        create_acl_handle = admin.create_acl(resource_type: resource_type, resource_name: "test_acl_topic_1", resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        create_acl_handle = admin.create_acl(resource_type: resource_type, resource_name: "test_acl_topic_2", resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        #describe_acl
+        describe_acl_handle = admin.describe_acl(resource_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_ANY, resource_name: nil, resource_pattern_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_ANY, principal: nil, host: nil, operation: Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_ANY, permission_type: Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ANY)
+        describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(describe_acl_handle[:response]).to eq(0)
+        expect(describe_acl_report.acls.length).to eq(2)
+      end
+    end
+
+    describe "#delete_acl" do
+      it "delete acl of a topic that does not exist" do
+        delete_acl_handle = admin.delete_acl(resource_type: resource_type, resource_name: non_existing_resource_name, resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.size).to eq(0)
+      end
+
+      it "create an acl and delete the newly created acl" do
+        #create_acl
+        create_acl_handle = admin.create_acl(resource_type: resource_type, resource_name: "test_acl_topic_1", resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        create_acl_handle = admin.create_acl(resource_type: resource_type, resource_name: "test_acl_topic_2", resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
+        create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(create_acl_report.rdkafka_response).to eq(0)
+        expect(create_acl_report.rdkafka_response_string).to eq("")
+
+        #delete_acl - resource_name nil - to delete all acls with any resource name and matching all other filters.
+        delete_acl_handle = admin.delete_acl(resource_type: resource_type, resource_name: nil, resource_pattern_type: resource_pattern_type, principal: principal, host: host, operation: operation, permission_type: permission_type)
+        delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
+        expect(delete_acl_handle[:response]).to eq(0)
+        expect(delete_acl_report.deleted_acls.length).to eq(2)
+
+      end
+    end
+  end
+
+  describe('Group tests') do
+    describe "#delete_group" do
+      describe("with an existing group") do
+        let(:consumer_config) { rdkafka_consumer_config('group.id': group_name) }
+        let(:producer_config) { rdkafka_producer_config }
+        let(:producer) { producer_config.producer }
+        let(:consumer) { consumer_config.consumer }
+
+        before do
+          # Create a topic, post a message to it, consume it and commit offsets, this will create a group that we can then delete.
+          admin.create_topic(topic_name, topic_partition_count, topic_replication_factor).wait(max_wait_timeout: 15.0)
+
+          producer.produce(topic: topic_name, payload: "test", key: "test").wait(max_wait_timeout: 15.0)
+
+          consumer.subscribe(topic_name)
+          wait_for_assignment(consumer)
+          message = consumer.poll(100)
+
+          expect(message).to_not be_nil
+
+          consumer.commit
+          consumer.close
+        end
+
+        after do
+          producer.close
+          consumer.close
+        end
+
+        it "deletes the group" do
+          delete_group_handle = admin.delete_group(group_name)
+          report = delete_group_handle.wait(max_wait_timeout: 15.0)
+
+          expect(report.result_name).to eql(group_name)
+        end
+      end
+
+      describe "called with invalid input" do
+        describe "with the name of a group that does not exist" do
+          it "raises an exception" do
+            delete_group_handle = admin.delete_group(group_name)
+
+            expect {
+              delete_group_handle.wait(max_wait_timeout: 15.0)
+            }.to raise_exception { |ex|
+              expect(ex).to be_a(Rdkafka::RdkafkaError)
+              expect(ex.message).to match(/Broker: The group id does not exist \(group_id_not_found\)/)
+            }
+          end
+        end
+      end
+
+    end
+  end
+
+  describe '#create_partitions' do
+    let(:metadata) { admin.metadata(topic_name).topics.first }
+
+    context 'when topic does not exist' do
+      it 'expect to fail due to unknown partition' do
+        expect { admin.create_partitions(topic_name, 10).wait }.to raise_error(Rdkafka::RdkafkaError, /unknown_topic_or_part/)
+      end
+    end
+
+    context 'when topic already has the desired number of partitions' do
+      before { admin.create_topic(topic_name, 2, 1).wait }
+
+      it 'expect not to change number of partitions' do
+        expect { admin.create_partitions(topic_name, 2).wait }.to raise_error(Rdkafka::RdkafkaError, /invalid_partitions/)
+        expect(metadata[:partition_count]).to eq(2)
+      end
+    end
+
+    context 'when topic has more than the requested number of partitions' do
+      before { admin.create_topic(topic_name, 5, 1).wait }
+
+      it 'expect not to change number of partitions' do
+        expect { admin.create_partitions(topic_name, 2).wait }.to raise_error(Rdkafka::RdkafkaError, /invalid_partitions/)
+        expect(metadata[:partition_count]).to eq(5)
+      end
+    end
+
+    context 'when topic has less then desired number of partitions' do
+      before { admin.create_topic(topic_name, 1, 1).wait }
+
+      it 'expect to change number of partitions' do
+        admin.create_partitions(topic_name, 10).wait
+        expect(metadata[:partition_count]).to eq(10)
+      end
+    end
+  end
 end

data/spec/rdkafka/bindings_spec.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
 require 'zlib'
 
 describe Rdkafka::Bindings do

data/spec/rdkafka/callbacks_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Callbacks do
 
   # The code in the call back functions is 100% covered by other specs.  Due to

data/spec/rdkafka/config_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Config do
   context "logger" do
     it "should have a default logger" do
@@ -115,6 +113,14 @@ describe Rdkafka::Config do
       consumer.close
     end
 
+    it "should create a consumer with consumer_poll_set set to false" do
+      config = rdkafka_consumer_config
+      config.consumer_poll_set = false
+      consumer = config.consumer
+      expect(consumer).to be_a Rdkafka::Consumer
+      consumer.close
+    end
+
     it "should raise an error when creating a consumer with invalid config" do
       config = Rdkafka::Config.new('invalid.key' => 'value')
       expect {

data/spec/rdkafka/consumer/headers_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Headers do
   let(:headers) do
     { # Note String keys!

data/spec/rdkafka/consumer/message_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Message do
   let(:native_client) { new_native_client }
   let(:native_topic) { new_native_topic(native_client: native_client) }

data/spec/rdkafka/consumer/partition_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::Partition do
   let(:offset) { 100 }
   let(:err) { 0 }

data/spec/rdkafka/consumer/topic_partition_list_spec.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require "spec_helper"
-
 describe Rdkafka::Consumer::TopicPartitionList do
   it "should create a new list and add unassigned topics" do
     list = Rdkafka::Consumer::TopicPartitionList.new
@@ -221,5 +219,24 @@ describe Rdkafka::Consumer::TopicPartitionList do
 
       expect(list).to eq other
     end
+
+    it "should create a native list with timetamp offsets if offsets are Time" do
+      list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic_and_partitions_with_offsets("topic", 0 => Time.at(1505069646, 250_000))
+      end
+
+      tpl = list.to_native_tpl
+
+      compare_list = Rdkafka::Consumer::TopicPartitionList.new.tap do |list|
+        list.add_topic_and_partitions_with_offsets(
+          "topic",
+          0 => (Time.at(1505069646, 250_000).to_f * 1000).floor
+        )
+      end
+
+      native_list = Rdkafka::Consumer::TopicPartitionList.from_native_tpl(tpl)
+
+      expect(native_list).to eq compare_list
+    end
   end
 end