karafka-rdkafka 0.12.0

data/spec/rdkafka/admin_spec.rb

@@ -0,0 +1,203 @@
+require "spec_helper"
+require "ostruct"
+
+describe Rdkafka::Admin do
+  let(:config)   { rdkafka_config }
+  let(:admin)    { config.admin }
+
+  after do
+    # Registry should always end up being empty
+    expect(Rdkafka::Admin::CreateTopicHandle::REGISTRY).to be_empty
+    admin.close
+  end
+
+  let(:topic_name)               { "test-topic-#{Random.new.rand(0..1_000_000)}" }
+  let(:topic_partition_count)    { 3 }
+  let(:topic_replication_factor) { 1 }
+  let(:topic_config)             { {"cleanup.policy" => "compact", "min.cleanable.dirty.ratio" => 0.8} }
+  let(:invalid_topic_config)     { {"cleeeeenup.policee" => "campact"} }
+
+  describe "#create_topic" do
+    describe "called with invalid input" do
+      describe "with an invalid topic name" do
+        # https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L29
+        # public static final String LEGAL_CHARS = "[a-zA-Z0-9._-]";
+        let(:topic_name) { "[!@#]" }
+
+        it "raises an exception" do
+          create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          expect {
+            create_topic_handle.wait(max_wait_timeout: 15.0)
+          }.to raise_exception { |ex|
+            expect(ex).to be_a(Rdkafka::RdkafkaError)
+            expect(ex.message).to match(/Broker: Invalid topic \(topic_exception\)/)
+            expect(ex.broker_message).to match(/Topic name.*is illegal, it contains a character other than ASCII alphanumerics/)
+          }
+        end
+      end
+
+      describe "with the name of a topic that already exists" do
+        let(:topic_name) { "empty_test_topic" } # created in spec_helper.rb
+
+        it "raises an exception" do
+          create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          expect {
+            create_topic_handle.wait(max_wait_timeout: 15.0)
+          }.to raise_exception { |ex|
+            expect(ex).to be_a(Rdkafka::RdkafkaError)
+            expect(ex.message).to match(/Broker: Topic already exists \(topic_already_exists\)/)
+            expect(ex.broker_message).to match(/Topic 'empty_test_topic' already exists/)
+          }
+        end
+      end
+
+      describe "with an invalid partition count" do
+        let(:topic_partition_count) { -999 }
+
+        it "raises an exception" do
+          expect {
+            admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          }.to raise_error Rdkafka::Config::ConfigError, /num_partitions out of expected range/
+        end
+      end
+
+      describe "with an invalid replication factor" do
+        let(:topic_replication_factor) { -2  }
+
+        it "raises an exception" do
+          expect {
+            admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          }.to raise_error Rdkafka::Config::ConfigError, /replication_factor out of expected range/
+        end
+      end
+
+      describe "with an invalid topic configuration" do
+        it "doesn't create the topic" do
+          create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor, invalid_topic_config)
+          expect {
+            create_topic_handle.wait(max_wait_timeout: 15.0)
+          }.to raise_error Rdkafka::RdkafkaError, /Broker: Configuration is invalid \(invalid_config\)/
+        end
+      end
+    end
+
+    context "edge case" do
+      context "where we are unable to get the background queue" do
+        before do
+          allow(Rdkafka::Bindings).to receive(:rd_kafka_queue_get_background).and_return(FFI::Pointer::NULL)
+        end
+
+        it "raises an exception" do
+          expect {
+            admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          }.to raise_error Rdkafka::Config::ConfigError, /rd_kafka_queue_get_background was NULL/
+        end
+      end
+
+      context "where rd_kafka_CreateTopics raises an exception" do
+        before do
+          allow(Rdkafka::Bindings).to receive(:rd_kafka_CreateTopics).and_raise(RuntimeError.new("oops"))
+        end
+
+        it "raises an exception" do
+          expect {
+            admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+          }.to raise_error RuntimeError, /oops/
+        end
+      end
+    end
+
+    it "creates a topic" do
+      create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor, topic_config)
+      create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)
+      expect(create_topic_report.error_string).to be_nil
+      expect(create_topic_report.result_name).to eq(topic_name)
+    end
+  end
+
+  describe "#delete_topic" do
+    describe "called with invalid input" do
+      # https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L29
+      # public static final String LEGAL_CHARS = "[a-zA-Z0-9._-]";
+      describe "with an invalid topic name" do
+        let(:topic_name) { "[!@#]" }
+
+        it "raises an exception" do
+          delete_topic_handle = admin.delete_topic(topic_name)
+          expect {
+            delete_topic_handle.wait(max_wait_timeout: 15.0)
+          }.to raise_exception { |ex|
+            expect(ex).to be_a(Rdkafka::RdkafkaError)
+            expect(ex.message).to match(/Broker: Unknown topic or partition \(unknown_topic_or_part\)/)
+            expect(ex.broker_message).to match(/Broker: Unknown topic or partition/)
+          }
+        end
+      end
+
+      describe "with the name of a topic that does not exist" do
+        it "raises an exception" do
+          delete_topic_handle = admin.delete_topic(topic_name)
+          expect {
+            delete_topic_handle.wait(max_wait_timeout: 15.0)
+          }.to raise_exception { |ex|
+            expect(ex).to be_a(Rdkafka::RdkafkaError)
+            expect(ex.message).to match(/Broker: Unknown topic or partition \(unknown_topic_or_part\)/)
+            expect(ex.broker_message).to match(/Broker: Unknown topic or partition/)
+          }
+        end
+      end
+    end
+
+    context "edge case" do
+      context "where we are unable to get the background queue" do
+        before do
+          allow(Rdkafka::Bindings).to receive(:rd_kafka_queue_get_background).and_return(FFI::Pointer::NULL)
+        end
+
+        it "raises an exception" do
+          expect {
+            admin.delete_topic(topic_name)
+          }.to raise_error Rdkafka::Config::ConfigError, /rd_kafka_queue_get_background was NULL/
+        end
+      end
+
+      context "where rd_kafka_DeleteTopics raises an exception" do
+        before do
+          allow(Rdkafka::Bindings).to receive(:rd_kafka_DeleteTopics).and_raise(RuntimeError.new("oops"))
+        end
+
+        it "raises an exception" do
+          expect {
+            admin.delete_topic(topic_name)
+          }.to raise_error RuntimeError, /oops/
+        end
+      end
+    end
+
+
+    it "deletes a topic that was newly created" do
+      create_topic_handle = admin.create_topic(topic_name, topic_partition_count, topic_replication_factor)
+      create_topic_report = create_topic_handle.wait(max_wait_timeout: 15.0)
+      expect(create_topic_report.error_string).to be_nil
+      expect(create_topic_report.result_name).to eq(topic_name)
+
+      # Retry topic deletion a few times. On CI Kafka seems to not
+      # always be ready for it immediately
+      delete_topic_report = nil
+      10.times do |i|
+        begin
+          delete_topic_handle = admin.delete_topic(topic_name)
+          delete_topic_report = delete_topic_handle.wait(max_wait_timeout: 15.0)
+          break
+        rescue Rdkafka::RdkafkaError => ex
+          if i > 3
+            raise ex
+          end
+        end
+      end
+
+      expect(delete_topic_report.error_string).to be_nil
+      expect(delete_topic_report.result_name).to eq(topic_name)
+    end
+  end
+end

data/spec/rdkafka/bindings_spec.rb

@@ -0,0 +1,134 @@
+require "spec_helper"
+require 'zlib'
+
+describe Rdkafka::Bindings do
+  it "should load librdkafka" do
+    expect(Rdkafka::Bindings.ffi_libraries.map(&:name).first).to include "librdkafka"
+  end
+
+  describe ".lib_extension" do
+    it "should know the lib extension for darwin" do
+      stub_const('RbConfig::CONFIG', 'host_os' =>'darwin')
+      expect(Rdkafka::Bindings.lib_extension).to eq "dylib"
+    end
+
+    it "should know the lib extension for linux" do
+      stub_const('RbConfig::CONFIG', 'host_os' =>'linux')
+      expect(Rdkafka::Bindings.lib_extension).to eq "so"
+    end
+  end
+
+  it "should successfully call librdkafka" do
+    expect {
+      Rdkafka::Bindings.rd_kafka_conf_new
+    }.not_to raise_error
+  end
+
+  describe "log callback" do
+    let(:log_queue) { Rdkafka::Config.log_queue }
+    before do
+      allow(log_queue).to receive(:<<)
+    end
+
+    it "should log fatal messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 0, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
+    end
+
+    it "should log error messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 3, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::ERROR, "rdkafka: log line"])
+    end
+
+    it "should log warning messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 4, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::WARN, "rdkafka: log line"])
+    end
+
+    it "should log info messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 5, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::INFO, "rdkafka: log line"])
+    end
+
+    it "should log debug messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 7, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::DEBUG, "rdkafka: log line"])
+    end
+
+    it "should log unknown messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 100, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::UNKNOWN, "rdkafka: log line"])
+    end
+  end
+
+  describe "partitioner" do
+    let(:partition_key) { ('a'..'z').to_a.shuffle.take(15).join('') }
+    let(:partition_count) { rand(50) + 1 }
+
+    it "should return the same partition for a similar string and the same partition count" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      result_2 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      expect(result_1).to eq(result_2)
+    end
+
+    it "should match the old partitioner" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
+      result_2 = (Zlib.crc32(partition_key) % partition_count)
+      expect(result_1).to eq(result_2)
+    end
+
+    it "should return the partition calculated by the specified partitioner" do
+      result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count, "murmur2")
+      ptr = FFI::MemoryPointer.from_string(partition_key)
+      result_2 = Rdkafka::Bindings.rd_kafka_msg_partitioner_murmur2(nil, ptr, partition_key.size, partition_count, nil, nil)
+      expect(result_1).to eq(result_2)
+    end
+  end
+
+  describe "stats callback" do
+    context "without a stats callback" do
+      it "should do nothing" do
+        expect {
+          Rdkafka::Bindings::StatsCallback.call(nil, "{}", 2, nil)
+        }.not_to raise_error
+      end
+    end
+
+    context "with a stats callback" do
+      before do
+        Rdkafka::Config.statistics_callback = lambda do |stats|
+          $received_stats = stats
+        end
+      end
+
+      it "should call the stats callback with a stats hash" do
+        Rdkafka::Bindings::StatsCallback.call(nil, "{\"received\":1}", 13, nil)
+        expect($received_stats).to eq({'received' => 1})
+      end
+    end
+  end
+
+  describe "error callback" do
+    context "without an error callback" do
+      it "should do nothing" do
+        expect {
+          Rdkafka::Bindings::ErrorCallback.call(nil, 1, "error", nil)
+        }.not_to raise_error
+      end
+    end
+
+    context "with an error callback" do
+      before do
+        Rdkafka::Config.error_callback = lambda do |error|
+          $received_error = error
+        end
+      end
+
+      it "should call the error callback with an Rdkafka::Error" do
+        Rdkafka::Bindings::ErrorCallback.call(nil, 8, "Broker not available", nil)
+        expect($received_error.code).to eq(:broker_not_available)
+        expect($received_error.broker_message).to eq("Broker not available")
+      end
+    end
+  end
+end

data/spec/rdkafka/callbacks_spec.rb

@@ -0,0 +1,20 @@
+require "spec_helper"
+
+describe Rdkafka::Callbacks do
+
+  # The code in the call back functions is 100% covered by other specs.  Due to
+  # the large number of collaborators, and the fact that FFI does not play
+  # nicely with doubles, it was very difficult to construct tests that were
+  # not over-mocked.
+
+  # For debugging purposes, if you suspect that you are running into trouble in
+  # one of the callback functions, it may be helpful to surround the inner body
+  # of the method with something like:
+  #
+  #   begin
+  #     <method body>
+  #   rescue => ex; puts ex.inspect; puts ex.backtrace; end;
+  #
+  # This will output to STDOUT any exceptions that are being raised in the callback.
+
+end

data/spec/rdkafka/config_spec.rb

@@ -0,0 +1,182 @@
+require "spec_helper"
+
+describe Rdkafka::Config do
+  context "logger" do
+    it "should have a default logger" do
+      expect(Rdkafka::Config.logger).to be_a Logger
+    end
+
+    it "should set the logger" do
+      logger = Logger.new(STDOUT)
+      expect(Rdkafka::Config.logger).not_to eq logger
+      Rdkafka::Config.logger = logger
+      expect(Rdkafka::Config.logger).to eq logger
+    end
+
+    it "should not accept a nil logger" do
+      expect {
+        Rdkafka::Config.logger = nil
+      }.to raise_error(Rdkafka::Config::NoLoggerError)
+    end
+
+    it "supports logging queue" do
+      log = StringIO.new
+      Rdkafka::Config.logger = Logger.new(log)
+
+      Rdkafka::Config.log_queue << [Logger::FATAL, "I love testing"]
+      20.times do
+        break if log.string != ""
+        sleep 0.05
+      end
+
+      expect(log.string).to include "FATAL -- : I love testing"
+    end
+  end
+
+  context "statistics callback" do
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.statistics_callback = lambda do |stats|
+            puts stats
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.statistics_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
+        end
+        expect {
+          Rdkafka::Config.statistics_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.statistics_callback).to respond_to :call
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        Rdkafka::Config.statistics_callback = 'a string'
+      }.to raise_error(TypeError)
+    end
+  end
+
+  context "error callback" do
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.error_callback = lambda do |error|
+            puts error
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.error_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
+        end
+        expect {
+          Rdkafka::Config.error_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.error_callback).to respond_to :call
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        Rdkafka::Config.error_callback = 'a string'
+      }.to raise_error(TypeError)
+    end
+  end
+
+  context "configuration" do
+    it "should store configuration" do
+      config = Rdkafka::Config.new
+      config[:"key"] = 'value'
+      expect(config[:"key"]).to eq 'value'
+    end
+
+    it "should use default configuration" do
+      config = Rdkafka::Config.new
+      expect(config[:"api.version.request"]).to eq true
+    end
+
+    it "should create a consumer with valid config" do
+      consumer = rdkafka_consumer_config.consumer
+      expect(consumer).to be_a Rdkafka::Consumer
+      consumer.close
+    end
+
+    it "should raise an error when creating a consumer with invalid config" do
+      config = Rdkafka::Config.new('invalid.key' => 'value')
+      expect {
+        config.consumer
+      }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
+    end
+
+    it "should raise an error when creating a consumer with a nil key in the config" do
+      config = Rdkafka::Config.new(nil => 'value')
+      expect {
+        config.consumer
+      }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"\"")
+    end
+
+    it "should treat a nil value as blank" do
+      config = Rdkafka::Config.new('security.protocol' => nil)
+      expect {
+        config.consumer
+        config.producer
+      }.to raise_error(Rdkafka::Config::ConfigError, "Configuration property \"security.protocol\" cannot be set to empty value")
+    end
+
+    it "should create a producer with valid config" do
+      producer = rdkafka_consumer_config.producer
+      expect(producer).to be_a Rdkafka::Producer
+      producer.close
+    end
+
+    it "should raise an error when creating a producer with invalid config" do
+      config = Rdkafka::Config.new('invalid.key' => 'value')
+      expect {
+        config.producer
+      }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
+    end
+
+    it "should allow configuring zstd compression" do
+      config = Rdkafka::Config.new('compression.codec' => 'zstd')
+      begin
+        expect(config.producer).to be_a Rdkafka::Producer
+        config.producer.close
+      rescue Rdkafka::Config::ConfigError => ex
+        pending "Zstd compression not supported on this machine"
+        raise ex
+      end
+    end
+
+    it "should raise an error when client creation fails for a consumer" do
+      config = Rdkafka::Config.new(
+        "security.protocol" => "SSL",
+        "ssl.ca.location" => "/nonsense"
+      )
+      expect {
+        config.consumer
+      }.to raise_error(Rdkafka::Config::ClientCreationError, /ssl.ca.location failed(.*)/)
+    end
+
+    it "should raise an error when client creation fails for a producer" do
+      config = Rdkafka::Config.new(
+        "security.protocol" => "SSL",
+        "ssl.ca.location" => "/nonsense"
+      )
+      expect {
+        config.producer
+      }.to raise_error(Rdkafka::Config::ClientCreationError, /ssl.ca.location failed(.*)/)
+    end
+  end
+end

data/spec/rdkafka/consumer/message_spec.rb

@@ -0,0 +1,139 @@
+require "spec_helper"
+
+describe Rdkafka::Consumer::Message do
+  let(:native_client) { new_native_client }
+  let(:native_topic) { new_native_topic(native_client: native_client) }
+  let(:payload) { nil }
+  let(:key) { nil }
+  let(:native_message) do
+    Rdkafka::Bindings::Message.new.tap do |message|
+      message[:rkt] = native_topic
+      message[:partition] = 3
+      message[:offset] = 100
+      if payload
+        ptr = FFI::MemoryPointer.new(:char, payload.bytesize)
+        ptr.put_bytes(0, payload)
+        message[:payload] = ptr
+        message[:len] = payload.bytesize
+      end
+      if key
+        ptr = FFI::MemoryPointer.new(:char, key.bytesize)
+        ptr.put_bytes(0, key)
+        message[:key] = ptr
+        message[:key_len] = key.bytesize
+      end
+    end
+  end
+
+  after(:each) do
+    Rdkafka::Bindings.rd_kafka_destroy(native_client)
+  end
+
+  subject { Rdkafka::Consumer::Message.new(native_message) }
+
+  before do
+    # mock headers, because it produces 'segmentation fault' while settings or reading headers for
+    # a message which is created from scratch
+    #
+    # Code dump example:
+    #
+    # ```
+    # frame #7: 0x000000010dacf5ab librdkafka.dylib`rd_list_destroy + 11
+    # frame #8: 0x000000010dae5a7e librdkafka.dylib`rd_kafka_headers_destroy + 14
+    # frame #9: 0x000000010da9ab40 librdkafka.dylib`rd_kafka_message_set_headers + 32
+    # ```
+    expect( Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(any_args) do
+      Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
+    end
+  end
+
+  it "should have a topic" do
+    expect(subject.topic).to eq "topic_name"
+  end
+
+  it "should have a partition" do
+    expect(subject.partition).to eq 3
+  end
+
+  context "payload" do
+    it "should have a nil payload when none is present" do
+      expect(subject.payload).to be_nil
+    end
+
+    context "present payload" do
+      let(:payload) { "payload content" }
+
+      it "should have a payload" do
+        expect(subject.payload).to eq "payload content"
+      end
+    end
+  end
+
+  context "key" do
+    it "should have a nil key when none is present" do
+      expect(subject.key).to be_nil
+    end
+
+    context "present key" do
+      let(:key) { "key content" }
+
+      it "should have a key" do
+        expect(subject.key).to eq "key content"
+      end
+    end
+  end
+
+  it "should have an offset" do
+    expect(subject.offset).to eq 100
+  end
+
+  describe "#timestamp" do
+    context "without a timestamp" do
+      before do
+        allow(Rdkafka::Bindings).to receive(:rd_kafka_message_timestamp).and_return(-1)
+      end
+
+      it "should have a nil timestamp if not present" do
+        expect(subject.timestamp).to be_nil
+      end
+    end
+
+    context "with a timestamp" do
+      before do
+        allow(Rdkafka::Bindings).to receive(:rd_kafka_message_timestamp).and_return(1505069646250)
+      end
+
+      it "should have timestamp if present" do
+        expect(subject.timestamp).to eq Time.at(1505069646, 250_000)
+      end
+    end
+  end
+
+  describe "#to_s" do
+    before do
+      allow(subject).to receive(:timestamp).and_return(1000)
+    end
+
+    it "should have a human readable representation" do
+      expect(subject.to_s).to eq "<Message in 'topic_name' with key '', payload '', partition 3, offset 100, timestamp 1000>"
+    end
+
+    context "with key and payload" do
+      let(:key) { "key" }
+      let(:payload) { "payload" }
+
+      it "should have a human readable representation" do
+        expect(subject.to_s).to eq "<Message in 'topic_name' with key 'key', payload 'payload', partition 3, offset 100, timestamp 1000>"
+      end
+    end
+
+    context "with a very long key and payload" do
+      let(:key) { "k" * 100_000 }
+      let(:payload) { "p" * 100_000 }
+
+      it "should have a human readable representation" do
+        expect(subject.to_s).to eq "<Message in 'topic_name' with key 'kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk...', payload 'pppppppppppppppppppppppppppppppppppppppp...', partition 3, offset 100, timestamp 1000>"
+      end
+    end
+  end
+end