karafka-rdkafka 0.20.0-arm64-darwin

data/spec/rdkafka/bindings_spec.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require 'zlib'
+
+describe Rdkafka::Bindings do
+  it "should load librdkafka" do
+    expect(Rdkafka::Bindings.ffi_libraries.map(&:name).first).to include "librdkafka"
+  end
+
+  describe ".lib_extension" do
+    it "should know the lib extension for darwin" do
+      stub_const('RbConfig::CONFIG', 'host_os' =>'darwin')
+      expect(Rdkafka::Bindings.lib_extension).to eq "dylib"
+    end
+
+    it "should know the lib extension for linux" do
+      stub_const('RbConfig::CONFIG', 'host_os' =>'linux')
+      expect(Rdkafka::Bindings.lib_extension).to eq "so"
+    end
+  end
+
+  it "should successfully call librdkafka" do
+    expect {
+      Rdkafka::Bindings.rd_kafka_conf_new
+    }.not_to raise_error
+  end
+
+  describe "log callback" do
+    let(:log_queue) { Rdkafka::Config.log_queue }
+    before do
+      allow(log_queue).to receive(:<<)
+    end
+
+    it "should log fatal messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 0, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
+    end
+
+    it "should log fatal messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 1, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
+    end
+
+    it "should log fatal messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 2, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::FATAL, "rdkafka: log line"])
+    end
+
+    it "should log error messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 3, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::ERROR, "rdkafka: log line"])
+    end
+
+    it "should log warning messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 4, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::WARN, "rdkafka: log line"])
+    end
+
+    it "should log info messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 5, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::INFO, "rdkafka: log line"])
+    end
+
+    it "should log info messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 6, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::INFO, "rdkafka: log line"])
+    end
+
+    it "should log debug messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 7, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::DEBUG, "rdkafka: log line"])
+    end
+
+    it "should log unknown messages" do
+      Rdkafka::Bindings::LogCallback.call(nil, 100, nil, "log line")
+      expect(log_queue).to have_received(:<<).with([Logger::UNKNOWN, "rdkafka: log line"])
+    end
+  end
+
+  describe "stats callback" do
+    context "without a stats callback" do
+      it "should do nothing" do
+        expect {
+          Rdkafka::Bindings::StatsCallback.call(nil, "{}", 2, nil)
+        }.not_to raise_error
+      end
+    end
+
+    context "with a stats callback" do
+      before do
+        Rdkafka::Config.statistics_callback = lambda do |stats|
+          $received_stats = stats
+        end
+      end
+
+      it "should call the stats callback with a stats hash" do
+        Rdkafka::Bindings::StatsCallback.call(nil, "{\"received\":1}", 13, nil)
+        expect($received_stats).to eq({'received' => 1})
+      end
+    end
+  end
+
+  describe "error callback" do
+    context "without an error callback" do
+      it "should do nothing" do
+        expect {
+          Rdkafka::Bindings::ErrorCallback.call(nil, 1, "error", nil)
+        }.not_to raise_error
+      end
+    end
+
+    context "with an error callback" do
+      before do
+        Rdkafka::Config.error_callback = lambda do |error|
+          $received_error = error
+        end
+      end
+
+      it "should call the error callback with an Rdkafka::Error" do
+        Rdkafka::Bindings::ErrorCallback.call(nil, 8, "Broker not available", nil)
+        expect($received_error.code).to eq(:broker_not_available)
+        expect($received_error.broker_message).to eq("Broker not available")
+      end
+    end
+  end
+
+  describe "oauthbearer set token" do
+    context "with args" do
+      before do
+        DEFAULT_TOKEN_EXPIRY_SECONDS = 900
+        $token_value = "token"
+        $md_lifetime_ms = Time.now.to_i*1000 + DEFAULT_TOKEN_EXPIRY_SECONDS * 1000
+        $md_principal_name = "kafka-cluster"
+        $extensions = nil
+        $extension_size = 0
+        $error_buffer = FFI::MemoryPointer.from_string(" " * 256)
+      end
+
+      it "should set token or capture failure" do
+        RdKafkaTestConsumer.with do |consumer_ptr|
+          response = Rdkafka::Bindings.rd_kafka_oauthbearer_set_token(consumer_ptr, $token_value, $md_lifetime_ms, $md_principal_name, $extensions, $extension_size, $error_buffer, 256)
+          expect(response).to eq(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__STATE)
+          expect($error_buffer.read_string).to eq("SASL/OAUTHBEARER is not the configured authentication mechanism")
+        end
+      end
+    end
+  end
+
+  describe "oauthbearer set token failure" do
+
+    context "without args" do
+
+      it "should fail" do
+        expect {
+          Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure
+        }.to raise_error(ArgumentError)
+      end
+    end
+
+    context "with args" do
+      it "should succeed" do
+        expect {
+          errstr = "error"
+          RdKafkaTestConsumer.with do |consumer_ptr|
+            Rdkafka::Bindings.rd_kafka_oauthbearer_set_token_failure(consumer_ptr, errstr)
+          end
+        }.to_not raise_error
+      end
+    end
+  end
+
+  describe "oauthbearer callback" do
+    context "without an oauthbearer callback" do
+      it "should do nothing" do
+        expect {
+          Rdkafka::Bindings::OAuthbearerTokenRefreshCallback.call(nil, "", nil)
+        }.not_to raise_error
+      end
+    end
+
+    context "with an oauthbearer callback" do
+      before do
+        Rdkafka::Config.oauthbearer_token_refresh_callback = lambda do |config, client_name|
+          $received_config = config
+          $received_client_name = client_name
+        end
+      end
+
+      it "should call the oauth bearer callback and receive config and client name" do
+        RdKafkaTestConsumer.with do |consumer_ptr|
+          Rdkafka::Bindings::OAuthbearerTokenRefreshCallback.call(consumer_ptr, "{}", nil)
+            expect($received_config).to eq("{}")
+            expect($received_client_name).to match(/consumer/)
+        end
+      end
+    end
+  end
+end

data/spec/rdkafka/callbacks_spec.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+describe Rdkafka::Callbacks do
+
+  # The code in the call back functions is 100% covered by other specs.  Due to
+  # the large number of collaborators, and the fact that FFI does not play
+  # nicely with doubles, it was very difficult to construct tests that were
+  # not over-mocked.
+
+  # For debugging purposes, if you suspect that you are running into trouble in
+  # one of the callback functions, it may be helpful to surround the inner body
+  # of the method with something like:
+  #
+  #   begin
+  #     <method body>
+  #   rescue => ex; puts ex.inspect; puts ex.backtrace; end;
+  #
+  # This will output to STDOUT any exceptions that are being raised in the callback.
+
+end

data/spec/rdkafka/config_spec.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+describe Rdkafka::Config do
+  context "logger" do
+    it "should have a default logger" do
+      expect(Rdkafka::Config.logger).to be_a Logger
+    end
+
+    it "should set the logger" do
+      logger = Logger.new(STDOUT)
+      expect(Rdkafka::Config.logger).not_to eq logger
+      Rdkafka::Config.logger = logger
+      expect(Rdkafka::Config.logger).to eq logger
+    end
+
+    it "should not accept a nil logger" do
+      expect {
+        Rdkafka::Config.logger = nil
+      }.to raise_error(Rdkafka::Config::NoLoggerError)
+    end
+
+    it "supports logging queue" do
+      log = StringIO.new
+      Rdkafka::Config.logger = Logger.new(log)
+      Rdkafka::Config.ensure_log_thread
+
+      Rdkafka::Config.log_queue << [Logger::FATAL, "I love testing"]
+      20.times do
+        break if log.string != ""
+        sleep 0.05
+      end
+
+      expect(log.string).to include "FATAL -- : I love testing"
+    end
+
+    unless RUBY_PLATFORM == 'java'
+      it "expect to start new logger thread after fork and work" do
+        reader, writer = IO.pipe
+
+        pid = fork do
+          $stdout.reopen(writer)
+          Rdkafka::Config.logger = Logger.new($stdout)
+          reader.close
+          producer = rdkafka_producer_config(debug: 'all').producer
+          producer.close
+          writer.close
+          sleep(1)
+        end
+
+        writer.close
+        Process.wait(pid)
+        output = reader.read
+        expect(output.split("\n").size).to be >= 20
+      end
+    end
+  end
+
+  context "statistics callback" do
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.statistics_callback = lambda do |stats|
+            puts stats
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.statistics_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
+        end
+        expect {
+          Rdkafka::Config.statistics_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.statistics_callback).to respond_to :call
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        Rdkafka::Config.statistics_callback = 'a string'
+      }.to raise_error(TypeError)
+    end
+  end
+
+  context "error callback" do
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.error_callback = lambda do |error|
+            puts error
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.error_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(stats); end
+        end
+        expect {
+          Rdkafka::Config.error_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.error_callback).to respond_to :call
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        Rdkafka::Config.error_callback = 'a string'
+      }.to raise_error(TypeError)
+    end
+  end
+
+  context "oauthbearer calllback" do
+    context "with a proc/lambda" do
+      it "should set the callback" do
+        expect {
+          Rdkafka::Config.oauthbearer_token_refresh_callback = lambda do |config, client_name|
+            puts config
+            puts client_name
+          end
+        }.not_to raise_error
+        expect(Rdkafka::Config.oauthbearer_token_refresh_callback).to respond_to :call
+      end
+    end
+
+    context "with a callable object" do
+      it "should set the callback" do
+        callback = Class.new do
+          def call(config, client_name); end
+        end
+
+        expect {
+          Rdkafka::Config.oauthbearer_token_refresh_callback = callback.new
+        }.not_to raise_error
+        expect(Rdkafka::Config.oauthbearer_token_refresh_callback).to respond_to :call
+      end
+    end
+
+    it "should not accept a callback that's not callable" do
+      expect {
+        Rdkafka::Config.oauthbearer_token_refresh_callback = 'not a callback'
+      }.to raise_error(TypeError)
+    end
+  end
+
+  context "configuration" do
+    it "should store configuration" do
+      config = Rdkafka::Config.new
+      config[:"key"] = 'value'
+      expect(config[:"key"]).to eq 'value'
+    end
+
+    it "should use default configuration" do
+      config = Rdkafka::Config.new
+      expect(config[:"api.version.request"]).to eq true
+    end
+
+    it "should create a consumer with valid config" do
+      consumer = rdkafka_consumer_config.consumer
+      expect(consumer).to be_a Rdkafka::Consumer
+      consumer.close
+    end
+
+    it "should create a consumer with consumer_poll_set set to false" do
+      config = rdkafka_consumer_config
+      config.consumer_poll_set = false
+      consumer = config.consumer
+      expect(consumer).to be_a Rdkafka::Consumer
+      consumer.close
+    end
+
+    it "should raise an error when creating a consumer with invalid config" do
+      config = Rdkafka::Config.new('invalid.key' => 'value')
+      expect {
+        config.consumer
+      }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
+    end
+
+    it "should raise an error when creating a consumer with a nil key in the config" do
+      config = Rdkafka::Config.new(nil => 'value')
+      expect {
+        config.consumer
+      }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"\"")
+    end
+
+    it "should treat a nil value as blank" do
+      config = Rdkafka::Config.new('security.protocol' => nil)
+      expect {
+        config.consumer
+        config.producer
+      }.to raise_error(Rdkafka::Config::ConfigError, "Configuration property \"security.protocol\" cannot be set to empty value")
+    end
+
+    it "should create a producer with valid config" do
+      producer = rdkafka_consumer_config.producer
+      expect(producer).to be_a Rdkafka::Producer
+      producer.close
+    end
+
+    it "should raise an error when creating a producer with invalid config" do
+      config = Rdkafka::Config.new('invalid.key' => 'value')
+      expect {
+        config.producer
+      }.to raise_error(Rdkafka::Config::ConfigError, "No such configuration property: \"invalid.key\"")
+    end
+
+    it "allows string partitioner key" do
+      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2").and_call_original
+      config = Rdkafka::Config.new("partitioner" => "murmur2")
+      config.producer.close
+    end
+
+    it "allows symbol partitioner key" do
+      expect(Rdkafka::Producer).to receive(:new).with(kind_of(Rdkafka::NativeKafka), "murmur2").and_call_original
+      config = Rdkafka::Config.new(:partitioner => "murmur2")
+      config.producer.close
+    end
+
+    it "should allow configuring zstd compression" do
+      config = Rdkafka::Config.new('compression.codec' => 'zstd')
+      begin
+        producer = config.producer
+        expect(producer).to be_a Rdkafka::Producer
+        producer.close
+      rescue Rdkafka::Config::ConfigError => ex
+        pending "Zstd compression not supported on this machine"
+        raise ex
+      end
+    end
+
+    it "should raise an error when client creation fails for a consumer" do
+      config = Rdkafka::Config.new(
+        "security.protocol" => "SSL",
+        "ssl.ca.location" => "/nonsense"
+      )
+      expect {
+        config.consumer
+      }.to raise_error(Rdkafka::Config::ClientCreationError, /ssl.ca.location failed(.*)/)
+    end
+
+    it "should raise an error when client creation fails for a producer" do
+      config = Rdkafka::Config.new(
+        "security.protocol" => "SSL",
+        "ssl.ca.location" => "/nonsense"
+      )
+      expect {
+        config.producer
+      }.to raise_error(Rdkafka::Config::ClientCreationError, /ssl.ca.location failed(.*)/)
+    end
+  end
+end

data/spec/rdkafka/consumer/headers_spec.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+describe Rdkafka::Consumer::Headers do
+  let(:headers) do
+    { # Note String keys!
+      "version" => ["2.1.3", "2.1.4"],
+      "type" => "String"
+    }
+  end
+  let(:native_message) { double('native message') }
+  let(:headers_ptr) { double('headers pointer') }
+
+  describe '.from_native' do
+    before do
+      expect(Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(native_message, anything) do |_, headers_ptrptr|
+        expect(headers_ptrptr).to receive(:read_pointer).and_return(headers_ptr)
+        Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+      end
+
+      # First version header
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 0, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 0", read_string_to_null: "version"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["version"][0].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 0", read_string: headers["version"][0]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      # Second version header
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 1, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 1", read_string_to_null: "version"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["version"][1].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 1", read_string: headers["version"][1]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      # Single type header
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 2, anything, anything, anything) do |_, _, name_ptrptr, value_ptrptr, size_ptr|
+              expect(name_ptrptr).to receive(:read_pointer).and_return(double("pointer 2", read_string_to_null: "type"))
+              expect(size_ptr).to receive(:[]).with(:value).and_return(headers["type"].size)
+              expect(value_ptrptr).to receive(:read_pointer).and_return(double("value pointer 2", read_string: headers["type"]))
+              Rdkafka::Bindings::RD_KAFKA_RESP_ERR_NO_ERROR
+            end
+
+      expect(Rdkafka::Bindings).to \
+        receive(:rd_kafka_header_get_all)
+          .with(headers_ptr, 3, anything, anything, anything)
+          .and_return(Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT)
+    end
+
+    subject { described_class.from_native(native_message) }
+
+    it { is_expected.to eq(headers) }
+    it { is_expected.to be_frozen }
+
+    it 'returns array for duplicate headers' do
+      expect(subject['version']).to eq(["2.1.3", "2.1.4"])
+    end
+
+    it 'returns string for single headers' do
+      expect(subject['type']).to eq("String")
+    end
+
+    it 'does not support symbols mappings' do
+      expect(subject.key?(:version)).to eq(false)
+    end
+  end
+end

data/spec/rdkafka/consumer/message_spec.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+describe Rdkafka::Consumer::Message do
+  let(:native_client) { new_native_client }
+  let(:native_topic) { new_native_topic(native_client: native_client) }
+  let(:payload) { nil }
+  let(:key) { nil }
+  let(:native_message) do
+    Rdkafka::Bindings::Message.new.tap do |message|
+      message[:rkt] = native_topic
+      message[:partition] = 3
+      message[:offset] = 100
+      if payload
+        ptr = FFI::MemoryPointer.new(:char, payload.bytesize)
+        ptr.put_bytes(0, payload)
+        message[:payload] = ptr
+        message[:len] = payload.bytesize
+      end
+      if key
+        ptr = FFI::MemoryPointer.new(:char, key.bytesize)
+        ptr.put_bytes(0, key)
+        message[:key] = ptr
+        message[:key_len] = key.bytesize
+      end
+    end
+  end
+
+  after(:each) do
+    Rdkafka::Bindings.rd_kafka_destroy(native_client)
+  end
+
+  subject { Rdkafka::Consumer::Message.new(native_message) }
+
+  before do
+    # mock headers, because it produces 'segmentation fault' while settings or reading headers for
+    # a message which is created from scratch
+    #
+    # Code dump example:
+    #
+    # ```
+    # frame #7: 0x000000010dacf5ab librdkafka.dylib`rd_list_destroy + 11
+    # frame #8: 0x000000010dae5a7e librdkafka.dylib`rd_kafka_headers_destroy + 14
+    # frame #9: 0x000000010da9ab40 librdkafka.dylib`rd_kafka_message_set_headers + 32
+    # ```
+    expect( Rdkafka::Bindings).to receive(:rd_kafka_message_headers).with(any_args) do
+      Rdkafka::Bindings::RD_KAFKA_RESP_ERR__NOENT
+    end
+  end
+
+  it "should have a topic" do
+    expect(subject.topic).to eq "topic_name"
+  end
+
+  it "should have a partition" do
+    expect(subject.partition).to eq 3
+  end
+
+  context "payload" do
+    it "should have a nil payload when none is present" do
+      expect(subject.payload).to be_nil
+    end
+
+    context "present payload" do
+      let(:payload) { "payload content" }
+
+      it "should have a payload" do
+        expect(subject.payload).to eq "payload content"
+      end
+    end
+  end
+
+  context "key" do
+    it "should have a nil key when none is present" do
+      expect(subject.key).to be_nil
+    end
+
+    context "present key" do
+      let(:key) { "key content" }
+
+      it "should have a key" do
+        expect(subject.key).to eq "key content"
+      end
+    end
+  end
+
+  it "should have an offset" do
+    expect(subject.offset).to eq 100
+  end
+
+  describe "#timestamp" do
+    context "without a timestamp" do
+      before do
+        allow(Rdkafka::Bindings).to receive(:rd_kafka_message_timestamp).and_return(-1)
+      end
+
+      it "should have a nil timestamp if not present" do
+        expect(subject.timestamp).to be_nil
+      end
+    end
+
+    context "with a timestamp" do
+      before do
+        allow(Rdkafka::Bindings).to receive(:rd_kafka_message_timestamp).and_return(1505069646250)
+      end
+
+      it "should have timestamp if present" do
+        expect(subject.timestamp).to eq Time.at(1505069646, 250_000)
+      end
+    end
+  end
+
+  describe "#to_s" do
+    before do
+      allow(subject).to receive(:timestamp).and_return(1000)
+    end
+
+    it "should have a human readable representation" do
+      expect(subject.to_s).to eq "<Message in 'topic_name' with key '', payload '', partition 3, offset 100, timestamp 1000>"
+    end
+
+    context "with key and payload" do
+      let(:key) { "key" }
+      let(:payload) { "payload" }
+
+      it "should have a human readable representation" do
+        expect(subject.to_s).to eq "<Message in 'topic_name' with key 'key', payload 'payload', partition 3, offset 100, timestamp 1000>"
+      end
+    end
+
+    context "with a very long key and payload" do
+      let(:key) { "k" * 100_000 }
+      let(:payload) { "p" * 100_000 }
+
+      it "should have a human readable representation" do
+        expect(subject.to_s).to eq "<Message in 'topic_name' with key 'kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk...', payload 'pppppppppppppppppppppppppppppppppppppppp...', partition 3, offset 100, timestamp 1000>"
+      end
+    end
+  end
+end