nulogy_message_bus_consumer 0.5.0 → 1.0.0.alpha

data/spec/support/kafka.rb

@@ -1,74 +0,0 @@
-require "open3"
-
-module Kafka
-  module_function
-
-  def kafka_config
-    config = {
-      "auto.offset.reset": "beginning",
-      "bootstrap.servers": test_bootstrap_servers,
-      "enable.auto.commit": false,
-      "group.id": random_consumer_group
-    }
-
-    Rdkafka::Config.new(config)
-  end
-
-  def random_topic_name
-    "test-topic-#{SecureRandom.uuid}"
-  end
-
-  def random_consumer_group
-    "ruby-test-consumer-group-#{SecureRandom.uuid}"
-  end
-
-  def test_bootstrap_servers
-    "#{ENV["MBC_KAFKA_HOST"]}:#{ENV["MBC_KAFKA_PORT"]}"
-  end
-
-  def setup_kafka_producer
-    kafka_config.producer
-  end
-
-  def setup_kafka_consumer(topic_name)
-    consumer = kafka_config.consumer
-    consumer.subscribe(topic_name)
-    NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(consumer)
-    consumer
-  end
-
-  def create_topic(topic_name)
-    run("kaf topic create #{topic_name} --brokers #{test_bootstrap_servers} --replicas 1 --partitions 3")
-  end
-
-  def delete_topic(topic_name)
-    run("kaf topic delete #{topic_name} --brokers #{test_bootstrap_servers}")
-  end
-
-  def list_topics
-    topics = run("kaf topics --brokers #{test_bootstrap_servers}")
-    topics.split(" ")
-  end
-
-  def run(command)
-    stdout, stderr, status = Open3.capture3(command)
-    raise <<~OUTPUT if status != 0
-      Command `#{command}` failed with:
-      STDOUT:
-      #{stdout}
-
-      STDERR:
-      #{stderr}
-    OUTPUT
-
-    stdout
-  end
-
-  def test_config(topic_name)
-    NulogyMessageBusConsumer::Config.new(
-      consumer_group_id: random_consumer_group,
-      bootstrap_servers: test_bootstrap_servers,
-      topic_name: topic_name
-    )
-  end
-end

data/spec/support/middleware_tap.rb

@@ -1,12 +0,0 @@
-class MiddlewareTap
-  attr_reader :arguments
-
-  def initialize
-    @arguments = {}
-  end
-
-  def call(**arguments)
-    @arguments = arguments
-    yield
-  end
-end

data/spec/support/test_topic.rb

@@ -1,48 +0,0 @@
-class TestTopic
-  attr_reader :topic_name
-
-  def initialize(topic_name: Kafka.random_topic_name)
-    @topic_name = topic_name
-  end
-
-  def consumer
-    @consumer ||= Kafka.setup_kafka_consumer(topic_name)
-  end
-
-  def producer
-    @producer ||= Kafka.setup_kafka_producer
-  end
-
-  def produce_one_message(key: "TEST KEY", payload: '{ "KEY": "TEST PAYLOAD" }', **kwargs)
-    if kwargs.key?(:topic)
-      raise "Do not specify the topic when producing with a TestTopic. Create a new TestTopic instead."
-    end
-
-    id = SecureRandom.uuid
-    if kwargs.key?(:event_json)
-      kwargs[:payload] = JSON.dump({id: id, event_json: JSON.dump(kwargs.delete(:event_json))})
-    end
-
-    producer.produce(
-      topic: topic_name,
-      key: key,
-      payload: payload,
-      **kwargs
-    ).wait
-
-    id
-  end
-
-  def consume_one_message
-    consumer.poll(250)
-  end
-
-  def config
-    Kafka.test_config(topic_name)
-  end
-
-  def close
-    @producer&.close
-    @consumer&.close
-  end
-end

data/spec/unit/nulogy_message_bus_consumer/config_spec.rb

@@ -1,20 +0,0 @@
-module NulogyMessageBusConsumer
-  RSpec.describe Config do
-    describe "defaults" do
-      it "lag_check_interval_seconds to 20" do
-        expect(Config.new.lag_check_interval_seconds).to be(20)
-      end
-
-      it "lag_checks to 6" do
-        expect(Config.new.lag_checks).to be(6)
-      end
-    end
-
-    it "merges defaults" do
-      config = Config.new(lag_checks: 3)
-
-      expect(config.lag_checks).to be(3)
-      expect(config.lag_check_interval_seconds).to be(20)
-    end
-  end
-end

data/spec/unit/nulogy_message_bus_consumer/lag_tracker.rb

@@ -1,35 +0,0 @@
-module NulogyMessageBusConsumer
-  RSpec.describe LagTracker do
-    it "does not fail when values change" do
-      tracker = described_class.new(failing_checks: 1)
-
-      tracker.update({"one" => {"0" => 1, "1" => 1}})
-      tracker.update({"one" => {"0" => 2, "1" => 0}})
-
-      expect(tracker).not_to be_failing
-    end
-
-    it "tracks failed when value unchanged enough" do
-      tracker = described_class.new(failing_checks: 2)
-
-      tracker.update({"one" => {"0" => 1, "1" => 1}})
-      tracker.update({"one" => {"0" => 1, "1" => 2}})
-      tracker.update({"one" => {"0" => 1, "1" => 3}})
-
-      expect(tracker).to be_failing
-      expect(tracker.failed).to eq({
-        "one" => ["0"]
-      })
-    end
-
-    it "ignores unchanged 0 values" do
-      tracker = described_class.new(failing_checks: 1)
-
-      tracker.update({"one" => {"0" => 0}})
-      tracker.update({"one" => {"0" => 0}})
-      tracker.update({"one" => {"0" => 0}})
-
-      expect(tracker).not_to be_failing
-    end
-  end
-end

data/spec/unit/nulogy_message_bus_consumer/message_spec.rb

@@ -1,84 +0,0 @@
-RSpec.describe NulogyMessageBusConsumer::Message do
-  describe ".from_kafka" do
-    let(:message_id) { SecureRandom.uuid }
-    let(:subscription_id) { SecureRandom.uuid }
-    let(:company_uuid) { SecureRandom.uuid }
-    let(:partition_key) { [company_uuid, subscription_id].join(",") }
-
-    it "builds a message from a kafka message" do
-      kafka_message = build_kafka_message(
-        payload: JSON.dump(
-          id: message_id,
-          subscription_id: subscription_id,
-          company_uuid: company_uuid,
-          event_json: JSON.dump(hello: "world")
-        )
-      )
-
-      built = NulogyMessageBusConsumer::Message.from_kafka(kafka_message)
-
-      expect(built).to have_attributes(
-        id: message_id,
-        subscription_id: subscription_id,
-        company_uuid: company_uuid,
-        event_data: {hello: "world"},
-        event_data_unparsed: include_json(hello: "world")
-      )
-    end
-
-    it "expects the event_json to be valid json" do
-      kafka_message = build_kafka_message(
-        payload: JSON.dump(event_json: "not json")
-      )
-
-      built = NulogyMessageBusConsumer::Message.from_kafka(kafka_message)
-
-      expect(built).to have_attributes(
-        event_data: {},
-        event_data_unparsed: "not json"
-      )
-    end
-
-    it "falls back on some depricated columns" do
-      kafka_message = build_kafka_message(
-        payload: JSON.dump(
-          public_subscription_id: subscription_id,
-          tenant_id: company_uuid
-        )
-      )
-
-      built = NulogyMessageBusConsumer::Message.from_kafka(kafka_message)
-
-      expect(built).to have_attributes(
-        subscription_id: subscription_id,
-        company_uuid: company_uuid
-      )
-    end
-  end
-
-  def build_kafka_message(overrides = {}) # rubocop:disable Metrics/MethodLength
-    attributes = {
-      topic: "some_topic",
-      partition: 3,
-      payload: JSON.dump(
-        id: message_id,
-        subscription_id: subscription_id,
-        company_uuid: company_uuid,
-        event_json: JSON.dump(
-          hello: "world"
-        )
-      ),
-      key: JSON.dump(
-        schema: {},
-        payload: {
-          partition_key: partition_key
-        }
-      ),
-      offset: 13,
-      timestamp: Time.zone.now,
-      headers: {key: "value"}
-    }.merge(overrides)
-
-    instance_double("Rdkafka::Consumer::Message", attributes)
-  end
-end

data/spec/unit/nulogy_message_bus_consumer/pipeline_spec.rb

@@ -1,49 +0,0 @@
-RSpec.describe NulogyMessageBusConsumer::Pipeline do
-  let(:handler) { spy }
-
-  describe "#invoke_pipeline" do
-    it "raises if handler is not provided" do
-      expect { invoke_pipeline }.to raise_error("Handlers are the end of the line. Do not use yield.")
-    end
-
-    it "calls the handler with message" do
-      message = NulogyMessageBusConsumer::Message.new
-
-      expect(handler).to receive(:call).with(message: message)
-
-      invoke_pipeline(
-        ->(**_, &block) { block.yield(message: message) },
-        handler
-      )
-    end
-
-    it "passes information along" do
-      message = NulogyMessageBusConsumer::Message.new
-
-      expect(handler).to receive(:call).with(message: message)
-
-      invoke_pipeline(
-        ->(**_, &block) { block.yield(message: message) },
-        ->(**_, &block) { block.yield },
-        handler
-      )
-    end
-
-    it "raises when overwriting an existing key" do
-      message = NulogyMessageBusConsumer::Message.new
-      other_message = NulogyMessageBusConsumer::Message.new
-
-      expect {
-        invoke_pipeline(
-          ->(**_, &block) { block.yield(message: message) },
-          ->(**_, &block) { block.yield(message: other_message) },
-          handler
-        )
-      }.to raise_error("Cannot override existing argument(s): message")
-    end
-  end
-
-  def invoke_pipeline(*steps)
-    NulogyMessageBusConsumer::Pipeline.new(steps).invoke
-  end
-end

data/spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb

@@ -1,58 +0,0 @@
-RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
-  subject(:step) { NulogyMessageBusConsumer::Steps::CommitOnSuccess.new }
-
-  let(:kafka_consumer) { spy }
-  let(:message) { NulogyMessageBusConsumer::Message.new }
-
-  context "when result is :success" do
-    it "returns the result" do
-      result = step.call(kafka_consumer: kafka_consumer, message: message) { :success }
-
-      expect(result).to be(:success)
-    end
-
-    it "commits the message offset" do
-      expect(kafka_consumer).to receive(:store_offset).with(message)
-      expect(kafka_consumer).to receive(:commit)
-
-      step.call(kafka_consumer: kafka_consumer, message: message) { :success }
-    end
-  end
-
-  context "when result is :failure" do
-    it "returns the result" do
-      result = step.call(kafka_consumer: kafka_consumer, message: message) { :failure }
-
-      expect(result).to be(:failure)
-    end
-
-    it "reconnects to Kafka when failure" do
-      # When subscribing, we subscribe to a topic (i.e. the keys)
-      topic_partition_list = {topic_name: %i[partition_1 partition_2]}
-      allow(kafka_consumer).to receive(:subscription).and_return(topic_partition_list)
-
-      expect(kafka_consumer).to receive(:unsubscribe)
-      expect(kafka_consumer).to receive(:subscribe).with(:topic_name)
-
-      step.call(kafka_consumer: kafka_consumer, message: message) { :failure }
-    end
-  end
-
-  context "when result is something else" do
-    it "raises" do
-      expect {
-        step.call(kafka_consumer: kafka_consumer, message: message) { :bogus_value }
-      }.to raise_error("'bogus_value' is not a valid processing outcome. Must be :success or :failure")
-    end
-  end
-
-  context "when downstream steps raise" do
-    it "does not commit" do
-      expect(kafka_consumer).not_to receive(:commit)
-
-      expect {
-        step.call(kafka_consumer: kafka_consumer, message: message) { raise "oopsie" }
-      }.to raise_error("oopsie")
-    end
-  end
-end

data/spec/unit/nulogy_message_bus_consumer/steps/deduplicate_messages_spec.rb

@@ -1,56 +0,0 @@
-RSpec.describe NulogyMessageBusConsumer::Steps::DeduplicateMessages do
-  subject(:deduplicate_messages) { NulogyMessageBusConsumer::Steps::DeduplicateMessages.new(logger) }
-
-  let(:logger) { spy }
-  let(:id) { SecureRandom.uuid }
-  let(:message) { NulogyMessageBusConsumer::Message.new(id: id) }
-
-  context "when a duplicate message is consumed" do
-    it "logs a duplicate was detected" do
-      expect(logger).to receive(:warn).with(include_json(
-        event: "duplicate_message_detected",
-        kafka_message_id: id
-      ))
-
-      deduplicate_messages.call(message: message) { :success }
-      deduplicate_messages.call(message: message) { raise "Not evaluated" }
-    end
-
-    it "returns :success" do
-      deduplicate_messages.call(message: message) { :success }
-
-      result = deduplicate_messages.call(message: message) { raise "Not evaluated" }
-
-      expect(result).to be(:success)
-    end
-  end
-
-  context "when a duplicate message is processed at the same time" do
-    it "logs a duplicate was detected" do
-      expect(logger).to receive(:warn).with(include_json(
-        event: "duplicate_message_detected",
-        kafka_message_id: id
-      ))
-
-      deduplicate_messages.call(message: message) do
-        NulogyMessageBusConsumer::ProcessedMessage.create!(id: id)
-        :success
-      end
-    end
-
-    it "returns :success" do
-      result = deduplicate_messages.call(message: message) {
-        NulogyMessageBusConsumer::ProcessedMessage.create!(id: id)
-        :success
-      }
-
-      expect(result).to be(:success)
-    end
-  end
-
-  it "records message was processed" do
-    deduplicate_messages.call(message: message) { :success }
-
-    expect(NulogyMessageBusConsumer::ProcessedMessage.exists?(id: id)).to be(true)
-  end
-end

data/spec/unit/nulogy_message_bus_consumer/steps/log_messages_spec.rb

@@ -1,70 +0,0 @@
-RSpec.describe NulogyMessageBusConsumer::Steps::LogMessages do
-  let(:logger) { spy }
-
-  it "logs the message was received" do
-    log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
-
-    expect(logger).to receive(:info).with(include_json(
-      event: "message_received",
-      kafka_message_id: "id",
-      message: "Received id"
-    ))
-
-    log_message.call(message: message) {}
-  end
-
-  it "logs the message was processed" do
-    log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
-
-    expect(logger).to receive(:info).with(include_json(
-      event: "message_processed",
-      kafka_message_id: "id",
-      message: match(/^Processed id/)
-    ))
-
-    log_message.call(message: message) {}
-  end
-
-  it "logs the message's time to processed" do
-    # clock always gives milliseconds since epoch
-    clock = instance_double(NulogyMessageBusConsumer::Clock, ms: 1_000_001_234)
-    log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger, clock: clock)
-
-    expect(logger).to receive(:info).with(include_json(
-      event: "message_processed",
-      kafka_message_id: "id",
-      time_to_processed: 1234
-    ))
-
-    # debezium converts time with zone to nanoseconds since epoch
-    log_message.call(message: message(created_at: 1_000_000_000_000)) {}
-  end
-
-  it "logs the message's result" do
-    log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
-
-    expect(logger).to receive(:info).with(include_json(
-      event: "message_processed",
-      result: "success"
-    ))
-
-    log_message.call(message: message) { :success }
-
-    expect(logger).to receive(:info).with(include_json(
-      event: "message_processed",
-      result: "failure"
-    ))
-
-    log_message.call(message: message) { :failure }
-  end
-
-  it "returns a result" do
-    log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
-
-    expect(log_message.call(message: message) { :return_value }).to eq(:return_value)
-  end
-
-  def message(id: "id", created_at: 1_000, **attrs)
-    NulogyMessageBusConsumer::Message.new(id: id, created_at: created_at, **attrs)
-  end
-end

data/spec/unit/nulogy_message_bus_consumer/steps/monitor_replication_lag/calculator_spec.rb

@@ -1,63 +0,0 @@
-RSpec.describe NulogyMessageBusConsumer::Steps::LogConsumerLag::Calculator do
-  subject(:calculator) { NulogyMessageBusConsumer::Steps::LogConsumerLag::Calculator }
-
-  describe ".add_max_lag" do
-    it "adds max lag per partition" do
-      lag_per_topic = {
-        topic_1: {
-          partition_1: 100,
-          partition_2: 900,
-          partition_3: 100
-        },
-        topic_2: {
-          partition_1: 100,
-          partition_2: 200,
-          partition_3: 100
-        }
-      }
-
-      lag_with_max = calculator.add_max_lag(lag_per_topic)
-
-      expect(lag_with_max).to match(
-        _max: 900,
-        topic_1: {
-          _max: 900,
-          partition_1: 100,
-          partition_2: 900,
-          partition_3: 100
-        },
-        topic_2: {
-          _max: 200,
-          partition_1: 100,
-          partition_2: 200,
-          partition_3: 100
-        }
-      )
-    end
-
-    it "sets the max to 0 when no partitions are subscribed to" do
-      lag_per_topic = {
-        topic_1: {}
-      }
-
-      lag_with_max = calculator.add_max_lag(lag_per_topic)
-
-      expect(lag_with_max).to match(
-        _max: 0,
-        topic_1: {
-          _max: 0
-        }
-      )
-    end
-
-    it "sets the max to 0 when no topics are subscribed to" do
-      lag_per_topic = {}
-
-      lag_with_max = calculator.add_max_lag(lag_per_topic)
-
-      expect(lag_with_max).to match(
-        _max: 0
-      )
-    end
-  end
-end

data/spec/unit/nulogy_message_bus_consumer/steps/stream_messages_spec.rb

@@ -1,35 +0,0 @@
-RSpec.describe NulogyMessageBusConsumer::Steps::StreamMessages do
-  subject(:step) { NulogyMessageBusConsumer::Steps::StreamMessages.new(logger) }
-
-  let(:config) { NulogyMessageBusConsumer::Config.new }
-  let(:logger) { spy }
-
-  it "logs errors while streaming from kafka" do
-    kafka_consumer = instance_double("Rdkafka::Consumer")
-    allow(kafka_consumer).to receive(:subscribe)
-    allow(kafka_consumer).to receive(:unsubscribe)
-    allow(kafka_consumer).to receive(:each).and_raise(StandardError, "streaming failed")
-
-    expect(logger).to receive(:error).with(include_json(
-      event: "message_processing_errored",
-      class: "StandardError",
-      message: "streaming failed"
-    ))
-
-    expect {
-      step.call(kafka_consumer: kafka_consumer)
-    }.to raise_error(StandardError)
-  end
-
-  context "when handler returns failure" do
-    it "does not commit the message in kafka" do
-      kafka_consumer = spy
-      allow(kafka_consumer).to receive(:each).and_yield(anything)
-      allow(NulogyMessageBusConsumer::Message).to receive(:from_kafka).and_return(NulogyMessageBusConsumer::Message.new)
-
-      expect(kafka_consumer).not_to receive(:commit)
-
-      step.call(kafka_consumer: kafka_consumer) { :failure }
-    end
-  end
-end

data/spec/unit/nulogy_message_bus_consumer_spec.rb

@@ -1,30 +0,0 @@
-RSpec.describe NulogyMessageBusConsumer do
-  describe "#configure" do
-    before do
-      config = NulogyMessageBusConsumer.config
-      config.instance_variable_names.each do |variable_name|
-        config.remove_instance_variable(variable_name)
-      end
-    end
-
-    it "allows configuration to be provided as arguments" do
-      NulogyMessageBusConsumer.configure(
-        bootstrap_servers: "some servers"
-      )
-
-      expect(NulogyMessageBusConsumer.config).to have_attributes(
-        bootstrap_servers: "some servers"
-      )
-    end
-
-    it "allows configuration to be provided as a block" do
-      NulogyMessageBusConsumer.configure do |config|
-        config.bootstrap_servers = "some servers"
-      end
-
-      expect(NulogyMessageBusConsumer.config).to have_attributes(
-        bootstrap_servers: "some servers"
-      )
-    end
-  end
-end