nulogy_message_bus_consumer 0.3.0 → 0.5.0

data/spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb

@@ -0,0 +1,54 @@
+RSpec.describe NulogyMessageBusConsumer::Steps::ConnectToMessageBus do
+  subject(:pipeline) do
+    pipeline = NulogyMessageBusConsumer.recommended_consumer_pipeline(config: config)
+    pipeline.insert(tap, after: NulogyMessageBusConsumer::Steps::ConnectToMessageBus)
+    pipeline.append(message_handler_spy)
+    pipeline
+  end
+
+  let(:topic) { TestTopic.new }
+  let(:config) { topic.config }
+  let(:tap) { MiddlewareTap.new }
+  let(:message_handler_spy) { double }
+
+  after { topic.close }
+
+  # TODO: This spec is terribly flakey.
+  xit "receives messages" do
+    called = false
+    expect(message_handler_spy).to receive(:call) do |message:, **_kargs|
+      expect(message).to have_attributes(event_data: {data: "Some Payload"})
+      called = true
+      :success
+    end
+
+    pipeline_thread = start(pipeline, tap)
+
+    topic.produce_one_message(
+      key: "Some Key",
+      payload: message_payload(data: "Some Payload")
+    )
+
+    NulogyMessageBusConsumer::KafkaUtils.wait_for { called }
+    Thread.kill(pipeline_thread)
+  end
+
+  def start(pipeline, tap)
+    thr = Thread.new { pipeline.invoke }
+    wait_for_partition_assignment(tap)
+    thr
+  end
+
+  def wait_for_partition_assignment(tap)
+    NulogyMessageBusConsumer::KafkaUtils.wait_for { tap.arguments[:kafka_consumer] }
+    NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(tap.arguments[:kafka_consumer])
+  end
+
+  def message_payload(**payload)
+    JSON.dump(
+      id: SecureRandom.uuid,
+      created_at: 1_000,
+      event_json: JSON.dump(payload)
+    )
+  end
+end

data/spec/integration/nulogy_message_bus_consumer/steps/supervise_consumer_lag_spec.rb

@@ -0,0 +1,54 @@
+module NulogyMessageBusConsumer
+  module Steps
+    RSpec.describe SuperviseConsumerLag do
+      # RSpec does not like when objects (spies, doubles, etc) are used
+      # outside of its scope, e.g.
+      # The use of doubles or partial doubles from rspec-mocks outside of the per-test lifecycle is not supported. (RSpec::Mocks::OutsideOfExampleError)
+      let(:thread_spy) do
+        Class.new do
+          attr_reader :killed
+
+          def initialize
+            @killed = false
+          end
+
+          def kill
+            @killed = true
+          end
+        end
+      end
+
+      it "kills the main thread after lag does not change" do
+        thread = thread_spy.new
+        logger = spy
+
+        consumer = instance_double(
+          Rdkafka::Consumer,
+          committed: nil,
+          lag: {}
+        )
+        # skip waiting for assignment
+        allow(KafkaUtils).to receive(:wait_for_assignment).with(consumer).and_return(nil)
+        tracker = instance_double(
+          LagTracker,
+          failing_checks: 3,
+          failing?: true,
+          failed: {"topic" => ["partition1", "partition2"]}
+        ).as_null_object
+
+        supervisor = described_class.new(
+          logger,
+          tracker: tracker,
+          killable: thread
+        )
+
+        supervisor.call(kafka_consumer: consumer) {}
+
+        KafkaUtils.wait_for { thread.killed }
+
+        expect(logger).to have_received(:warn).with(/Assigned partition lag has not changed/)
+        expect(thread.killed).to be(true)
+      end
+    end
+  end
+end

data/spec/integration/test_topic_spec.rb

@@ -0,0 +1,39 @@
+RSpec.describe TestTopic do
+  let(:topic) { TestTopic.new }
+
+  after { topic.close }
+
+  context "smoke test for specs" do
+    it "publishes and receives messages" do
+      topic.produce_one_message(
+        key: "Some Key",
+        payload: "Some Payload"
+      )
+
+      message = topic.consume_one_message
+      expect(message).to have_attributes(
+        key: "Some Key",
+        payload: "Some Payload"
+      )
+    end
+
+    it "receives returns nil when no messages are received" do
+      consumer = topic.consumer
+      message = consumer.poll(1)
+      expect(message).to be(nil)
+    end
+  end
+
+  context "spec helpers" do
+    it "creates topics" do
+      create_topic(topic.topic_name)
+      expect(list_topics).to include(topic.topic_name)
+    end
+
+    it "deletes topics" do
+      create_topic(topic.topic_name)
+      delete_topic(topic.topic_name)
+      expect(list_topics).not_to include(topic.topic_name)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,49 @@
+require "bundler/setup"
+require "rspec/json_expectations"
+ENV["RAILS_ENV"] ||= "test"
+require File.expand_path("dummy/config/environment", __dir__)
+abort("The Rails environment is running in production mode!") if Rails.env.production?
+require "rspec/rails"
+require "dotenv"
+
+env_file = ENV["CI"] == "true" ? ".env.ci" : ".env.local"
+raise "Expected #{env_file}" unless File.exist?(env_file)
+
+Dotenv.load(env_file)
+
+# Load RSpec helpers.
+ENGINE_ROOT ||= File.expand_path("..", __dir__)
+Dir[File.join(ENGINE_ROOT, "spec/support/**/*.rb")].sort.each { |f| require f }
+
+# Load migrations from the dummy app.
+# ActiveRecord::Migrator.migrations_paths = File.join(ENGINE_ROOT, 'spec/dummy/db/migrate')
+begin
+  ActiveRecord::Migration.maintain_test_schema!
+rescue ActiveRecord::PendingMigrationError => e
+  puts e.to_s.strip
+  exit 1
+end
+
+RSpec.configure do |config|
+  config.example_status_persistence_file_path = ".rspec_status"
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.use_transactional_fixtures = true
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+    c.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+  config.filter_rails_from_backtrace!
+  config.use_transactional_fixtures = true
+
+  config.include(Kafka)
+end

data/spec/support/kafka.rb

@@ -0,0 +1,74 @@
+require "open3"
+
+module Kafka
+  module_function
+
+  def kafka_config
+    config = {
+      "auto.offset.reset": "beginning",
+      "bootstrap.servers": test_bootstrap_servers,
+      "enable.auto.commit": false,
+      "group.id": random_consumer_group
+    }
+
+    Rdkafka::Config.new(config)
+  end
+
+  def random_topic_name
+    "test-topic-#{SecureRandom.uuid}"
+  end
+
+  def random_consumer_group
+    "ruby-test-consumer-group-#{SecureRandom.uuid}"
+  end
+
+  def test_bootstrap_servers
+    "#{ENV["MBC_KAFKA_HOST"]}:#{ENV["MBC_KAFKA_PORT"]}"
+  end
+
+  def setup_kafka_producer
+    kafka_config.producer
+  end
+
+  def setup_kafka_consumer(topic_name)
+    consumer = kafka_config.consumer
+    consumer.subscribe(topic_name)
+    NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(consumer)
+    consumer
+  end
+
+  def create_topic(topic_name)
+    run("kaf topic create #{topic_name} --brokers #{test_bootstrap_servers} --replicas 1 --partitions 3")
+  end
+
+  def delete_topic(topic_name)
+    run("kaf topic delete #{topic_name} --brokers #{test_bootstrap_servers}")
+  end
+
+  def list_topics
+    topics = run("kaf topics --brokers #{test_bootstrap_servers}")
+    topics.split(" ")
+  end
+
+  def run(command)
+    stdout, stderr, status = Open3.capture3(command)
+    raise <<~OUTPUT if status != 0
+      Command `#{command}` failed with:
+      STDOUT:
+      #{stdout}
+
+      STDERR:
+      #{stderr}
+    OUTPUT
+
+    stdout
+  end
+
+  def test_config(topic_name)
+    NulogyMessageBusConsumer::Config.new(
+      consumer_group_id: random_consumer_group,
+      bootstrap_servers: test_bootstrap_servers,
+      topic_name: topic_name
+    )
+  end
+end

data/spec/support/middleware_tap.rb

@@ -0,0 +1,12 @@
+class MiddlewareTap
+  attr_reader :arguments
+
+  def initialize
+    @arguments = {}
+  end
+
+  def call(**arguments)
+    @arguments = arguments
+    yield
+  end
+end

data/spec/support/test_topic.rb

@@ -0,0 +1,48 @@
+class TestTopic
+  attr_reader :topic_name
+
+  def initialize(topic_name: Kafka.random_topic_name)
+    @topic_name = topic_name
+  end
+
+  def consumer
+    @consumer ||= Kafka.setup_kafka_consumer(topic_name)
+  end
+
+  def producer
+    @producer ||= Kafka.setup_kafka_producer
+  end
+
+  def produce_one_message(key: "TEST KEY", payload: '{ "KEY": "TEST PAYLOAD" }', **kwargs)
+    if kwargs.key?(:topic)
+      raise "Do not specify the topic when producing with a TestTopic. Create a new TestTopic instead."
+    end
+
+    id = SecureRandom.uuid
+    if kwargs.key?(:event_json)
+      kwargs[:payload] = JSON.dump({id: id, event_json: JSON.dump(kwargs.delete(:event_json))})
+    end
+
+    producer.produce(
+      topic: topic_name,
+      key: key,
+      payload: payload,
+      **kwargs
+    ).wait
+
+    id
+  end
+
+  def consume_one_message
+    consumer.poll(250)
+  end
+
+  def config
+    Kafka.test_config(topic_name)
+  end
+
+  def close
+    @producer&.close
+    @consumer&.close
+  end
+end

data/spec/unit/nulogy_message_bus_consumer/config_spec.rb

@@ -0,0 +1,20 @@
+module NulogyMessageBusConsumer
+  RSpec.describe Config do
+    describe "defaults" do
+      it "lag_check_interval_seconds to 20" do
+        expect(Config.new.lag_check_interval_seconds).to be(20)
+      end
+
+      it "lag_checks to 6" do
+        expect(Config.new.lag_checks).to be(6)
+      end
+    end
+
+    it "merges defaults" do
+      config = Config.new(lag_checks: 3)
+
+      expect(config.lag_checks).to be(3)
+      expect(config.lag_check_interval_seconds).to be(20)
+    end
+  end
+end

data/spec/unit/nulogy_message_bus_consumer/lag_tracker.rb

@@ -0,0 +1,35 @@
+module NulogyMessageBusConsumer
+  RSpec.describe LagTracker do
+    it "does not fail when values change" do
+      tracker = described_class.new(failing_checks: 1)
+
+      tracker.update({"one" => {"0" => 1, "1" => 1}})
+      tracker.update({"one" => {"0" => 2, "1" => 0}})
+
+      expect(tracker).not_to be_failing
+    end
+
+    it "tracks failed when value unchanged enough" do
+      tracker = described_class.new(failing_checks: 2)
+
+      tracker.update({"one" => {"0" => 1, "1" => 1}})
+      tracker.update({"one" => {"0" => 1, "1" => 2}})
+      tracker.update({"one" => {"0" => 1, "1" => 3}})
+
+      expect(tracker).to be_failing
+      expect(tracker.failed).to eq({
+        "one" => ["0"]
+      })
+    end
+
+    it "ignores unchanged 0 values" do
+      tracker = described_class.new(failing_checks: 1)
+
+      tracker.update({"one" => {"0" => 0}})
+      tracker.update({"one" => {"0" => 0}})
+      tracker.update({"one" => {"0" => 0}})
+
+      expect(tracker).not_to be_failing
+    end
+  end
+end

data/spec/unit/nulogy_message_bus_consumer/message_spec.rb

@@ -0,0 +1,84 @@
+RSpec.describe NulogyMessageBusConsumer::Message do
+  describe ".from_kafka" do
+    let(:message_id) { SecureRandom.uuid }
+    let(:subscription_id) { SecureRandom.uuid }
+    let(:company_uuid) { SecureRandom.uuid }
+    let(:partition_key) { [company_uuid, subscription_id].join(",") }
+
+    it "builds a message from a kafka message" do
+      kafka_message = build_kafka_message(
+        payload: JSON.dump(
+          id: message_id,
+          subscription_id: subscription_id,
+          company_uuid: company_uuid,
+          event_json: JSON.dump(hello: "world")
+        )
+      )
+
+      built = NulogyMessageBusConsumer::Message.from_kafka(kafka_message)
+
+      expect(built).to have_attributes(
+        id: message_id,
+        subscription_id: subscription_id,
+        company_uuid: company_uuid,
+        event_data: {hello: "world"},
+        event_data_unparsed: include_json(hello: "world")
+      )
+    end
+
+    it "expects the event_json to be valid json" do
+      kafka_message = build_kafka_message(
+        payload: JSON.dump(event_json: "not json")
+      )
+
+      built = NulogyMessageBusConsumer::Message.from_kafka(kafka_message)
+
+      expect(built).to have_attributes(
+        event_data: {},
+        event_data_unparsed: "not json"
+      )
+    end
+
+    it "falls back on some depricated columns" do
+      kafka_message = build_kafka_message(
+        payload: JSON.dump(
+          public_subscription_id: subscription_id,
+          tenant_id: company_uuid
+        )
+      )
+
+      built = NulogyMessageBusConsumer::Message.from_kafka(kafka_message)
+
+      expect(built).to have_attributes(
+        subscription_id: subscription_id,
+        company_uuid: company_uuid
+      )
+    end
+  end
+
+  def build_kafka_message(overrides = {}) # rubocop:disable Metrics/MethodLength
+    attributes = {
+      topic: "some_topic",
+      partition: 3,
+      payload: JSON.dump(
+        id: message_id,
+        subscription_id: subscription_id,
+        company_uuid: company_uuid,
+        event_json: JSON.dump(
+          hello: "world"
+        )
+      ),
+      key: JSON.dump(
+        schema: {},
+        payload: {
+          partition_key: partition_key
+        }
+      ),
+      offset: 13,
+      timestamp: Time.zone.now,
+      headers: {key: "value"}
+    }.merge(overrides)
+
+    instance_double("Rdkafka::Consumer::Message", attributes)
+  end
+end