nulogy_message_bus_consumer 0.3.2 → 1.0.0

data/spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb

@@ -0,0 +1,53 @@
+RSpec.describe NulogyMessageBusConsumer::Steps::ConnectToMessageBus do
+  subject(:pipeline) do
+    pipeline = NulogyMessageBusConsumer.recommended_consumer_pipeline(config: config)
+    pipeline.insert(tap, after: NulogyMessageBusConsumer::Steps::ConnectToMessageBus)
+    pipeline.append(message_handler_spy)
+    pipeline
+  end
+
+  let(:topic) { TestTopic.new }
+  let(:config) { topic.config }
+  let(:tap) { MiddlewareTap.new }
+  let(:message_handler_spy) { double }
+
+  after { topic.close }
+
+  it "receives messages", skip: flakey_in_ci do
+    called = false
+    expect(message_handler_spy).to receive(:call) do |message:, **_kargs|
+      expect(message).to have_attributes(event_data: {data: "Some Payload"})
+      called = true
+      :success
+    end
+
+    pipeline_thread = start(pipeline, tap)
+
+    topic.produce_one_message(
+      key: "Some Key",
+      payload: message_payload(data: "Some Payload")
+    )
+
+    NulogyMessageBusConsumer::KafkaUtils.wait_for { called }
+    Thread.kill(pipeline_thread)
+  end
+
+  def start(pipeline, tap)
+    thr = Thread.new { pipeline.invoke }
+    wait_for_partition_assignment(tap)
+    thr
+  end
+
+  def wait_for_partition_assignment(tap)
+    NulogyMessageBusConsumer::KafkaUtils.wait_for { tap.arguments[:kafka_consumer] }
+    NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(tap.arguments[:kafka_consumer])
+  end
+
+  def message_payload(**payload)
+    JSON.dump(
+      id: SecureRandom.uuid,
+      created_at: 1_000,
+      event_json: JSON.dump(payload)
+    )
+  end
+end

data/spec/integration/nulogy_message_bus_consumer/tasks/prune_processed_messages_spec.rb

@@ -0,0 +1,32 @@
+module NulogyMessageBusConsumer
+  module Tasks
+    RSpec.describe PruneProcessedMessages do
+      let(:logger) { spy }
+      let(:task) { described_class.new(logger, 10, 1.minute) }
+
+      it "prunes old messages" do
+        processed_message(2.minutes.ago)
+        processed_message(1.minute.ago)
+
+        task.call
+
+        expect(ProcessedMessage.count).to be(0)
+        expect(logger).to have_received(:info).with(/Pruned 2 processed messages/)
+      end
+
+      it "does not prune newer messages" do
+        processed_message(2.minutes.ago)
+        processed_message(1.minute.ago)
+        keep = processed_message(1.minute.from_now)
+
+        task.call
+
+        expect(ProcessedMessage.ids).to contain_exactly(keep.id)
+      end
+
+      def processed_message(ago)
+        ProcessedMessage.create!(id: SecureRandom.uuid, created_at: ago)
+      end
+    end
+  end
+end

data/spec/integration/nulogy_message_bus_consumer/tasks/supervise_consumer_lag_spec.rb

@@ -0,0 +1,33 @@
+module NulogyMessageBusConsumer
+  module Tasks
+    RSpec.describe SuperviseConsumerLag do
+      let(:killable) { spy }
+      let(:logger) { spy }
+      let(:tracker) { LagTracker.new(failing_checks: 2) }
+      let(:task) { described_class.new(logger, tracker: tracker, killable: killable) }
+
+      it "kills the main thread after lag does not change" do
+        lag = {
+          "partition-1" => 2,
+          "partition-2" => 3
+        }
+        tracker.update({"topic" => lag})
+        tracker.update({"topic" => lag})
+        consumer = instance_double(
+          Rdkafka::Consumer,
+          committed: nil,
+          lag: {"topic" => lag}
+        )
+        # skip waiting for assignment
+        allow(KafkaUtils).to receive(:wait_for_assignment).with(consumer).and_return(nil)
+        allow(Thread.current).to receive(:exit)
+        task.extract_args(kafka_consumer: consumer)
+
+        task.call
+
+        expect(logger).to have_received(:warn).with(/Assigned partition lag has not changed/)
+        expect(killable).to have_received(:kill)
+      end
+    end
+  end
+end

data/spec/integration/test_topic_spec.rb

@@ -0,0 +1,39 @@
+RSpec.describe TestTopic do
+  let(:topic) { TestTopic.new }
+
+  after { topic.close }
+
+  context "smoke test for specs" do
+    it "publishes and receives messages" do
+      topic.produce_one_message(
+        key: "Some Key",
+        payload: "Some Payload"
+      )
+
+      message = topic.consume_one_message
+      expect(message).to have_attributes(
+        key: "Some Key",
+        payload: "Some Payload"
+      )
+    end
+
+    it "receives returns nil when no messages are received" do
+      consumer = topic.consumer
+      message = consumer.poll(1)
+      expect(message).to be(nil)
+    end
+  end
+
+  context "spec helpers" do
+    it "creates topics" do
+      create_topic(topic.topic_name)
+      expect(list_topics).to include(topic.topic_name)
+    end
+
+    it "deletes topics" do
+      create_topic(topic.topic_name)
+      delete_topic(topic.topic_name)
+      expect(list_topics).not_to include(topic.topic_name)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,50 @@
+require "bundler/setup"
+require "rspec/json_expectations"
+ENV["RAILS_ENV"] ||= "test"
+require File.expand_path("dummy/config/environment", __dir__)
+abort("The Rails environment is running in production mode!") if Rails.env.production?
+require "rspec/rails"
+require "dotenv"
+
+env_file = ENV["CI"] == "true" ? ".env.ci" : ".env.local"
+raise "Expected #{env_file}" unless File.exist?(env_file)
+
+Dotenv.load(env_file)
+
+# Load RSpec helpers.
+ENGINE_ROOT ||= File.expand_path("..", __dir__)
+Dir[File.join(ENGINE_ROOT, "spec/support/**/*.rb")].sort.each { |f| require f }
+
+# Load migrations from the dummy app.
+# ActiveRecord::Migrator.migrations_paths = File.join(ENGINE_ROOT, 'spec/dummy/db/migrate')
+begin
+  ActiveRecord::Migration.maintain_test_schema!
+rescue ActiveRecord::PendingMigrationError => e
+  puts e.to_s.strip
+  exit 1
+end
+
+RSpec.configure do |config|
+  config.example_status_persistence_file_path = ".rspec_status"
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.use_transactional_fixtures = true
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+    c.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+  config.filter_rails_from_backtrace!
+  config.use_transactional_fixtures = true
+
+  config.include(Kafka)
+  config.extend(Skip)
+end

data/spec/support/kafka.rb

@@ -0,0 +1,74 @@
+require "open3"
+
+module Kafka
+  module_function
+
+  def kafka_config
+    config = {
+      "auto.offset.reset": "beginning",
+      "bootstrap.servers": test_bootstrap_servers,
+      "enable.auto.commit": false,
+      "group.id": random_consumer_group
+    }
+
+    Rdkafka::Config.new(config)
+  end
+
+  def random_topic_name
+    "test-topic-#{SecureRandom.uuid}"
+  end
+
+  def random_consumer_group
+    "ruby-test-consumer-group-#{SecureRandom.uuid}"
+  end
+
+  def test_bootstrap_servers
+    "#{ENV["MBC_KAFKA_HOST"]}:#{ENV["MBC_KAFKA_PORT"]}"
+  end
+
+  def setup_kafka_producer
+    kafka_config.producer
+  end
+
+  def setup_kafka_consumer(topic_name)
+    consumer = kafka_config.consumer
+    consumer.subscribe(topic_name)
+    NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(consumer)
+    consumer
+  end
+
+  def create_topic(topic_name)
+    run("kaf topic create #{topic_name} --brokers #{test_bootstrap_servers} --replicas 1 --partitions 3")
+  end
+
+  def delete_topic(topic_name)
+    run("kaf topic delete #{topic_name} --brokers #{test_bootstrap_servers}")
+  end
+
+  def list_topics
+    topics = run("kaf topics --brokers #{test_bootstrap_servers}")
+    topics.split(" ")
+  end
+
+  def run(command)
+    stdout, stderr, status = Open3.capture3(command)
+    raise <<~OUTPUT if status != 0
+      Command `#{command}` failed with:
+      STDOUT:
+      #{stdout}
+
+      STDERR:
+      #{stderr}
+    OUTPUT
+
+    stdout
+  end
+
+  def test_config(topic_name)
+    NulogyMessageBusConsumer::Config.new(
+      consumer_group_id: random_consumer_group,
+      bootstrap_servers: test_bootstrap_servers,
+      topic_name: topic_name
+    )
+  end
+end

data/spec/support/middleware_tap.rb

@@ -0,0 +1,12 @@
+class MiddlewareTap
+  attr_reader :arguments
+
+  def initialize
+    @arguments = {}
+  end
+
+  def call(**arguments)
+    @arguments = arguments
+    yield
+  end
+end

data/spec/support/skip.rb

@@ -0,0 +1,9 @@
+module Skip
+  def flakey_in_ci
+    if ENV.fetch("CI", false)
+      "This spec is terribly flakey in CI, so we only run it locally"
+    else
+      false
+    end
+  end
+end

data/spec/support/test_topic.rb

@@ -0,0 +1,48 @@
+class TestTopic
+  attr_reader :topic_name
+
+  def initialize(topic_name: Kafka.random_topic_name)
+    @topic_name = topic_name
+  end
+
+  def consumer
+    @consumer ||= Kafka.setup_kafka_consumer(topic_name)
+  end
+
+  def producer
+    @producer ||= Kafka.setup_kafka_producer
+  end
+
+  def produce_one_message(key: "TEST KEY", payload: '{ "KEY": "TEST PAYLOAD" }', **kwargs)
+    if kwargs.key?(:topic)
+      raise "Do not specify the topic when producing with a TestTopic. Create a new TestTopic instead."
+    end
+
+    id = SecureRandom.uuid
+    if kwargs.key?(:event_json)
+      kwargs[:payload] = JSON.dump({id: id, event_json: JSON.dump(kwargs.delete(:event_json))})
+    end
+
+    producer.produce(
+      topic: topic_name,
+      key: key,
+      payload: payload,
+      **kwargs
+    ).wait
+
+    id
+  end
+
+  def consume_one_message
+    consumer.poll(250)
+  end
+
+  def config
+    Kafka.test_config(topic_name)
+  end
+
+  def close
+    @producer&.close
+    @consumer&.close
+  end
+end

data/spec/unit/nulogy_message_bus_consumer/config_spec.rb

@@ -0,0 +1,20 @@
+module NulogyMessageBusConsumer
+  RSpec.describe Config do
+    describe "defaults" do
+      it "lag_check_interval_seconds to 20" do
+        expect(Config.new.lag_check_interval_seconds).to be(20)
+      end
+
+      it "lag_checks to 6" do
+        expect(Config.new.lag_checks).to be(6)
+      end
+    end
+
+    it "merges defaults" do
+      config = Config.new(lag_checks: 3)
+
+      expect(config.lag_checks).to be(3)
+      expect(config.lag_check_interval_seconds).to be(20)
+    end
+  end
+end

data/spec/unit/nulogy_message_bus_consumer/lag_tracker.rb

@@ -0,0 +1,35 @@
+module NulogyMessageBusConsumer
+  RSpec.describe LagTracker do
+    it "does not fail when values change" do
+      tracker = described_class.new(failing_checks: 1)
+
+      tracker.update({"one" => {"0" => 1, "1" => 1}})
+      tracker.update({"one" => {"0" => 2, "1" => 0}})
+
+      expect(tracker).not_to be_failing
+    end
+
+    it "tracks failed when value unchanged enough" do
+      tracker = described_class.new(failing_checks: 2)
+
+      tracker.update({"one" => {"0" => 1, "1" => 1}})
+      tracker.update({"one" => {"0" => 1, "1" => 2}})
+      tracker.update({"one" => {"0" => 1, "1" => 3}})
+
+      expect(tracker).to be_failing
+      expect(tracker.failed).to eq({
+        "one" => ["0"]
+      })
+    end
+
+    it "ignores unchanged 0 values" do
+      tracker = described_class.new(failing_checks: 1)
+
+      tracker.update({"one" => {"0" => 0}})
+      tracker.update({"one" => {"0" => 0}})
+      tracker.update({"one" => {"0" => 0}})
+
+      expect(tracker).not_to be_failing
+    end
+  end
+end

data/spec/unit/nulogy_message_bus_consumer/message_spec.rb

@@ -0,0 +1,84 @@
+RSpec.describe NulogyMessageBusConsumer::Message do
+  describe ".from_kafka" do
+    let(:message_id) { SecureRandom.uuid }
+    let(:subscription_id) { SecureRandom.uuid }
+    let(:company_uuid) { SecureRandom.uuid }
+    let(:partition_key) { [company_uuid, subscription_id].join(",") }
+
+    it "builds a message from a kafka message" do
+      kafka_message = build_kafka_message(
+        payload: JSON.dump(
+          id: message_id,
+          subscription_id: subscription_id,
+          company_uuid: company_uuid,
+          event_json: JSON.dump(hello: "world")
+        )
+      )
+
+      built = NulogyMessageBusConsumer::Message.from_kafka(kafka_message)
+
+      expect(built).to have_attributes(
+        id: message_id,
+        subscription_id: subscription_id,
+        company_uuid: company_uuid,
+        event_data: {hello: "world"},
+        event_data_unparsed: include_json(hello: "world")
+      )
+    end
+
+    it "expects the event_json to be valid json" do
+      kafka_message = build_kafka_message(
+        payload: JSON.dump(event_json: "not json")
+      )
+
+      built = NulogyMessageBusConsumer::Message.from_kafka(kafka_message)
+
+      expect(built).to have_attributes(
+        event_data: {},
+        event_data_unparsed: "not json"
+      )
+    end
+
+    it "falls back on some depricated columns" do
+      kafka_message = build_kafka_message(
+        payload: JSON.dump(
+          public_subscription_id: subscription_id,
+          tenant_id: company_uuid
+        )
+      )
+
+      built = NulogyMessageBusConsumer::Message.from_kafka(kafka_message)
+
+      expect(built).to have_attributes(
+        subscription_id: subscription_id,
+        company_uuid: company_uuid
+      )
+    end
+  end
+
+  def build_kafka_message(overrides = {}) # rubocop:disable Metrics/MethodLength
+    attributes = {
+      topic: "some_topic",
+      partition: 3,
+      payload: JSON.dump(
+        id: message_id,
+        subscription_id: subscription_id,
+        company_uuid: company_uuid,
+        event_json: JSON.dump(
+          hello: "world"
+        )
+      ),
+      key: JSON.dump(
+        schema: {},
+        payload: {
+          partition_key: partition_key
+        }
+      ),
+      offset: 13,
+      timestamp: Time.zone.now,
+      headers: {key: "value"}
+    }.merge(overrides)
+
+    instance_double("Rdkafka::Consumer::Message", attributes)
+  end
+end