nulogy_message_bus_consumer 2.0.0 → 3.0.0

data/spec/integration/nulogy_message_bus_consumer/auditor_spec.rb

@@ -4,7 +4,8 @@ RSpec.describe "Auditing pipeline" do # rubocop:disable RSpec/DescribeClass
     NulogyMessageBusConsumer::Config.new(
       consumer_group_id: random_consumer_group,
       bootstrap_servers: test_bootstrap_servers,
-      topic_name: topic.topic_name
+      topic_name: topic.topic_name,
+      security_protocol: "plaintext"
     )
   end
   let(:logger) { spy }
@@ -33,7 +34,7 @@ RSpec.describe "Auditing pipeline" do # rubocop:disable RSpec/DescribeClass
 
       run_audit_pipeline
 
-      expect(logger).to have_not_received(:warn)
+      expect(logger).not_to receive(:warn)
     end
   end
 

data/spec/integration/nulogy_message_bus_consumer/end_to_end_spec.rb

@@ -0,0 +1,54 @@
+RSpec.describe "End to end" do
+  let(:logger) { NulogyMessageBusConsumer::NullLogger.new }
+  let(:topic) { TestTopic.new }
+  let(:tap) { MiddlewareTap.new }
+  let(:message_handler_spy) { double }
+  subject(:pipeline) do
+    pipeline = NulogyMessageBusConsumer.recommended_consumer_pipeline(config: topic.config)
+    pipeline.insert(tap, after: NulogyMessageBusConsumer::Steps::ConnectToMessageBus)
+    pipeline.append(message_handler_spy)
+    pipeline
+  end
+
+  after { topic.close }
+
+  it "receives messages using the full pipeline" do
+    create_topic(topic.topic_name)
+
+    called = false
+    expect(message_handler_spy).to receive(:call) do |message:, **_kargs|
+      expect(message).to have_attributes(event_data: {data: "Some Payload"})
+      called = true
+      :success
+    end
+
+    pipeline_thread = start(pipeline, tap)
+
+    topic.produce_one_message(
+      key: "Some Key",
+      payload: message_payload(data: "Some Payload")
+    )
+
+    NulogyMessageBusConsumer::KafkaUtils.wait_for { called }
+    Thread.kill(pipeline_thread)
+  end
+
+  def start(pipeline, tap)
+    thr = Thread.new { pipeline.invoke }
+    wait_for_partition_assignment(tap)
+    thr
+  end
+
+  def wait_for_partition_assignment(tap)
+    NulogyMessageBusConsumer::KafkaUtils.wait_for { tap.arguments.fetch(:kafka_consumer, nil) }
+    NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(tap.arguments[:kafka_consumer])
+  end
+
+  def message_payload(**payload)
+    JSON.dump(
+      id: SecureRandom.uuid,
+      created_at: 1_000,
+      event_json: JSON.dump(payload)
+    )
+  end
+end

data/spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb

@@ -31,7 +31,7 @@ RSpec.describe NulogyMessageBusConsumer::KafkaUtils do
     it "does not keep the connection open when there are no messages" do
       topic.produce_one_message(payload: "The Only Message")
 
-      enum = utils.every_message_until_none_are_left(topic.consumer)
+      enum = utils.every_message_until_none_are_left(topic.consumer, Kafka::CONSUMER_POLL_TIMEOUT)
 
       expect(enum).to match([
         have_attributes(payload: "The Only Message")

data/spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb

@@ -3,22 +3,22 @@ RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
   let(:consumer) { test_topic.consumer }
   let(:logger) { NulogyMessageBusConsumer::NullLogger.new }
   let(:handler) { spy }
+  let(:subscribe_to_partition) { NulogyMessageBusConsumer::Steps::SubscribeToPartition.new }
   let(:pipeline) do
     NulogyMessageBusConsumer::Pipeline.new([
       NulogyMessageBusConsumer::Steps::ConnectToMessageBus.new(test_topic.config, logger, kafka_consumer: consumer),
-      NulogyMessageBusConsumer::Steps::SeekBeginningOfTopic.new,
-      NulogyMessageBusConsumer::Steps::StreamMessagesUntilNoneAreLeft.new(logger),
-      NulogyMessageBusConsumer::Steps::CommitOnSuccess.new,
+      NulogyMessageBusConsumer::Steps::StreamMessagesUntilNoneAreLeft.new(logger, Kafka::CONSUMER_POLL_TIMEOUT),
+      NulogyMessageBusConsumer::Steps::CommitOnSuccess.new(logger),
       handler
     ])
   end
   let(:deduped_pipeline) do
     NulogyMessageBusConsumer::Pipeline.new([
       NulogyMessageBusConsumer::Steps::ConnectToMessageBus.new(test_topic.config, logger, kafka_consumer: consumer),
-      NulogyMessageBusConsumer::Steps::SeekBeginningOfTopic.new,
-      NulogyMessageBusConsumer::Steps::StreamMessagesUntilNoneAreLeft.new(logger),
+      subscribe_to_partition,
+      NulogyMessageBusConsumer::Steps::StreamMessagesUntilNoneAreLeft.new(logger, Kafka::CONSUMER_POLL_TIMEOUT),
+      NulogyMessageBusConsumer::Steps::CommitOnSuccess.new(logger),
       NulogyMessageBusConsumer::Steps::DeduplicateMessages.new(logger),
-      NulogyMessageBusConsumer::Steps::CommitOnSuccess.new,
       handler
     ])
   end
@@ -27,6 +27,8 @@ RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
 
   context "when successful" do
     it "commits and processes the next message" do
+      create_topic(test_topic.topic_name)
+
       expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:success)
       expect(handler).to receive(:call).with(a_message_with(key: "test 2")).and_return(:success)
       expect(consumer).to receive(:commit).twice
@@ -40,10 +42,11 @@ RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
 
   context "when failing by :failure" do
     it "reprocesses the message" do
-      expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:failure)
-      expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:success)
+      create_topic(test_topic.topic_name)
+
+      expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:failure, :success)
       expect(handler).to receive(:call).with(a_message_with(key: "test 2")).and_return(:success)
-      expect(consumer).to receive(:commit).twice
+      expect(consumer).to receive(:commit).twice.and_call_original
 
       test_topic.produce_one_message(key: "test 1")
       test_topic.produce_one_message(key: "test 2")
@@ -55,36 +58,20 @@ RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
     # This test is more illustrative of how we expect it to work.
     # Specifically, testing the "auto.offset.store" setting for the consumer.
     context "when a partition has a failing message" do
-      let(:handler) { ->(message:, **_) { message.event_data[:type] == "good" ? :success : :failure } }
+      let(:handler) { ->(message:, **_) { message.event_data[:type].start_with?("good") ? :success : :failure } }
 
-      it "processes messages from other partitions without committing offsets for partitions with failing messages" do
-        Kafka.create_topic(test_topic.topic_name)
+      it "it blocks subsequent messages from being processed" do
+        create_topic(test_topic.topic_name)
 
         # Produce message to a single partition. This partition will be blocked by the second message.
-        test_topic.produce_one_message(partition: 1, event_json: {type: "good"}) # success
+        success_id = test_topic.produce_one_message(partition: 1, event_json: {type: "good 1"}) # success
         test_topic.produce_one_message(partition: 1, event_json: {type: "bad"}) # failure
-        blocked_id = test_topic.produce_one_message(partition: 1, event_json: {type: "good"}) # blocked
-
-        consume_from_partition(1) do
-          deduped_pipeline.invoke
-        end
-
-        # produce to another partition
-        success_id = test_topic.produce_one_message(partition: 2, event_json: {type: "good"}) # success
-
-        consume_from_partition(2) do
-          deduped_pipeline.invoke
-        end
+        blocked_id = test_topic.produce_one_message(partition: 1, event_json: {type: "good 2"}) # blocked
 
-        # try consuming from all partitions again -- it will fail on the blocked one again
         deduped_pipeline.invoke
 
-        # Wait for assignment after a reconnect
-        NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(consumer)
-
         lag = consumer.lag(consumer.committed)
-        expect(lag.dig(test_topic.topic_name, 1)).to be >= 1
-        expect(lag.dig(test_topic.topic_name, 2)).to be(0)
+        expect(lag.dig(test_topic.topic_name, 1)).to be 2
         expect(NulogyMessageBusConsumer::ProcessedMessage.exists?(success_id)).to be(true)
         expect(NulogyMessageBusConsumer::ProcessedMessage.exists?(blocked_id)).to be(false)
       end
@@ -93,10 +80,12 @@ RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
 
   context "when failing by exception" do
     it "reprocesses the message" do
+      create_topic(test_topic.topic_name)
+
       expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_raise("intentional error")
       expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:success)
       expect(handler).to receive(:call).with(a_message_with(key: "test 2")).and_return(:success)
-      expect(consumer).to receive(:commit).twice
+      expect(consumer).to receive(:commit).twice.and_call_original
 
       test_topic.produce_one_message(key: "test 1")
       test_topic.produce_one_message(key: "test 2")
@@ -114,18 +103,36 @@ RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
       message: have_attributes(matcher)
     )
   end
+end
 
-  def consume_from_partition(partition_number)
-    original_assignment = consumer.assignment
-    topic_partitions = original_assignment
-      .to_h
-      .transform_values { |values| values.select { |t| t.partition == partition_number } }
-    new_assignment = Rdkafka::Consumer::TopicPartitionList.new(topic_partitions)
-
-    consumer.assign(new_assignment)
+module NulogyMessageBusConsumer
+  module Steps
+    class SubscribeToPartition
+      attr_writer :partition_number
+
+      def call(kafka_consumer:, **_)
+        original_assignment = kafka_consumer.assignment
+        if @partition_number
+          topic_partitions = original_assignment
+            .to_h
+            .transform_values { |values| values.select { |t| t.partition == @partition_number } }
+          new_assignment = Rdkafka::Consumer::TopicPartitionList.new(topic_partitions)
+
+          kafka_consumer.assign(new_assignment)
+          NulogyMessageBusConsumer::KafkaUtils.wait_for do
+            kafka_consumer.assignment == new_assignment
+          end
+        end
 
-    yield
-  ensure
-    consumer.assign(original_assignment)
+        yield
+      ensure
+        if @partition_number
+          kafka_consumer.assign(original_assignment)
+          NulogyMessageBusConsumer::KafkaUtils.wait_for do
+            kafka_consumer.assignment == original_assignment
+          end
+        end
+      end
+    end
   end
 end

data/spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb

@@ -1,46 +1,32 @@
 RSpec.describe NulogyMessageBusConsumer::Steps::ConnectToMessageBus do
-  subject(:pipeline) do
-    pipeline = NulogyMessageBusConsumer.recommended_consumer_pipeline(config: config)
-    pipeline.insert(tap, after: NulogyMessageBusConsumer::Steps::ConnectToMessageBus)
-    pipeline.append(message_handler_spy)
-    pipeline
-  end
-
+  let(:logger) { NulogyMessageBusConsumer::NullLogger.new }
   let(:topic) { TestTopic.new }
-  let(:config) { topic.config }
   let(:tap) { MiddlewareTap.new }
   let(:message_handler_spy) { double }
+  let(:pipeline) do
+    NulogyMessageBusConsumer::Pipeline.new([
+      NulogyMessageBusConsumer::Steps::ConnectToMessageBus.new(topic.config, logger, kafka_consumer: topic.consumer),
+      NulogyMessageBusConsumer::Steps::StreamMessagesUntilNoneAreLeft.new(logger, Kafka::CONSUMER_POLL_TIMEOUT),
+      message_handler_spy
+    ])
+  end
 
   after { topic.close }
 
-  it "receives messages", skip: flakey_in_ci do
-    called = false
+  it "receives messages" do
+    create_topic(topic.topic_name)
+
     expect(message_handler_spy).to receive(:call) do |message:, **_kargs|
       expect(message).to have_attributes(event_data: {data: "Some Payload"})
-      called = true
       :success
     end
 
-    pipeline_thread = start(pipeline, tap)
-
     topic.produce_one_message(
       key: "Some Key",
       payload: message_payload(data: "Some Payload")
     )
 
-    NulogyMessageBusConsumer::KafkaUtils.wait_for { called }
-    Thread.kill(pipeline_thread)
-  end
-
-  def start(pipeline, tap)
-    thr = Thread.new { pipeline.invoke }
-    wait_for_partition_assignment(tap)
-    thr
-  end
-
-  def wait_for_partition_assignment(tap)
-    NulogyMessageBusConsumer::KafkaUtils.wait_for { tap.arguments[:kafka_consumer] }
-    NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(tap.arguments[:kafka_consumer])
+    pipeline.invoke
   end
 
   def message_payload(**payload)

data/spec/integration/test_topic_spec.rb

@@ -5,6 +5,8 @@ RSpec.describe TestTopic do
 
   context "smoke test for specs" do
     it "publishes and receives messages" do
+      create_topic(topic.topic_name)
+
       topic.produce_one_message(
         key: "Some Key",
         payload: "Some Payload"
@@ -18,6 +20,8 @@ RSpec.describe TestTopic do
     end
 
     it "receives returns nil when no messages are received" do
+      create_topic(topic.topic_name)
+
       consumer = topic.consumer
       message = consumer.poll(1)
       expect(message).to be(nil)

data/spec/support/kafka.rb

@@ -3,16 +3,7 @@ require "open3"
 module Kafka
   module_function
 
-  def kafka_config
-    config = {
-      "auto.offset.reset": "beginning",
-      "bootstrap.servers": test_bootstrap_servers,
-      "enable.auto.commit": false,
-      "group.id": random_consumer_group
-    }
-
-    Rdkafka::Config.new(config)
-  end
+  CONSUMER_POLL_TIMEOUT = 1000
 
   def random_topic_name
     "test-topic-#{SecureRandom.uuid}"
@@ -27,16 +18,36 @@ module Kafka
   end
 
   def setup_kafka_producer
-    kafka_config.producer
+    kafka_producer_config.producer
+  end
+
+  def kafka_producer_config
+    config = {"bootstrap.servers": test_bootstrap_servers}
+
+    Rdkafka::Config.new(config)
   end
 
   def setup_kafka_consumer(topic_name)
-    consumer = kafka_config.consumer
+    consumer = kafka_consumer_config.consumer
     consumer.subscribe(topic_name)
     NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(consumer)
     consumer
   end
 
+  # TODO: Why is this duplicated from ConnectToMessageBus
+  def kafka_consumer_config
+    config = {
+      "auto.offset.reset": "beginning",
+      "bootstrap.servers": test_bootstrap_servers,
+      "enable.auto.commit": false,
+      "enable.auto.offset.store": false,
+      "group.id": random_consumer_group,
+      "security.protocol": "plaintext"
+    }
+
+    Rdkafka::Config.new(config)
+  end
+
   def create_topic(topic_name)
     run("kaf topic create #{topic_name} --brokers #{test_bootstrap_servers} --replicas 1 --partitions 3")
   end
@@ -68,7 +79,8 @@ module Kafka
     NulogyMessageBusConsumer::Config.new(
       consumer_group_id: random_consumer_group,
       bootstrap_servers: test_bootstrap_servers,
-      topic_name: topic_name
+      topic_name: topic_name,
+      security_protocol: "plaintext"
     )
   end
 end

data/spec/support/test_topic.rb

@@ -34,7 +34,7 @@ class TestTopic
   end
 
   def consume_one_message
-    consumer.poll(250)
+    consumer.poll(Kafka::CONSUMER_POLL_TIMEOUT)
   end
 
   def config

data/spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb

@@ -1,5 +1,6 @@
 RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
-  subject(:step) { NulogyMessageBusConsumer::Steps::CommitOnSuccess.new }
+  let(:logger) { NulogyMessageBusConsumer::NullLogger.new }
+  subject(:step) { NulogyMessageBusConsumer::Steps::CommitOnSuccess.new(logger) }
 
   let(:kafka_consumer) { spy }
   let(:message) { NulogyMessageBusConsumer::Message.new }
@@ -21,6 +22,9 @@ RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
 
   context "when result is :failure" do
     it "returns the result" do
+      topic_partition_list = {topic_name: %i[partition_1 partition_2]}
+      allow(kafka_consumer).to receive(:assignment).and_return(topic_partition_list)
+
       result = step.call(kafka_consumer: kafka_consumer, message: message) { :failure }
 
       expect(result).to be(:failure)
@@ -33,6 +37,8 @@ RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
 
       expect(kafka_consumer).to receive(:unsubscribe)
       expect(kafka_consumer).to receive(:subscribe).with(:topic_name)
+      expect(kafka_consumer).to receive(:assignment).and_return({})
+      expect(kafka_consumer).to receive(:assignment).and_return(topic_partition_list)
 
       step.call(kafka_consumer: kafka_consumer, message: message) { :failure }
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nulogy_message_bus_consumer
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Nulogy
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-29 00:00:00.000000000 Z
+date: 2024-12-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -56,16 +56,16 @@ dependencies:
   name: bundler-audit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.0.1
+        version: 0.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.7.0.1
+        version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
@@ -84,16 +84,16 @@ dependencies:
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.3
+        version: '1.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.3
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +128,28 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.0.3.5
+        version: 6.0.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.0.3.5
+        version: 6.0.4
+- !ruby/object:Gem::Dependency
+  name: psych
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -309,6 +323,7 @@ files:
 - spec/dummy/public/favicon.ico
 - spec/dummy/tmp/development_secret.txt
 - spec/integration/nulogy_message_bus_consumer/auditor_spec.rb
+- spec/integration/nulogy_message_bus_consumer/end_to_end_spec.rb
 - spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb
 - spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
 - spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb
@@ -349,7 +364,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: Code for accessing the Nulogy Message Bus
@@ -407,6 +422,7 @@ test_files:
 - spec/dummy/public/favicon.ico
 - spec/dummy/tmp/development_secret.txt
 - spec/integration/nulogy_message_bus_consumer/auditor_spec.rb
+- spec/integration/nulogy_message_bus_consumer/end_to_end_spec.rb
 - spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb
 - spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
 - spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb