RubyGems - nulogy_message_bus_consumer - Versions diffs - 0.3.0 → 0.5.0 - Mend

nulogy_message_bus_consumer 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (99) hide show

data/spec/dummy/public/404.html ADDED Viewed

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>The page you were looking for doesn't exist (404)</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <style>
+  .rails-default-error-page {
+    background-color: #EFEFEF;
+    color: #2E2F30;
+    text-align: center;
+    font-family: arial, sans-serif;
+    margin: 0;
+  }
+  .rails-default-error-page div.dialog {
+    width: 95%;
+    max-width: 33em;
+    margin: 4em auto 0;
+  }
+  .rails-default-error-page div.dialog > div {
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #BBB;
+    border-top: #B00100 solid 4px;
+    border-top-left-radius: 9px;
+    border-top-right-radius: 9px;
+    background-color: white;
+    padding: 7px 12% 0;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  .rails-default-error-page h1 {
+    font-size: 100%;
+    color: #730E15;
+    line-height: 1.5em;
+  }
+  .rails-default-error-page div.dialog > p {
+    margin: 0 0 1em;
+    padding: 1em;
+    background-color: #F7F7F7;
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #999;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-top-color: #DADADA;
+    color: #666;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  </style>
+</head>
+<body class="rails-default-error-page">
+  <!-- This file lives in public/404.html -->
+  <div class="dialog">
+    <div>
+      <h1>The page you were looking for doesn't exist.</h1>
+      <p>You may have mistyped the address or the page may have moved.</p>
+    </div>
+    <p>If you are the application owner check the logs for more information.</p>
+  </div>
+</body>
+</html>

data/spec/dummy/public/422.html ADDED Viewed

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>The change you wanted was rejected (422)</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <style>
+  .rails-default-error-page {
+    background-color: #EFEFEF;
+    color: #2E2F30;
+    text-align: center;
+    font-family: arial, sans-serif;
+    margin: 0;
+  }
+  .rails-default-error-page div.dialog {
+    width: 95%;
+    max-width: 33em;
+    margin: 4em auto 0;
+  }
+  .rails-default-error-page div.dialog > div {
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #BBB;
+    border-top: #B00100 solid 4px;
+    border-top-left-radius: 9px;
+    border-top-right-radius: 9px;
+    background-color: white;
+    padding: 7px 12% 0;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  .rails-default-error-page h1 {
+    font-size: 100%;
+    color: #730E15;
+    line-height: 1.5em;
+  }
+  .rails-default-error-page div.dialog > p {
+    margin: 0 0 1em;
+    padding: 1em;
+    background-color: #F7F7F7;
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #999;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-top-color: #DADADA;
+    color: #666;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  </style>
+</head>
+<body class="rails-default-error-page">
+  <!-- This file lives in public/422.html -->
+  <div class="dialog">
+    <div>
+      <h1>The change you wanted was rejected.</h1>
+      <p>Maybe you tried to change something you didn't have access to.</p>
+    </div>
+    <p>If you are the application owner check the logs for more information.</p>
+  </div>
+</body>
+</html>

data/spec/dummy/public/500.html ADDED Viewed

@@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>We're sorry, but something went wrong (500)</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <style>
+  .rails-default-error-page {
+    background-color: #EFEFEF;
+    color: #2E2F30;
+    text-align: center;
+    font-family: arial, sans-serif;
+    margin: 0;
+  }
+  .rails-default-error-page div.dialog {
+    width: 95%;
+    max-width: 33em;
+    margin: 4em auto 0;
+  }
+  .rails-default-error-page div.dialog > div {
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #BBB;
+    border-top: #B00100 solid 4px;
+    border-top-left-radius: 9px;
+    border-top-right-radius: 9px;
+    background-color: white;
+    padding: 7px 12% 0;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  .rails-default-error-page h1 {
+    font-size: 100%;
+    color: #730E15;
+    line-height: 1.5em;
+  }
+  .rails-default-error-page div.dialog > p {
+    margin: 0 0 1em;
+    padding: 1em;
+    background-color: #F7F7F7;
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #999;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-top-color: #DADADA;
+    color: #666;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  </style>
+</head>
+<body class="rails-default-error-page">
+  <!-- This file lives in public/500.html -->
+  <div class="dialog">
+    <div>
+      <h1>We're sorry, but something went wrong.</h1>
+    </div>
+    <p>If you are the application owner check the logs for more information.</p>
+  </div>
+</body>
+</html>

data/spec/dummy/public/apple-touch-icon-precomposed.png ADDED Viewed

File without changes

data/spec/dummy/public/apple-touch-icon.png ADDED Viewed

File without changes

data/spec/dummy/public/favicon.ico ADDED Viewed

File without changes

data/spec/dummy/tmp/development_secret.txt ADDED Viewed

	@@ -0,0 +1 @@
1	+ f68518d3020fd6430af89b8a7e2261ff0bcc4e6325685baca3256f5f99bf51e8003353fa85ce95984de17cf3a19223bcd8d2cfa7df0fc07887ef937919493208

data/spec/integration/nulogy_message_bus_consumer/auditor_spec.rb ADDED Viewed

@@ -0,0 +1,59 @@
+RSpec.describe "Auditing pipeline" do # rubocop:disable RSpec/DescribeClass
+  let(:topic) { TestTopic.new }
+  let(:config) do
+    NulogyMessageBusConsumer::Config.new(
+      consumer_group_id: random_consumer_group,
+      bootstrap_servers: test_bootstrap_servers,
+      topic_name: topic.topic_name
+    )
+  end
+  let(:logger) { spy }
+  after { topic.close }
+  context "when some messages have not been processed" do
+    it "logs the list of unprocessed messages" do
+      produce_message(id: uuid(1))
+      process_message(id: uuid(1))
+      produce_message(id: uuid(2))
+      expect(logger).to receive(:warn).with(include_json(
+        event: "unprocessed_message",
+        kafka_message: {id: uuid(2)}
+      ))
+      run_audit_pipeline
+    end
+  end
+  context "when all messages have been processed" do
+    it "does not log anything" do
+      produce_message(id: uuid(1))
+      process_message(id: uuid(1))
+      run_audit_pipeline
+      expect(logger).to have_not_received(:warn)
+    end
+  end
+  def run_audit_pipeline
+    NulogyMessageBusConsumer
+      .consumer_audit_pipeline(config: config, logger: logger)
+      .invoke
+  end
+  def produce_message(id:)
+    topic.produce_one_message(
+      payload: JSON.dump(id: id)
+    )
+  end
+  def process_message(id:)
+    NulogyMessageBusConsumer::ProcessedMessage.create!(id: id)
+  end
+  def uuid(id)
+    format("00000000-0000-0000-0000-%012d", id)
+  end
+end

data/spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb ADDED Viewed

@@ -0,0 +1,41 @@
+RSpec.describe NulogyMessageBusConsumer::KafkaUtils do
+  subject(:utils) { NulogyMessageBusConsumer::KafkaUtils }
+  let(:topic) { TestTopic.new }
+  after { topic.close }
+  describe "#seek_beginning" do
+    it "updates the consumer offset to the beginning of the topic" do
+      topic.produce_one_message(payload: "First Message")
+      expect(topic.consume_one_message).to have_attributes(payload: "First Message")
+      expect(topic.consume_one_message).to eq(nil)
+      utils.seek_beginning(topic.consumer)
+      expect(topic.consume_one_message).to have_attributes(payload: "First Message")
+    end
+  end
+  describe "#seek_end" do
+    it "updates the consumer offset to the end of the topic" do
+      topic.produce_one_message(payload: "First Message")
+      utils.seek_ending(topic.consumer)
+      expect(topic.consume_one_message).to eq(nil)
+    end
+  end
+  describe "#every_message_until_none_are_left" do
+    it "does not keep the connection open when there are no messages" do
+      topic.produce_one_message(payload: "The Only Message")
+      enum = utils.every_message_until_none_are_left(topic.consumer)
+      expect(enum).to match([
+        have_attributes(payload: "The Only Message")
+      ])
+    end
+  end
+end

data/spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb ADDED Viewed

@@ -0,0 +1,131 @@
+RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
+  let(:test_topic) { TestTopic.new }
+  let(:consumer) { test_topic.consumer }
+  let(:logger) { NulogyMessageBusConsumer::NullLogger.new }
+  let(:handler) { spy }
+  let(:pipeline) do
+    NulogyMessageBusConsumer::Pipeline.new([
+      NulogyMessageBusConsumer::Steps::ConnectToMessageBus.new(test_topic.config, logger, kafka_consumer: consumer),
+      NulogyMessageBusConsumer::Steps::SeekBeginningOfTopic.new,
+      NulogyMessageBusConsumer::Steps::StreamMessagesUntilNoneAreLeft.new(logger),
+      NulogyMessageBusConsumer::Steps::CommitOnSuccess.new,
+      handler
+    ])
+  end
+  let(:deduped_pipeline) do
+    NulogyMessageBusConsumer::Pipeline.new([
+      NulogyMessageBusConsumer::Steps::ConnectToMessageBus.new(test_topic.config, logger, kafka_consumer: consumer),
+      NulogyMessageBusConsumer::Steps::SeekBeginningOfTopic.new,
+      NulogyMessageBusConsumer::Steps::StreamMessagesUntilNoneAreLeft.new(logger),
+      NulogyMessageBusConsumer::Steps::DeduplicateMessages.new(logger),
+      NulogyMessageBusConsumer::Steps::CommitOnSuccess.new,
+      handler
+    ])
+  end
+  after { test_topic.close }
+  context "when successful" do
+    it "commits and processes the next message" do
+      expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:success)
+      expect(handler).to receive(:call).with(a_message_with(key: "test 2")).and_return(:success)
+      expect(consumer).to receive(:commit).twice
+      test_topic.produce_one_message(key: "test 1")
+      test_topic.produce_one_message(key: "test 2")
+      pipeline.invoke
+    end
+  end
+  context "when failing by :failure" do
+    it "reprocesses the message" do
+      expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:failure)
+      expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:success)
+      expect(handler).to receive(:call).with(a_message_with(key: "test 2")).and_return(:success)
+      expect(consumer).to receive(:commit).twice
+      test_topic.produce_one_message(key: "test 1")
+      test_topic.produce_one_message(key: "test 2")
+      pipeline.invoke
+      pipeline.invoke
+    end
+    # This test is more illustrative of how we expect it to work.
+    # Specifically, testing the "auto.offset.store" setting for the consumer.
+    context "when a partition has a failing message" do
+      let(:handler) { ->(message:, **_) { message.event_data[:type] == "good" ? :success : :failure } }
+      it "processes messages from other partitions without committing offsets for partitions with failing messages" do
+        Kafka.create_topic(test_topic.topic_name)
+        # Produce message to a single partition. This partition will be blocked by the second message.
+        test_topic.produce_one_message(partition: 1, event_json: {type: "good"}) # success
+        test_topic.produce_one_message(partition: 1, event_json: {type: "bad"}) # failure
+        blocked_id = test_topic.produce_one_message(partition: 1, event_json: {type: "good"}) # blocked
+        consume_from_partition(1) do
+          deduped_pipeline.invoke
+        end
+        # produce to another partition
+        success_id = test_topic.produce_one_message(partition: 2, event_json: {type: "good"}) # success
+        consume_from_partition(2) do
+          deduped_pipeline.invoke
+        end
+        # try consuming from all partitions again -- it will fail on the blocked one again
+        deduped_pipeline.invoke
+        # Wait for assignment after a reconnect
+        NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(consumer)
+        lag = consumer.lag(consumer.committed)
+        expect(lag.dig(test_topic.topic_name, 1)).to be >= 1
+        expect(lag.dig(test_topic.topic_name, 2)).to be(0)
+        expect(NulogyMessageBusConsumer::ProcessedMessage.exists?(success_id)).to be(true)
+        expect(NulogyMessageBusConsumer::ProcessedMessage.exists?(blocked_id)).to be(false)
+      end
+    end
+  end
+  context "when failing by exception" do
+    it "reprocesses the message" do
+      expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_raise("intentional error")
+      expect(handler).to receive(:call).with(a_message_with(key: "test 1")).and_return(:success)
+      expect(handler).to receive(:call).with(a_message_with(key: "test 2")).and_return(:success)
+      expect(consumer).to receive(:commit).twice
+      test_topic.produce_one_message(key: "test 1")
+      test_topic.produce_one_message(key: "test 2")
+      expect {
+        pipeline.invoke
+      }.to raise_error("intentional error")
+      pipeline.invoke
+    end
+  end
+  def a_message_with(matcher)
+    hash_including(
+      message: have_attributes(matcher)
+    )
+  end
+  def consume_from_partition(partition_number)
+    original_assignment = consumer.assignment
+    topic_partitions = original_assignment
+      .to_h
+      .transform_values { |values| values.select { |t| t.partition == partition_number } }
+    new_assignment = Rdkafka::Consumer::TopicPartitionList.new(topic_partitions)
+    consumer.assign(new_assignment)
+    yield
+  ensure
+    consumer.assign(original_assignment)
+  end
+end