nulogy_message_bus_consumer 1.0.0.alpha → 1.0.0

data/spec/unit/nulogy_message_bus_consumer/pipeline_spec.rb

@@ -0,0 +1,49 @@
+RSpec.describe NulogyMessageBusConsumer::Pipeline do
+  let(:handler) { spy }
+
+  describe "#invoke_pipeline" do
+    it "raises if handler is not provided" do
+      expect { invoke_pipeline }.to raise_error("Handlers are the end of the line. Do not use yield.")
+    end
+
+    it "calls the handler with message" do
+      message = NulogyMessageBusConsumer::Message.new
+
+      expect(handler).to receive(:call).with(message: message)
+
+      invoke_pipeline(
+        ->(**_, &block) { block.yield(message: message) },
+        handler
+      )
+    end
+
+    it "passes information along" do
+      message = NulogyMessageBusConsumer::Message.new
+
+      expect(handler).to receive(:call).with(message: message)
+
+      invoke_pipeline(
+        ->(**_, &block) { block.yield(message: message) },
+        ->(**_, &block) { block.yield },
+        handler
+      )
+    end
+
+    it "raises when overwriting an existing key" do
+      message = NulogyMessageBusConsumer::Message.new
+      other_message = NulogyMessageBusConsumer::Message.new
+
+      expect {
+        invoke_pipeline(
+          ->(**_, &block) { block.yield(message: message) },
+          ->(**_, &block) { block.yield(message: other_message) },
+          handler
+        )
+      }.to raise_error("Cannot override existing argument(s): message")
+    end
+  end
+
+  def invoke_pipeline(*steps)
+    NulogyMessageBusConsumer::Pipeline.new(steps).invoke
+  end
+end

data/spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb

@@ -0,0 +1,58 @@
+RSpec.describe NulogyMessageBusConsumer::Steps::CommitOnSuccess do
+  subject(:step) { NulogyMessageBusConsumer::Steps::CommitOnSuccess.new }
+
+  let(:kafka_consumer) { spy }
+  let(:message) { NulogyMessageBusConsumer::Message.new }
+
+  context "when result is :success" do
+    it "returns the result" do
+      result = step.call(kafka_consumer: kafka_consumer, message: message) { :success }
+
+      expect(result).to be(:success)
+    end
+
+    it "commits the message offset" do
+      expect(kafka_consumer).to receive(:store_offset).with(message)
+      expect(kafka_consumer).to receive(:commit)
+
+      step.call(kafka_consumer: kafka_consumer, message: message) { :success }
+    end
+  end
+
+  context "when result is :failure" do
+    it "returns the result" do
+      result = step.call(kafka_consumer: kafka_consumer, message: message) { :failure }
+
+      expect(result).to be(:failure)
+    end
+
+    it "reconnects to Kafka when failure" do
+      # When subscribing, we subscribe to a topic (i.e. the keys)
+      topic_partition_list = {topic_name: %i[partition_1 partition_2]}
+      allow(kafka_consumer).to receive(:subscription).and_return(topic_partition_list)
+
+      expect(kafka_consumer).to receive(:unsubscribe)
+      expect(kafka_consumer).to receive(:subscribe).with(:topic_name)
+
+      step.call(kafka_consumer: kafka_consumer, message: message) { :failure }
+    end
+  end
+
+  context "when result is something else" do
+    it "raises" do
+      expect {
+        step.call(kafka_consumer: kafka_consumer, message: message) { :bogus_value }
+      }.to raise_error("'bogus_value' is not a valid processing outcome. Must be :success or :failure")
+    end
+  end
+
+  context "when downstream steps raise" do
+    it "does not commit" do
+      expect(kafka_consumer).not_to receive(:commit)
+
+      expect {
+        step.call(kafka_consumer: kafka_consumer, message: message) { raise "oopsie" }
+      }.to raise_error("oopsie")
+    end
+  end
+end

data/spec/unit/nulogy_message_bus_consumer/steps/deduplicate_messages_spec.rb

@@ -0,0 +1,56 @@
+RSpec.describe NulogyMessageBusConsumer::Steps::DeduplicateMessages do
+  subject(:deduplicate_messages) { NulogyMessageBusConsumer::Steps::DeduplicateMessages.new(logger) }
+
+  let(:logger) { spy }
+  let(:id) { SecureRandom.uuid }
+  let(:message) { NulogyMessageBusConsumer::Message.new(id: id) }
+
+  context "when a duplicate message is consumed" do
+    it "logs a duplicate was detected" do
+      expect(logger).to receive(:warn).with(include_json(
+        event: "duplicate_message_detected",
+        kafka_message_id: id
+      ))
+
+      deduplicate_messages.call(message: message) { :success }
+      deduplicate_messages.call(message: message) { raise "Not evaluated" }
+    end
+
+    it "returns :success" do
+      deduplicate_messages.call(message: message) { :success }
+
+      result = deduplicate_messages.call(message: message) { raise "Not evaluated" }
+
+      expect(result).to be(:success)
+    end
+  end
+
+  context "when a duplicate message is processed at the same time" do
+    it "logs a duplicate was detected" do
+      expect(logger).to receive(:warn).with(include_json(
+        event: "duplicate_message_detected",
+        kafka_message_id: id
+      ))
+
+      deduplicate_messages.call(message: message) do
+        NulogyMessageBusConsumer::ProcessedMessage.create!(id: id)
+        :success
+      end
+    end
+
+    it "returns :success" do
+      result = deduplicate_messages.call(message: message) {
+        NulogyMessageBusConsumer::ProcessedMessage.create!(id: id)
+        :success
+      }
+
+      expect(result).to be(:success)
+    end
+  end
+
+  it "records message was processed" do
+    deduplicate_messages.call(message: message) { :success }
+
+    expect(NulogyMessageBusConsumer::ProcessedMessage.exists?(id: id)).to be(true)
+  end
+end

data/spec/unit/nulogy_message_bus_consumer/steps/log_messages_spec.rb

@@ -0,0 +1,70 @@
+RSpec.describe NulogyMessageBusConsumer::Steps::LogMessages do
+  let(:logger) { spy }
+
+  it "logs the message was received" do
+    log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
+
+    expect(logger).to receive(:info).with(include_json(
+      event: "message_received",
+      kafka_message_id: "id",
+      message: "Received id"
+    ))
+
+    log_message.call(message: message) {}
+  end
+
+  it "logs the message was processed" do
+    log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
+
+    expect(logger).to receive(:info).with(include_json(
+      event: "message_processed",
+      kafka_message_id: "id",
+      message: match(/^Processed id/)
+    ))
+
+    log_message.call(message: message) {}
+  end
+
+  it "logs the message's time to processed" do
+    # clock always gives milliseconds since epoch
+    clock = instance_double(NulogyMessageBusConsumer::Clock, ms: 1_000_001_234)
+    log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger, clock: clock)
+
+    expect(logger).to receive(:info).with(include_json(
+      event: "message_processed",
+      kafka_message_id: "id",
+      time_to_processed: 1234
+    ))
+
+    # debezium converts time with zone to nanoseconds since epoch
+    log_message.call(message: message(created_at: 1_000_000_000_000)) {}
+  end
+
+  it "logs the message's result" do
+    log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
+
+    expect(logger).to receive(:info).with(include_json(
+      event: "message_processed",
+      result: "success"
+    ))
+
+    log_message.call(message: message) { :success }
+
+    expect(logger).to receive(:info).with(include_json(
+      event: "message_processed",
+      result: "failure"
+    ))
+
+    log_message.call(message: message) { :failure }
+  end
+
+  it "returns a result" do
+    log_message = NulogyMessageBusConsumer::Steps::LogMessages.new(logger)
+
+    expect(log_message.call(message: message) { :return_value }).to eq(:return_value)
+  end
+
+  def message(id: "id", created_at: 1_000, **attrs)
+    NulogyMessageBusConsumer::Message.new(id: id, created_at: created_at, **attrs)
+  end
+end

data/spec/unit/nulogy_message_bus_consumer/steps/stream_messages_spec.rb

@@ -0,0 +1,35 @@
+RSpec.describe NulogyMessageBusConsumer::Steps::StreamMessages do
+  subject(:step) { NulogyMessageBusConsumer::Steps::StreamMessages.new(logger) }
+
+  let(:config) { NulogyMessageBusConsumer::Config.new }
+  let(:logger) { spy }
+
+  it "logs errors while streaming from kafka" do
+    kafka_consumer = instance_double("Rdkafka::Consumer")
+    allow(kafka_consumer).to receive(:subscribe)
+    allow(kafka_consumer).to receive(:unsubscribe)
+    allow(kafka_consumer).to receive(:each).and_raise(StandardError, "streaming failed")
+
+    expect(logger).to receive(:error).with(include_json(
+      event: "message_processing_errored",
+      class: "StandardError",
+      message: "streaming failed"
+    ))
+
+    expect {
+      step.call(kafka_consumer: kafka_consumer)
+    }.to raise_error(StandardError)
+  end
+
+  context "when handler returns failure" do
+    it "does not commit the message in kafka" do
+      kafka_consumer = spy
+      allow(kafka_consumer).to receive(:each).and_yield(anything)
+      allow(NulogyMessageBusConsumer::Message).to receive(:from_kafka).and_return(NulogyMessageBusConsumer::Message.new)
+
+      expect(kafka_consumer).not_to receive(:commit)
+
+      step.call(kafka_consumer: kafka_consumer) { :failure }
+    end
+  end
+end

data/spec/unit/nulogy_message_bus_consumer/tasks/calculator_spec.rb

@@ -0,0 +1,67 @@
+module NulogyMessageBusConsumer
+  module Tasks
+    RSpec.describe LogConsumerLag::Calculator do
+      subject(:calculator) { LogConsumerLag::Calculator }
+
+      describe ".add_max_lag" do
+        it "adds max lag per partition" do
+          lag_per_topic = {
+            topic_1: {
+              partition_1: 100,
+              partition_2: 900,
+              partition_3: 100
+            },
+            topic_2: {
+              partition_1: 100,
+              partition_2: 200,
+              partition_3: 100
+            }
+          }
+
+          lag_with_max = calculator.add_max_lag(lag_per_topic)
+
+          expect(lag_with_max).to match(
+            _max: 900,
+            topic_1: {
+              _max: 900,
+              partition_1: 100,
+              partition_2: 900,
+              partition_3: 100
+            },
+            topic_2: {
+              _max: 200,
+              partition_1: 100,
+              partition_2: 200,
+              partition_3: 100
+            }
+          )
+        end
+
+        it "sets the max to 0 when no partitions are subscribed to" do
+          lag_per_topic = {
+            topic_1: {}
+          }
+
+          lag_with_max = calculator.add_max_lag(lag_per_topic)
+
+          expect(lag_with_max).to match(
+            _max: 0,
+            topic_1: {
+              _max: 0
+            }
+          )
+        end
+
+        it "sets the max to 0 when no topics are subscribed to" do
+          lag_per_topic = {}
+
+          lag_with_max = calculator.add_max_lag(lag_per_topic)
+
+          expect(lag_with_max).to match(
+            _max: 0
+          )
+        end
+      end
+    end
+  end
+end

data/spec/unit/nulogy_message_bus_consumer_spec.rb

@@ -0,0 +1,30 @@
+RSpec.describe NulogyMessageBusConsumer do
+  describe "#configure" do
+    before do
+      config = NulogyMessageBusConsumer.config
+      config.instance_variable_names.each do |variable_name|
+        config.remove_instance_variable(variable_name)
+      end
+    end
+
+    it "allows configuration to be provided as arguments" do
+      NulogyMessageBusConsumer.configure(
+        bootstrap_servers: "some servers"
+      )
+
+      expect(NulogyMessageBusConsumer.config).to have_attributes(
+        bootstrap_servers: "some servers"
+      )
+    end
+
+    it "allows configuration to be provided as a block" do
+      NulogyMessageBusConsumer.configure do |config|
+        config.bootstrap_servers = "some servers"
+      end
+
+      expect(NulogyMessageBusConsumer.config).to have_attributes(
+        bootstrap_servers: "some servers"
+      )
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nulogy_message_bus_consumer
 version: !ruby/object:Gem::Version
-  version: 1.0.0.alpha
+  version: 1.0.0
 platform: ruby
 authors:
 - Nulogy
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-13 00:00:00.000000000 Z
+date: 2021-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -154,16 +154,16 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.9.0
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: rspec-json_expectations
   requirement: !ruby/object:Gem::Requirement
@@ -261,14 +261,92 @@ files:
 - lib/nulogy_message_bus_consumer/steps/commit_on_success.rb
 - lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb
 - lib/nulogy_message_bus_consumer/steps/deduplicate_messages.rb
-- lib/nulogy_message_bus_consumer/steps/log_consumer_lag.rb
 - lib/nulogy_message_bus_consumer/steps/log_messages.rb
 - lib/nulogy_message_bus_consumer/steps/seek_beginning_of_topic.rb
 - lib/nulogy_message_bus_consumer/steps/stream_messages.rb
 - lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb
-- lib/nulogy_message_bus_consumer/steps/supervise_consumer_lag.rb
+- lib/nulogy_message_bus_consumer/steps/timed_task.rb
+- lib/nulogy_message_bus_consumer/tasks/log_consumer_lag.rb
+- lib/nulogy_message_bus_consumer/tasks/prune_processed_messages.rb
+- lib/nulogy_message_bus_consumer/tasks/supervise_consumer_lag.rb
 - lib/nulogy_message_bus_consumer/version.rb
 - lib/tasks/engine/message_bus_consumer.rake
+- spec/dummy/Rakefile
+- spec/dummy/app/assets/config/manifest.js
+- spec/dummy/app/assets/stylesheets/application.css
+- spec/dummy/app/channels/application_cable/channel.rb
+- spec/dummy/app/channels/application_cable/connection.rb
+- spec/dummy/app/controllers/application_controller.rb
+- spec/dummy/app/helpers/application_helper.rb
+- spec/dummy/app/javascript/packs/application.js
+- spec/dummy/app/jobs/application_job.rb
+- spec/dummy/app/mailers/application_mailer.rb
+- spec/dummy/app/models/application_record.rb
+- spec/dummy/app/views/layouts/application.html.erb
+- spec/dummy/app/views/layouts/mailer.html.erb
+- spec/dummy/app/views/layouts/mailer.text.erb
+- spec/dummy/bin/rails
+- spec/dummy/bin/rake
+- spec/dummy/bin/setup
+- spec/dummy/config.ru
+- spec/dummy/config/application.rb
+- spec/dummy/config/boot.rb
+- spec/dummy/config/cable.yml
+- spec/dummy/config/credentials/message-bus-us-east-1.key
+- spec/dummy/config/credentials/message-bus-us-east-1.yml.enc
+- spec/dummy/config/database.yml
+- spec/dummy/config/environment.rb
+- spec/dummy/config/environments/development.rb
+- spec/dummy/config/environments/production.rb
+- spec/dummy/config/environments/test.rb
+- spec/dummy/config/initializers/application_controller_renderer.rb
+- spec/dummy/config/initializers/assets.rb
+- spec/dummy/config/initializers/backtrace_silencers.rb
+- spec/dummy/config/initializers/content_security_policy.rb
+- spec/dummy/config/initializers/cookies_serializer.rb
+- spec/dummy/config/initializers/filter_parameter_logging.rb
+- spec/dummy/config/initializers/inflections.rb
+- spec/dummy/config/initializers/message_bus_consumer.rb
+- spec/dummy/config/initializers/mime_types.rb
+- spec/dummy/config/initializers/wrap_parameters.rb
+- spec/dummy/config/locales/en.yml
+- spec/dummy/config/puma.rb
+- spec/dummy/config/routes.rb
+- spec/dummy/config/spring.rb
+- spec/dummy/config/storage.yml
+- spec/dummy/db/schema.rb
+- spec/dummy/log/development.log
+- spec/dummy/log/production.log
+- spec/dummy/log/test.log
+- spec/dummy/public/404.html
+- spec/dummy/public/422.html
+- spec/dummy/public/500.html
+- spec/dummy/public/apple-touch-icon-precomposed.png
+- spec/dummy/public/apple-touch-icon.png
+- spec/dummy/public/favicon.ico
+- spec/dummy/tmp/development_secret.txt
+- spec/integration/nulogy_message_bus_consumer/auditor_spec.rb
+- spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb
+- spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
+- spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb
+- spec/integration/nulogy_message_bus_consumer/tasks/prune_processed_messages_spec.rb
+- spec/integration/nulogy_message_bus_consumer/tasks/supervise_consumer_lag_spec.rb
+- spec/integration/test_topic_spec.rb
+- spec/spec_helper.rb
+- spec/support/kafka.rb
+- spec/support/middleware_tap.rb
+- spec/support/skip.rb
+- spec/support/test_topic.rb
+- spec/unit/nulogy_message_bus_consumer/config_spec.rb
+- spec/unit/nulogy_message_bus_consumer/lag_tracker.rb
+- spec/unit/nulogy_message_bus_consumer/message_spec.rb
+- spec/unit/nulogy_message_bus_consumer/pipeline_spec.rb
+- spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
+- spec/unit/nulogy_message_bus_consumer/steps/deduplicate_messages_spec.rb
+- spec/unit/nulogy_message_bus_consumer/steps/log_messages_spec.rb
+- spec/unit/nulogy_message_bus_consumer/steps/stream_messages_spec.rb
+- spec/unit/nulogy_message_bus_consumer/tasks/calculator_spec.rb
+- spec/unit/nulogy_message_bus_consumer_spec.rb
 homepage: https://github.com/nulogy/message-bus/tree/master/gems/nulogy_message_bus_consumer
 licenses: []
 metadata:
@@ -281,15 +359,91 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.15
 signing_key:
 specification_version: 4
 summary: Code for accessing the Nulogy Message Bus
-test_files: []
+test_files:
+- spec/dummy/Rakefile
+- spec/dummy/app/assets/config/manifest.js
+- spec/dummy/app/assets/stylesheets/application.css
+- spec/dummy/app/channels/application_cable/channel.rb
+- spec/dummy/app/channels/application_cable/connection.rb
+- spec/dummy/app/controllers/application_controller.rb
+- spec/dummy/app/helpers/application_helper.rb
+- spec/dummy/app/javascript/packs/application.js
+- spec/dummy/app/jobs/application_job.rb
+- spec/dummy/app/mailers/application_mailer.rb
+- spec/dummy/app/models/application_record.rb
+- spec/dummy/app/views/layouts/application.html.erb
+- spec/dummy/app/views/layouts/mailer.html.erb
+- spec/dummy/app/views/layouts/mailer.text.erb
+- spec/dummy/bin/rails
+- spec/dummy/bin/rake
+- spec/dummy/bin/setup
+- spec/dummy/config/application.rb
+- spec/dummy/config/boot.rb
+- spec/dummy/config/cable.yml
+- spec/dummy/config/credentials/message-bus-us-east-1.key
+- spec/dummy/config/credentials/message-bus-us-east-1.yml.enc
+- spec/dummy/config/database.yml
+- spec/dummy/config/environment.rb
+- spec/dummy/config/environments/development.rb
+- spec/dummy/config/environments/production.rb
+- spec/dummy/config/environments/test.rb
+- spec/dummy/config/initializers/application_controller_renderer.rb
+- spec/dummy/config/initializers/assets.rb
+- spec/dummy/config/initializers/backtrace_silencers.rb
+- spec/dummy/config/initializers/content_security_policy.rb
+- spec/dummy/config/initializers/cookies_serializer.rb
+- spec/dummy/config/initializers/filter_parameter_logging.rb
+- spec/dummy/config/initializers/inflections.rb
+- spec/dummy/config/initializers/message_bus_consumer.rb
+- spec/dummy/config/initializers/mime_types.rb
+- spec/dummy/config/initializers/wrap_parameters.rb
+- spec/dummy/config/locales/en.yml
+- spec/dummy/config/puma.rb
+- spec/dummy/config/routes.rb
+- spec/dummy/config/spring.rb
+- spec/dummy/config/storage.yml
+- spec/dummy/config.ru
+- spec/dummy/db/schema.rb
+- spec/dummy/log/development.log
+- spec/dummy/log/production.log
+- spec/dummy/log/test.log
+- spec/dummy/public/404.html
+- spec/dummy/public/422.html
+- spec/dummy/public/500.html
+- spec/dummy/public/apple-touch-icon-precomposed.png
+- spec/dummy/public/apple-touch-icon.png
+- spec/dummy/public/favicon.ico
+- spec/dummy/tmp/development_secret.txt
+- spec/integration/nulogy_message_bus_consumer/auditor_spec.rb
+- spec/integration/nulogy_message_bus_consumer/kafka_utils_spec.rb
+- spec/integration/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
+- spec/integration/nulogy_message_bus_consumer/steps/connect_to_message_bus_spec.rb
+- spec/integration/nulogy_message_bus_consumer/tasks/prune_processed_messages_spec.rb
+- spec/integration/nulogy_message_bus_consumer/tasks/supervise_consumer_lag_spec.rb
+- spec/integration/test_topic_spec.rb
+- spec/spec_helper.rb
+- spec/support/kafka.rb
+- spec/support/middleware_tap.rb
+- spec/support/skip.rb
+- spec/support/test_topic.rb
+- spec/unit/nulogy_message_bus_consumer/config_spec.rb
+- spec/unit/nulogy_message_bus_consumer/lag_tracker.rb
+- spec/unit/nulogy_message_bus_consumer/message_spec.rb
+- spec/unit/nulogy_message_bus_consumer/pipeline_spec.rb
+- spec/unit/nulogy_message_bus_consumer/steps/commit_on_success_spec.rb
+- spec/unit/nulogy_message_bus_consumer/steps/deduplicate_messages_spec.rb
+- spec/unit/nulogy_message_bus_consumer/steps/log_messages_spec.rb
+- spec/unit/nulogy_message_bus_consumer/steps/stream_messages_spec.rb
+- spec/unit/nulogy_message_bus_consumer/tasks/calculator_spec.rb
+- spec/unit/nulogy_message_bus_consumer_spec.rb