nulogy_message_bus_consumer 0.3.0 → 1.0.0.alpha

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2faa5c725b717d217b37a0c9019ecd3f3ae9d40885a2e3b1ce898e626bbb2d3c
-  data.tar.gz: 865137198c57cfd60bdea31c482a66240a284c34d4f1149033160c28fb2a6cef
+  metadata.gz: 1d17557a978d8762a83afcb1a8027296854d516cde57540243d48c80cc46c012
+  data.tar.gz: 02c45bfab242f35a0143d090a08387f80db0030854a28b161792cb09efded53c
 SHA512:
-  metadata.gz: 82c5cf164680d27183b17b341b362c5f1ec70a26fecf79d4f8d982ea3506e107ac8e216604c5182469c627892f3c423336053a2502c8e10c574c85af5dd82402
-  data.tar.gz: d2109e0aefef9eecebfb57425316cf36dd7cdb92a4356253efb73271f2fed5e4c1ed59e56a46b44a2dc869dd3acb5285b88ae11d6d72b15f50a7506ed85058a2
+  metadata.gz: 8c7ef5e49a3aff7e09e963592fc79f7435a29d3567764d0cec1e98d52fea31d18f1ee2ea4b3d429c2e550307f0fab539733d700716b89b45c05cfee269e87955
+  data.tar.gz: 0be71a225c18e458d440f80eefadfb5b1a841ad57d619af434e12832c48ff31c1ce2614aaf5786330e2282e418838a223f91c82ebae74cedda3801446620fbb2

data/Rakefile

@@ -18,13 +18,14 @@ APP_RAKEFILE = File.expand_path("spec/dummy/Rakefile", __dir__)
 load "rails/tasks/engine.rake"
 load "rails/tasks/statistics.rake"
 
+require "rspec/core"
+require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
-require "rubocop/rake_task"
-RuboCop::RakeTask.new
-task default: [:spec, :rubocop]
+require "standard/rake"
+task default: %i[spec standard]
 
 require "rake/release"
 
 Rake::Release::Task.load_all do |spec|
-  spec.version_tag = "nulogy_message_bus_producer-v#{spec.version}"
+  spec.version_tag = "nulogy_message_bus_consumer-v#{spec.version}"
 end

data/config/credentials/message-bus-us-east-1.key

@@ -0,0 +1 @@
+dfa19863b2709390893da4c2fb85579a

data/config/credentials/message-bus-us-east-1.yml.enc

@@ -0,0 +1 @@
+/WXdqUYePaHqAq3P0iTEsrLiMfRKzp2qYYh7K+q6LUNgi4eHNv0+SoLdI1bUNb9UaGvDPGNT3fCwAdkurk5Iud16ok3b4wD6yZ7UkfqbXqZaKH/dciQ5s63p9Hiuq1rbfcqoZ3KR1SXYAwvy8vNqbdwbAzz2N66B1wE5fNibZZlrWzXJjLReiTcNyxNbCPz6vwEwFF52RntuYlIJo4Nkm8vEk3No+HWrOkM8xptr5qApbd+RowLCLZ4/kcAMDB/XPiGobf0AOFv1NUR/9ChEy20usa8Fqd6HtEn4A25HnAC0uaN0K8ZRXjxhMpnXtfMBItn7yxyJ1ubjRZK5a1xVBRU7L/CVV9ZuIsqAHL1++gH5FBrEe83ZIUhN7AzngMDlOPKGiCLiZLrm18I1AEQrD7tJLyXos15AeAzj--cER8cN0iMLwu8Le+--FL7dhMTgr6xL6SkMnYKmeg==

data/lib/nulogy_message_bus_consumer.rb

@@ -1,10 +1,14 @@
+require "active_record/railtie"
+require "active_support/core_ext/time/zones"
 require "rdkafka"
 
 require "nulogy_message_bus_consumer/engine"
-
+require "nulogy_message_bus_consumer/clock"
 require "nulogy_message_bus_consumer/config"
+require "nulogy_message_bus_consumer/deployment/ecs"
 require "nulogy_message_bus_consumer/handlers/log_unprocessed_messages"
 require "nulogy_message_bus_consumer/kafka_utils"
+require "nulogy_message_bus_consumer/lag_tracker"
 require "nulogy_message_bus_consumer/message"
 require "nulogy_message_bus_consumer/null_logger"
 require "nulogy_message_bus_consumer/pipeline"
@@ -12,11 +16,12 @@ require "nulogy_message_bus_consumer/processed_message"
 require "nulogy_message_bus_consumer/steps/commit_on_success"
 require "nulogy_message_bus_consumer/steps/connect_to_message_bus"
 require "nulogy_message_bus_consumer/steps/deduplicate_messages"
+require "nulogy_message_bus_consumer/steps/log_consumer_lag"
 require "nulogy_message_bus_consumer/steps/log_messages"
-require "nulogy_message_bus_consumer/steps/monitor_replication_lag"
 require "nulogy_message_bus_consumer/steps/seek_beginning_of_topic"
 require "nulogy_message_bus_consumer/steps/stream_messages"
 require "nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left"
+require "nulogy_message_bus_consumer/steps/supervise_consumer_lag"
 
 module NulogyMessageBusConsumer
   module_function
@@ -31,7 +36,7 @@ module NulogyMessageBusConsumer
   end
 
   def logger
-    @@logger ||= NullLogger.new
+    @logger ||= NullLogger.new
   end
 
   def invoke_pipeline(*steps)
@@ -40,14 +45,21 @@ module NulogyMessageBusConsumer
 
   def recommended_consumer_pipeline(config: self.config, logger: self.logger)
     Pipeline.new([
-      # The first three are really system processing steps
+      # System processing/health steps.
+      # Note: that since they are before `StreamMessages`, they will only
+      # be called once, without any messages.
       Steps::ConnectToMessageBus.new(config, logger),
-      Steps::MonitorReplicationLag.new(logger),
+      Steps::LogConsumerLag.new(logger),
+      Steps::SuperviseConsumerLag.new(
+        logger,
+        check_interval_seconds: config.lag_check_interval_seconds,
+        tracker: LagTracker.new(failing_checks: config.lag_checks)
+      ),
       Steps::StreamMessages.new(logger),
       # Message processing steps start here.
       Steps::LogMessages.new(logger),
       Steps::CommitOnSuccess.new,
-      Steps::DeduplicateMessages.new(logger),
+      Steps::DeduplicateMessages.new(logger)
     ])
   end
 

data/lib/nulogy_message_bus_consumer/clock.rb

@@ -0,0 +1,13 @@
+module NulogyMessageBusConsumer
+  # Note: Since this calls Time.zone, it is NOt thread-safe
+  class Clock
+    def now
+      Time.zone.now.to_datetime
+    end
+
+    # milliseconds since epoch
+    def ms
+      now.strftime("%Q").to_i
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/config.rb

@@ -1,11 +1,19 @@
 module NulogyMessageBusConsumer
   class Config
-    attr_accessor :consumer_group_id
-    attr_accessor :bootstrap_servers
-    attr_accessor :topic_name
+    attr_accessor :bootstrap_servers,
+      :client_id,
+      :consumer_group_id,
+      :lag_check_interval_seconds,
+      :lag_checks,
+      :topic_name
 
     def initialize(options = {})
-      update(options)
+      defaults = {
+        lag_check_interval_seconds: 20,
+        lag_checks: 6
+      }
+
+      update(defaults.merge(options))
     end
 
     def update(options = {})

data/lib/nulogy_message_bus_consumer/deployment/ecs.rb

@@ -0,0 +1,23 @@
+module NulogyMessageBusConsumer
+  module Deployment
+    module ECS
+      module_function
+
+      # Try to get the TaskID from metadata server:
+      # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4.html
+      # Otherwise, return nil
+      def task_id
+        data = `curl --silent "$ECS_CONTAINER_METADATA_URI_V4/task"`
+
+        return if data.empty?
+
+        json = JSON.parse(data)
+        arn = json["TaskARN"]
+
+        return unless arn
+
+        arn.split("/").last
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/handlers/log_unprocessed_messages.rb

@@ -7,9 +7,10 @@ module NulogyMessageBusConsumer
 
       def call(message:, **_)
         return if ProcessedMessage.exists?(id: message.id)
+
         @logger.warn(JSON.dump(
           event: "unprocessed_message",
-          kafka_message: message.to_h,
+          kafka_message: message.to_h
         ))
       end
     end

data/lib/nulogy_message_bus_consumer/kafka_utils.rb

@@ -13,13 +13,14 @@ module NulogyMessageBusConsumer
     def wait_for(attempts: 100, interval: 0.1)
       attempts.times do
         break if yield
+
         sleep interval
       end
     end
 
     def every_message_until_none_are_left(consumer)
       Enumerator.new do |yielder|
-        while message = consumer.poll(250)
+        while (message = consumer.poll(250))
           yielder.yield(message)
         end
       end

data/lib/nulogy_message_bus_consumer/lag_tracker.rb

@@ -0,0 +1,53 @@
+require "set"
+
+module NulogyMessageBusConsumer
+  # Keeps track of how many times a topic's partition has not changed (non-zero) lag between update calls.
+  class LagTracker
+    attr_reader :failing_checks
+
+    def initialize(failing_checks: 3)
+      @failing_checks = failing_checks
+      @tracked = Hash.new { |h, topic| h[topic] = {} }
+      @failed = Hash.new { |h, topic| h[topic] = Set.new }
+    end
+
+    def update(topic_partitions)
+      topic_partitions.each_pair do |topic, partitions|
+        partitions.each_pair do |partition, value|
+          update_topic_partition(topic, partition, value)
+        end
+      end
+    end
+
+    def failing?
+      @failed.any?
+    end
+
+    def failed
+      @failed.transform_values { |v| v.to_a.sort }
+    end
+
+    private
+
+    def update_topic_partition(topic, partition, value)
+      current_value, count = @tracked.dig(topic, partition)
+
+      new_value, new_count =
+        if current_value == value && !value.zero?
+          [current_value, count + 1]
+        else
+          [value, 0]
+        end
+
+      @tracked[topic][partition] = [new_value, new_count]
+
+      if new_count >= @failing_checks
+        @failed[topic] << partition
+      end
+    end
+
+    def exists?(topic, partition)
+      @tracked.dig(topic, partition)
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/message.rb

@@ -1,23 +1,30 @@
 module NulogyMessageBusConsumer
   class Message
+    attr_reader :company_uuid,
+      :created_at,
+      :event_data,
+      :event_data_unparsed,
+      :id,
+      :key,
+      :offset,
+      :partition,
+      :subscription_id,
+      :timestamp,
+      :topic,
+      :type
+
     def initialize(attrs = {})
       attrs.each { |key, value| instance_variable_set("@#{key}", value) }
     end
 
-    attr_reader :event_data
-    attr_reader :event_data_unparsed
-    attr_reader :id
-    attr_reader :key
-    attr_reader :offset
-    attr_reader :partition
-    attr_reader :subscription_id
-    attr_reader :company_uuid
-    attr_reader :timestamp
-    attr_reader :topic
-
     def self.from_kafka(kafka_message)
       envelope_data = JSON.parse(kafka_message.payload, symbolize_names: true)
-      event_data = JSON.parse(envelope_data[:event_json], symbolize_names: true) rescue {}
+      event_data =
+        begin
+          JSON.parse(envelope_data[:event_json], symbolize_names: true)
+        rescue
+          {}
+        end
 
       new(
         event_data: event_data,
@@ -30,6 +37,8 @@ module NulogyMessageBusConsumer
         company_uuid: envelope_data[:company_uuid] || envelope_data[:tenant_id],
         timestamp: kafka_message.timestamp,
         topic: kafka_message.topic,
+        type: envelope_data[:type],
+        created_at: envelope_data[:created_at]
       )
     end
 

data/lib/nulogy_message_bus_consumer/null_logger.rb

@@ -1,9 +1,12 @@
 module NulogyMessageBusConsumer
   class NullLogger
-    def info(*_) end
+    def info(*_)
+    end
 
-    def error(*_) end
+    def error(*_)
+    end
 
-    def warn(*_) end
+    def warn(*_)
+    end
   end
 end

data/lib/nulogy_message_bus_consumer/pipeline.rb

@@ -9,7 +9,7 @@ module NulogyMessageBusConsumer
     end
 
     def insert(step, after:)
-      index = @steps.find_index { |step| step.is_a?(after) }
+      index = @steps.find_index { |s| s.is_a?(after) }
       @steps.insert(index + 1, step)
     end
 
@@ -25,7 +25,7 @@ module NulogyMessageBusConsumer
       @steps.reverse.reduce(last_step) do |composed_steps, previous_step|
         lambda do |**args|
           invoke_next = compose_with_merged_args(args, composed_steps)
-            previous_step.call(**args, &invoke_next)
+          previous_step.call(**args, &invoke_next)
         end
       end
     end
@@ -33,7 +33,10 @@ module NulogyMessageBusConsumer
     def compose_with_merged_args(existing_args, func)
       lambda do |**yielded_args|
         args_to_be_overridden = existing_args.keys & yielded_args.keys
-        raise "Cannot override existing argument(s): #{args_to_be_overridden.join(', ')}" if args_to_be_overridden.any?
+        if args_to_be_overridden.any?
+          raise "Cannot override existing argument(s): #{args_to_be_overridden.join(", ")}"
+        end
+
         func.call(**existing_args.merge(yielded_args))
       end
     end

data/lib/nulogy_message_bus_consumer/steps/commit_on_success.rb

@@ -7,6 +7,7 @@ module NulogyMessageBusConsumer
         raise_if_invalid(result)
 
         if result == :success
+          kafka_consumer.store_offset(message)
           kafka_consumer.commit
         else
           reconnect_to_reprocess_same_message(kafka_consumer)

data/lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb

@@ -1,33 +1,52 @@
 module NulogyMessageBusConsumer
   module Steps
     class ConnectToMessageBus
-      def initialize(config, logger)
+      def initialize(config, logger, kafka_consumer: nil)
         @config = config
         @logger = logger
+        @kafka_consumer = kafka_consumer
       end
 
       def call(**_)
         @logger.info("Connecting to the MessageBus")
-        consumer = Rdkafka::Config.new(consumer_config).consumer
         @logger.info("Using consumer group id: #{@config.consumer_group_id}")
 
-        consumer.subscribe(@config.topic_name)
-        @logger.info("Listening for kafka messages on topic #{@config.topic_name}")
+        subscribe
+
+        trap("TERM") { kafka_consumer.close }
 
-        trap("TERM") { consumer.close }
+        wait_for_assignment
 
-        KafkaUtils.wait_for_assignment(consumer)
-        yield(kafka_consumer: consumer)
+        yield(kafka_consumer: kafka_consumer)
       end
 
       private
 
+      def kafka_consumer
+        @kafka_consumer ||= Rdkafka::Config.new(consumer_config).consumer
+      end
+
       def consumer_config
-        {
+        config = {
           "bootstrap.servers": @config.bootstrap_servers,
           "enable.auto.commit": false,
           "group.id": @config.consumer_group_id,
+          "enable.auto.offset.store": false
         }
+
+        config["client.id"] = @config.client_id if @config.client_id
+
+        config
+      end
+
+      def subscribe
+        kafka_consumer.subscribe(@config.topic_name)
+        @logger.info("Listening for kafka messages on topic #{@config.topic_name}")
+      end
+
+      def wait_for_assignment
+        KafkaUtils.wait_for_assignment(kafka_consumer)
+        @logger.info("Connected as client: #{kafka_consumer.member_id}")
       end
     end
   end

data/lib/nulogy_message_bus_consumer/steps/deduplicate_messages.rb

@@ -41,7 +41,7 @@ module NulogyMessageBusConsumer
       def log_duplicate(message)
         @logger.warn(JSON.dump({
           event: "duplicate_message_detected",
-          kafka_message_id: message.id,
+          kafka_message_id: message.id
         }))
       end
     end

data/lib/nulogy_message_bus_consumer/steps/{monitor_replication_lag.rb → log_consumer_lag.rb}

@@ -1,6 +1,6 @@
 module NulogyMessageBusConsumer
   module Steps
-    class MonitorReplicationLag
+    class LogConsumerLag
       def initialize(logger)
         @logger = logger
       end
@@ -22,9 +22,9 @@ module NulogyMessageBusConsumer
 
             @logger.info(JSON.dump({
               event: "consumer_lag",
-              topics: Calculator.add_max_lag(lag_per_topic),
+              topics: Calculator.add_max_lag(lag_per_topic)
             }))
-            STDOUT.flush
+            $stdout.flush
 
             sleep 60
           end

data/lib/nulogy_message_bus_consumer/steps/log_messages.rb

@@ -1,28 +1,39 @@
 module NulogyMessageBusConsumer
   module Steps
     class LogMessages
-      def initialize(logger)
+      def initialize(logger, clock: Clock.new)
         @logger = logger
+        @clock = clock
       end
 
       def call(message:, **_)
         @logger.info(JSON.dump({
           event: "message_received",
           kafka_message_id: message.id,
-          message: "Received #{message.id}",
+          message: "Received #{message.id}"
         }))
 
         result = yield
 
+        millis = diff_millis(message.created_at, @clock.ms)
         @logger.info(JSON.dump({
           event: "message_processed",
           kafka_message_id: message.id,
-          message: "Processed #{message.id}",
+          message: "Processed #{message.id} (#{message.topic}##{message.partition}@#{message.offset})",
           result: result,
+          time_to_processed: millis
         }))
 
         result
       end
+
+      # Debezium appears to be giving us nanos since epoch
+      # https://github.com/debezium/debezium/blob/5a115e902cdc1dc399ec02758dd1039a33e99bc2/debezium-core/src/main/java/io/debezium/jdbc/JdbcValueConverters.java#L237
+      def diff_millis(oldest_nanos, newest_millis)
+        old_millis = oldest_nanos / 1000
+
+        newest_millis - old_millis
+      end
     end
   end
 end

data/lib/nulogy_message_bus_consumer/steps/stream_messages.rb

@@ -12,11 +12,11 @@ module NulogyMessageBusConsumer
             kafka_message: kafka_message
           )
         end
-      rescue StandardError => e
+      rescue => e
         @logger.error(JSON.dump({
           event: "message_processing_errored",
           class: e.class,
-          message: e.message,
+          message: e.message
         }))
 
         raise

data/lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb

@@ -12,11 +12,11 @@ module NulogyMessageBusConsumer
             kafka_message: kafka_message
           )
         end
-      rescue StandardError => e
+      rescue => e
         @logger.error(JSON.dump({
           event: "message_processing_errored",
           class: e.class,
-          message: e.message,
+          message: e.message
         }))
 
         raise

data/lib/nulogy_message_bus_consumer/steps/supervise_consumer_lag.rb

@@ -0,0 +1,76 @@
+module NulogyMessageBusConsumer
+  module Steps
+    # Supervises the consumer's lag.
+    #
+    # If a partition's lag is non-zero and does not change for an extended period
+    # of time, then kill the main thread.
+    #
+    # That period of time is check_interval_seconds * LagTracker#failing_checks
+    # With the defaults, that would be 20 * 6 ~ 120 seconds = 2 minutes.
+    #
+    # Note that this strategy may not work for a busy integration.
+    # Consumer lag monitoring should alert in that case.
+    # However, this strategy may help alleviate alerts for low traffic or off-peak
+    # environments.
+    #
+    # We've come across cases where the consumer lag is still being logged,
+    # messages are being processed, but the consumer is not consuming messages
+    # in particular topics.
+    #
+    # Killing the main thread causes ECS to restart the task.
+    class SuperviseConsumerLag
+      def initialize(logger, tracker: NulogyMessageBusConsumer::LagTracker.new(failing_checks: 6), killable: nil, check_interval_seconds: 20)
+        @logger = logger
+        @tracker = tracker
+        @killable = killable
+        @check_interval_seconds = check_interval_seconds
+      end
+
+      def call(kafka_consumer:, **_)
+        @consumer = kafka_consumer
+        @killable ||= Thread.current
+
+        run
+
+        yield
+      end
+
+      private
+
+      def run
+        Thread.abort_on_exception = true
+
+        Thread.new do
+          NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(@consumer)
+
+          loop do
+            @tracker.update(@consumer.lag(@consumer.committed))
+
+            if @tracker.failing?
+              log_failed_partitions
+
+              @killable.kill
+              Thread.current.exit
+            end
+
+            sleep @check_interval_seconds
+          end
+        end
+      end
+
+      def log_failed_partitions
+        seconds = @check_interval_seconds * @tracker.failing_checks
+        failed = @tracker
+          .failed
+          .map { |topic, partitions| "#{topic}: #{partitions.join(",")}" }
+          .join(", ")
+
+        @logger.warn(JSON.dump({
+          event: "message_processing_warning",
+          message: "Assigned partition lag has not changed in #{seconds} seconds: #{failed}"
+        }))
+        $stdout.flush
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/version.rb

@@ -1,3 +1,3 @@
 module NulogyMessageBusConsumer
-  VERSION = "0.3.0"
+  VERSION = "1.0.0.alpha".freeze
 end

data/lib/tasks/engine/message_bus_consumer.rake

@@ -1,16 +1,15 @@
 namespace :message_bus_consumer do
   desc "Verifies that the messages in the message bus have been processed"
-  task :audit => :environment do
-    config = build_audit_config
+  task audit: :environment do
+    logger = Rails.logger
+    config = NulogyMessageBusConsumer::Config.new(
+      consumer_group_id: ENV.fetch("MB_AUDIT_GROUP"),
+      bootstrap_servers: ENV.fetch("MB_BOOTSTRAP_SERVERS"),
+      topic_name: ENV.fetch("MB_CONSUMER_TOPIC")
+    )
 
     NulogyMessageBusConsumer
-      .consumer_audit_pipeline(config: config)
+      .consumer_audit_pipeline(config: config, logger: logger)
       .invoke
   end
-
-  def build_audit_config
-    audit_config = NulogyMessageBusConsumer.config.dup
-    audit_config.consumer_group_id = "#{audit_config.consumer_group_id}_consumer_audit"
-    NulogyMessageBusConsumer.config
-  end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nulogy_message_bus_consumer
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.0.alpha
 platform: ruby
 authors:
 - Nulogy
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-05 00:00:00.000000000 Z
+date: 2021-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -52,34 +52,104 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler-audit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.0.1
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.7.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.7.6
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.0.3
+        version: 6.0.3.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.0.3
+        version: 6.0.3.5
 - !ruby/object:Gem::Dependency
   name: rake-release
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 1.3.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.2.1
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -95,75 +165,75 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.9.0
 - !ruby/object:Gem::Dependency
-  name: rspec-rails
+  name: rspec-json_expectations
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.0.1
+        version: 2.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 4.0.1
+        version: 2.2.0
 - !ruby/object:Gem::Dependency
-  name: rspec-json_expectations
+  name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: 4.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 2.2.0
+        version: 4.0.1
 - !ruby/object:Gem::Dependency
-  name: rubocop
+  name: standard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.81.0
+        version: 0.11.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.81.0
+        version: 0.11.0
 - !ruby/object:Gem::Dependency
-  name: rubocop-rspec
+  name: rubocop-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.38.1
+        version: 2.5.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.38.1
+        version: 2.5.2
 - !ruby/object:Gem::Dependency
-  name: pg
+  name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.2.3
+        version: 1.38.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.2.3
+        version: 1.38.1
 description:
 email:
 - tass@nulogy.com
@@ -172,13 +242,18 @@ extensions: []
 extra_rdoc_files: []
 files:
 - Rakefile
+- config/credentials/message-bus-us-east-1.key
+- config/credentials/message-bus-us-east-1.yml.enc
 - config/routes.rb
 - db/migrate/20200509095105_create_message_bus_processed_messages.rb
 - lib/nulogy_message_bus_consumer.rb
+- lib/nulogy_message_bus_consumer/clock.rb
 - lib/nulogy_message_bus_consumer/config.rb
+- lib/nulogy_message_bus_consumer/deployment/ecs.rb
 - lib/nulogy_message_bus_consumer/engine.rb
 - lib/nulogy_message_bus_consumer/handlers/log_unprocessed_messages.rb
 - lib/nulogy_message_bus_consumer/kafka_utils.rb
+- lib/nulogy_message_bus_consumer/lag_tracker.rb
 - lib/nulogy_message_bus_consumer/message.rb
 - lib/nulogy_message_bus_consumer/null_logger.rb
 - lib/nulogy_message_bus_consumer/pipeline.rb
@@ -186,11 +261,12 @@ files:
 - lib/nulogy_message_bus_consumer/steps/commit_on_success.rb
 - lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb
 - lib/nulogy_message_bus_consumer/steps/deduplicate_messages.rb
+- lib/nulogy_message_bus_consumer/steps/log_consumer_lag.rb
 - lib/nulogy_message_bus_consumer/steps/log_messages.rb
-- lib/nulogy_message_bus_consumer/steps/monitor_replication_lag.rb
 - lib/nulogy_message_bus_consumer/steps/seek_beginning_of_topic.rb
 - lib/nulogy_message_bus_consumer/steps/stream_messages.rb
 - lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb
+- lib/nulogy_message_bus_consumer/steps/supervise_consumer_lag.rb
 - lib/nulogy_message_bus_consumer/version.rb
 - lib/tasks/engine/message_bus_consumer.rake
 homepage: https://github.com/nulogy/message-bus/tree/master/gems/nulogy_message_bus_consumer
@@ -208,9 +284,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.3
 signing_key: