nulogy_message_bus_consumer 0.1.0 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c30491ceed8cd0f6640c89cc9d9cdf285bdfd92db7b2f1539db46b1732833901
-  data.tar.gz: 3ecec8b99782d45a3b1654a82ecfb8f86162d315531a01d5be9264282f80c2fd
+  metadata.gz: c47fb7fab95cd7d64e9654c0ce43fec64f77f8718331f8028afc6c18ab993f81
+  data.tar.gz: 3c656a7dea18d149f04ccfbc3dc4db04e9aa7b3cd6183be7b0a1a45337c3a975
 SHA512:
-  metadata.gz: 4375656d3a099245716be814300f55f63758bcbf8d9a44fc4f7f13e413e828e1918a19001ee3b87252a383e1405bf8f31a91990ebe917da0b38b8b185fc245a5
-  data.tar.gz: 24e4599d94648a59e415a1ed984ffceda98265ad244a374e4754ad4ee659f07ed17a75b681c7b82527a559ae3055e70fc1d1bb50d2212b0ca31dd0ede14b74a9
+  metadata.gz: 04320f62a752bcfcc4bad93bd0611ba0f8945018dfb1680016ab739b70ba66cb7ebcf5062fb12f316745e59386714613c44f17f7feea4006a56b8625da119e23
+  data.tar.gz: c480ced5fb50feeb8d655cb8c0b45c19a12bd733088db6fe4bf78ce33d08a7051594eb86a011227d2cc1363ea058e1544435c318824a6c9549be2e32fb6f65ef

data/Rakefile

@@ -16,7 +16,17 @@ end
 
 APP_RAKEFILE = File.expand_path("spec/dummy/Rakefile", __dir__)
 load "rails/tasks/engine.rake"
-
 load "rails/tasks/statistics.rake"
 
-require "bundler/gem_tasks"
+require "rspec/core"
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec)
+require "rubocop/rake_task"
+RuboCop::RakeTask.new
+task default: %i[spec rubocop]
+
+require "rake/release"
+
+Rake::Release::Task.load_all do |spec|
+  spec.version_tag = "nulogy_message_bus_consumer-v#{spec.version}"
+end

data/config/credentials/message-bus-us-east-1.key

@@ -0,0 +1 @@
+dfa19863b2709390893da4c2fb85579a

data/config/credentials/message-bus-us-east-1.yml.enc

@@ -0,0 +1 @@
+/WXdqUYePaHqAq3P0iTEsrLiMfRKzp2qYYh7K+q6LUNgi4eHNv0+SoLdI1bUNb9UaGvDPGNT3fCwAdkurk5Iud16ok3b4wD6yZ7UkfqbXqZaKH/dciQ5s63p9Hiuq1rbfcqoZ3KR1SXYAwvy8vNqbdwbAzz2N66B1wE5fNibZZlrWzXJjLReiTcNyxNbCPz6vwEwFF52RntuYlIJo4Nkm8vEk3No+HWrOkM8xptr5qApbd+RowLCLZ4/kcAMDB/XPiGobf0AOFv1NUR/9ChEy20usa8Fqd6HtEn4A25HnAC0uaN0K8ZRXjxhMpnXtfMBItn7yxyJ1ubjRZK5a1xVBRU7L/CVV9ZuIsqAHL1++gH5FBrEe83ZIUhN7AzngMDlOPKGiCLiZLrm18I1AEQrD7tJLyXos15AeAzj--cER8cN0iMLwu8Le+--FL7dhMTgr6xL6SkMnYKmeg==

data/lib/nulogy_message_bus_consumer.rb

@@ -1,52 +1,62 @@
+require "rdkafka"
+
 require "nulogy_message_bus_consumer/engine"
+
 require "nulogy_message_bus_consumer/config"
+require "nulogy_message_bus_consumer/handlers/log_unprocessed_messages"
+require "nulogy_message_bus_consumer/kafka_utils"
 require "nulogy_message_bus_consumer/message"
+require "nulogy_message_bus_consumer/null_logger"
+require "nulogy_message_bus_consumer/pipeline"
 require "nulogy_message_bus_consumer/processed_message"
+require "nulogy_message_bus_consumer/steps/commit_on_success"
 require "nulogy_message_bus_consumer/steps/connect_to_message_bus"
 require "nulogy_message_bus_consumer/steps/deduplicate_messages"
 require "nulogy_message_bus_consumer/steps/log_messages"
 require "nulogy_message_bus_consumer/steps/monitor_replication_lag"
-require "nulogy_message_bus_consumer/steps/result_validation"
-require "nulogy_message_bus_consumer/steps/stream_individual_messages"
+require "nulogy_message_bus_consumer/steps/seek_beginning_of_topic"
+require "nulogy_message_bus_consumer/steps/stream_messages"
+require "nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left"
 
 module NulogyMessageBusConsumer
   module_function
 
+  mattr_accessor :config
+  mattr_accessor :logger
+
+  def configure(options = {})
+    self.config ||= Config.new
+    config.update(options) if options.present?
+    yield(config) if block_given?
+  end
+
+  def logger
+    @logger ||= NullLogger.new
+  end
+
   def invoke_pipeline(*steps)
-    build_pipeline(*steps).call
+    Pipeline.new(steps).invoke
   end
 
-  def default_steps(config, logger)
-    [
+  def recommended_consumer_pipeline(config: self.config, logger: self.logger)
+    Pipeline.new([
       # The first three are really system processing steps
       Steps::ConnectToMessageBus.new(config, logger),
       Steps::MonitorReplicationLag.new(logger),
-      Steps::StreamIndividualMessages.new(config, logger),
+      Steps::StreamMessages.new(logger),
       # Message processing steps start here.
       Steps::LogMessages.new(logger),
+      Steps::CommitOnSuccess.new,
       Steps::DeduplicateMessages.new(logger),
-      Steps::ResultValidation.new,
-    ]
+    ])
   end
 
-  def build_pipeline(*steps)
-    last_step = ->(**_) { raise "Handlers are the end of the line. Do not use yield." }
-
-    steps.reverse.reduce(last_step) do |composed_steps, previous_step|
-      lambda do |**args|
-        invoke_next = compose_with_merged_args(args, composed_steps)
-        previous_step.call(**args, &invoke_next)
-      end
-    end
-  end
-
-  def compose_with_merged_args(context, func)
-    lambda do |**inner_args|
-      existing = context.keys & inner_args.keys
-
-      raise "Cannot override existing key(s): #{existing.join(', ')}" if existing.any?
-
-      func.call(**context.merge(inner_args))
-    end
+  def consumer_audit_pipeline(config: self.config, logger: self.logger)
+    Pipeline.new([
+      Steps::ConnectToMessageBus.new(config, logger),
+      Steps::SeekBeginningOfTopic.new,
+      Steps::StreamMessagesUntilNoneAreLeft.new(logger),
+      Handlers::LogUnprocessedMessages.new(logger),
+    ])
   end
 end

data/lib/nulogy_message_bus_consumer/config.rb

@@ -5,6 +5,10 @@ module NulogyMessageBusConsumer
     attr_accessor :topic_name
 
     def initialize(options = {})
+      update(options)
+    end
+
+    def update(options = {})
       options.each { |key, value| public_send("#{key}=", value) }
     end
   end

data/lib/nulogy_message_bus_consumer/handlers/log_unprocessed_messages.rb

@@ -0,0 +1,18 @@
+module NulogyMessageBusConsumer
+  module Handlers
+    class LogUnprocessedMessages
+      def initialize(logger)
+        @logger = logger
+      end
+
+      def call(message:, **_)
+        return if ProcessedMessage.exists?(id: message.id)
+
+        @logger.warn(JSON.dump(
+          event: "unprocessed_message",
+          kafka_message: message.to_h
+        ))
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/kafka_utils.rb

@@ -0,0 +1,62 @@
+module NulogyMessageBusConsumer
+  module KafkaUtils
+    module_function
+
+    def wait_for_assignment(consumer)
+      wait_for { !consumer.assignment.empty? }
+    end
+
+    def wait_for_unassignment(consumer)
+      wait_for { consumer.assignment.empty? }
+    end
+
+    def wait_for(attempts: 100, interval: 0.1)
+      attempts.times do
+        break if yield
+
+        sleep interval
+      end
+    end
+
+    def every_message_until_none_are_left(consumer)
+      Enumerator.new do |yielder|
+        while (message = consumer.poll(250))
+          yielder.yield(message)
+        end
+      end
+    end
+
+    def seek_beginning(consumer)
+      wait_for_assignment(consumer)
+      assigned_partitions(consumer).each do |topic_name, partition|
+        message = Message.new(
+          topic: topic_name,
+          partition: partition,
+          offset: BEGINNING_OFFSET
+        )
+        consumer.seek(message)
+      end
+    end
+
+    def seek_ending(consumer)
+      wait_for_assignment(consumer)
+      assigned_partitions(consumer).each do |topic_name, partition|
+        message = Message.new(
+          topic: topic_name,
+          partition: partition,
+          offset: END_OFFSET
+        )
+        consumer.seek(message)
+      end
+    end
+
+    def assigned_partitions(consumer)
+      consumer.assignment.to_h
+        .flat_map { |topic_name, partitions| [topic_name].product(partitions) }
+        .map { |topic_name, partition| [topic_name, partition.partition] }
+    end
+
+    BEGINNING_OFFSET = 0
+    END_OFFSET = -1
+  end
+end

data/lib/nulogy_message_bus_consumer/message.rb

@@ -1,21 +1,50 @@
 module NulogyMessageBusConsumer
-  Message = Struct.new(
-    :id,
-    :public_subscription_id,
-    :tenant_id,
-    :event_data,
-    :raw_event_data,
-    keyword_init: true
-  ) do
+  class Message
+    attr_reader :event_data
+    attr_reader :event_data_unparsed
+    attr_reader :id
+    attr_reader :key
+    attr_reader :offset
+    attr_reader :partition
+    attr_reader :subscription_id
+    attr_reader :company_uuid
+    attr_reader :timestamp
+    attr_reader :topic
+    attr_reader :created_at
+
+    def initialize(attrs = {})
+      attrs.each { |key, value| instance_variable_set("@#{key}", value) }
+    end
+
     def self.from_kafka(kafka_message)
-      envelope_data = JSON.parse(kafka_message.payload)
+      envelope_data = JSON.parse(kafka_message.payload, symbolize_names: true)
+      event_data =
+        begin
+          JSON.parse(envelope_data[:event_json], symbolize_names: true)
+        rescue StandardError
+          {}
+        end
 
       new(
-        id: envelope_data["id"],
-        public_subscription_id: envelope_data["public_subscription_id"],
-        tenant_id: envelope_data["tenant_id"],
-        event_data: envelope_data["event_json"]
+        event_data: event_data,
+        event_data_unparsed: envelope_data[:event_json],
+        id: envelope_data[:id],
+        key: kafka_message.key,
+        offset: kafka_message.offset,
+        partition: kafka_message.partition,
+        subscription_id: envelope_data[:subscription_id] || envelope_data[:public_subscription_id],
+        company_uuid: envelope_data[:company_uuid] || envelope_data[:tenant_id],
+        timestamp: kafka_message.timestamp,
+        topic: kafka_message.topic,
+        created_at: envelope_data[:created_at]
       )
     end
+
+    def to_h
+      instance_variable_names.each_with_object({}) do |attribute_name, hash|
+        attribute_name.sub!("@", "")
+        hash[attribute_name] = public_send(attribute_name)
+      end
+    end
   end
 end

data/lib/nulogy_message_bus_consumer/null_logger.rb

@@ -0,0 +1,9 @@
+module NulogyMessageBusConsumer
+  class NullLogger
+    def info(*_) end
+
+    def error(*_) end
+
+    def warn(*_) end
+  end
+end

data/lib/nulogy_message_bus_consumer/pipeline.rb

@@ -0,0 +1,44 @@
+module NulogyMessageBusConsumer
+  class Pipeline
+    def initialize(steps)
+      @steps = steps
+    end
+
+    def invoke
+      convert_steps_into_lambda.call
+    end
+
+    def insert(step, after:)
+      index = @steps.find_index { |s| s.is_a?(after) }
+      @steps.insert(index + 1, step)
+    end
+
+    def append(step)
+      @steps << step
+    end
+
+    private
+
+    def convert_steps_into_lambda
+      last_step = ->(**_) { raise "Handlers are the end of the line. Do not use yield." }
+
+      @steps.reverse.reduce(last_step) do |composed_steps, previous_step|
+        lambda do |**args|
+          invoke_next = compose_with_merged_args(args, composed_steps)
+          previous_step.call(**args, &invoke_next)
+        end
+      end
+    end
+
+    def compose_with_merged_args(existing_args, func)
+      lambda do |**yielded_args|
+        args_to_be_overridden = existing_args.keys & yielded_args.keys
+        if args_to_be_overridden.any?
+          raise "Cannot override existing argument(s): #{args_to_be_overridden.join(', ')}"
+        end
+
+        func.call(**existing_args.merge(yielded_args))
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/steps/commit_on_success.rb

@@ -0,0 +1,37 @@
+module NulogyMessageBusConsumer
+  module Steps
+    class CommitOnSuccess
+      def call(kafka_consumer:, message:, **_)
+        result = yield
+
+        raise_if_invalid(result)
+
+        if result == :success
+          kafka_consumer.store_offset(message)
+          kafka_consumer.commit
+        else
+          reconnect_to_reprocess_same_message(kafka_consumer)
+        end
+
+        result
+      end
+
+      private
+
+      def reconnect_to_reprocess_same_message(kafka_consumer)
+        subscriptions = kafka_consumer.subscription
+        kafka_consumer.unsubscribe
+        kafka_consumer.subscribe(*subscriptions.to_h.keys)
+      end
+
+      def raise_if_invalid(result)
+        return if %i[success failure].include?(result)
+
+        raise(
+          StandardError,
+          "'#{result}' is not a valid processing outcome. Must be :success or :failure"
+        )
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb

@@ -1,28 +1,38 @@
 module NulogyMessageBusConsumer
   module Steps
     class ConnectToMessageBus
-      def initialize(config, logger)
+      def initialize(config, logger, kafka_consumer: nil)
         @config = config
         @logger = logger
+        @kafka_consumer = kafka_consumer
       end
 
       def call(**_)
         @logger.info("Connecting to the MessageBus")
-        consumer = Rdkafka::Config.new(consumer_config).consumer
+        consumer = build_consumer
         @logger.info("Using consumer group id: #{@config.consumer_group_id}")
 
+        consumer.subscribe(@config.topic_name)
+        @logger.info("Listening for kafka messages on topic #{@config.topic_name}")
+
         trap("TERM") { consumer.close }
 
+        KafkaUtils.wait_for_assignment(consumer)
         yield(kafka_consumer: consumer)
       end
 
       private
 
+      def build_consumer
+        @kafka_consumer || Rdkafka::Config.new(consumer_config).consumer
+      end
+
       def consumer_config
         {
           "bootstrap.servers": @config.bootstrap_servers,
           "enable.auto.commit": false,
           "group.id": @config.consumer_group_id,
+          "enable.auto.offset.store": false,
         }
       end
     end

data/lib/nulogy_message_bus_consumer/steps/log_messages.rb

@@ -1,8 +1,16 @@
 module NulogyMessageBusConsumer
   module Steps
+    class Clock
+      # milliseconds since epoch
+      def now
+        Time.zone.now.to_datetime.strftime("%Q").to_i
+      end
+    end
+
     class LogMessages
-      def initialize(logger)
+      def initialize(logger, clock: Clock.new)
         @logger = logger
+        @clock = clock
       end
 
       def call(message:, **_)
@@ -14,15 +22,25 @@ module NulogyMessageBusConsumer
 
         result = yield
 
+        millis = diff_millis(message.created_at, @clock.now)
         @logger.info(JSON.dump({
           event: "message_processed",
           kafka_message_id: message.id,
           message: "Processed #{message.id}",
           result: result,
+          time_to_processed: millis,
         }))
 
         result
       end
+
+      # Debezium appears to be giving us nanos since epoch
+      # https://github.com/debezium/debezium/blob/5a115e902cdc1dc399ec02758dd1039a33e99bc2/debezium-core/src/main/java/io/debezium/jdbc/JdbcValueConverters.java#L237
+      def diff_millis(oldest_nanos, newest_millis)
+        old_millis = oldest_nanos / 1000
+
+        newest_millis - old_millis
+      end
     end
   end
 end

data/lib/nulogy_message_bus_consumer/steps/monitor_replication_lag.rb

@@ -15,7 +15,7 @@ module NulogyMessageBusConsumer
           # Delayed start. If we attempt to read consumer#committed immediately, it may fail.
           # We suspect this is because the consumer#committed is called before the consumer
           # has finished connecting. There appears to be a race condition.
-          sleep 30
+          KafkaUtils.wait_for_assignment(kafka_consumer)
 
           loop do
             lag_per_topic = kafka_consumer.lag(kafka_consumer.committed)

data/lib/nulogy_message_bus_consumer/steps/seek_beginning_of_topic.rb

@@ -0,0 +1,10 @@
+module NulogyMessageBusConsumer
+  module Steps
+    class SeekBeginningOfTopic
+      def call(kafka_consumer:, **_)
+        KafkaUtils.seek_beginning(kafka_consumer)
+        yield
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/steps/{stream_individual_messages.rb → stream_messages.rb}

@@ -1,22 +1,16 @@
 module NulogyMessageBusConsumer
   module Steps
-    class StreamIndividualMessages
-      def initialize(config, logger)
-        @config = config
+    class StreamMessages
+      def initialize(logger)
         @logger = logger
       end
 
       def call(kafka_consumer:, **_)
-        kafka_consumer.subscribe(@config.topic_name)
-        @logger.info "Listening for kafka messages on topic #{@config.topic_name}"
-
         kafka_consumer.each do |kafka_message|
-          result = yield(
-            message: NulogyMessageBusConsumer::Message.from_kafka(kafka_message),
+          yield(
+            message: Message.from_kafka(kafka_message),
             kafka_message: kafka_message
           )
-
-          kafka_consumer.commit if result == :success
         end
       rescue StandardError => e
         @logger.error(JSON.dump({
@@ -25,8 +19,6 @@ module NulogyMessageBusConsumer
           message: e.message,
         }))
 
-        kafka_consumer.unsubscribe
-
         raise
       end
     end

data/lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb

@@ -0,0 +1,26 @@
+module NulogyMessageBusConsumer
+  module Steps
+    class StreamMessagesUntilNoneAreLeft
+      def initialize(logger)
+        @logger = logger
+      end
+
+      def call(kafka_consumer:, **_)
+        KafkaUtils.every_message_until_none_are_left(kafka_consumer).each do |kafka_message|
+          yield(
+            message: Message.from_kafka(kafka_message),
+            kafka_message: kafka_message
+          )
+        end
+      rescue StandardError => e
+        @logger.error(JSON.dump({
+          event: "message_processing_errored",
+          class: e.class,
+          message: e.message,
+        }))
+
+        raise
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/version.rb

@@ -1,3 +1,3 @@
 module NulogyMessageBusConsumer
-  VERSION = "0.1.0"
+  VERSION = "0.3.3"
 end

data/lib/tasks/engine/message_bus_consumer.rake

@@ -0,0 +1,15 @@
+namespace :message_bus_consumer do
+  desc "Verifies that the messages in the message bus have been processed"
+  task audit: :environment do
+    logger = Rails.logger
+    config = NulogyMessageBusConsumer::Config.new(
+      consumer_group_id: ENV.fetch("MB_AUDIT_GROUP"),
+      bootstrap_servers: ENV.fetch("MB_BOOTSTRAP_SERVERS"),
+      topic_name: ENV.fetch("MB_CONSUMER_TOPIC")
+    )
+
+    NulogyMessageBusConsumer
+      .consumer_audit_pipeline(config: config, logger: logger)
+      .invoke
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: nulogy_message_bus_consumer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.3
 platform: ruby
 authors:
 - Nulogy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-05 00:00:00.000000000 Z
+date: 2020-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -38,20 +38,118 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: rdkafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler-audit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.7.0.1
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.7.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.7.6
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.3
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.0.3
+        version: 6.0.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 6.0.3
+        version: 6.0.3.1
+- !ruby/object:Gem::Dependency
+  name: rake-release
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.2.1
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +178,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.2.0
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.0.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.0.1
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -95,33 +207,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.81.0
 - !ruby/object:Gem::Dependency
-  name: rubocop-rspec
+  name: rubocop-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.38.1
+        version: 2.5.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.38.1
+        version: 2.5.2
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.4.2
+        version: 1.38.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.4.2
+        version: 1.38.1
 description: 
 email:
 - tass@nulogy.com
@@ -130,20 +242,29 @@ extensions: []
 extra_rdoc_files: []
 files:
 - Rakefile
+- config/credentials/message-bus-us-east-1.key
+- config/credentials/message-bus-us-east-1.yml.enc
 - config/routes.rb
 - db/migrate/20200509095105_create_message_bus_processed_messages.rb
 - lib/nulogy_message_bus_consumer.rb
 - lib/nulogy_message_bus_consumer/config.rb
 - lib/nulogy_message_bus_consumer/engine.rb
+- lib/nulogy_message_bus_consumer/handlers/log_unprocessed_messages.rb
+- lib/nulogy_message_bus_consumer/kafka_utils.rb
 - lib/nulogy_message_bus_consumer/message.rb
+- lib/nulogy_message_bus_consumer/null_logger.rb
+- lib/nulogy_message_bus_consumer/pipeline.rb
 - lib/nulogy_message_bus_consumer/processed_message.rb
+- lib/nulogy_message_bus_consumer/steps/commit_on_success.rb
 - lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb
 - lib/nulogy_message_bus_consumer/steps/deduplicate_messages.rb
 - lib/nulogy_message_bus_consumer/steps/log_messages.rb
 - lib/nulogy_message_bus_consumer/steps/monitor_replication_lag.rb
-- lib/nulogy_message_bus_consumer/steps/result_validation.rb
-- lib/nulogy_message_bus_consumer/steps/stream_individual_messages.rb
+- lib/nulogy_message_bus_consumer/steps/seek_beginning_of_topic.rb
+- lib/nulogy_message_bus_consumer/steps/stream_messages.rb
+- lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb
 - lib/nulogy_message_bus_consumer/version.rb
+- lib/tasks/engine/message_bus_consumer.rake
 homepage: https://github.com/nulogy/message-bus/tree/master/gems/nulogy_message_bus_consumer
 licenses: []
 metadata:

data/lib/nulogy_message_bus_consumer/steps/result_validation.rb

@@ -1,16 +0,0 @@
-module NulogyMessageBusConsumer
-  module Steps
-    class ResultValidation
-      def call(**_)
-        result = yield
-
-        return result if %i[success failure].include?(result)
-
-        raise(
-          StandardError,
-          "'#{result}' is not a valid processing outcome. Must be :success or :failure"
-        )
-      end
-    end
-  end
-end