nulogy_message_bus_consumer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c30491ceed8cd0f6640c89cc9d9cdf285bdfd92db7b2f1539db46b1732833901
+  data.tar.gz: 3ecec8b99782d45a3b1654a82ecfb8f86162d315531a01d5be9264282f80c2fd
+SHA512:
+  metadata.gz: 4375656d3a099245716be814300f55f63758bcbf8d9a44fc4f7f13e413e828e1918a19001ee3b87252a383e1405bf8f31a91990ebe917da0b38b8b185fc245a5
+  data.tar.gz: 24e4599d94648a59e415a1ed984ffceda98265ad244a374e4754ad4ee659f07ed17a75b681c7b82527a559ae3055e70fc1d1bb50d2212b0ca31dd0ede14b74a9

data/Rakefile

@@ -0,0 +1,22 @@
+begin
+  require "bundler/setup"
+rescue LoadError
+  puts "You must `gem install bundler` and `bundle install` to run rake tasks"
+end
+
+require "rdoc/task"
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = "rdoc"
+  rdoc.title = "MessageBus"
+  rdoc.options << "--line-numbers"
+  rdoc.rdoc_files.include("README.md")
+  rdoc.rdoc_files.include("lib/**/*.rb")
+end
+
+APP_RAKEFILE = File.expand_path("spec/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+load "rails/tasks/statistics.rake"
+
+require "bundler/gem_tasks"

data/config/routes.rb

@@ -0,0 +1,2 @@
+NulogyMessageBusConsumer::Engine.routes.draw do
+end

data/db/migrate/20200509095105_create_message_bus_processed_messages.rb

@@ -0,0 +1,7 @@
+class CreateMessageBusProcessedMessages < ActiveRecord::Migration[6.0]
+  def change
+    create_table :message_bus_processed_messages, id: :uuid, default: nil do |t|
+      t.datetime :created_at, null: false
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer.rb

@@ -0,0 +1,52 @@
+require "nulogy_message_bus_consumer/engine"
+require "nulogy_message_bus_consumer/config"
+require "nulogy_message_bus_consumer/message"
+require "nulogy_message_bus_consumer/processed_message"
+require "nulogy_message_bus_consumer/steps/connect_to_message_bus"
+require "nulogy_message_bus_consumer/steps/deduplicate_messages"
+require "nulogy_message_bus_consumer/steps/log_messages"
+require "nulogy_message_bus_consumer/steps/monitor_replication_lag"
+require "nulogy_message_bus_consumer/steps/result_validation"
+require "nulogy_message_bus_consumer/steps/stream_individual_messages"
+
+module NulogyMessageBusConsumer
+  module_function
+
+  def invoke_pipeline(*steps)
+    build_pipeline(*steps).call
+  end
+
+  def default_steps(config, logger)
+    [
+      # The first three are really system processing steps
+      Steps::ConnectToMessageBus.new(config, logger),
+      Steps::MonitorReplicationLag.new(logger),
+      Steps::StreamIndividualMessages.new(config, logger),
+      # Message processing steps start here.
+      Steps::LogMessages.new(logger),
+      Steps::DeduplicateMessages.new(logger),
+      Steps::ResultValidation.new,
+    ]
+  end
+
+  def build_pipeline(*steps)
+    last_step = ->(**_) { raise "Handlers are the end of the line. Do not use yield." }
+
+    steps.reverse.reduce(last_step) do |composed_steps, previous_step|
+      lambda do |**args|
+        invoke_next = compose_with_merged_args(args, composed_steps)
+        previous_step.call(**args, &invoke_next)
+      end
+    end
+  end
+
+  def compose_with_merged_args(context, func)
+    lambda do |**inner_args|
+      existing = context.keys & inner_args.keys
+
+      raise "Cannot override existing key(s): #{existing.join(', ')}" if existing.any?
+
+      func.call(**context.merge(inner_args))
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/config.rb

@@ -0,0 +1,11 @@
+module NulogyMessageBusConsumer
+  class Config
+    attr_accessor :consumer_group_id
+    attr_accessor :bootstrap_servers
+    attr_accessor :topic_name
+
+    def initialize(options = {})
+      options.each { |key, value| public_send("#{key}=", value) }
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/engine.rb

@@ -0,0 +1,5 @@
+module NulogyMessageBusConsumer
+  class Engine < ::Rails::Engine
+    isolate_namespace NulogyMessageBusConsumer
+  end
+end

data/lib/nulogy_message_bus_consumer/message.rb

@@ -0,0 +1,21 @@
+module NulogyMessageBusConsumer
+  Message = Struct.new(
+    :id,
+    :public_subscription_id,
+    :tenant_id,
+    :event_data,
+    :raw_event_data,
+    keyword_init: true
+  ) do
+    def self.from_kafka(kafka_message)
+      envelope_data = JSON.parse(kafka_message.payload)
+
+      new(
+        id: envelope_data["id"],
+        public_subscription_id: envelope_data["public_subscription_id"],
+        tenant_id: envelope_data["tenant_id"],
+        event_data: envelope_data["event_json"]
+      )
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/processed_message.rb

@@ -0,0 +1,5 @@
+module NulogyMessageBusConsumer
+  class ProcessedMessage < ActiveRecord::Base
+    self.table_name = "message_bus_processed_messages"
+  end
+end

data/lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb

@@ -0,0 +1,30 @@
+module NulogyMessageBusConsumer
+  module Steps
+    class ConnectToMessageBus
+      def initialize(config, logger)
+        @config = config
+        @logger = logger
+      end
+
+      def call(**_)
+        @logger.info("Connecting to the MessageBus")
+        consumer = Rdkafka::Config.new(consumer_config).consumer
+        @logger.info("Using consumer group id: #{@config.consumer_group_id}")
+
+        trap("TERM") { consumer.close }
+
+        yield(kafka_consumer: consumer)
+      end
+
+      private
+
+      def consumer_config
+        {
+          "bootstrap.servers": @config.bootstrap_servers,
+          "enable.auto.commit": false,
+          "group.id": @config.consumer_group_id,
+        }
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/steps/deduplicate_messages.rb

@@ -0,0 +1,49 @@
+module NulogyMessageBusConsumer
+  module Steps
+    class DeduplicateMessages
+      def initialize(logger)
+        @logger = logger
+      end
+
+      def call(message:, **_)
+        return :success if duplicate_exists?(message)
+
+        result = :unknown
+        ProcessedMessage.transaction(joinable: false) do
+          result = yield
+
+          record_processed_message(message) if result == :success
+        end
+
+        result
+      end
+
+      private
+
+      def duplicate_exists?(message)
+        if ProcessedMessage.exists?(id: message.id)
+          log_duplicate(message)
+
+          return true
+        end
+
+        false
+      end
+
+      def record_processed_message(message)
+        ProcessedMessage.create!(id: message.id)
+      rescue ActiveRecord::RecordNotUnique
+        log_duplicate(message)
+
+        raise ActiveRecord::Rollback
+      end
+
+      def log_duplicate(message)
+        @logger.warn(JSON.dump({
+          event: "duplicate_message_detected",
+          kafka_message_id: message.id,
+        }))
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/steps/log_messages.rb

@@ -0,0 +1,28 @@
+module NulogyMessageBusConsumer
+  module Steps
+    class LogMessages
+      def initialize(logger)
+        @logger = logger
+      end
+
+      def call(message:, **_)
+        @logger.info(JSON.dump({
+          event: "message_received",
+          kafka_message_id: message.id,
+          message: "Received #{message.id}",
+        }))
+
+        result = yield
+
+        @logger.info(JSON.dump({
+          event: "message_processed",
+          kafka_message_id: message.id,
+          message: "Processed #{message.id}",
+          result: result,
+        }))
+
+        result
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/steps/monitor_replication_lag.rb

@@ -0,0 +1,51 @@
+module NulogyMessageBusConsumer
+  module Steps
+    class MonitorReplicationLag
+      def initialize(logger)
+        @logger = logger
+      end
+
+      def call(kafka_consumer:, **_)
+        # Ensure that the process is terminated if there is a problem getting the consumption lag.
+        # This also ensures that the process will terminate on-boot if it cannot connect to Kafka,
+        # allowing the container to be terminated by ECS.
+        Thread.abort_on_exception = true
+
+        Thread.new do
+          # Delayed start. If we attempt to read consumer#committed immediately, it may fail.
+          # We suspect this is because the consumer#committed is called before the consumer
+          # has finished connecting. There appears to be a race condition.
+          sleep 30
+
+          loop do
+            lag_per_topic = kafka_consumer.lag(kafka_consumer.committed)
+
+            @logger.info(JSON.dump({
+              event: "consumer_lag",
+              topics: Calculator.add_max_lag(lag_per_topic),
+            }))
+            STDOUT.flush
+
+            sleep 60
+          end
+        end
+
+        yield
+      end
+
+      module Calculator
+        def self.add_max_lag(lag_by_topic)
+          lag_by_topic.each_value do |lag_by_partition|
+            lag_by_partition[:_max] = lag_by_partition.values.max || 0
+          end
+
+          lag_by_topic[:_max] = lag_by_topic
+            .map { |_topic, lag_by_partition| lag_by_partition[:_max] }
+            .max || 0
+
+          lag_by_topic
+        end
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/steps/result_validation.rb

@@ -0,0 +1,16 @@
+module NulogyMessageBusConsumer
+  module Steps
+    class ResultValidation
+      def call(**_)
+        result = yield
+
+        return result if %i[success failure].include?(result)
+
+        raise(
+          StandardError,
+          "'#{result}' is not a valid processing outcome. Must be :success or :failure"
+        )
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/steps/stream_individual_messages.rb

@@ -0,0 +1,34 @@
+module NulogyMessageBusConsumer
+  module Steps
+    class StreamIndividualMessages
+      def initialize(config, logger)
+        @config = config
+        @logger = logger
+      end
+
+      def call(kafka_consumer:, **_)
+        kafka_consumer.subscribe(@config.topic_name)
+        @logger.info "Listening for kafka messages on topic #{@config.topic_name}"
+
+        kafka_consumer.each do |kafka_message|
+          result = yield(
+            message: NulogyMessageBusConsumer::Message.from_kafka(kafka_message),
+            kafka_message: kafka_message
+          )
+
+          kafka_consumer.commit if result == :success
+        end
+      rescue StandardError => e
+        @logger.error(JSON.dump({
+          event: "message_processing_errored",
+          class: e.class,
+          message: e.message,
+        }))
+
+        kafka_consumer.unsubscribe
+
+        raise
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/version.rb

@@ -0,0 +1,3 @@
+module NulogyMessageBusConsumer
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,170 @@
+--- !ruby/object:Gem::Specification
+name: nulogy_message_bus_consumer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Nulogy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-06-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 6.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 6.0.3
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.9.0
+- !ruby/object:Gem::Dependency
+  name: rspec-json_expectations
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.81.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.81.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.38.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.38.1
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+description: 
+email:
+- tass@nulogy.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Rakefile
+- config/routes.rb
+- db/migrate/20200509095105_create_message_bus_processed_messages.rb
+- lib/nulogy_message_bus_consumer.rb
+- lib/nulogy_message_bus_consumer/config.rb
+- lib/nulogy_message_bus_consumer/engine.rb
+- lib/nulogy_message_bus_consumer/message.rb
+- lib/nulogy_message_bus_consumer/processed_message.rb
+- lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb
+- lib/nulogy_message_bus_consumer/steps/deduplicate_messages.rb
+- lib/nulogy_message_bus_consumer/steps/log_messages.rb
+- lib/nulogy_message_bus_consumer/steps/monitor_replication_lag.rb
+- lib/nulogy_message_bus_consumer/steps/result_validation.rb
+- lib/nulogy_message_bus_consumer/steps/stream_individual_messages.rb
+- lib/nulogy_message_bus_consumer/version.rb
+homepage: https://github.com/nulogy/message-bus/tree/master/gems/nulogy_message_bus_consumer
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org/
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Code for accessing the Nulogy Message Bus
+test_files: []