nulogy_message_bus_consumer 0.3.0 → 0.5.0

data/lib/nulogy_message_bus_consumer/steps/stream_messages.rb

@@ -12,11 +12,11 @@ module NulogyMessageBusConsumer
             kafka_message: kafka_message
           )
         end
-      rescue StandardError => e
+      rescue => e
         @logger.error(JSON.dump({
           event: "message_processing_errored",
           class: e.class,
-          message: e.message,
+          message: e.message
         }))
 
         raise

data/lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb

@@ -12,11 +12,11 @@ module NulogyMessageBusConsumer
             kafka_message: kafka_message
           )
         end
-      rescue StandardError => e
+      rescue => e
         @logger.error(JSON.dump({
           event: "message_processing_errored",
           class: e.class,
-          message: e.message,
+          message: e.message
         }))
 
         raise

data/lib/nulogy_message_bus_consumer/steps/supervise_consumer_lag.rb

@@ -0,0 +1,76 @@
+module NulogyMessageBusConsumer
+  module Steps
+    # Supervises the consumer's lag.
+    #
+    # If a partition's lag is non-zero and does not change for an extended period
+    # of time, then kill the main thread.
+    #
+    # That period of time is check_interval_seconds * LagTracker#failing_checks
+    # With the defaults, that would be 20 * 6 ~ 120 seconds = 2 minutes.
+    #
+    # Note that this strategy may not work for a busy integration.
+    # Consumer lag monitoring should alert in that case.
+    # However, this strategy may help alleviate alerts for low traffic or off-peak
+    # environments.
+    #
+    # We've come across cases where the consumer lag is still being logged,
+    # messages are being processed, but the consumer is not consuming messages
+    # in particular topics.
+    #
+    # Killing the main thread causes ECS to restart the task.
+    class SuperviseConsumerLag
+      def initialize(logger, tracker: NulogyMessageBusConsumer::LagTracker.new(failing_checks: 6), killable: nil, check_interval_seconds: 20)
+        @logger = logger
+        @tracker = tracker
+        @killable = killable
+        @check_interval_seconds = check_interval_seconds
+      end
+
+      def call(kafka_consumer:, **_)
+        @consumer = kafka_consumer
+        @killable ||= Thread.current
+
+        run
+
+        yield
+      end
+
+      private
+
+      def run
+        Thread.abort_on_exception = true
+
+        Thread.new do
+          NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(@consumer)
+
+          loop do
+            @tracker.update(@consumer.lag(@consumer.committed))
+
+            if @tracker.failing?
+              log_failed_partitions
+
+              @killable.kill
+              Thread.current.exit
+            end
+
+            sleep @check_interval_seconds
+          end
+        end
+      end
+
+      def log_failed_partitions
+        seconds = @check_interval_seconds * @tracker.failing_checks
+        failed = @tracker
+          .failed
+          .map { |topic, partitions| "#{topic}: #{partitions.join(",")}" }
+          .join(", ")
+
+        @logger.warn(JSON.dump({
+          event: "message_processing_warning",
+          message: "Assigned partition lag has not changed in #{seconds} seconds: #{failed}"
+        }))
+        $stdout.flush
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/version.rb

@@ -1,3 +1,3 @@
 module NulogyMessageBusConsumer
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/lib/tasks/engine/message_bus_consumer.rake

@@ -1,16 +1,15 @@
 namespace :message_bus_consumer do
   desc "Verifies that the messages in the message bus have been processed"
-  task :audit => :environment do
-    config = build_audit_config
+  task audit: :environment do
+    logger = Rails.logger
+    config = NulogyMessageBusConsumer::Config.new(
+      consumer_group_id: ENV.fetch("MB_AUDIT_GROUP"),
+      bootstrap_servers: ENV.fetch("MB_BOOTSTRAP_SERVERS"),
+      topic_name: ENV.fetch("MB_CONSUMER_TOPIC")
+    )
 
     NulogyMessageBusConsumer
-      .consumer_audit_pipeline(config: config)
+      .consumer_audit_pipeline(config: config, logger: logger)
       .invoke
   end
-
-  def build_audit_config
-    audit_config = NulogyMessageBusConsumer.config.dup
-    audit_config.consumer_group_id = "#{audit_config.consumer_group_id}_consumer_audit"
-    NulogyMessageBusConsumer.config
-  end
-end
+end

data/spec/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require_relative "config/application"
+
+Rails.application.load_tasks

data/spec/dummy/app/assets/config/manifest.js

@@ -0,0 +1,3 @@
+//= link_tree ../images
+//= link_directory ../stylesheets .css
+//= link message_bus_manifest.js

data/spec/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/spec/dummy/app/channels/application_cable/channel.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Channel < ActionCable::Channel::Base
+  end
+end

data/spec/dummy/app/channels/application_cable/connection.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Connection < ActionCable::Connection::Base
+  end
+end

data/spec/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,2 @@
+class ApplicationController < ActionController::Base
+end

data/spec/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/spec/dummy/app/javascript/packs/application.js

@@ -0,0 +1,15 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require rails-ujs
+//= require activestorage
+//= require_tree .

data/spec/dummy/app/jobs/application_job.rb

@@ -0,0 +1,7 @@
+class ApplicationJob < ActiveJob::Base
+  # Automatically retry jobs that encountered a deadlock
+  # retry_on ActiveRecord::Deadlocked
+
+  # Most jobs are safe to ignore if the underlying records are no longer available
+  # discard_on ActiveJob::DeserializationError
+end

data/spec/dummy/app/mailers/application_mailer.rb

@@ -0,0 +1,4 @@
+class ApplicationMailer < ActionMailer::Base
+  default from: "from@example.com"
+  layout "mailer"
+end

data/spec/dummy/app/models/application_record.rb

@@ -0,0 +1,3 @@
+class ApplicationRecord < ActiveRecord::Base
+  self.abstract_class = true
+end

data/spec/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Dummy</title>
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+
+    <%= stylesheet_link_tag    'application', media: 'all' %>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/spec/dummy/app/views/layouts/mailer.html.erb

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <style>
+      /* Email styles need to be inline */
+    </style>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/spec/dummy/app/views/layouts/mailer.text.erb

@@ -0,0 +1 @@
+<%= yield %>

data/spec/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path("../config/application", __dir__)
+require_relative "../config/boot"
+require "rails/commands"

data/spec/dummy/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative "../config/boot"
+require "rake"
+Rake.application.run

data/spec/dummy/bin/setup

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+require "fileutils"
+
+# path to your application root.
+APP_ROOT = File.expand_path("..", __dir__)
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+FileUtils.chdir APP_ROOT do
+  # This script is a way to setup or update your development environment automatically.
+  # This script is idempotent, so that you can run it at anytime and get an expectable outcome.
+  # Add necessary setup steps to this file.
+
+  puts "== Installing dependencies =="
+  system! "gem install bundler --conservative"
+  system("bundle check") || system!("bundle install")
+
+  # puts "\n== Copying sample files =="
+  # unless File.exist?('config/database.yml')
+  #   FileUtils.cp 'config/database.yml.sample', 'config/database.yml'
+  # end
+
+  puts "\n== Preparing database =="
+  system! "bin/rails db:prepare"
+
+  puts "\n== Removing old logs and tempfiles =="
+  system! "bin/rails log:clear tmp:clear"
+
+  puts "\n== Restarting application server =="
+  system! "bin/rails restart"
+end

data/spec/dummy/config.ru

@@ -0,0 +1,5 @@
+# This file is used by Rack-based servers to start the application.
+
+require_relative "config/environment"
+
+run Rails.application

data/spec/dummy/config/application.rb

@@ -0,0 +1,29 @@
+require_relative "boot"
+
+require "rails"
+# Pick the frameworks you want:
+require "active_model/railtie"
+require "active_job/railtie"
+require "active_record/railtie"
+require "active_storage/engine"
+require "action_controller/railtie"
+require "action_mailer/railtie"
+require "action_view/railtie"
+require "action_cable/engine"
+require "sprockets/railtie"
+# require "rails/test_unit/railtie"
+
+Bundler.require(*Rails.groups)
+require "nulogy_message_bus_consumer"
+
+module Dummy
+  class Application < Rails::Application
+    # Initialize configuration defaults for originally generated Rails version.
+    config.load_defaults 6.0
+
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration can go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded after loading
+    # the framework and any gems in your application.
+  end
+end

data/spec/dummy/config/boot.rb

@@ -0,0 +1,5 @@
+# Set up gems listed in the Gemfile.
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../Gemfile", __dir__)
+
+require "bundler/setup" if File.exist?(ENV["BUNDLE_GEMFILE"])
+$LOAD_PATH.unshift File.expand_path("../../../lib", __dir__)

data/spec/dummy/config/cable.yml

@@ -0,0 +1,10 @@
+development:
+  adapter: async
+
+test:
+  adapter: test
+
+production:
+  adapter: redis
+  url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
+  channel_prefix: dummy_production

data/spec/dummy/config/credentials/message-bus-us-east-1.key

@@ -0,0 +1 @@
+dfa19863b2709390893da4c2fb85579a

data/spec/dummy/config/credentials/message-bus-us-east-1.yml.enc

@@ -0,0 +1 @@
+../../../../config/credentials/message-bus-us-east-1.yml.enc

data/spec/dummy/config/database.yml

@@ -0,0 +1,27 @@
+# Copy this file to 'database.yml' and modify the copy as needed.
+
+defaults: &defaults
+  adapter: postgresql
+  username: nulogy
+  password: <%= ENV['DATABASE_PASS'] || "Nulogy4Ever" %>
+  host: <%= ENV['DATABASE_HOST'] || "localhost" %>
+  port: 5437
+  min_messages: warning
+  schema_search_path: "\"$user\", public, extensions"
+
+development: &development
+  <<: *defaults
+  database: <%= ENV['DEVELOPMENT_DATABASE'] || "message_bus_consumer_development" %>
+
+development_slave:
+  <<: *development
+
+# Warning: The database defined as 'test' will be erased and
+# re-generated from your development database when you run 'rake'.
+# Do not set this db to the same as development or production.
+test: &test
+  <<: *defaults
+  database: <%= ENV['TEST_DATABASE'] || "message_bus_consumer_test" %>
+
+test_slave:
+  <<: *test

data/spec/dummy/config/environment.rb

@@ -0,0 +1,5 @@
+# Load the Rails application.
+require_relative "application"
+
+# Initialize the Rails application.
+Rails.application.initialize!

data/spec/dummy/config/environments/development.rb

@@ -0,0 +1,62 @@
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # In the development environment your application's code is reloaded on
+  # every request. This slows down response time but is perfect for development
+  # since you don't have to restart the web server when you make code changes.
+  config.cache_classes = false
+
+  # Do not eager load code on boot.
+  config.eager_load = false
+
+  # Show full error reports.
+  config.consider_all_requests_local = true
+
+  # Enable/disable caching. By default caching is disabled.
+  # Run rails dev:cache to toggle caching.
+  if Rails.root.join("tmp/caching-dev.txt").exist?
+    config.action_controller.perform_caching = true
+    config.action_controller.enable_fragment_cache_logging = true
+
+    config.cache_store = :memory_store
+    config.public_file_server.headers = {
+      "Cache-Control" => "public, max-age=#{2.days.to_i}"
+    }
+  else
+    config.action_controller.perform_caching = false
+
+    config.cache_store = :null_store
+  end
+
+  # Store uploaded files on the local file system (see config/storage.yml for options).
+  config.active_storage.service = :local
+
+  # Don't care if the mailer can't send.
+  config.action_mailer.raise_delivery_errors = false
+
+  config.action_mailer.perform_caching = false
+
+  # Print deprecation notices to the Rails logger.
+  config.active_support.deprecation = :log
+
+  # Raise an error on page load if there are pending migrations.
+  config.active_record.migration_error = :page_load
+
+  # Highlight code that triggered database queries in logs.
+  config.active_record.verbose_query_logs = true
+
+  # Debug mode disables concatenation and preprocessing of assets.
+  # This option may cause significant delays in view rendering with a large
+  # number of complex assets.
+  config.assets.debug = true
+
+  # Suppress logger output for asset requests.
+  config.assets.quiet = true
+
+  # Raises error for missing translations.
+  # config.action_view.raise_on_missing_translations = true
+
+  # Use an evented file watcher to asynchronously detect changes in source code,
+  # routes, locales, etc. This feature depends on the listen gem.
+  # config.file_watcher = ActiveSupport::EventedFileUpdateChecker
+end