nulogy_message_bus_consumer 0.3.2 → 1.0.0

data/lib/nulogy_message_bus_consumer/tasks/log_consumer_lag.rb

@@ -0,0 +1,45 @@
+module NulogyMessageBusConsumer
+  module Tasks
+    class LogConsumerLag
+      attr_reader :interval
+
+      def initialize(logger, interval_seconds)
+        @logger = logger
+        @interval = interval_seconds
+      end
+
+      def extract_args(kafka_consumer:, **_)
+        @kafka_consumer = kafka_consumer
+      end
+
+      def call
+        # Delayed start. If we attempt to read consumer#committed immediately, it may fail.
+        # We suspect this is because the consumer#committed is called before the consumer
+        # has finished connecting. There appears to be a race condition.
+        KafkaUtils.wait_for_assignment(@kafka_consumer)
+
+        lag_per_topic = @kafka_consumer.lag(@kafka_consumer.committed)
+
+        @logger.info(JSON.dump({
+          event: "consumer_lag",
+          topics: Calculator.add_max_lag(lag_per_topic)
+        }))
+        $stdout.flush
+      end
+
+      module Calculator
+        def self.add_max_lag(lag_by_topic)
+          lag_by_topic.each_value do |lag_by_partition|
+            lag_by_partition[:_max] = lag_by_partition.values.max || 0
+          end
+
+          lag_by_topic[:_max] = lag_by_topic
+            .map { |_topic, lag_by_partition| lag_by_partition[:_max] }
+            .max || 0
+
+          lag_by_topic
+        end
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/tasks/prune_processed_messages.rb

@@ -0,0 +1,37 @@
+module NulogyMessageBusConsumer
+  module Tasks
+    # Deletes any ProcessedMessage rows that are older than max_age
+    #
+    # interval_seconds: how many seconds between invocations
+    # max_age: ActiveSupport::Duration of how old ProcessedMessage are
+    # removed. This calls `#ago` on it.
+    # For example, setting this to `1.week` will delete records more than
+    # `1.week.ago` on each invocation.
+    class PruneProcessedMessages
+      attr_reader :interval
+
+      def initialize(logger, interval_seconds, max_age)
+        @logger = logger
+        @interval = interval_seconds
+        @max_age = max_age
+      end
+
+      def extract_args(**_)
+      end
+
+      def call
+        deleted = prune_stale
+
+        @logger.info("Pruned #{deleted} processed messages")
+      end
+
+      private
+
+      def prune_stale
+        ProcessedMessage
+          .where(ProcessedMessage.arel_table[:created_at].lt(@max_age.ago))
+          .delete_all
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/tasks/supervise_consumer_lag.rb

@@ -0,0 +1,65 @@
+module NulogyMessageBusConsumer
+  module Tasks
+    # Supervises the consumer's lag.
+    #
+    # If a partition's lag is non-zero and does not change for an extended period
+    # of time, then kill the main thread.
+    #
+    # That period of time is check_interval_seconds * LagTracker#failing_checks
+    # With the defaults, that would be 20 * 6 ~ 120 seconds = 2 minutes.
+    #
+    # Note that this strategy may not work for a busy integration.
+    # Consumer lag monitoring should alert in that case.
+    # However, this strategy may help alleviate alerts for low traffic or off-peak
+    # environments.
+    #
+    # We've come across cases where the consumer lag is still being logged,
+    # messages are being processed, but the consumer is not consuming messages
+    # in particular topics.
+    #
+    # Killing the main thread causes ECS to restart the task.
+    class SuperviseConsumerLag
+      attr_reader :interval
+
+      def initialize(logger, tracker: NulogyMessageBusConsumer::LagTracker.new(failing_checks: 6), killable: nil, check_interval_seconds: 20)
+        @logger = logger
+        @tracker = tracker
+        @killable = killable
+        @interval = check_interval_seconds
+      end
+
+      def extract_args(kafka_consumer:, **_)
+        @consumer = kafka_consumer
+        @killable ||= Thread.current
+      end
+
+      def call
+        NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(@consumer)
+
+        @tracker.update(@consumer.lag(@consumer.committed))
+        if @tracker.failing?
+          log_failed_partitions
+
+          @killable.kill
+          Thread.current.exit
+        end
+      end
+
+      private
+
+      def log_failed_partitions
+        seconds = @interval * @tracker.failing_checks
+        failed = @tracker
+          .failed
+          .map { |topic, partitions| "#{topic}: #{partitions.join(",")}" }
+          .join(", ")
+
+        @logger.warn(JSON.dump({
+          event: "message_processing_warning",
+          message: "Assigned partition lag has not changed in #{seconds} seconds: #{failed}"
+        }))
+        $stdout.flush
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/version.rb

@@ -1,3 +1,3 @@
 module NulogyMessageBusConsumer
-  VERSION = "0.3.2"
+  VERSION = "1.0.0"
 end

data/lib/tasks/engine/message_bus_consumer.rake

@@ -1,16 +1,15 @@
 namespace :message_bus_consumer do
   desc "Verifies that the messages in the message bus have been processed"
   task audit: :environment do
-    config = build_audit_config
+    logger = Rails.logger
+    config = NulogyMessageBusConsumer::Config.new(
+      consumer_group_id: ENV.fetch("MB_AUDIT_GROUP"),
+      bootstrap_servers: ENV.fetch("MB_BOOTSTRAP_SERVERS"),
+      topic_name: ENV.fetch("MB_CONSUMER_TOPIC")
+    )
 
     NulogyMessageBusConsumer
-      .consumer_audit_pipeline(config: config)
+      .consumer_audit_pipeline(config: config, logger: logger)
       .invoke
   end
-
-  def build_audit_config
-    audit_config = NulogyMessageBusConsumer.config.dup
-    audit_config.consumer_group_id = "#{audit_config.consumer_group_id}_consumer_audit"
-    NulogyMessageBusConsumer.config
-  end
 end

data/spec/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require_relative "config/application"
+
+Rails.application.load_tasks

data/spec/dummy/app/assets/config/manifest.js

@@ -0,0 +1,3 @@
+//= link_tree ../images
+//= link_directory ../stylesheets .css
+//= link message_bus_manifest.js

data/spec/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/spec/dummy/app/channels/application_cable/channel.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Channel < ActionCable::Channel::Base
+  end
+end

data/spec/dummy/app/channels/application_cable/connection.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Connection < ActionCable::Connection::Base
+  end
+end

data/spec/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,2 @@
+class ApplicationController < ActionController::Base
+end

data/spec/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/spec/dummy/app/javascript/packs/application.js

@@ -0,0 +1,15 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require rails-ujs
+//= require activestorage
+//= require_tree .

data/spec/dummy/app/jobs/application_job.rb

@@ -0,0 +1,7 @@
+class ApplicationJob < ActiveJob::Base
+  # Automatically retry jobs that encountered a deadlock
+  # retry_on ActiveRecord::Deadlocked
+
+  # Most jobs are safe to ignore if the underlying records are no longer available
+  # discard_on ActiveJob::DeserializationError
+end

data/spec/dummy/app/mailers/application_mailer.rb

@@ -0,0 +1,4 @@
+class ApplicationMailer < ActionMailer::Base
+  default from: "from@example.com"
+  layout "mailer"
+end

data/spec/dummy/app/models/application_record.rb

@@ -0,0 +1,3 @@
+class ApplicationRecord < ActiveRecord::Base
+  self.abstract_class = true
+end

data/spec/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Dummy</title>
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+
+    <%= stylesheet_link_tag    'application', media: 'all' %>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/spec/dummy/app/views/layouts/mailer.html.erb

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <style>
+      /* Email styles need to be inline */
+    </style>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/spec/dummy/app/views/layouts/mailer.text.erb

@@ -0,0 +1 @@
+<%= yield %>

data/spec/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path("../config/application", __dir__)
+require_relative "../config/boot"
+require "rails/commands"

data/spec/dummy/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative "../config/boot"
+require "rake"
+Rake.application.run

data/spec/dummy/bin/setup

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+require "fileutils"
+
+# path to your application root.
+APP_ROOT = File.expand_path("..", __dir__)
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+FileUtils.chdir APP_ROOT do
+  # This script is a way to setup or update your development environment automatically.
+  # This script is idempotent, so that you can run it at anytime and get an expectable outcome.
+  # Add necessary setup steps to this file.
+
+  puts "== Installing dependencies =="
+  system! "gem install bundler --conservative"
+  system("bundle check") || system!("bundle install")
+
+  # puts "\n== Copying sample files =="
+  # unless File.exist?('config/database.yml')
+  #   FileUtils.cp 'config/database.yml.sample', 'config/database.yml'
+  # end
+
+  puts "\n== Preparing database =="
+  system! "bin/rails db:prepare"
+
+  puts "\n== Removing old logs and tempfiles =="
+  system! "bin/rails log:clear tmp:clear"
+
+  puts "\n== Restarting application server =="
+  system! "bin/rails restart"
+end

data/spec/dummy/config.ru

@@ -0,0 +1,5 @@
+# This file is used by Rack-based servers to start the application.
+
+require_relative "config/environment"
+
+run Rails.application

data/spec/dummy/config/application.rb

@@ -0,0 +1,29 @@
+require_relative "boot"
+
+require "rails"
+# Pick the frameworks you want:
+require "active_model/railtie"
+require "active_job/railtie"
+require "active_record/railtie"
+require "active_storage/engine"
+require "action_controller/railtie"
+require "action_mailer/railtie"
+require "action_view/railtie"
+require "action_cable/engine"
+require "sprockets/railtie"
+# require "rails/test_unit/railtie"
+
+Bundler.require(*Rails.groups)
+require "nulogy_message_bus_consumer"
+
+module Dummy
+  class Application < Rails::Application
+    # Initialize configuration defaults for originally generated Rails version.
+    config.load_defaults 6.0
+
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration can go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded after loading
+    # the framework and any gems in your application.
+  end
+end

data/spec/dummy/config/boot.rb

@@ -0,0 +1,5 @@
+# Set up gems listed in the Gemfile.
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../Gemfile", __dir__)
+
+require "bundler/setup" if File.exist?(ENV["BUNDLE_GEMFILE"])
+$LOAD_PATH.unshift File.expand_path("../../../lib", __dir__)

data/spec/dummy/config/cable.yml

@@ -0,0 +1,10 @@
+development:
+  adapter: async
+
+test:
+  adapter: test
+
+production:
+  adapter: redis
+  url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
+  channel_prefix: dummy_production

data/spec/dummy/config/credentials/message-bus-us-east-1.key

@@ -0,0 +1 @@
+dfa19863b2709390893da4c2fb85579a

data/spec/dummy/config/credentials/message-bus-us-east-1.yml.enc

@@ -0,0 +1 @@
+../../../../config/credentials/message-bus-us-east-1.yml.enc

data/spec/dummy/config/database.yml

@@ -0,0 +1,27 @@
+# Copy this file to 'database.yml' and modify the copy as needed.
+
+defaults: &defaults
+  adapter: postgresql
+  username: nulogy
+  password: <%= ENV['DATABASE_PASS'] || "Nulogy4Ever" %>
+  host: <%= ENV['DATABASE_HOST'] || "localhost" %>
+  port: 5437
+  min_messages: warning
+  schema_search_path: "\"$user\", public, extensions"
+
+development: &development
+  <<: *defaults
+  database: <%= ENV['DEVELOPMENT_DATABASE'] || "message_bus_consumer_development" %>
+
+development_slave:
+  <<: *development
+
+# Warning: The database defined as 'test' will be erased and
+# re-generated from your development database when you run 'rake'.
+# Do not set this db to the same as development or production.
+test: &test
+  <<: *defaults
+  database: <%= ENV['TEST_DATABASE'] || "message_bus_consumer_test" %>
+
+test_slave:
+  <<: *test