nulogy_message_bus_consumer 1.0.0.alpha → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d17557a978d8762a83afcb1a8027296854d516cde57540243d48c80cc46c012
-  data.tar.gz: 02c45bfab242f35a0143d090a08387f80db0030854a28b161792cb09efded53c
+  metadata.gz: bc67ef0e9ad8a12ee285d6bf7898e262d5af7140ddfc50ce1265086f313dab5d
+  data.tar.gz: 968ef8091b7ba117dad2a90f265a9e89844f1604404449d703868f343f41c35d
 SHA512:
-  metadata.gz: 8c7ef5e49a3aff7e09e963592fc79f7435a29d3567764d0cec1e98d52fea31d18f1ee2ea4b3d429c2e550307f0fab539733d700716b89b45c05cfee269e87955
-  data.tar.gz: 0be71a225c18e458d440f80eefadfb5b1a841ad57d619af434e12832c48ff31c1ce2614aaf5786330e2282e418838a223f91c82ebae74cedda3801446620fbb2
+  metadata.gz: 7def0d572755c4b66d8f3ca603fadc64dd3aa159a3a6e868ec38efa589d08d50bdd9ba187cb89ef8773fbe02a5616dea705b6366b4c7371bec3e3979788fca39
+  data.tar.gz: 89b43ff862329c5710f5441a765e20924d0411a0f012905d2e697fb6029b3f9df716b9704d290e37e357bbd4f196d30dd86774ae25bcc017cb6087ad1a2e7845

data/Rakefile

@@ -23,9 +23,3 @@ require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 require "standard/rake"
 task default: %i[spec standard]
-
-require "rake/release"
-
-Rake::Release::Task.load_all do |spec|
-  spec.version_tag = "nulogy_message_bus_consumer-v#{spec.version}"
-end

data/lib/nulogy_message_bus_consumer/config.rb

@@ -5,12 +5,20 @@ module NulogyMessageBusConsumer
       :consumer_group_id,
       :lag_check_interval_seconds,
       :lag_checks,
+      :lag_timeout_milliseconds,
+      :log_lag_interval_seconds,
+      :prune_interval_seconds,
+      :prune_max_age,
       :topic_name
 
     def initialize(options = {})
       defaults = {
         lag_check_interval_seconds: 20,
-        lag_checks: 6
+        lag_checks: 6,
+        lag_timeout_milliseconds: 200,
+        log_lag_interval_seconds: 1.minute.to_i,
+        prune_interval_seconds: 1.hour.to_i,
+        prune_max_age: 8.days
       }
 
       update(defaults.merge(options))

data/lib/nulogy_message_bus_consumer/message.rb

@@ -1,17 +1,16 @@
 module NulogyMessageBusConsumer
   class Message
-    attr_reader :company_uuid,
-      :created_at,
-      :event_data,
-      :event_data_unparsed,
-      :id,
-      :key,
-      :offset,
-      :partition,
-      :subscription_id,
-      :timestamp,
-      :topic,
-      :type
+    attr_reader :event_data
+    attr_reader :event_data_unparsed
+    attr_reader :id
+    attr_reader :key
+    attr_reader :offset
+    attr_reader :partition
+    attr_reader :subscription_id
+    attr_reader :company_uuid
+    attr_reader :timestamp
+    attr_reader :topic
+    attr_reader :created_at
 
     def initialize(attrs = {})
       attrs.each { |key, value| instance_variable_set("@#{key}", value) }
@@ -37,7 +36,6 @@ module NulogyMessageBusConsumer
         company_uuid: envelope_data[:company_uuid] || envelope_data[:tenant_id],
         timestamp: kafka_message.timestamp,
         topic: kafka_message.topic,
-        type: envelope_data[:type],
         created_at: envelope_data[:created_at]
       )
     end

data/lib/nulogy_message_bus_consumer/steps/timed_task.rb

@@ -0,0 +1,42 @@
+module NulogyMessageBusConsumer
+  module Steps
+    # A generic class to run a "task" on a timer (in a separate thread!)
+    # This class runs the code, the Task does the work
+    #
+    # A Task must implement the methods called in this class:
+    # - #extract_args(kwargs)
+    #   Called with the keyword arguments (kwargs) that is passed to this step.
+    #   This is a chance to pull out references to pipeline variables (e.g. kafka_consumer)
+    # - #call
+    #   The work the Task should perform
+    # - #interval
+    #   The time, in seconds, between invocations of #call
+    class TimedTask
+      def initialize(task)
+        @task = task
+      end
+
+      def call(**kwargs)
+        @task.extract_args(**kwargs)
+
+        # Ensure that the process is terminated if there is a problem getting the consumption lag.
+        # This also ensures that the process will terminate on-boot if it cannot connect to Kafka,
+        # allowing the container to be terminated by ECS.
+        Thread.abort_on_exception = true
+        Thread.new { run }
+
+        yield
+      end
+
+      private
+
+      def run
+        loop do
+          @task.call
+
+          sleep @task.interval
+        end
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/tasks/log_consumer_lag.rb

@@ -0,0 +1,51 @@
+module NulogyMessageBusConsumer
+  module Tasks
+    class LogConsumerLag
+      attr_reader :interval
+
+      def initialize(logger, interval_seconds, lag_timeout)
+        @logger = logger
+        @interval = interval_seconds
+        @lag_timeout = lag_timeout
+      end
+
+      def extract_args(kafka_consumer:, **_)
+        @kafka_consumer = kafka_consumer
+      end
+
+      def call
+        # Delayed start. If we attempt to read consumer#committed immediately, it may fail.
+        # We suspect this is because the consumer#committed is called before the consumer
+        # has finished connecting. There appears to be a race condition.
+        KafkaUtils.wait_for_assignment(@kafka_consumer)
+
+        # Note: consumer#committed has a timeout of 1200ms. To respect our lag_timeout, use the largest.
+        committed_timeout = [1200, @lag_timeout].max
+        # The first parameter is a TopicPartitionList. When nil, it uses all the assigned ones.
+        committed_offsets = @kafka_consumer.committed(nil, committed_timeout)
+
+        lag_per_topic = @kafka_consumer.lag(committed_offsets, @lag_timeout)
+
+        @logger.info(JSON.dump({
+          event: "consumer_lag",
+          topics: Calculator.add_max_lag(lag_per_topic)
+        }))
+        $stdout.flush
+      end
+
+      module Calculator
+        def self.add_max_lag(lag_by_topic)
+          lag_by_topic.each_value do |lag_by_partition|
+            lag_by_partition[:_max] = lag_by_partition.values.max || 0
+          end
+
+          lag_by_topic[:_max] = lag_by_topic
+            .map { |_topic, lag_by_partition| lag_by_partition[:_max] }
+            .max || 0
+
+          lag_by_topic
+        end
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/tasks/prune_processed_messages.rb

@@ -0,0 +1,37 @@
+module NulogyMessageBusConsumer
+  module Tasks
+    # Deletes any ProcessedMessage rows that are older than max_age
+    #
+    # interval_seconds: how many seconds between invocations
+    # max_age: ActiveSupport::Duration of how old ProcessedMessage are
+    # removed. This calls `#ago` on it.
+    # For example, setting this to `1.week` will delete records more than
+    # `1.week.ago` on each invocation.
+    class PruneProcessedMessages
+      attr_reader :interval
+
+      def initialize(logger, interval_seconds, max_age)
+        @logger = logger
+        @interval = interval_seconds
+        @max_age = max_age
+      end
+
+      def extract_args(**_)
+      end
+
+      def call
+        deleted = prune_stale
+
+        @logger.info("Pruned #{deleted} processed messages")
+      end
+
+      private
+
+      def prune_stale
+        ProcessedMessage
+          .where(ProcessedMessage.arel_table[:created_at].lt(@max_age.ago))
+          .delete_all
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/{steps → tasks}/supervise_consumer_lag.rb

@@ -1,5 +1,5 @@
 module NulogyMessageBusConsumer
-  module Steps
+  module Tasks
     # Supervises the consumer's lag.
     #
     # If a partition's lag is non-zero and does not change for an extended period
@@ -19,47 +19,36 @@ module NulogyMessageBusConsumer
     #
     # Killing the main thread causes ECS to restart the task.
     class SuperviseConsumerLag
+      attr_reader :interval
+
       def initialize(logger, tracker: NulogyMessageBusConsumer::LagTracker.new(failing_checks: 6), killable: nil, check_interval_seconds: 20)
         @logger = logger
         @tracker = tracker
         @killable = killable
-        @check_interval_seconds = check_interval_seconds
+        @interval = check_interval_seconds
       end
 
-      def call(kafka_consumer:, **_)
+      def extract_args(kafka_consumer:, **_)
         @consumer = kafka_consumer
         @killable ||= Thread.current
-
-        run
-
-        yield
       end
 
-      private
-
-      def run
-        Thread.abort_on_exception = true
+      def call
+        NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(@consumer)
 
-        Thread.new do
-          NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(@consumer)
+        @tracker.update(@consumer.lag(@consumer.committed))
+        if @tracker.failing?
+          log_failed_partitions
 
-          loop do
-            @tracker.update(@consumer.lag(@consumer.committed))
-
-            if @tracker.failing?
-              log_failed_partitions
-
-              @killable.kill
-              Thread.current.exit
-            end
-
-            sleep @check_interval_seconds
-          end
+          @killable.kill
+          Thread.current.exit
         end
       end
 
+      private
+
       def log_failed_partitions
-        seconds = @check_interval_seconds * @tracker.failing_checks
+        seconds = @interval * @tracker.failing_checks
         failed = @tracker
           .failed
           .map { |topic, partitions| "#{topic}: #{partitions.join(",")}" }

data/lib/nulogy_message_bus_consumer/version.rb

@@ -1,3 +1,3 @@
 module NulogyMessageBusConsumer
-  VERSION = "1.0.0.alpha".freeze
+  VERSION = "2.0.0"
 end

data/lib/nulogy_message_bus_consumer.rb

@@ -16,12 +16,14 @@ require "nulogy_message_bus_consumer/processed_message"
 require "nulogy_message_bus_consumer/steps/commit_on_success"
 require "nulogy_message_bus_consumer/steps/connect_to_message_bus"
 require "nulogy_message_bus_consumer/steps/deduplicate_messages"
-require "nulogy_message_bus_consumer/steps/log_consumer_lag"
 require "nulogy_message_bus_consumer/steps/log_messages"
 require "nulogy_message_bus_consumer/steps/seek_beginning_of_topic"
 require "nulogy_message_bus_consumer/steps/stream_messages"
 require "nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left"
-require "nulogy_message_bus_consumer/steps/supervise_consumer_lag"
+require "nulogy_message_bus_consumer/steps/timed_task"
+require "nulogy_message_bus_consumer/tasks/log_consumer_lag"
+require "nulogy_message_bus_consumer/tasks/prune_processed_messages"
+require "nulogy_message_bus_consumer/tasks/supervise_consumer_lag"
 
 module NulogyMessageBusConsumer
   module_function
@@ -49,11 +51,18 @@ module NulogyMessageBusConsumer
       # Note: that since they are before `StreamMessages`, they will only
       # be called once, without any messages.
       Steps::ConnectToMessageBus.new(config, logger),
-      Steps::LogConsumerLag.new(logger),
-      Steps::SuperviseConsumerLag.new(
-        logger,
-        check_interval_seconds: config.lag_check_interval_seconds,
-        tracker: LagTracker.new(failing_checks: config.lag_checks)
+      Steps::TimedTask.new(
+        Tasks::LogConsumerLag.new(logger, config.log_lag_interval_seconds, config.lag_timeout_milliseconds)
+      ),
+      Steps::TimedTask.new(
+        Tasks::PruneProcessedMessages.new(logger, config.prune_interval_seconds, config.prune_max_age)
+      ),
+      Steps::TimedTask.new(
+        Tasks::SuperviseConsumerLag.new(
+          logger,
+          check_interval_seconds: config.lag_check_interval_seconds,
+          tracker: LagTracker.new(failing_checks: config.lag_checks)
+        )
       ),
       Steps::StreamMessages.new(logger),
       # Message processing steps start here.

data/spec/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require_relative "config/application"
+
+Rails.application.load_tasks

data/spec/dummy/app/assets/config/manifest.js

@@ -0,0 +1,3 @@
+//= link_tree ../images
+//= link_directory ../stylesheets .css
+//= link message_bus_manifest.js

data/spec/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/spec/dummy/app/channels/application_cable/channel.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Channel < ActionCable::Channel::Base
+  end
+end

data/spec/dummy/app/channels/application_cable/connection.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Connection < ActionCable::Connection::Base
+  end
+end

data/spec/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,2 @@
+class ApplicationController < ActionController::Base
+end

data/spec/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/spec/dummy/app/javascript/packs/application.js

@@ -0,0 +1,15 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require rails-ujs
+//= require activestorage
+//= require_tree .

data/spec/dummy/app/jobs/application_job.rb

@@ -0,0 +1,7 @@
+class ApplicationJob < ActiveJob::Base
+  # Automatically retry jobs that encountered a deadlock
+  # retry_on ActiveRecord::Deadlocked
+
+  # Most jobs are safe to ignore if the underlying records are no longer available
+  # discard_on ActiveJob::DeserializationError
+end

data/spec/dummy/app/mailers/application_mailer.rb

@@ -0,0 +1,4 @@
+class ApplicationMailer < ActionMailer::Base
+  default from: "from@example.com"
+  layout "mailer"
+end

data/spec/dummy/app/models/application_record.rb

@@ -0,0 +1,3 @@
+class ApplicationRecord < ActiveRecord::Base
+  self.abstract_class = true
+end

data/spec/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Dummy</title>
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+
+    <%= stylesheet_link_tag    'application', media: 'all' %>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/spec/dummy/app/views/layouts/mailer.html.erb

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <style>
+      /* Email styles need to be inline */
+    </style>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/spec/dummy/app/views/layouts/mailer.text.erb

@@ -0,0 +1 @@
+<%= yield %>

data/spec/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path("../config/application", __dir__)
+require_relative "../config/boot"
+require "rails/commands"

data/spec/dummy/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative "../config/boot"
+require "rake"
+Rake.application.run

data/spec/dummy/bin/setup

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+require "fileutils"
+
+# path to your application root.
+APP_ROOT = File.expand_path("..", __dir__)
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+FileUtils.chdir APP_ROOT do
+  # This script is a way to setup or update your development environment automatically.
+  # This script is idempotent, so that you can run it at anytime and get an expectable outcome.
+  # Add necessary setup steps to this file.
+
+  puts "== Installing dependencies =="
+  system! "gem install bundler --conservative"
+  system("bundle check") || system!("bundle install")
+
+  # puts "\n== Copying sample files =="
+  # unless File.exist?('config/database.yml')
+  #   FileUtils.cp 'config/database.yml.sample', 'config/database.yml'
+  # end
+
+  puts "\n== Preparing database =="
+  system! "bin/rails db:prepare"
+
+  puts "\n== Removing old logs and tempfiles =="
+  system! "bin/rails log:clear tmp:clear"
+
+  puts "\n== Restarting application server =="
+  system! "bin/rails restart"
+end

data/spec/dummy/config/application.rb

@@ -0,0 +1,29 @@
+require_relative "boot"
+
+require "rails"
+# Pick the frameworks you want:
+require "active_model/railtie"
+require "active_job/railtie"
+require "active_record/railtie"
+require "active_storage/engine"
+require "action_controller/railtie"
+require "action_mailer/railtie"
+require "action_view/railtie"
+require "action_cable/engine"
+require "sprockets/railtie"
+# require "rails/test_unit/railtie"
+
+Bundler.require(*Rails.groups)
+require "nulogy_message_bus_consumer"
+
+module Dummy
+  class Application < Rails::Application
+    # Initialize configuration defaults for originally generated Rails version.
+    config.load_defaults 6.0
+
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration can go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded after loading
+    # the framework and any gems in your application.
+  end
+end

data/spec/dummy/config/boot.rb

@@ -0,0 +1,5 @@
+# Set up gems listed in the Gemfile.
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../Gemfile", __dir__)
+
+require "bundler/setup" if File.exist?(ENV["BUNDLE_GEMFILE"])
+$LOAD_PATH.unshift File.expand_path("../../../lib", __dir__)

data/spec/dummy/config/cable.yml

@@ -0,0 +1,10 @@
+development:
+  adapter: async
+
+test:
+  adapter: test
+
+production:
+  adapter: redis
+  url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
+  channel_prefix: dummy_production

data/spec/dummy/config/credentials/message-bus-us-east-1.yml.enc

@@ -0,0 +1 @@
+../../../../config/credentials/message-bus-us-east-1.yml.enc

data/spec/dummy/config/database.yml

@@ -0,0 +1,27 @@
+# Copy this file to 'database.yml' and modify the copy as needed.
+
+defaults: &defaults
+  adapter: postgresql
+  username: nulogy
+  password: <%= ENV['DATABASE_PASS'] || "Nulogy4Ever" %>
+  host: <%= ENV['DATABASE_HOST'] || "localhost" %>
+  port: 5437
+  min_messages: warning
+  schema_search_path: "\"$user\", public, extensions"
+
+development: &development
+  <<: *defaults
+  database: <%= ENV['DEVELOPMENT_DATABASE'] || "message_bus_consumer_development" %>
+
+development_slave:
+  <<: *development
+
+# Warning: The database defined as 'test' will be erased and
+# re-generated from your development database when you run 'rake'.
+# Do not set this db to the same as development or production.
+test: &test
+  <<: *defaults
+  database: <%= ENV['TEST_DATABASE'] || "message_bus_consumer_test" %>
+
+test_slave:
+  <<: *test

data/spec/dummy/config/environment.rb

@@ -0,0 +1,5 @@
+# Load the Rails application.
+require_relative "application"
+
+# Initialize the Rails application.
+Rails.application.initialize!

data/spec/dummy/config/environments/development.rb

@@ -0,0 +1,62 @@
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # In the development environment your application's code is reloaded on
+  # every request. This slows down response time but is perfect for development
+  # since you don't have to restart the web server when you make code changes.
+  config.cache_classes = false
+
+  # Do not eager load code on boot.
+  config.eager_load = false
+
+  # Show full error reports.
+  config.consider_all_requests_local = true
+
+  # Enable/disable caching. By default caching is disabled.
+  # Run rails dev:cache to toggle caching.
+  if Rails.root.join("tmp/caching-dev.txt").exist?
+    config.action_controller.perform_caching = true
+    config.action_controller.enable_fragment_cache_logging = true
+
+    config.cache_store = :memory_store
+    config.public_file_server.headers = {
+      "Cache-Control" => "public, max-age=#{2.days.to_i}"
+    }
+  else
+    config.action_controller.perform_caching = false
+
+    config.cache_store = :null_store
+  end
+
+  # Store uploaded files on the local file system (see config/storage.yml for options).
+  config.active_storage.service = :local
+
+  # Don't care if the mailer can't send.
+  config.action_mailer.raise_delivery_errors = false
+
+  config.action_mailer.perform_caching = false
+
+  # Print deprecation notices to the Rails logger.
+  config.active_support.deprecation = :log
+
+  # Raise an error on page load if there are pending migrations.
+  config.active_record.migration_error = :page_load
+
+  # Highlight code that triggered database queries in logs.
+  config.active_record.verbose_query_logs = true
+
+  # Debug mode disables concatenation and preprocessing of assets.
+  # This option may cause significant delays in view rendering with a large
+  # number of complex assets.
+  config.assets.debug = true
+
+  # Suppress logger output for asset requests.
+  config.assets.quiet = true
+
+  # Raises error for missing translations.
+  # config.action_view.raise_on_missing_translations = true
+
+  # Use an evented file watcher to asynchronously detect changes in source code,
+  # routes, locales, etc. This feature depends on the listen gem.
+  # config.file_watcher = ActiveSupport::EventedFileUpdateChecker
+end