nulogy_message_bus_consumer 1.0.0.alpha → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d17557a978d8762a83afcb1a8027296854d516cde57540243d48c80cc46c012
-  data.tar.gz: 02c45bfab242f35a0143d090a08387f80db0030854a28b161792cb09efded53c
+  metadata.gz: '0989adbcf8a099ac67362106e1724097f7266523a290a36d568d2f8d195a085a'
+  data.tar.gz: da702aedd0055c0fb3ee4269999bc72009bc6d0823f68264770caa11c10c445c
 SHA512:
-  metadata.gz: 8c7ef5e49a3aff7e09e963592fc79f7435a29d3567764d0cec1e98d52fea31d18f1ee2ea4b3d429c2e550307f0fab539733d700716b89b45c05cfee269e87955
-  data.tar.gz: 0be71a225c18e458d440f80eefadfb5b1a841ad57d619af434e12832c48ff31c1ce2614aaf5786330e2282e418838a223f91c82ebae74cedda3801446620fbb2
+  metadata.gz: 690db182773fca7bb41d230eb2a5a2c866f61eae733d1afd63b3f7b4693a747b0d1a82d169f7c09b745d70dd5e945e0c745eebb138e7c2405d608ccb736572ba
+  data.tar.gz: 2f5bf0079b7932e15cf9996518d17ec9309882f601cef507a8952ebf22f4c9da01ba5ede9d77674bc491c2dbc6a04b6fb00d095e106075c9d2ad81a001421b49

data/lib/nulogy_message_bus_consumer.rb

@@ -16,12 +16,14 @@ require "nulogy_message_bus_consumer/processed_message"
 require "nulogy_message_bus_consumer/steps/commit_on_success"
 require "nulogy_message_bus_consumer/steps/connect_to_message_bus"
 require "nulogy_message_bus_consumer/steps/deduplicate_messages"
-require "nulogy_message_bus_consumer/steps/log_consumer_lag"
 require "nulogy_message_bus_consumer/steps/log_messages"
 require "nulogy_message_bus_consumer/steps/seek_beginning_of_topic"
 require "nulogy_message_bus_consumer/steps/stream_messages"
 require "nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left"
-require "nulogy_message_bus_consumer/steps/supervise_consumer_lag"
+require "nulogy_message_bus_consumer/steps/timed_task"
+require "nulogy_message_bus_consumer/tasks/log_consumer_lag"
+require "nulogy_message_bus_consumer/tasks/prune_processed_messages"
+require "nulogy_message_bus_consumer/tasks/supervise_consumer_lag"
 
 module NulogyMessageBusConsumer
   module_function
@@ -49,11 +51,18 @@ module NulogyMessageBusConsumer
       # Note: that since they are before `StreamMessages`, they will only
       # be called once, without any messages.
       Steps::ConnectToMessageBus.new(config, logger),
-      Steps::LogConsumerLag.new(logger),
-      Steps::SuperviseConsumerLag.new(
-        logger,
-        check_interval_seconds: config.lag_check_interval_seconds,
-        tracker: LagTracker.new(failing_checks: config.lag_checks)
+      Steps::TimedTask.new(
+        Tasks::LogConsumerLag.new(logger, config.log_lag_interval_seconds)
+      ),
+      Steps::TimedTask.new(
+        Tasks::PruneProcessedMessages.new(logger, config.prune_interval_seconds, config.prune_max_age)
+      ),
+      Steps::TimedTask.new(
+        Tasks::SuperviseConsumerLag.new(
+          logger,
+          check_interval_seconds: config.lag_check_interval_seconds,
+          tracker: LagTracker.new(failing_checks: config.lag_checks)
+        )
       ),
       Steps::StreamMessages.new(logger),
       # Message processing steps start here.

data/lib/nulogy_message_bus_consumer/config.rb

@@ -5,12 +5,18 @@ module NulogyMessageBusConsumer
       :consumer_group_id,
       :lag_check_interval_seconds,
       :lag_checks,
+      :log_lag_interval_seconds,
+      :prune_interval_seconds,
+      :prune_max_age,
       :topic_name
 
     def initialize(options = {})
       defaults = {
         lag_check_interval_seconds: 20,
-        lag_checks: 6
+        lag_checks: 6,
+        log_lag_interval_seconds: 1.minute.to_i,
+        prune_interval_seconds: 1.hour.to_i,
+        prune_max_age: 8.days
       }
 
       update(defaults.merge(options))

data/lib/nulogy_message_bus_consumer/message.rb

@@ -1,17 +1,16 @@
 module NulogyMessageBusConsumer
   class Message
-    attr_reader :company_uuid,
-      :created_at,
-      :event_data,
-      :event_data_unparsed,
-      :id,
-      :key,
-      :offset,
-      :partition,
-      :subscription_id,
-      :timestamp,
-      :topic,
-      :type
+    attr_reader :event_data
+    attr_reader :event_data_unparsed
+    attr_reader :id
+    attr_reader :key
+    attr_reader :offset
+    attr_reader :partition
+    attr_reader :subscription_id
+    attr_reader :company_uuid
+    attr_reader :timestamp
+    attr_reader :topic
+    attr_reader :created_at
 
     def initialize(attrs = {})
       attrs.each { |key, value| instance_variable_set("@#{key}", value) }
@@ -37,7 +36,6 @@ module NulogyMessageBusConsumer
         company_uuid: envelope_data[:company_uuid] || envelope_data[:tenant_id],
         timestamp: kafka_message.timestamp,
         topic: kafka_message.topic,
-        type: envelope_data[:type],
         created_at: envelope_data[:created_at]
       )
     end

data/lib/nulogy_message_bus_consumer/steps/timed_task.rb

@@ -0,0 +1,42 @@
+module NulogyMessageBusConsumer
+  module Steps
+    # A generic class to run a "task" on a timer (in a separate thread!)
+    # This class runs the code, the Task does the work
+    #
+    # A Task must implement the methods called in this class:
+    # - #extract_args(kwargs)
+    #   Called with the keyword arguments (kwargs) that is passed to this step.
+    #   This is a chance to pull out references to pipeline variables (e.g. kafka_consumer)
+    # - #call
+    #   The work the Task should perform
+    # - #interval
+    #   The time, in seconds, between invocations of #call
+    class TimedTask
+      def initialize(task)
+        @task = task
+      end
+
+      def call(**kwargs)
+        @task.extract_args(**kwargs)
+
+        # Ensure that the process is terminated if there is a problem getting the consumption lag.
+        # This also ensures that the process will terminate on-boot if it cannot connect to Kafka,
+        # allowing the container to be terminated by ECS.
+        Thread.abort_on_exception = true
+        Thread.new { run }
+
+        yield
+      end
+
+      private
+
+      def run
+        loop do
+          @task.call
+
+          sleep @task.interval
+        end
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/tasks/log_consumer_lag.rb

@@ -0,0 +1,45 @@
+module NulogyMessageBusConsumer
+  module Tasks
+    class LogConsumerLag
+      attr_reader :interval
+
+      def initialize(logger, interval_seconds)
+        @logger = logger
+        @interval = interval_seconds
+      end
+
+      def extract_args(kafka_consumer:, **_)
+        @kafka_consumer = kafka_consumer
+      end
+
+      def call
+        # Delayed start. If we attempt to read consumer#committed immediately, it may fail.
+        # We suspect this is because the consumer#committed is called before the consumer
+        # has finished connecting. There appears to be a race condition.
+        KafkaUtils.wait_for_assignment(@kafka_consumer)
+
+        lag_per_topic = @kafka_consumer.lag(@kafka_consumer.committed)
+
+        @logger.info(JSON.dump({
+          event: "consumer_lag",
+          topics: Calculator.add_max_lag(lag_per_topic)
+        }))
+        $stdout.flush
+      end
+
+      module Calculator
+        def self.add_max_lag(lag_by_topic)
+          lag_by_topic.each_value do |lag_by_partition|
+            lag_by_partition[:_max] = lag_by_partition.values.max || 0
+          end
+
+          lag_by_topic[:_max] = lag_by_topic
+            .map { |_topic, lag_by_partition| lag_by_partition[:_max] }
+            .max || 0
+
+          lag_by_topic
+        end
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/tasks/prune_processed_messages.rb

@@ -0,0 +1,37 @@
+module NulogyMessageBusConsumer
+  module Tasks
+    # Deletes any ProcessedMessage rows that are older than max_age
+    #
+    # interval_seconds: how many seconds between invocations
+    # max_age: ActiveSupport::Duration of how old ProcessedMessage are
+    # removed. This calls `#ago` on it.
+    # For example, setting this to `1.week` will delete records more than
+    # `1.week.ago` on each invocation.
+    class PruneProcessedMessages
+      attr_reader :interval
+
+      def initialize(logger, interval_seconds, max_age)
+        @logger = logger
+        @interval = interval_seconds
+        @max_age = max_age
+      end
+
+      def extract_args(**_)
+      end
+
+      def call
+        deleted = prune_stale
+
+        @logger.info("Pruned #{deleted} processed messages")
+      end
+
+      private
+
+      def prune_stale
+        ProcessedMessage
+          .where(ProcessedMessage.arel_table[:created_at].lt(@max_age.ago))
+          .delete_all
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/{steps → tasks}/supervise_consumer_lag.rb

@@ -1,5 +1,5 @@
 module NulogyMessageBusConsumer
-  module Steps
+  module Tasks
     # Supervises the consumer's lag.
     #
     # If a partition's lag is non-zero and does not change for an extended period
@@ -19,47 +19,36 @@ module NulogyMessageBusConsumer
     #
     # Killing the main thread causes ECS to restart the task.
     class SuperviseConsumerLag
+      attr_reader :interval
+
       def initialize(logger, tracker: NulogyMessageBusConsumer::LagTracker.new(failing_checks: 6), killable: nil, check_interval_seconds: 20)
         @logger = logger
         @tracker = tracker
         @killable = killable
-        @check_interval_seconds = check_interval_seconds
+        @interval = check_interval_seconds
       end
 
-      def call(kafka_consumer:, **_)
+      def extract_args(kafka_consumer:, **_)
         @consumer = kafka_consumer
         @killable ||= Thread.current
-
-        run
-
-        yield
       end
 
-      private
-
-      def run
-        Thread.abort_on_exception = true
+      def call
+        NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(@consumer)
 
-        Thread.new do
-          NulogyMessageBusConsumer::KafkaUtils.wait_for_assignment(@consumer)
+        @tracker.update(@consumer.lag(@consumer.committed))
+        if @tracker.failing?
+          log_failed_partitions
 
-          loop do
-            @tracker.update(@consumer.lag(@consumer.committed))
-
-            if @tracker.failing?
-              log_failed_partitions
-
-              @killable.kill
-              Thread.current.exit
-            end
-
-            sleep @check_interval_seconds
-          end
+          @killable.kill
+          Thread.current.exit
         end
       end
 
+      private
+
       def log_failed_partitions
-        seconds = @check_interval_seconds * @tracker.failing_checks
+        seconds = @interval * @tracker.failing_checks
         failed = @tracker
           .failed
           .map { |topic, partitions| "#{topic}: #{partitions.join(",")}" }

data/lib/nulogy_message_bus_consumer/version.rb

@@ -1,3 +1,3 @@
 module NulogyMessageBusConsumer
-  VERSION = "1.0.0.alpha".freeze
+  VERSION = "1.0.0"
 end

data/spec/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require_relative "config/application"
+
+Rails.application.load_tasks

data/spec/dummy/app/assets/config/manifest.js

@@ -0,0 +1,3 @@
+//= link_tree ../images
+//= link_directory ../stylesheets .css
+//= link message_bus_manifest.js

data/spec/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/spec/dummy/app/channels/application_cable/channel.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Channel < ActionCable::Channel::Base
+  end
+end

data/spec/dummy/app/channels/application_cable/connection.rb

@@ -0,0 +1,4 @@
+module ApplicationCable
+  class Connection < ActionCable::Connection::Base
+  end
+end

data/spec/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,2 @@
+class ApplicationController < ActionController::Base
+end

data/spec/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/spec/dummy/app/javascript/packs/application.js

@@ -0,0 +1,15 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require rails-ujs
+//= require activestorage
+//= require_tree .

data/spec/dummy/app/jobs/application_job.rb

@@ -0,0 +1,7 @@
+class ApplicationJob < ActiveJob::Base
+  # Automatically retry jobs that encountered a deadlock
+  # retry_on ActiveRecord::Deadlocked
+
+  # Most jobs are safe to ignore if the underlying records are no longer available
+  # discard_on ActiveJob::DeserializationError
+end

data/spec/dummy/app/mailers/application_mailer.rb

@@ -0,0 +1,4 @@
+class ApplicationMailer < ActionMailer::Base
+  default from: "from@example.com"
+  layout "mailer"
+end

data/spec/dummy/app/models/application_record.rb

@@ -0,0 +1,3 @@
+class ApplicationRecord < ActiveRecord::Base
+  self.abstract_class = true
+end

data/spec/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Dummy</title>
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+
+    <%= stylesheet_link_tag    'application', media: 'all' %>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/spec/dummy/app/views/layouts/mailer.html.erb

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <style>
+      /* Email styles need to be inline */
+    </style>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/spec/dummy/app/views/layouts/mailer.text.erb

@@ -0,0 +1 @@
+<%= yield %>

data/spec/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path("../config/application", __dir__)
+require_relative "../config/boot"
+require "rails/commands"