nulogy_message_bus_consumer 0.3.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 06d80940b2a655a32c8835712e4b05d1c11e03842906838bc41747387b29fe9c
-  data.tar.gz: 97088e64abcb4ce26caf0dd6e1152c40d94c7c3ef6cff5e5934724ab2ee3b72c
+  metadata.gz: '0989adbcf8a099ac67362106e1724097f7266523a290a36d568d2f8d195a085a'
+  data.tar.gz: da702aedd0055c0fb3ee4269999bc72009bc6d0823f68264770caa11c10c445c
 SHA512:
-  metadata.gz: b29bbf882128c1c4d47b91a4929de09e254e6428648504b3dc14a29e7e16d43c5d22ccb0c9ea0fd87b7b062c76446ad13ad8dae39d17f96bf02ae19423c09099
-  data.tar.gz: '095efec4ae0cba16311a8f97b9c2f0db81642875fb50dccc32aebb3e3b7e287f2761826bb7197aca3004f821d06a6606e6d0f07956853b3f32cd34d4faf0bd1c'
+  metadata.gz: 690db182773fca7bb41d230eb2a5a2c866f61eae733d1afd63b3f7b4693a747b0d1a82d169f7c09b745d70dd5e945e0c745eebb138e7c2405d608ccb736572ba
+  data.tar.gz: 2f5bf0079b7932e15cf9996518d17ec9309882f601cef507a8952ebf22f4c9da01ba5ede9d77674bc491c2dbc6a04b6fb00d095e106075c9d2ad81a001421b49

data/Rakefile

@@ -21,9 +21,8 @@ load "rails/tasks/statistics.rake"
 require "rspec/core"
 require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
-require "rubocop/rake_task"
-RuboCop::RakeTask.new
-task default: %i[spec rubocop]
+require "standard/rake"
+task default: %i[spec standard]
 
 require "rake/release"
 

data/config/credentials/message-bus-us-east-1.key

@@ -0,0 +1 @@
+dfa19863b2709390893da4c2fb85579a

data/config/credentials/message-bus-us-east-1.yml.enc

@@ -0,0 +1 @@
+/WXdqUYePaHqAq3P0iTEsrLiMfRKzp2qYYh7K+q6LUNgi4eHNv0+SoLdI1bUNb9UaGvDPGNT3fCwAdkurk5Iud16ok3b4wD6yZ7UkfqbXqZaKH/dciQ5s63p9Hiuq1rbfcqoZ3KR1SXYAwvy8vNqbdwbAzz2N66B1wE5fNibZZlrWzXJjLReiTcNyxNbCPz6vwEwFF52RntuYlIJo4Nkm8vEk3No+HWrOkM8xptr5qApbd+RowLCLZ4/kcAMDB/XPiGobf0AOFv1NUR/9ChEy20usa8Fqd6HtEn4A25HnAC0uaN0K8ZRXjxhMpnXtfMBItn7yxyJ1ubjRZK5a1xVBRU7L/CVV9ZuIsqAHL1++gH5FBrEe83ZIUhN7AzngMDlOPKGiCLiZLrm18I1AEQrD7tJLyXos15AeAzj--cER8cN0iMLwu8Le+--FL7dhMTgr6xL6SkMnYKmeg==

data/lib/nulogy_message_bus_consumer.rb

@@ -1,10 +1,14 @@
+require "active_record/railtie"
+require "active_support/core_ext/time/zones"
 require "rdkafka"
 
 require "nulogy_message_bus_consumer/engine"
-
+require "nulogy_message_bus_consumer/clock"
 require "nulogy_message_bus_consumer/config"
+require "nulogy_message_bus_consumer/deployment/ecs"
 require "nulogy_message_bus_consumer/handlers/log_unprocessed_messages"
 require "nulogy_message_bus_consumer/kafka_utils"
+require "nulogy_message_bus_consumer/lag_tracker"
 require "nulogy_message_bus_consumer/message"
 require "nulogy_message_bus_consumer/null_logger"
 require "nulogy_message_bus_consumer/pipeline"
@@ -13,10 +17,13 @@ require "nulogy_message_bus_consumer/steps/commit_on_success"
 require "nulogy_message_bus_consumer/steps/connect_to_message_bus"
 require "nulogy_message_bus_consumer/steps/deduplicate_messages"
 require "nulogy_message_bus_consumer/steps/log_messages"
-require "nulogy_message_bus_consumer/steps/monitor_replication_lag"
 require "nulogy_message_bus_consumer/steps/seek_beginning_of_topic"
 require "nulogy_message_bus_consumer/steps/stream_messages"
 require "nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left"
+require "nulogy_message_bus_consumer/steps/timed_task"
+require "nulogy_message_bus_consumer/tasks/log_consumer_lag"
+require "nulogy_message_bus_consumer/tasks/prune_processed_messages"
+require "nulogy_message_bus_consumer/tasks/supervise_consumer_lag"
 
 module NulogyMessageBusConsumer
   module_function
@@ -40,14 +47,28 @@ module NulogyMessageBusConsumer
 
   def recommended_consumer_pipeline(config: self.config, logger: self.logger)
     Pipeline.new([
-      # The first three are really system processing steps
+      # System processing/health steps.
+      # Note: that since they are before `StreamMessages`, they will only
+      # be called once, without any messages.
       Steps::ConnectToMessageBus.new(config, logger),
-      Steps::MonitorReplicationLag.new(logger),
+      Steps::TimedTask.new(
+        Tasks::LogConsumerLag.new(logger, config.log_lag_interval_seconds)
+      ),
+      Steps::TimedTask.new(
+        Tasks::PruneProcessedMessages.new(logger, config.prune_interval_seconds, config.prune_max_age)
+      ),
+      Steps::TimedTask.new(
+        Tasks::SuperviseConsumerLag.new(
+          logger,
+          check_interval_seconds: config.lag_check_interval_seconds,
+          tracker: LagTracker.new(failing_checks: config.lag_checks)
+        )
+      ),
       Steps::StreamMessages.new(logger),
       # Message processing steps start here.
       Steps::LogMessages.new(logger),
       Steps::CommitOnSuccess.new,
-      Steps::DeduplicateMessages.new(logger),
+      Steps::DeduplicateMessages.new(logger)
     ])
   end
 
@@ -56,7 +77,7 @@ module NulogyMessageBusConsumer
       Steps::ConnectToMessageBus.new(config, logger),
       Steps::SeekBeginningOfTopic.new,
       Steps::StreamMessagesUntilNoneAreLeft.new(logger),
-      Handlers::LogUnprocessedMessages.new(logger),
+      Handlers::LogUnprocessedMessages.new(logger)
     ])
   end
 end

data/lib/nulogy_message_bus_consumer/clock.rb

@@ -0,0 +1,13 @@
+module NulogyMessageBusConsumer
+  # Note: Since this calls Time.zone, it is NOt thread-safe
+  class Clock
+    def now
+      Time.zone.now.to_datetime
+    end
+
+    # milliseconds since epoch
+    def ms
+      now.strftime("%Q").to_i
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/config.rb

@@ -1,11 +1,25 @@
 module NulogyMessageBusConsumer
   class Config
-    attr_accessor :consumer_group_id
-    attr_accessor :bootstrap_servers
-    attr_accessor :topic_name
+    attr_accessor :bootstrap_servers,
+      :client_id,
+      :consumer_group_id,
+      :lag_check_interval_seconds,
+      :lag_checks,
+      :log_lag_interval_seconds,
+      :prune_interval_seconds,
+      :prune_max_age,
+      :topic_name
 
     def initialize(options = {})
-      update(options)
+      defaults = {
+        lag_check_interval_seconds: 20,
+        lag_checks: 6,
+        log_lag_interval_seconds: 1.minute.to_i,
+        prune_interval_seconds: 1.hour.to_i,
+        prune_max_age: 8.days
+      }
+
+      update(defaults.merge(options))
     end
 
     def update(options = {})

data/lib/nulogy_message_bus_consumer/deployment/ecs.rb

@@ -0,0 +1,23 @@
+module NulogyMessageBusConsumer
+  module Deployment
+    module ECS
+      module_function
+
+      # Try to get the TaskID from metadata server:
+      # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4.html
+      # Otherwise, return nil
+      def task_id
+        data = `curl --silent "$ECS_CONTAINER_METADATA_URI_V4/task"`
+
+        return if data.empty?
+
+        json = JSON.parse(data)
+        arn = json["TaskARN"]
+
+        return unless arn
+
+        arn.split("/").last
+      end
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/lag_tracker.rb

@@ -0,0 +1,53 @@
+require "set"
+
+module NulogyMessageBusConsumer
+  # Keeps track of how many times a topic's partition has not changed (non-zero) lag between update calls.
+  class LagTracker
+    attr_reader :failing_checks
+
+    def initialize(failing_checks: 3)
+      @failing_checks = failing_checks
+      @tracked = Hash.new { |h, topic| h[topic] = {} }
+      @failed = Hash.new { |h, topic| h[topic] = Set.new }
+    end
+
+    def update(topic_partitions)
+      topic_partitions.each_pair do |topic, partitions|
+        partitions.each_pair do |partition, value|
+          update_topic_partition(topic, partition, value)
+        end
+      end
+    end
+
+    def failing?
+      @failed.any?
+    end
+
+    def failed
+      @failed.transform_values { |v| v.to_a.sort }
+    end
+
+    private
+
+    def update_topic_partition(topic, partition, value)
+      current_value, count = @tracked.dig(topic, partition)
+
+      new_value, new_count =
+        if current_value == value && !value.zero?
+          [current_value, count + 1]
+        else
+          [value, 0]
+        end
+
+      @tracked[topic][partition] = [new_value, new_count]
+
+      if new_count >= @failing_checks
+        @failed[topic] << partition
+      end
+    end
+
+    def exists?(topic, partition)
+      @tracked.dig(topic, partition)
+    end
+  end
+end

data/lib/nulogy_message_bus_consumer/message.rb

@@ -21,7 +21,7 @@ module NulogyMessageBusConsumer
       event_data =
         begin
           JSON.parse(envelope_data[:event_json], symbolize_names: true)
-        rescue StandardError
+        rescue
           {}
         end
 

data/lib/nulogy_message_bus_consumer/null_logger.rb

@@ -1,9 +1,12 @@
 module NulogyMessageBusConsumer
   class NullLogger
-    def info(*_) end
+    def info(*_)
+    end
 
-    def error(*_) end
+    def error(*_)
+    end
 
-    def warn(*_) end
+    def warn(*_)
+    end
   end
 end

data/lib/nulogy_message_bus_consumer/pipeline.rb

@@ -34,7 +34,7 @@ module NulogyMessageBusConsumer
       lambda do |**yielded_args|
         args_to_be_overridden = existing_args.keys & yielded_args.keys
         if args_to_be_overridden.any?
-          raise "Cannot override existing argument(s): #{args_to_be_overridden.join(', ')}"
+          raise "Cannot override existing argument(s): #{args_to_be_overridden.join(", ")}"
         end
 
         func.call(**existing_args.merge(yielded_args))

data/lib/nulogy_message_bus_consumer/steps/commit_on_success.rb

@@ -1,12 +1,13 @@
 module NulogyMessageBusConsumer
   module Steps
     class CommitOnSuccess
-      def call(kafka_consumer:, **_)
+      def call(kafka_consumer:, message:, **_)
         result = yield
 
         raise_if_invalid(result)
 
         if result == :success
+          kafka_consumer.store_offset(message)
           kafka_consumer.commit
         else
           reconnect_to_reprocess_same_message(kafka_consumer)

data/lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb

@@ -9,30 +9,44 @@ module NulogyMessageBusConsumer
 
       def call(**_)
         @logger.info("Connecting to the MessageBus")
-        consumer = build_consumer
         @logger.info("Using consumer group id: #{@config.consumer_group_id}")
 
-        consumer.subscribe(@config.topic_name)
-        @logger.info("Listening for kafka messages on topic #{@config.topic_name}")
+        subscribe
+
+        trap("TERM") { kafka_consumer.close }
 
-        trap("TERM") { consumer.close }
+        wait_for_assignment
 
-        KafkaUtils.wait_for_assignment(consumer)
-        yield(kafka_consumer: consumer)
+        yield(kafka_consumer: kafka_consumer)
       end
 
       private
 
-      def build_consumer
-        @kafka_consumer || Rdkafka::Config.new(consumer_config).consumer
+      def kafka_consumer
+        @kafka_consumer ||= Rdkafka::Config.new(consumer_config).consumer
       end
 
       def consumer_config
-        {
+        config = {
           "bootstrap.servers": @config.bootstrap_servers,
           "enable.auto.commit": false,
           "group.id": @config.consumer_group_id,
+          "enable.auto.offset.store": false
         }
+
+        config["client.id"] = @config.client_id if @config.client_id
+
+        config
+      end
+
+      def subscribe
+        kafka_consumer.subscribe(@config.topic_name)
+        @logger.info("Listening for kafka messages on topic #{@config.topic_name}")
+      end
+
+      def wait_for_assignment
+        KafkaUtils.wait_for_assignment(kafka_consumer)
+        @logger.info("Connected as client: #{kafka_consumer.member_id}")
       end
     end
   end

data/lib/nulogy_message_bus_consumer/steps/deduplicate_messages.rb

@@ -41,7 +41,7 @@ module NulogyMessageBusConsumer
       def log_duplicate(message)
         @logger.warn(JSON.dump({
           event: "duplicate_message_detected",
-          kafka_message_id: message.id,
+          kafka_message_id: message.id
         }))
       end
     end

data/lib/nulogy_message_bus_consumer/steps/log_messages.rb

@@ -1,12 +1,5 @@
 module NulogyMessageBusConsumer
   module Steps
-    class Clock
-      # milliseconds since epoch
-      def now
-        Time.zone.now.to_datetime.strftime("%Q").to_i
-      end
-    end
-
     class LogMessages
       def initialize(logger, clock: Clock.new)
         @logger = logger
@@ -17,18 +10,18 @@ module NulogyMessageBusConsumer
         @logger.info(JSON.dump({
           event: "message_received",
           kafka_message_id: message.id,
-          message: "Received #{message.id}",
+          message: "Received #{message.id}"
         }))
 
         result = yield
 
-        millis = diff_millis(message.created_at, @clock.now)
+        millis = diff_millis(message.created_at, @clock.ms)
         @logger.info(JSON.dump({
           event: "message_processed",
           kafka_message_id: message.id,
-          message: "Processed #{message.id}",
+          message: "Processed #{message.id} (#{message.topic}##{message.partition}@#{message.offset})",
           result: result,
-          time_to_processed: millis,
+          time_to_processed: millis
         }))
 
         result

data/lib/nulogy_message_bus_consumer/steps/stream_messages.rb

@@ -12,11 +12,11 @@ module NulogyMessageBusConsumer
             kafka_message: kafka_message
           )
         end
-      rescue StandardError => e
+      rescue => e
         @logger.error(JSON.dump({
           event: "message_processing_errored",
           class: e.class,
-          message: e.message,
+          message: e.message
         }))
 
         raise

data/lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb

@@ -12,11 +12,11 @@ module NulogyMessageBusConsumer
             kafka_message: kafka_message
           )
         end
-      rescue StandardError => e
+      rescue => e
         @logger.error(JSON.dump({
           event: "message_processing_errored",
           class: e.class,
-          message: e.message,
+          message: e.message
         }))
 
         raise

data/lib/nulogy_message_bus_consumer/steps/timed_task.rb

@@ -0,0 +1,42 @@
+module NulogyMessageBusConsumer
+  module Steps
+    # A generic class to run a "task" on a timer (in a separate thread!)
+    # This class runs the code, the Task does the work
+    #
+    # A Task must implement the methods called in this class:
+    # - #extract_args(kwargs)
+    #   Called with the keyword arguments (kwargs) that is passed to this step.
+    #   This is a chance to pull out references to pipeline variables (e.g. kafka_consumer)
+    # - #call
+    #   The work the Task should perform
+    # - #interval
+    #   The time, in seconds, between invocations of #call
+    class TimedTask
+      def initialize(task)
+        @task = task
+      end
+
+      def call(**kwargs)
+        @task.extract_args(**kwargs)
+
+        # Ensure that the process is terminated if there is a problem getting the consumption lag.
+        # This also ensures that the process will terminate on-boot if it cannot connect to Kafka,
+        # allowing the container to be terminated by ECS.
+        Thread.abort_on_exception = true
+        Thread.new { run }
+
+        yield
+      end
+
+      private
+
+      def run
+        loop do
+          @task.call
+
+          sleep @task.interval
+        end
+      end
+    end
+  end
+end