nulogy_message_bus_consumer 1.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0989adbcf8a099ac67362106e1724097f7266523a290a36d568d2f8d195a085a'
-  data.tar.gz: da702aedd0055c0fb3ee4269999bc72009bc6d0823f68264770caa11c10c445c
+  metadata.gz: b061b1d275f3492d62c8ecfd64caaae755f9d9b60a0a4f7466ad72833788ea6b
+  data.tar.gz: df51d614f618b900c5cd432668d68bf1843ba74b4bf54a8ad39544812dd56ede
 SHA512:
-  metadata.gz: 690db182773fca7bb41d230eb2a5a2c866f61eae733d1afd63b3f7b4693a747b0d1a82d169f7c09b745d70dd5e945e0c745eebb138e7c2405d608ccb736572ba
-  data.tar.gz: 2f5bf0079b7932e15cf9996518d17ec9309882f601cef507a8952ebf22f4c9da01ba5ede9d77674bc491c2dbc6a04b6fb00d095e106075c9d2ad81a001421b49
+  metadata.gz: dd39cb403d86779b3466f7b95ca1403adc6ed464c36c6270e6a37b5dca92117c0507ca3b9d1e4fb6632f737db57ebb2178cd484e65c163083e2ffab321d0775c
+  data.tar.gz: 8548a30f820e217a2bcfd1467d5967767649f096298c5fe6d5b407f6bc0da4aefd4a9170da9b9109b3621c7facc6bd3b9dcad03d5dcea337c3dd9443c4e61718

data/Rakefile

@@ -23,9 +23,3 @@ require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 require "standard/rake"
 task default: %i[spec standard]
-
-require "rake/release"
-
-Rake::Release::Task.load_all do |spec|
-  spec.version_tag = "nulogy_message_bus_consumer-v#{spec.version}"
-end

data/lib/nulogy_message_bus_consumer/config.rb

@@ -5,6 +5,7 @@ module NulogyMessageBusConsumer
       :consumer_group_id,
       :lag_check_interval_seconds,
       :lag_checks,
+      :lag_timeout_milliseconds,
       :log_lag_interval_seconds,
       :prune_interval_seconds,
       :prune_max_age,
@@ -14,6 +15,7 @@ module NulogyMessageBusConsumer
       defaults = {
         lag_check_interval_seconds: 20,
         lag_checks: 6,
+        lag_timeout_milliseconds: 200,
         log_lag_interval_seconds: 1.minute.to_i,
         prune_interval_seconds: 1.hour.to_i,
         prune_max_age: 8.days

data/lib/nulogy_message_bus_consumer/kafka_utils.rb

@@ -10,17 +10,18 @@ module NulogyMessageBusConsumer
       wait_for { consumer.assignment.empty? }
     end
 
-    def wait_for(attempts: 100, interval: 0.1)
+    def wait_for(attempts: 1000, interval: 0.1)
       attempts.times do
-        break if yield
+        return if yield
 
         sleep interval
       end
+      raise "wait_for timed out!"
     end
 
-    def every_message_until_none_are_left(consumer)
+    def every_message_until_none_are_left(consumer, timeout = 250)
       Enumerator.new do |yielder|
-        while (message = consumer.poll(250))
+        while (message = consumer.poll(timeout))
           yielder.yield(message)
         end
       end

data/lib/nulogy_message_bus_consumer/message.rb

@@ -41,9 +41,9 @@ module NulogyMessageBusConsumer
     end
 
     def to_h
-      instance_variable_names.each_with_object({}) do |attribute_name, hash|
-        attribute_name.sub!("@", "")
-        hash[attribute_name] = public_send(attribute_name)
+      instance_variables.each_with_object({}) do |instance_variable, hash|
+        attribute_name = instance_variable.to_s.delete("@")
+        hash[attribute_name.to_sym] = public_send(attribute_name)
       end
     end
   end

data/lib/nulogy_message_bus_consumer/steps/commit_on_success.rb

@@ -1,6 +1,10 @@
 module NulogyMessageBusConsumer
   module Steps
     class CommitOnSuccess
+      def initialize(logger)
+        @logger = logger
+      end
+
       def call(kafka_consumer:, message:, **_)
         result = yield
 
@@ -9,8 +13,18 @@ module NulogyMessageBusConsumer
         if result == :success
           kafka_consumer.store_offset(message)
           kafka_consumer.commit
+          @logger.info(JSON.dump({
+            event: "message_committed",
+            kafka_message_id: message.id,
+            message: message.to_h
+          }))
         else
           reconnect_to_reprocess_same_message(kafka_consumer)
+          @logger.info(JSON.dump({
+            event: "message_failed",
+            kafka_message_id: message.id,
+            message: message.to_h
+          }))
         end
 
         result
@@ -22,6 +36,7 @@ module NulogyMessageBusConsumer
         subscriptions = kafka_consumer.subscription
         kafka_consumer.unsubscribe
         kafka_consumer.subscribe(*subscriptions.to_h.keys)
+        KafkaUtils.wait_for_assignment(kafka_consumer)
       end
 
       def raise_if_invalid(result)

data/lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb

@@ -1,17 +1,23 @@
 module NulogyMessageBusConsumer
   module Steps
     class StreamMessagesUntilNoneAreLeft
-      def initialize(logger)
+      def initialize(logger, timeout = 250)
         @logger = logger
+        @timeout = timeout
       end
 
       def call(kafka_consumer:, **_)
-        KafkaUtils.every_message_until_none_are_left(kafka_consumer).each do |kafka_message|
-          yield(
+        KafkaUtils.every_message_until_none_are_left(kafka_consumer, @timeout).each do |kafka_message|
+          result = yield(
             message: Message.from_kafka(kafka_message),
             kafka_message: kafka_message
           )
+          if result == :failure
+            # stop reading on failure or else we'll get stuck in a loop
+            return :failure
+          end
         end
+        :success
       rescue => e
         @logger.error(JSON.dump({
           event: "message_processing_errored",

data/lib/nulogy_message_bus_consumer/tasks/log_consumer_lag.rb

@@ -3,9 +3,10 @@ module NulogyMessageBusConsumer
     class LogConsumerLag
       attr_reader :interval
 
-      def initialize(logger, interval_seconds)
+      def initialize(logger, interval_seconds, lag_timeout)
         @logger = logger
         @interval = interval_seconds
+        @lag_timeout = lag_timeout
       end
 
       def extract_args(kafka_consumer:, **_)
@@ -18,7 +19,12 @@ module NulogyMessageBusConsumer
         # has finished connecting. There appears to be a race condition.
         KafkaUtils.wait_for_assignment(@kafka_consumer)
 
-        lag_per_topic = @kafka_consumer.lag(@kafka_consumer.committed)
+        # Note: consumer#committed has a timeout of 1200ms. To respect our lag_timeout, use the largest.
+        committed_timeout = [1200, @lag_timeout].max
+        # The first parameter is a TopicPartitionList. When nil, it uses all the assigned ones.
+        committed_offsets = @kafka_consumer.committed(nil, committed_timeout)
+
+        lag_per_topic = @kafka_consumer.lag(committed_offsets, @lag_timeout)
 
         @logger.info(JSON.dump({
           event: "consumer_lag",

data/lib/nulogy_message_bus_consumer/version.rb

@@ -1,3 +1,3 @@
 module NulogyMessageBusConsumer
-  VERSION = "1.0.0"
+  VERSION = "2.0.1"
 end

data/lib/nulogy_message_bus_consumer.rb

@@ -52,7 +52,7 @@ module NulogyMessageBusConsumer
       # be called once, without any messages.
       Steps::ConnectToMessageBus.new(config, logger),
       Steps::TimedTask.new(
-        Tasks::LogConsumerLag.new(logger, config.log_lag_interval_seconds)
+        Tasks::LogConsumerLag.new(logger, config.log_lag_interval_seconds, config.lag_timeout_milliseconds)
       ),
       Steps::TimedTask.new(
         Tasks::PruneProcessedMessages.new(logger, config.prune_interval_seconds, config.prune_max_age)
@@ -67,7 +67,7 @@ module NulogyMessageBusConsumer
       Steps::StreamMessages.new(logger),
       # Message processing steps start here.
       Steps::LogMessages.new(logger),
-      Steps::CommitOnSuccess.new,
+      Steps::CommitOnSuccess.new(logger),
       Steps::DeduplicateMessages.new(logger)
     ])
   end

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20_200_612_184_425) do
+ActiveRecord::Schema.define(version: 2020_06_12_184425) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
   enable_extension "uuid-ossp"