nulogy_message_bus_consumer 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc67ef0e9ad8a12ee285d6bf7898e262d5af7140ddfc50ce1265086f313dab5d
-  data.tar.gz: 968ef8091b7ba117dad2a90f265a9e89844f1604404449d703868f343f41c35d
+  metadata.gz: b061b1d275f3492d62c8ecfd64caaae755f9d9b60a0a4f7466ad72833788ea6b
+  data.tar.gz: df51d614f618b900c5cd432668d68bf1843ba74b4bf54a8ad39544812dd56ede
 SHA512:
-  metadata.gz: 7def0d572755c4b66d8f3ca603fadc64dd3aa159a3a6e868ec38efa589d08d50bdd9ba187cb89ef8773fbe02a5616dea705b6366b4c7371bec3e3979788fca39
-  data.tar.gz: 89b43ff862329c5710f5441a765e20924d0411a0f012905d2e697fb6029b3f9df716b9704d290e37e357bbd4f196d30dd86774ae25bcc017cb6087ad1a2e7845
+  metadata.gz: dd39cb403d86779b3466f7b95ca1403adc6ed464c36c6270e6a37b5dca92117c0507ca3b9d1e4fb6632f737db57ebb2178cd484e65c163083e2ffab321d0775c
+  data.tar.gz: 8548a30f820e217a2bcfd1467d5967767649f096298c5fe6d5b407f6bc0da4aefd4a9170da9b9109b3621c7facc6bd3b9dcad03d5dcea337c3dd9443c4e61718

data/lib/nulogy_message_bus_consumer/kafka_utils.rb

@@ -10,17 +10,18 @@ module NulogyMessageBusConsumer
       wait_for { consumer.assignment.empty? }
     end
 
-    def wait_for(attempts: 100, interval: 0.1)
+    def wait_for(attempts: 1000, interval: 0.1)
       attempts.times do
-        break if yield
+        return if yield
 
         sleep interval
       end
+      raise "wait_for timed out!"
     end
 
-    def every_message_until_none_are_left(consumer)
+    def every_message_until_none_are_left(consumer, timeout = 250)
       Enumerator.new do |yielder|
-        while (message = consumer.poll(250))
+        while (message = consumer.poll(timeout))
           yielder.yield(message)
         end
       end

data/lib/nulogy_message_bus_consumer/message.rb

@@ -41,9 +41,9 @@ module NulogyMessageBusConsumer
     end
 
     def to_h
-      instance_variable_names.each_with_object({}) do |attribute_name, hash|
-        attribute_name.sub!("@", "")
-        hash[attribute_name] = public_send(attribute_name)
+      instance_variables.each_with_object({}) do |instance_variable, hash|
+        attribute_name = instance_variable.to_s.delete("@")
+        hash[attribute_name.to_sym] = public_send(attribute_name)
       end
     end
   end

data/lib/nulogy_message_bus_consumer/steps/commit_on_success.rb

@@ -1,6 +1,10 @@
 module NulogyMessageBusConsumer
   module Steps
     class CommitOnSuccess
+      def initialize(logger)
+        @logger = logger
+      end
+
       def call(kafka_consumer:, message:, **_)
         result = yield
 
@@ -9,8 +13,18 @@ module NulogyMessageBusConsumer
         if result == :success
           kafka_consumer.store_offset(message)
           kafka_consumer.commit
+          @logger.info(JSON.dump({
+            event: "message_committed",
+            kafka_message_id: message.id,
+            message: message.to_h
+          }))
         else
           reconnect_to_reprocess_same_message(kafka_consumer)
+          @logger.info(JSON.dump({
+            event: "message_failed",
+            kafka_message_id: message.id,
+            message: message.to_h
+          }))
         end
 
         result
@@ -22,6 +36,7 @@ module NulogyMessageBusConsumer
         subscriptions = kafka_consumer.subscription
         kafka_consumer.unsubscribe
         kafka_consumer.subscribe(*subscriptions.to_h.keys)
+        KafkaUtils.wait_for_assignment(kafka_consumer)
       end
 
       def raise_if_invalid(result)

data/lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb

@@ -1,17 +1,23 @@
 module NulogyMessageBusConsumer
   module Steps
     class StreamMessagesUntilNoneAreLeft
-      def initialize(logger)
+      def initialize(logger, timeout = 250)
         @logger = logger
+        @timeout = timeout
       end
 
       def call(kafka_consumer:, **_)
-        KafkaUtils.every_message_until_none_are_left(kafka_consumer).each do |kafka_message|
-          yield(
+        KafkaUtils.every_message_until_none_are_left(kafka_consumer, @timeout).each do |kafka_message|
+          result = yield(
             message: Message.from_kafka(kafka_message),
             kafka_message: kafka_message
           )
+          if result == :failure
+            # stop reading on failure or else we'll get stuck in a loop
+            return :failure
+          end
         end
+        :success
       rescue => e
         @logger.error(JSON.dump({
           event: "message_processing_errored",

data/lib/nulogy_message_bus_consumer/version.rb

@@ -1,3 +1,3 @@
 module NulogyMessageBusConsumer
-  VERSION = "2.0.0"
+  VERSION = "2.0.1"
 end

data/lib/nulogy_message_bus_consumer.rb

@@ -67,7 +67,7 @@ module NulogyMessageBusConsumer
       Steps::StreamMessages.new(logger),
       # Message processing steps start here.
       Steps::LogMessages.new(logger),
-      Steps::CommitOnSuccess.new,
+      Steps::CommitOnSuccess.new(logger),
       Steps::DeduplicateMessages.new(logger)
     ])
   end

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20_200_612_184_425) do
+ActiveRecord::Schema.define(version: 2020_06_12_184425) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
   enable_extension "uuid-ossp"