nulogy_message_bus_consumer 2.0.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc67ef0e9ad8a12ee285d6bf7898e262d5af7140ddfc50ce1265086f313dab5d
-  data.tar.gz: 968ef8091b7ba117dad2a90f265a9e89844f1604404449d703868f343f41c35d
+  metadata.gz: 862741ddaf92ca7e2d6fde54ed615ba825bba19a4d273d1ebed423251d6b533d
+  data.tar.gz: 600762f9aa198ddbb83fbed2f515408da1eb92fa79906e53de946310e44814ce
 SHA512:
-  metadata.gz: 7def0d572755c4b66d8f3ca603fadc64dd3aa159a3a6e868ec38efa589d08d50bdd9ba187cb89ef8773fbe02a5616dea705b6366b4c7371bec3e3979788fca39
-  data.tar.gz: 89b43ff862329c5710f5441a765e20924d0411a0f012905d2e697fb6029b3f9df716b9704d290e37e357bbd4f196d30dd86774ae25bcc017cb6087ad1a2e7845
+  metadata.gz: 675fe6e4b74840afdfcb044ba179f3c82878eb60740d5bfe5abb8568a25d45771b2aafbc33a77bdad97aac75bdc5d3fb047e450a7d970787f23b437ac626d2f6
+  data.tar.gz: e322f8822ebd5cc1c709f4f608cf17ee2ed66fe6e844089467a90304a7b3e73a529ef8847def2264b63208fbf41c17466b82454ce793120fda2879ca4405d84f

data/lib/nulogy_message_bus_consumer/config.rb

@@ -3,6 +3,7 @@ module NulogyMessageBusConsumer
     attr_accessor :bootstrap_servers,
       :client_id,
       :consumer_group_id,
+      :security_protocol,
       :lag_check_interval_seconds,
       :lag_checks,
       :lag_timeout_milliseconds,
@@ -13,6 +14,7 @@ module NulogyMessageBusConsumer
 
     def initialize(options = {})
       defaults = {
+        security_protocol: "ssl",
         lag_check_interval_seconds: 20,
         lag_checks: 6,
         lag_timeout_milliseconds: 200,

data/lib/nulogy_message_bus_consumer/kafka_utils.rb

@@ -10,17 +10,18 @@ module NulogyMessageBusConsumer
       wait_for { consumer.assignment.empty? }
     end
 
-    def wait_for(attempts: 100, interval: 0.1)
+    def wait_for(attempts: 1000, interval: 0.1)
       attempts.times do
-        break if yield
+        return if yield
 
         sleep interval
       end
+      raise "wait_for timed out!"
     end
 
-    def every_message_until_none_are_left(consumer)
+    def every_message_until_none_are_left(consumer, timeout = 250)
       Enumerator.new do |yielder|
-        while (message = consumer.poll(250))
+        while (message = consumer.poll(timeout))
           yielder.yield(message)
         end
       end

data/lib/nulogy_message_bus_consumer/message.rb

@@ -41,9 +41,9 @@ module NulogyMessageBusConsumer
     end
 
     def to_h
-      instance_variable_names.each_with_object({}) do |attribute_name, hash|
-        attribute_name.sub!("@", "")
-        hash[attribute_name] = public_send(attribute_name)
+      instance_variables.each_with_object({}) do |instance_variable, hash|
+        attribute_name = instance_variable.to_s.delete("@")
+        hash[attribute_name.to_sym] = public_send(attribute_name)
       end
     end
   end

data/lib/nulogy_message_bus_consumer/steps/commit_on_success.rb

@@ -1,6 +1,10 @@
 module NulogyMessageBusConsumer
   module Steps
     class CommitOnSuccess
+      def initialize(logger)
+        @logger = logger
+      end
+
       def call(kafka_consumer:, message:, **_)
         result = yield
 
@@ -9,8 +13,18 @@ module NulogyMessageBusConsumer
         if result == :success
           kafka_consumer.store_offset(message)
           kafka_consumer.commit
+          @logger.info(JSON.dump({
+            event: "message_committed",
+            kafka_message_id: message.id,
+            message: message.to_h
+          }))
         else
           reconnect_to_reprocess_same_message(kafka_consumer)
+          @logger.info(JSON.dump({
+            event: "message_failed",
+            kafka_message_id: message.id,
+            message: message.to_h
+          }))
         end
 
         result
@@ -22,6 +36,7 @@ module NulogyMessageBusConsumer
         subscriptions = kafka_consumer.subscription
         kafka_consumer.unsubscribe
         kafka_consumer.subscribe(*subscriptions.to_h.keys)
+        KafkaUtils.wait_for_assignment(kafka_consumer)
       end
 
       def raise_if_invalid(result)

data/lib/nulogy_message_bus_consumer/steps/connect_to_message_bus.rb

@@ -31,7 +31,12 @@ module NulogyMessageBusConsumer
           "bootstrap.servers": @config.bootstrap_servers,
           "enable.auto.commit": false,
           "group.id": @config.consumer_group_id,
-          "enable.auto.offset.store": false
+          "enable.auto.offset.store": false,
+          # Connect to the broker(s) using TLS:
+          # See "SSL" section in:
+          # https://github.com/confluentinc/librdkafka/blob/cb8c19c43011b66c4b08b25e5150455a247e1ff3/INTRODUCTION.md
+          # librdkafka will try to use CA certs from OpenSSL if installed
+          "security.protocol": @config.security_protocol
         }
 
         config["client.id"] = @config.client_id if @config.client_id

data/lib/nulogy_message_bus_consumer/steps/stream_messages_until_none_are_left.rb

@@ -1,17 +1,23 @@
 module NulogyMessageBusConsumer
   module Steps
     class StreamMessagesUntilNoneAreLeft
-      def initialize(logger)
+      def initialize(logger, timeout = 250)
         @logger = logger
+        @timeout = timeout
       end
 
       def call(kafka_consumer:, **_)
-        KafkaUtils.every_message_until_none_are_left(kafka_consumer).each do |kafka_message|
-          yield(
+        KafkaUtils.every_message_until_none_are_left(kafka_consumer, @timeout).each do |kafka_message|
+          result = yield(
             message: Message.from_kafka(kafka_message),
             kafka_message: kafka_message
           )
+          if result == :failure
+            # stop reading on failure or else we'll get stuck in a loop
+            return :failure
+          end
         end
+        :success
       rescue => e
         @logger.error(JSON.dump({
           event: "message_processing_errored",

data/lib/nulogy_message_bus_consumer/version.rb

@@ -1,3 +1,3 @@
 module NulogyMessageBusConsumer
-  VERSION = "2.0.0"
+  VERSION = "3.0.0"
 end

data/lib/nulogy_message_bus_consumer.rb

@@ -67,7 +67,7 @@ module NulogyMessageBusConsumer
       Steps::StreamMessages.new(logger),
       # Message processing steps start here.
       Steps::LogMessages.new(logger),
-      Steps::CommitOnSuccess.new,
+      Steps::CommitOnSuccess.new(logger),
       Steps::DeduplicateMessages.new(logger)
     ])
   end

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20_200_612_184_425) do
+ActiveRecord::Schema.define(version: 2020_06_12_184425) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
   enable_extension "uuid-ossp"