nulogy_message_bus_consumer 1.0.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0989adbcf8a099ac67362106e1724097f7266523a290a36d568d2f8d195a085a'
-  data.tar.gz: da702aedd0055c0fb3ee4269999bc72009bc6d0823f68264770caa11c10c445c
+  metadata.gz: bc67ef0e9ad8a12ee285d6bf7898e262d5af7140ddfc50ce1265086f313dab5d
+  data.tar.gz: 968ef8091b7ba117dad2a90f265a9e89844f1604404449d703868f343f41c35d
 SHA512:
-  metadata.gz: 690db182773fca7bb41d230eb2a5a2c866f61eae733d1afd63b3f7b4693a747b0d1a82d169f7c09b745d70dd5e945e0c745eebb138e7c2405d608ccb736572ba
-  data.tar.gz: 2f5bf0079b7932e15cf9996518d17ec9309882f601cef507a8952ebf22f4c9da01ba5ede9d77674bc491c2dbc6a04b6fb00d095e106075c9d2ad81a001421b49
+  metadata.gz: 7def0d572755c4b66d8f3ca603fadc64dd3aa159a3a6e868ec38efa589d08d50bdd9ba187cb89ef8773fbe02a5616dea705b6366b4c7371bec3e3979788fca39
+  data.tar.gz: 89b43ff862329c5710f5441a765e20924d0411a0f012905d2e697fb6029b3f9df716b9704d290e37e357bbd4f196d30dd86774ae25bcc017cb6087ad1a2e7845

data/Rakefile

@@ -23,9 +23,3 @@ require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 require "standard/rake"
 task default: %i[spec standard]
-
-require "rake/release"
-
-Rake::Release::Task.load_all do |spec|
-  spec.version_tag = "nulogy_message_bus_consumer-v#{spec.version}"
-end

data/lib/nulogy_message_bus_consumer/config.rb

@@ -5,6 +5,7 @@ module NulogyMessageBusConsumer
       :consumer_group_id,
       :lag_check_interval_seconds,
       :lag_checks,
+      :lag_timeout_milliseconds,
       :log_lag_interval_seconds,
       :prune_interval_seconds,
       :prune_max_age,
@@ -14,6 +15,7 @@ module NulogyMessageBusConsumer
       defaults = {
         lag_check_interval_seconds: 20,
         lag_checks: 6,
+        lag_timeout_milliseconds: 200,
         log_lag_interval_seconds: 1.minute.to_i,
         prune_interval_seconds: 1.hour.to_i,
         prune_max_age: 8.days

data/lib/nulogy_message_bus_consumer/tasks/log_consumer_lag.rb

@@ -3,9 +3,10 @@ module NulogyMessageBusConsumer
     class LogConsumerLag
       attr_reader :interval
 
-      def initialize(logger, interval_seconds)
+      def initialize(logger, interval_seconds, lag_timeout)
         @logger = logger
         @interval = interval_seconds
+        @lag_timeout = lag_timeout
       end
 
       def extract_args(kafka_consumer:, **_)
@@ -18,7 +19,12 @@ module NulogyMessageBusConsumer
         # has finished connecting. There appears to be a race condition.
         KafkaUtils.wait_for_assignment(@kafka_consumer)
 
-        lag_per_topic = @kafka_consumer.lag(@kafka_consumer.committed)
+        # Note: consumer#committed has a timeout of 1200ms. To respect our lag_timeout, use the largest.
+        committed_timeout = [1200, @lag_timeout].max
+        # The first parameter is a TopicPartitionList. When nil, it uses all the assigned ones.
+        committed_offsets = @kafka_consumer.committed(nil, committed_timeout)
+
+        lag_per_topic = @kafka_consumer.lag(committed_offsets, @lag_timeout)
 
         @logger.info(JSON.dump({
           event: "consumer_lag",

data/lib/nulogy_message_bus_consumer/version.rb

@@ -1,3 +1,3 @@
 module NulogyMessageBusConsumer
-  VERSION = "1.0.0"
+  VERSION = "2.0.0"
 end

data/lib/nulogy_message_bus_consumer.rb

@@ -52,7 +52,7 @@ module NulogyMessageBusConsumer
       # be called once, without any messages.
       Steps::ConnectToMessageBus.new(config, logger),
       Steps::TimedTask.new(
-        Tasks::LogConsumerLag.new(logger, config.log_lag_interval_seconds)
+        Tasks::LogConsumerLag.new(logger, config.log_lag_interval_seconds, config.lag_timeout_milliseconds)
       ),
       Steps::TimedTask.new(
         Tasks::PruneProcessedMessages.new(logger, config.prune_interval_seconds, config.prune_max_age)

data/spec/dummy/log/development.log

@@ -1,4 +1,83 @@
-Connecting to the MessageBus
-Using consumer group id: test-jasons-partitions-1-audit
-Listening for kafka messages on topic test-jasons-partitions-1
-{"event":"message_processing_errored","class":"ActiveRecord::NoDatabaseError","message":"FATAL:  database \"message_bus_consumer_development\" does not exist\n"}
+   (0.9ms)  DROP DATABASE IF EXISTS "message_bus_consumer_development"
+   (22.2ms)  DROP DATABASE IF EXISTS "message_bus_consumer_test"
+   (76.6ms)  CREATE DATABASE "message_bus_consumer_development" ENCODING = 'utf8'
+   (88.5ms)  CREATE DATABASE "message_bus_consumer_test" ENCODING = 'utf8'
+  SQL (0.9ms)  CREATE EXTENSION IF NOT EXISTS "plpgsql"
+  SQL (3.5ms)  CREATE EXTENSION IF NOT EXISTS "uuid-ossp"
+   (0.9ms)  DROP TABLE IF EXISTS "message_bus_processed_messages" CASCADE
+   (3.6ms)  CREATE TABLE "message_bus_processed_messages" ("id" uuid NOT NULL PRIMARY KEY, "created_at" timestamp NOT NULL)
+   (5.9ms)  CREATE TABLE "schema_migrations" ("version" character varying NOT NULL PRIMARY KEY)
+   (1.1ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC
+   (1.4ms)  INSERT INTO "schema_migrations" (version) VALUES (20200612184425)
+   (1.1ms)  INSERT INTO "schema_migrations" (version) VALUES
+(20200509095105);
+
+
+   (5.6ms)  CREATE TABLE "ar_internal_metadata" ("key" character varying NOT NULL PRIMARY KEY, "value" character varying, "created_at" timestamp(6) NOT NULL, "updated_at" timestamp(6) NOT NULL)
+  ActiveRecord::InternalMetadata Load (0.9ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "environment"], ["LIMIT", 1]]
+   (0.9ms)  BEGIN
+  ActiveRecord::InternalMetadata Create (1.2ms)  INSERT INTO "ar_internal_metadata" ("key", "value", "created_at", "updated_at") VALUES ($1, $2, $3, $4) RETURNING "key"  [["key", "environment"], ["value", "development"], ["created_at", "2022-11-22 15:53:48.407450"], ["updated_at", "2022-11-22 15:53:48.407450"]]
+   (1.1ms)  COMMIT
+  ActiveRecord::InternalMetadata Load (0.8ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "environment"], ["LIMIT", 1]]
+  ActiveRecord::InternalMetadata Load (0.8ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "schema_sha1"], ["LIMIT", 1]]
+   (0.7ms)  BEGIN
+  ActiveRecord::InternalMetadata Create (0.9ms)  INSERT INTO "ar_internal_metadata" ("key", "value", "created_at", "updated_at") VALUES ($1, $2, $3, $4) RETURNING "key"  [["key", "schema_sha1"], ["value", "dde38963e534769353e4aa869d2984e2f2b3db17"], ["created_at", "2022-11-22 15:53:48.418014"], ["updated_at", "2022-11-22 15:53:48.418014"]]
+   (1.2ms)  COMMIT
+  SQL (1.0ms)  CREATE EXTENSION IF NOT EXISTS "plpgsql"
+  SQL (5.0ms)  CREATE EXTENSION IF NOT EXISTS "uuid-ossp"
+   (1.3ms)  DROP TABLE IF EXISTS "message_bus_processed_messages" CASCADE
+   (6.9ms)  CREATE TABLE "message_bus_processed_messages" ("id" uuid NOT NULL PRIMARY KEY, "created_at" timestamp NOT NULL)
+   (5.4ms)  CREATE TABLE "schema_migrations" ("version" character varying NOT NULL PRIMARY KEY)
+   (1.0ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC
+   (1.4ms)  INSERT INTO "schema_migrations" (version) VALUES (20200612184425)
+   (1.3ms)  INSERT INTO "schema_migrations" (version) VALUES
+(20200509095105);
+
+
+   (4.4ms)  CREATE TABLE "ar_internal_metadata" ("key" character varying NOT NULL PRIMARY KEY, "value" character varying, "created_at" timestamp(6) NOT NULL, "updated_at" timestamp(6) NOT NULL)
+  ActiveRecord::InternalMetadata Load (0.9ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "environment"], ["LIMIT", 1]]
+   (0.7ms)  BEGIN
+  ActiveRecord::InternalMetadata Create (1.7ms)  INSERT INTO "ar_internal_metadata" ("key", "value", "created_at", "updated_at") VALUES ($1, $2, $3, $4) RETURNING "key"  [["key", "environment"], ["value", "development"], ["created_at", "2022-11-22 15:53:48.493798"], ["updated_at", "2022-11-22 15:53:48.493798"]]
+   (1.1ms)  COMMIT
+  ActiveRecord::InternalMetadata Load (2.0ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "environment"], ["LIMIT", 1]]
+   (0.7ms)  BEGIN
+  ActiveRecord::InternalMetadata Update (2.0ms)  UPDATE "ar_internal_metadata" SET "value" = $1, "updated_at" = $2 WHERE "ar_internal_metadata"."key" = $3  [["value", "test"], ["updated_at", "2022-11-22 15:53:48.503739"], ["key", "environment"]]
+   (1.1ms)  COMMIT
+  ActiveRecord::InternalMetadata Load (0.8ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "schema_sha1"], ["LIMIT", 1]]
+   (0.7ms)  BEGIN
+  ActiveRecord::InternalMetadata Create (0.8ms)  INSERT INTO "ar_internal_metadata" ("key", "value", "created_at", "updated_at") VALUES ($1, $2, $3, $4) RETURNING "key"  [["key", "schema_sha1"], ["value", "dde38963e534769353e4aa869d2984e2f2b3db17"], ["created_at", "2022-11-22 15:53:48.511603"], ["updated_at", "2022-11-22 15:53:48.511603"]]
+   (1.3ms)  COMMIT
+   (1.1ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC
+   (1.3ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC
+   (0.9ms)  SELECT "ar_internal_metadata"."value" FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1  [["key", "environment"]]
+   (0.9ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC
+   (0.8ms)  SELECT "ar_internal_metadata"."value" FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1  [["key", "environment"]]
+   (0.8ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC
+   (0.8ms)  SELECT "ar_internal_metadata"."value" FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1  [["key", "environment"]]
+  SQL (0.8ms)  CREATE EXTENSION IF NOT EXISTS "plpgsql"
+  SQL (0.8ms)  CREATE EXTENSION IF NOT EXISTS "uuid-ossp"
+   (2.3ms)  DROP TABLE IF EXISTS "message_bus_processed_messages" CASCADE
+   (2.9ms)  CREATE TABLE "message_bus_processed_messages" ("id" uuid NOT NULL PRIMARY KEY, "created_at" timestamp NOT NULL)
+   (1.0ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC
+  ActiveRecord::InternalMetadata Load (0.9ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "environment"], ["LIMIT", 1]]
+  ActiveRecord::InternalMetadata Load (0.9ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "environment"], ["LIMIT", 1]]
+  ActiveRecord::InternalMetadata Load (0.9ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "schema_sha1"], ["LIMIT", 1]]
+  SQL (0.8ms)  CREATE EXTENSION IF NOT EXISTS "plpgsql"
+  SQL (0.8ms)  CREATE EXTENSION IF NOT EXISTS "uuid-ossp"
+   (2.0ms)  DROP TABLE IF EXISTS "message_bus_processed_messages" CASCADE
+   (4.0ms)  CREATE TABLE "message_bus_processed_messages" ("id" uuid NOT NULL PRIMARY KEY, "created_at" timestamp NOT NULL)
+   (1.0ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC
+  ActiveRecord::InternalMetadata Load (0.9ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "environment"], ["LIMIT", 1]]
+   (0.8ms)  BEGIN
+  ActiveRecord::InternalMetadata Update (1.5ms)  UPDATE "ar_internal_metadata" SET "value" = $1, "updated_at" = $2 WHERE "ar_internal_metadata"."key" = $3  [["value", "development"], ["updated_at", "2022-11-22 15:54:02.152053"], ["key", "environment"]]
+   (1.1ms)  COMMIT
+  ActiveRecord::InternalMetadata Load (0.9ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "environment"], ["LIMIT", 1]]
+   (0.8ms)  BEGIN
+  ActiveRecord::InternalMetadata Update (2.1ms)  UPDATE "ar_internal_metadata" SET "value" = $1, "updated_at" = $2 WHERE "ar_internal_metadata"."key" = $3  [["value", "test"], ["updated_at", "2022-11-22 15:54:02.160153"], ["key", "environment"]]
+   (1.2ms)  COMMIT
+  ActiveRecord::InternalMetadata Load (0.9ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "schema_sha1"], ["LIMIT", 1]]
+   (1.2ms)  SELECT pg_try_advisory_lock(5775182565276900090)
+   (2.0ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC
+  ActiveRecord::InternalMetadata Load (0.9ms)  SELECT "ar_internal_metadata".* FROM "ar_internal_metadata" WHERE "ar_internal_metadata"."key" = $1 LIMIT $2  [["key", "environment"], ["LIMIT", 1]]
+   (0.9ms)  SELECT pg_advisory_unlock(5775182565276900090)
+   (1.7ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" ORDER BY "schema_migrations"."version" ASC