@lucaapp/service-utils 5.17.0 → 5.18.0

package/dist/lib/kafka/kafkaClient.d.ts

@@ -12,19 +12,23 @@ declare class KafkaClient {
     private readonly admin;
     private readonly producer;
     private readonly consumers;
+    private readonly topics;
     readonly serviceIdentity: ServiceIdentity;
     readonly encryptionEnabled: boolean;
     constructor(parentLogger: Logger, kafkaConfig: KafkaConfiguration, topicSecrets: Partial<Record<KafkaTopic, string>>, serviceIdentity: ServiceIdentity);
     connect: () => Promise<void>;
-    private getTopic;
+    private topicPrefix;
+    private getTopicName;
+    private isForeignEvent;
     private getTopicSecret;
     private encryptValue;
     private decryptValue;
     private generateSignature;
     private verifySignature;
     private parseValue;
-    private ensureTopics;
+    private sendEnsured;
     consume: <T extends KafkaTopic>(kafkaTopic: T, handler: EventPayloadHandler<T>, fromBeginning?: boolean) => Promise<Consumer>;
+    private runConsumer;
     produce: <T extends KafkaTopic>(kafkaTopic: T, key: string, value: KafkaEvent<T>) => Promise<void>;
     /**
      * Register a custom topic (not in KafkaTopic enum)
@@ -34,7 +38,7 @@ declare class KafkaClient {
     /**
      * Get full topic name for custom topic
      */
-    private getCustomTopic;
+    private getCustomTopicName;
     /**
      * Produce message to custom topic
      */
@@ -43,10 +47,16 @@ declare class KafkaClient {
      * Consume messages from custom topic
      */
     consumeCustom: <T = unknown>(topicName: string, handler: GenericEventPayloadHandler<T>, fromBeginning?: boolean, onPoison?: boolean | CustomTopicPoisonHandler) => Promise<Consumer>;
+    private runCustomConsumer;
     private defaultPoisonHandler;
     private encryptCustomValue;
     private decryptCustomValue;
     private parseCustomValue;
+    deleteEmptyUnconsumedTopics: () => Promise<{
+        deleted: string[];
+        skippedActive: string[];
+        skippedNonEmpty: string[];
+    }>;
     shutdown: () => Promise<void>;
 }
 export { KafkaClient };

package/dist/lib/kafka/kafkaClient.js

@@ -41,34 +41,9 @@ const serviceIdentity_1 = require("../serviceIdentity");
 const defaultPoisonHandler_1 = require("./defaultPoisonHandler");
 const events_1 = require("./events");
 const utils_1 = require("../../utils/utils");
-const metrics_1 = require("../metrics");
+const topicLifecycle_1 = require("./topicLifecycle");
+const metrics_1 = require("./metrics");
 const KEY_ALG = 'ES256';
-const messageProducedSizeCounter = new metrics_1.metricsClient.Histogram({
-    name: 'kafka_message_produce_size_bytes',
-    help: 'Size of messages produced',
-    labelNames: ['topic'],
-    buckets: [64, 128, 256, 512, 1024, 2048, 4096, 8182],
-});
-const messageProduceError = new metrics_1.metricsClient.Counter({
-    name: 'kafka_message_produce_error_count',
-    help: 'Total number of errors during message produce',
-    labelNames: ['topic'],
-});
-const messageConsumedCounter = new metrics_1.metricsClient.Counter({
-    name: 'kafka_message_consume_count',
-    help: 'Total number of messages consumed',
-    labelNames: ['topic', 'groupId'],
-});
-const messageConsumedErrorCounter = new metrics_1.metricsClient.Counter({
-    name: 'kafka_message_consume_error_count',
-    help: 'Total number of errors during message consume',
-    labelNames: ['topic', 'groupId'],
-});
-const messageAcknowledgedCounter = new metrics_1.metricsClient.Counter({
-    name: 'kafka_message_consume_ack_count',
-    help: 'Total number of messages acknowledged',
-    labelNames: ['topic', 'groupId'],
-});
 const getIssuer = (kafkaTopic) => {
     return events_1.MessageIssuer[kafkaTopic].valueOf();
 };
@@ -95,10 +70,18 @@ class KafkaClient {
                 throw (0, utils_1.logAndGetError)(this.logger, 'Unable to connect kafkaClient', error);
             }
         };
-        this.getTopic = async (kafkaTopic) => {
-            const topic = `${this.environment}_${getIssuer(kafkaTopic)}_${kafkaTopic}`;
-            await this.ensureTopics(topic);
-            return topic;
+        this.topicPrefix = () => this.environment === serviceIdentity_1.Environment.PROD ||
+            this.environment === serviceIdentity_1.Environment.LOCAL
+            ? this.environment.valueOf()
+            : 'shared';
+        this.getTopicName = (kafkaTopic) => `${this.topicPrefix()}_${getIssuer(kafkaTopic)}_${kafkaTopic}`;
+        this.isForeignEvent = (headers) => {
+            if (this.environment === serviceIdentity_1.Environment.PROD ||
+                this.environment === serviceIdentity_1.Environment.LOCAL) {
+                return false;
+            }
+            const source = headerToString(headers?.sourceEnvironment);
+            return !!source && source !== this.environment.valueOf();
         };
         this.getTopicSecret = (topic) => {
             const topicSecret = this.topicSecrets[topic];
@@ -175,78 +158,90 @@ class KafkaClient {
             }
             return JSON.parse(value.toString());
         };
-        this.ensureTopics = async (topic) => {
-            const existingTopics = await this.admin.listTopics();
-            if (existingTopics.includes(topic)) {
-                this.logger.debug(`Topic=${topic} already exists. Not creating.`);
-                return;
+        this.sendEnsured = async (topic, record) => {
+            await this.topics.ensure(topic);
+            try {
+                await this.producer.send(record);
+            }
+            catch (error) {
+                if (!(0, topicLifecycle_1.isUnknownTopicError)(error)) {
+                    throw error;
+                }
+                this.logger.warn({ topic }, 'Topic missing on produce. Re-ensuring and retrying once.');
+                this.topics.uncache(topic);
+                await this.topics.ensure(topic);
+                await this.producer.send(record);
             }
-            const numPartitions = this.environment == serviceIdentity_1.Environment.PROD ? 3 : 1;
-            const replicationFactor = this.environment == serviceIdentity_1.Environment.LOCAL ? 1 : 3;
-            await this.admin.createTopics({
-                topics: [
-                    {
-                        topic,
-                        numPartitions,
-                        replicationFactor,
-                    },
-                ],
-            });
-            this.logger.debug(`Topic=${topic} created.`);
         };
         this.consume = async (kafkaTopic, handler, fromBeginning = false) => {
-            const topic = await this.getTopic(kafkaTopic);
+            const topic = this.getTopicName(kafkaTopic);
             const groupId = `${this.environment.valueOf()}_${kafkaTopic.valueOf()}_${this.serviceIdentity.identityName}`;
             try {
                 const consumer = this.kafkaClient.consumer({
                     groupId,
                     sessionTimeout: 20000, // 20 seconds
                     heartbeatInterval: 3000, // 3 seconds
+                    allowAutoTopicCreation: false,
                 });
                 this.consumers.push(consumer);
                 await consumer.connect();
-                await consumer.subscribe({ topic, fromBeginning });
-                await consumer.run({
-                    autoCommit: true,
-                    eachMessage: async ({ message }) => {
-                        try {
-                            messageConsumedCounter
-                                .labels({ topic: kafkaTopic.valueOf(), groupId })
-                                .inc();
-                            const decryptedValue = await this.decryptValue(kafkaTopic, message.value);
-                            await this.verifySignature(kafkaTopic, decryptedValue, message.headers);
-                            const value = this.parseValue(decryptedValue);
-                            this.logger.debug({
-                                key: message.key?.toString(),
-                                value,
-                                timestamp: message.timestamp,
-                            }, 'Record received');
-                            try {
-                                await handler({ ...message, value });
-                            }
-                            catch (error) {
-                                throw (0, utils_1.logAndGetError)(this.logger, `Could not consume message for groupId=${groupId}, topic=${topic}, message=${value}`, error);
-                            }
-                            messageAcknowledgedCounter
-                                .labels({ topic: kafkaTopic.valueOf(), groupId })
-                                .inc();
-                        }
-                        catch (error) {
-                            messageConsumedErrorCounter
-                                .labels({ topic: kafkaTopic.valueOf(), groupId })
-                                .inc();
-                            throw error;
-                        }
-                    },
-                });
+                const start = () => this.runConsumer(consumer, topic, fromBeginning, kafkaTopic, handler, groupId);
+                if (await this.topics.exists(topic)) {
+                    await start();
+                }
+                else {
+                    void this.topics.runWhenExists(topic, groupId, start);
+                }
                 return consumer;
             }
             catch (error) {
                 throw (0, utils_1.logAndGetError)(this.logger, `Could not create consumer for groupId=${groupId}, topic=${topic}`, error);
             }
         };
+        this.runConsumer = async (consumer, topic, fromBeginning, kafkaTopic, handler, groupId) => {
+            await consumer.subscribe({ topic, fromBeginning });
+            await consumer.run({
+                autoCommit: true,
+                eachMessage: async ({ message }) => {
+                    if (this.isForeignEvent(message.headers)) {
+                        metrics_1.messageSkippedForeignEnvCounter
+                            .labels({ topic: kafkaTopic.valueOf(), groupId })
+                            .inc();
+                        return;
+                    }
+                    try {
+                        metrics_1.messageConsumedCounter
+                            .labels({ topic: kafkaTopic.valueOf(), groupId })
+                            .inc();
+                        const decryptedValue = await this.decryptValue(kafkaTopic, message.value);
+                        await this.verifySignature(kafkaTopic, decryptedValue, message.headers);
+                        const value = this.parseValue(decryptedValue);
+                        this.logger.debug({
+                            key: message.key?.toString(),
+                            value,
+                            timestamp: message.timestamp,
+                        }, 'Record received');
+                        try {
+                            await handler({ ...message, value });
+                        }
+                        catch (error) {
+                            throw (0, utils_1.logAndGetError)(this.logger, `Could not consume message for groupId=${groupId}, topic=${topic}, message=${value}`, error);
+                        }
+                        metrics_1.messageAcknowledgedCounter
+                            .labels({ topic: kafkaTopic.valueOf(), groupId })
+                            .inc();
+                    }
+                    catch (error) {
+                        metrics_1.messageConsumedErrorCounter
+                            .labels({ topic: kafkaTopic.valueOf(), groupId })
+                            .inc();
+                        throw error;
+                    }
+                },
+            });
+        };
         this.produce = async (kafkaTopic, key, value) => {
-            const topic = await this.getTopic(kafkaTopic);
+            const topic = this.getTopicName(kafkaTopic);
             const serializedValue = JSON.stringify(value);
             const encryptedValue = await this.encryptValue(kafkaTopic, serializedValue);
             const signature = await this.generateSignature(serializedValue);
@@ -260,18 +255,19 @@ class KafkaClient {
                             headers: {
                                 signature,
                                 signatureIssuer: this.serviceIdentity.identityName,
+                                sourceEnvironment: this.environment.valueOf(),
                             },
                         },
                     ],
                 };
-                await this.producer.send(producerRecord);
+                await this.sendEnsured(topic, producerRecord);
                 this.logger.debug(producerRecord, 'Record sent');
-                messageProducedSizeCounter
+                metrics_1.messageProducedSizeCounter
                     .labels({ topic })
                     .observe(Buffer.byteLength(encryptedValue));
             }
             catch (error) {
-                messageProduceError.labels({ topic }).inc();
+                metrics_1.messageProduceError.labels({ topic }).inc();
                 throw (0, utils_1.logAndGetError)(this.logger, `Could not produce message for topic=${topic}`, error);
             }
         };
@@ -286,20 +282,18 @@ class KafkaClient {
         /**
          * Get full topic name for custom topic
          */
-        this.getCustomTopic = async (topicName) => {
+        this.getCustomTopicName = (topicName) => {
             const config = this.customTopics.get(topicName);
             if (!config) {
                 throw (0, utils_1.logAndGetError)(this.logger, `Custom topic ${topicName} not registered. Call registerCustomTopic() first.`);
             }
-            const topic = `${this.environment}_${config.issuer}_${topicName}`;
-            await this.ensureTopics(topic);
-            return topic;
+            return `${this.topicPrefix()}_${config.issuer}_${topicName}`;
         };
         /**
          * Produce message to custom topic
          */
         this.produceCustom = async (topicName, key, value) => {
-            const topic = await this.getCustomTopic(topicName);
+            const topic = this.getCustomTopicName(topicName);
             const serializedValue = JSON.stringify(value);
             // For custom topics, encryption is optional (skip if no secret provided)
             const config = this.customTopics.get(topicName);
@@ -314,18 +308,21 @@ class KafkaClient {
                         {
                             key,
                             value: encryptedValue,
-                            headers: { signature },
+                            headers: {
+                                signature,
+                                sourceEnvironment: this.environment.valueOf(),
+                            },
                         },
                     ],
                 };
-                await this.producer.send(producerRecord);
+                await this.sendEnsured(topic, producerRecord);
                 this.logger.debug(producerRecord, 'Custom topic record sent');
-                messageProducedSizeCounter
+                metrics_1.messageProducedSizeCounter
                     .labels({ topic })
                     .observe(Buffer.byteLength(encryptedValue));
             }
             catch (error) {
-                messageProduceError.labels({ topic }).inc();
+                metrics_1.messageProduceError.labels({ topic }).inc();
                 throw (0, utils_1.logAndGetError)(this.logger, `Could not produce message for custom topic=${topicName}`, error);
             }
         };
@@ -336,71 +333,87 @@ class KafkaClient {
             const poisonHandler = onPoison === true
                 ? this.defaultPoisonHandler(topicName)
                 : onPoison || undefined;
-            const topic = await this.getCustomTopic(topicName);
+            const topic = this.getCustomTopicName(topicName);
             const groupId = `${this.environment.valueOf()}_${topicName}_${this.serviceIdentity.identityName}`;
             try {
                 const consumer = this.kafkaClient.consumer({
                     groupId,
                     sessionTimeout: 20000,
                     heartbeatInterval: 3000,
+                    allowAutoTopicCreation: false,
                 });
                 this.consumers.push(consumer);
                 await consumer.connect();
-                await consumer.subscribe({ topic, fromBeginning });
-                await consumer.run({
-                    autoCommit: true,
-                    eachMessage: async ({ message, partition }) => {
-                        try {
-                            messageConsumedCounter.labels({ topic: topicName, groupId }).inc();
-                            // Decrypt if secret was provided
-                            const config = this.customTopics.get(topicName);
-                            const decryptedValue = config.secret && this.encryptionEnabled
-                                ? await this.decryptCustomValue(config.secret, message.value)
-                                : message.value;
-                            const value = this.parseCustomValue(decryptedValue);
-                            this.logger.debug({
-                                key: message.key?.toString(),
-                                value,
-                                timestamp: message.timestamp,
-                            }, 'Custom topic record received');
-                            await handler({ ...message, value });
-                            messageAcknowledgedCounter
-                                .labels({ topic: topicName, groupId })
-                                .inc();
-                        }
-                        catch (error) {
-                            if (poisonHandler) {
-                                this.logger.warn(error, 'Poison message detected, forwarding to DLQ');
-                                await poisonHandler({
-                                    error,
-                                    decryptFailed: false,
-                                    message,
-                                    partition,
-                                    primaryTopicFullName: topic,
-                                    rawPayloadUtf8: message.value
-                                        ? Buffer.from(message.value).toString('utf8')
-                                        : '',
-                                });
-                                messageAcknowledgedCounter
-                                    .labels({ topic: topicName, groupId })
-                                    .inc();
-                                return;
-                            }
-                            else {
-                                messageConsumedErrorCounter
-                                    .labels({ topic: topicName, groupId })
-                                    .inc();
-                                throw (0, utils_1.logAndGetError)(this.logger, `Could not consume message for custom topic=${topicName}`, error);
-                            }
-                        }
-                    },
-                });
+                const start = () => this.runCustomConsumer(consumer, topic, fromBeginning, topicName, handler, groupId, poisonHandler);
+                if (await this.topics.exists(topic)) {
+                    await start();
+                }
+                else {
+                    void this.topics.runWhenExists(topic, groupId, start);
+                }
                 return consumer;
             }
             catch (error) {
                 throw (0, utils_1.logAndGetError)(this.logger, `Could not create consumer for custom topic=${topicName}`, error);
             }
         };
+        this.runCustomConsumer = async (consumer, topic, fromBeginning, topicName, handler, groupId, poisonHandler) => {
+            await consumer.subscribe({ topic, fromBeginning });
+            await consumer.run({
+                autoCommit: true,
+                eachMessage: async ({ message, partition }) => {
+                    if (this.isForeignEvent(message.headers)) {
+                        metrics_1.messageSkippedForeignEnvCounter
+                            .labels({ topic: topicName, groupId })
+                            .inc();
+                        return;
+                    }
+                    try {
+                        metrics_1.messageConsumedCounter.labels({ topic: topicName, groupId }).inc();
+                        // Decrypt if secret was provided
+                        const config = this.customTopics.get(topicName);
+                        const decryptedValue = config.secret && this.encryptionEnabled
+                            ? await this.decryptCustomValue(config.secret, message.value)
+                            : message.value;
+                        const value = this.parseCustomValue(decryptedValue);
+                        this.logger.debug({
+                            key: message.key?.toString(),
+                            value,
+                            timestamp: message.timestamp,
+                        }, 'Custom topic record received');
+                        await handler({ ...message, value });
+                        metrics_1.messageAcknowledgedCounter
+                            .labels({ topic: topicName, groupId })
+                            .inc();
+                    }
+                    catch (error) {
+                        if (poisonHandler) {
+                            this.logger.warn(error, 'Poison message detected, forwarding to DLQ');
+                            await poisonHandler({
+                                error,
+                                decryptFailed: false,
+                                message,
+                                partition,
+                                primaryTopicFullName: topic,
+                                rawPayloadUtf8: message.value
+                                    ? Buffer.from(message.value).toString('utf8')
+                                    : '',
+                            });
+                            metrics_1.messageAcknowledgedCounter
+                                .labels({ topic: topicName, groupId })
+                                .inc();
+                            return;
+                        }
+                        else {
+                            metrics_1.messageConsumedErrorCounter
+                                .labels({ topic: topicName, groupId })
+                                .inc();
+                            throw (0, utils_1.logAndGetError)(this.logger, `Could not consume message for custom topic=${topicName}`, error);
+                        }
+                    }
+                },
+            });
+        };
         this.defaultPoisonHandler = (topicName) => (0, defaultPoisonHandler_1.createDefaultPoisonHandler)(topicName, {
             logger: this.logger,
             produceCustom: (t, k, v) => this.produceCustom(t, k, v),
@@ -422,7 +435,9 @@ class KafkaClient {
             }
             return JSON.parse(value.toString());
         };
+        this.deleteEmptyUnconsumedTopics = () => this.topics.deleteEmptyUnconsumed();
         this.shutdown = async () => {
+            this.topics.stop();
             try {
                 for (const consumer of this.consumers) {
                     await consumer.disconnect();
@@ -465,8 +480,11 @@ class KafkaClient {
                 }),
             });
             this.admin = this.kafkaClient.admin();
-            this.producer = this.kafkaClient.producer();
+            this.producer = this.kafkaClient.producer({
+                allowAutoTopicCreation: false,
+            });
             this.consumers = [];
+            this.topics = new topicLifecycle_1.TopicLifecycle(this.admin, this.logger, this.environment);
         }
         catch (error) {
             throw (0, utils_1.logAndGetError)(this.logger, 'Unable to connect to Kafka', error);

package/dist/lib/kafka/metrics.d.ts

@@ -0,0 +1,8 @@
+import { metricsClient } from '../metrics';
+declare const messageProducedSizeCounter: metricsClient.Histogram<"topic">;
+declare const messageProduceError: metricsClient.Counter<"topic">;
+declare const messageConsumedCounter: metricsClient.Counter<"topic" | "groupId">;
+declare const messageConsumedErrorCounter: metricsClient.Counter<"topic" | "groupId">;
+declare const messageAcknowledgedCounter: metricsClient.Counter<"topic" | "groupId">;
+declare const messageSkippedForeignEnvCounter: metricsClient.Counter<"topic" | "groupId">;
+export { messageProducedSizeCounter, messageProduceError, messageConsumedCounter, messageConsumedErrorCounter, messageAcknowledgedCounter, messageSkippedForeignEnvCounter, };

package/dist/lib/kafka/metrics.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.messageSkippedForeignEnvCounter = exports.messageAcknowledgedCounter = exports.messageConsumedErrorCounter = exports.messageConsumedCounter = exports.messageProduceError = exports.messageProducedSizeCounter = void 0;
+const metrics_1 = require("../metrics");
+const messageProducedSizeCounter = new metrics_1.metricsClient.Histogram({
+    name: 'kafka_message_produce_size_bytes',
+    help: 'Size of messages produced',
+    labelNames: ['topic'],
+    buckets: [64, 128, 256, 512, 1024, 2048, 4096, 8182],
+});
+exports.messageProducedSizeCounter = messageProducedSizeCounter;
+const messageProduceError = new metrics_1.metricsClient.Counter({
+    name: 'kafka_message_produce_error_count',
+    help: 'Total number of errors during message produce',
+    labelNames: ['topic'],
+});
+exports.messageProduceError = messageProduceError;
+const messageConsumedCounter = new metrics_1.metricsClient.Counter({
+    name: 'kafka_message_consume_count',
+    help: 'Total number of messages consumed',
+    labelNames: ['topic', 'groupId'],
+});
+exports.messageConsumedCounter = messageConsumedCounter;
+const messageConsumedErrorCounter = new metrics_1.metricsClient.Counter({
+    name: 'kafka_message_consume_error_count',
+    help: 'Total number of errors during message consume',
+    labelNames: ['topic', 'groupId'],
+});
+exports.messageConsumedErrorCounter = messageConsumedErrorCounter;
+const messageAcknowledgedCounter = new metrics_1.metricsClient.Counter({
+    name: 'kafka_message_consume_ack_count',
+    help: 'Total number of messages acknowledged',
+    labelNames: ['topic', 'groupId'],
+});
+exports.messageAcknowledgedCounter = messageAcknowledgedCounter;
+const messageSkippedForeignEnvCounter = new metrics_1.metricsClient.Counter({
+    name: 'kafka_message_skipped_foreign_env_count',
+    help: 'Total number of messages skipped because they belong to another environment',
+    labelNames: ['topic', 'groupId'],
+});
+exports.messageSkippedForeignEnvCounter = messageSkippedForeignEnvCounter;

package/dist/lib/kafka/topicLifecycle.d.ts

@@ -0,0 +1,23 @@
+import { Admin } from 'kafkajs';
+import { Logger } from 'pino';
+import { Environment } from '../serviceIdentity';
+declare const isUnknownTopicError: (error: unknown) => boolean;
+declare class TopicLifecycle {
+    private readonly admin;
+    private readonly logger;
+    private readonly environment;
+    private readonly ensuredTopics;
+    private stopped;
+    constructor(admin: Admin, logger: Logger, environment: Environment);
+    stop: () => void;
+    exists: (topic: string) => Promise<boolean>;
+    uncache: (topic: string) => void;
+    ensure: (topic: string) => Promise<void>;
+    runWhenExists: (topic: string, groupId: string, start: () => Promise<void>) => Promise<void>;
+    deleteEmptyUnconsumed: () => Promise<{
+        deleted: string[];
+        skippedActive: string[];
+        skippedNonEmpty: string[];
+    }>;
+}
+export { TopicLifecycle, isUnknownTopicError };

package/dist/lib/kafka/topicLifecycle.js

@@ -0,0 +1,146 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isUnknownTopicError = exports.TopicLifecycle = void 0;
+const kafkajs_1 = require("kafkajs");
+const serviceIdentity_1 = require("../serviceIdentity");
+const utils_1 = require("../../utils/utils");
+const TOPIC_WAIT_INTERVAL_MS = 30_000;
+const TOPIC_DELETE_BATCH_SIZE = 20;
+const sleep = (ms) => new Promise(resolve => {
+    const timer = setTimeout(resolve, ms);
+    timer.unref?.();
+});
+const isUnknownTopicError = (error) => {
+    if (!error || typeof error !== 'object') {
+        return false;
+    }
+    const err = error;
+    if (err.type === 'UNKNOWN_TOPIC_OR_PARTITION' || err.code === 3) {
+        return true;
+    }
+    if (err.originalError && isUnknownTopicError(err.originalError)) {
+        return true;
+    }
+    if (err.cause && isUnknownTopicError(err.cause)) {
+        return true;
+    }
+    return Array.isArray(err.errors) && err.errors.some(isUnknownTopicError);
+};
+exports.isUnknownTopicError = isUnknownTopicError;
+class TopicLifecycle {
+    constructor(admin, logger, environment) {
+        this.admin = admin;
+        this.logger = logger;
+        this.environment = environment;
+        this.ensuredTopics = new Set();
+        this.stopped = false;
+        this.stop = () => {
+            this.stopped = true;
+        };
+        this.exists = async (topic) => {
+            const existingTopics = await this.admin.listTopics();
+            return existingTopics.includes(topic);
+        };
+        this.uncache = (topic) => {
+            this.ensuredTopics.delete(topic);
+        };
+        this.ensure = async (topic) => {
+            if (this.ensuredTopics.has(topic)) {
+                return;
+            }
+            if (!(await this.exists(topic))) {
+                const numPartitions = this.environment == serviceIdentity_1.Environment.PROD ? 3 : 1;
+                const replicationFactor = this.environment == serviceIdentity_1.Environment.LOCAL ? 1 : 3;
+                try {
+                    await this.admin.createTopics({
+                        topics: [
+                            {
+                                topic,
+                                numPartitions,
+                                replicationFactor,
+                            },
+                        ],
+                    });
+                    this.logger.info(`Topic=${topic} created.`);
+                }
+                catch (error) {
+                    throw (0, utils_1.logAndGetError)(this.logger, `Unable to create topic=${topic} (broker partition quota exhausted?)`, error);
+                }
+            }
+            this.ensuredTopics.add(topic);
+        };
+        this.runWhenExists = async (topic, groupId, start) => {
+            this.logger.info({ topic, groupId }, 'Topic does not exist yet. Consumer waits for a producer to create it.');
+            try {
+                let attempt = 0;
+                while (!this.stopped) {
+                    await sleep(TOPIC_WAIT_INTERVAL_MS);
+                    const exists = await this.exists(topic).catch(() => false);
+                    if (exists) {
+                        await start();
+                        this.logger.info({ topic, groupId, attempt }, 'Topic appeared. Consumer subscribed.');
+                        return;
+                    }
+                    attempt += 1;
+                    if (attempt % 20 === 0) {
+                        this.logger.info({ topic, groupId, attempt }, 'Still waiting for topic before subscribing');
+                    }
+                }
+            }
+            catch (error) {
+                this.logger.error({ err: error, topic, groupId }, 'Failed to subscribe consumer after topic appeared');
+            }
+        };
+        this.deleteEmptyUnconsumed = async () => {
+            if (this.environment === serviceIdentity_1.Environment.PROD) {
+                throw (0, utils_1.logAndGetError)(this.logger, 'Refusing to delete kafka topics in production');
+            }
+            const topics = (await this.admin.listTopics()).filter(topic => !topic.startsWith('__'));
+            const activeTopics = new Set();
+            const { groups } = await this.admin.listGroups();
+            if (groups.length > 0) {
+                const described = await this.admin.describeGroups(groups.map(group => group.groupId));
+                for (const group of described.groups) {
+                    for (const member of group.members) {
+                        try {
+                            const { assignment } = kafkajs_1.AssignerProtocol.MemberAssignment.decode(member.memberAssignment) ?? { assignment: {} };
+                            Object.keys(assignment).forEach(topic => activeTopics.add(topic));
+                        }
+                        catch {
+                            continue;
+                        }
+                    }
+                }
+            }
+            const deleted = [];
+            const skippedActive = [];
+            const skippedNonEmpty = [];
+            for (const topic of topics) {
+                if (activeTopics.has(topic)) {
+                    skippedActive.push(topic);
+                    continue;
+                }
+                const offsets = await this.admin.fetchTopicOffsets(topic);
+                const isEmpty = offsets.every(partition => partition.high === partition.low);
+                if (!isEmpty) {
+                    skippedNonEmpty.push(topic);
+                    continue;
+                }
+                deleted.push(topic);
+            }
+            for (let i = 0; i < deleted.length; i += TOPIC_DELETE_BATCH_SIZE) {
+                await this.admin.deleteTopics({
+                    topics: deleted.slice(i, i + TOPIC_DELETE_BATCH_SIZE),
+                });
+            }
+            deleted.forEach(topic => this.ensuredTopics.delete(topic));
+            this.logger.info({
+                deletedCount: deleted.length,
+                skippedActiveCount: skippedActive.length,
+                skippedNonEmptyCount: skippedNonEmpty.length,
+            }, 'Deleted empty unconsumed kafka topics');
+            return { deleted, skippedActive, skippedNonEmpty };
+        };
+    }
+}
+exports.TopicLifecycle = TopicLifecycle;

package/dist/lib/pgBoss/controller/routes.schema.d.ts

@@ -206,11 +206,11 @@ export declare const successResponseSchema: z.ZodObject<{
 }, "strip", z.ZodTypeAny, {
     success: boolean;
     name?: string | undefined;
-    jobId?: string | undefined;
     deleted?: string | undefined;
+    jobId?: string | undefined;
 }, {
     success: boolean;
     name?: string | undefined;
-    jobId?: string | undefined;
     deleted?: string | undefined;
+    jobId?: string | undefined;
 }>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lucaapp/service-utils",
-  "version": "5.17.0",
+  "version": "5.18.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [