@basaltkit/queue-kafka 1.0.2 → 1.1.0

package/README.md

@@ -94,6 +94,7 @@ The worker's concurrency is passed as `partitionsConsumedConcurrently` — actua
 | `retrySuffix` | `string` | `'.retry'` | Suffix for the retry topic. |
 | `deadSuffix` | `string` | `'.dead'` | Suffix for the dead-letter topic. |
 | `client` | `KafkaClient` | kafkajs | Injectable client — used in tests without a broker. |
+| `onError` | `(error, { source, queue? }) => void` | contextual `console.error` | Infrastructure-fault hook (same pattern as rabbitmq/sqs): a worker's connect/subscribe/run failing at boot (`source: 'consumer'` — previously an unhandled, process-fatal rejection and an invisible zero-worker app), or a retry/dead-letter re-publish failing (`source: 'producer'` — reported, then rethrown so the offset is not committed and Kafka redelivers; a producer outage cannot silently lose a failing job). |
 
 Implements the `QueueDriver` contract from `@basaltkit/queue`.
 

package/dist/index.d.ts

@@ -47,6 +47,17 @@ export interface KafkaDriverOptions {
     deadSuffix?: string;
     /** Injectable client — defaults to kafkajs. Tests pass a fake. */
     client?: KafkaClient;
+    /**
+     * Called on infrastructure faults the driver cannot recover in line — a
+     * worker's connect/subscribe/run failing at boot (otherwise the app reports
+     * healthy with ZERO workers and the rejection is process-fatal), or a
+     * retry/dead-letter re-publish failing inside the consume callback. Same
+     * pattern as the rabbitmq/sqs drivers. Default: console.error with context.
+     */
+    onError?: (error: unknown, info: {
+        source: 'consumer' | 'producer';
+        queue?: string;
+    }) => void;
 }
 /**
  * Kafka driver for `@basaltkit/queue`. Kafka is a log, not a task queue, so this
@@ -73,6 +84,7 @@ export declare class KafkaQueueDriver implements QueueDriver {
     private readonly groupId;
     private readonly retrySuffix;
     private readonly deadSuffix;
+    private readonly onError;
     constructor(options: KafkaDriverOptions);
     setExecutor(executor: JobExecutor): void;
     add(queue: string, jobName: string, data: unknown, options: AddJobOptions): Promise<void>;

package/dist/index.js

@@ -41,11 +41,15 @@ export class KafkaQueueDriver {
     groupId;
     retrySuffix;
     deadSuffix;
+    onError;
     constructor(options) {
         this.options = options;
         this.groupId = options.groupId ?? 'basalt-queue';
         this.retrySuffix = options.retrySuffix ?? '.retry';
         this.deadSuffix = options.deadSuffix ?? '.dead';
+        this.onError =
+            options.onError ??
+                ((error, info) => console.error(`[basalt:queue] kafka ${info.source} error${info.queue ? ` (queue "${info.queue}")` : ''}:`, error));
     }
     setExecutor(executor) {
         this.executor = executor;
@@ -70,7 +74,8 @@ export class KafkaQueueDriver {
         });
     }
     startWorker(queue, options = {}) {
-        void (async () => {
+        ;
+        (async () => {
             const consumer = (await this.client()).consumer({ groupId: this.groupId });
             this.consumers.push(consumer);
             await consumer.connect();
@@ -80,7 +85,12 @@ export class KafkaQueueDriver {
                 eachMessage: ({ message }) => this.handle(queue, message),
                 ...(options.concurrency !== undefined ? { partitionsConsumedConcurrently: options.concurrency } : {}),
             });
-        })();
+        })().catch((error) => {
+            // A broker-connect/subscribe failure at boot must be VISIBLE — otherwise
+            // the app reports healthy with zero workers, and the floating rejection
+            // would be process-fatal by Node's default.
+            this.onError(error, { source: 'consumer', queue });
+        });
     }
     async close() {
         const producer = await this.producerPromise?.catch(() => undefined);
@@ -102,15 +112,26 @@ export class KafkaQueueDriver {
             // Clamp the max-attempts read from the (untrusted) message to a hard
             // ceiling so a crafted `attempts` can't drive a retry-amplification loop.
             const attempts = Math.min(Number(headers[HEADER.attempts] ?? 1) || 1, MAX_ATTEMPTS);
-            const producer = await this.producer();
-            if (attempt < attempts) {
-                await producer.send({
-                    topic: this.retryTopic(queue),
-                    messages: [{ value, headers: { ...headers, [HEADER.attempt]: String(attempt + 1) } }],
-                });
+            try {
+                const producer = await this.producer();
+                if (attempt < attempts) {
+                    await producer.send({
+                        topic: this.retryTopic(queue),
+                        messages: [{ value, headers: { ...headers, [HEADER.attempt]: String(attempt + 1) } }],
+                    });
+                }
+                else {
+                    await producer.send({ topic: this.deadTopic(queue), messages: [{ value, headers }] });
+                }
             }
-            else {
-                await producer.send({ topic: this.deadTopic(queue), messages: [{ value, headers }] });
+            catch (publishError) {
+                // The failed job could not be re-routed. Surface the fault, then
+                // RETHROW so eachMessage rejects and the offset is NOT committed —
+                // kafkajs redelivers the message (at-least-once) instead of the job
+                // silently vanishing on a producer outage. (Rabbitmq keeps the message
+                // unacked for the same reason; Kafka's equivalent is not committing.)
+                this.onError(publishError, { source: 'producer', queue });
+                throw publishError;
             }
         }
         // Offsets auto-commit after eachMessage resolves — a re-routed failure is

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@basaltkit/queue-kafka",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "description": "Kafka driver for @basaltkit/queue: produce/consume jobs with retry and dead-letter topics (no delayed delivery or priority — Kafka has neither).",
   "license": "MIT",
   "type": "module",
@@ -14,7 +14,7 @@
     "dist"
   ],
   "dependencies": {
-    "@basaltkit/queue": "^1.2.1"
+    "@basaltkit/queue": "^1.3.1"
   },
   "peerDependencies": {
     "kafkajs": "^2.0.0"