@basaltkit/queue-kafka 1.0.1 → 1.1.0

package/README.md

@@ -1,3 +1,9 @@
+<p align="center">
+  <a href="https://basaltkit-docs.pages.dev">
+    <img src="https://basaltkit-docs.pages.dev/social-card.png" alt="Basalt" width="440">
+  </a>
+</p>
+
 # @basaltkit/queue-kafka
 
 **Apache Kafka** driver for [`@basaltkit/queue`](https://www.npmjs.com/package/@basaltkit/queue): runs your jobs by producing and consuming messages on Kafka topics, without changing your job code. You need this package when your data platform is already built on Kafka and you want to process background work on the same infrastructure.
@@ -88,6 +94,7 @@ The worker's concurrency is passed as `partitionsConsumedConcurrently` — actua
 | `retrySuffix` | `string` | `'.retry'` | Suffix for the retry topic. |
 | `deadSuffix` | `string` | `'.dead'` | Suffix for the dead-letter topic. |
 | `client` | `KafkaClient` | kafkajs | Injectable client — used in tests without a broker. |
+| `onError` | `(error, { source, queue? }) => void` | contextual `console.error` | Infrastructure-fault hook (same pattern as rabbitmq/sqs): a worker's connect/subscribe/run failing at boot (`source: 'consumer'` — previously an unhandled, process-fatal rejection and an invisible zero-worker app), or a retry/dead-letter re-publish failing (`source: 'producer'` — reported, then rethrown so the offset is not committed and Kafka redelivers; a producer outage cannot silently lose a failing job). |
 
 Implements the `QueueDriver` contract from `@basaltkit/queue`.
 

package/dist/index.d.ts

@@ -1,13 +1,12 @@
-import { QueueDriver, DriverCapabilities, JobExecutor, AddJobOptions } from '@basaltkit/queue';
-
+import type { AddJobOptions, DriverCapabilities, JobExecutor, QueueDriver } from '@basaltkit/queue';
 /** The subset of a kafkajs client this driver uses. */
-interface KafkaClient {
+export interface KafkaClient {
     producer(): KafkaProducer;
     consumer(config: {
         groupId: string;
     }): KafkaConsumer;
 }
-interface KafkaProducer {
+export interface KafkaProducer {
     connect(): Promise<void>;
     send(record: {
         topic: string;
@@ -18,11 +17,11 @@ interface KafkaProducer {
     }): Promise<unknown>;
     disconnect(): Promise<void>;
 }
-interface KafkaMessage {
+export interface KafkaMessage {
     value: Uint8Array | string | null;
     headers?: Record<string, Uint8Array | string | undefined>;
 }
-interface KafkaConsumer {
+export interface KafkaConsumer {
     connect(): Promise<void>;
     subscribe(subscription: {
         topic: string;
@@ -37,7 +36,7 @@ interface KafkaConsumer {
     }): Promise<void>;
     disconnect(): Promise<void>;
 }
-interface KafkaDriverOptions {
+export interface KafkaDriverOptions {
     brokers: string[];
     clientId?: string;
     /** Consumer group used by workers. Default 'basalt-queue'. */
@@ -48,6 +47,17 @@ interface KafkaDriverOptions {
     deadSuffix?: string;
     /** Injectable client — defaults to kafkajs. Tests pass a fake. */
     client?: KafkaClient;
+    /**
+     * Called on infrastructure faults the driver cannot recover in line — a
+     * worker's connect/subscribe/run failing at boot (otherwise the app reports
+     * healthy with ZERO workers and the rejection is process-fatal), or a
+     * retry/dead-letter re-publish failing inside the consume callback. Same
+     * pattern as the rabbitmq/sqs drivers. Default: console.error with context.
+     */
+    onError?: (error: unknown, info: {
+        source: 'consumer' | 'producer';
+        queue?: string;
+    }) => void;
 }
 /**
  * Kafka driver for `@basaltkit/queue`. Kafka is a log, not a task queue, so this
@@ -63,7 +73,7 @@ interface KafkaDriverOptions {
  * Worker concurrency is bounded by the topic's partition count, not the
  * `concurrency` number (passed through as `partitionsConsumedConcurrently`).
  */
-declare class KafkaQueueDriver implements QueueDriver {
+export declare class KafkaQueueDriver implements QueueDriver {
     private readonly options;
     readonly name = "kafka";
     readonly capabilities: DriverCapabilities;
@@ -74,6 +84,7 @@ declare class KafkaQueueDriver implements QueueDriver {
     private readonly groupId;
     private readonly retrySuffix;
     private readonly deadSuffix;
+    private readonly onError;
     constructor(options: KafkaDriverOptions);
     setExecutor(executor: JobExecutor): void;
     add(queue: string, jobName: string, data: unknown, options: AddJobOptions): Promise<void>;
@@ -87,5 +98,3 @@ declare class KafkaQueueDriver implements QueueDriver {
     private retryTopic;
     private deadTopic;
 }
-
-export { type KafkaClient, type KafkaConsumer, type KafkaDriverOptions, type KafkaMessage, type KafkaProducer, KafkaQueueDriver };

package/dist/index.js

@@ -1,128 +1,173 @@
-// src/index.ts
-var HEADER = {
-  job: "x-basalt-job",
-  attempt: "x-basalt-attempt",
-  attempts: "x-basalt-attempts"
+const HEADER = {
+    job: 'x-basalt-job',
+    attempt: 'x-basalt-attempt',
+    attempts: 'x-basalt-attempts',
 };
-var MAX_ATTEMPTS = 50;
-var defaultClient = async (options) => {
-  const specifier = "kafkajs";
-  const mod = await import(specifier);
-  return new mod.Kafka({ clientId: options.clientId ?? "basalt", brokers: options.brokers });
+/** Hard ceiling on retries, so a crafted message can't request unbounded ones. */
+const MAX_ATTEMPTS = 50;
+const defaultClient = async (options) => {
+    // Opaque specifier keeps kafkajs an optional peer dependency (runtime-resolved).
+    const specifier = 'kafkajs';
+    const mod = (await import(specifier));
+    return new mod.Kafka({ clientId: options.clientId ?? 'basalt', brokers: options.brokers });
 };
-var KafkaQueueDriver = class {
-  constructor(options) {
-    this.options = options;
-    this.groupId = options.groupId ?? "basalt-queue";
-    this.retrySuffix = options.retrySuffix ?? ".retry";
-    this.deadSuffix = options.deadSuffix ?? ".dead";
-  }
-  options;
-  name = "kafka";
-  capabilities = {
-    delayed: false,
-    priority: false,
-    retries: true,
-    backoff: false
-  };
-  executor;
-  clientPromise;
-  producerPromise;
-  consumers = [];
-  groupId;
-  retrySuffix;
-  deadSuffix;
-  setExecutor(executor) {
-    this.executor = executor;
-  }
-  async add(queue, jobName, data, options) {
-    void options;
-    const producer = await this.producer();
-    await producer.send({
-      topic: queue,
-      messages: [
-        {
-          value: JSON.stringify(data),
-          headers: {
-            [HEADER.job]: jobName,
-            [HEADER.attempt]: "1",
-            [HEADER.attempts]: String(options.attempts)
-          }
-        }
-      ]
-    });
-  }
-  startWorker(queue, options = {}) {
-    void (async () => {
-      const consumer = (await this.client()).consumer({ groupId: this.groupId });
-      this.consumers.push(consumer);
-      await consumer.connect();
-      await consumer.subscribe({ topic: queue, fromBeginning: false });
-      await consumer.subscribe({ topic: this.retryTopic(queue), fromBeginning: false });
-      await consumer.run({
-        eachMessage: ({ message }) => this.handle(queue, message),
-        ...options.concurrency !== void 0 ? { partitionsConsumedConcurrently: options.concurrency } : {}
-      });
-    })();
-  }
-  async close() {
-    const producer = await this.producerPromise?.catch(() => void 0);
-    await producer?.disconnect();
-    await Promise.all(this.consumers.map((consumer) => consumer.disconnect()));
-  }
-  // --- internals -----------------------------------------------------------
-  async handle(queue, message) {
-    if (message.value === null) return;
-    const headers = decodeHeaders(message.headers);
-    const jobName = headers[HEADER.job] ?? "";
-    const value = asString(message.value);
-    try {
-      await this.executor?.(jobName, JSON.parse(value));
-    } catch {
-      const attempt = Number(headers[HEADER.attempt] ?? 1);
-      const attempts = Math.min(Number(headers[HEADER.attempts] ?? 1) || 1, MAX_ATTEMPTS);
-      const producer = await this.producer();
-      if (attempt < attempts) {
+/**
+ * Kafka driver for `@basaltkit/queue`. Kafka is a log, not a task queue, so this
+ * driver is deliberately honest about what it can't do:
+ *
+ * - `delayed` and `priority` are NOT supported (Kafka has neither). With the
+ *   queue's `onUnsupported: 'throw'` policy a delayed/priority dispatch fails
+ *   loudly; with the default 'warn' it logs and proceeds without them.
+ * - `retries` are supported via a retry topic (`<topic>.retry`) the worker also
+ *   consumes; exhausted jobs go to `<topic>.dead`. There is no backoff delay
+ *   (Kafka can't defer a message), so `backoff` is not supported either.
+ *
+ * Worker concurrency is bounded by the topic's partition count, not the
+ * `concurrency` number (passed through as `partitionsConsumedConcurrently`).
+ */
+export class KafkaQueueDriver {
+    options;
+    name = 'kafka';
+    capabilities = {
+        delayed: false,
+        priority: false,
+        retries: true,
+        backoff: false,
+    };
+    executor;
+    clientPromise;
+    producerPromise;
+    consumers = [];
+    groupId;
+    retrySuffix;
+    deadSuffix;
+    onError;
+    constructor(options) {
+        this.options = options;
+        this.groupId = options.groupId ?? 'basalt-queue';
+        this.retrySuffix = options.retrySuffix ?? '.retry';
+        this.deadSuffix = options.deadSuffix ?? '.dead';
+        this.onError =
+            options.onError ??
+                ((error, info) => console.error(`[basalt:queue] kafka ${info.source} error${info.queue ? ` (queue "${info.queue}")` : ''}:`, error));
+    }
+    setExecutor(executor) {
+        this.executor = executor;
+    }
+    async add(queue, jobName, data, options) {
+        // delay/priority are unsupported (see capabilities); the QueueManager has
+        // already applied its onUnsupported policy, so they are simply not encoded.
+        void options;
+        const producer = await this.producer();
         await producer.send({
-          topic: this.retryTopic(queue),
-          messages: [{ value, headers: { ...headers, [HEADER.attempt]: String(attempt + 1) } }]
+            topic: queue,
+            messages: [
+                {
+                    value: JSON.stringify(data),
+                    headers: {
+                        [HEADER.job]: jobName,
+                        [HEADER.attempt]: '1',
+                        [HEADER.attempts]: String(options.attempts),
+                    },
+                },
+            ],
         });
-      } else {
-        await producer.send({ topic: this.deadTopic(queue), messages: [{ value, headers }] });
-      }
     }
-  }
-  client() {
-    if (!this.clientPromise) {
-      this.clientPromise = this.options.client ? Promise.resolve(this.options.client) : defaultClient(this.options);
+    startWorker(queue, options = {}) {
+        ;
+        (async () => {
+            const consumer = (await this.client()).consumer({ groupId: this.groupId });
+            this.consumers.push(consumer);
+            await consumer.connect();
+            await consumer.subscribe({ topic: queue, fromBeginning: false });
+            await consumer.subscribe({ topic: this.retryTopic(queue), fromBeginning: false });
+            await consumer.run({
+                eachMessage: ({ message }) => this.handle(queue, message),
+                ...(options.concurrency !== undefined ? { partitionsConsumedConcurrently: options.concurrency } : {}),
+            });
+        })().catch((error) => {
+            // A broker-connect/subscribe failure at boot must be VISIBLE — otherwise
+            // the app reports healthy with zero workers, and the floating rejection
+            // would be process-fatal by Node's default.
+            this.onError(error, { source: 'consumer', queue });
+        });
     }
-    return this.clientPromise;
-  }
-  producer() {
-    if (!this.producerPromise) {
-      this.producerPromise = (async () => {
-        const producer = (await this.client()).producer();
-        await producer.connect();
-        return producer;
-      })();
+    async close() {
+        const producer = await this.producerPromise?.catch(() => undefined);
+        await producer?.disconnect();
+        await Promise.all(this.consumers.map((consumer) => consumer.disconnect()));
     }
-    return this.producerPromise;
-  }
-  retryTopic(queue) {
-    return `${queue}${this.retrySuffix}`;
-  }
-  deadTopic(queue) {
-    return `${queue}${this.deadSuffix}`;
-  }
-};
-var asString = (value) => typeof value === "string" ? value : new TextDecoder().decode(value);
-var decodeHeaders = (headers) => {
-  const out = {};
-  for (const [key, value] of Object.entries(headers ?? {})) {
-    if (value !== void 0) out[key] = asString(value);
-  }
-  return out;
-};
-export {
-  KafkaQueueDriver
+    // --- internals -----------------------------------------------------------
+    async handle(queue, message) {
+        if (message.value === null)
+            return;
+        const headers = decodeHeaders(message.headers);
+        const jobName = headers[HEADER.job] ?? '';
+        const value = asString(message.value);
+        try {
+            await this.executor?.(jobName, JSON.parse(value));
+        }
+        catch {
+            const attempt = Number(headers[HEADER.attempt] ?? 1);
+            // Clamp the max-attempts read from the (untrusted) message to a hard
+            // ceiling so a crafted `attempts` can't drive a retry-amplification loop.
+            const attempts = Math.min(Number(headers[HEADER.attempts] ?? 1) || 1, MAX_ATTEMPTS);
+            try {
+                const producer = await this.producer();
+                if (attempt < attempts) {
+                    await producer.send({
+                        topic: this.retryTopic(queue),
+                        messages: [{ value, headers: { ...headers, [HEADER.attempt]: String(attempt + 1) } }],
+                    });
+                }
+                else {
+                    await producer.send({ topic: this.deadTopic(queue), messages: [{ value, headers }] });
+                }
+            }
+            catch (publishError) {
+                // The failed job could not be re-routed. Surface the fault, then
+                // RETHROW so eachMessage rejects and the offset is NOT committed —
+                // kafkajs redelivers the message (at-least-once) instead of the job
+                // silently vanishing on a producer outage. (Rabbitmq keeps the message
+                // unacked for the same reason; Kafka's equivalent is not committing.)
+                this.onError(publishError, { source: 'producer', queue });
+                throw publishError;
+            }
+        }
+        // Offsets auto-commit after eachMessage resolves — a re-routed failure is
+        // considered handled so the same message is not redelivered by Kafka.
+    }
+    client() {
+        if (!this.clientPromise) {
+            this.clientPromise = this.options.client
+                ? Promise.resolve(this.options.client)
+                : defaultClient(this.options);
+        }
+        return this.clientPromise;
+    }
+    producer() {
+        if (!this.producerPromise) {
+            this.producerPromise = (async () => {
+                const producer = (await this.client()).producer();
+                await producer.connect();
+                return producer;
+            })();
+        }
+        return this.producerPromise;
+    }
+    retryTopic(queue) {
+        return `${queue}${this.retrySuffix}`;
+    }
+    deadTopic(queue) {
+        return `${queue}${this.deadSuffix}`;
+    }
+}
+const asString = (value) => typeof value === 'string' ? value : new TextDecoder().decode(value);
+const decodeHeaders = (headers) => {
+    const out = {};
+    for (const [key, value] of Object.entries(headers ?? {})) {
+        if (value !== undefined)
+            out[key] = asString(value);
+    }
+    return out;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@basaltkit/queue-kafka",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "Kafka driver for @basaltkit/queue: produce/consume jobs with retry and dead-letter topics (no delayed delivery or priority — Kafka has neither).",
   "license": "MIT",
   "type": "module",
@@ -14,16 +14,15 @@
     "dist"
   ],
   "dependencies": {
-    "@basaltkit/queue": "^1.1.0"
+    "@basaltkit/queue": "^1.3.1"
   },
   "peerDependencies": {
     "kafkajs": "^2.0.0"
   },
   "devDependencies": {
-    "@types/node": "^22.15.0",
-    "tsup": "^8.4.0",
-    "typescript": "^5.8.0",
-    "vitest": "^3.1.0",
+    "@types/node": "^26.3.0",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.11",
     "@basaltkit/tsconfig": "^0.24.0"
   },
   "publishConfig": {
@@ -31,11 +30,11 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/Zebedeu/basalt.git",
+    "url": "git+https://github.com/basaltkit/basalt.git",
     "directory": "packages/queue-kafka"
   },
-  "homepage": "https://github.com/Zebedeu/basalt/tree/main/packages/queue-kafka#readme",
-  "bugs": "https://github.com/Zebedeu/basalt/issues",
+  "homepage": "https://github.com/basaltkit/basalt/tree/main/packages/queue-kafka#readme",
+  "bugs": "https://github.com/basaltkit/basalt/issues",
   "keywords": [
     "basalt",
     "typescript",
@@ -44,7 +43,7 @@
     "kafkajs"
   ],
   "scripts": {
-    "build": "tsup src/index.ts --format esm --dts --clean",
+    "build": "tsc -p tsconfig.build.json",
     "test": "vitest run",
     "typecheck": "tsc --noEmit"
   }