@basaltkit/queue-kafka 1.0.1 → 1.0.2

package/README.md

@@ -1,3 +1,9 @@
+<p align="center">
+  <a href="https://basaltkit-docs.pages.dev">
+    <img src="https://basaltkit-docs.pages.dev/social-card.png" alt="Basalt" width="440">
+  </a>
+</p>
+
 # @basaltkit/queue-kafka
 
 **Apache Kafka** driver for [`@basaltkit/queue`](https://www.npmjs.com/package/@basaltkit/queue): runs your jobs by producing and consuming messages on Kafka topics, without changing your job code. You need this package when your data platform is already built on Kafka and you want to process background work on the same infrastructure.

package/dist/index.d.ts

@@ -1,13 +1,12 @@
-import { QueueDriver, DriverCapabilities, JobExecutor, AddJobOptions } from '@basaltkit/queue';
-
+import type { AddJobOptions, DriverCapabilities, JobExecutor, QueueDriver } from '@basaltkit/queue';
 /** The subset of a kafkajs client this driver uses. */
-interface KafkaClient {
+export interface KafkaClient {
     producer(): KafkaProducer;
     consumer(config: {
         groupId: string;
     }): KafkaConsumer;
 }
-interface KafkaProducer {
+export interface KafkaProducer {
     connect(): Promise<void>;
     send(record: {
         topic: string;
@@ -18,11 +17,11 @@ interface KafkaProducer {
     }): Promise<unknown>;
     disconnect(): Promise<void>;
 }
-interface KafkaMessage {
+export interface KafkaMessage {
     value: Uint8Array | string | null;
     headers?: Record<string, Uint8Array | string | undefined>;
 }
-interface KafkaConsumer {
+export interface KafkaConsumer {
     connect(): Promise<void>;
     subscribe(subscription: {
         topic: string;
@@ -37,7 +36,7 @@ interface KafkaConsumer {
     }): Promise<void>;
     disconnect(): Promise<void>;
 }
-interface KafkaDriverOptions {
+export interface KafkaDriverOptions {
     brokers: string[];
     clientId?: string;
     /** Consumer group used by workers. Default 'basalt-queue'. */
@@ -63,7 +62,7 @@ interface KafkaDriverOptions {
  * Worker concurrency is bounded by the topic's partition count, not the
  * `concurrency` number (passed through as `partitionsConsumedConcurrently`).
  */
-declare class KafkaQueueDriver implements QueueDriver {
+export declare class KafkaQueueDriver implements QueueDriver {
     private readonly options;
     readonly name = "kafka";
     readonly capabilities: DriverCapabilities;
@@ -87,5 +86,3 @@ declare class KafkaQueueDriver implements QueueDriver {
     private retryTopic;
     private deadTopic;
 }
-
-export { type KafkaClient, type KafkaConsumer, type KafkaDriverOptions, type KafkaMessage, type KafkaProducer, KafkaQueueDriver };

package/dist/index.js

@@ -1,128 +1,152 @@
-// src/index.ts
-var HEADER = {
-  job: "x-basalt-job",
-  attempt: "x-basalt-attempt",
-  attempts: "x-basalt-attempts"
+const HEADER = {
+    job: 'x-basalt-job',
+    attempt: 'x-basalt-attempt',
+    attempts: 'x-basalt-attempts',
 };
-var MAX_ATTEMPTS = 50;
-var defaultClient = async (options) => {
-  const specifier = "kafkajs";
-  const mod = await import(specifier);
-  return new mod.Kafka({ clientId: options.clientId ?? "basalt", brokers: options.brokers });
+/** Hard ceiling on retries, so a crafted message can't request unbounded ones. */
+const MAX_ATTEMPTS = 50;
+const defaultClient = async (options) => {
+    // Opaque specifier keeps kafkajs an optional peer dependency (runtime-resolved).
+    const specifier = 'kafkajs';
+    const mod = (await import(specifier));
+    return new mod.Kafka({ clientId: options.clientId ?? 'basalt', brokers: options.brokers });
 };
-var KafkaQueueDriver = class {
-  constructor(options) {
-    this.options = options;
-    this.groupId = options.groupId ?? "basalt-queue";
-    this.retrySuffix = options.retrySuffix ?? ".retry";
-    this.deadSuffix = options.deadSuffix ?? ".dead";
-  }
-  options;
-  name = "kafka";
-  capabilities = {
-    delayed: false,
-    priority: false,
-    retries: true,
-    backoff: false
-  };
-  executor;
-  clientPromise;
-  producerPromise;
-  consumers = [];
-  groupId;
-  retrySuffix;
-  deadSuffix;
-  setExecutor(executor) {
-    this.executor = executor;
-  }
-  async add(queue, jobName, data, options) {
-    void options;
-    const producer = await this.producer();
-    await producer.send({
-      topic: queue,
-      messages: [
-        {
-          value: JSON.stringify(data),
-          headers: {
-            [HEADER.job]: jobName,
-            [HEADER.attempt]: "1",
-            [HEADER.attempts]: String(options.attempts)
-          }
-        }
-      ]
-    });
-  }
-  startWorker(queue, options = {}) {
-    void (async () => {
-      const consumer = (await this.client()).consumer({ groupId: this.groupId });
-      this.consumers.push(consumer);
-      await consumer.connect();
-      await consumer.subscribe({ topic: queue, fromBeginning: false });
-      await consumer.subscribe({ topic: this.retryTopic(queue), fromBeginning: false });
-      await consumer.run({
-        eachMessage: ({ message }) => this.handle(queue, message),
-        ...options.concurrency !== void 0 ? { partitionsConsumedConcurrently: options.concurrency } : {}
-      });
-    })();
-  }
-  async close() {
-    const producer = await this.producerPromise?.catch(() => void 0);
-    await producer?.disconnect();
-    await Promise.all(this.consumers.map((consumer) => consumer.disconnect()));
-  }
-  // --- internals -----------------------------------------------------------
-  async handle(queue, message) {
-    if (message.value === null) return;
-    const headers = decodeHeaders(message.headers);
-    const jobName = headers[HEADER.job] ?? "";
-    const value = asString(message.value);
-    try {
-      await this.executor?.(jobName, JSON.parse(value));
-    } catch {
-      const attempt = Number(headers[HEADER.attempt] ?? 1);
-      const attempts = Math.min(Number(headers[HEADER.attempts] ?? 1) || 1, MAX_ATTEMPTS);
-      const producer = await this.producer();
-      if (attempt < attempts) {
+/**
+ * Kafka driver for `@basaltkit/queue`. Kafka is a log, not a task queue, so this
+ * driver is deliberately honest about what it can't do:
+ *
+ * - `delayed` and `priority` are NOT supported (Kafka has neither). With the
+ *   queue's `onUnsupported: 'throw'` policy a delayed/priority dispatch fails
+ *   loudly; with the default 'warn' it logs and proceeds without them.
+ * - `retries` are supported via a retry topic (`<topic>.retry`) the worker also
+ *   consumes; exhausted jobs go to `<topic>.dead`. There is no backoff delay
+ *   (Kafka can't defer a message), so `backoff` is not supported either.
+ *
+ * Worker concurrency is bounded by the topic's partition count, not the
+ * `concurrency` number (passed through as `partitionsConsumedConcurrently`).
+ */
+export class KafkaQueueDriver {
+    options;
+    name = 'kafka';
+    capabilities = {
+        delayed: false,
+        priority: false,
+        retries: true,
+        backoff: false,
+    };
+    executor;
+    clientPromise;
+    producerPromise;
+    consumers = [];
+    groupId;
+    retrySuffix;
+    deadSuffix;
+    constructor(options) {
+        this.options = options;
+        this.groupId = options.groupId ?? 'basalt-queue';
+        this.retrySuffix = options.retrySuffix ?? '.retry';
+        this.deadSuffix = options.deadSuffix ?? '.dead';
+    }
+    setExecutor(executor) {
+        this.executor = executor;
+    }
+    async add(queue, jobName, data, options) {
+        // delay/priority are unsupported (see capabilities); the QueueManager has
+        // already applied its onUnsupported policy, so they are simply not encoded.
+        void options;
+        const producer = await this.producer();
         await producer.send({
-          topic: this.retryTopic(queue),
-          messages: [{ value, headers: { ...headers, [HEADER.attempt]: String(attempt + 1) } }]
+            topic: queue,
+            messages: [
+                {
+                    value: JSON.stringify(data),
+                    headers: {
+                        [HEADER.job]: jobName,
+                        [HEADER.attempt]: '1',
+                        [HEADER.attempts]: String(options.attempts),
+                    },
+                },
+            ],
         });
-      } else {
-        await producer.send({ topic: this.deadTopic(queue), messages: [{ value, headers }] });
-      }
     }
-  }
-  client() {
-    if (!this.clientPromise) {
-      this.clientPromise = this.options.client ? Promise.resolve(this.options.client) : defaultClient(this.options);
+    startWorker(queue, options = {}) {
+        void (async () => {
+            const consumer = (await this.client()).consumer({ groupId: this.groupId });
+            this.consumers.push(consumer);
+            await consumer.connect();
+            await consumer.subscribe({ topic: queue, fromBeginning: false });
+            await consumer.subscribe({ topic: this.retryTopic(queue), fromBeginning: false });
+            await consumer.run({
+                eachMessage: ({ message }) => this.handle(queue, message),
+                ...(options.concurrency !== undefined ? { partitionsConsumedConcurrently: options.concurrency } : {}),
+            });
+        })();
     }
-    return this.clientPromise;
-  }
-  producer() {
-    if (!this.producerPromise) {
-      this.producerPromise = (async () => {
-        const producer = (await this.client()).producer();
-        await producer.connect();
-        return producer;
-      })();
+    async close() {
+        const producer = await this.producerPromise?.catch(() => undefined);
+        await producer?.disconnect();
+        await Promise.all(this.consumers.map((consumer) => consumer.disconnect()));
     }
-    return this.producerPromise;
-  }
-  retryTopic(queue) {
-    return `${queue}${this.retrySuffix}`;
-  }
-  deadTopic(queue) {
-    return `${queue}${this.deadSuffix}`;
-  }
-};
-var asString = (value) => typeof value === "string" ? value : new TextDecoder().decode(value);
-var decodeHeaders = (headers) => {
-  const out = {};
-  for (const [key, value] of Object.entries(headers ?? {})) {
-    if (value !== void 0) out[key] = asString(value);
-  }
-  return out;
-};
-export {
-  KafkaQueueDriver
+    // --- internals -----------------------------------------------------------
+    async handle(queue, message) {
+        if (message.value === null)
+            return;
+        const headers = decodeHeaders(message.headers);
+        const jobName = headers[HEADER.job] ?? '';
+        const value = asString(message.value);
+        try {
+            await this.executor?.(jobName, JSON.parse(value));
+        }
+        catch {
+            const attempt = Number(headers[HEADER.attempt] ?? 1);
+            // Clamp the max-attempts read from the (untrusted) message to a hard
+            // ceiling so a crafted `attempts` can't drive a retry-amplification loop.
+            const attempts = Math.min(Number(headers[HEADER.attempts] ?? 1) || 1, MAX_ATTEMPTS);
+            const producer = await this.producer();
+            if (attempt < attempts) {
+                await producer.send({
+                    topic: this.retryTopic(queue),
+                    messages: [{ value, headers: { ...headers, [HEADER.attempt]: String(attempt + 1) } }],
+                });
+            }
+            else {
+                await producer.send({ topic: this.deadTopic(queue), messages: [{ value, headers }] });
+            }
+        }
+        // Offsets auto-commit after eachMessage resolves — a re-routed failure is
+        // considered handled so the same message is not redelivered by Kafka.
+    }
+    client() {
+        if (!this.clientPromise) {
+            this.clientPromise = this.options.client
+                ? Promise.resolve(this.options.client)
+                : defaultClient(this.options);
+        }
+        return this.clientPromise;
+    }
+    producer() {
+        if (!this.producerPromise) {
+            this.producerPromise = (async () => {
+                const producer = (await this.client()).producer();
+                await producer.connect();
+                return producer;
+            })();
+        }
+        return this.producerPromise;
+    }
+    retryTopic(queue) {
+        return `${queue}${this.retrySuffix}`;
+    }
+    deadTopic(queue) {
+        return `${queue}${this.deadSuffix}`;
+    }
+}
+const asString = (value) => typeof value === 'string' ? value : new TextDecoder().decode(value);
+const decodeHeaders = (headers) => {
+    const out = {};
+    for (const [key, value] of Object.entries(headers ?? {})) {
+        if (value !== undefined)
+            out[key] = asString(value);
+    }
+    return out;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@basaltkit/queue-kafka",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Kafka driver for @basaltkit/queue: produce/consume jobs with retry and dead-letter topics (no delayed delivery or priority — Kafka has neither).",
   "license": "MIT",
   "type": "module",
@@ -14,16 +14,15 @@
     "dist"
   ],
   "dependencies": {
-    "@basaltkit/queue": "^1.1.0"
+    "@basaltkit/queue": "^1.2.1"
   },
   "peerDependencies": {
     "kafkajs": "^2.0.0"
   },
   "devDependencies": {
-    "@types/node": "^22.15.0",
-    "tsup": "^8.4.0",
-    "typescript": "^5.8.0",
-    "vitest": "^3.1.0",
+    "@types/node": "^26.3.0",
+    "typescript": "^7.0.2",
+    "vitest": "^4.1.11",
     "@basaltkit/tsconfig": "^0.24.0"
   },
   "publishConfig": {
@@ -31,11 +30,11 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/Zebedeu/basalt.git",
+    "url": "git+https://github.com/basaltkit/basalt.git",
     "directory": "packages/queue-kafka"
   },
-  "homepage": "https://github.com/Zebedeu/basalt/tree/main/packages/queue-kafka#readme",
-  "bugs": "https://github.com/Zebedeu/basalt/issues",
+  "homepage": "https://github.com/basaltkit/basalt/tree/main/packages/queue-kafka#readme",
+  "bugs": "https://github.com/basaltkit/basalt/issues",
   "keywords": [
     "basalt",
     "typescript",
@@ -44,7 +43,7 @@
     "kafkajs"
   ],
   "scripts": {
-    "build": "tsup src/index.ts --format esm --dts --clean",
+    "build": "tsc -p tsconfig.build.json",
     "test": "vitest run",
     "typecheck": "tsc --noEmit"
   }