@blokjs/trigger-pubsub 0.2.3 → 0.6.1

package/src/adapters/NATSPubSubAdapter.ts

@@ -0,0 +1,326 @@
+/**
+ * NATSPubSubAdapter — v0.7 PR 6 — Pub/Sub adapter backed by NATS.
+ *
+ * Two modes:
+ *   - **Fan-out** (default, when `consumerGroup` is absent): every
+ *     subscriber receives every message on the subject. NATS Core
+ *     publish/subscribe semantics — cheapest path, no persistence.
+ *   - **Competing-consumer** (when `consumerGroup` is set): NATS
+ *     Queue Group — exactly one subscriber in the named group
+ *     receives each message. Pure NATS Core.
+ *   - **Durable** (when `durable: true`): subscribe via NATS
+ *     JetStream consumer so the subscription survives restarts and
+ *     replays missed messages from `startFrom`. Required for the
+ *     `{seq}` / `{timestamp}` replay cursors.
+ *
+ * Subject wildcards (`orders.*.created`, `orders.>`) are honored by
+ * NATS natively in both modes.
+ *
+ * Requires `nats` as a peer dependency:
+ *
+ *     bun add nats
+ *
+ * Environment variables:
+ *   - `NATS_SERVERS`   — comma-separated URLs (default `localhost:4222`).
+ *   - `NATS_TOKEN`     — bearer token authentication.
+ *   - `NATS_USER` / `NATS_PASS`  — userpass authentication.
+ */
+
+import type { PubSubTriggerOpts } from "@blokjs/helper";
+import { v4 as uuid } from "uuid";
+import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
+
+export interface NATSPubSubConfig {
+	servers: string[];
+	token?: string;
+	user?: string;
+	pass?: string;
+}
+
+interface NatsSubscription {
+	unsubscribe: () => void | Promise<void>;
+}
+
+interface NatsConnection {
+	close: () => Promise<void>;
+	drain: () => Promise<void>;
+	subscribe: (
+		subject: string,
+		opts?: { queue?: string; callback?: (err: Error | null, msg: NatsMsg) => void },
+	) => NatsSubscription;
+	publish: (subject: string, payload: Uint8Array) => void;
+	jetstream?: () => NatsJetStream;
+	jetstreamManager?: () => Promise<NatsJetStreamManager>;
+}
+
+interface NatsJetStream {
+	subscribe: (
+		subject: string,
+		opts?: unknown,
+	) => Promise<{
+		[Symbol.asyncIterator]: () => AsyncIterator<NatsJsMsg>;
+		unsubscribe: () => Promise<void> | void;
+	}>;
+	publish: (subject: string, payload: Uint8Array) => Promise<unknown>;
+}
+
+interface NatsJetStreamManager {
+	streams: { add: (config: unknown) => Promise<unknown>; info: (name: string) => Promise<unknown> };
+	consumers: { add: (stream: string, config: unknown) => Promise<unknown> };
+}
+
+interface NatsMsg {
+	subject: string;
+	data: Uint8Array;
+	sid: number;
+	respond?: (data?: Uint8Array) => boolean;
+}
+
+interface NatsJsMsg {
+	subject: string;
+	data: Uint8Array;
+	seq: number;
+	ack: () => void;
+	nak: (millis?: number) => void;
+	info: { stream: string; consumer: string; redeliveryCount: number; timestampNanos?: number };
+}
+
+const TEXT_DECODER = new TextDecoder();
+const TEXT_ENCODER = new TextEncoder();
+
+export class NATSPubSubAdapter implements PubSubAdapter {
+	readonly provider = "nats" as const;
+	private readonly config: NATSPubSubConfig;
+	private conn: NatsConnection | null = null;
+	private subscriptions: Map<string, NatsSubscription | { unsubscribe: () => Promise<void> | void }> = new Map();
+	private connected = false;
+
+	constructor(config?: Partial<NATSPubSubConfig>) {
+		this.config = {
+			servers: config?.servers ?? (process.env.NATS_SERVERS ?? "localhost:4222").split(",").map((s) => s.trim()),
+			token: config?.token ?? process.env.NATS_TOKEN,
+			user: config?.user ?? process.env.NATS_USER,
+			pass: config?.pass ?? process.env.NATS_PASS,
+		};
+	}
+
+	async connect(): Promise<void> {
+		if (this.connected) return;
+		try {
+			// biome-ignore lint/suspicious/noExplicitAny: nats is a runtime peer dep.
+			const nats: any = await import("nats");
+			this.conn = (await nats.connect({
+				servers: this.config.servers,
+				token: this.config.token,
+				user: this.config.user,
+				pass: this.config.pass,
+			})) as NatsConnection;
+			this.connected = true;
+		} catch (err) {
+			throw new Error(
+				`[blok][pubsub-nats] connect failed: ${(err as Error).message}. Install nats as a peer dependency: bun add nats`,
+			);
+		}
+	}
+
+	async disconnect(): Promise<void> {
+		if (!this.connected) return;
+		for (const sub of this.subscriptions.values()) {
+			try {
+				const result = sub.unsubscribe();
+				if (result instanceof Promise) await result;
+			} catch {
+				/* ignore */
+			}
+		}
+		this.subscriptions.clear();
+		try {
+			await this.conn?.drain();
+		} catch {
+			/* ignore */
+		}
+		this.conn = null;
+		this.connected = false;
+	}
+
+	async subscribe(config: PubSubTriggerOpts, handler: (message: PubSubMessage) => Promise<void>): Promise<void> {
+		if (!this.connected || !this.conn) throw new Error("[blok][pubsub-nats] not connected. Call connect() first.");
+		const subKey = `${config.topic}#${config.consumerGroup ?? ""}`;
+
+		if (config.durable === true) {
+			// JetStream durable subscription — survives restarts.
+			if (!this.conn.jetstream) {
+				throw new Error("[blok][pubsub-nats] durable subscriptions require JetStream support in the nats client");
+			}
+			const jsm = await this.conn.jetstreamManager?.();
+			if (jsm) {
+				// Auto-create a stream covering the subject if one doesn't
+				// exist. Production deployments should pre-provision via
+				// `nats stream add` to control retention.
+				try {
+					await jsm.streams.add({
+						name: `blok-${(config.topic ?? "").replace(/[^a-zA-Z0-9_]/g, "_")}`,
+						subjects: [config.topic],
+					});
+				} catch {
+					/* stream already exists — ignore */
+				}
+			}
+			const js = this.conn.jetstream();
+			const startSeq =
+				typeof config.startFrom === "object" && config.startFrom && "seq" in config.startFrom
+					? config.startFrom.seq
+					: undefined;
+			const deliverPolicy =
+				config.startFrom === "earliest"
+					? "all"
+					: config.startFrom === "latest"
+						? "new"
+						: startSeq !== undefined
+							? "by_start_sequence"
+							: "all";
+			const sub = await js.subscribe(config.topic, {
+				config: {
+					durable_name:
+						config.consumerGroup ??
+						`blok-${(config.subscription ?? config.topic ?? "default").replace(/[^a-zA-Z0-9_]/g, "_")}`,
+					deliver_policy: deliverPolicy,
+					opt_start_seq: startSeq,
+					ack_policy: config.ack === false ? "none" : "explicit",
+				},
+			});
+			this.subscriptions.set(subKey, { unsubscribe: () => sub.unsubscribe() });
+			// Drive the async iterator in a background loop.
+			void (async () => {
+				try {
+					for await (const msg of sub as unknown as AsyncIterable<NatsJsMsg>) {
+						await this.dispatchJsMessage(msg, config, handler);
+					}
+				} catch (err) {
+					// Subscription closed or connection lost — let the trigger
+					// re-listen via HMR/reconnect logic. Log for visibility.
+					console.error(`[blok][pubsub-nats] subscription error: ${(err as Error).message}`);
+				}
+			})();
+			return;
+		}
+
+		// Core NATS subscription — fire-and-forget, with optional queue
+		// group for competing-consumer semantics.
+		const sub = this.conn.subscribe(config.topic, {
+			queue: config.consumerGroup,
+			callback: (err, msg) => {
+				if (err) {
+					console.error(`[blok][pubsub-nats] subscribe error: ${err.message}`);
+					return;
+				}
+				void this.dispatchCoreMessage(msg, config, handler);
+			},
+		});
+		this.subscriptions.set(subKey, sub);
+	}
+
+	private async dispatchJsMessage(
+		msg: NatsJsMsg,
+		config: PubSubTriggerOpts,
+		handler: (message: PubSubMessage) => Promise<void>,
+	): Promise<void> {
+		const text = TEXT_DECODER.decode(msg.data);
+		let body: unknown = text;
+		try {
+			body = text.length > 0 ? JSON.parse(text) : null;
+		} catch {
+			/* leave as text */
+		}
+		const message: PubSubMessage = {
+			id: `${msg.info.stream}:${msg.seq}`,
+			body,
+			attributes: { subject: msg.subject },
+			raw: msg,
+			topic: msg.subject,
+			subscription: msg.info.consumer,
+			publishTime: msg.info.timestampNanos ? new Date(msg.info.timestampNanos / 1e6) : undefined,
+			ack: async () => {
+				msg.ack();
+			},
+			nack: async () => {
+				msg.nak();
+			},
+		};
+		try {
+			await handler(message);
+			if (config.ack !== false) msg.ack();
+		} catch {
+			msg.nak();
+		}
+	}
+
+	private async dispatchCoreMessage(
+		msg: NatsMsg,
+		_config: PubSubTriggerOpts,
+		handler: (message: PubSubMessage) => Promise<void>,
+	): Promise<void> {
+		const text = TEXT_DECODER.decode(msg.data);
+		let body: unknown = text;
+		try {
+			body = text.length > 0 ? JSON.parse(text) : null;
+		} catch {
+			/* leave as text */
+		}
+		const message: PubSubMessage = {
+			id: `${msg.subject}:${msg.sid}:${uuid()}`,
+			body,
+			attributes: { subject: msg.subject },
+			raw: msg,
+			topic: msg.subject,
+			ack: async () => {
+				/* core NATS has no explicit ack */
+			},
+			nack: async () => {
+				/* core NATS has no explicit nack */
+			},
+		};
+		try {
+			await handler(message);
+		} catch (err) {
+			console.error(`[blok][pubsub-nats] handler error: ${(err as Error).message}`);
+		}
+	}
+
+	async unsubscribe(subscription: string): Promise<void> {
+		const sub = this.subscriptions.get(subscription);
+		if (!sub) return;
+		try {
+			const result = sub.unsubscribe();
+			if (result instanceof Promise) await result;
+		} catch {
+			/* ignore */
+		}
+		this.subscriptions.delete(subscription);
+	}
+
+	async publish(topic: string, payload: unknown): Promise<void> {
+		if (!this.connected || !this.conn) throw new Error("[blok][pubsub-nats] not connected. Call connect() first.");
+		const body = typeof payload === "string" ? payload : JSON.stringify(payload);
+		const data = TEXT_ENCODER.encode(body);
+		// Use core NATS publish. When a durable subscriber has been
+		// installed for this subject, the subscribe() path created a
+		// JetStream stream with a subject filter that captures any
+		// publish to `topic` — so durable consumers see the message via
+		// the stream while core subscribers see it directly. The earlier
+		// "try js.publish first, fall back to core" logic caused
+		// **double-delivery**: js.publish to a subject covered by a
+		// stream goes to BOTH the stream and core subscribers, then the
+		// fallback would publish AGAIN if js.publish timed out (vs
+		// returning 503 fast). Sticking to core publish avoids the race.
+		this.conn.publish(topic, data);
+	}
+
+	isConnected(): boolean {
+		return this.connected;
+	}
+
+	async healthCheck(): Promise<boolean> {
+		return this.connected;
+	}
+}

package/src/adapters/RedisStreamsPubSubAdapter.ts

@@ -0,0 +1,225 @@
+/**
+ * RedisStreamsPubSubAdapter — v0.7 PR 6 — Pub/Sub adapter backed by
+ * Redis Streams via `ioredis`.
+ *
+ * Pub/Sub vs Worker semantics: this adapter uses **distinct consumer
+ * groups per subscriber** so multiple subscribers each receive every
+ * message (fan-out). When `consumerGroup` is explicitly set on the
+ * workflow, all subscribers in the group compete (1 of N gets each).
+ *
+ * Replay cursors:
+ *   - `"earliest"` / unset → `$` (only new messages by default).
+ *   - `"earliest"` with explicit intent → `0` (replay full stream).
+ *   - `{seq: N}` → resume from stream id `N-0`.
+ *
+ * Requires `ioredis` as a peer dependency.
+ *
+ * Environment variables:
+ *   - `REDIS_HOST` (default `localhost`).
+ *   - `REDIS_PORT` (default `6379`).
+ *   - `REDIS_PASSWORD`.
+ *   - `REDIS_DB`.
+ */
+
+import type { PubSubTriggerOpts } from "@blokjs/helper";
+import { v4 as uuid } from "uuid";
+import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
+
+export interface RedisStreamsPubSubConfig {
+	host: string;
+	port: number;
+	password?: string;
+	db?: number;
+	blockMs: number;
+	count: number;
+}
+
+interface RedisClient {
+	xadd(stream: string, ...args: string[]): Promise<string>;
+	xreadgroup(...args: string[]): Promise<Array<[string, Array<[string, string[]]>]> | null>;
+	xgroup(...args: string[]): Promise<string>;
+	xack(stream: string, group: string, ...ids: string[]): Promise<number>;
+	ping(): Promise<string>;
+	quit(): Promise<string>;
+}
+
+interface ActiveSubscription {
+	stop: () => void;
+}
+
+export class RedisStreamsPubSubAdapter implements PubSubAdapter {
+	readonly provider = "redis-streams" as const;
+	private readonly config: RedisStreamsPubSubConfig;
+	private client: RedisClient | null = null;
+	private subscriptions: Map<string, ActiveSubscription> = new Map();
+	private connected = false;
+	private consumerName = `blok-pubsub-${uuid().slice(0, 8)}`;
+
+	constructor(config?: Partial<RedisStreamsPubSubConfig>) {
+		this.config = {
+			host: config?.host ?? process.env.REDIS_HOST ?? "localhost",
+			port: config?.port ?? Number.parseInt(process.env.REDIS_PORT ?? "6379", 10),
+			password: config?.password ?? process.env.REDIS_PASSWORD,
+			db: config?.db ?? Number.parseInt(process.env.REDIS_DB ?? "0", 10),
+			blockMs: config?.blockMs ?? 5000,
+			count: config?.count ?? 10,
+		};
+	}
+
+	async connect(): Promise<void> {
+		if (this.connected) return;
+		try {
+			// biome-ignore lint/suspicious/noExplicitAny: ioredis is a runtime peer dep.
+			const ioredis: any = await import("ioredis");
+			const IORedis = ioredis.default ?? ioredis.Redis ?? ioredis;
+			this.client = new IORedis({
+				host: this.config.host,
+				port: this.config.port,
+				password: this.config.password,
+				db: this.config.db,
+			}) as RedisClient;
+			await this.client.ping();
+			this.connected = true;
+		} catch (err) {
+			throw new Error(
+				`[blok][pubsub-redis] connect failed: ${(err as Error).message}. Install ioredis as a peer dependency: bun add ioredis`,
+			);
+		}
+	}
+
+	async disconnect(): Promise<void> {
+		if (!this.connected) return;
+		for (const sub of this.subscriptions.values()) sub.stop();
+		this.subscriptions.clear();
+		try {
+			await this.client?.quit();
+		} catch {
+			/* ignore */
+		}
+		this.client = null;
+		this.connected = false;
+	}
+
+	async subscribe(config: PubSubTriggerOpts, handler: (message: PubSubMessage) => Promise<void>): Promise<void> {
+		if (!this.connected || !this.client) throw new Error("[blok][pubsub-redis] not connected. Call connect() first.");
+		const client = this.client;
+		const stream = config.topic;
+		// Fan-out: each subscriber gets its own group (unique per instance).
+		// Competing-consumer: explicit `consumerGroup` makes all subscribers
+		// share work.
+		const group = config.consumerGroup ?? `blok-fanout-${this.consumerName}-${stream.replace(/[^a-zA-Z0-9_]/g, "_")}`;
+		const startId =
+			config.startFrom === "earliest"
+				? "0"
+				: config.startFrom === "latest" || config.startFrom === undefined
+					? "$"
+					: typeof config.startFrom === "object" && "seq" in config.startFrom
+						? `${config.startFrom.seq}-0`
+						: "$";
+
+		try {
+			await client.xgroup("CREATE", stream, group, startId, "MKSTREAM");
+		} catch (err) {
+			if (!/BUSYGROUP/i.test((err as Error).message)) throw err;
+		}
+
+		let stopped = false;
+		const sub: ActiveSubscription = {
+			stop: () => {
+				stopped = true;
+			},
+		};
+		this.subscriptions.set(`${stream}#${group}`, sub);
+
+		void (async () => {
+			while (!stopped) {
+				let entries: Array<[string, Array<[string, string[]]>]> | null = null;
+				try {
+					entries = await client.xreadgroup(
+						"GROUP",
+						group,
+						this.consumerName,
+						"COUNT",
+						String(this.config.count),
+						"BLOCK",
+						String(this.config.blockMs),
+						"STREAMS",
+						stream,
+						">",
+					);
+				} catch {
+					await new Promise((r) => setTimeout(r, 1000));
+					continue;
+				}
+				if (!entries) continue;
+				for (const [, msgs] of entries) {
+					for (const [id, fields] of msgs) {
+						if (stopped) break;
+						const payload = this.fieldsToObject(fields);
+						const dataString = typeof payload.data === "string" ? payload.data : "";
+						let body: unknown;
+						try {
+							body = dataString.length > 0 ? JSON.parse(dataString) : null;
+						} catch {
+							body = dataString;
+						}
+						const message: PubSubMessage = {
+							id,
+							body,
+							attributes: payload,
+							raw: { id, fields },
+							topic: stream,
+							subscription: group,
+							publishTime: new Date(Number.parseInt(id.split("-")[0] ?? String(Date.now()), 10)),
+							ack: async () => {
+								await client.xack(stream, group, id);
+							},
+							nack: async () => {
+								/* leave unacked — picked up by XAUTOCLAIM */
+							},
+						};
+						try {
+							await handler(message);
+							if (config.ack !== false) await client.xack(stream, group, id);
+						} catch {
+							// Leave unacked — pending entries are visible in XPENDING.
+						}
+					}
+				}
+			}
+		})();
+	}
+
+	private fieldsToObject(fields: string[]): Record<string, string> {
+		const out: Record<string, string> = {};
+		for (let i = 0; i < fields.length; i += 2) out[fields[i]] = fields[i + 1];
+		return out;
+	}
+
+	async unsubscribe(subscription: string): Promise<void> {
+		const sub = this.subscriptions.get(subscription);
+		if (!sub) return;
+		sub.stop();
+		this.subscriptions.delete(subscription);
+	}
+
+	async publish(topic: string, payload: unknown): Promise<void> {
+		if (!this.connected || !this.client) throw new Error("[blok][pubsub-redis] not connected. Call connect() first.");
+		const body = typeof payload === "string" ? payload : JSON.stringify(payload);
+		await this.client.xadd(topic, "*", "data", body);
+	}
+
+	isConnected(): boolean {
+		return this.connected;
+	}
+
+	async healthCheck(): Promise<boolean> {
+		if (!this.connected || !this.client) return false;
+		try {
+			const pong = await this.client.ping();
+			return pong === "PONG";
+		} catch {
+			return false;
+		}
+	}
+}

package/src/adapters/factory.test.ts

@@ -0,0 +1,87 @@
+/**
+ * Pub/Sub adapter factory unit tests — v0.7 PR 6.
+ *
+ * Mirrors the worker factory tests (PR 5) — provider resolution
+ * order, constructor lookup, pool sharing, reset utility.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { _resetAdapterPoolForTests, createPubSubAdapter, getOrCreateAdapter, resolveProvider } from "./factory";
+
+describe("pubsub adapter factory — v0.7 PR 6", () => {
+	beforeEach(() => {
+		_resetAdapterPoolForTests();
+		process.env.BLOK_PUBSUB_ADAPTER = undefined;
+	});
+
+	afterEach(() => {
+		_resetAdapterPoolForTests();
+		process.env.BLOK_PUBSUB_ADAPTER = undefined;
+	});
+
+	describe("resolveProvider()", () => {
+		it("returns the explicit provider when set", () => {
+			expect(resolveProvider("kafka")).toBe("kafka");
+		});
+
+		it("falls back to BLOK_PUBSUB_ADAPTER env var", () => {
+			process.env.BLOK_PUBSUB_ADAPTER = "redis-streams";
+			expect(resolveProvider(undefined)).toBe("redis-streams");
+		});
+
+		it("falls back to nats when neither is set", () => {
+			expect(resolveProvider(undefined)).toBe("nats");
+		});
+
+		it("ignores invalid env values and falls back to nats", () => {
+			process.env.BLOK_PUBSUB_ADAPTER = "not-a-provider";
+			expect(resolveProvider(undefined)).toBe("nats");
+		});
+
+		it("explicit provider wins over the env var", () => {
+			process.env.BLOK_PUBSUB_ADAPTER = "gcp";
+			expect(resolveProvider("aws")).toBe("aws");
+		});
+	});
+
+	describe("createPubSubAdapter()", () => {
+		it("returns the correct provider name for each built-in", () => {
+			expect(createPubSubAdapter("nats").provider).toBe("nats");
+			expect(createPubSubAdapter("redis-streams").provider).toBe("redis-streams");
+			expect(createPubSubAdapter("kafka").provider).toBe("kafka");
+			expect(createPubSubAdapter("gcp").provider).toBe("gcp");
+			expect(createPubSubAdapter("aws").provider).toBe("aws");
+			expect(createPubSubAdapter("azure").provider).toBe("azure");
+		});
+
+		it("each call returns a fresh instance", () => {
+			const a = createPubSubAdapter("nats");
+			const b = createPubSubAdapter("nats");
+			expect(a).not.toBe(b);
+		});
+	});
+
+	describe("getOrCreateAdapter()", () => {
+		it("returns the same instance on repeated calls for the same provider", () => {
+			const a = getOrCreateAdapter("nats");
+			const b = getOrCreateAdapter("nats");
+			expect(a).toBe(b);
+		});
+
+		it("returns different instances for different providers", () => {
+			const nats = getOrCreateAdapter("nats");
+			const kafka = getOrCreateAdapter("kafka");
+			expect(nats).not.toBe(kafka);
+			expect(nats.provider).toBe("nats");
+			expect(kafka.provider).toBe("kafka");
+		});
+
+		it("_resetAdapterPoolForTests() drops cached instances", () => {
+			const first = getOrCreateAdapter("nats");
+			_resetAdapterPoolForTests();
+			const second = getOrCreateAdapter("nats");
+			expect(first).not.toBe(second);
+		});
+	});
+});