blokctl 1.3.0 → 1.4.0

package/dist/scaffold-repo/triggers/pubsub/src/adapters/KafkaPubSubAdapter.ts.bak

@@ -0,0 +1,251 @@
+/**
+ * KafkaPubSubAdapter — v0.7 PR 6 — Pub/Sub adapter backed by Apache
+ * Kafka via `kafkajs`.
+ *
+ * Pub/Sub vs Worker semantics: this adapter uses **per-subscriber
+ * consumer groups** so multiple subscribers each receive every message
+ * (fan-out). When `consumerGroup` is explicitly set, all subscribers
+ * share that group and compete (1 of N gets each message).
+ *
+ * Replay cursors via `startFrom`:
+ *   - `"earliest"` → `fromBeginning: true` on first subscribe.
+ *   - `"latest"` (default) → only new messages.
+ *   - `{seq: N}` / `{timestamp: ms}` — provider-specific. Cleanest
+ *     long-term path is Kafka's `admin.seek({offset|timestamp})` post-
+ *     subscribe; v1 honors `earliest` / `latest` and `{seq}` via
+ *     `auto.offset.reset` only.
+ *
+ * Requires `kafkajs` as a peer dependency.
+ *
+ * Environment variables:
+ *   - `KAFKA_BROKERS`           — comma-separated (default `localhost:9092`).
+ *   - `KAFKA_CLIENT_ID`         — default `"blok-pubsub"`.
+ *   - `KAFKA_SASL_USERNAME` / `KAFKA_SASL_PASSWORD`  — SASL/PLAIN.
+ *   - `KAFKA_SSL`               — `"true"` enables TLS.
+ */
+
+import type { PubSubTriggerOpts } from "@blokjs/helper";
+import { v4 as uuid } from "uuid";
+import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
+
+export interface KafkaPubSubConfig {
+	brokers: string[];
+	clientId: string;
+	saslUsername?: string;
+	saslPassword?: string;
+	ssl: boolean;
+}
+
+interface KafkaConsumerHandle {
+	disconnect: () => Promise<void>;
+	stop: () => Promise<void>;
+}
+
+export class KafkaPubSubAdapter implements PubSubAdapter {
+	readonly provider = "kafka" as const;
+	private readonly config: KafkaPubSubConfig;
+	// biome-ignore lint/suspicious/noExplicitAny: kafkajs's exported `Kafka` constructor is loosely typed.
+	private kafka: any = null;
+	// biome-ignore lint/suspicious/noExplicitAny: producer is created lazily and typed loosely.
+	private producer: any = null;
+	private consumers: Map<string, KafkaConsumerHandle> = new Map();
+	private connected = false;
+
+	constructor(config?: Partial<KafkaPubSubConfig>) {
+		this.config = {
+			brokers: config?.brokers ?? (process.env.KAFKA_BROKERS ?? "localhost:9092").split(",").map((s) => s.trim()),
+			clientId: config?.clientId ?? process.env.KAFKA_CLIENT_ID ?? "blok-pubsub",
+			saslUsername: config?.saslUsername ?? process.env.KAFKA_SASL_USERNAME,
+			saslPassword: config?.saslPassword ?? process.env.KAFKA_SASL_PASSWORD,
+			ssl: config?.ssl ?? process.env.KAFKA_SSL === "true",
+		};
+	}
+
+	async connect(): Promise<void> {
+		if (this.connected) return;
+		try {
+			// biome-ignore lint/suspicious/noExplicitAny: kafkajs is a runtime peer dep.
+			const kafkajs: any = await import("kafkajs");
+			const sasl =
+				this.config.saslUsername && this.config.saslPassword
+					? { mechanism: "plain", username: this.config.saslUsername, password: this.config.saslPassword }
+					: undefined;
+			this.kafka = new kafkajs.Kafka({
+				clientId: this.config.clientId,
+				brokers: this.config.brokers,
+				ssl: this.config.ssl,
+				sasl,
+			});
+			this.producer = this.kafka.producer();
+			await this.producer.connect();
+			this.connected = true;
+		} catch (err) {
+			throw new Error(
+				`[blok][pubsub-kafka] connect failed: ${(err as Error).message}. Install kafkajs as a peer dependency: bun add kafkajs`,
+			);
+		}
+	}
+
+	async disconnect(): Promise<void> {
+		if (!this.connected) return;
+		for (const consumer of this.consumers.values()) {
+			try {
+				await consumer.disconnect();
+			} catch {
+				/* ignore */
+			}
+		}
+		this.consumers.clear();
+		try {
+			await this.producer?.disconnect();
+		} catch {
+			/* ignore */
+		}
+		this.producer = null;
+		this.connected = false;
+	}
+
+	async subscribe(config: PubSubTriggerOpts, handler: (message: PubSubMessage) => Promise<void>): Promise<void> {
+		if (!this.connected) throw new Error("[blok][pubsub-kafka] not connected. Call connect() first.");
+		// Fan-out: distinct group id per subscriber instance. Competing-
+		// consumer: explicit consumerGroup shared across all subscribers.
+		const groupId =
+			config.consumerGroup ?? `blok-fanout-${uuid().slice(0, 8)}-${config.topic.replace(/[^a-zA-Z0-9_]/g, "_")}`;
+		const consumer = this.kafka.consumer({ groupId });
+		await consumer.connect();
+		// `{seq}`/`{timestamp}` cursors are applied via consumer.seek() AFTER
+		// run() (see below); they still need the offsets to exist in the log,
+		// so subscribe fromBeginning for any explicit cursor. Plain "earliest"
+		// keeps fromBeginning; "latest"/unset only wants new messages.
+		const seekCursor = typeof config.startFrom === "object" ? config.startFrom : undefined;
+		const fromBeginning = config.startFrom === "earliest" || seekCursor !== undefined;
+		await consumer.subscribe({ topic: config.topic, fromBeginning });
+		this.consumers.set(`${config.topic}#${groupId}`, consumer);
+
+		await consumer.run({
+			autoCommit: config.ack !== false,
+			eachMessage: async ({
+				message,
+			}: {
+				message: { key?: Buffer; value?: Buffer; offset: string; timestamp: string; headers?: Record<string, Buffer> };
+			}) => {
+				const text = message.value?.toString("utf8") ?? "";
+				let body: unknown = text;
+				try {
+					body = text.length > 0 ? JSON.parse(text) : null;
+				} catch {
+					/* leave as text */
+				}
+				const attributes: Record<string, string> = {};
+				if (message.headers) {
+					for (const [k, v] of Object.entries(message.headers)) attributes[k] = v?.toString("utf8") ?? "";
+				}
+				// kafkajs has no per-message nack; the ONLY way to suppress the
+				// auto-commit of a failed offset is to throw out of eachMessage.
+				// PubSubTrigger.handleMessage catches handler errors and calls
+				// nack() (it never re-throws), so without this flag the offset
+				// would auto-commit and the failed message would be lost. We set
+				// a flag on nack() and throw after the handler returns.
+				let nacked = false;
+				const msg: PubSubMessage = {
+					id: `${config.topic}:${message.offset}`,
+					body,
+					attributes,
+					raw: message,
+					topic: config.topic,
+					subscription: groupId,
+					publishTime: new Date(Number.parseInt(message.timestamp, 10)),
+					ack: async () => {
+						/* autoCommit handles ack */
+					},
+					nack: async () => {
+						nacked = true;
+					},
+				};
+				await handler(msg);
+				// At-least-once (default): a nacked message must NOT be
+				// committed — throw so kafkajs redelivers it. At-most-once
+				// (ack:false) never commits per-message anyway, so a nack is a
+				// no-op there (the message is considered consumed once).
+				if (nacked && config.ack !== false) {
+					throw new Error(`[blok][pubsub-kafka] message ${msg.id} nacked — suppressing offset commit for redelivery`);
+				}
+			},
+		});
+
+		// Apply `{seq}` / `{timestamp}` cursors. kafkajs requires seek() AFTER
+		// run() — it queues the seek and re-fetches from the requested offset
+		// once partitions are assigned. "earliest"/"latest" are already handled
+		// by fromBeginning above and need no seek.
+		if (seekCursor) await this.applyStartFrom(consumer, config.topic, seekCursor);
+	}
+
+	/**
+	 * Translate a `{seq}` / `{timestamp}` cursor into per-partition
+	 * `consumer.seek()` calls.
+	 *   - `{ seq }`       → seek every partition to that absolute offset.
+	 *   - `{ timestamp }` → resolve per-partition offsets at that wall-clock
+	 *     instant (unix SECONDS, matching the shared `startFrom` contract —
+	 *     kafkajs wants milliseconds, so ×1000) and seek each.
+	 */
+	private async applyStartFrom(
+		// biome-ignore lint/suspicious/noExplicitAny: kafkajs consumer is loosely typed.
+		consumer: any,
+		topic: string,
+		cursor: { seq: number } | { timestamp: number },
+	): Promise<void> {
+		const admin = this.kafka.admin();
+		await admin.connect();
+		try {
+			if ("seq" in cursor) {
+				const offset = String(cursor.seq);
+				const meta = await admin.fetchTopicMetadata({ topics: [topic] });
+				const partitions: number[] = meta.topics[0].partitions.map((p: { partitionId: number }) => p.partitionId);
+				for (const partition of partitions) consumer.seek({ topic, partition, offset });
+				return;
+			}
+			// unix SECONDS in the shared contract → kafkajs wants milliseconds.
+			const offsets: Array<{ partition: number; offset: string }> = await admin.fetchTopicOffsetsByTimestamp(
+				topic,
+				cursor.timestamp * 1000,
+			);
+			for (const { partition, offset } of offsets) consumer.seek({ topic, partition, offset });
+		} finally {
+			await admin.disconnect().catch(() => {});
+		}
+	}
+
+	async unsubscribe(subscription: string): Promise<void> {
+		const consumer = this.consumers.get(subscription);
+		if (!consumer) return;
+		try {
+			await consumer.stop();
+			await consumer.disconnect();
+		} catch {
+			/* ignore */
+		}
+		this.consumers.delete(subscription);
+	}
+
+	async publish(
+		topic: string,
+		payload: unknown,
+		opts?: { partitionKey?: string; orderingKey?: string },
+	): Promise<void> {
+		if (!this.connected || !this.producer) throw new Error("[blok][pubsub-kafka] not connected. Call connect() first.");
+		const body = typeof payload === "string" ? payload : JSON.stringify(payload);
+		const key = opts?.partitionKey ?? opts?.orderingKey;
+		await this.producer.send({
+			topic,
+			messages: [{ key, value: body }],
+		});
+	}
+
+	isConnected(): boolean {
+		return this.connected;
+	}
+
+	async healthCheck(): Promise<boolean> {
+		return this.connected;
+	}
+}

package/dist/scaffold-repo/triggers/pubsub/src/adapters/NATSPubSubAdapter.ts

@@ -0,0 +1,333 @@
+/**
+ * NATSPubSubAdapter — v0.7 PR 6 — Pub/Sub adapter backed by NATS.
+ *
+ * Two modes:
+ *   - **Fan-out** (default, when `consumerGroup` is absent): every
+ *     subscriber receives every message on the subject. NATS Core
+ *     publish/subscribe semantics — cheapest path, no persistence.
+ *   - **Competing-consumer** (when `consumerGroup` is set): NATS
+ *     Queue Group — exactly one subscriber in the named group
+ *     receives each message. Pure NATS Core.
+ *   - **Durable** (when `durable: true`): subscribe via NATS
+ *     JetStream consumer so the subscription survives restarts and
+ *     replays missed messages from `startFrom`. Required for the
+ *     `{seq}` / `{timestamp}` replay cursors.
+ *
+ * Subject wildcards (`orders.*.created`, `orders.>`) are honored by
+ * NATS natively in both modes.
+ *
+ * Requires `nats` as a peer dependency:
+ *
+ *     bun add nats
+ *
+ * Environment variables:
+ *   - `NATS_SERVERS`   — comma-separated URLs (default `localhost:4222`).
+ *   - `NATS_TOKEN`     — bearer token authentication.
+ *   - `NATS_USER` / `NATS_PASS`  — userpass authentication.
+ */
+
+import type { PubSubTriggerOpts } from "@blokjs/helper";
+import { v4 as uuid } from "uuid";
+import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
+
+export interface NATSPubSubConfig {
+	servers: string[];
+	token?: string;
+	user?: string;
+	pass?: string;
+}
+
+interface NatsSubscription {
+	unsubscribe: () => void | Promise<void>;
+}
+
+interface NatsConnection {
+	close: () => Promise<void>;
+	drain: () => Promise<void>;
+	subscribe: (
+		subject: string,
+		opts?: { queue?: string; callback?: (err: Error | null, msg: NatsMsg) => void },
+	) => NatsSubscription;
+	publish: (subject: string, payload: Uint8Array) => void;
+	jetstream?: () => NatsJetStream;
+	jetstreamManager?: () => Promise<NatsJetStreamManager>;
+}
+
+interface NatsJetStream {
+	subscribe: (
+		subject: string,
+		opts?: unknown,
+	) => Promise<{
+		[Symbol.asyncIterator]: () => AsyncIterator<NatsJsMsg>;
+		unsubscribe: () => Promise<void> | void;
+	}>;
+	publish: (subject: string, payload: Uint8Array) => Promise<unknown>;
+}
+
+interface NatsJetStreamManager {
+	streams: { add: (config: unknown) => Promise<unknown>; info: (name: string) => Promise<unknown> };
+	consumers: { add: (stream: string, config: unknown) => Promise<unknown> };
+}
+
+interface NatsMsg {
+	subject: string;
+	data: Uint8Array;
+	sid: number;
+	respond?: (data?: Uint8Array) => boolean;
+}
+
+interface NatsJsMsg {
+	subject: string;
+	data: Uint8Array;
+	seq: number;
+	ack: () => void;
+	nak: (millis?: number) => void;
+	info: { stream: string; consumer: string; redeliveryCount: number; timestampNanos?: number };
+}
+
+const TEXT_DECODER = new TextDecoder();
+const TEXT_ENCODER = new TextEncoder();
+
+export class NATSPubSubAdapter implements PubSubAdapter {
+	readonly provider = "nats" as const;
+	private readonly config: NATSPubSubConfig;
+	private conn: NatsConnection | null = null;
+	private subscriptions: Map<string, NatsSubscription | { unsubscribe: () => Promise<void> | void }> = new Map();
+	private connected = false;
+
+	constructor(config?: Partial<NATSPubSubConfig>) {
+		this.config = {
+			servers: config?.servers ?? (process.env.NATS_SERVERS ?? "localhost:4222").split(",").map((s) => s.trim()),
+			token: config?.token ?? process.env.NATS_TOKEN,
+			user: config?.user ?? process.env.NATS_USER,
+			pass: config?.pass ?? process.env.NATS_PASS,
+		};
+	}
+
+	async connect(): Promise<void> {
+		if (this.connected) return;
+		try {
+			// biome-ignore lint/suspicious/noExplicitAny: nats is a runtime peer dep.
+			const nats: any = await import("nats");
+			this.conn = (await nats.connect({
+				servers: this.config.servers,
+				token: this.config.token,
+				user: this.config.user,
+				pass: this.config.pass,
+			})) as NatsConnection;
+			this.connected = true;
+		} catch (err) {
+			throw new Error(
+				`[blok][pubsub-nats] connect failed: ${(err as Error).message}. Install nats as a peer dependency: bun add nats`,
+			);
+		}
+	}
+
+	async disconnect(): Promise<void> {
+		if (!this.connected) return;
+		for (const sub of this.subscriptions.values()) {
+			try {
+				const result = sub.unsubscribe();
+				if (result instanceof Promise) await result;
+			} catch {
+				/* ignore */
+			}
+		}
+		this.subscriptions.clear();
+		try {
+			await this.conn?.drain();
+		} catch {
+			/* ignore */
+		}
+		this.conn = null;
+		this.connected = false;
+	}
+
+	async subscribe(config: PubSubTriggerOpts, handler: (message: PubSubMessage) => Promise<void>): Promise<void> {
+		if (!this.connected || !this.conn) throw new Error("[blok][pubsub-nats] not connected. Call connect() first.");
+		const subKey = `${config.topic}#${config.consumerGroup ?? ""}`;
+
+		if (config.durable === true) {
+			// JetStream durable subscription — survives restarts.
+			if (!this.conn.jetstream) {
+				throw new Error("[blok][pubsub-nats] durable subscriptions require JetStream support in the nats client");
+			}
+			const jsm = await this.conn.jetstreamManager?.();
+			if (jsm) {
+				// Auto-create a stream covering the subject if one doesn't
+				// exist. Production deployments should pre-provision via
+				// `nats stream add` to control retention.
+				try {
+					await jsm.streams.add({
+						name: `blok-${(config.topic ?? "").replace(/[^a-zA-Z0-9_]/g, "_")}`,
+						subjects: [config.topic],
+					});
+				} catch {
+					/* stream already exists — ignore */
+				}
+			}
+			const js = this.conn.jetstream();
+			// nats.js `js.subscribe` needs a ConsumerOptsBuilder, NOT a raw
+			// `{ config }` object — only the builder installs the push
+			// `deliver_subject` JetStream requires. A raw config throws
+			// "push consumer requires deliver_subject" at runtime (the typed
+			// API accepts it, so tsc never catches this).
+			// biome-ignore lint/suspicious/noExplicitAny: nats is a runtime peer dep.
+			const nats: any = await import("nats");
+			const durableName =
+				config.consumerGroup ??
+				`blok-${(config.subscription ?? config.topic ?? "default").replace(/[^a-zA-Z0-9_]/g, "_")}`;
+			const opts = nats.consumerOpts();
+			opts.durable(durableName);
+			opts.deliverTo(nats.createInbox());
+			opts.manualAck(); // dispatchJsMessage ack/naks explicitly
+			if (config.ack === false) opts.ackNone();
+			else opts.ackExplicit();
+			// Replay position: `earliest` = full retained history; `{seq}` /
+			// `{timestamp}` resume from a cursor; `latest` (and the default,
+			// per the schema) = new messages only.
+			if (config.startFrom === "earliest") {
+				opts.deliverAll();
+			} else if (typeof config.startFrom === "object" && config.startFrom && "seq" in config.startFrom) {
+				opts.startSequence(config.startFrom.seq);
+			} else if (typeof config.startFrom === "object" && config.startFrom && "timestamp" in config.startFrom) {
+				opts.startTime(new Date(config.startFrom.timestamp));
+			} else {
+				opts.deliverNew();
+			}
+			const sub = await js.subscribe(config.topic, opts);
+			this.subscriptions.set(subKey, { unsubscribe: () => sub.unsubscribe() });
+			// Drive the async iterator in a background loop.
+			void (async () => {
+				try {
+					for await (const msg of sub as unknown as AsyncIterable<NatsJsMsg>) {
+						await this.dispatchJsMessage(msg, config, handler);
+					}
+				} catch (err) {
+					// Subscription closed or connection lost — let the trigger
+					// re-listen via HMR/reconnect logic. Log for visibility.
+					console.error(`[blok][pubsub-nats] subscription error: ${(err as Error).message}`);
+				}
+			})();
+			return;
+		}
+
+		// Core NATS subscription — fire-and-forget, with optional queue
+		// group for competing-consumer semantics.
+		const sub = this.conn.subscribe(config.topic, {
+			queue: config.consumerGroup,
+			callback: (err, msg) => {
+				if (err) {
+					console.error(`[blok][pubsub-nats] subscribe error: ${err.message}`);
+					return;
+				}
+				void this.dispatchCoreMessage(msg, config, handler);
+			},
+		});
+		this.subscriptions.set(subKey, sub);
+	}
+
+	private async dispatchJsMessage(
+		msg: NatsJsMsg,
+		config: PubSubTriggerOpts,
+		handler: (message: PubSubMessage) => Promise<void>,
+	): Promise<void> {
+		const text = TEXT_DECODER.decode(msg.data);
+		let body: unknown = text;
+		try {
+			body = text.length > 0 ? JSON.parse(text) : null;
+		} catch {
+			/* leave as text */
+		}
+		const message: PubSubMessage = {
+			id: `${msg.info.stream}:${msg.seq}`,
+			body,
+			attributes: { subject: msg.subject },
+			raw: msg,
+			topic: msg.subject,
+			subscription: msg.info.consumer,
+			publishTime: msg.info.timestampNanos ? new Date(msg.info.timestampNanos / 1e6) : undefined,
+			ack: async () => {
+				msg.ack();
+			},
+			nack: async () => {
+				msg.nak();
+			},
+		};
+		try {
+			await handler(message);
+			if (config.ack !== false) msg.ack();
+		} catch {
+			msg.nak();
+		}
+	}
+
+	private async dispatchCoreMessage(
+		msg: NatsMsg,
+		_config: PubSubTriggerOpts,
+		handler: (message: PubSubMessage) => Promise<void>,
+	): Promise<void> {
+		const text = TEXT_DECODER.decode(msg.data);
+		let body: unknown = text;
+		try {
+			body = text.length > 0 ? JSON.parse(text) : null;
+		} catch {
+			/* leave as text */
+		}
+		const message: PubSubMessage = {
+			id: `${msg.subject}:${msg.sid}:${uuid()}`,
+			body,
+			attributes: { subject: msg.subject },
+			raw: msg,
+			topic: msg.subject,
+			ack: async () => {
+				/* core NATS has no explicit ack */
+			},
+			nack: async () => {
+				/* core NATS has no explicit nack */
+			},
+		};
+		try {
+			await handler(message);
+		} catch (err) {
+			console.error(`[blok][pubsub-nats] handler error: ${(err as Error).message}`);
+		}
+	}
+
+	async unsubscribe(subscription: string): Promise<void> {
+		const sub = this.subscriptions.get(subscription);
+		if (!sub) return;
+		try {
+			const result = sub.unsubscribe();
+			if (result instanceof Promise) await result;
+		} catch {
+			/* ignore */
+		}
+		this.subscriptions.delete(subscription);
+	}
+
+	async publish(topic: string, payload: unknown): Promise<void> {
+		if (!this.connected || !this.conn) throw new Error("[blok][pubsub-nats] not connected. Call connect() first.");
+		const body = typeof payload === "string" ? payload : JSON.stringify(payload);
+		const data = TEXT_ENCODER.encode(body);
+		// Use core NATS publish. When a durable subscriber has been
+		// installed for this subject, the subscribe() path created a
+		// JetStream stream with a subject filter that captures any
+		// publish to `topic` — so durable consumers see the message via
+		// the stream while core subscribers see it directly. The earlier
+		// "try js.publish first, fall back to core" logic caused
+		// **double-delivery**: js.publish to a subject covered by a
+		// stream goes to BOTH the stream and core subscribers, then the
+		// fallback would publish AGAIN if js.publish timed out (vs
+		// returning 503 fast). Sticking to core publish avoids the race.
+		this.conn.publish(topic, data);
+	}
+
+	isConnected(): boolean {
+		return this.connected;
+	}
+
+	async healthCheck(): Promise<boolean> {
+		return this.connected;
+	}
+}