blokctl 1.3.0 → 1.4.1

package/dist/scaffold-repo/triggers/pubsub/src/adapters/GCPPubSubAdapter.ts

@@ -0,0 +1,327 @@
+/**
+ * GCPPubSubAdapter - Google Cloud Pub/Sub adapter for PubSubTrigger
+ *
+ * Uses @google-cloud/pubsub for GCP Pub/Sub connectivity.
+ * Requires: npm install @google-cloud/pubsub
+ *
+ * Environment variables:
+ * - GOOGLE_CLOUD_PROJECT: GCP project ID
+ * - GOOGLE_APPLICATION_CREDENTIALS: Path to service account key file (optional if using default credentials)
+ * - PUBSUB_EMULATOR_HOST: Pub/Sub emulator host for local development (optional)
+ */
+
+import type { PubSubTriggerOpts } from "@blokjs/helper";
+import type { Message, PubSub, Subscription } from "@google-cloud/pubsub";
+import { v4 as uuid } from "uuid";
+import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
+
+/**
+ * GCP Pub/Sub configuration
+ */
+export interface GCPPubSubConfig {
+	projectId?: string;
+	credentials?: {
+		client_email: string;
+		private_key: string;
+	};
+}
+
+/**
+ * GCPPubSubAdapter - Google Cloud Pub/Sub implementation
+ */
+export class GCPPubSubAdapter implements PubSubAdapter {
+	readonly provider = "gcp" as const;
+
+	private client: PubSub | undefined;
+	private subscriptions: Map<string, Subscription> = new Map();
+	private connected = false;
+	private config: GCPPubSubConfig;
+
+	/**
+	 * Type-narrowing accessor for `this.client`. Field is undefined until
+	 * `connect()` runs.
+	 */
+	private requireClient(): PubSub {
+		if (!this.client) {
+			throw new Error("[GCPPubSubAdapter] client is not initialised — call connect() first");
+		}
+		return this.client;
+	}
+
+	constructor(config?: GCPPubSubConfig) {
+		this.config = {
+			projectId: config?.projectId || process.env.GOOGLE_CLOUD_PROJECT,
+			credentials: config?.credentials,
+		};
+	}
+
+	/**
+	 * Connect to GCP Pub/Sub
+	 */
+	async connect(): Promise<void> {
+		if (this.connected) return;
+
+		try {
+			// Dynamic import of @google-cloud/pubsub
+			const { PubSub } = await import("@google-cloud/pubsub");
+
+			this.client = new PubSub({
+				projectId: this.config.projectId,
+				credentials: this.config.credentials,
+			});
+
+			this.connected = true;
+			console.log(`[GCPPubSubAdapter] Connected to GCP Pub/Sub: ${this.config.projectId}`);
+		} catch (error) {
+			throw new Error(
+				`Failed to connect to GCP Pub/Sub: ${(error as Error).message}. Make sure @google-cloud/pubsub is installed: npm install @google-cloud/pubsub`,
+			);
+		}
+	}
+
+	/**
+	 * Disconnect from GCP Pub/Sub
+	 */
+	async disconnect(): Promise<void> {
+		if (!this.connected) return;
+
+		try {
+			// Close all subscriptions
+			for (const [name, subscription] of this.subscriptions) {
+				await subscription.close();
+			}
+			this.subscriptions.clear();
+
+			await this.requireClient().close();
+			this.connected = false;
+			console.log("[GCPPubSubAdapter] Disconnected from GCP Pub/Sub");
+		} catch (error) {
+			console.error(`[GCPPubSubAdapter] Error disconnecting: ${(error as Error).message}`);
+		}
+	}
+
+	/**
+	 * Subscribe to a GCP Pub/Sub topic
+	 */
+	async subscribe(config: PubSubTriggerOpts, handler: (message: PubSubMessage) => Promise<void>): Promise<void> {
+		if (!this.connected) {
+			throw new Error("Not connected to GCP Pub/Sub. Call connect() first.");
+		}
+
+		if (!config.subscription) {
+			throw new Error("[GCPPubSubAdapter] `subscription` is required — must be the GCP subscription name.");
+		}
+		const subscriptionName = config.subscription;
+
+		// Get the subscription. The GCP SDK calls the per-subscription
+		// deadline `maxAckDeadline` (a `Duration`); our author-facing
+		// field is `ackDeadline` (seconds) for parity with other
+		// providers. `Duration` is the GCP SDK's tc39-Temporal-shaped
+		// shim — construct via `Duration.from({seconds})`.
+		const { Duration } = await import("@google-cloud/pubsub");
+		const subscription = this.requireClient().subscription(subscriptionName, {
+			flowControl: {
+				maxMessages: config.maxMessages || 10,
+			},
+			maxAckDeadline: Duration.from({ seconds: config.ackDeadline || 30 }),
+		});
+
+		// `deadLetterTopic` / `filter` are set at the GCP subscription
+		// resource level — they can only be provisioned at create time (or
+		// via setMetadata for the dead-letter policy). Reconcile them here:
+		// create the subscription with the requested policy, or if it
+		// already exists, verify it matches and throw on a mismatch rather
+		// than silently ignoring the author's intent.
+		await this.reconcileSubscriptionConfig(subscription, subscriptionName, config);
+
+		// Translate `startFrom` into a GCP seek BEFORE attaching listeners,
+		// so replay happens before the streaming pull starts delivering.
+		// GCP seeks by timestamp/snapshot; there is no sequence-number seek.
+		await this.applyStartFrom(subscription, config.startFrom);
+
+		// Message handler
+		const messageHandler = async (gcpMessage: Message) => {
+			// Parse message data
+			let body: unknown;
+			try {
+				const data = gcpMessage.data.toString();
+				body = JSON.parse(data);
+			} catch {
+				body = gcpMessage.data.toString();
+			}
+
+			// Create pub/sub message
+			const pubsubMessage: PubSubMessage = {
+				id: gcpMessage.id || uuid(),
+				body,
+				attributes: gcpMessage.attributes || {},
+				raw: gcpMessage,
+				topic: config.topic,
+				subscription: subscriptionName,
+				publishTime: gcpMessage.publishTime ? new Date(gcpMessage.publishTime) : new Date(),
+				ack: async () => {
+					gcpMessage.ack();
+				},
+				nack: async () => {
+					gcpMessage.nack();
+				},
+			};
+
+			// Process message
+			try {
+				await handler(pubsubMessage);
+			} catch (error) {
+				console.error(`[GCPPubSubAdapter] Error processing message: ${(error as Error).message}`);
+			}
+		};
+
+		// Error handler
+		const errorHandler = (error: Error) => {
+			console.error(`[GCPPubSubAdapter] Subscription error: ${error.message}`);
+		};
+
+		// Attach listeners
+		subscription.on("message", messageHandler);
+		subscription.on("error", errorHandler);
+
+		// Store subscription reference
+		this.subscriptions.set(subscriptionName, subscription);
+
+		console.log(`[GCPPubSubAdapter] Subscribed to: ${subscriptionName}`);
+	}
+
+	/**
+	 * Translate `startFrom` into a GCP `subscription.seek()`.
+	 *
+	 * - `"latest"` / unset → no-op (streaming pull already delivers only
+	 *   messages published after the subscriber attaches).
+	 * - `"earliest"` → seek to the epoch; GCP clamps to the retention floor
+	 *   and replays everything still retained.
+	 * - `{ timestamp }` (unix seconds) → seek to that wall-clock instant.
+	 * - `{ seq }` → rejected: GCP Pub/Sub has no sequence-number cursor.
+	 */
+	private async applyStartFrom(subscription: Subscription, startFrom: PubSubTriggerOpts["startFrom"]): Promise<void> {
+		if (startFrom === undefined || startFrom === "latest") return;
+		if (startFrom === "earliest") {
+			await subscription.seek(new Date(0));
+			return;
+		}
+		if ("timestamp" in startFrom) {
+			await subscription.seek(new Date(startFrom.timestamp * 1000));
+			return;
+		}
+		// { seq } — GCP Pub/Sub seeks by timestamp or snapshot only; there is
+		// no offset/sequence cursor. Reject loudly instead of silently
+		// dropping the author's replay intent.
+		throw new Error(
+			"[GCPPubSubAdapter] startFrom `{seq}` is not supported — GCP Pub/Sub seeks by timestamp or snapshot only. Use `{timestamp}`, `earliest`, or `latest`.",
+		);
+	}
+
+	/**
+	 * Provision `deadLetterTopic` / `filter` on the subscription resource.
+	 *
+	 * These are create-time properties in GCP. When the subscription does
+	 * not yet exist we create it with the requested policy/filter. When it
+	 * already exists we verify the live metadata matches — `filter` is
+	 * immutable so a mismatch throws; a dead-letter mismatch is patched via
+	 * `setMetadata` (the one field GCP lets you change after creation).
+	 */
+	private async reconcileSubscriptionConfig(
+		subscription: Subscription,
+		subscriptionName: string,
+		config: PubSubTriggerOpts,
+	): Promise<void> {
+		if (!config.deadLetterTopic && !config.filter) return;
+
+		const deadLetterPolicy = config.deadLetterTopic
+			? { deadLetterTopic: this.requireClient().topic(config.deadLetterTopic).name }
+			: undefined;
+
+		const [exists] = await subscription.exists();
+		if (!exists) {
+			// A subscription obtained via `client.subscription(name)` is
+			// detached from its topic and the SDK refuses to `.create()` it
+			// ("Subscriptions can only be created when accessed through
+			// Topics"). Create through the topic instead.
+			await this.requireClient()
+				.topic(config.topic)
+				.createSubscription(subscriptionName, {
+					...(config.filter ? { filter: config.filter } : {}),
+					...(deadLetterPolicy ? { deadLetterPolicy } : {}),
+				});
+			return;
+		}
+
+		const [metadata] = await subscription.getMetadata();
+
+		// `filter` is immutable after creation — a mismatch means the caller
+		// asked for a filter the pre-existing subscription can't honor.
+		if (config.filter && (metadata.filter || "") !== config.filter) {
+			throw new Error(
+				`[GCPPubSubAdapter] subscription "${subscriptionName}" already exists with filter "${metadata.filter || ""}" which differs from the requested filter "${config.filter}". GCP filters are immutable; delete and recreate the subscription.`,
+			);
+		}
+
+		if (deadLetterPolicy && metadata.deadLetterPolicy?.deadLetterTopic !== deadLetterPolicy.deadLetterTopic) {
+			await subscription.setMetadata({ deadLetterPolicy });
+		}
+	}
+
+	/**
+	 * Unsubscribe from a GCP Pub/Sub subscription
+	 */
+	async unsubscribe(subscriptionName: string): Promise<void> {
+		const subscription = this.subscriptions.get(subscriptionName);
+		if (subscription) {
+			await subscription.close();
+			this.subscriptions.delete(subscriptionName);
+			console.log(`[GCPPubSubAdapter] Unsubscribed from: ${subscriptionName}`);
+		}
+	}
+
+	/**
+	 * Check if connected to GCP Pub/Sub
+	 */
+	isConnected(): boolean {
+		return this.connected;
+	}
+
+	/**
+	 * v0.7 PR 6 — publish to a GCP Pub/Sub topic.
+	 *
+	 * `partitionKey` / `orderingKey` map to GCP's `orderingKey` (the
+	 * topic must have message ordering enabled — otherwise the field
+	 * is ignored by the broker).
+	 */
+	async publish(
+		topic: string,
+		payload: unknown,
+		opts?: { partitionKey?: string; orderingKey?: string },
+	): Promise<void> {
+		if (!this.connected) throw new Error("[blok][pubsub-gcp] not connected. Call connect() first.");
+		const body = typeof payload === "string" ? payload : JSON.stringify(payload);
+		const t = this.requireClient().topic(topic);
+		await t.publishMessage({
+			data: Buffer.from(body),
+			orderingKey: opts?.orderingKey ?? opts?.partitionKey,
+		});
+	}
+
+	/**
+	 * Health check - verify GCP Pub/Sub connectivity
+	 */
+	async healthCheck(): Promise<boolean> {
+		if (!this.connected) return false;
+
+		try {
+			// List subscriptions as a health check
+			const [subscriptions] = await this.requireClient().getSubscriptions({ pageSize: 1 });
+			return true;
+		} catch {
+			return false;
+		}
+	}
+}
+
+export default GCPPubSubAdapter;

package/dist/scaffold-repo/triggers/pubsub/src/adapters/KafkaPubSubAdapter.ts

@@ -0,0 +1,247 @@
+/**
+ * KafkaPubSubAdapter — v0.7 PR 6 — Pub/Sub adapter backed by Apache
+ * Kafka via `kafkajs`.
+ *
+ * Pub/Sub vs Worker semantics: this adapter uses **per-subscriber
+ * consumer groups** so multiple subscribers each receive every message
+ * (fan-out). When `consumerGroup` is explicitly set, all subscribers
+ * share that group and compete (1 of N gets each message).
+ *
+ * Replay cursors via `startFrom`:
+ *   - `"earliest"` → `fromBeginning: true` on first subscribe.
+ *   - `"latest"` (default) → only new messages.
+ *   - `{seq: N}` → `consumer.seek()` every assigned partition to offset N.
+ *   - `{timestamp: ms}` → resolve broker offsets at that timestamp, then seek.
+ *
+ * Requires `kafkajs` as a peer dependency.
+ *
+ * Environment variables:
+ *   - `KAFKA_BROKERS`           — comma-separated (default `localhost:9092`).
+ *   - `KAFKA_CLIENT_ID`         — default `"blok-pubsub"`.
+ *   - `KAFKA_SASL_USERNAME` / `KAFKA_SASL_PASSWORD`  — SASL/PLAIN.
+ *   - `KAFKA_SSL`               — `"true"` enables TLS.
+ */
+
+import type { PubSubTriggerOpts } from "@blokjs/helper";
+import { v4 as uuid } from "uuid";
+import type { PubSubAdapter, PubSubMessage } from "../PubSubTrigger";
+
+export interface KafkaPubSubConfig {
+	brokers: string[];
+	clientId: string;
+	saslUsername?: string;
+	saslPassword?: string;
+	ssl: boolean;
+}
+
+type KafkaSeekCursor = Extract<PubSubTriggerOpts["startFrom"], { seq: number } | { timestamp: number }>;
+
+interface KafkaConsumerHandle {
+	connect: () => Promise<void>;
+	disconnect: () => Promise<void>;
+	run: (opts: {
+		autoCommit: boolean;
+		eachMessage: (payload: {
+			message: { key?: Buffer; value?: Buffer; offset: string; timestamp: string; headers?: Record<string, Buffer> };
+		}) => Promise<void>;
+	}) => Promise<void>;
+	seek: (opts: { topic: string; partition: number; offset: string }) => void;
+	stop: () => Promise<void>;
+	subscribe: (opts: { topic: string; fromBeginning: boolean }) => Promise<void>;
+}
+
+interface KafkaAdminClient {
+	connect: () => Promise<void>;
+	disconnect: () => Promise<void>;
+	fetchTopicMetadata: (opts: { topics: string[] }) => Promise<{
+		topics: Array<{ partitions: Array<{ partitionId: number }> }>;
+	}>;
+	fetchTopicOffsetsByTimestamp: (
+		topic: string,
+		timestamp: number,
+	) => Promise<Array<{ partition: number; offset: string }>>;
+}
+
+export class KafkaPubSubAdapter implements PubSubAdapter {
+	readonly provider = "kafka" as const;
+	private readonly config: KafkaPubSubConfig;
+	// biome-ignore lint/suspicious/noExplicitAny: kafkajs's exported `Kafka` constructor is loosely typed.
+	private kafka: any = null;
+	// biome-ignore lint/suspicious/noExplicitAny: producer is created lazily and typed loosely.
+	private producer: any = null;
+	private consumers: Map<string, KafkaConsumerHandle> = new Map();
+	private connected = false;
+
+	constructor(config?: Partial<KafkaPubSubConfig>) {
+		this.config = {
+			brokers: config?.brokers ?? (process.env.KAFKA_BROKERS ?? "localhost:9092").split(",").map((s) => s.trim()),
+			clientId: config?.clientId ?? process.env.KAFKA_CLIENT_ID ?? "blok-pubsub",
+			saslUsername: config?.saslUsername ?? process.env.KAFKA_SASL_USERNAME,
+			saslPassword: config?.saslPassword ?? process.env.KAFKA_SASL_PASSWORD,
+			ssl: config?.ssl ?? process.env.KAFKA_SSL === "true",
+		};
+	}
+
+	async connect(): Promise<void> {
+		if (this.connected) return;
+		try {
+			// biome-ignore lint/suspicious/noExplicitAny: kafkajs is a runtime peer dep.
+			const kafkajs: any = await import("kafkajs");
+			const sasl =
+				this.config.saslUsername && this.config.saslPassword
+					? { mechanism: "plain", username: this.config.saslUsername, password: this.config.saslPassword }
+					: undefined;
+			this.kafka = new kafkajs.Kafka({
+				clientId: this.config.clientId,
+				brokers: this.config.brokers,
+				ssl: this.config.ssl,
+				sasl,
+			});
+			this.producer = this.kafka.producer();
+			await this.producer.connect();
+			this.connected = true;
+		} catch (err) {
+			throw new Error(
+				`[blok][pubsub-kafka] connect failed: ${(err as Error).message}. Install kafkajs as a peer dependency: bun add kafkajs`,
+			);
+		}
+	}
+
+	async disconnect(): Promise<void> {
+		if (!this.connected) return;
+		for (const consumer of this.consumers.values()) {
+			try {
+				await consumer.disconnect();
+			} catch {
+				/* ignore */
+			}
+		}
+		this.consumers.clear();
+		try {
+			await this.producer?.disconnect();
+		} catch {
+			/* ignore */
+		}
+		this.producer = null;
+		this.connected = false;
+	}
+
+	async subscribe(config: PubSubTriggerOpts, handler: (message: PubSubMessage) => Promise<void>): Promise<void> {
+		if (!this.connected) throw new Error("[blok][pubsub-kafka] not connected. Call connect() first.");
+		// Fan-out: distinct group id per subscriber instance. Competing-
+		// consumer: explicit consumerGroup shared across all subscribers.
+		const groupId =
+			config.consumerGroup ?? `blok-fanout-${uuid().slice(0, 8)}-${config.topic.replace(/[^a-zA-Z0-9_]/g, "_")}`;
+		const consumer = this.kafka.consumer({ groupId }) as KafkaConsumerHandle;
+		await consumer.connect();
+		const seekCursor = typeof config.startFrom === "object" ? config.startFrom : undefined;
+		const fromBeginning = config.startFrom === "earliest";
+		await consumer.subscribe({ topic: config.topic, fromBeginning });
+		this.consumers.set(`${config.topic}#${groupId}`, consumer);
+
+		await consumer.run({
+			autoCommit: config.ack !== false,
+			eachMessage: async ({
+				message,
+			}: {
+				message: { key?: Buffer; value?: Buffer; offset: string; timestamp: string; headers?: Record<string, Buffer> };
+			}) => {
+				const text = message.value?.toString("utf8") ?? "";
+				let body: unknown = text;
+				try {
+					body = text.length > 0 ? JSON.parse(text) : null;
+				} catch {
+					/* leave as text */
+				}
+				const attributes: Record<string, string> = {};
+				if (message.headers) {
+					for (const [k, v] of Object.entries(message.headers)) attributes[k] = v?.toString("utf8") ?? "";
+				}
+				// kafkajs has no per-message nack; the ONLY way to suppress the
+				// auto-commit of a failed offset is to throw out of eachMessage.
+				// PubSubTrigger.handleMessage catches handler errors and calls
+				// nack() (it never re-throws), so without this flag the offset
+				// would auto-commit and the failed message would be lost. We set
+				// a flag on nack() and throw after the handler returns.
+				let nacked = false;
+				const msg: PubSubMessage = {
+					id: `${config.topic}:${message.offset}`,
+					body,
+					attributes,
+					raw: message,
+					topic: config.topic,
+					subscription: groupId,
+					publishTime: new Date(Number.parseInt(message.timestamp, 10)),
+					ack: async () => {
+						/* autoCommit handles ack */
+					},
+					nack: async () => {
+						nacked = true;
+					},
+				};
+				await handler(msg);
+				// At-least-once (default): a nacked message must NOT be
+				// committed — throw so kafkajs redelivers it. At-most-once
+				// (ack:false) never commits per-message anyway, so a nack is a
+				// no-op there (the message is considered consumed once).
+				if (nacked && config.ack !== false) {
+					throw new Error(`[blok][pubsub-kafka] message ${msg.id} nacked — suppressing offset commit for redelivery`);
+				}
+			},
+		});
+		if (seekCursor) await this.applyStartFrom(consumer, config.topic, seekCursor);
+	}
+
+	private async applyStartFrom(consumer: KafkaConsumerHandle, topic: string, cursor: KafkaSeekCursor): Promise<void> {
+		const admin = this.kafka.admin() as KafkaAdminClient;
+		await admin.connect();
+		try {
+			if ("seq" in cursor) {
+				const offset = String(cursor.seq);
+				const metadata = await admin.fetchTopicMetadata({ topics: [topic] });
+				for (const partition of metadata.topics[0]?.partitions ?? []) {
+					consumer.seek({ topic, partition: partition.partitionId, offset });
+				}
+				return;
+			}
+			const offsets = await admin.fetchTopicOffsetsByTimestamp(topic, cursor.timestamp);
+			for (const { partition, offset } of offsets) consumer.seek({ topic, partition, offset });
+		} finally {
+			await admin.disconnect().catch(() => {});
+		}
+	}
+
+	async unsubscribe(subscription: string): Promise<void> {
+		const consumer = this.consumers.get(subscription);
+		if (!consumer) return;
+		try {
+			await consumer.stop();
+			await consumer.disconnect();
+		} catch {
+			/* ignore */
+		}
+		this.consumers.delete(subscription);
+	}
+
+	async publish(
+		topic: string,
+		payload: unknown,
+		opts?: { partitionKey?: string; orderingKey?: string },
+	): Promise<void> {
+		if (!this.connected || !this.producer) throw new Error("[blok][pubsub-kafka] not connected. Call connect() first.");
+		const body = typeof payload === "string" ? payload : JSON.stringify(payload);
+		const key = opts?.partitionKey ?? opts?.orderingKey;
+		await this.producer.send({
+			topic,
+			messages: [{ key, value: body }],
+		});
+	}
+
+	isConnected(): boolean {
+		return this.connected;
+	}
+
+	async healthCheck(): Promise<boolean> {
+		return this.connected;
+	}
+}