blokctl 1.3.0 → 1.4.0

package/dist/scaffold-repo/triggers/worker/src/adapters/PgBossAdapter.ts

@@ -0,0 +1,319 @@
+/**
+ * PgBossAdapter — v0.7 PR 5 — Worker adapter backed by `pg-boss`,
+ * which stores jobs in PostgreSQL. Operationally attractive when
+ * Postgres is already in the stack — no extra broker infra needed.
+ *
+ * Semantics:
+ *   - `pg-boss` handles concurrency, retries, dead-letter, priority,
+ *     delays, and exactly-once scheduling natively — the adapter
+ *     forwards `config.concurrency`, `config.retries`, etc. to the
+ *     `boss.work(queue, opts, handler)` and `boss.send(queue, data, opts)`
+ *     calls directly.
+ *   - **Single connection per process** — `pg-boss` manages its own
+ *     pool; we instantiate one `PgBoss` instance per `PgBossAdapter`.
+ *   - **Schema**: `pg-boss` creates its own schema (`pgboss` by
+ *     default) on first start. Tables are migrated automatically.
+ *
+ * Requires `pg-boss` as a peer dependency:
+ *
+ *     bun add pg-boss
+ *
+ * Environment variables:
+ *   - `DATABASE_URL` — Postgres connection string (default
+ *     `postgres://localhost:5432/blok`).
+ *   - `PG_BOSS_SCHEMA` — schema name (default `"pgboss"`).
+ */
+
+import type { WorkerTriggerOpts } from "@blokjs/helper";
+import { v4 as uuid } from "uuid";
+import type { WorkerAdapter, WorkerJob, WorkerQueueStats } from "../WorkerTrigger";
+
+export interface PgBossConfig {
+	connectionString: string;
+	schema?: string;
+}
+
+interface PgBossInstance {
+	start(): Promise<unknown>;
+	stop(opts?: { graceful?: boolean }): Promise<void>;
+	createQueue(queue: string, opts?: Record<string, unknown>): Promise<unknown>;
+	send(queue: string, data: unknown, opts?: Record<string, unknown>): Promise<string | null>;
+	work(
+		queue: string,
+		opts: Record<string, unknown>,
+		handler: (job: PgBossJob | PgBossJob[]) => Promise<unknown>,
+	): Promise<string>;
+	offWork(queue: string): Promise<void>;
+	getQueueSize(queue: string): Promise<number>;
+}
+
+interface PgBossJob {
+	id: string;
+	data: unknown;
+	name: string;
+}
+
+interface QueueStatsCounters {
+	completed: number;
+	failed: number;
+	active: number;
+}
+
+export class PgBossAdapter implements WorkerAdapter {
+	readonly provider = "pg-boss" as const;
+	private readonly config: PgBossConfig;
+	private boss: PgBossInstance | null = null;
+	private workIds: Map<string, string> = new Map();
+	private connected = false;
+	private stats: Map<string, QueueStatsCounters> = new Map();
+
+	constructor(config?: Partial<PgBossConfig>) {
+		this.config = {
+			connectionString: config?.connectionString ?? process.env.DATABASE_URL ?? "postgres://localhost:5432/blok",
+			schema: config?.schema ?? process.env.PG_BOSS_SCHEMA ?? "pgboss",
+		};
+	}
+
+	async connect(): Promise<void> {
+		if (this.connected) return;
+		try {
+			// Indirect specifier so tsc doesn't try to resolve types for
+			// the optional peer dependency at build time.
+			const moduleName = "pg-boss";
+			// biome-ignore lint/suspicious/noExplicitAny: pg-boss is a runtime peer dep.
+			const mod: any = await import(moduleName);
+			// biome-ignore lint/suspicious/noExplicitAny: pg-boss is a runtime peer dep.
+			const PgBoss: any = mod.default ?? mod;
+			this.boss = new PgBoss({
+				connectionString: this.config.connectionString,
+				schema: this.config.schema,
+			}) as PgBossInstance;
+			await this.boss.start();
+			this.connected = true;
+		} catch (err) {
+			throw new Error(
+				`[blok][pg-boss] connect failed: ${(err as Error).message}. Install pg-boss as a peer dependency: bun add pg-boss`,
+			);
+		}
+	}
+
+	async disconnect(): Promise<void> {
+		if (!this.connected) return;
+		try {
+			await this.boss?.stop({ graceful: true });
+		} catch {
+			/* ignore */
+		}
+		this.workIds.clear();
+		this.boss = null;
+		this.connected = false;
+	}
+
+	async process(config: WorkerTriggerOpts, handler: (job: WorkerJob) => Promise<void>): Promise<void> {
+		if (!this.connected || !this.boss) throw new Error("[blok][pg-boss] not connected. Call connect() first.");
+		this.stats.set(config.queue, { completed: 0, failed: 0, active: 0 });
+		const stats = this.stats.get(config.queue) as QueueStatsCounters;
+
+		// pg-boss v10 requires explicit queue creation before send/work.
+		// The call is idempotent — succeeds whether the queue exists or
+		// not — so this is safe to repeat across re-entries.
+		await this.ensureQueue(config.queue, config);
+
+		// pg-boss v10's handler receives an ARRAY of jobs (batch); the
+		// `batchSize` option caps how many jobs one poll fetches. pg-boss
+		// has no per-job `concurrency`/`teamSize` knob — its unit of
+		// parallelism is the batch the handler is handed — so we map
+		// `config.concurrency` (the requested parallelism) to `batchSize`.
+		// The handler loop below processes the batch; jobs within it run
+		// sequentially in this process, but a larger batch means more jobs
+		// drained per poll (= higher effective throughput). `concurrency`
+		// defaults to 1 in the worker schema → single-job fetch, matching
+		// the prior behaviour. `pollingIntervalSeconds` defaults to 2s
+		// which adds ~2s latency on the happy path — tighten via config.
+		const batchSize = config.concurrency && config.concurrency > 0 ? config.concurrency : 1;
+		const id = await this.boss.work(
+			config.queue,
+			{
+				batchSize,
+				includeMetadata: true,
+			},
+			async (jobOrBatch) => {
+				const jobs = Array.isArray(jobOrBatch) ? jobOrBatch : [jobOrBatch];
+				for (const job of jobs) {
+					await this.runOneJob(config, handler, stats, job);
+				}
+			},
+		);
+		this.workIds.set(config.queue, id);
+	}
+
+	private async runOneJob(
+		config: WorkerTriggerOpts,
+		handler: (job: WorkerJob) => Promise<void>,
+		stats: QueueStatsCounters,
+		job: PgBossJob,
+	): Promise<void> {
+		stats.active += 1;
+		// Tracks whether stats have already been counted by the
+		// WorkerJob API (`complete` / `fail`). Without this, both
+		// the callback and the wrapper's try/catch credit the same
+		// outcome, double-counting `stats.completed` /
+		// `stats.failed`. pg-boss itself handles ack/retry/DLQ via
+		// the handler's return-vs-throw — these callbacks exist
+		// purely for stats parity with the other worker adapters.
+		let settled = false;
+		const workerJob: WorkerJob = {
+			id: job.id,
+			data: job.data,
+			headers: {},
+			queue: config.queue,
+			priority: config.priority ?? 0,
+			attempts: 0,
+			maxRetries: config.retries ?? 0,
+			createdAt: new Date(),
+			timeout: config.timeout,
+			raw: job,
+			complete: async () => {
+				if (settled) return;
+				stats.completed += 1;
+				settled = true;
+			},
+			fail: async (err: Error) => {
+				if (settled) return;
+				stats.failed += 1;
+				settled = true;
+				// Re-throw so pg-boss's `boss.work` sees the handler
+				// failure and schedules a retry / drops to DLQ per
+				// the queue config — the contract is "handler throws
+				// => job failed".
+				throw err;
+			},
+		};
+		try {
+			await handler(workerJob);
+			if (!settled) {
+				stats.completed += 1;
+				settled = true;
+			}
+		} catch (err) {
+			if (!settled) {
+				stats.failed += 1;
+				settled = true;
+			}
+			throw err;
+		} finally {
+			stats.active = Math.max(0, stats.active - 1);
+		}
+	}
+
+	private async ensureQueue(queue: string, config?: WorkerTriggerOpts): Promise<void> {
+		if (!this.boss) return;
+		// Per-queue routing/retry policy. pg-boss 10 applies these at the
+		// QUEUE level (the `createQueue` options), not on `work()` — a
+		// failed job is retried `retryLimit` times then routed to the
+		// `deadLetter` queue automatically. The DLQ is itself a queue, so
+		// it must exist before it can receive dead-lettered jobs.
+		const queueOpts: Record<string, unknown> = { name: queue };
+		const dlq = config?.deadLetterQueue;
+		if (typeof dlq === "string" && dlq.length > 0) {
+			queueOpts.deadLetter = dlq;
+			await this.createQueueIdempotent(dlq, { name: dlq });
+		}
+		if (typeof config?.retries === "number") queueOpts.retryLimit = config.retries;
+		if (typeof config?.timeout === "number" && config.timeout > 0) {
+			queueOpts.expireInSeconds = Math.ceil(config.timeout / 1000);
+		}
+		await this.createQueueIdempotent(queue, queueOpts);
+	}
+
+	private async createQueueIdempotent(queue: string, opts: Record<string, unknown>): Promise<void> {
+		if (!this.boss) return;
+		try {
+			await this.boss.createQueue(queue, opts);
+		} catch (err) {
+			// pg-boss v10's createQueue is idempotent but may throw on
+			// race conditions when multiple processes call it concurrently
+			// against a fresh schema — those errors are harmless once the
+			// queue exists. Swallow + let downstream send/work surface a
+			// real problem if the queue isn't usable.
+			const message = (err as Error).message ?? "";
+			if (!message.includes("already exists")) {
+				// Log the unexpected case but don't propagate — operators
+				// shouldn't see a queue-creation race blow up their
+				// trigger boot path.
+				console.warn(`[blok][pg-boss] ensureQueue(${queue}) warning:`, message);
+			}
+		}
+	}
+
+	async addJob(
+		queue: string,
+		data: unknown,
+		opts?: { priority?: number; delay?: number; retries?: number; timeout?: number; jobId?: string },
+	): Promise<string> {
+		if (!this.connected || !this.boss) throw new Error("[blok][pg-boss] not connected. Call connect() first.");
+		// pg-boss v10 requires the queue to exist before send. The
+		// call is idempotent — cheap to repeat per add.
+		await this.ensureQueue(queue);
+		// pg-boss v10's `attorney.checkSendArgs` rejects any explicitly
+		// undefined `priority` / `retryLimit` / `startAfter` /
+		// `expireInSeconds` / `singletonKey` with "X must be an integer".
+		// Build the options object with only the keys we actually want
+		// set — caught by the real-broker integration test in
+		// `__tests__/integration/pgboss-adapter.real-pg.test.ts`.
+		const sendOpts: Record<string, unknown> = {};
+		if (typeof opts?.priority === "number") sendOpts.priority = opts.priority;
+		if (typeof opts?.delay === "number" && opts.delay > 0) {
+			sendOpts.startAfter = Math.ceil(opts.delay / 1000);
+		}
+		if (typeof opts?.retries === "number") sendOpts.retryLimit = opts.retries;
+		if (typeof opts?.timeout === "number") sendOpts.expireInSeconds = Math.ceil(opts.timeout / 1000);
+		if (typeof opts?.jobId === "string" && opts.jobId.length > 0) sendOpts.singletonKey = opts.jobId;
+
+		const jobId = await this.boss.send(queue, data, sendOpts);
+		return jobId ?? opts?.jobId ?? uuid();
+	}
+
+	async stopProcessing(queue: string): Promise<void> {
+		if (!this.connected || !this.boss) return;
+		try {
+			await this.boss.offWork(queue);
+		} catch {
+			/* ignore */
+		}
+		this.workIds.delete(queue);
+	}
+
+	isConnected(): boolean {
+		return this.connected;
+	}
+
+	async healthCheck(): Promise<boolean> {
+		if (!this.connected || !this.boss) return false;
+		try {
+			await this.boss.getQueueSize("__health_check__");
+			return true;
+		} catch {
+			// getQueueSize on a non-existent queue may throw — fall back to
+			// "connection is live" by checking the boss instance attribute.
+			return this.connected;
+		}
+	}
+
+	async getQueueStats(queue: string): Promise<WorkerQueueStats> {
+		const counters = this.stats.get(queue) ?? { completed: 0, failed: 0, active: 0 };
+		let waiting = 0;
+		try {
+			waiting = (await this.boss?.getQueueSize(queue)) ?? 0;
+		} catch {
+			/* ignore */
+		}
+		return {
+			waiting,
+			active: counters.active,
+			completed: counters.completed,
+			failed: counters.failed,
+			delayed: 0,
+		};
+	}
+}

package/dist/scaffold-repo/triggers/worker/src/adapters/RabbitMQAdapter.ts

@@ -0,0 +1,318 @@
+/**
+ * RabbitMQAdapter — v0.7 PR 5 — Worker adapter backed by RabbitMQ via
+ * the `amqplib` driver. Direct-exchange / queue model — the `queue`
+ * field maps to an AMQP queue name; messages are consumed with
+ * manual ACK. The default exchange (`""`) is used for publishing.
+ *
+ * Features:
+ *   - Concurrency via `prefetch(N)` per channel.
+ *   - Retries via `nack` with requeue=true until `retries` is hit, then
+ *     drop or DLQ-route based on `deadLetterQueue` config.
+ *   - Priorities via the `x-max-priority` queue arg (AMQP standard).
+ *   - Delayed delivery requires the `rabbitmq_delayed_message_exchange`
+ *     plugin; `opts.delay` throws a clear config error when used against a
+ *     broker without it rather than silently delivering immediately.
+ *
+ * Requires `amqplib` as a peer dependency:
+ *
+ *     bun add amqplib
+ *
+ * Environment variables:
+ *   - `AMQP_URL` — full AMQP connection string (default `amqp://localhost`).
+ *   - `AMQP_VHOST` — virtual host (default `/`).
+ */
+
+import type { WorkerTriggerOpts } from "@blokjs/helper";
+import { v4 as uuid } from "uuid";
+import type { WorkerAdapter, WorkerJob, WorkerQueueStats } from "../WorkerTrigger";
+
+export interface RabbitMQConfig {
+	url: string;
+	vhost?: string;
+}
+
+// AMQP priority queues cap at 255 (x-max-priority is a uint8). Pin the
+// declared ceiling so any caller-supplied priority is honoured without the
+// queue needing per-call re-declaration.
+const MAX_PRIORITY = 255;
+
+/**
+ * Inject a vhost into an AMQP connection string. amqplib reads the vhost from
+ * the URL path for string URLs, so the configured vhost has to live there.
+ * The default vhost "/" must be percent-encoded (`%2F`) — a literal slash in
+ * the path would be parsed as a sub-path, not the root vhost.
+ */
+function withVhost(url: string, vhost: string): string {
+	const u = new URL(url);
+	u.pathname = `/${encodeURIComponent(vhost)}`;
+	return u.toString();
+}
+
+interface RabbitChannel {
+	prefetch(n: number): Promise<void>;
+	assertQueue(
+		name: string,
+		opts?: Record<string, unknown>,
+	): Promise<{ queue: string; messageCount: number; consumerCount: number }>;
+	checkQueue(name: string): Promise<{ queue: string; messageCount: number; consumerCount: number }>;
+	consume(
+		queue: string,
+		cb: (
+			msg: {
+				content: Buffer;
+				fields: { deliveryTag: number; redelivered: boolean };
+				properties: { messageId?: string; priority?: number; timestamp?: number; headers?: Record<string, unknown> };
+			} | null,
+		) => void,
+		opts?: { noAck?: boolean; consumerTag?: string },
+	): Promise<{ consumerTag: string }>;
+	cancel(consumerTag: string): Promise<void>;
+	ack(msg: { fields: { deliveryTag: number } }): void;
+	nack(msg: { fields: { deliveryTag: number } }, allUpTo?: boolean, requeue?: boolean): void;
+	sendToQueue(queue: string, content: Buffer, opts?: Record<string, unknown>): boolean;
+	close(): Promise<void>;
+}
+
+interface RabbitConnection {
+	createChannel(): Promise<RabbitChannel>;
+	close(): Promise<void>;
+}
+
+interface QueueStatsCounters {
+	completed: number;
+	failed: number;
+	active: number;
+}
+
+export class RabbitMQAdapter implements WorkerAdapter {
+	readonly provider = "rabbitmq" as const;
+	private readonly config: RabbitMQConfig;
+	private conn: RabbitConnection | null = null;
+	private channels: Map<string, { channel: RabbitChannel; consumerTag?: string }> = new Map();
+	private connected = false;
+	private stats: Map<string, QueueStatsCounters> = new Map();
+
+	constructor(config?: Partial<RabbitMQConfig>) {
+		this.config = {
+			url: config?.url ?? process.env.AMQP_URL ?? "amqp://localhost",
+			vhost: config?.vhost ?? process.env.AMQP_VHOST,
+		};
+	}
+
+	async connect(): Promise<void> {
+		if (this.connected) return;
+		try {
+			// biome-ignore lint/suspicious/noExplicitAny: amqplib's connect returns a loose ConnectionLike.
+			const amqp: any = await import("amqplib");
+			// amqplib reads the vhost from the URL PATH for string URLs; the
+			// second arg is socket options only (vhost there is silently
+			// dropped). Encode the configured vhost into the path so it
+			// actually takes effect. The default vhost "/" must be encoded as
+			// %2F or it parses as an empty path.
+			const url = this.config.vhost ? withVhost(this.config.url, this.config.vhost) : this.config.url;
+			this.conn = (await amqp.connect(url)) as RabbitConnection;
+			this.connected = true;
+		} catch (err) {
+			throw new Error(
+				`[blok][rabbitmq] connect failed: ${(err as Error).message}. Install amqplib as a peer dependency: bun add amqplib`,
+			);
+		}
+	}
+
+	async disconnect(): Promise<void> {
+		if (!this.connected) return;
+		for (const [, entry] of this.channels) {
+			try {
+				if (entry.consumerTag) await entry.channel.cancel(entry.consumerTag);
+				await entry.channel.close();
+			} catch {
+				/* ignore */
+			}
+		}
+		this.channels.clear();
+		try {
+			await this.conn?.close();
+		} catch {
+			/* ignore */
+		}
+		this.conn = null;
+		this.connected = false;
+	}
+
+	async process(config: WorkerTriggerOpts, handler: (job: WorkerJob) => Promise<void>): Promise<void> {
+		if (!this.connected || !this.conn) throw new Error("[blok][rabbitmq] not connected. Call connect() first.");
+		const channel = await this.conn.createChannel();
+		await channel.prefetch(config.concurrency ?? 1);
+
+		const queueArgs: Record<string, unknown> = {};
+		if (config.deadLetterQueue) {
+			queueArgs["x-dead-letter-exchange"] = "";
+			queueArgs["x-dead-letter-routing-key"] = config.deadLetterQueue;
+			await channel.assertQueue(config.deadLetterQueue, { durable: true });
+		}
+		await channel.assertQueue(config.queue, { durable: true, maxPriority: MAX_PRIORITY, arguments: queueArgs });
+
+		this.stats.set(config.queue, { completed: 0, failed: 0, active: 0 });
+		const stats = this.stats.get(config.queue) as QueueStatsCounters;
+		const maxAttempts = (config.retries ?? 3) + 1;
+
+		const { consumerTag } = await channel.consume(
+			config.queue,
+			(msg) => {
+				if (!msg) return;
+				void (async () => {
+					const payloadString = msg.content.toString("utf8");
+					let data: unknown;
+					try {
+						data = payloadString.length > 0 ? JSON.parse(payloadString) : null;
+					} catch {
+						data = payloadString;
+					}
+					const headers: Record<string, string> = {};
+					for (const [k, v] of Object.entries(msg.properties.headers ?? {})) headers[k] = String(v);
+					const attempts = Number.parseInt(String(msg.properties.headers?.["x-blok-attempt"] ?? 0), 10);
+					// Tracks whether the message has already been ack/nack'd via
+					// the WorkerJob API (`complete` / `fail`). The wrapping
+					// try/catch below also wants to ack on handler success and
+					// nack on handler throw — without this flag we'd
+					// double-ack the same delivery tag and Rabbit closes the
+					// channel with PRECONDITION_FAILED (caught by the
+					// real-broker integration test in
+					// `__tests__/integration/rabbitmq-adapter.real-rabbitmq.test.ts`).
+					let settled = false;
+					const job: WorkerJob = {
+						id: msg.properties.messageId ?? `${config.queue}:${msg.fields.deliveryTag}`,
+						data,
+						headers,
+						queue: config.queue,
+						priority: msg.properties.priority ?? config.priority ?? 0,
+						attempts,
+						maxRetries: config.retries ?? 3,
+						createdAt: msg.properties.timestamp ? new Date(msg.properties.timestamp) : new Date(),
+						timeout: config.timeout,
+						raw: msg,
+						complete: async () => {
+							if (settled) return;
+							channel.ack(msg);
+							stats.completed += 1;
+							settled = true;
+						},
+						fail: async (err: Error, requeue?: boolean) => {
+							if (settled) return;
+							stats.failed += 1;
+							const exceeded = attempts + 1 >= maxAttempts;
+							channel.nack(msg, false, !exceeded && requeue !== false);
+							settled = true;
+						},
+					};
+					stats.active += 1;
+					try {
+						await handler(job);
+						if (!settled && config.ack !== false) {
+							channel.ack(msg);
+							stats.completed += 1;
+							settled = true;
+						}
+					} catch {
+						if (!settled) {
+							stats.failed += 1;
+							const exceeded = attempts + 1 >= maxAttempts;
+							channel.nack(msg, false, !exceeded);
+							settled = true;
+						}
+					} finally {
+						stats.active = Math.max(0, stats.active - 1);
+					}
+				})();
+			},
+			{ noAck: config.ack === false },
+		);
+		this.channels.set(config.queue, { channel, consumerTag });
+	}
+
+	async addJob(
+		queue: string,
+		data: unknown,
+		opts?: { priority?: number; delay?: number; retries?: number; timeout?: number; jobId?: string },
+	): Promise<string> {
+		if (!this.connected || !this.conn) throw new Error("[blok][rabbitmq] not connected. Call connect() first.");
+		// Delayed delivery requires the rabbitmq_delayed_message_exchange
+		// plugin (x-delay header → x-delayed-message exchange). Stock
+		// rabbitmq images do not ship it, so silently dropping the delay
+		// would deliver the job immediately and lie to the caller. Reject
+		// loudly instead.
+		if (typeof opts?.delay === "number" && opts.delay > 0) {
+			throw new Error(
+				"[blok][rabbitmq] delayed delivery (opts.delay) requires the rabbitmq_delayed_message_exchange plugin, which is not enabled on this broker. Enable the plugin or remove the delay.",
+			);
+		}
+		let channel = this.channels.get(queue)?.channel;
+		if (!channel) {
+			channel = await this.conn.createChannel();
+			await channel.assertQueue(queue, { durable: true, maxPriority: MAX_PRIORITY });
+		}
+		const messageId = opts?.jobId ?? uuid();
+		const headers: Record<string, unknown> = {};
+		const ok = channel.sendToQueue(queue, Buffer.from(typeof data === "string" ? data : JSON.stringify(data)), {
+			persistent: true,
+			messageId,
+			priority: opts?.priority,
+			timestamp: Date.now(),
+			headers,
+		});
+		if (!ok) {
+			// Channel is in flow-controlled state. The send is still
+			// accepted; the channel will emit a 'drain' event when ready.
+			// We don't currently surface backpressure to callers.
+		}
+		return messageId;
+	}
+
+	async stopProcessing(queue: string): Promise<void> {
+		const entry = this.channels.get(queue);
+		if (!entry) return;
+		try {
+			if (entry.consumerTag) await entry.channel.cancel(entry.consumerTag);
+			await entry.channel.close();
+		} catch {
+			/* ignore */
+		}
+		this.channels.delete(queue);
+	}
+
+	isConnected(): boolean {
+		return this.connected;
+	}
+
+	async healthCheck(): Promise<boolean> {
+		if (!this.connected || !this.conn) return false;
+		try {
+			const channel = await this.conn.createChannel();
+			await channel.close();
+			return true;
+		} catch {
+			return false;
+		}
+	}
+
+	async getQueueStats(queue: string): Promise<WorkerQueueStats> {
+		const counters = this.stats.get(queue) ?? { completed: 0, failed: 0, active: 0 };
+		let waiting = 0;
+		const entry = this.channels.get(queue);
+		if (entry) {
+			try {
+				const info = await entry.channel.checkQueue(queue);
+				waiting = info.messageCount;
+			} catch {
+				/* ignore */
+			}
+		}
+		return {
+			waiting,
+			active: counters.active,
+			completed: counters.completed,
+			failed: counters.failed,
+			delayed: 0,
+		};
+	}
+}