@shivaedev/jobs 0.0.0-stage → 0.19.0

package/src/server/internal/jobControl.ts

@@ -0,0 +1,44 @@
+import { Effect, Option } from "effect";
+import type { PgBossClient } from "#client.ts";
+import { toPgBossError } from "#error.ts";
+import { decodeStoredData } from "#internal/storedRows.ts";
+import type { PgBossService } from "#service.ts";
+
+export function getJobFor(client: PgBossClient): PgBossService["getJob"] {
+	return (queue, id) =>
+		Effect.tryPromise({
+			catch: (error) => toPgBossError("getJob", error, queue.name),
+			try: async () => {
+				if (client.findJobs === undefined) {
+					throw new TypeError("This pg-boss client does not support findJobs");
+				}
+				return (await client.findJobs(queue.name, { id }))[0];
+			},
+		}).pipe(
+			Effect.flatMap((job) =>
+				job === undefined
+					? Effect.succeed(Option.none())
+					: decodeStoredData(queue, job.data).pipe(Effect.map((data) => Option.some({ ...job, data }))),
+			),
+		);
+}
+
+export function retryFor(client: PgBossClient): PgBossService["retry"] {
+	return (queue, id) =>
+		Effect.tryPromise({
+			catch: (error) => toPgBossError("retry", error, queue.name),
+			try: async () => {
+				if (client.findJobs === undefined || client.retry === undefined || client.update === undefined) {
+					throw new TypeError("This pg-boss client does not support findJobs, retry and update");
+				}
+				const job = (await client.findJobs(queue.name, { id }))[0];
+				if (job?.state !== "failed" && job?.state !== "retry") {
+					return;
+				}
+				if (job.state === "failed") {
+					await client.retry(queue.name, id);
+				}
+				await client.update(queue.name, undefined, { id, startAfter: 0 });
+			},
+		});
+}

package/src/server/internal/lifecycle.ts

@@ -0,0 +1,68 @@
+import { Effect } from "effect";
+import type { ConstructorOptions, StopOptions } from "pg-boss";
+import { defaultClientFactory, type PgBossClient, type PgBossClientFactory } from "#client.ts";
+import { PgBossClientOptionsError, type PgBossError, toPgBossError } from "#error.ts";
+import { deferralStorePlan } from "#internal/deferralStorePlan.ts";
+import { acquireCachedClient, releaseCachedClient } from "./clientCache.ts";
+import { logClientError, observeClientErrors } from "./clientErrors.ts";
+
+export interface AcquireClientOptions {
+	readonly clientCacheKey?: string | symbol | undefined;
+	readonly clientFactory?: PgBossClientFactory | undefined;
+	readonly constructor: ConstructorOptions;
+}
+
+export interface AcquiredClient {
+	readonly cacheKey?: string | symbol;
+	readonly client: PgBossClient;
+	readonly reused: boolean;
+}
+
+async function startClient(options: AcquireClientOptions, report: (error: Error) => void): Promise<PgBossClient> {
+	const client = (options.clientFactory ?? defaultClientFactory)(options.constructor);
+	observeClientErrors(client, report);
+	try {
+		await client.start();
+		if (options.constructor.migrate !== false && client.getDb !== undefined) {
+			await client.getDb().executeSql(deferralStorePlan(options.constructor.schema ?? "pgboss"));
+		}
+		return client;
+	} catch (error) {
+		try {
+			await client.stop();
+		} catch {
+			// Preserve the startup failure; stop is best effort on failed acquisition.
+		}
+		throw error;
+	}
+}
+
+export function acquireClient(options: AcquireClientOptions): Effect.Effect<AcquiredClient, PgBossError | PgBossClientOptionsError> {
+	return Effect.contextWith((context) => {
+		const run = Effect.runForkWith(context);
+		function report(error: Error) {
+			run(logClientError(error));
+		}
+		return Effect.tryPromise({
+			catch: (error) => (error instanceof PgBossClientOptionsError ? error : toPgBossError("start", error)),
+			try: async () => {
+				const cacheKey = options.clientCacheKey;
+				return cacheKey === undefined
+					? { client: await startClient(options, report), reused: false }
+					: { cacheKey, ...(await acquireCachedClient(cacheKey, options.constructor, () => startClient(options, report))) };
+			},
+		});
+	});
+}
+
+export function releaseClient(acquired: AcquiredClient, stopOptions?: StopOptions): Effect.Effect<void> {
+	return Effect.tryPromise({
+		catch: (error) => toPgBossError("stop", error),
+		try: async () => {
+			const client = acquired.cacheKey === undefined ? acquired.client : await releaseCachedClient(acquired.cacheKey);
+			if (client !== undefined) {
+				await client.stop(stopOptions);
+			}
+		},
+	}).pipe(Effect.catch((error) => Effect.logError(error)));
+}

package/src/server/internal/listJobs.ts

@@ -0,0 +1,98 @@
+import { Effect, Option, Schema } from "effect";
+import type { PgBossClient } from "#client.ts";
+import type { JobPayloadSchema, QueueDefinition } from "#definition.ts";
+import { type PgBossError, type PgBossPayloadError, toPgBossError } from "#error.ts";
+import { type DeadLetteredJob, deadLetterQueueName, type JobPage, type JobPageOptions } from "#health.ts";
+import { decodeStoredData, quotedSchema } from "#internal/storedRows.ts";
+import type { PgBossService } from "#service.ts";
+
+const Cursor = Schema.StringFromBase64Url.pipe(Schema.decodeTo(Schema.fromJsonString(Schema.Tuple([Schema.String, Schema.String]))));
+
+const Row = Schema.Struct({
+	completedOn: Schema.OptionFromNullOr(Schema.DateTimeUtcFromDate),
+	createdOn: Schema.DateTimeUtcFromDate,
+	cursorAt: Schema.String,
+	data: Schema.Unknown,
+	id: Schema.String,
+	output: Schema.Unknown,
+	queue: Schema.String,
+	retryCount: Schema.Number,
+	retryLimit: Schema.Number,
+	sourceId: Schema.OptionFromNullOr(Schema.String),
+	sourceOutput: Schema.Unknown,
+	startedOn: Schema.OptionFromNullOr(Schema.DateTimeUtcFromDate),
+	state: Schema.Literals(["created", "retry", "active", "completed", "cancelled", "failed"]),
+});
+
+const Rows = Schema.Array(Row);
+
+type Listing = "deadLetteredJobs" | "failedJobs";
+
+export function failedJobsFor(client: PgBossClient, schema: string): PgBossService["failedJobs"] {
+	return (queue, options) =>
+		listJobs(client, schema, "failedJobs", queue, options).pipe(
+			Effect.map(({ jobs, next }) => ({ jobs: jobs.map(({ sourceId: _id, sourceOutput: _output, ...job }) => job), next })),
+		);
+}
+
+export function deadLetteredJobsFor(client: PgBossClient, schema: string): PgBossService["deadLetteredJobs"] {
+	return (queue, options) => listJobs(client, schema, "deadLetteredJobs", queue, options);
+}
+
+function listJobs<const TName extends string, const TPayload extends JobPayloadSchema>(
+	client: PgBossClient,
+	schema: string,
+	listing: Listing,
+	queue: Pick<QueueDefinition<TName, TPayload>, "name" | "schema">,
+	options: JobPageOptions,
+): Effect.Effect<JobPage<DeadLetteredJob<TPayload["Type"]>>, PgBossError | PgBossPayloadError, TPayload["DecodingServices"]> {
+	return Effect.tryPromise({
+		catch: (error) => toPgBossError(listing, error, queue.name),
+		try: async () => {
+			if (client.getDb === undefined) {
+				throw new TypeError("This pg-boss client does not support getDb");
+			}
+			if (!Number.isSafeInteger(options.limit) || options.limit < 1) {
+				throw new RangeError("The page limit must be a positive integer");
+			}
+			const after = options.cursor === undefined ? undefined : await Schema.decodeUnknownPromise(Cursor)(options.cursor);
+			const quoted = quotedSchema(schema);
+			const { rows } = await client.getDb().executeSql(
+				`SELECT
+						id,
+						name AS queue,
+						state,
+						retry_count AS "retryCount",
+						retry_limit AS "retryLimit",
+						created_on AS "createdOn",
+						started_on AS "startedOn",
+						completed_on AS "completedOn",
+						output,
+						data,
+						source_id AS "sourceId",
+						source_output AS "sourceOutput",
+						to_json(created_on) #>> '{}' AS "cursorAt"
+					FROM ${quoted}.job
+					WHERE name = $1
+						${listing === "failedJobs" ? "AND state = 'failed'" : ""}
+						AND ($2::timestamptz IS NULL OR (created_on, id) < ($2::timestamptz, $3::uuid))
+					ORDER BY created_on DESC, id DESC
+					LIMIT $4`,
+				[listing === "failedJobs" ? queue.name : deadLetterQueueName(queue.name), after?.[0] ?? null, after?.[1] ?? null, options.limit + 1],
+			);
+			return Schema.decodeUnknownPromise(Rows)(rows);
+		},
+	}).pipe(
+		Effect.flatMap((rows) => {
+			const page = rows.slice(0, options.limit);
+			const last = page.at(-1);
+			const next =
+				rows.length > options.limit && last !== undefined ? Option.some(Schema.encodeSync(Cursor)([last.cursorAt, last.id])) : Option.none();
+			return Effect.forEach(page, ({ cursorAt: _, data, ...row }) =>
+				decodeStoredData(queue, data).pipe(
+					Effect.map((decoded) => ({ ...row, attempt: Option.isSome(row.startedOn) ? row.retryCount + 1 : 0, data: decoded })),
+				),
+			).pipe(Effect.map((jobs) => ({ jobs, next })));
+		}),
+	);
+}

package/src/server/internal/makeService.ts

@@ -0,0 +1,19 @@
+import type { PgBossClient } from "#client.ts";
+import { countsFor } from "#internal/countsFor.ts";
+import { enqueueFor } from "#internal/enqueueFor.ts";
+import { healthFor } from "#internal/healthFor.ts";
+import { getJobFor, retryFor } from "#internal/jobControl.ts";
+import { deadLetteredJobsFor, failedJobsFor } from "#internal/listJobs.ts";
+import type { PgBossService } from "#service.ts";
+
+export function makeService(client: PgBossClient, names: readonly string[], schema = "pgboss"): PgBossService {
+	return {
+		counts: (queue) => countsFor(client, schema, queue.name),
+		deadLetteredJobs: deadLetteredJobsFor(client, schema),
+		enqueue: enqueueFor(client),
+		failedJobs: failedJobsFor(client, schema),
+		getJob: getJobFor(client),
+		health: healthFor(client, names),
+		retry: retryFor(client),
+	};
+}

package/src/server/internal/nativeClient.ts

@@ -0,0 +1,30 @@
+import { Socket } from "node:net";
+import { type ConstructorOptions, PgBoss } from "pg-boss";
+
+export function nativeClient(options: ConstructorOptions) {
+	const disconnected = new Set<Promise<void>>();
+	const config = {
+		...options,
+		stream: () => {
+			const socket = new Socket();
+			const closed = new Promise<void>((resolve) =>
+				socket.once("close", () => {
+					disconnected.delete(closed);
+					resolve();
+				}),
+			);
+			disconnected.add(closed);
+			return socket;
+		},
+	};
+	const client = new PgBoss(config);
+	const stop = client.stop.bind(client);
+	client.stop = async (stopOptions) => {
+		await stop(stopOptions);
+		// pg-boss's pool shutdown does not wait for its transports to disconnect.
+		if (stopOptions?.close !== false) {
+			await Promise.all([...disconnected]);
+		}
+	};
+	return client;
+}

package/src/server/internal/observeTestingSettlements.ts

@@ -0,0 +1,36 @@
+import type { PgBoss } from "pg-boss";
+
+export function observeTestingSettlements(client: PgBoss, settled: (name: string) => void) {
+	const observers = new Map<string, () => void>();
+	return {
+		close() {
+			for (const close of observers.values()) {
+				close();
+			}
+			observers.clear();
+		},
+		watch(name: string) {
+			if (observers.has(name)) {
+				return;
+			}
+			const spy = client.getSpy(name);
+			let open = true;
+			async function observe() {
+				while (true) {
+					await Promise.race([spy.waitForJob(() => true, "completed"), spy.waitForJob(() => true, "failed")]);
+					if (!open) {
+						return;
+					}
+					// This controller owns the spy; clear history and the losing wait before rearming both outcomes.
+					spy.clear();
+					settled(name);
+				}
+			}
+			observers.set(name, () => {
+				open = false;
+				spy.clear();
+			});
+			void observe().catch((error: unknown) => client.emit("error", error instanceof Error ? error : new Error(String(error))));
+		},
+	};
+}

package/src/server/internal/register.ts

@@ -0,0 +1,145 @@
+import { type Context, Effect, Redacted, Schema, type Scope } from "effect";
+import type { Job } from "pg-boss";
+import type { PgBossClient } from "#client.ts";
+import {
+	type JobRegistration,
+	jobContext,
+	type QueueDefinition,
+	type QueueWorker,
+	type ScheduleDefinition,
+	type ScheduledWorker,
+} from "#definition.ts";
+import { PgBossError, PgBossPayloadError, PgBossQueueOptionsError, toPgBossError } from "#error.ts";
+import { ensureQueue } from "#internal/ensureQueue.ts";
+import { registerWorker } from "#internal/registerWorker.ts";
+import { payloadOf } from "#internal/tracing.ts";
+
+function isQueueWorker(registration: JobRegistration): registration is QueueWorker {
+	return registration._tag === "QueueWorker";
+}
+
+function isScheduledWorker(registration: JobRegistration): registration is ScheduledWorker {
+	return registration._tag === "ScheduledWorker";
+}
+
+function isQueueDefinition(registration: JobRegistration): registration is QueueDefinition {
+	return registration._tag === "QueueDefinition";
+}
+
+function registrationDefinition(registration: JobRegistration): QueueDefinition | ScheduleDefinition {
+	if (isQueueDefinition(registration)) {
+		return registration;
+	}
+	if (isQueueWorker(registration)) {
+		return registration.queue;
+	}
+	if (isScheduledWorker(registration)) {
+		return registration.schedule;
+	}
+	throw new TypeError(`Unknown pg-boss registration: ${registration._tag}`);
+}
+
+export function registrationName(registration: JobRegistration): string {
+	return registrationDefinition(registration).name;
+}
+
+async function prepareQueues(client: PgBossClient, registrations: readonly JobRegistration[], replaceWorkers: boolean): Promise<void> {
+	for (const registration of registrations) {
+		if (replaceWorkers && !isQueueDefinition(registration)) {
+			await client.offWork(registrationName(registration));
+		}
+	}
+	for (const registration of registrations) {
+		const definition = registrationDefinition(registration);
+		try {
+			await ensureQueue(client, definition.name, definition.queueOptions);
+		} catch (error) {
+			throw error instanceof PgBossQueueOptionsError ? error : toPgBossError("register", error, definition.name);
+		}
+	}
+}
+
+// The layer requires RegistrationRequirements of its jobs, so its context provides every registered worker.
+function workerContext<R>(context: Context.Context<R>): Context.Context<unknown>;
+function workerContext(context: unknown): unknown {
+	return context;
+}
+
+function runQueueWorker(worker: QueueWorker, job: Job<unknown>) {
+	return Schema.decodeUnknownEffect(worker.queue.schema)(payloadOf(job.data)).pipe(
+		Effect.mapError(
+			(error) =>
+				new PgBossPayloadError({
+					direction: "decode",
+					original: Redacted.make(error),
+					queue: worker.queue.name,
+				}),
+		),
+		Effect.flatMap((payload) => worker.handler(payload, jobContext(job))),
+	);
+}
+
+async function registerQueue(
+	client: PgBossClient,
+	registration: QueueWorker | QueueDefinition,
+	context: Context.Context<unknown>,
+	scope: Scope.Scope,
+	schema: string,
+): Promise<void> {
+	const queue = registration._tag === "QueueWorker" ? registration.queue : registration;
+	const name = queue.name;
+	if (registration._tag === "QueueDefinition") {
+		return;
+	}
+	await registerWorker(client, name, queue.workerOptions, (job) => runQueueWorker(registration, job), scope, schema, queue.maxDeferrals, context);
+}
+
+async function registerSchedule(
+	client: PgBossClient,
+	worker: ScheduledWorker,
+	context: Context.Context<unknown>,
+	scope: Scope.Scope,
+	schema: string,
+): Promise<void> {
+	const { schedule } = worker;
+	await registerWorker(client, schedule.name, schedule.workerOptions, () => worker.effect, scope, schema, schedule.maxDeferrals, context);
+	await client.schedule(schedule.name, schedule.cron, null, schedule.scheduleOptions);
+}
+
+export function registrationNames(registrations: readonly JobRegistration[]): readonly string[] {
+	const names = registrations.map(registrationName);
+	if (new Set(names).size !== names.length) {
+		throw new TypeError("pg-boss job names must be unique");
+	}
+	return names;
+}
+
+export function registerJobs<R>(
+	client: PgBossClient,
+	registrations: readonly JobRegistration[],
+	context: Context.Context<R>,
+	replaceWorkers: boolean,
+	schema = "pgboss",
+	startWorkers = true,
+): Effect.Effect<void, PgBossError | PgBossQueueOptionsError, Scope.Scope> {
+	return Effect.flatMap(Effect.scope, (scope) =>
+		Effect.tryPromise({
+			catch: (error) => (error instanceof PgBossQueueOptionsError || error instanceof PgBossError ? error : toPgBossError("register", error)),
+			try: async () => {
+				registrationNames(registrations);
+				await prepareQueues(client, registrations, replaceWorkers);
+				if (!startWorkers) {
+					return;
+				}
+				const provided = workerContext(context);
+				for (const registration of registrations) {
+					if (isQueueWorker(registration) || isQueueDefinition(registration)) {
+						await registerQueue(client, registration, provided, scope, schema);
+					} else if (isScheduledWorker(registration)) {
+						await registerSchedule(client, registration, provided, scope, schema);
+					}
+				}
+			},
+		}),
+	);
+}

package/src/server/internal/registerReleasedJobs.ts

@@ -0,0 +1,44 @@
+import { type Context, Effect, Option, Scope } from "effect";
+import { WorkerRelease } from "@shivaedev/sql/WorkerRelease.ts";
+import type { PgBossClient } from "#client.ts";
+import type { JobRegistration } from "#definition.ts";
+import { toPgBossError } from "#error.ts";
+import { registerJobs, registrationName } from "./register.ts";
+
+export const registerReleasedJobs = Effect.fnUntraced(function* <R>(
+	client: PgBossClient,
+	jobs: readonly JobRegistration[],
+	context: Context.Context<R>,
+	replaceWorkers: boolean,
+	schema?: string,
+) {
+	const signal = yield* Effect.serviceOption(WorkerRelease);
+	const workers = jobs.filter((job) => job._tag !== "QueueDefinition");
+	const scope = yield* Effect.scope;
+	const register = registerJobs(client, jobs, context, replaceWorkers, schema).pipe(Effect.provideService(Scope.Scope, scope));
+	if (Option.isNone(signal) || workers.length === 0) {
+		return yield* register;
+	}
+	yield* registerJobs(client, jobs, context, replaceWorkers, schema, false);
+	let working = replaceWorkers;
+	let paused = false;
+	yield* signal.value.register((active) =>
+		Effect.gen(function* () {
+			if (active) {
+				yield* register;
+				working = true;
+				if (paused) {
+					yield* Effect.logInfo("Workers resumed for database release");
+				}
+			} else if (working) {
+				yield* Effect.tryPromise({
+					catch: (error) => toPgBossError("register", error),
+					try: () => Promise.all(workers.map((job) => client.offWork(registrationName(job)))),
+				});
+				working = false;
+				paused = true;
+				yield* Effect.logInfo("Workers paused for another database release");
+			}
+		}).pipe(Effect.provide(context)),
+	);
+});

package/src/server/internal/registerWorker.ts

@@ -0,0 +1,56 @@
+import { Clock, type Context, Effect, type Scope } from "effect";
+import type { Job, WorkOptions } from "pg-boss";
+import type { PgBossClient } from "#client.ts";
+import { batchHandler } from "#internal/batchHandler.ts";
+import { deferrableAttempt } from "#internal/deferrableAttempt.ts";
+import { traceJob } from "#internal/tracing.ts";
+
+export function registerWorker(
+	client: PgBossClient,
+	name: string,
+	options: WorkOptions,
+	run: (job: Job<unknown>) => Effect.Effect<unknown, unknown, unknown>,
+	scope: Scope.Scope,
+	schema: string,
+	maxDeferrals: number,
+	context: Context.Context<unknown>,
+): Promise<string> {
+	const registration = client.work(
+		name,
+		{ ...options, includeMetadata: true },
+		batchHandler(
+			client,
+			name,
+			(job) => {
+				const attempt = traceJob(deferrableAttempt(run(job), client, schema, job, maxDeferrals), job, name);
+				return Effect.provide(attempt, context);
+			},
+			async (attempt) => {
+				if (client.findJobs === undefined || client.notifyWorker === undefined) {
+					return;
+				}
+				const job = (await client.findJobs(name, { id: attempt.id }))[0];
+				if (job?.state !== "retry") {
+					return;
+				}
+				const workId = await registration;
+				await Effect.runPromise(
+					Effect.gen(function* () {
+						const now = yield* Clock.currentTimeMillis;
+						// PostgreSQL timestamps retain sub-millisecond precision that Date drops.
+						yield* Effect.sleep(Math.max(0, job.startAfter.getTime() + 1 - now));
+						const workerIds = client
+							.getWipData?.()
+							.filter((worker) => worker.workId === workId)
+							.map((worker) => worker.id) ?? [workId];
+						for (const workerId of workerIds) {
+							client.notifyWorker?.(workerId);
+						}
+					}).pipe(Effect.forkIn(scope)),
+				);
+			},
+			Effect.runPromiseWith(context),
+		),
+	);
+	return registration;
+}

package/src/server/internal/runSchemaPlan.ts

@@ -0,0 +1,32 @@
+import type { Effect } from "effect";
+import type { Connection } from "effect/sql/SqlConnection";
+import { pendingIndexInserts } from "./schemaIndexBuilds.ts";
+
+export async function runSchemaPlan(
+	connection: Connection,
+	plan: string,
+	run: <A, E>(effect: Effect.Effect<A, E>) => Promise<A>,
+	upgrade?: { readonly schema: string; readonly version: number },
+): Promise<void> {
+	// Native exported plans append concurrent index DDL after the migration transaction.
+	const parts = /^\s*BEGIN;(?<body>[\s\S]*)\n {4}COMMIT;(?<concurrent>[\s\S]*)$/u.exec(plan);
+	if (parts?.groups === undefined) {
+		throw new Error("Unrecognized pg-boss schema plan transaction");
+	}
+	const { body = "", concurrent = "" } = parts.groups;
+	function quote(text: string) {
+		return `'${text.replaceAll("'", "''")}'`;
+	}
+	const commands = concurrent
+		.trim()
+		.split(/;\s*(?:\n|$)/u)
+		.filter((statement) => statement.trim() !== "");
+	const pending = upgrade === undefined ? "" : pendingIndexInserts(upgrade.schema, upgrade.version, commands, quote);
+	// Effect SQL uses the extended protocol, which accepts one statement per command.
+	await run(connection.executeRaw(`DO ${quote(`BEGIN EXECUTE ${quote(`${body}\n${pending}`)}; END`)}`, []));
+	if (upgrade === undefined) {
+		for (const statement of commands) {
+			await run(connection.executeRaw(statement, []));
+		}
+	}
+}

package/src/server/internal/scheduleDefinition.ts

@@ -0,0 +1,21 @@
+import type { DefineScheduleOptions, ScheduleDefinition } from "#definition.ts";
+import { maxDeferrals, nativeQueueOptions } from "#internal/definitionOptions.ts";
+
+export function scheduleDefinition<const TName extends string>(options: DefineScheduleOptions<TName>): ScheduleDefinition<TName> {
+	const definition: ScheduleDefinition<TName> = {
+		_tag: "ScheduleDefinition",
+		cron: options.cron,
+		maxDeferrals: maxDeferrals(options.queue),
+		name: options.name,
+		queueOptions: nativeQueueOptions(options.queue),
+		run: (effect) =>
+			Object.freeze({
+				_tag: "ScheduledWorker" as const,
+				effect,
+				schedule: definition,
+			}),
+		scheduleOptions: Object.freeze({ tz: "UTC", ...options.schedule }),
+		workerOptions: Object.freeze({ batchSize: 10, burstWhenBatchFull: true, ...options.worker }),
+	};
+	return Object.freeze(definition);
+}

package/src/server/internal/schemaIndexBuilds.ts

@@ -0,0 +1,44 @@
+import type { PgBoss } from "pg-boss";
+
+const BUILD_PREFIX = "effect-pg-boss-schema:";
+
+function indexName(command: string): string {
+	const match =
+		/^\s*(?:CREATE (?:UNIQUE )?INDEX CONCURRENTLY IF NOT EXISTS |DROP INDEX CONCURRENTLY IF EXISTS (?:(?:"[^"]+"|\w+)\.))(?<name>\w+)/mu.exec(
+			command,
+		);
+	const name = match?.groups?.name;
+	if (name === undefined) {
+		throw new Error("Unrecognized pg-boss concurrent index command");
+	}
+	return name;
+}
+
+export function pendingIndexInserts(schema: string, version: number, commands: readonly string[], quote: (text: string) => string): string {
+	return commands
+		.map((command, position) => {
+			const table = indexName(command).replace(/_i\d+$/u, "");
+			const name = `${BUILD_PREFIX}${String(position).padStart(6, "0")}`;
+			return `INSERT INTO ${schema}.bam (name, version, table_name, command) VALUES (${quote(name)}, ${version}, ${quote(table)}, ${quote(command)});`;
+		})
+		.join("\n");
+}
+
+export async function resumeSchemaIndexes(client: PgBoss, schema: string): Promise<void> {
+	const db = client.getDb();
+	const { rows } = await db.executeSql(
+		`SELECT id, command FROM ${schema}.bam WHERE starts_with(name, $1) AND status <> 'completed' ORDER BY created_on, name`,
+		[BUILD_PREFIX],
+	);
+	for (const row of rows) {
+		const name = indexName(row.command);
+		if (/^\s*CREATE /mu.test(row.command)) {
+			const probe = await db.executeSql("SELECT NOT indisvalid AS invalid FROM pg_index WHERE indexrelid = to_regclass($1)", [`${schema}.${name}`]);
+			if (probe.rows[0]?.invalid === true) {
+				await db.executeSql(`DROP INDEX CONCURRENTLY IF EXISTS ${schema}."${name}"`);
+			}
+		}
+		await db.executeSql(row.command);
+		await db.executeSql(`UPDATE ${schema}.bam SET status = 'completed', completed_on = clock_timestamp() WHERE id = $1`, [row.id]);
+	}
+}