@shivaedev/jobs 0.22.1 → 0.23.0

package/src/server/internal/registerReleasedJobs.ts

@@ -3,6 +3,7 @@ import { WorkerRelease } from "@shivaedev/sql/WorkerRelease.ts";
 import type { PgBossClient } from "#client.ts";
 import type { JobRegistration } from "#definition.ts";
 import { toPgBossError } from "#error.ts";
+import type { PgBossRunner } from "#internal/batchHandler.ts";
 import { registerJobs, registrationName } from "./register.ts";
 
 export const registerReleasedJobs = Effect.fnUntraced(function* <R>(
@@ -10,16 +11,20 @@ export const registerReleasedJobs = Effect.fnUntraced(function* <R>(
 	jobs: readonly JobRegistration[],
 	context: Context.Context<R>,
 	replaceWorkers: boolean,
-	schema?: string,
+	execute: PgBossRunner,
+	schema: string | undefined,
+	graceful: boolean,
 ) {
 	const signal = yield* Effect.serviceOption(WorkerRelease);
 	const workers = jobs.filter((job) => job._tag !== "QueueDefinition");
 	const scope = yield* Effect.scope;
-	const register = registerJobs(client, jobs, context, replaceWorkers, schema).pipe(Effect.provideService(Scope.Scope, scope));
+	const register = registerJobs(client, jobs, context, replaceWorkers, execute, schema, true, graceful).pipe(
+		Effect.provideService(Scope.Scope, scope),
+	);
 	if (Option.isNone(signal) || workers.length === 0) {
 		return yield* register;
 	}
-	yield* registerJobs(client, jobs, context, replaceWorkers, schema, false);
+	yield* registerJobs(client, jobs, context, replaceWorkers, execute, schema, false, graceful);
 	let working = replaceWorkers;
 	let paused = false;
 	yield* signal.value.register((active) =>

package/src/server/internal/registerWorker.ts

@@ -1,11 +1,13 @@
-import { Clock, type Context, Effect, type Scope } from "effect";
+import { type Context, Effect, Fiber, Scope } from "effect";
 import type { Job, WorkOptions } from "pg-boss";
 import type { PgBossClient } from "#client.ts";
-import { batchHandler } from "#internal/batchHandler.ts";
+import { toPgBossError } from "#error.ts";
+import { batchHandler, type PgBossRunner } from "#internal/batchHandler.ts";
 import { deferrableAttempt } from "#internal/deferrableAttempt.ts";
+import { retryWake } from "#internal/retryWake.ts";
 import { traceJob } from "#internal/tracing.ts";
 
-export function registerWorker(
+export const registerWorker = Effect.fnUntraced(function* (
 	client: PgBossClient,
 	name: string,
 	options: WorkOptions,
@@ -14,47 +16,34 @@ export function registerWorker(
 	schema: string,
 	maxDeferrals: number,
 	context: Context.Context<unknown>,
-): Promise<string> {
-	const registration = client.work(
-		name,
-		{ ...options, includeMetadata: true },
-		batchHandler(
-			client,
-			name,
-			(job) => {
-				const attempt = traceJob(deferrableAttempt(run(job), client, schema, job, maxDeferrals), job, name);
-				return Effect.provide(attempt, context);
-			},
-			async (attempt) => {
-				if (client.findJobs === undefined || client.notifyWorker === undefined) {
-					return;
-				}
-				const job = (await client.findJobs(name, { id: attempt.id }))[0];
-				if (job?.state !== "retry") {
-					return;
-				}
-				const workId = await registration;
-				await Effect.runPromise(
-					Effect.gen(function* () {
-						// Host timers can fire early; PostgreSQL also retains precision that Date drops.
-						const deadline = job.startAfter.getTime() + 1;
-						let remaining = deadline - (yield* Clock.currentTimeMillis);
-						while (remaining > 0) {
-							yield* Effect.sleep(remaining);
-							remaining = deadline - (yield* Clock.currentTimeMillis);
-						}
-						const workerIds = client
-							.getWipData?.()
-							.filter((worker) => worker.workId === workId)
-							.map((worker) => worker.id) ?? [workId];
-						for (const workerId of workerIds) {
-							client.notifyWorker?.(workerId);
-						}
-					}).pipe(Effect.forkIn(scope)),
-				);
-			},
-			Effect.runPromiseWith(context),
-		),
+	execute: PgBossRunner,
+	graceful: boolean,
+) {
+	const fibers = yield* Scope.fork(scope);
+	function runJob(job: Job<unknown>) {
+		return Effect.provide(traceJob(deferrableAttempt(run(job), client, schema, job, maxDeferrals), job, name), context);
+	}
+	function afterFailure(attempt: Job<unknown>) {
+		return retryWake(client, name, attempt.id, registration).pipe(Effect.forkIn(fibers), Effect.asVoid);
+	}
+	const handle = yield* Effect.try({
+		catch: (error) => toPgBossError("register", error, name),
+		try: () => batchHandler(client, name, runJob, afterFailure),
+	});
+	const registration: Promise<string> = yield* Effect.try({
+		catch: (error) => toPgBossError("register", error, name),
+		try: () =>
+			client.work(name, { ...options, includeMetadata: true }, (jobs) =>
+				execute(handle(jobs).pipe(Effect.forkIn(fibers), Effect.flatMap(Fiber.join))),
+			),
+	});
+	const id = yield* Effect.tryPromise({ catch: (error) => toPgBossError("register", error, name), try: () => registration });
+	yield* Scope.addFinalizer(
+		scope,
+		Effect.tryPromise({
+			catch: (error) => toPgBossError("stop", error, name),
+			try: () => client.offWork(name, { id, wait: graceful }),
+		}).pipe(Effect.catch(Effect.logError)),
 	);
-	return registration;
-}
+	return id;
+});

package/src/server/internal/retryWake.ts

@@ -0,0 +1,33 @@
+import { Clock, Effect } from "effect";
+import { redactDiagnosticCause } from "@shivaedev/errors/redactCause.ts";
+import type { PgBossClient } from "#client.ts";
+import { toPgBossError } from "#error.ts";
+
+export function retryWake(client: PgBossClient, name: string, id: string, registration: Promise<string>): Effect.Effect<void> {
+	return Effect.gen(function* () {
+		const findJobs = client.findJobs?.bind(client);
+		if (findJobs === undefined || client.notifyWorker === undefined) {
+			return;
+		}
+		const jobs = yield* Effect.tryPromise({ catch: (error) => toPgBossError("settle", error, name), try: () => findJobs(name, { id }) });
+		const job = jobs[0];
+		if (job?.state !== "retry") {
+			return;
+		}
+		const workId = yield* Effect.tryPromise({ catch: (error) => toPgBossError("register", error, name), try: () => registration });
+		// Host timers can fire early; PostgreSQL retains precision that Date drops.
+		const deadline = job.startAfter.getTime() + 1;
+		let remaining = deadline - (yield* Clock.currentTimeMillis);
+		while (remaining > 0) {
+			yield* Effect.sleep(remaining);
+			remaining = deadline - (yield* Clock.currentTimeMillis);
+		}
+		const workerIds = client
+			.getWipData?.()
+			.filter((worker) => worker.workId === workId)
+			.map((worker) => worker.id) ?? [workId];
+		for (const workerId of workerIds) {
+			client.notifyWorker(workerId);
+		}
+	}).pipe(Effect.catchCause((cause) => Effect.logError(redactDiagnosticCause(cause))));
+}

package/src/server/internal/runQueueWorker.ts

@@ -0,0 +1,19 @@
+import { Effect, Redacted, Schema } from "effect";
+import type { Job } from "pg-boss";
+import { jobContext, type QueueWorker } from "#definition.ts";
+import { PgBossPayloadError } from "#error.ts";
+import { payloadOf } from "#internal/tracing.ts";
+
+export function runQueueWorker(worker: QueueWorker, job: Job<unknown>) {
+	return Schema.decodeUnknownEffect(worker.queue.schema)(payloadOf(job.data)).pipe(
+		Effect.mapError(
+			(error) =>
+				new PgBossPayloadError({
+					direction: "decode",
+					original: Redacted.make(error),
+					queue: worker.queue.name,
+				}),
+		),
+		Effect.flatMap((payload) => worker.handler(payload, jobContext(job))),
+	);
+}

package/src/server/internal/schemaIndexBuilds.ts

@@ -1,5 +1,9 @@
+import { Schema } from "effect";
 import type { PgBoss } from "pg-boss";
 
+const IndexBuilds = Schema.Array(Schema.Struct({ command: Schema.String, id: Schema.String }));
+const IndexValidity = Schema.Array(Schema.Struct({ invalid: Schema.Boolean }));
+
 const BUILD_PREFIX = "effect-pg-boss-schema:";
 
 function indexName(command: string): string {
@@ -30,11 +34,11 @@ export async function resumeSchemaIndexes(client: PgBoss, schema: string): Promi
 		`SELECT id, command FROM ${schema}.bam WHERE starts_with(name, $1) AND status <> 'completed' ORDER BY created_on, name`,
 		[BUILD_PREFIX],
 	);
-	for (const row of rows) {
+	for (const row of Schema.decodeUnknownSync(IndexBuilds)(rows)) {
 		const name = indexName(row.command);
 		if (/^\s*CREATE /mu.test(row.command)) {
 			const probe = await db.executeSql("SELECT NOT indisvalid AS invalid FROM pg_index WHERE indexrelid = to_regclass($1)", [`${schema}.${name}`]);
-			if (probe.rows[0]?.invalid === true) {
+			if (Schema.decodeUnknownSync(IndexValidity)(probe.rows)[0]?.invalid === true) {
 				await db.executeSql(`DROP INDEX CONCURRENTLY IF EXISTS ${schema}."${name}"`);
 			}
 		}

package/src/server/schema.ts

@@ -1,7 +1,7 @@
-import { Effect } from "effect";
+import { Effect, Schema } from "effect";
 import { SqlClient } from "effect/sql";
 import type { Connection } from "effect/sql/SqlConnection";
-import { getConstructionPlans, getMigrationPlans, type MigrationPartition, PgBoss } from "pg-boss";
+import { getConstructionPlans, getMigrationPlans, PgBoss } from "pg-boss";
 import pgBossPackage from "pg-boss/package.json" with { type: "json" };
 import type { SchemaComponent } from "@shivaedev/sql/SchemaComponent.ts";
 import { type PgBossError, toPgBossError } from "#error.ts";
@@ -9,6 +9,15 @@ import { deferralStorePlan } from "#internal/deferralStorePlan.ts";
 import { runSchemaPlan } from "#internal/runSchemaPlan.ts";
 import { resumeSchemaIndexes } from "#internal/schemaIndexBuilds.ts";
 
+const Partitions = Schema.Array(
+	Schema.Struct({
+		policy: Schema.String,
+		tableName: Schema.String,
+	}),
+);
+const InstalledSchema = Schema.Array(Schema.Struct({ name: Schema.NullOr(Schema.String) }));
+const InstalledVersion = Schema.Array(Schema.Struct({ version: Schema.Number }));
+
 export interface PgBossSchemaOptions {
 	readonly schema?: string;
 }
@@ -39,10 +48,13 @@ async function installSchema(
 		await resumeSchemaIndexes(client, schema);
 	}
 	if (version < pgBossPackage.pgboss.schema) {
-		const partitionTables: MigrationPartition[] = Array.from(
+		const partitionTables = Schema.decodeUnknownSync(Partitions)(
 			await run(connection.executeUnprepared(`SELECT table_name AS "tableName", policy FROM ${schema}.queue WHERE partition = true`, [], undefined)),
 		);
-		await runSchemaPlan(connection, getMigrationPlans(schema, version, { partitionTables }), run, { schema, version: pgBossPackage.pgboss.schema });
+		await runSchemaPlan(connection, getMigrationPlans(schema, version, { partitionTables: Array.from(partitionTables) }), run, {
+			schema,
+			version: pgBossPackage.pgboss.schema,
+		});
 		await resumeSchemaIndexes(client, schema);
 	}
 }
@@ -72,9 +84,13 @@ export function pgBossSchema(options: PgBossSchemaOptions = {}): SchemaComponent
 		pending: Effect.gen(function* () {
 			const sql = yield* SqlClient.SqlClient;
 			const table = sql`${sql(schema)}.${sql("version")}`;
-			const [installed] = yield* sql<{ name: string | null }>`select to_regclass(${table.compile()[0]})::text as name`.withoutTransform;
+			const [installed] = yield* sql`select to_regclass(${table.compile()[0]})::text as name`.withoutTransform.pipe(
+				Effect.flatMap(Schema.decodeUnknownEffect(InstalledSchema)),
+			);
 			const [row] =
-				installed === undefined || installed.name === null ? [] : yield* sql<{ version: number }>`select version from ${table}`.withoutTransform;
+				installed === undefined || installed.name === null
+					? []
+					: yield* sql`select version from ${table}`.withoutTransform.pipe(Effect.flatMap(Schema.decodeUnknownEffect(InstalledVersion)));
 			const required = pgBossPackage.pgboss.schema;
 			return row === undefined || row.version < required
 				? `schema ${schema}, installed version ${row?.version ?? "missing"}, required version ${required}`

package/src/server/service.ts

@@ -1,5 +1,7 @@
-import { Context, Effect, Layer, type Option, type Schema } from "effect";
+import { Context, Effect, Exit, Layer, type Option, Redacted, type Schema, Scope } from "effect";
 import type { ConstructorOptions, JobWithMetadata, SendOptions, StopOptions } from "pg-boss";
+import { defineService } from "@shivaedev/service/defineService.ts";
+import { genericMethod } from "@shivaedev/service/genericMethod.ts";
 import type { PgBossClientFactory } from "#client.ts";
 import type { JobPayloadSchema, JobRegistration, QueueDefinition, RegistrationRequirements } from "#definition.ts";
 import type { PgBossClientOptionsError, PgBossError, PgBossPayloadError, PgBossQueueOptionsError } from "#error.ts";
@@ -9,13 +11,14 @@ import { acquireClient, releaseClient } from "#internal/lifecycle.ts";
 import { makeService } from "#internal/makeService.ts";
 import { registrationName } from "#internal/register.ts";
 import { registerReleasedJobs } from "#internal/registerReleasedJobs.ts";
+import { PgBoss } from "#PgBoss.ts";
 
 export interface PgBossService {
 	readonly counts: (queue: Pick<QueueDefinition, "name">) => Effect.Effect<QueueCounts, PgBossError>;
 	readonly deadLetteredJobs: <const TName extends string, const TPayload extends JobPayloadSchema | typeof Schema.Null>(
 		queue: { readonly name: TName; readonly schema: TPayload },
 		options: JobPageOptions,
-	) => Effect.Effect<JobPage<DeadLetteredJob<TPayload["Type"]>>, PgBossError | PgBossPayloadError, TPayload["DecodingServices"]>;
+	) => Effect.Effect<JobPage<DeadLetteredJob<TPayload["Type"]>>, PgBossError | PgBossPayloadError | Schema.SchemaError, TPayload["DecodingServices"]>;
 	readonly enqueue: <const TName extends string, const TPayload extends JobPayloadSchema, const TQueue extends QueueDefinition<TName, TPayload>>(
 		queue: QueueDefinition<TName, TPayload> & TQueue,
 		payload: NoInfer<Schema.Schema.Type<TPayload>>,
@@ -24,77 +27,109 @@ export interface PgBossService {
 	readonly failedJobs: <const TName extends string, const TPayload extends JobPayloadSchema | typeof Schema.Null>(
 		queue: { readonly name: TName; readonly schema: TPayload },
 		options: JobPageOptions,
-	) => Effect.Effect<JobPage<ListedJob<TPayload["Type"]>>, PgBossError | PgBossPayloadError, TPayload["DecodingServices"]>;
+	) => Effect.Effect<JobPage<ListedJob<TPayload["Type"]>>, PgBossError | PgBossPayloadError | Schema.SchemaError, TPayload["DecodingServices"]>;
 	readonly getJob: <const TName extends string, const TPayload extends JobPayloadSchema | typeof Schema.Null>(
 		queue: { readonly name: TName; readonly schema: TPayload },
 		id: string,
 	) => Effect.Effect<Option.Option<JobWithMetadata<TPayload["Type"]>>, PgBossError | PgBossPayloadError, TPayload["DecodingServices"]>;
-	readonly health: Effect.Effect<JobsHealth, PgBossError>;
+	readonly health: () => Effect.Effect<JobsHealth, PgBossError>;
 	readonly retry: (queue: Pick<QueueDefinition, "name">, id: string) => Effect.Effect<void, PgBossError>;
 }
 
-export type PgBossLayerOptions<TRegistrations extends readonly JobRegistration[], TErrorRequirements> = ConstructorOptions & {
+export type PgBossOptions = Omit<ConstructorOptions, "connectionString"> & {
+	readonly connectionString?: Redacted.Redacted<string>;
 	readonly clientFactory?: PgBossClientFactory;
-	// One started client is reference-counted per key, so development module reloads reuse it instead of starting another.
 	readonly clientCacheKey?: string | symbol | undefined;
-	readonly jobs: TRegistrations;
-	readonly onError?: (error: Error) => Effect.Effect<unknown, never, TErrorRequirements>;
 	readonly stop?: StopOptions;
 };
 
+export function pgBossLayer(options: PgBossOptions): Layer.Layer<PgBoss, PgBossError | PgBossClientOptionsError> {
+	const { connectionString, clientFactory, clientCacheKey, stop, ...settings } = options;
+	return Layer.effect(
+		PgBoss,
+		Effect.gen(function* () {
+			const dependencies = yield* Effect.context<never>();
+			const execute = Effect.runPromiseWith(dependencies);
+			const acquired = yield* Effect.acquireRelease(
+				acquireClient(
+					{
+						clientCacheKey,
+						clientFactory,
+						constructor: {
+							...settings,
+							...(connectionString === undefined ? {} : { connectionString: Redacted.value(connectionString) }),
+						},
+					},
+					(error) => {
+						void execute(logClientError(error));
+					},
+				),
+				(client) => releaseClient(client, stop),
+			);
+			return { ...acquired, schema: settings.schema, stop };
+		}),
+	);
+}
+
+export interface PgBossLayerOptions<TRegistrations extends readonly JobRegistration[], TErrorRequirements> {
+	readonly jobs: TRegistrations;
+	readonly onError?: (error: Error) => Effect.Effect<unknown, never, TErrorRequirements>;
+}
+
 export interface PgBossDefinition<TIdentifier extends string = string> extends Context.Service<TIdentifier, PgBossService> {
 	readonly layer: <const TRegistrations extends readonly JobRegistration[], TErrorRequirements = never>(
 		options: PgBossLayerOptions<TRegistrations, TErrorRequirements>,
 	) => Layer.Layer<
 		TIdentifier,
-		PgBossError | PgBossQueueOptionsError | PgBossClientOptionsError,
-		Exclude<RegistrationRequirements<TRegistrations>, TIdentifier> | TErrorRequirements
+		PgBossError | PgBossQueueOptionsError,
+		PgBoss | Exclude<RegistrationRequirements<TRegistrations>, TIdentifier> | TErrorRequirements
 	>;
 }
 
 export function makePgBoss<const TIdentifier extends string>(identifier: TIdentifier): PgBossDefinition<TIdentifier> {
-	const Service = Context.Service<TIdentifier, PgBossService>(identifier);
-
+	function definition<R>(options: PgBossLayerOptions<readonly JobRegistration[], R>, dependencies: Context.Context<R>) {
+		return defineService({
+			id: identifier,
+			initialize: Effect.gen(function* () {
+				const acquired = yield* PgBoss;
+				const execute = Effect.runPromiseWith(dependencies);
+				const workerScope = yield* Scope.make();
+				const service = makeService(acquired.client, options.jobs.map(registrationName), acquired.schema);
+				const context = Context.add(dependencies, Service, service);
+				const reportError = options.onError ?? logClientError;
+				const unsubscribe = subscribeClientErrors(acquired.client, (error) => {
+					void execute(Effect.exit(Effect.provide(reportError(error), context)));
+				});
+				yield* Effect.addFinalizer(() => Scope.close(workerScope, Exit.void).pipe(Effect.ensuring(Effect.sync(unsubscribe))));
+				yield* registerReleasedJobs(
+					acquired.client,
+					options.jobs,
+					context,
+					acquired.reused,
+					(effect) => execute(effect),
+					acquired.schema,
+					acquired.stop?.graceful !== false,
+				).pipe(Effect.provideService(Scope.Scope, workerScope), Effect.provide(dependencies));
+				return service;
+			}),
+			methods: (service) => ({
+				counts: service.counts,
+				deadLetteredJobs: genericMethod(service.deadLetteredJobs),
+				enqueue: genericMethod(service.enqueue),
+				failedJobs: genericMethod(service.failedJobs),
+				getJob: genericMethod(service.getJob),
+				health: service.health,
+				retry: service.retry,
+			}),
+			requires: [PgBoss],
+		});
+	}
+	const Service: Context.Service<TIdentifier, PgBossService> = definition({ jobs: [] }, Context.empty());
 	function layer<const TRegistrations extends readonly JobRegistration[], TErrorRequirements = never>(
 		options: PgBossLayerOptions<TRegistrations, TErrorRequirements>,
-	): Layer.Layer<
-		TIdentifier,
-		PgBossError | PgBossQueueOptionsError | PgBossClientOptionsError,
-		Exclude<RegistrationRequirements<TRegistrations>, TIdentifier> | TErrorRequirements
-	> {
+	) {
 		type Requirements = Exclude<RegistrationRequirements<TRegistrations>, TIdentifier> | TErrorRequirements;
-		const names = options.jobs.map(registrationName);
-		const { clientFactory, clientCacheKey, jobs, onError, stop, ...constructorOptions } = options;
-
-		const acquire = Effect.acquireRelease(
-			Effect.gen(function* () {
-				const dependencies = yield* Effect.context<Requirements>();
-				const acquired = yield* acquireClient({
-					clientCacheKey,
-					clientFactory,
-					constructor: constructorOptions,
-				});
-				const service = makeService(acquired.client, names, constructorOptions.schema);
-				const context = Context.add(dependencies, Service, service);
-				const reportError: (error: Error) => Effect.Effect<unknown, never, Requirements | TIdentifier> = onError ?? logClientError;
-				function errorListener(error: Error): void {
-					void Effect.runPromise(Effect.exit(Effect.provide(reportError(error), context)));
-				}
-				const unsubscribe = subscribeClientErrors(acquired.client, errorListener);
-				const registration = registerReleasedJobs(acquired.client, jobs, context, acquired.reused, constructorOptions.schema);
-				return yield* registration.pipe(
-					Effect.as({ acquired, service, unsubscribe }),
-					Effect.onError(() => releaseClient(acquired, stop).pipe(Effect.ensuring(Effect.sync(unsubscribe)))),
-				);
-			}),
-			(resource) => releaseClient(resource.acquired, stop).pipe(Effect.ensuring(Effect.sync(resource.unsubscribe))),
-		);
-
-		return Layer.effect(
-			Service,
-			Effect.map(acquire, ({ service }) => service),
-		);
+		return Layer.unwrap(Effect.map(Effect.context<Requirements>(), (dependencies) => definition(options, dependencies).layer));
 	}
-
 	return Object.assign(Service, { layer });
 }