@shivaedev/effect-pg-boss 0.10.1 → 0.12.0

package/src/server/internal/service.ts

@@ -33,7 +33,7 @@ export const makeService = (client: PgBossClient, names: readonly string[], sche
 					Effect.flatMap((data) =>
 						Effect.tryPromise({
 							catch: (error) => toPgBossError("enqueue", error, queue.name),
-							try: () => client.send(queue.name, data, options),
+							try: () => client.send(queue.name, data, queue.key === undefined ? options : { ...options, singletonKey: queue.key(payload) }),
 						}),
 					),
 					Effect.map(Option.fromNullishOr),

package/src/server/internal/settleDeferral.ts

@@ -0,0 +1,53 @@
+import { Schema } from "effect";
+import type { Job } from "pg-boss";
+import type { PgBossClient } from "#client.ts";
+import { STATELY_DEFERRAL_POLICY } from "./deferral.ts";
+
+const Settlement = Schema.Struct({ claimed: Schema.Boolean, deferred: Schema.Boolean });
+
+export async function settleDeferral(client: PgBossClient, schema: string, job: Job<unknown>, at: string, reason: string, maxDeferrals: number) {
+	if (client.getDb === undefined) {
+		throw new TypeError("This pg-boss client does not support getDb for RetryAt");
+	}
+	const quoted = `"${schema.replaceAll('"', '""')}"`;
+	// Native failure settlement deletes and reinserts jobs, so a cascading job foreign key would erase the counter.
+	const { rows } = await client.getDb().executeSql(
+		`WITH claimed AS MATERIALIZED (
+			SELECT name, id, created_on,
+				GREATEST(keep_until, $4::timestamptz + (keep_until - start_after)) AS keep_until
+			FROM ${quoted}.job
+			WHERE name = $1 AND id = $2 AND state = 'active' AND retry_count = $3
+			AND date_trunc('milliseconds', started_on) = $6::timestamptz
+			FOR UPDATE
+		), counted AS (
+			INSERT INTO ${quoted}.effect_job_deferral AS previous (name, id, created_on, count, keep_until)
+			SELECT name, id, created_on, 1, keep_until FROM claimed
+			ON CONFLICT (name, id) DO UPDATE SET
+				count = CASE WHEN previous.created_on = EXCLUDED.created_on THEN previous.count + 1 ELSE 1 END,
+				created_on = EXCLUDED.created_on, keep_until = EXCLUDED.keep_until
+			WHERE previous.created_on <> EXCLUDED.created_on OR previous.count < $7
+			RETURNING name, id, keep_until
+		), deferred AS (
+			UPDATE ${quoted}.job AS job SET state = 'retry', start_after = $4::timestamptz,
+				policy = CASE WHEN job.policy = 'stately' THEN '${STATELY_DEFERRAL_POLICY}' ELSE job.policy END,
+				keep_until = counted.keep_until, started_on = NULL, heartbeat_on = NULL,
+				output = jsonb_build_object('jobDeferral', jsonb_build_object('at', $4::text, 'reason', $5::text))
+			FROM counted WHERE job.name = counted.name AND job.id = counted.id
+			RETURNING job.id
+		), cleaned AS (
+			DELETE FROM ${quoted}.effect_job_deferral WHERE (name, id) IN (
+				SELECT stale.name, stale.id FROM ${quoted}.effect_job_deferral AS stale
+				WHERE EXISTS (SELECT 1 FROM counted) AND stale.keep_until < ${quoted}.job_now()
+				AND (stale.name, stale.id) <> ($1, $2)
+				AND NOT EXISTS (
+					SELECT 1 FROM ${quoted}.job AS live
+					WHERE live.name = stale.name AND live.id = stale.id AND live.created_on = stale.created_on
+				)
+				ORDER BY stale.name, stale.id LIMIT 100 FOR UPDATE OF stale SKIP LOCKED
+			)
+		)
+		SELECT EXISTS (SELECT 1 FROM claimed) AS claimed, EXISTS (SELECT 1 FROM deferred) AS deferred`,
+		[job.name, job.id, job.retryCount, at, reason, Reflect.get(job, "startedOn"), maxDeferrals],
+	);
+	return Schema.decodeUnknownSync(Settlement)(rows[0]);
+}

package/src/server/internal/testingClient.ts

@@ -1,5 +1,6 @@
 import type { Job, PgBoss, SendOptions, UpdateOptions, UpdateRequest, UpdateResponse } from "pg-boss";
 import type { PgBossClient } from "#client.ts";
+import { deferralObservers } from "#internal/deferral.ts";
 import { expireAttempt } from "#internal/expireAttempt.ts";
 import { observeTestingSettlements } from "#internal/observeTestingSettlements.ts";
 import { testingExpirations } from "#internal/testingExpirations.ts";
@@ -102,7 +103,7 @@ export function testingClient(client: PgBoss, schema: string, controls: TestingC
 			});
 		},
 	};
-	return new Proxy(client, {
+	const wrapped = new Proxy(client, {
 		get(target, property) {
 			if (Object.hasOwn(overrides, property)) {
 				const override: unknown = Reflect.get(overrides, property);
@@ -112,4 +113,6 @@ export function testingClient(client: PgBoss, schema: string, controls: TestingC
 			return typeof value === "function" ? value.bind(target) : value;
 		},
 	});
+	deferralObservers.set(wrapped, controls.changed);
+	return wrapped;
 }

package/src/server/internal/testingDeferrals.ts

@@ -0,0 +1,43 @@
+import { DateTime } from "effect";
+import type { PgBoss } from "pg-boss";
+import type { QueueDefinition } from "#definition.ts";
+import { deferralOf } from "#internal/deferral.ts";
+import { testingOperation } from "#internal/testingReadout.ts";
+
+export function testingDeferrals(client: () => PgBoss, schema: () => string, changed: () => void) {
+	return {
+		deferred: (queue: Pick<QueueDefinition, "name">) =>
+			testingOperation(
+				"deferred",
+				async () => {
+					const jobs = await client().findJobs(queue.name);
+					return jobs.flatMap((job) => {
+						const deferral = job.state === "retry" && job.startedOn === null ? deferralOf(job) : undefined;
+						return deferral === undefined ? [] : [{ at: DateTime.makeUnsafe(job.startAfter), reason: deferral.reason }];
+					});
+				},
+				queue.name,
+			),
+		runDeferredNow: (queue: Pick<QueueDefinition, "name">) =>
+			testingOperation(
+				"runDeferredNow",
+				async () => {
+					const quotedSchema = `"${schema().replaceAll('"', '""')}"`;
+					await client()
+						.getDb()
+						.executeSql(
+							`UPDATE ${quotedSchema}.job SET start_after = ${quotedSchema}.job_now()
+				WHERE name = $1 AND state = 'retry' AND output ? 'jobDeferral' AND started_on IS NULL`,
+							[queue.name],
+						);
+					changed();
+					for (const worker of client()
+						.getWipData()
+						.filter((candidate) => candidate.name === queue.name)) {
+						client().notifyWorker(worker.id);
+					}
+				},
+				queue.name,
+			),
+	};
+}

package/src/server/schema.ts

@@ -4,6 +4,7 @@ import type { Connection } from "effect/sql/SqlConnection";
 import { getConstructionPlans, getMigrationPlans, type MigrationPartition, PgBoss } from "pg-boss";
 import pgBossPackage from "pg-boss/package.json" with { type: "json" };
 import { type PgBossError, toPgBossError } from "#error.ts";
+import { deferralStorePlan } from "#internal/deferralStorePlan.ts";
 import { runSchemaPlan } from "#internal/runSchemaPlan.ts";
 import { resumeSchemaIndexes } from "#internal/schemaIndexBuilds.ts";
 
@@ -51,7 +52,13 @@ export function installPgBossSchema(options: PgBossSchemaOptions = {}): Effect.E
 			const sql = yield* SqlClient.SqlClient;
 			const connection = yield* sql.reserve;
 			const run = Effect.runPromiseWith(yield* Effect.context<never>());
-			yield* Effect.tryPromise({ catch: (error) => error, try: () => installSchema(connection, options, run) });
+			yield* Effect.tryPromise({
+				catch: (error) => error,
+				try: async () => {
+					await installSchema(connection, options, run);
+					await run(connection.executeRaw(deferralStorePlan(options.schema ?? "pgboss"), []));
+				},
+			});
 		}).pipe(Effect.mapError((error) => toPgBossError("install", error))),
 	).pipe(Effect.uninterruptible);
 }

package/src/server/service.ts

@@ -10,18 +10,18 @@ import { registerJobs, registrationName } from "#internal/register.ts";
 import { makeService } from "#internal/service.ts";
 
 export interface PgBossService {
-	readonly counts: (queue: QueueDefinition) => Effect.Effect<QueueCounts, PgBossError>;
-	readonly enqueue: <const TName extends string, const TPayload extends JobPayloadSchema>(
-		queue: QueueDefinition<TName, TPayload>,
-		payload: Schema.Schema.Type<TPayload>,
-		options?: SendOptions,
+	readonly counts: (queue: Pick<QueueDefinition, "name">) => Effect.Effect<QueueCounts, PgBossError>;
+	readonly enqueue: <const TName extends string, const TPayload extends JobPayloadSchema, const TQueue extends QueueDefinition<TName, TPayload>>(
+		queue: QueueDefinition<TName, TPayload> & TQueue,
+		payload: NoInfer<Schema.Schema.Type<TPayload>>,
+		options?: NoInfer<TQueue["key"] extends undefined ? SendOptions : Omit<SendOptions, "singletonKey"> & { readonly singletonKey?: never }>,
 	) => Effect.Effect<Option.Option<string>, PgBossError | PgBossPayloadError, TPayload["EncodingServices"]>;
 	readonly getJob: <const TName extends string, const TPayload extends JobPayloadSchema>(
 		queue: QueueDefinition<TName, TPayload>,
 		id: string,
 	) => Effect.Effect<Option.Option<JobWithMetadata<TPayload["Type"]>>, PgBossError | PgBossPayloadError, TPayload["DecodingServices"]>;
 	readonly health: Effect.Effect<JobsHealth, PgBossError>;
-	readonly retry: (queue: QueueDefinition, id: string) => Effect.Effect<void, PgBossError>;
+	readonly retry: (queue: Pick<QueueDefinition, "name">, id: string) => Effect.Effect<void, PgBossError>;
 }
 
 export type PgBossLayerOptions<TRegistrations extends readonly JobRegistration[], TErrorRequirements> = ConstructorOptions & {
@@ -72,7 +72,7 @@ export const makePgBoss = <const TIdentifier extends string>(identifier: TIdenti
 					void Effect.runPromise(Effect.exit(Effect.provide(reportError(error), context)));
 				};
 				const unsubscribe = subscribeClientErrors(acquired.client, errorListener);
-				const registration = registerJobs(acquired.client, jobs, context, acquired.reused);
+				const registration = registerJobs(acquired.client, jobs, context, acquired.reused, constructorOptions.schema);
 				return yield* registration.pipe(
 					Effect.as({ acquired, service, unsubscribe }),
 					Effect.onError(() => releaseClient(acquired, stop).pipe(Effect.ensuring(Effect.sync(unsubscribe)))),

package/src/server/testing.ts

@@ -4,6 +4,7 @@ import type { PgBossClientFactory } from "#client.ts";
 import type { QueueDefinition } from "#definition.ts";
 import { PgBossTestingError } from "#error.ts";
 import { testingClient } from "#internal/testingClient.ts";
+import { testingDeferrals } from "#internal/testingDeferrals.ts";
 import { testingOperation, testingReadout } from "#internal/testingReadout.ts";
 import { testingSignal } from "#internal/testingSignal.ts";
 
@@ -15,6 +16,7 @@ export interface PgBossTestingOptions {
 
 export function makePgBossTesting(options: PgBossTestingOptions = {}) {
 	let current: PgBoss | undefined;
+	let schema = "pgboss";
 	const signal = testingSignal();
 	const controls = {
 		changed: signal.changed,
@@ -32,11 +34,13 @@ export function makePgBossTesting(options: PgBossTestingOptions = {}) {
 		if (current !== undefined) {
 			throw new Error("A testing controller owns one jobs client; create a controller per runtime");
 		}
+		schema = clientOptions.schema ?? "pgboss";
 		current = new PgBoss({ ...clientOptions, "__test__enableSpies": true });
 		return testingClient(current, clientOptions.schema ?? "pgboss", controls);
 	};
 	return {
-		checkBatchWindow: (queue: QueueDefinition, millisecondsPerJob: number) =>
+		...testingDeferrals(client, () => schema, signal.changed),
+		checkBatchWindow: (queue: Pick<QueueDefinition, "name" | "workerOptions">, millisecondsPerJob: number) =>
 			testingOperation(
 				"checkBatchWindow",
 				async () => {
@@ -64,7 +68,7 @@ export function makePgBossTesting(options: PgBossTestingOptions = {}) {
 				}
 			}),
 		clientFactory,
-		expireAfter: (queue: QueueDefinition, attempts: number) =>
+		expireAfter: (queue: Pick<QueueDefinition, "name">, attempts: number) =>
 			Effect.try({
 				catch: (original) => new PgBossTestingError({ operation: "expireAfter", original: Redacted.make(original), queue: queue.name }),
 				try: () => {
@@ -74,7 +78,7 @@ export function makePgBossTesting(options: PgBossTestingOptions = {}) {
 					controls.expirations.set(queue.name, attempts);
 				},
 			}),
-		failNextSend: (queue: QueueDefinition) => Effect.sync(() => controls.failures.add(queue.name)).pipe(Effect.asVoid),
+		failNextSend: (queue: Pick<QueueDefinition, "name">) => Effect.sync(() => controls.failures.add(queue.name)).pipe(Effect.asVoid),
 		jobs: testingReadout(client),
 		revision: signal.revision,
 		schemaVersion: testingOperation("schemaVersion", () => client().schemaVersion()),