@shivaedev/effect-pg-boss 0.8.1 → 0.10.0

package/src/server/internal/testingExpirations.ts

@@ -0,0 +1,31 @@
+import type { Job, PgBoss, SendOptions } from "pg-boss";
+
+export function testingExpirations(client: PgBoss) {
+	const snapshots = new Map<string, Promise<number | undefined>>();
+	return {
+		attemptsFor: (job: Job<unknown>) => snapshots.get(`${job.name}:${job.id}`),
+		send(name: string, data: object | null | undefined, options: SendOptions, attempts: number | undefined) {
+			if (attempts === undefined && options.id === undefined) {
+				return client.send(name, data, options);
+			}
+			const id = options.id ?? crypto.randomUUID();
+			const key = `${name}:${id}`;
+			const previous = snapshots.get(key);
+			const sending = client.send(name, data, { ...options, id });
+			// A worker may claim the row before send returns. Wait for its result before applying the snapshot.
+			const snapshot = sending
+				.then(
+					(sent) => (sent === id ? attempts : previous),
+					() => previous,
+				)
+				.then((value) => {
+					if (value === undefined && snapshots.get(key) === snapshot) {
+						snapshots.delete(key);
+					}
+					return value;
+				});
+			snapshots.set(key, snapshot);
+			return sending;
+		},
+	};
+}

package/src/server/internal/testingReadout.ts

@@ -0,0 +1,26 @@
+import { Effect, Redacted, Schema } from "effect";
+import type { FindJobsOptions, PgBoss } from "pg-boss";
+import type { JobPayloadSchema, QueueDefinition } from "#definition.ts";
+import { PgBossPayloadError, PgBossTestingError } from "#error.ts";
+import { payloadOf } from "#internal/tracing.ts";
+
+export function testingOperation<A>(operation: string, run: () => Promise<A>, queue?: string) {
+	return Effect.tryPromise({
+		catch: (original) => new PgBossTestingError({ operation, original: Redacted.make(original), queue }),
+		try: run,
+	});
+}
+
+export function testingReadout(client: () => PgBoss) {
+	return <TName extends string, TPayload extends JobPayloadSchema>(queue: QueueDefinition<TName, TPayload>, options?: FindJobsOptions) =>
+		testingOperation("jobs", () => client().findJobs<unknown>(queue.name, options), queue.name).pipe(
+			Effect.flatMap((jobs) =>
+				Effect.forEach(jobs, (job) =>
+					Schema.decodeUnknownEffect(queue.schema)(payloadOf(job.data)).pipe(
+						Effect.mapError((original) => new PgBossPayloadError({ direction: "decode", original: Redacted.make(original), queue: queue.name })),
+						Effect.map((data) => ({ ...job, data })),
+					),
+				),
+			),
+		);
+}

package/src/server/internal/testingSignal.ts

@@ -0,0 +1,28 @@
+import { Effect } from "effect";
+
+export function testingSignal() {
+	let revision = 0;
+	const listeners = new Set<(revision: number) => void>();
+	return {
+		changed() {
+			revision += 1;
+			for (const listener of listeners) {
+				listener(revision);
+			}
+			listeners.clear();
+		},
+		revision: Effect.sync(() => revision),
+		waitForChange: (after: number): Effect.Effect<number> =>
+			Effect.callback((resume) => {
+				if (revision !== after) {
+					resume(Effect.succeed(revision));
+					return;
+				}
+				function listener(current: number) {
+					resume(Effect.succeed(current));
+				}
+				listeners.add(listener);
+				return Effect.sync(() => listeners.delete(listener));
+			}),
+	};
+}

package/src/server/schema.ts

@@ -0,0 +1,57 @@
+import { Effect } from "effect";
+import { SqlClient } from "effect/sql";
+import type { Connection } from "effect/sql/SqlConnection";
+import { getConstructionPlans, getMigrationPlans, type MigrationPartition, PgBoss } from "pg-boss";
+import pgBossPackage from "pg-boss/package.json" with { type: "json" };
+import { type PgBossError, toPgBossError } from "#error.ts";
+import { runSchemaPlan } from "#internal/runSchemaPlan.ts";
+import { resumeSchemaIndexes } from "#internal/schemaIndexBuilds.ts";
+
+export interface PgBossSchemaOptions {
+	readonly schema?: string;
+}
+
+async function installSchema(
+	connection: Connection,
+	options: PgBossSchemaOptions,
+	run: <A, E>(effect: Effect.Effect<A, E>) => Promise<A>,
+): Promise<void> {
+	const schema = options.schema ?? "pgboss";
+	const construction = getConstructionPlans(schema);
+	const client = new PgBoss({
+		db: { executeSql: async (text, values = []) => ({ rows: Array.from(await run(connection.executeUnprepared(text, values, undefined))) }) },
+		schema,
+	});
+	if (!(await client.isInstalled())) {
+		await runSchemaPlan(connection, construction, run);
+		return;
+	}
+	const version = await client.schemaVersion();
+	if (version === null) {
+		throw new Error("The pg-boss version table is empty");
+	}
+	if (version > pgBossPackage.pgboss.schema) {
+		return;
+	}
+	if (version >= 27) {
+		await resumeSchemaIndexes(client, schema);
+	}
+	if (version < pgBossPackage.pgboss.schema) {
+		const partitionTables: MigrationPartition[] = Array.from(
+			await run(connection.executeUnprepared(`SELECT table_name AS "tableName", policy FROM ${schema}.queue WHERE partition = true`, [], undefined)),
+		);
+		await runSchemaPlan(connection, getMigrationPlans(schema, version, { partitionTables }), run, { schema, version: pgBossPackage.pgboss.schema });
+		await resumeSchemaIndexes(client, schema);
+	}
+}
+
+export function installPgBossSchema(options: PgBossSchemaOptions = {}): Effect.Effect<void, PgBossError, SqlClient.SqlClient> {
+	return Effect.scoped(
+		Effect.gen(function* () {
+			const sql = yield* SqlClient.SqlClient;
+			const connection = yield* sql.reserve;
+			const run = Effect.runPromiseWith(yield* Effect.context<never>());
+			yield* Effect.tryPromise({ catch: (error) => error, try: () => installSchema(connection, options, run) });
+		}).pipe(Effect.mapError((error) => toPgBossError("install", error))),
+	).pipe(Effect.uninterruptible);
+}

package/src/server/testing.ts

@@ -0,0 +1,85 @@
+import { Effect, Redacted } from "effect";
+import { PgBoss } from "pg-boss";
+import type { PgBossClientFactory } from "#client.ts";
+import type { QueueDefinition } from "#definition.ts";
+import { PgBossTestingError } from "#error.ts";
+import { testingClient } from "#internal/testingClient.ts";
+import { testingOperation, testingReadout } from "#internal/testingReadout.ts";
+import { testingSignal } from "#internal/testingSignal.ts";
+
+export const testRetryPolicy = Object.freeze({ retryBackoff: false, retryDelay: 0 });
+
+export interface PgBossTestingOptions {
+	readonly retryPolicy?: typeof testRetryPolicy;
+}
+
+export function makePgBossTesting(options: PgBossTestingOptions = {}) {
+	let current: PgBoss | undefined;
+	const signal = testingSignal();
+	const controls = {
+		changed: signal.changed,
+		expirations: new Map<string, number>(),
+		failures: new Set<string>(),
+		retryPolicy: options.retryPolicy ?? {},
+	};
+	function client(): PgBoss {
+		if (current === undefined) {
+			throw new Error("Provide the testing clientFactory to a jobs Layer first");
+		}
+		return current;
+	}
+	const clientFactory: PgBossClientFactory = (clientOptions) => {
+		if (current !== undefined) {
+			throw new Error("A testing controller owns one jobs client; create a controller per runtime");
+		}
+		current = new PgBoss({ ...clientOptions, "__test__enableSpies": true });
+		return testingClient(current, clientOptions.schema ?? "pgboss", controls);
+	};
+	return {
+		checkBatchWindow: (queue: QueueDefinition, millisecondsPerJob: number) =>
+			testingOperation(
+				"checkBatchWindow",
+				async () => {
+					const stored = await client().getQueue(queue.name);
+					if (stored === null) {
+						throw new Error("Queue is not registered");
+					}
+					if (stored.expireInSeconds === undefined) {
+						throw new Error("Queue has no expiry window");
+					}
+					if (!Number.isFinite(millisecondsPerJob) || millisecondsPerJob < 0) {
+						throw new RangeError("Job duration must be nonnegative and finite");
+					}
+					const required = (queue.workerOptions.batchSize ?? 10) * millisecondsPerJob;
+					if (required >= stored.expireInSeconds * 1000) {
+						throw new Error("Sequential batch work must fit within the queue expiry window");
+					}
+				},
+				queue.name,
+			),
+		checkSchemaVersion: (expected: number) =>
+			testingOperation("checkSchemaVersion", async () => {
+				if ((await client().schemaVersion()) !== expected) {
+					throw new Error(`Expected pg-boss schema version ${expected}`);
+				}
+			}),
+		clientFactory,
+		expireAfter: (queue: QueueDefinition, attempts: number) =>
+			Effect.try({
+				catch: (original) => new PgBossTestingError({ operation: "expireAfter", original: Redacted.make(original), queue: queue.name }),
+				try: () => {
+					if (!Number.isSafeInteger(attempts) || attempts < 1) {
+						throw new RangeError("Attempts must be a positive integer");
+					}
+					controls.expirations.set(queue.name, attempts);
+				},
+			}),
+		failNextSend: (queue: QueueDefinition) => Effect.sync(() => controls.failures.add(queue.name)).pipe(Effect.asVoid),
+		jobs: testingReadout(client),
+		revision: signal.revision,
+		schemaVersion: testingOperation("schemaVersion", () => client().schemaVersion()),
+		waitForChange: signal.waitForChange,
+	};
+}
+
+export type PgBossTesting = ReturnType<typeof makePgBossTesting>;

package/docs/north-star.md

@@ -1,48 +0,0 @@
-# North star
-
-## The problem
-
-A durable job crosses two boundaries at once. Data leaves the producer's process and is read later by a worker; execution leaves the request's Effect context and is called later by pg-boss. When applications bridge those boundaries by hand, they repeat payload validation, dependency capture, queue registration and shutdown logic. A producer can drift from its worker's payload shape, a worker can rebuild services differently from the rest of the application, and a client can outlive the scope that should own it.
-
-The package exists to make that boundary one reusable path. pg-boss remains the PostgreSQL queue engine. Effect remains the application's language for services, errors and resources. The adapter should join those pieces without making the application learn another execution model.
-
-## The ideal
-
-A feature declares one object Schema for a queue, then attaches an ordinary Effect handler. The producer uses its decoded type; the durable boundary uses its encoded type; the handler receives decoded data. Domain code never has to guess what an old or malformed stored payload means.
-
-A scheduled feature declares a name and cron expression, then attaches a payload-free Effect. Queue workers and scheduled workers become one list of registrations at the application's runtime boundary. The application's jobs Layer supplies the workers' services and owns the client that pg-boss calls through.
-
-An application that needs only queueing should not import a repository framework, transport or UI package. A worker that needs application services should declare them through ordinary Effect requirements. The jobs adapter should preserve those requirements rather than hiding them in a global runtime or making each callback rebuild its dependencies.
-
-## What good looks like
-
-- The payload contract is authored once, including transformations between application and durable representations.
-- Invalid stored data is rejected before the worker's domain handler executes.
-- Worker services are supplied where the jobs Layer is built, and their requirements remain visible to TypeScript.
-- The client starts under an Effect scope. Startup failure, registration failure and scope release have a clear resource owner.
-- Client reuse is requested with a key. Applications choose the environments and registrations that should share a client.
-- pg-boss options remain pg-boss options. The package removes repeated boundary plumbing, not the engine's vocabulary.
-- Operational observations are data, such as queue counts; an application decides what those observations mean for its health endpoint and alerts.
-- Documentation distinguishes a fake client's observed calls from durable behavior demonstrated against PostgreSQL.
-
-## Trade-offs
-
-**The durable contract wins over permissive input.** A queue Schema must describe object data on both sides. A malformed payload should not become a partially trusted value in domain code merely to keep a worker running.
-
-**Explicit dependencies win over callback convenience.** The jobs Layer requires the services its handlers use. Adding a hidden runtime or service locator would make a worker shorter while moving its real dependencies out of the type system.
-
-**Resource ownership wins over reuse.** Sharing a client can avoid duplicate starts across Layer builds, but a cache key is an application choice. It must not become a reason to ignore cleanup or silently infer environment policy.
-
-**A thin boundary wins over orchestration features.** pg-boss already owns queue execution and scheduling. A new helper earns its place by removing repeated application code at the Effect boundary, with tests for the semantics it preserves.
-
-**Proven behavior wins over a broader pitch.** Passing `retryBackoff: true` proves registration supplied that option. It does not prove retry timing. A fake client cannot establish a PostgreSQL delivery guarantee, and a PostgreSQL library test cannot establish application adoption or deployment behavior.
-
-## What it deliberately leaves out
-
-- A replacement for pg-boss storage, cron interpretation, queue policy or retry execution.
-- Domain workers, authorization rules, external-provider interfaces and application logging or health policy.
-- Another general service declaration system; native Context and Layer stay available, and general declaration ergonomics belong to `effect-service`.
-- Application repositories or migration ownership; those belong to `effect-sql`.
-- Transactional enqueueing: jobs use a separate connection from application SQL transactions. Applications own any required outbox policy; after-commit changes have their own path through `effect-changes`.
-- A promise of exactly-once domain side effects or provider idempotency. Those need application-specific decisions and evidence.
-- Browser, device or offline synchronization behavior. A PostgreSQL queue boundary cannot establish those product contracts.

package/docs/roadmap.md

@@ -1,35 +0,0 @@
-# Roadmap
-
-This file owns the package's implementation inventory and open API choices. The [framework roadmap](https://github.com/ShivaeDev/platform/blob/main/docs/framework/roadmap.md#next-application-and-deployment-acceptance) owns jobs integration with application services and external providers, including deployment and device validation.
-
-## Built
-
-- [x] `makePgBoss` retains its literal service identity in `PgBossDefinition` and its Layer type, so a different jobs Layer does not satisfy the required service. [Compiler evidence](https://github.com/ShivaeDev/platform/blob/main/packages/effect-pg-boss/src/server/service.typecheck.test.ts).
-- [x] Object payload Schema definitions, decoded enqueue types and queue workers. Type tests reject primitive schemas, missing fields and encoded values passed as decoded payloads.
-- [x] Encoding at enqueue and decoding before the handler. The PostgreSQL test round-trips a transformed payload and checks that malformed stored data reaches the dead-letter queue without calling the domain handler.
-- [x] Queue and schedule registration, with dead-letter queues, default retry settings and UTC schedule registration. The [PostgreSQL schedule test](../src/server/schedulePostgres.spec.ts) reads the stored cron, explicit UTC option, default retries and dead-letter queue through pg-boss. Fake-client tests also verify a retry-limit override.
-- [x] Producer-only queue definitions create queues and dead-letter queues through `Jobs.layer({ jobs: [Queue] })`, without starting a consumer. The [PostgreSQL producer test](../src/server/producerPostgres.spec.ts) enqueues a transformed payload and observes worker retry exhaustion and dead-letter delivery. The [starter producer](../../../examples/starter/src/server/noteProducer.ts) uses this API.
-- [x] Native pg-boss burst defaults (`batchSize: 10`, `burstWhenBatchFull: true`) and configurable batches and local concurrency. [PostgreSQL backlog evidence](../src/server/throughputPostgres.spec.ts) drains 30 jobs within 2.5 seconds.
-- [x] Handlers and scheduled effects can yield their owning jobs service without an external self requirement. [PostgreSQL follow-up evidence](../src/server/followUpPostgres.spec.ts) completes a worker chain and enqueues to a producer-only queue using one scoped client. All declared queues exist before consumers start.
-- [x] Worker and scheduled Effect service capture. Runtime and packed-consumer type tests preserve handler requirements.
-- [x] Scoped startup and shutdown, including cleanup when startup or registration fails.
-- [x] Explicit keyed client reuse and unkeyed per-build acquisition. Lifecycle tests cover two builds sharing one client and replacing the registered worker.
-- [x] Full pg-boss queue creation options, excluding the adapter's name and dead-letter setup. [PostgreSQL evidence](../src/server/notifyPostgres.spec.ts) verifies notification delivery to a waiting worker and immediate live counts after completion; [type tests](../src/server/service.typecheck.test.ts) enforce option ownership.
-- [x] Schema-decoded lookup by job ID and retry under the same ID for failed or retry-waiting jobs. [PostgreSQL evidence](../src/server/jobControlPostgres.spec.ts) verifies both states complete after a manual retry, missing lookups return `None` and duplicate-ID sends return `None`.
-- [x] Startup reconciles declared mutable queue settings and rejects immutable policy or partition changes with a typed error. [PostgreSQL restart evidence](../src/server/queueOptionsPostgres.spec.ts) checks expiry and retry-limit changes on an existing queue.
-- [x] Queue health totals for active, failed, queued, ready and dead-lettered counts. The [PostgreSQL health test](../src/server/healthPostgres.spec.ts) checks two populated queues and their totals after the pg-boss monitor refreshes its cached counts, including delayed jobs in queued counts.
-- [x] Background error events reach the captured reporting service and the listener is removed on release, verified with a [fake client](../src/server/edgeCases.spec.ts). Job AbortSignal cancellation interrupts the handler and leaves the rest of the batch to pg-boss expiry, verified with a [fake client](../src/server/settlement.spec.ts).
-- [x] Public defining-module imports and an installed-package type fixture.
-
-## Next
-
-No further package API expansion is selected. Let the framework's jobs integration work identify the next package change, and preserve the existing payload and lifecycle contracts while it does. That integration's status stays in the framework roadmap.
-
-## Decisions
-
-Enqueueing is separate from application SQL transactions. The package supplies no transactional enqueue adapter. Applications use transactions for database invariants and handle queue errors without failing those transactions. The starter enqueues after its note write transaction and logs queue failures; its [PostgreSQL queue-failure story](../../../examples/starter/src/server/queueFailure.spec.ts) proves that rejected enqueueing leaves create and edit writes saved. Its [rollback story](../../../examples/starter/src/server/committedJobs.spec.ts) also proves that an independently enqueued job survives an outer application rollback.
-
-## Open questions
-
-- Should a shared client key be checked against constructor settings and registration changes? The lifecycle fixture uses the same Layer twice; it does not settle incompatible configurations under one key.
-- What evidence is needed before the guide makes stronger claims about actual cron execution? PostgreSQL tests verify schedule registration; cron execution timing remains untested.