@shivaedev/effect-pg-boss 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +25 -5
- package/dist/server/client.d.ts +2 -0
- package/dist/server/client.d.ts.map +1 -1
- package/dist/server/client.js.map +1 -1
- package/dist/server/definition.d.ts +15 -3
- package/dist/server/definition.d.ts.map +1 -1
- package/dist/server/definition.js +21 -3
- package/dist/server/definition.js.map +1 -1
- package/dist/server/internal/batchHandler.d.ts +1 -1
- package/dist/server/internal/batchHandler.d.ts.map +1 -1
- package/dist/server/internal/batchHandler.js +14 -7
- package/dist/server/internal/batchHandler.js.map +1 -1
- package/dist/server/internal/deferrableAttempt.d.ts +7 -0
- package/dist/server/internal/deferrableAttempt.d.ts.map +1 -0
- package/dist/server/internal/deferrableAttempt.js +28 -0
- package/dist/server/internal/deferrableAttempt.js.map +1 -0
- package/dist/server/internal/deferral.d.ts +15 -0
- package/dist/server/internal/deferral.d.ts.map +1 -0
- package/dist/server/internal/deferral.js +14 -0
- package/dist/server/internal/deferral.js.map +1 -0
- package/dist/server/internal/deferralStorePlan.d.ts +2 -0
- package/dist/server/internal/deferralStorePlan.d.ts.map +1 -0
- package/dist/server/internal/deferralStorePlan.js +34 -0
- package/dist/server/internal/deferralStorePlan.js.map +1 -0
- package/dist/server/internal/expireAttempt.d.ts +1 -1
- package/dist/server/internal/expireAttempt.d.ts.map +1 -1
- package/dist/server/internal/expireAttempt.js +4 -2
- package/dist/server/internal/expireAttempt.js.map +1 -1
- package/dist/server/internal/lifecycle.d.ts.map +1 -1
- package/dist/server/internal/lifecycle.js +4 -0
- package/dist/server/internal/lifecycle.js.map +1 -1
- package/dist/server/internal/register.d.ts +2 -2
- package/dist/server/internal/register.d.ts.map +1 -1
- package/dist/server/internal/register.js +12 -12
- package/dist/server/internal/register.js.map +1 -1
- package/dist/server/internal/registerWorker.d.ts +5 -0
- package/dist/server/internal/registerWorker.d.ts.map +1 -0
- package/dist/server/internal/registerWorker.js +33 -0
- package/dist/server/internal/registerWorker.js.map +1 -0
- package/dist/server/internal/settleDeferral.d.ts +7 -0
- package/dist/server/internal/settleDeferral.d.ts.map +1 -0
- package/dist/server/internal/settleDeferral.js +47 -0
- package/dist/server/internal/settleDeferral.js.map +1 -0
- package/dist/server/internal/testingClient.d.ts.map +1 -1
- package/dist/server/internal/testingClient.js +4 -1
- package/dist/server/internal/testingClient.js.map +1 -1
- package/dist/server/internal/testingDeferrals.d.ts +11 -0
- package/dist/server/internal/testingDeferrals.d.ts.map +1 -0
- package/dist/server/internal/testingDeferrals.js +28 -0
- package/dist/server/internal/testingDeferrals.js.map +1 -0
- package/dist/server/schema.d.ts.map +1 -1
- package/dist/server/schema.js +8 -1
- package/dist/server/schema.js.map +1 -1
- package/dist/server/service.js +1 -1
- package/dist/server/service.js.map +1 -1
- package/dist/server/testing.d.ts +5 -0
- package/dist/server/testing.d.ts.map +1 -1
- package/dist/server/testing.js +4 -0
- package/dist/server/testing.js.map +1 -1
- package/package.json +5 -5
- package/src/server/client.ts +2 -0
- package/src/server/definition.ts +29 -5
- package/src/server/internal/batchHandler.ts +27 -6
- package/src/server/internal/deferrableAttempt.ts +41 -0
- package/src/server/internal/deferral.ts +19 -0
- package/src/server/internal/deferralStorePlan.ts +34 -0
- package/src/server/internal/expireAttempt.ts +4 -2
- package/src/server/internal/lifecycle.ts +4 -0
- package/src/server/internal/register.ts +38 -33
- package/src/server/internal/registerWorker.ts +55 -0
- package/src/server/internal/settleDeferral.ts +53 -0
- package/src/server/internal/testingClient.ts +4 -1
- package/src/server/internal/testingDeferrals.ts +43 -0
- package/src/server/schema.ts +8 -1
- package/src/server/service.ts +1 -1
- package/src/server/testing.ts +4 -0
|
@@ -2,18 +2,43 @@ import { Cause, Clock, Effect, Exit } from "effect";
|
|
|
2
2
|
import type { Job } from "pg-boss";
|
|
3
3
|
import type { PgBossClient } from "#client.ts";
|
|
4
4
|
import { deadlineOf } from "#internal/deadlineOf.ts";
|
|
5
|
+
import { deferredAttempt } from "#internal/deferral.ts";
|
|
5
6
|
|
|
6
7
|
function failureOutput(cause: Cause.Cause<unknown>): object {
|
|
7
8
|
const error = Cause.squash(cause);
|
|
8
9
|
return typeof error === "object" && error !== null ? error : { message: String(error) };
|
|
9
10
|
}
|
|
10
11
|
|
|
11
|
-
|
|
12
|
+
async function settleAttempt(
|
|
13
|
+
complete: NonNullable<PgBossClient["complete"]>,
|
|
14
|
+
fail: NonNullable<PgBossClient["fail"]>,
|
|
15
|
+
name: string,
|
|
16
|
+
job: Job<unknown>,
|
|
17
|
+
exit: Exit.Exit<unknown, unknown>,
|
|
18
|
+
afterFailure?: (job: Job<unknown>) => Promise<void>,
|
|
19
|
+
) {
|
|
20
|
+
if (Exit.isSuccess(exit)) {
|
|
21
|
+
if (exit.value !== deferredAttempt) {
|
|
22
|
+
await complete(name, job);
|
|
23
|
+
}
|
|
24
|
+
} else {
|
|
25
|
+
await fail(name, job, failureOutput(exit.cause));
|
|
26
|
+
await afterFailure?.(job);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function batchHandler(
|
|
31
|
+
client: PgBossClient,
|
|
32
|
+
name: string,
|
|
33
|
+
run: (job: Job<unknown>) => Effect.Effect<unknown, unknown>,
|
|
34
|
+
afterFailure?: (job: Job<unknown>) => Promise<void>,
|
|
35
|
+
) {
|
|
12
36
|
const complete = client.complete?.bind(client);
|
|
13
37
|
const fail = client.fail?.bind(client);
|
|
14
38
|
if (complete === undefined || fail === undefined) {
|
|
15
39
|
throw new TypeError("This pg-boss client does not support complete and fail");
|
|
16
40
|
}
|
|
41
|
+
|
|
17
42
|
return async (jobs: readonly Job<unknown>[]): Promise<void> => {
|
|
18
43
|
let expired = false;
|
|
19
44
|
for (const job of jobs) {
|
|
@@ -33,11 +58,7 @@ export function batchHandler(client: PgBossClient, name: string, run: (job: Job<
|
|
|
33
58
|
// Returning would let pg-boss complete the unprocessed jobs; its expiry owns them instead.
|
|
34
59
|
return new Promise<never>(() => undefined);
|
|
35
60
|
}
|
|
36
|
-
|
|
37
|
-
await complete(name, job);
|
|
38
|
-
} else {
|
|
39
|
-
await fail(name, job, failureOutput(exit.cause));
|
|
40
|
-
}
|
|
61
|
+
await settleAttempt(complete, fail, name, job, exit, afterFailure);
|
|
41
62
|
}
|
|
42
63
|
if (expired) {
|
|
43
64
|
return new Promise<never>(() => undefined);
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { Clock, DateTime, Effect, Schema } from "effect";
|
|
2
|
+
import type { Job } from "pg-boss";
|
|
3
|
+
import type { PgBossClient } from "#client.ts";
|
|
4
|
+
import { RetryAt } from "#definition.ts";
|
|
5
|
+
import { deferralObservers, deferredAttempt } from "#internal/deferral.ts";
|
|
6
|
+
import { settleDeferral } from "#internal/settleDeferral.ts";
|
|
7
|
+
|
|
8
|
+
const recordDeferral = Effect.fnUntraced(function* (at: string, reason: string) {
|
|
9
|
+
const span = yield* Effect.currentSpan.pipe(Effect.orDie);
|
|
10
|
+
span.event("job.deferred", yield* Clock.currentTimeNanos, { at, reason });
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
export function deferrableAttempt<A, E, R>(
|
|
14
|
+
effect: Effect.Effect<A, E, R>,
|
|
15
|
+
client: PgBossClient,
|
|
16
|
+
schema: string,
|
|
17
|
+
job: Job<unknown>,
|
|
18
|
+
maxDeferrals: number,
|
|
19
|
+
) {
|
|
20
|
+
return effect.pipe(
|
|
21
|
+
Effect.catchIf(Schema.is(RetryAt), (error) => {
|
|
22
|
+
if (maxDeferrals === 0) {
|
|
23
|
+
return Effect.fail(error);
|
|
24
|
+
}
|
|
25
|
+
const at = DateTime.formatIso(error.at);
|
|
26
|
+
return Effect.promise(() => settleDeferral(client, schema, job, at, error.reason, maxDeferrals)).pipe(
|
|
27
|
+
Effect.flatMap((settled) => {
|
|
28
|
+
if (settled.claimed && !settled.deferred) {
|
|
29
|
+
return Effect.fail(error);
|
|
30
|
+
}
|
|
31
|
+
// Native batch settlement must not match a sibling's new claim at the unchanged retry count.
|
|
32
|
+
job.retryCount = Number.MAX_SAFE_INTEGER;
|
|
33
|
+
if (settled.deferred) {
|
|
34
|
+
deferralObservers.get(client)?.();
|
|
35
|
+
}
|
|
36
|
+
return (settled.deferred ? recordDeferral(at, error.reason) : Effect.void).pipe(Effect.as(deferredAttempt));
|
|
37
|
+
}),
|
|
38
|
+
);
|
|
39
|
+
}),
|
|
40
|
+
);
|
|
41
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { Schema } from "effect";
|
|
2
|
+
import type { Job } from "pg-boss";
|
|
3
|
+
import type { PgBossClient } from "#client.ts";
|
|
4
|
+
|
|
5
|
+
export const deferredAttempt = Symbol("deferredAttempt");
|
|
6
|
+
export const deferralObservers = new WeakMap<PgBossClient, () => void>();
|
|
7
|
+
export const STATELY_DEFERRAL_POLICY = "effect-stately-deferred";
|
|
8
|
+
|
|
9
|
+
export const Deferral = Schema.Struct({
|
|
10
|
+
at: Schema.String,
|
|
11
|
+
reason: Schema.String,
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
const Output = Schema.Struct({ jobDeferral: Deferral });
|
|
15
|
+
|
|
16
|
+
export function deferralOf(job: Job<unknown>) {
|
|
17
|
+
const output: unknown = Reflect.get(job, "output");
|
|
18
|
+
return Schema.is(Output)(output) ? output.jobDeferral : undefined;
|
|
19
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { STATELY_DEFERRAL_POLICY } from "./deferral.ts";
|
|
2
|
+
|
|
3
|
+
export function deferralStorePlan(schema: string): string {
|
|
4
|
+
const quoted = `"${schema.replaceAll('"', '""')}"`;
|
|
5
|
+
const jobTable = `${quoted}.job`.replaceAll("'", "''");
|
|
6
|
+
const key = `${schema}.effect_job_deferral`.replaceAll("'", "''");
|
|
7
|
+
const body = `BEGIN
|
|
8
|
+
PERFORM pg_advisory_xact_lock(hashtextextended('${key}', 0));
|
|
9
|
+
CREATE TABLE IF NOT EXISTS ${quoted}.effect_job_deferral (
|
|
10
|
+
name text NOT NULL,
|
|
11
|
+
id uuid NOT NULL,
|
|
12
|
+
created_on timestamptz NOT NULL,
|
|
13
|
+
count bigint NOT NULL,
|
|
14
|
+
keep_until timestamptz NOT NULL,
|
|
15
|
+
PRIMARY KEY (name, id)
|
|
16
|
+
);
|
|
17
|
+
CREATE INDEX IF NOT EXISTS effect_job_deferral_keep_until ON ${quoted}.effect_job_deferral (keep_until);
|
|
18
|
+
CREATE OR REPLACE FUNCTION ${quoted}.effect_stately_deferral() RETURNS trigger LANGUAGE plpgsql AS $claim$
|
|
19
|
+
BEGIN
|
|
20
|
+
IF NEW.policy = '${STATELY_DEFERRAL_POLICY}' AND NEW.state = 'active' THEN
|
|
21
|
+
NEW.policy := 'stately';
|
|
22
|
+
END IF;
|
|
23
|
+
RETURN NEW;
|
|
24
|
+
END
|
|
25
|
+
$claim$;
|
|
26
|
+
IF NOT EXISTS (
|
|
27
|
+
SELECT 1 FROM pg_trigger WHERE tgrelid = '${jobTable}'::regclass AND tgname = 'effect_stately_deferral'
|
|
28
|
+
) THEN
|
|
29
|
+
CREATE TRIGGER effect_stately_deferral BEFORE UPDATE OF state, policy ON ${quoted}.job
|
|
30
|
+
FOR EACH ROW EXECUTE FUNCTION ${quoted}.effect_stately_deferral();
|
|
31
|
+
END IF;
|
|
32
|
+
END`;
|
|
33
|
+
return `DO '${body.replaceAll("'", "''")}'`;
|
|
34
|
+
}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import type { Job, PgBoss } from "pg-boss";
|
|
2
2
|
|
|
3
|
-
export async function expireAttempt(client: PgBoss, schema: string, job: Job<unknown
|
|
3
|
+
export async function expireAttempt(client: PgBoss, schema: string, job: Job<unknown>, cause: "expiry" | "heartbeat" = "expiry"): Promise<void> {
|
|
4
4
|
if ((await client.schemaVersion()) !== 45) {
|
|
5
5
|
throw new Error("Testing expiry requires pg-boss schema version 45");
|
|
6
6
|
}
|
|
7
7
|
const quoted = `"${schema.replaceAll('"', '""')}"`;
|
|
8
|
+
const timestamp = cause === "expiry" ? "started_on" : "heartbeat_on";
|
|
9
|
+
const seconds = cause === "expiry" ? "expire_seconds" : "heartbeat_seconds";
|
|
8
10
|
await client.getDb().executeSql(
|
|
9
|
-
`UPDATE ${quoted}.job SET
|
|
11
|
+
`UPDATE ${quoted}.job SET ${timestamp} = ${quoted}.job_now() - (${seconds} + 1) * interval '1 second'
|
|
10
12
|
WHERE name = $1 AND id = $2 AND retry_count = $3 AND state = 'active'`,
|
|
11
13
|
[job.name, job.id, job.retryCount],
|
|
12
14
|
);
|
|
@@ -2,6 +2,7 @@ import { Effect } from "effect";
|
|
|
2
2
|
import type { ConstructorOptions, StopOptions } from "pg-boss";
|
|
3
3
|
import { defaultClientFactory, type PgBossClient, type PgBossClientFactory } from "#client.ts";
|
|
4
4
|
import { toPgBossError } from "#error.ts";
|
|
5
|
+
import { deferralStorePlan } from "#internal/deferralStorePlan.ts";
|
|
5
6
|
import { acquireCachedClient, releaseCachedClient } from "./clientCache.ts";
|
|
6
7
|
import { logClientError, observeClientErrors } from "./clientErrors.ts";
|
|
7
8
|
|
|
@@ -22,6 +23,9 @@ async function startClient(options: AcquireClientOptions, report: (error: Error)
|
|
|
22
23
|
observeClientErrors(client, report);
|
|
23
24
|
try {
|
|
24
25
|
await client.start();
|
|
26
|
+
if (options.constructor.migrate !== false && client.getDb !== undefined) {
|
|
27
|
+
await client.getDb().executeSql(deferralStorePlan(options.constructor.schema ?? "pgboss"));
|
|
28
|
+
}
|
|
25
29
|
return client;
|
|
26
30
|
} catch (error) {
|
|
27
31
|
try {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Context, Effect, Redacted, Schema } from "effect";
|
|
1
|
+
import { type Context, Effect, Redacted, Schema, type Scope } from "effect";
|
|
2
2
|
import type { Job } from "pg-boss";
|
|
3
3
|
import type { PgBossClient } from "#client.ts";
|
|
4
4
|
import {
|
|
@@ -10,9 +10,9 @@ import {
|
|
|
10
10
|
type ScheduledWorker,
|
|
11
11
|
} from "#definition.ts";
|
|
12
12
|
import { PgBossError, PgBossPayloadError, PgBossQueueOptionsError, toPgBossError } from "#error.ts";
|
|
13
|
-
import { batchHandler } from "#internal/batchHandler.ts";
|
|
14
13
|
import { ensureQueue } from "#internal/ensureQueue.ts";
|
|
15
|
-
import {
|
|
14
|
+
import { registerWorker } from "#internal/registerWorker.ts";
|
|
15
|
+
import { payloadOf } from "#internal/tracing.ts";
|
|
16
16
|
|
|
17
17
|
function isQueueWorker(registration: JobRegistration): registration is QueueWorker {
|
|
18
18
|
return registration._tag === "QueueWorker";
|
|
@@ -65,7 +65,7 @@ function workerContext(context: unknown): unknown {
|
|
|
65
65
|
return context;
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
function runQueueWorker(worker: QueueWorker, job: Job<unknown
|
|
68
|
+
function runQueueWorker(worker: QueueWorker, job: Job<unknown>) {
|
|
69
69
|
return Schema.decodeUnknownEffect(worker.queue.schema)(payloadOf(job.data)).pipe(
|
|
70
70
|
Effect.mapError(
|
|
71
71
|
(error) =>
|
|
@@ -76,31 +76,33 @@ function runQueueWorker(worker: QueueWorker, job: Job<unknown>, context: Context
|
|
|
76
76
|
}),
|
|
77
77
|
),
|
|
78
78
|
Effect.flatMap((payload) => worker.handler(payload, jobContext(job))),
|
|
79
|
-
(effect) => traceJob(effect, job, worker.queue.name),
|
|
80
|
-
Effect.provide(context),
|
|
81
79
|
);
|
|
82
80
|
}
|
|
83
81
|
|
|
84
|
-
async function registerQueue(
|
|
82
|
+
async function registerQueue(
|
|
83
|
+
client: PgBossClient,
|
|
84
|
+
registration: QueueWorker | QueueDefinition,
|
|
85
|
+
context: Context.Context<unknown>,
|
|
86
|
+
scope: Scope.Scope,
|
|
87
|
+
schema: string,
|
|
88
|
+
): Promise<void> {
|
|
85
89
|
const queue = registration._tag === "QueueWorker" ? registration.queue : registration;
|
|
86
90
|
const name = queue.name;
|
|
87
91
|
if (registration._tag === "QueueDefinition") {
|
|
88
92
|
return;
|
|
89
93
|
}
|
|
90
|
-
await client.
|
|
91
|
-
name,
|
|
92
|
-
{ ...queue.workerOptions, includeMetadata: true },
|
|
93
|
-
batchHandler(client, name, (job) => runQueueWorker(registration, job, context)),
|
|
94
|
-
);
|
|
94
|
+
await registerWorker(client, name, queue.workerOptions, (job) => runQueueWorker(registration, job), scope, schema, queue.maxDeferrals, context);
|
|
95
95
|
}
|
|
96
96
|
|
|
97
|
-
async function registerSchedule(
|
|
97
|
+
async function registerSchedule(
|
|
98
|
+
client: PgBossClient,
|
|
99
|
+
worker: ScheduledWorker,
|
|
100
|
+
context: Context.Context<unknown>,
|
|
101
|
+
scope: Scope.Scope,
|
|
102
|
+
schema: string,
|
|
103
|
+
): Promise<void> {
|
|
98
104
|
const { schedule } = worker;
|
|
99
|
-
await client.
|
|
100
|
-
schedule.name,
|
|
101
|
-
{ ...schedule.workerOptions, includeMetadata: true },
|
|
102
|
-
batchHandler(client, schedule.name, (job) => Effect.provide(traceJob(worker.effect, job, schedule.name), context)),
|
|
103
|
-
);
|
|
105
|
+
await registerWorker(client, schedule.name, schedule.workerOptions, () => worker.effect, scope, schema, schedule.maxDeferrals, context);
|
|
104
106
|
await client.schedule(schedule.name, schedule.cron, null, schedule.scheduleOptions);
|
|
105
107
|
}
|
|
106
108
|
|
|
@@ -117,19 +119,22 @@ export const registerJobs = <R>(
|
|
|
117
119
|
registrations: readonly JobRegistration[],
|
|
118
120
|
context: Context.Context<R>,
|
|
119
121
|
replaceWorkers: boolean,
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
122
|
+
schema = "pgboss",
|
|
123
|
+
): Effect.Effect<void, PgBossError | PgBossQueueOptionsError, Scope.Scope> =>
|
|
124
|
+
Effect.flatMap(Effect.scope, (scope) =>
|
|
125
|
+
Effect.tryPromise({
|
|
126
|
+
catch: (error) => (error instanceof PgBossQueueOptionsError || error instanceof PgBossError ? error : toPgBossError("register", error)),
|
|
127
|
+
try: async () => {
|
|
128
|
+
registrationNames(registrations);
|
|
129
|
+
await prepareQueues(client, registrations, replaceWorkers);
|
|
130
|
+
const provided = workerContext(context);
|
|
131
|
+
for (const registration of registrations) {
|
|
132
|
+
if (isQueueWorker(registration) || isQueueDefinition(registration)) {
|
|
133
|
+
await registerQueue(client, registration, provided, scope, schema);
|
|
134
|
+
} else if (isScheduledWorker(registration)) {
|
|
135
|
+
await registerSchedule(client, registration, provided, scope, schema);
|
|
136
|
+
}
|
|
132
137
|
}
|
|
133
|
-
}
|
|
134
|
-
},
|
|
135
|
-
|
|
138
|
+
},
|
|
139
|
+
}),
|
|
140
|
+
);
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { Clock, type Context, Effect, type Scope } from "effect";
|
|
2
|
+
import type { Job, WorkOptions } from "pg-boss";
|
|
3
|
+
import type { PgBossClient } from "#client.ts";
|
|
4
|
+
import { batchHandler } from "#internal/batchHandler.ts";
|
|
5
|
+
import { deferrableAttempt } from "#internal/deferrableAttempt.ts";
|
|
6
|
+
import { traceJob } from "#internal/tracing.ts";
|
|
7
|
+
|
|
8
|
+
export function registerWorker(
|
|
9
|
+
client: PgBossClient,
|
|
10
|
+
name: string,
|
|
11
|
+
options: WorkOptions,
|
|
12
|
+
run: (job: Job<unknown>) => Effect.Effect<unknown, unknown, unknown>,
|
|
13
|
+
scope: Scope.Scope,
|
|
14
|
+
schema: string,
|
|
15
|
+
maxDeferrals: number,
|
|
16
|
+
context: Context.Context<unknown>,
|
|
17
|
+
): Promise<string> {
|
|
18
|
+
const registration = client.work(
|
|
19
|
+
name,
|
|
20
|
+
{ ...options, includeMetadata: true },
|
|
21
|
+
batchHandler(
|
|
22
|
+
client,
|
|
23
|
+
name,
|
|
24
|
+
(job) => {
|
|
25
|
+
const attempt = traceJob(deferrableAttempt(run(job), client, schema, job, maxDeferrals), job, name);
|
|
26
|
+
return Effect.provide(attempt, context);
|
|
27
|
+
},
|
|
28
|
+
async (attempt) => {
|
|
29
|
+
if (client.findJobs === undefined || client.notifyWorker === undefined) {
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
const job = (await client.findJobs(name, { id: attempt.id }))[0];
|
|
33
|
+
if (job?.state !== "retry") {
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
const workId = await registration;
|
|
37
|
+
await Effect.runPromise(
|
|
38
|
+
Effect.gen(function* () {
|
|
39
|
+
const now = yield* Clock.currentTimeMillis;
|
|
40
|
+
// PostgreSQL timestamps retain sub-millisecond precision that Date drops.
|
|
41
|
+
yield* Effect.sleep(Math.max(0, job.startAfter.getTime() + 1 - now));
|
|
42
|
+
const workerIds = client
|
|
43
|
+
.getWipData?.()
|
|
44
|
+
.filter((worker) => worker.workId === workId)
|
|
45
|
+
.map((worker) => worker.id) ?? [workId];
|
|
46
|
+
for (const workerId of workerIds) {
|
|
47
|
+
client.notifyWorker?.(workerId);
|
|
48
|
+
}
|
|
49
|
+
}).pipe(Effect.forkIn(scope)),
|
|
50
|
+
);
|
|
51
|
+
},
|
|
52
|
+
),
|
|
53
|
+
);
|
|
54
|
+
return registration;
|
|
55
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { Schema } from "effect";
|
|
2
|
+
import type { Job } from "pg-boss";
|
|
3
|
+
import type { PgBossClient } from "#client.ts";
|
|
4
|
+
import { STATELY_DEFERRAL_POLICY } from "./deferral.ts";
|
|
5
|
+
|
|
6
|
+
const Settlement = Schema.Struct({ claimed: Schema.Boolean, deferred: Schema.Boolean });
|
|
7
|
+
|
|
8
|
+
export async function settleDeferral(client: PgBossClient, schema: string, job: Job<unknown>, at: string, reason: string, maxDeferrals: number) {
|
|
9
|
+
if (client.getDb === undefined) {
|
|
10
|
+
throw new TypeError("This pg-boss client does not support getDb for RetryAt");
|
|
11
|
+
}
|
|
12
|
+
const quoted = `"${schema.replaceAll('"', '""')}"`;
|
|
13
|
+
// Native failure settlement deletes and reinserts jobs, so a cascading job foreign key would erase the counter.
|
|
14
|
+
const { rows } = await client.getDb().executeSql(
|
|
15
|
+
`WITH claimed AS MATERIALIZED (
|
|
16
|
+
SELECT name, id, created_on,
|
|
17
|
+
GREATEST(keep_until, $4::timestamptz + (keep_until - start_after)) AS keep_until
|
|
18
|
+
FROM ${quoted}.job
|
|
19
|
+
WHERE name = $1 AND id = $2 AND state = 'active' AND retry_count = $3
|
|
20
|
+
AND date_trunc('milliseconds', started_on) = $6::timestamptz
|
|
21
|
+
FOR UPDATE
|
|
22
|
+
), counted AS (
|
|
23
|
+
INSERT INTO ${quoted}.effect_job_deferral AS previous (name, id, created_on, count, keep_until)
|
|
24
|
+
SELECT name, id, created_on, 1, keep_until FROM claimed
|
|
25
|
+
ON CONFLICT (name, id) DO UPDATE SET
|
|
26
|
+
count = CASE WHEN previous.created_on = EXCLUDED.created_on THEN previous.count + 1 ELSE 1 END,
|
|
27
|
+
created_on = EXCLUDED.created_on, keep_until = EXCLUDED.keep_until
|
|
28
|
+
WHERE previous.created_on <> EXCLUDED.created_on OR previous.count < $7
|
|
29
|
+
RETURNING name, id, keep_until
|
|
30
|
+
), deferred AS (
|
|
31
|
+
UPDATE ${quoted}.job AS job SET state = 'retry', start_after = $4::timestamptz,
|
|
32
|
+
policy = CASE WHEN job.policy = 'stately' THEN '${STATELY_DEFERRAL_POLICY}' ELSE job.policy END,
|
|
33
|
+
keep_until = counted.keep_until, started_on = NULL, heartbeat_on = NULL,
|
|
34
|
+
output = jsonb_build_object('jobDeferral', jsonb_build_object('at', $4::text, 'reason', $5::text))
|
|
35
|
+
FROM counted WHERE job.name = counted.name AND job.id = counted.id
|
|
36
|
+
RETURNING job.id
|
|
37
|
+
), cleaned AS (
|
|
38
|
+
DELETE FROM ${quoted}.effect_job_deferral WHERE (name, id) IN (
|
|
39
|
+
SELECT stale.name, stale.id FROM ${quoted}.effect_job_deferral AS stale
|
|
40
|
+
WHERE EXISTS (SELECT 1 FROM counted) AND stale.keep_until < ${quoted}.job_now()
|
|
41
|
+
AND (stale.name, stale.id) <> ($1, $2)
|
|
42
|
+
AND NOT EXISTS (
|
|
43
|
+
SELECT 1 FROM ${quoted}.job AS live
|
|
44
|
+
WHERE live.name = stale.name AND live.id = stale.id AND live.created_on = stale.created_on
|
|
45
|
+
)
|
|
46
|
+
ORDER BY stale.name, stale.id LIMIT 100 FOR UPDATE OF stale SKIP LOCKED
|
|
47
|
+
)
|
|
48
|
+
)
|
|
49
|
+
SELECT EXISTS (SELECT 1 FROM claimed) AS claimed, EXISTS (SELECT 1 FROM deferred) AS deferred`,
|
|
50
|
+
[job.name, job.id, job.retryCount, at, reason, Reflect.get(job, "startedOn"), maxDeferrals],
|
|
51
|
+
);
|
|
52
|
+
return Schema.decodeUnknownSync(Settlement)(rows[0]);
|
|
53
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { Job, PgBoss, SendOptions, UpdateOptions, UpdateRequest, UpdateResponse } from "pg-boss";
|
|
2
2
|
import type { PgBossClient } from "#client.ts";
|
|
3
|
+
import { deferralObservers } from "#internal/deferral.ts";
|
|
3
4
|
import { expireAttempt } from "#internal/expireAttempt.ts";
|
|
4
5
|
import { observeTestingSettlements } from "#internal/observeTestingSettlements.ts";
|
|
5
6
|
import { testingExpirations } from "#internal/testingExpirations.ts";
|
|
@@ -102,7 +103,7 @@ export function testingClient(client: PgBoss, schema: string, controls: TestingC
|
|
|
102
103
|
});
|
|
103
104
|
},
|
|
104
105
|
};
|
|
105
|
-
|
|
106
|
+
const wrapped = new Proxy(client, {
|
|
106
107
|
get(target, property) {
|
|
107
108
|
if (Object.hasOwn(overrides, property)) {
|
|
108
109
|
const override: unknown = Reflect.get(overrides, property);
|
|
@@ -112,4 +113,6 @@ export function testingClient(client: PgBoss, schema: string, controls: TestingC
|
|
|
112
113
|
return typeof value === "function" ? value.bind(target) : value;
|
|
113
114
|
},
|
|
114
115
|
});
|
|
116
|
+
deferralObservers.set(wrapped, controls.changed);
|
|
117
|
+
return wrapped;
|
|
115
118
|
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { DateTime } from "effect";
|
|
2
|
+
import type { PgBoss } from "pg-boss";
|
|
3
|
+
import type { QueueDefinition } from "#definition.ts";
|
|
4
|
+
import { deferralOf } from "#internal/deferral.ts";
|
|
5
|
+
import { testingOperation } from "#internal/testingReadout.ts";
|
|
6
|
+
|
|
7
|
+
export function testingDeferrals(client: () => PgBoss, schema: () => string, changed: () => void) {
|
|
8
|
+
return {
|
|
9
|
+
deferred: (queue: QueueDefinition) =>
|
|
10
|
+
testingOperation(
|
|
11
|
+
"deferred",
|
|
12
|
+
async () => {
|
|
13
|
+
const jobs = await client().findJobs(queue.name);
|
|
14
|
+
return jobs.flatMap((job) => {
|
|
15
|
+
const deferral = job.state === "retry" && job.startedOn === null ? deferralOf(job) : undefined;
|
|
16
|
+
return deferral === undefined ? [] : [{ at: DateTime.makeUnsafe(job.startAfter), reason: deferral.reason }];
|
|
17
|
+
});
|
|
18
|
+
},
|
|
19
|
+
queue.name,
|
|
20
|
+
),
|
|
21
|
+
runDeferredNow: (queue: QueueDefinition) =>
|
|
22
|
+
testingOperation(
|
|
23
|
+
"runDeferredNow",
|
|
24
|
+
async () => {
|
|
25
|
+
const quotedSchema = `"${schema().replaceAll('"', '""')}"`;
|
|
26
|
+
await client()
|
|
27
|
+
.getDb()
|
|
28
|
+
.executeSql(
|
|
29
|
+
`UPDATE ${quotedSchema}.job SET start_after = ${quotedSchema}.job_now()
|
|
30
|
+
WHERE name = $1 AND state = 'retry' AND output ? 'jobDeferral' AND started_on IS NULL`,
|
|
31
|
+
[queue.name],
|
|
32
|
+
);
|
|
33
|
+
changed();
|
|
34
|
+
for (const worker of client()
|
|
35
|
+
.getWipData()
|
|
36
|
+
.filter((candidate) => candidate.name === queue.name)) {
|
|
37
|
+
client().notifyWorker(worker.id);
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
queue.name,
|
|
41
|
+
),
|
|
42
|
+
};
|
|
43
|
+
}
|
package/src/server/schema.ts
CHANGED
|
@@ -4,6 +4,7 @@ import type { Connection } from "effect/sql/SqlConnection";
|
|
|
4
4
|
import { getConstructionPlans, getMigrationPlans, type MigrationPartition, PgBoss } from "pg-boss";
|
|
5
5
|
import pgBossPackage from "pg-boss/package.json" with { type: "json" };
|
|
6
6
|
import { type PgBossError, toPgBossError } from "#error.ts";
|
|
7
|
+
import { deferralStorePlan } from "#internal/deferralStorePlan.ts";
|
|
7
8
|
import { runSchemaPlan } from "#internal/runSchemaPlan.ts";
|
|
8
9
|
import { resumeSchemaIndexes } from "#internal/schemaIndexBuilds.ts";
|
|
9
10
|
|
|
@@ -51,7 +52,13 @@ export function installPgBossSchema(options: PgBossSchemaOptions = {}): Effect.E
|
|
|
51
52
|
const sql = yield* SqlClient.SqlClient;
|
|
52
53
|
const connection = yield* sql.reserve;
|
|
53
54
|
const run = Effect.runPromiseWith(yield* Effect.context<never>());
|
|
54
|
-
yield* Effect.tryPromise({
|
|
55
|
+
yield* Effect.tryPromise({
|
|
56
|
+
catch: (error) => error,
|
|
57
|
+
try: async () => {
|
|
58
|
+
await installSchema(connection, options, run);
|
|
59
|
+
await run(connection.executeRaw(deferralStorePlan(options.schema ?? "pgboss"), []));
|
|
60
|
+
},
|
|
61
|
+
});
|
|
55
62
|
}).pipe(Effect.mapError((error) => toPgBossError("install", error))),
|
|
56
63
|
).pipe(Effect.uninterruptible);
|
|
57
64
|
}
|
package/src/server/service.ts
CHANGED
|
@@ -72,7 +72,7 @@ export const makePgBoss = <const TIdentifier extends string>(identifier: TIdenti
|
|
|
72
72
|
void Effect.runPromise(Effect.exit(Effect.provide(reportError(error), context)));
|
|
73
73
|
};
|
|
74
74
|
const unsubscribe = subscribeClientErrors(acquired.client, errorListener);
|
|
75
|
-
const registration = registerJobs(acquired.client, jobs, context, acquired.reused);
|
|
75
|
+
const registration = registerJobs(acquired.client, jobs, context, acquired.reused, constructorOptions.schema);
|
|
76
76
|
return yield* registration.pipe(
|
|
77
77
|
Effect.as({ acquired, service, unsubscribe }),
|
|
78
78
|
Effect.onError(() => releaseClient(acquired, stop).pipe(Effect.ensuring(Effect.sync(unsubscribe)))),
|
package/src/server/testing.ts
CHANGED
|
@@ -4,6 +4,7 @@ import type { PgBossClientFactory } from "#client.ts";
|
|
|
4
4
|
import type { QueueDefinition } from "#definition.ts";
|
|
5
5
|
import { PgBossTestingError } from "#error.ts";
|
|
6
6
|
import { testingClient } from "#internal/testingClient.ts";
|
|
7
|
+
import { testingDeferrals } from "#internal/testingDeferrals.ts";
|
|
7
8
|
import { testingOperation, testingReadout } from "#internal/testingReadout.ts";
|
|
8
9
|
import { testingSignal } from "#internal/testingSignal.ts";
|
|
9
10
|
|
|
@@ -15,6 +16,7 @@ export interface PgBossTestingOptions {
|
|
|
15
16
|
|
|
16
17
|
export function makePgBossTesting(options: PgBossTestingOptions = {}) {
|
|
17
18
|
let current: PgBoss | undefined;
|
|
19
|
+
let schema = "pgboss";
|
|
18
20
|
const signal = testingSignal();
|
|
19
21
|
const controls = {
|
|
20
22
|
changed: signal.changed,
|
|
@@ -32,10 +34,12 @@ export function makePgBossTesting(options: PgBossTestingOptions = {}) {
|
|
|
32
34
|
if (current !== undefined) {
|
|
33
35
|
throw new Error("A testing controller owns one jobs client; create a controller per runtime");
|
|
34
36
|
}
|
|
37
|
+
schema = clientOptions.schema ?? "pgboss";
|
|
35
38
|
current = new PgBoss({ ...clientOptions, "__test__enableSpies": true });
|
|
36
39
|
return testingClient(current, clientOptions.schema ?? "pgboss", controls);
|
|
37
40
|
};
|
|
38
41
|
return {
|
|
42
|
+
...testingDeferrals(client, () => schema, signal.changed),
|
|
39
43
|
checkBatchWindow: (queue: QueueDefinition, millisecondsPerJob: number) =>
|
|
40
44
|
testingOperation(
|
|
41
45
|
"checkBatchWindow",
|