@shivaedev/jobs 0.22.1 → 0.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +47 -53
- package/dist/JobFailure.d.ts +7 -0
- package/dist/JobFailure.d.ts.map +1 -0
- package/dist/JobFailure.js +8 -0
- package/dist/JobFailure.js.map +1 -0
- package/dist/server/PgBoss.d.ts +11 -0
- package/dist/server/PgBoss.d.ts.map +1 -0
- package/dist/server/PgBoss.js +4 -0
- package/dist/server/PgBoss.js.map +1 -0
- package/dist/server/client.d.ts +1 -1
- package/dist/server/client.d.ts.map +1 -1
- package/dist/server/health.d.ts +3 -2
- package/dist/server/health.d.ts.map +1 -1
- package/dist/server/health.js.map +1 -1
- package/dist/server/internal/batchHandler.d.ts +2 -1
- package/dist/server/internal/batchHandler.d.ts.map +1 -1
- package/dist/server/internal/batchHandler.js +49 -31
- package/dist/server/internal/batchHandler.js.map +1 -1
- package/dist/server/internal/clientErrors.d.ts.map +1 -1
- package/dist/server/internal/clientErrors.js +2 -1
- package/dist/server/internal/clientErrors.js.map +1 -1
- package/dist/server/internal/deferrableAttempt.d.ts +1 -1
- package/dist/server/internal/deferrableAttempt.d.ts.map +1 -1
- package/dist/server/internal/deferrableAttempt.js +5 -1
- package/dist/server/internal/deferrableAttempt.js.map +1 -1
- package/dist/server/internal/ensureQueue.d.ts +2 -1
- package/dist/server/internal/ensureQueue.d.ts.map +1 -1
- package/dist/server/internal/ensureQueue.js +16 -21
- package/dist/server/internal/ensureQueue.js.map +1 -1
- package/dist/server/internal/failureOutput.d.ts +4 -0
- package/dist/server/internal/failureOutput.d.ts.map +1 -0
- package/dist/server/internal/failureOutput.js +9 -0
- package/dist/server/internal/failureOutput.js.map +1 -0
- package/dist/server/internal/lifecycle.d.ts +1 -1
- package/dist/server/internal/lifecycle.d.ts.map +1 -1
- package/dist/server/internal/lifecycle.js +10 -16
- package/dist/server/internal/lifecycle.js.map +1 -1
- package/dist/server/internal/listJobs.d.ts.map +1 -1
- package/dist/server/internal/listJobs.js +44 -39
- package/dist/server/internal/listJobs.js.map +1 -1
- package/dist/server/internal/makeService.js +1 -1
- package/dist/server/internal/makeService.js.map +1 -1
- package/dist/server/internal/register.d.ts +4 -3
- package/dist/server/internal/register.d.ts.map +1 -1
- package/dist/server/internal/register.js +34 -48
- package/dist/server/internal/register.js.map +1 -1
- package/dist/server/internal/registerReleasedJobs.d.ts +2 -1
- package/dist/server/internal/registerReleasedJobs.d.ts.map +1 -1
- package/dist/server/internal/registerReleasedJobs.js +3 -3
- package/dist/server/internal/registerReleasedJobs.js.map +1 -1
- package/dist/server/internal/registerWorker.d.ts +3 -2
- package/dist/server/internal/registerWorker.d.ts.map +1 -1
- package/dist/server/internal/registerWorker.js +26 -33
- package/dist/server/internal/registerWorker.js.map +1 -1
- package/dist/server/internal/retryWake.d.ts +4 -0
- package/dist/server/internal/retryWake.d.ts.map +1 -0
- package/dist/server/internal/retryWake.js +32 -0
- package/dist/server/internal/retryWake.js.map +1 -0
- package/dist/server/internal/runQueueWorker.d.ts +5 -0
- package/dist/server/internal/runQueueWorker.d.ts.map +1 -0
- package/dist/server/internal/runQueueWorker.js +12 -0
- package/dist/server/internal/runQueueWorker.js.map +1 -0
- package/dist/server/internal/schemaIndexBuilds.d.ts.map +1 -1
- package/dist/server/internal/schemaIndexBuilds.js +5 -2
- package/dist/server/internal/schemaIndexBuilds.js.map +1 -1
- package/dist/server/schema.d.ts.map +1 -1
- package/dist/server/schema.js +16 -5
- package/dist/server/schema.js.map +1 -1
- package/dist/server/service.d.ts +13 -8
- package/dist/server/service.d.ts.map +1 -1
- package/dist/server/service.js +53 -22
- package/dist/server/service.js.map +1 -1
- package/package.json +14 -8
- package/src/JobFailure.ts +9 -0
- package/src/server/PgBoss.ts +8 -0
- package/src/server/client.ts +1 -1
- package/src/server/health.ts +3 -2
- package/src/server/internal/batchHandler.ts +61 -42
- package/src/server/internal/clientErrors.ts +2 -1
- package/src/server/internal/deferrableAttempt.ts +5 -1
- package/src/server/internal/ensureQueue.ts +18 -20
- package/src/server/internal/failureOutput.ts +9 -0
- package/src/server/internal/lifecycle.ts +13 -16
- package/src/server/internal/listJobs.ts +57 -47
- package/src/server/internal/makeService.ts +1 -1
- package/src/server/internal/register.ts +68 -66
- package/src/server/internal/registerReleasedJobs.ts +8 -3
- package/src/server/internal/registerWorker.ts +35 -46
- package/src/server/internal/retryWake.ts +33 -0
- package/src/server/internal/runQueueWorker.ts +19 -0
- package/src/server/internal/schemaIndexBuilds.ts +6 -2
- package/src/server/schema.ts +22 -6
- package/src/server/service.ts +84 -49
|
@@ -1,71 +1,90 @@
|
|
|
1
1
|
import { Cause, Clock, Effect, Exit } from "effect";
|
|
2
2
|
import type { Job } from "pg-boss";
|
|
3
3
|
import type { PgBossClient } from "#client.ts";
|
|
4
|
+
import { toPgBossError } from "#error.ts";
|
|
4
5
|
import { deadlineOf } from "#internal/deadlineOf.ts";
|
|
5
6
|
import { deferredAttempt } from "#internal/deferral.ts";
|
|
7
|
+
import { failureOutput } from "#internal/failureOutput.ts";
|
|
6
8
|
import { waitForBatchExpiry } from "#internal/waitForBatchExpiry.ts";
|
|
7
9
|
|
|
8
|
-
function
|
|
9
|
-
const error = Cause.squash(cause);
|
|
10
|
-
return typeof error === "object" && error !== null ? error : { message: String(error) };
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
async function settleAttempt(
|
|
10
|
+
function settleAttempt(
|
|
14
11
|
complete: NonNullable<PgBossClient["complete"]>,
|
|
15
12
|
fail: NonNullable<PgBossClient["fail"]>,
|
|
16
13
|
name: string,
|
|
17
14
|
job: Job<unknown>,
|
|
18
15
|
exit: Exit.Exit<unknown, unknown>,
|
|
19
|
-
afterFailure?: (job: Job<unknown>) => Promise<void>,
|
|
20
16
|
) {
|
|
21
|
-
if (Exit.isSuccess(exit)) {
|
|
22
|
-
|
|
23
|
-
await complete(name, job);
|
|
24
|
-
}
|
|
25
|
-
} else {
|
|
26
|
-
await fail(name, job, failureOutput(exit.cause));
|
|
27
|
-
await afterFailure?.(job);
|
|
17
|
+
if (Exit.isSuccess(exit) && exit.value === deferredAttempt) {
|
|
18
|
+
return Effect.void;
|
|
28
19
|
}
|
|
20
|
+
return Effect.tryPromise({
|
|
21
|
+
catch: (error) => toPgBossError("settle", error, name),
|
|
22
|
+
try: () => (Exit.isSuccess(exit) ? complete(name, job) : fail(name, job, failureOutput(exit.cause))),
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export type PgBossRunner = <A, E>(effect: Effect.Effect<A, E>) => Promise<A>;
|
|
27
|
+
|
|
28
|
+
function aborted(signal: AbortSignal) {
|
|
29
|
+
return Effect.callback<never>((resume) => {
|
|
30
|
+
function interrupt() {
|
|
31
|
+
resume(Effect.interrupt);
|
|
32
|
+
}
|
|
33
|
+
if (signal.aborted) {
|
|
34
|
+
interrupt();
|
|
35
|
+
} else {
|
|
36
|
+
signal.addEventListener("abort", interrupt, { once: true });
|
|
37
|
+
}
|
|
38
|
+
return Effect.sync(() => signal.removeEventListener("abort", interrupt));
|
|
39
|
+
});
|
|
29
40
|
}
|
|
30
41
|
|
|
31
42
|
export function batchHandler(
|
|
32
43
|
client: PgBossClient,
|
|
33
44
|
name: string,
|
|
34
45
|
run: (job: Job<unknown>) => Effect.Effect<unknown, unknown>,
|
|
35
|
-
afterFailure
|
|
36
|
-
execute: typeof Effect.runPromise = Effect.runPromise,
|
|
46
|
+
afterFailure: (job: Job<unknown>) => Effect.Effect<void, unknown> = () => Effect.void,
|
|
37
47
|
) {
|
|
38
48
|
const complete = client.complete?.bind(client);
|
|
39
49
|
const fail = client.fail?.bind(client);
|
|
40
50
|
if (complete === undefined || fail === undefined) {
|
|
41
51
|
throw new TypeError("This pg-boss client does not support complete and fail");
|
|
42
52
|
}
|
|
43
|
-
function
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
return async (jobs: readonly Job<unknown>[]): Promise<void> => {
|
|
48
|
-
let expired = false;
|
|
49
|
-
for (const job of jobs) {
|
|
50
|
-
const deadline = deadlineOf(job);
|
|
51
|
-
if ((await Effect.runPromise(Clock.currentTimeMillis)) >= deadline) {
|
|
52
|
-
expired = true;
|
|
53
|
-
continue;
|
|
54
|
-
}
|
|
55
|
-
const exit = await Effect.runPromiseExit(
|
|
56
|
-
Effect.flatMap(Clock.currentTimeMillis, (now) => {
|
|
57
|
-
const remaining = deadline - now;
|
|
58
|
-
return remaining <= 0 ? Effect.interrupt : run(job).pipe(Effect.timeoutOrElse({ duration: remaining, orElse: () => Effect.interrupt }));
|
|
59
|
-
}),
|
|
60
|
-
{ signal: job.signal },
|
|
61
|
-
);
|
|
62
|
-
if (job.signal.aborted || (Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause))) {
|
|
63
|
-
return waitForExpiry(jobs);
|
|
64
|
-
}
|
|
65
|
-
await settleAttempt(complete, fail, name, job, exit, afterFailure);
|
|
53
|
+
const handle = Effect.fnUntraced(function* (job: Job<unknown>) {
|
|
54
|
+
if (job.signal.aborted) {
|
|
55
|
+
return "interrupted";
|
|
66
56
|
}
|
|
67
|
-
|
|
68
|
-
|
|
57
|
+
const remaining = deadlineOf(job) - (yield* Clock.currentTimeMillis);
|
|
58
|
+
if (remaining <= 0) {
|
|
59
|
+
return "expired";
|
|
69
60
|
}
|
|
70
|
-
|
|
61
|
+
const exit = yield* run(job).pipe(
|
|
62
|
+
Effect.timeoutOrElse({ duration: remaining, orElse: () => Effect.interrupt }),
|
|
63
|
+
Effect.raceFirst(aborted(job.signal)),
|
|
64
|
+
Effect.exit,
|
|
65
|
+
);
|
|
66
|
+
if (job.signal.aborted || (Exit.isFailure(exit) && Cause.hasInterrupts(exit.cause))) {
|
|
67
|
+
return "interrupted";
|
|
68
|
+
}
|
|
69
|
+
yield* settleAttempt(complete, fail, name, job, exit);
|
|
70
|
+
if (Exit.isFailure(exit)) {
|
|
71
|
+
yield* afterFailure(job);
|
|
72
|
+
}
|
|
73
|
+
return "settled";
|
|
74
|
+
});
|
|
75
|
+
return (jobs: readonly Job<unknown>[]) =>
|
|
76
|
+
Effect.gen(function* () {
|
|
77
|
+
const waitForExpiry = waitForBatchExpiry(client, name, jobs).pipe(Effect.tapError(Effect.logError));
|
|
78
|
+
let expired = false;
|
|
79
|
+
for (const job of jobs) {
|
|
80
|
+
const outcome = yield* handle(job);
|
|
81
|
+
if (outcome === "interrupted") {
|
|
82
|
+
return yield* waitForExpiry;
|
|
83
|
+
}
|
|
84
|
+
expired ||= outcome === "expired";
|
|
85
|
+
}
|
|
86
|
+
if (expired) {
|
|
87
|
+
yield* waitForExpiry;
|
|
88
|
+
}
|
|
89
|
+
});
|
|
71
90
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Effect } from "effect";
|
|
2
|
+
import { isSensitiveKey, redactText } from "@shivaedev/errors/sensitive.ts";
|
|
2
3
|
import type { PgBossClient } from "#client.ts";
|
|
3
4
|
|
|
4
5
|
type Listener = (error: Error) => void;
|
|
@@ -14,7 +15,7 @@ const observers: WeakMap<WeakKey, unknown> = shared instanceof WeakMap ? shared
|
|
|
14
15
|
Reflect.set(globalThis, key, observers);
|
|
15
16
|
|
|
16
17
|
export function logClientError(error: Error): Effect.Effect<void> {
|
|
17
|
-
const message = { event: "pg_boss_error", message: error.message };
|
|
18
|
+
const message = { event: "pg_boss_error", message: redactText(error.message, isSensitiveKey) };
|
|
18
19
|
return Reflect.get(error, "code") === "57P01" ? Effect.logDebug(message) : Effect.logError(message);
|
|
19
20
|
}
|
|
20
21
|
|
|
@@ -2,6 +2,7 @@ import { Clock, DateTime, Effect, Schema } from "effect";
|
|
|
2
2
|
import type { Job } from "pg-boss";
|
|
3
3
|
import type { PgBossClient } from "#client.ts";
|
|
4
4
|
import { RetryAt } from "#definition.ts";
|
|
5
|
+
import { toPgBossError } from "#error.ts";
|
|
5
6
|
import { deferralObservers, deferredAttempt } from "#internal/deferral.ts";
|
|
6
7
|
import { settleDeferral } from "#internal/settleDeferral.ts";
|
|
7
8
|
|
|
@@ -23,7 +24,10 @@ export function deferrableAttempt<A, E, R>(
|
|
|
23
24
|
return Effect.fail(error);
|
|
24
25
|
}
|
|
25
26
|
const at = DateTime.formatIso(error.at);
|
|
26
|
-
return Effect.
|
|
27
|
+
return Effect.tryPromise({
|
|
28
|
+
catch: (failure) => toPgBossError("settle", failure, job.name),
|
|
29
|
+
try: () => settleDeferral(client, schema, job, at, error.reason, maxDeferrals),
|
|
30
|
+
}).pipe(
|
|
27
31
|
Effect.flatMap((settled) => {
|
|
28
32
|
if (settled.claimed && !settled.deferred) {
|
|
29
33
|
return Effect.fail(error);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Effect, Schedule } from "effect";
|
|
1
2
|
import type { Queue } from "pg-boss";
|
|
2
3
|
import type { PgBossClient } from "#client.ts";
|
|
3
4
|
import { PgBossQueueOptionsError } from "#error.ts";
|
|
@@ -8,40 +9,37 @@ const CREATE_QUEUE_ATTEMPTS = 6;
|
|
|
8
9
|
|
|
9
10
|
// A partitioned queue attaches its table to pg-boss's job table, which can deadlock with job traffic on other queues.
|
|
10
11
|
// PostgreSQL rolls the whole creation back, so trying it again is safe.
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
}
|
|
19
|
-
await new Promise((resolve) => setTimeout(resolve, 25 * 2 ** (attempt - 1)));
|
|
20
|
-
}
|
|
21
|
-
}
|
|
12
|
+
function createQueue(client: PgBossClient, name: string, options?: Omit<Queue, "name">) {
|
|
13
|
+
return Effect.tryPromise({ catch: (error) => error, try: () => client.createQueue(name, options) }).pipe(
|
|
14
|
+
Effect.retry({
|
|
15
|
+
schedule: Schedule.exponential(25).pipe(Schedule.upTo({ times: CREATE_QUEUE_ATTEMPTS - 1 })),
|
|
16
|
+
while: (error) => error instanceof Error && Reflect.get(error, "code") === DEADLOCK_DETECTED,
|
|
17
|
+
}),
|
|
18
|
+
);
|
|
22
19
|
}
|
|
23
20
|
|
|
24
|
-
export
|
|
21
|
+
export const ensureQueue = Effect.fnUntraced(function* (client: PgBossClient, name: string, options: Omit<Queue, "name" | "deadLetter">) {
|
|
25
22
|
const deadLetter = deadLetterQueueName(name);
|
|
26
23
|
const desired = { retryBackoff: true, retryLimit: 3, ...options, deadLetter };
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
const existing =
|
|
24
|
+
yield* createQueue(client, deadLetter);
|
|
25
|
+
yield* createQueue(client, name, desired);
|
|
26
|
+
const existing = yield* Effect.tryPromise({ catch: (error) => error, try: () => client.getQueue(name) });
|
|
30
27
|
if (existing === null) {
|
|
31
28
|
return;
|
|
32
29
|
}
|
|
33
30
|
const immutable = (["partition", "policy"] as const).filter((key) => desired[key] !== undefined && desired[key] !== existing[key]);
|
|
34
31
|
if (immutable.length > 0) {
|
|
35
|
-
|
|
32
|
+
return yield* new PgBossQueueOptionsError({ options: immutable, queue: name });
|
|
36
33
|
}
|
|
34
|
+
const updateQueue = client.updateQueue?.bind(client);
|
|
37
35
|
const { partition: _partition, policy: _policy, ...mutable } = desired;
|
|
38
36
|
const changed = Object.entries(mutable).some(([key, value]) => value !== undefined && value !== Reflect.get(existing, key));
|
|
39
37
|
if (!changed) {
|
|
40
38
|
return;
|
|
41
39
|
}
|
|
42
|
-
if (
|
|
43
|
-
|
|
40
|
+
if (updateQueue === undefined) {
|
|
41
|
+
return yield* Effect.fail(new TypeError("This pg-boss client does not support updateQueue"));
|
|
44
42
|
}
|
|
45
43
|
// pg-boss validates retryDelayMax against retryBackoff in the same update.
|
|
46
|
-
|
|
47
|
-
}
|
|
44
|
+
yield* Effect.tryPromise({ catch: (error) => error, try: () => updateQueue(name, mutable) });
|
|
45
|
+
});
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Cause, Predicate, Schema } from "effect";
|
|
2
|
+
import { JobFailure } from "#JobFailure.ts";
|
|
3
|
+
|
|
4
|
+
export function failureOutput(cause: Cause.Cause<unknown>): typeof JobFailure.Encoded {
|
|
5
|
+
const error = Cause.squash(cause);
|
|
6
|
+
const tag = Predicate.hasProperty(error, "_tag") && typeof error._tag === "string" ? error._tag : "Error";
|
|
7
|
+
const message = Predicate.hasProperty(error, "message") && typeof error.message === "string" ? error.message : String(error);
|
|
8
|
+
return Schema.encodeSync(JobFailure)({ _tag: tag, message });
|
|
9
|
+
}
|
|
@@ -4,7 +4,7 @@ import { defaultClientFactory, type PgBossClient, type PgBossClientFactory } fro
|
|
|
4
4
|
import { PgBossClientOptionsError, type PgBossError, toPgBossError } from "#error.ts";
|
|
5
5
|
import { deferralStorePlan } from "#internal/deferralStorePlan.ts";
|
|
6
6
|
import { acquireCachedClient, releaseCachedClient } from "./clientCache.ts";
|
|
7
|
-
import {
|
|
7
|
+
import { observeClientErrors } from "./clientErrors.ts";
|
|
8
8
|
|
|
9
9
|
export interface AcquireClientOptions {
|
|
10
10
|
readonly clientCacheKey?: string | symbol | undefined;
|
|
@@ -37,21 +37,18 @@ async function startClient(options: AcquireClientOptions, report: (error: Error)
|
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
export function acquireClient(
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
: { cacheKey, ...(await acquireCachedClient(cacheKey, options.constructor, () => startClient(options, report))) };
|
|
53
|
-
},
|
|
54
|
-
});
|
|
40
|
+
export function acquireClient(
|
|
41
|
+
options: AcquireClientOptions,
|
|
42
|
+
report: (error: Error) => void,
|
|
43
|
+
): Effect.Effect<AcquiredClient, PgBossError | PgBossClientOptionsError> {
|
|
44
|
+
return Effect.tryPromise({
|
|
45
|
+
catch: (error) => (error instanceof PgBossClientOptionsError ? error : toPgBossError("start", error)),
|
|
46
|
+
try: async () => {
|
|
47
|
+
const cacheKey = options.clientCacheKey;
|
|
48
|
+
return cacheKey === undefined
|
|
49
|
+
? { client: await startClient(options, report), reused: false }
|
|
50
|
+
: { cacheKey, ...(await acquireCachedClient(cacheKey, options.constructor, () => startClient(options, report))) };
|
|
51
|
+
},
|
|
55
52
|
});
|
|
56
53
|
}
|
|
57
54
|
|
|
@@ -1,12 +1,26 @@
|
|
|
1
|
-
import { Effect, Option, Schema } from "effect";
|
|
1
|
+
import { Effect, Option, Redacted, Schema } from "effect";
|
|
2
2
|
import type { PgBossClient } from "#client.ts";
|
|
3
3
|
import type { JobPayloadSchema } from "#definition.ts";
|
|
4
|
-
import { type PgBossError,
|
|
4
|
+
import { type PgBossError, PgBossPayloadError, toPgBossError } from "#error.ts";
|
|
5
5
|
import { type DeadLetteredJob, deadLetterQueueName, type JobPage, type JobPageOptions } from "#health.ts";
|
|
6
6
|
import { decodeStoredData, quotedSchema } from "#internal/storedRows.ts";
|
|
7
|
+
import { JobFailure } from "#JobFailure.ts";
|
|
7
8
|
import type { PgBossService } from "#service.ts";
|
|
8
9
|
|
|
9
|
-
const Cursor = Schema.StringFromBase64Url.pipe(
|
|
10
|
+
const Cursor = Schema.StringFromBase64Url.pipe(
|
|
11
|
+
Schema.decodeTo(
|
|
12
|
+
Schema.fromJsonString(
|
|
13
|
+
Schema.Tuple([
|
|
14
|
+
Schema.String.check(Schema.makeFilter((value) => Option.isSome(Schema.decodeUnknownOption(Schema.DateTimeUtcFromString)(value)))),
|
|
15
|
+
Schema.String.check(Schema.isUUID()),
|
|
16
|
+
]),
|
|
17
|
+
),
|
|
18
|
+
),
|
|
19
|
+
);
|
|
20
|
+
const PageOptions = Schema.Struct({
|
|
21
|
+
cursor: Schema.optionalKey(Schema.UndefinedOr(Cursor)),
|
|
22
|
+
limit: Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)),
|
|
23
|
+
});
|
|
10
24
|
|
|
11
25
|
const Row = Schema.Struct({
|
|
12
26
|
completedOn: Schema.OptionFromNullOr(Schema.DateTimeUtcFromDate),
|
|
@@ -14,12 +28,12 @@ const Row = Schema.Struct({
|
|
|
14
28
|
cursorAt: Schema.String,
|
|
15
29
|
data: Schema.Unknown,
|
|
16
30
|
id: Schema.String,
|
|
17
|
-
output: Schema.
|
|
31
|
+
output: Schema.NullOr(JobFailure),
|
|
18
32
|
queue: Schema.String,
|
|
19
33
|
retryCount: Schema.Number,
|
|
20
34
|
retryLimit: Schema.Number,
|
|
21
35
|
sourceId: Schema.OptionFromNullOr(Schema.String),
|
|
22
|
-
sourceOutput: Schema.
|
|
36
|
+
sourceOutput: Schema.NullOr(JobFailure),
|
|
23
37
|
startedOn: Schema.OptionFromNullOr(Schema.DateTimeUtcFromDate),
|
|
24
38
|
state: Schema.Literals(["created", "retry", "active", "completed", "cancelled", "failed"]),
|
|
25
39
|
});
|
|
@@ -39,60 +53,56 @@ export function deadLetteredJobsFor(client: PgBossClient, schema: string): PgBos
|
|
|
39
53
|
return (queue, options) => listJobs(client, schema, "deadLetteredJobs", queue, options);
|
|
40
54
|
}
|
|
41
55
|
|
|
42
|
-
function
|
|
56
|
+
const listJobs = Effect.fnUntraced(function* <const TName extends string, const TPayload extends JobPayloadSchema | typeof Schema.Null>(
|
|
43
57
|
client: PgBossClient,
|
|
44
58
|
schema: string,
|
|
45
59
|
listing: Listing,
|
|
46
60
|
queue: { readonly name: TName; readonly schema: TPayload },
|
|
47
61
|
options: JobPageOptions,
|
|
48
|
-
): Effect.
|
|
49
|
-
|
|
62
|
+
): Effect.fn.Return<JobPage<DeadLetteredJob<TPayload["Type"]>>, PgBossError | PgBossPayloadError | Schema.SchemaError, TPayload["DecodingServices"]> {
|
|
63
|
+
const { cursor: after, limit } = yield* Schema.decodeUnknownEffect(PageOptions)(options);
|
|
64
|
+
const rows = yield* Effect.tryPromise({
|
|
50
65
|
catch: (error) => toPgBossError(listing, error, queue.name),
|
|
51
66
|
try: async () => {
|
|
52
67
|
if (client.getDb === undefined) {
|
|
53
68
|
throw new TypeError("This pg-boss client does not support getDb");
|
|
54
69
|
}
|
|
55
|
-
if (!Number.isSafeInteger(options.limit) || options.limit < 1) {
|
|
56
|
-
throw new RangeError("The page limit must be a positive integer");
|
|
57
|
-
}
|
|
58
|
-
const after = options.cursor === undefined ? undefined : await Schema.decodeUnknownPromise(Cursor)(options.cursor);
|
|
59
70
|
const quoted = quotedSchema(schema);
|
|
60
|
-
const { rows } = await client.getDb().executeSql(
|
|
71
|
+
const { rows: storedRows } = await client.getDb().executeSql(
|
|
61
72
|
`SELECT
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
[listing === "failedJobs" ? queue.name : deadLetterQueueName(queue.name), after?.[0] ?? null, after?.[1] ?? null,
|
|
73
|
+
id,
|
|
74
|
+
name AS queue,
|
|
75
|
+
state,
|
|
76
|
+
retry_count AS "retryCount",
|
|
77
|
+
retry_limit AS "retryLimit",
|
|
78
|
+
created_on AS "createdOn",
|
|
79
|
+
started_on AS "startedOn",
|
|
80
|
+
completed_on AS "completedOn",
|
|
81
|
+
output,
|
|
82
|
+
data,
|
|
83
|
+
source_id AS "sourceId",
|
|
84
|
+
source_output AS "sourceOutput",
|
|
85
|
+
to_json(created_on) #>> '{}' AS "cursorAt"
|
|
86
|
+
FROM ${quoted}.job
|
|
87
|
+
WHERE name = $1
|
|
88
|
+
${listing === "failedJobs" ? "AND state = 'failed'" : ""}
|
|
89
|
+
AND ($2::timestamptz IS NULL OR (created_on, id) < ($2::timestamptz, $3::uuid))
|
|
90
|
+
ORDER BY created_on DESC, id DESC
|
|
91
|
+
LIMIT $4`,
|
|
92
|
+
[listing === "failedJobs" ? queue.name : deadLetterQueueName(queue.name), after?.[0] ?? null, after?.[1] ?? null, limit + 1],
|
|
82
93
|
);
|
|
83
|
-
return
|
|
94
|
+
return storedRows;
|
|
84
95
|
},
|
|
85
|
-
})
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
const last = page.at(-1);
|
|
89
|
-
const next =
|
|
90
|
-
rows.length > options.limit && last !== undefined ? Option.some(Schema.encodeSync(Cursor)([last.cursorAt, last.id])) : Option.none();
|
|
91
|
-
return Effect.forEach(page, ({ cursorAt: _, data, ...row }) =>
|
|
92
|
-
decodeStoredData(queue, data).pipe(
|
|
93
|
-
Effect.map((decoded) => ({ ...row, attempt: Option.isSome(row.startedOn) ? row.retryCount + 1 : 0, data: decoded })),
|
|
94
|
-
),
|
|
95
|
-
).pipe(Effect.map((jobs) => ({ jobs, next })));
|
|
96
|
-
}),
|
|
96
|
+
});
|
|
97
|
+
const decodedRows = yield* Schema.decodeUnknownEffect(Rows)(rows).pipe(
|
|
98
|
+
Effect.mapError((error) => new PgBossPayloadError({ direction: "decode", original: Redacted.make(error), queue: queue.name })),
|
|
97
99
|
);
|
|
98
|
-
|
|
100
|
+
const page = decodedRows.slice(0, limit);
|
|
101
|
+
const last = page.at(-1);
|
|
102
|
+
const next = decodedRows.length > limit && last !== undefined ? Option.some(Schema.encodeSync(Cursor)([last.cursorAt, last.id])) : Option.none();
|
|
103
|
+
return yield* Effect.forEach(page, ({ cursorAt: _, data, ...row }) =>
|
|
104
|
+
decodeStoredData(queue, data).pipe(
|
|
105
|
+
Effect.map((decoded) => ({ ...row, attempt: Option.isSome(row.startedOn) ? row.retryCount + 1 : 0, data: decoded })),
|
|
106
|
+
),
|
|
107
|
+
).pipe(Effect.map((jobs) => ({ jobs, next })));
|
|
108
|
+
});
|
|
@@ -13,7 +13,7 @@ export function makeService(client: PgBossClient, names: readonly string[], sche
|
|
|
13
13
|
enqueue: enqueueFor(client),
|
|
14
14
|
failedJobs: failedJobsFor(client, schema),
|
|
15
15
|
getJob: getJobFor(client),
|
|
16
|
-
health: healthFor(client, names),
|
|
16
|
+
health: () => healthFor(client, names),
|
|
17
17
|
retry: retryFor(client),
|
|
18
18
|
};
|
|
19
19
|
}
|
|
@@ -1,18 +1,11 @@
|
|
|
1
|
-
import { type Context, Effect,
|
|
2
|
-
import type { Job } from "pg-boss";
|
|
1
|
+
import { type Context, Effect, type Scope } from "effect";
|
|
3
2
|
import type { PgBossClient } from "#client.ts";
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
type QueueDefinition,
|
|
8
|
-
type QueueWorker,
|
|
9
|
-
type ScheduleDefinition,
|
|
10
|
-
type ScheduledWorker,
|
|
11
|
-
} from "#definition.ts";
|
|
12
|
-
import { PgBossError, PgBossPayloadError, PgBossQueueOptionsError, toPgBossError } from "#error.ts";
|
|
3
|
+
import type { JobRegistration, QueueDefinition, QueueWorker, ScheduleDefinition, ScheduledWorker } from "#definition.ts";
|
|
4
|
+
import { PgBossQueueOptionsError, toPgBossError } from "#error.ts";
|
|
5
|
+
import type { PgBossRunner } from "#internal/batchHandler.ts";
|
|
13
6
|
import { ensureQueue } from "#internal/ensureQueue.ts";
|
|
14
7
|
import { registerWorker } from "#internal/registerWorker.ts";
|
|
15
|
-
import {
|
|
8
|
+
import { runQueueWorker } from "#internal/runQueueWorker.ts";
|
|
16
9
|
|
|
17
10
|
function isQueueWorker(registration: JobRegistration): registration is QueueWorker {
|
|
18
11
|
return registration._tag === "QueueWorker";
|
|
@@ -43,21 +36,19 @@ export function registrationName(registration: JobRegistration): string {
|
|
|
43
36
|
return registrationDefinition(registration).name;
|
|
44
37
|
}
|
|
45
38
|
|
|
46
|
-
|
|
39
|
+
const prepareQueues = Effect.fnUntraced(function* (client: PgBossClient, registrations: readonly JobRegistration[], replaceWorkers: boolean) {
|
|
47
40
|
for (const registration of registrations) {
|
|
48
41
|
if (replaceWorkers && !isQueueDefinition(registration)) {
|
|
49
|
-
|
|
42
|
+
yield* Effect.tryPromise({ catch: (error) => toPgBossError("register", error), try: () => client.offWork(registrationName(registration)) });
|
|
50
43
|
}
|
|
51
44
|
}
|
|
52
45
|
for (const registration of registrations) {
|
|
53
46
|
const definition = registrationDefinition(registration);
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
throw error instanceof PgBossQueueOptionsError ? error : toPgBossError("register", error, definition.name);
|
|
58
|
-
}
|
|
47
|
+
yield* ensureQueue(client, definition.name, definition.queueOptions).pipe(
|
|
48
|
+
Effect.mapError((error) => (error instanceof PgBossQueueOptionsError ? error : toPgBossError("register", error, definition.name))),
|
|
49
|
+
);
|
|
59
50
|
}
|
|
60
|
-
}
|
|
51
|
+
});
|
|
61
52
|
|
|
62
53
|
// The layer requires RegistrationRequirements of its jobs, so its context provides every registered worker.
|
|
63
54
|
function workerContext<R>(context: Context.Context<R>): Context.Context<unknown>;
|
|
@@ -65,46 +56,61 @@ function workerContext(context: unknown): unknown {
|
|
|
65
56
|
return context;
|
|
66
57
|
}
|
|
67
58
|
|
|
68
|
-
|
|
69
|
-
return Schema.decodeUnknownEffect(worker.queue.schema)(payloadOf(job.data)).pipe(
|
|
70
|
-
Effect.mapError(
|
|
71
|
-
(error) =>
|
|
72
|
-
new PgBossPayloadError({
|
|
73
|
-
direction: "decode",
|
|
74
|
-
original: Redacted.make(error),
|
|
75
|
-
queue: worker.queue.name,
|
|
76
|
-
}),
|
|
77
|
-
),
|
|
78
|
-
Effect.flatMap((payload) => worker.handler(payload, jobContext(job))),
|
|
79
|
-
);
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
async function registerQueue(
|
|
59
|
+
const registerQueue = Effect.fnUntraced(function* (
|
|
83
60
|
client: PgBossClient,
|
|
84
61
|
registration: QueueWorker | QueueDefinition,
|
|
85
62
|
context: Context.Context<unknown>,
|
|
86
63
|
scope: Scope.Scope,
|
|
87
64
|
schema: string,
|
|
88
|
-
|
|
65
|
+
execute: PgBossRunner,
|
|
66
|
+
graceful: boolean,
|
|
67
|
+
) {
|
|
89
68
|
const queue = registration._tag === "QueueWorker" ? registration.queue : registration;
|
|
90
69
|
const name = queue.name;
|
|
91
70
|
if (registration._tag === "QueueDefinition") {
|
|
92
71
|
return;
|
|
93
72
|
}
|
|
94
|
-
|
|
95
|
-
|
|
73
|
+
yield* registerWorker(
|
|
74
|
+
client,
|
|
75
|
+
name,
|
|
76
|
+
queue.workerOptions,
|
|
77
|
+
(job) => runQueueWorker(registration, job),
|
|
78
|
+
scope,
|
|
79
|
+
schema,
|
|
80
|
+
queue.maxDeferrals,
|
|
81
|
+
context,
|
|
82
|
+
execute,
|
|
83
|
+
graceful,
|
|
84
|
+
);
|
|
85
|
+
});
|
|
96
86
|
|
|
97
|
-
|
|
87
|
+
const registerSchedule = Effect.fnUntraced(function* (
|
|
98
88
|
client: PgBossClient,
|
|
99
89
|
worker: ScheduledWorker,
|
|
100
90
|
context: Context.Context<unknown>,
|
|
101
91
|
scope: Scope.Scope,
|
|
102
92
|
schema: string,
|
|
103
|
-
|
|
93
|
+
execute: PgBossRunner,
|
|
94
|
+
graceful: boolean,
|
|
95
|
+
) {
|
|
104
96
|
const { schedule } = worker;
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
97
|
+
yield* registerWorker(
|
|
98
|
+
client,
|
|
99
|
+
schedule.name,
|
|
100
|
+
schedule.workerOptions,
|
|
101
|
+
() => worker.effect,
|
|
102
|
+
scope,
|
|
103
|
+
schema,
|
|
104
|
+
schedule.maxDeferrals,
|
|
105
|
+
context,
|
|
106
|
+
execute,
|
|
107
|
+
graceful,
|
|
108
|
+
);
|
|
109
|
+
yield* Effect.tryPromise({
|
|
110
|
+
catch: (error) => toPgBossError("register", error, schedule.name),
|
|
111
|
+
try: () => client.schedule(schedule.name, schedule.cron, null, schedule.scheduleOptions),
|
|
112
|
+
});
|
|
113
|
+
});
|
|
108
114
|
|
|
109
115
|
export function registrationNames(registrations: readonly JobRegistration[]): readonly string[] {
|
|
110
116
|
const names = registrations.map(registrationName);
|
|
@@ -114,32 +120,28 @@ export function registrationNames(registrations: readonly JobRegistration[]): re
|
|
|
114
120
|
return names;
|
|
115
121
|
}
|
|
116
122
|
|
|
117
|
-
export function
|
|
123
|
+
export const registerJobs = Effect.fnUntraced(function* <R>(
|
|
118
124
|
client: PgBossClient,
|
|
119
125
|
registrations: readonly JobRegistration[],
|
|
120
126
|
context: Context.Context<R>,
|
|
121
127
|
replaceWorkers: boolean,
|
|
128
|
+
execute: PgBossRunner,
|
|
122
129
|
schema = "pgboss",
|
|
123
130
|
startWorkers = true,
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
}
|
|
142
|
-
},
|
|
143
|
-
}),
|
|
144
|
-
);
|
|
145
|
-
}
|
|
131
|
+
graceful = true,
|
|
132
|
+
) {
|
|
133
|
+
const scope = yield* Effect.scope;
|
|
134
|
+
yield* Effect.try({ catch: (error) => toPgBossError("register", error), try: () => registrationNames(registrations) });
|
|
135
|
+
yield* prepareQueues(client, registrations, replaceWorkers);
|
|
136
|
+
if (!startWorkers) {
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
const provided = workerContext(context);
|
|
140
|
+
for (const registration of registrations) {
|
|
141
|
+
if (isQueueWorker(registration) || isQueueDefinition(registration)) {
|
|
142
|
+
yield* registerQueue(client, registration, provided, scope, schema, execute, graceful);
|
|
143
|
+
} else if (isScheduledWorker(registration)) {
|
|
144
|
+
yield* registerSchedule(client, registration, provided, scope, schema, execute, graceful);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
});
|