@intelligo-dev/jobs 1.0.0-beta.1 → 1.0.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +6 -0
- package/README.md +78 -0
- package/dist/db/schema.d.ts +1 -4
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +1 -4
- package/dist/db/schema.js.map +1 -1
- package/dist/index.d.ts +6 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +43 -26
- package/dist/index.js.map +1 -1
- package/package.json +38 -10
- package/src/db/schema.ts +50 -0
- package/src/index.ts +198 -0
package/NOTICE
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
Intelligo
|
|
2
|
+
Copyright 2026 Turtuvshin Byambaa and the Intelligo contributors
|
|
3
|
+
|
|
4
|
+
This product is licensed under the Apache License, Version 2.0 (see
|
|
5
|
+
LICENSE). The Intelligo name and logo are trademarks; see TRADEMARK.md
|
|
6
|
+
in the source repository, https://github.com/intelligo-dev/intelligo.
|
package/README.md
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# @intelligo-dev/jobs
|
|
2
|
+
|
|
3
|
+
A Postgres-backed job queue: enqueue, claim with SKIP LOCKED, retry with backoff and prune. No Redis.
|
|
4
|
+
|
|
5
|
+
Part of [Intelligo](https://intelligo.dev), an application framework and
|
|
6
|
+
operational platform for vertical AI SaaS products. Every `@intelligo-dev/*`
|
|
7
|
+
package is released at one version and shares one database schema;
|
|
8
|
+
`pnpm dlx @intelligo-dev/cli@beta create my-app` installs the set an
|
|
9
|
+
application needs. Documentation:
|
|
10
|
+
[intelligo.dev/docs/packages/jobs](https://intelligo.dev/docs/packages/jobs).
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pnpm add @intelligo-dev/jobs@beta drizzle-orm
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
`drizzle-orm` is a peer. The queue is one table, `jobs`, in the framework's
|
|
19
|
+
schema: `intelligo migrate` from
|
|
20
|
+
[`@intelligo-dev/cli`](https://www.npmjs.com/package/@intelligo-dev/cli)
|
|
21
|
+
creates it, and the connection is the `DATABASE_URL` that
|
|
22
|
+
`@intelligo-dev/core` reads.
|
|
23
|
+
|
|
24
|
+
## Use
|
|
25
|
+
|
|
26
|
+
Enqueue from anywhere that can reach the database:
|
|
27
|
+
|
|
28
|
+
```ts
|
|
29
|
+
import { enqueue } from "@intelligo-dev/jobs";
|
|
30
|
+
|
|
31
|
+
await enqueue({
|
|
32
|
+
kind: "digest.send",
|
|
33
|
+
workspaceId,
|
|
34
|
+
payload: { userId },
|
|
35
|
+
runAt: new Date(Date.now() + 60_000), // optional: not before
|
|
36
|
+
maxAttempts: 5, // optional: 3 by default
|
|
37
|
+
});
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Drain from a cron route, a long-running worker or a script — handlers are
|
|
41
|
+
keyed by `kind`:
|
|
42
|
+
|
|
43
|
+
```ts
|
|
44
|
+
import { drain } from "@intelligo-dev/jobs";
|
|
45
|
+
|
|
46
|
+
const result = await drain(
|
|
47
|
+
{
|
|
48
|
+
"digest.send": async (job) => {
|
|
49
|
+
await sendDigest(job.payload);
|
|
50
|
+
},
|
|
51
|
+
},
|
|
52
|
+
{ limit: 25 }
|
|
53
|
+
);
|
|
54
|
+
// { claimed, succeeded, failed, unhandled }
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
## What the queue guarantees
|
|
58
|
+
|
|
59
|
+
- **No double-processing.** Claiming is `FOR UPDATE SKIP LOCKED`, so several
|
|
60
|
+
workers drain the same queue and each takes different rows.
|
|
61
|
+
- **Retry with backoff.** A handler that throws puts the job back, one more
|
|
62
|
+
minute later per attempt; after `maxAttempts` it is `failed` and keeps its
|
|
63
|
+
last error. `listFailedJobs()` reads them, newest first.
|
|
64
|
+
- **A dead worker does not strand a job.** One still `running` thirty minutes
|
|
65
|
+
after its handler started is claimed again, so a handler must finish well
|
|
66
|
+
inside that.
|
|
67
|
+
- **An unknown kind costs nothing.** A job with no handler in this worker goes
|
|
68
|
+
back without spending an attempt, a minute behind the jobs the worker can
|
|
69
|
+
run — another deployment may know it.
|
|
70
|
+
|
|
71
|
+
`pruneJobs(before)` deletes succeeded jobs older than the cutoff and returns
|
|
72
|
+
how many. `postgresJobQueue` is the same `enqueue` and `drain` as one
|
|
73
|
+
`JobQueue` value; a different queue implements that type, and call sites stay
|
|
74
|
+
as they are.
|
|
75
|
+
|
|
76
|
+
## Licence
|
|
77
|
+
|
|
78
|
+
Apache-2.0
|
package/dist/db/schema.d.ts
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @intelligo-dev/jobs database schema.
|
|
3
|
-
*
|
|
4
2
|
* A DB-backed job queue — the default adapter behind the job contract.
|
|
5
3
|
* No Redis, no external broker: Postgres `FOR UPDATE SKIP LOCKED` is
|
|
6
|
-
* sufficient at this scale and keeps self-hosting to one dependency
|
|
7
|
-
* (the same reasoning as the rate limiter).
|
|
4
|
+
* sufficient at this scale and keeps self-hosting to one dependency.
|
|
8
5
|
*
|
|
9
6
|
* Scanned by drizzle-kit alongside the core schema directory — see
|
|
10
7
|
* packages/core/drizzle.config.ts.
|
package/dist/db/schema.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/db/schema.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/db/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAYH,eAAO,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA2BhB,CAAC;AAEF,MAAM,MAAM,GAAG,GAAG,OAAO,IAAI,CAAC,YAAY,CAAC;AAC3C,MAAM,MAAM,SAAS,GAAG,OAAO,IAAI,CAAC,YAAY,CAAC"}
|
package/dist/db/schema.js
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* @intelligo-dev/jobs database schema.
|
|
3
|
-
*
|
|
4
2
|
* A DB-backed job queue — the default adapter behind the job contract.
|
|
5
3
|
* No Redis, no external broker: Postgres `FOR UPDATE SKIP LOCKED` is
|
|
6
|
-
* sufficient at this scale and keeps self-hosting to one dependency
|
|
7
|
-
* (the same reasoning as the rate limiter).
|
|
4
|
+
* sufficient at this scale and keeps self-hosting to one dependency.
|
|
8
5
|
*
|
|
9
6
|
* Scanned by drizzle-kit alongside the core schema directory — see
|
|
10
7
|
* packages/core/drizzle.config.ts.
|
package/dist/db/schema.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/db/schema.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/db/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EACL,OAAO,EACP,IAAI,EACJ,SAAS,EACT,OAAO,EACP,KAAK,EACL,KAAK,GACN,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAE7D,MAAM,CAAC,MAAM,IAAI,GAAG,OAAO,CACzB,MAAM,EACN;IACE,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,kEAAkE;IAClE,WAAW,EAAE,IAAI,CAAC,cAAc,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE;QAClE,QAAQ,EAAE,SAAS;KACpB,CAAC;IACF,wDAAwD;IACxD,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE;IAC5B,OAAO,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,KAAK,EAA2B;IAC1D,6CAA6C;IAC7C,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC;IACnD,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClD,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACzD,iDAAiD;IACjD,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE;IACjD,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC;IAClC,UAAU,EAAE,SAAS,CAAC,aAAa,CAAC;IACpC,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC;IAC7B,SAAS,EAAE,SAAS,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE;CAC1D,EACD,CAAC,KAAK,EAAE,EAAE,CAAC;IACT,KAAK,CAAC,gBAAgB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC;IACrD,KAAK,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;IACnD,KAAK,CAAC,oBAAoB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC;CAClD,CACF,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
* @intelligo-dev/jobs — job contract plus a Postgres-backed default adapter.
|
|
3
3
|
*
|
|
4
4
|
* The contract is what consumers code against; the adapter is what runs
|
|
5
|
-
* when nobody supplies
|
|
6
|
-
*
|
|
5
|
+
* when nobody supplies another. Swapping in a different queue means
|
|
6
|
+
* implementing `JobQueue`, not rewriting call sites.
|
|
7
7
|
*
|
|
8
8
|
* Claiming uses `FOR UPDATE SKIP LOCKED`, so several workers (Vercel
|
|
9
9
|
* cron invocations, a long-running process, a local script) can drain
|
|
10
10
|
* the same queue without double-processing.
|
|
11
11
|
*/
|
|
12
|
-
import type { Job } from "./db/schema";
|
|
12
|
+
import type { Job } from "./db/schema.js";
|
|
13
13
|
export type EnqueueInput = {
|
|
14
14
|
kind: string;
|
|
15
15
|
payload?: Record<string, unknown>;
|
|
@@ -37,7 +37,7 @@ export declare function enqueue(input: EnqueueInput): Promise<string>;
|
|
|
37
37
|
export declare function drain(handlers: Record<string, JobHandler>, options?: {
|
|
38
38
|
limit?: number;
|
|
39
39
|
}): Promise<DrainResult>;
|
|
40
|
-
/** Jobs that exhausted their attempts — surfaced in the admin console. */
|
|
40
|
+
/** Jobs that exhausted their attempts, newest first — surfaced in the admin console. */
|
|
41
41
|
export declare function listFailedJobs(limit?: number): Promise<{
|
|
42
42
|
id: string;
|
|
43
43
|
workspaceId: string | null;
|
|
@@ -56,6 +56,6 @@ export declare function listFailedJobs(limit?: number): Promise<{
|
|
|
56
56
|
export declare function pruneJobs(before: Date): Promise<number>;
|
|
57
57
|
/** The default Postgres-backed queue. */
|
|
58
58
|
export declare const postgresJobQueue: JobQueue;
|
|
59
|
-
export { jobs } from "./db/schema";
|
|
60
|
-
export type { Job, InsertJob } from "./db/schema";
|
|
59
|
+
export { jobs } from "./db/schema.js";
|
|
60
|
+
export type { Job, InsertJob } from "./db/schema.js";
|
|
61
61
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAOH,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAOH,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAOvC,MAAM,MAAM,YAAY,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,8CAA8C;IAC9C,KAAK,CAAC,EAAE,IAAI,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAErD,MAAM,MAAM,QAAQ,GAAG;IACrB,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9C,mEAAmE;IACnE,KAAK,CACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,EACpC,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAC3B,OAAO,CAAC,WAAW,CAAC,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB,CAAC;AAEF,wBAAsB,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAWlE;AAoCD,wBAAsB,KAAK,CACzB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,EACpC,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAC/B,OAAO,CAAC,WAAW,CAAC,CAsEtB;AAED,wFAAwF;AACxF,wBAAsB,cAAc,CAAC,KAAK,SAAK;;;;;;;;;;;;;KAO9C;AAED,mDAAmD;AACnD,wBAAsB,SAAS,CAAC,MAAM,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAM7D;AAED,yCAAyC;AACzC,eAAO,MAAM,gBAAgB,EAAE,QAA6B,CAAC;AAE7D,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,YAAY,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
* @intelligo-dev/jobs — job contract plus a Postgres-backed default adapter.
|
|
3
3
|
*
|
|
4
4
|
* The contract is what consumers code against; the adapter is what runs
|
|
5
|
-
* when nobody supplies
|
|
6
|
-
*
|
|
5
|
+
* when nobody supplies another. Swapping in a different queue means
|
|
6
|
+
* implementing `JobQueue`, not rewriting call sites.
|
|
7
7
|
*
|
|
8
8
|
* Claiming uses `FOR UPDATE SKIP LOCKED`, so several workers (Vercel
|
|
9
9
|
* cron invocations, a long-running process, a local script) can drain
|
|
@@ -11,9 +11,11 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { db } from "@intelligo-dev/core/db";
|
|
13
13
|
import { createLogger } from "@intelligo-dev/core/logger";
|
|
14
|
-
import { and, eq, lte, sql } from "drizzle-orm";
|
|
15
|
-
import { jobs } from "./db/schema";
|
|
14
|
+
import { and, desc, eq, inArray, lt, lte, or, sql } from "drizzle-orm";
|
|
15
|
+
import { jobs } from "./db/schema.js";
|
|
16
16
|
const log = createLogger("Jobs");
|
|
17
|
+
const UNHANDLED_DELAY_MS = 60000;
|
|
18
|
+
const ABANDONED_AFTER_MS = 30 * 60000;
|
|
17
19
|
export async function enqueue(input) {
|
|
18
20
|
const id = crypto.randomUUID();
|
|
19
21
|
await db.insert(jobs).values({
|
|
@@ -28,27 +30,30 @@ export async function enqueue(input) {
|
|
|
28
30
|
}
|
|
29
31
|
/**
|
|
30
32
|
* Claim due jobs atomically. SKIP LOCKED means a concurrent worker
|
|
31
|
-
* takes different rows rather than blocking on ours.
|
|
33
|
+
* takes different rows rather than blocking on ours. A job whose
|
|
34
|
+
* handler started more than `ABANDONED_AFTER_MS` ago and is still
|
|
35
|
+
* `running` belongs to a worker that died mid-handler, and is claimed
|
|
36
|
+
* again — so a handler must finish well within that.
|
|
32
37
|
*/
|
|
33
38
|
async function claim(limit) {
|
|
34
39
|
const now = new Date();
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
40
|
+
const abandonedBefore = new Date(now.getTime() - ABANDONED_AFTER_MS);
|
|
41
|
+
const due = db
|
|
42
|
+
.select({ id: jobs.id })
|
|
43
|
+
.from(jobs)
|
|
44
|
+
.where(or(and(eq(jobs.status, "pending"), lte(jobs.runAt, now)), and(eq(jobs.status, "running"), lt(jobs.startedAt, abandonedBefore))))
|
|
45
|
+
.orderBy(jobs.runAt)
|
|
46
|
+
.limit(limit)
|
|
47
|
+
.for("update", { skipLocked: true });
|
|
48
|
+
return db
|
|
49
|
+
.update(jobs)
|
|
50
|
+
.set({
|
|
51
|
+
status: "running",
|
|
52
|
+
attempts: sql `${jobs.attempts} + 1`,
|
|
53
|
+
startedAt: now,
|
|
54
|
+
})
|
|
55
|
+
.where(inArray(jobs.id, due))
|
|
56
|
+
.returning();
|
|
52
57
|
}
|
|
53
58
|
export async function drain(handlers, options = {}) {
|
|
54
59
|
const claimedJobs = await claim(options.limit ?? 10);
|
|
@@ -63,14 +68,26 @@ export async function drain(handlers, options = {}) {
|
|
|
63
68
|
if (!handler) {
|
|
64
69
|
// Put it back rather than burning an attempt on a kind this
|
|
65
70
|
// worker simply doesn't know about — another deployment might.
|
|
71
|
+
// Moving `runAt` sends it behind the due jobs this worker can run,
|
|
72
|
+
// so a backlog of unknown kinds cannot fill every claim.
|
|
66
73
|
result.unhandled.push(job.kind);
|
|
67
74
|
await db
|
|
68
75
|
.update(jobs)
|
|
69
|
-
.set({
|
|
76
|
+
.set({
|
|
77
|
+
status: "pending",
|
|
78
|
+
attempts: sql `${jobs.attempts} - 1`,
|
|
79
|
+
runAt: new Date(Date.now() + UNHANDLED_DELAY_MS),
|
|
80
|
+
})
|
|
70
81
|
.where(eq(jobs.id, job.id));
|
|
71
82
|
continue;
|
|
72
83
|
}
|
|
73
84
|
try {
|
|
85
|
+
// A batch runs one job at a time; the clock that decides a job was
|
|
86
|
+
// abandoned starts when its own handler does.
|
|
87
|
+
await db
|
|
88
|
+
.update(jobs)
|
|
89
|
+
.set({ startedAt: new Date() })
|
|
90
|
+
.where(eq(jobs.id, job.id));
|
|
74
91
|
await handler(job);
|
|
75
92
|
await db
|
|
76
93
|
.update(jobs)
|
|
@@ -106,13 +123,13 @@ export async function drain(handlers, options = {}) {
|
|
|
106
123
|
}
|
|
107
124
|
return result;
|
|
108
125
|
}
|
|
109
|
-
/** Jobs that exhausted their attempts — surfaced in the admin console. */
|
|
126
|
+
/** Jobs that exhausted their attempts, newest first — surfaced in the admin console. */
|
|
110
127
|
export async function listFailedJobs(limit = 50) {
|
|
111
128
|
return db
|
|
112
129
|
.select()
|
|
113
130
|
.from(jobs)
|
|
114
131
|
.where(eq(jobs.status, "failed"))
|
|
115
|
-
.orderBy(jobs.finishedAt)
|
|
132
|
+
.orderBy(desc(jobs.finishedAt))
|
|
116
133
|
.limit(limit);
|
|
117
134
|
}
|
|
118
135
|
/** Delete succeeded jobs older than the cutoff. */
|
|
@@ -125,5 +142,5 @@ export async function pruneJobs(before) {
|
|
|
125
142
|
}
|
|
126
143
|
/** The default Postgres-backed queue. */
|
|
127
144
|
export const postgresJobQueue = { enqueue, drain };
|
|
128
|
-
export { jobs } from "./db/schema";
|
|
145
|
+
export { jobs } from "./db/schema.js";
|
|
129
146
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,EAAE,EAAE,MAAM,wBAAwB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,EAAE,EAAE,MAAM,wBAAwB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAEvE,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAGnC,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;AAEjC,MAAM,kBAAkB,GAAG,KAAM,CAAC;AAClC,MAAM,kBAAkB,GAAG,EAAE,GAAG,KAAM,CAAC;AA8BvC,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAmB;IAC/C,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;IAC/B,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC3B,EAAE;QACF,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,IAAI;QACtC,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,IAAI;QAC9B,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE;QAChC,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC;KACpC,CAAC,CAAC;IACH,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,KAAK,CAAC,KAAa;IAChC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,eAAe,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,kBAAkB,CAAC,CAAC;IACrE,MAAM,GAAG,GAAG,EAAE;SACX,MAAM,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;SACvB,IAAI,CAAC,IAAI,CAAC;SACV,KAAK,CACJ,EAAE,CACA,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,EACrD,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC,CACrE,CACF;SACA,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;SACnB,KAAK,CAAC,KAAK,CAAC;SACZ,GAAG,CAAC,QAAQ,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvC,OAAO,EAAE;SACN,MAAM,CAAC,IAAI,CAAC;SACZ,GAAG,CAAC;QACH,MAAM,EAAE,SAAS;QACjB,QAAQ,EAAE,GAAG,CAAA,GAAG,IAAI,CAAC,QAAQ,MAAM;QACnC,SAAS,EAAE,GAAG;KACf,CAAC;SACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;SAC5B,SAAS,EAAE,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CACzB,QAAoC,EACpC,UAA8B,EAAE;IAEhC,MAAM,WAAW,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACrD,MAAM,MAAM,GAAgB;QAC1B,OAAO,EAAE,WAAW,CAAC,MAAM;QAC3B,SAAS,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC;QACT,SAAS,EAAE,EAAE;KACd,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEnC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,4DAA4D;YAC5D,+DAA+D;YAC/D,mEAAmE;YACnE,yDAAyD;YACzD,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,EAAE;iBACL,MAAM,CAAC,IAAI,CAAC;iBACZ,GAAG,CAAC;gBACH,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,GAAG,CAAA,GAAG,IAAI,CAAC,QAAQ,MAAM;gBACnC,KAAK,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC;aACjD,CAAC;iBACD,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QAED,IAAI,CAAC;YACH,mEAAmE;YACnE,8CAA8C;YAC9C,MAAM,EAAE;iBACL,MAAM,CAAC,IAAI,CAAC;iBACZ,GAAG,CAAC,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC;iBAC9B,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YACnB,MAAM,EAAE;iBACL,MAAM,CAAC,IAAI,CAAC;iBACZ,GAAG,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;iBACrE,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9B,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,WAAW,CAAC;YAClD,MAAM,EAAE;iBACL,MAAM,CAAC,IAAI,CAAC;iBACZ,GAAG,CAAC;gBACH,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;gBACxC,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;gBACzC,uDAAuD;gBACvD,wBAAwB;gBACxB,KAAK,EAAE,SAAS;oBACd,CAAC,CAAC,GAAG,CAAC,KAAK;oBACX,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,QAAQ,GAAG,KAAM,CAAC;gBAChD,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;aAClC,CAAC;iBACD,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9B,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;YACnB,GAAG,CAAC,KAAK,CAAC,YAAY,EAAE;gBACtB,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC7B,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;gBAC5B,KAAK,EAAE,OAAO;aACf,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,wFAAwF;AACxF,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAK,GAAG,EAAE;IAC7C,OAAO,EAAE;SACN,MAAM,EAAE;SACR,IAAI,CAAC,IAAI,CAAC;SACV,KAAK,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;SAChC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;SAC9B,KAAK,CAAC,KAAK,CAAC,CAAC;AAClB,CAAC;AAED,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAY;IAC1C,MAAM,OAAO,GAAG,MAAM,EAAE;SACrB,MAAM,CAAC,IAAI,CAAC;SACZ,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;SACtE,SAAS,EAAE,CAAC;IACf,OAAO,OAAO,CAAC,MAAM,CAAC;AACxB,CAAC;AAED,yCAAyC;AACzC,MAAM,CAAC,MAAM,gBAAgB,GAAa,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAE7D,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,15 +1,26 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@intelligo-dev/jobs",
|
|
3
|
-
"version": "1.0.0-beta.
|
|
3
|
+
"version": "1.0.0-beta.13",
|
|
4
|
+
"description": "A Postgres-backed job queue: enqueue, claim with SKIP LOCKED, retry with backoff and prune. No Redis.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"intelligo",
|
|
7
|
+
"ai-saas",
|
|
8
|
+
"saas",
|
|
9
|
+
"typescript",
|
|
10
|
+
"job-queue",
|
|
11
|
+
"background-jobs",
|
|
12
|
+
"postgres"
|
|
13
|
+
],
|
|
4
14
|
"license": "Apache-2.0",
|
|
15
|
+
"author": "Turtuvshin Byambaa <toroo.byamba@gmail.com>",
|
|
5
16
|
"repository": {
|
|
6
17
|
"type": "git",
|
|
7
|
-
"url": "git+https://github.com/intelligo-
|
|
18
|
+
"url": "git+https://github.com/intelligo-dev/intelligo.git",
|
|
8
19
|
"directory": "packages/jobs"
|
|
9
20
|
},
|
|
10
|
-
"homepage": "https://
|
|
21
|
+
"homepage": "https://intelligo.dev/docs/packages/jobs",
|
|
11
22
|
"bugs": {
|
|
12
|
-
"url": "https://github.com/intelligo-
|
|
23
|
+
"url": "https://github.com/intelligo-dev/intelligo/issues"
|
|
13
24
|
},
|
|
14
25
|
"type": "module",
|
|
15
26
|
"exports": {
|
|
@@ -23,22 +34,39 @@
|
|
|
23
34
|
}
|
|
24
35
|
},
|
|
25
36
|
"dependencies": {
|
|
26
|
-
"
|
|
27
|
-
|
|
37
|
+
"@intelligo-dev/core": "1.0.0-beta.13"
|
|
38
|
+
},
|
|
39
|
+
"peerDependencies": {
|
|
40
|
+
"drizzle-orm": "^0.45.1"
|
|
28
41
|
},
|
|
29
42
|
"devDependencies": {
|
|
30
|
-
"@types/node": "^
|
|
31
|
-
"
|
|
43
|
+
"@types/node": "^26.6.1",
|
|
44
|
+
"@types/pg": "^8.23.1",
|
|
45
|
+
"drizzle-orm": "^0.45.2",
|
|
46
|
+
"pg": "8.18.0",
|
|
47
|
+
"typescript": "^5.9.3"
|
|
32
48
|
},
|
|
33
49
|
"files": [
|
|
34
|
-
"dist"
|
|
50
|
+
"dist",
|
|
51
|
+
"src",
|
|
52
|
+
"!src/**/*.test.ts",
|
|
53
|
+
"!src/**/*.test.tsx",
|
|
54
|
+
"!src/**/__tests__/**",
|
|
55
|
+
"LICENSE",
|
|
56
|
+
"NOTICE",
|
|
57
|
+
"README.md",
|
|
58
|
+
"!dist/**/*.tsbuildinfo"
|
|
35
59
|
],
|
|
36
60
|
"publishConfig": {
|
|
37
61
|
"access": "public"
|
|
38
62
|
},
|
|
63
|
+
"engines": {
|
|
64
|
+
"node": ">=22.14"
|
|
65
|
+
},
|
|
66
|
+
"sideEffects": false,
|
|
39
67
|
"scripts": {
|
|
40
68
|
"type-check": "tsc --noEmit",
|
|
41
69
|
"lint": "eslint .",
|
|
42
|
-
"build": "tsc -p tsconfig.build.json"
|
|
70
|
+
"build": "tsc -p tsconfig.build.json && node ../../scripts/fix-esm-extensions.mjs dist"
|
|
43
71
|
}
|
|
44
72
|
}
|
package/src/db/schema.ts
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A DB-backed job queue — the default adapter behind the job contract.
|
|
3
|
+
* No Redis, no external broker: Postgres `FOR UPDATE SKIP LOCKED` is
|
|
4
|
+
* sufficient at this scale and keeps self-hosting to one dependency.
|
|
5
|
+
*
|
|
6
|
+
* Scanned by drizzle-kit alongside the core schema directory — see
|
|
7
|
+
* packages/core/drizzle.config.ts.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import {
|
|
11
|
+
pgTable,
|
|
12
|
+
text,
|
|
13
|
+
timestamp,
|
|
14
|
+
integer,
|
|
15
|
+
jsonb,
|
|
16
|
+
index,
|
|
17
|
+
} from "drizzle-orm/pg-core";
|
|
18
|
+
import { organization } from "@intelligo-dev/core/db/schema";
|
|
19
|
+
|
|
20
|
+
export const jobs = pgTable(
|
|
21
|
+
"jobs",
|
|
22
|
+
{
|
|
23
|
+
id: text("id").primaryKey(),
|
|
24
|
+
/** Null for platform-wide jobs (cleanup crons, health sweeps). */
|
|
25
|
+
workspaceId: text("workspace_id").references(() => organization.id, {
|
|
26
|
+
onDelete: "cascade",
|
|
27
|
+
}),
|
|
28
|
+
/** Handler key, e.g. "credits.cleanup-reservations". */
|
|
29
|
+
kind: text("kind").notNull(),
|
|
30
|
+
payload: jsonb("payload").$type<Record<string, unknown>>(),
|
|
31
|
+
/** pending | running | succeeded | failed */
|
|
32
|
+
status: text("status").notNull().default("pending"),
|
|
33
|
+
attempts: integer("attempts").notNull().default(0),
|
|
34
|
+
maxAttempts: integer("max_attempts").notNull().default(3),
|
|
35
|
+
/** Earliest time a worker may claim this job. */
|
|
36
|
+
runAt: timestamp("run_at").notNull().defaultNow(),
|
|
37
|
+
startedAt: timestamp("started_at"),
|
|
38
|
+
finishedAt: timestamp("finished_at"),
|
|
39
|
+
lastError: text("last_error"),
|
|
40
|
+
createdAt: timestamp("created_at").notNull().defaultNow(),
|
|
41
|
+
},
|
|
42
|
+
(table) => [
|
|
43
|
+
index("jobs_claim_idx").on(table.status, table.runAt),
|
|
44
|
+
index("jobs_kind_idx").on(table.kind, table.status),
|
|
45
|
+
index("jobs_workspace_idx").on(table.workspaceId),
|
|
46
|
+
]
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
export type Job = typeof jobs.$inferSelect;
|
|
50
|
+
export type InsertJob = typeof jobs.$inferInsert;
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @intelligo-dev/jobs — job contract plus a Postgres-backed default adapter.
|
|
3
|
+
*
|
|
4
|
+
* The contract is what consumers code against; the adapter is what runs
|
|
5
|
+
* when nobody supplies another. Swapping in a different queue means
|
|
6
|
+
* implementing `JobQueue`, not rewriting call sites.
|
|
7
|
+
*
|
|
8
|
+
* Claiming uses `FOR UPDATE SKIP LOCKED`, so several workers (Vercel
|
|
9
|
+
* cron invocations, a long-running process, a local script) can drain
|
|
10
|
+
* the same queue without double-processing.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { db } from "@intelligo-dev/core/db";
|
|
14
|
+
import { createLogger } from "@intelligo-dev/core/logger";
|
|
15
|
+
import { and, desc, eq, inArray, lt, lte, or, sql } from "drizzle-orm";
|
|
16
|
+
|
|
17
|
+
import { jobs } from "./db/schema";
|
|
18
|
+
import type { Job } from "./db/schema";
|
|
19
|
+
|
|
20
|
+
const log = createLogger("Jobs");
|
|
21
|
+
|
|
22
|
+
const UNHANDLED_DELAY_MS = 60_000;
|
|
23
|
+
const ABANDONED_AFTER_MS = 30 * 60_000;
|
|
24
|
+
|
|
25
|
+
export type EnqueueInput = {
|
|
26
|
+
kind: string;
|
|
27
|
+
payload?: Record<string, unknown>;
|
|
28
|
+
workspaceId?: string | null;
|
|
29
|
+
/** Delay before the job becomes claimable. */
|
|
30
|
+
runAt?: Date;
|
|
31
|
+
maxAttempts?: number;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export type JobHandler = (job: Job) => Promise<void>;
|
|
35
|
+
|
|
36
|
+
export type JobQueue = {
|
|
37
|
+
enqueue(input: EnqueueInput): Promise<string>;
|
|
38
|
+
/** Claim and run up to `limit` due jobs; returns what happened. */
|
|
39
|
+
drain(
|
|
40
|
+
handlers: Record<string, JobHandler>,
|
|
41
|
+
options?: { limit?: number }
|
|
42
|
+
): Promise<DrainResult>;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export type DrainResult = {
|
|
46
|
+
claimed: number;
|
|
47
|
+
succeeded: number;
|
|
48
|
+
failed: number;
|
|
49
|
+
/** Jobs whose kind had no registered handler — left pending. */
|
|
50
|
+
unhandled: string[];
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
export async function enqueue(input: EnqueueInput): Promise<string> {
|
|
54
|
+
const id = crypto.randomUUID();
|
|
55
|
+
await db.insert(jobs).values({
|
|
56
|
+
id,
|
|
57
|
+
workspaceId: input.workspaceId ?? null,
|
|
58
|
+
kind: input.kind,
|
|
59
|
+
payload: input.payload ?? null,
|
|
60
|
+
runAt: input.runAt ?? new Date(),
|
|
61
|
+
maxAttempts: input.maxAttempts ?? 3,
|
|
62
|
+
});
|
|
63
|
+
return id;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Claim due jobs atomically. SKIP LOCKED means a concurrent worker
|
|
68
|
+
* takes different rows rather than blocking on ours. A job whose
|
|
69
|
+
* handler started more than `ABANDONED_AFTER_MS` ago and is still
|
|
70
|
+
* `running` belongs to a worker that died mid-handler, and is claimed
|
|
71
|
+
* again — so a handler must finish well within that.
|
|
72
|
+
*/
|
|
73
|
+
async function claim(limit: number): Promise<Job[]> {
|
|
74
|
+
const now = new Date();
|
|
75
|
+
const abandonedBefore = new Date(now.getTime() - ABANDONED_AFTER_MS);
|
|
76
|
+
const due = db
|
|
77
|
+
.select({ id: jobs.id })
|
|
78
|
+
.from(jobs)
|
|
79
|
+
.where(
|
|
80
|
+
or(
|
|
81
|
+
and(eq(jobs.status, "pending"), lte(jobs.runAt, now)),
|
|
82
|
+
and(eq(jobs.status, "running"), lt(jobs.startedAt, abandonedBefore))
|
|
83
|
+
)
|
|
84
|
+
)
|
|
85
|
+
.orderBy(jobs.runAt)
|
|
86
|
+
.limit(limit)
|
|
87
|
+
.for("update", { skipLocked: true });
|
|
88
|
+
|
|
89
|
+
return db
|
|
90
|
+
.update(jobs)
|
|
91
|
+
.set({
|
|
92
|
+
status: "running",
|
|
93
|
+
attempts: sql`${jobs.attempts} + 1`,
|
|
94
|
+
startedAt: now,
|
|
95
|
+
})
|
|
96
|
+
.where(inArray(jobs.id, due))
|
|
97
|
+
.returning();
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export async function drain(
|
|
101
|
+
handlers: Record<string, JobHandler>,
|
|
102
|
+
options: { limit?: number } = {}
|
|
103
|
+
): Promise<DrainResult> {
|
|
104
|
+
const claimedJobs = await claim(options.limit ?? 10);
|
|
105
|
+
const result: DrainResult = {
|
|
106
|
+
claimed: claimedJobs.length,
|
|
107
|
+
succeeded: 0,
|
|
108
|
+
failed: 0,
|
|
109
|
+
unhandled: [],
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
for (const job of claimedJobs) {
|
|
113
|
+
const handler = handlers[job.kind];
|
|
114
|
+
|
|
115
|
+
if (!handler) {
|
|
116
|
+
// Put it back rather than burning an attempt on a kind this
|
|
117
|
+
// worker simply doesn't know about — another deployment might.
|
|
118
|
+
// Moving `runAt` sends it behind the due jobs this worker can run,
|
|
119
|
+
// so a backlog of unknown kinds cannot fill every claim.
|
|
120
|
+
result.unhandled.push(job.kind);
|
|
121
|
+
await db
|
|
122
|
+
.update(jobs)
|
|
123
|
+
.set({
|
|
124
|
+
status: "pending",
|
|
125
|
+
attempts: sql`${jobs.attempts} - 1`,
|
|
126
|
+
runAt: new Date(Date.now() + UNHANDLED_DELAY_MS),
|
|
127
|
+
})
|
|
128
|
+
.where(eq(jobs.id, job.id));
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
try {
|
|
133
|
+
// A batch runs one job at a time; the clock that decides a job was
|
|
134
|
+
// abandoned starts when its own handler does.
|
|
135
|
+
await db
|
|
136
|
+
.update(jobs)
|
|
137
|
+
.set({ startedAt: new Date() })
|
|
138
|
+
.where(eq(jobs.id, job.id));
|
|
139
|
+
await handler(job);
|
|
140
|
+
await db
|
|
141
|
+
.update(jobs)
|
|
142
|
+
.set({ status: "succeeded", finishedAt: new Date(), lastError: null })
|
|
143
|
+
.where(eq(jobs.id, job.id));
|
|
144
|
+
result.succeeded += 1;
|
|
145
|
+
} catch (error) {
|
|
146
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
147
|
+
const exhausted = job.attempts >= job.maxAttempts;
|
|
148
|
+
await db
|
|
149
|
+
.update(jobs)
|
|
150
|
+
.set({
|
|
151
|
+
status: exhausted ? "failed" : "pending",
|
|
152
|
+
finishedAt: exhausted ? new Date() : null,
|
|
153
|
+
// Back off linearly; a failing dependency shouldn't be
|
|
154
|
+
// hammered every drain.
|
|
155
|
+
runAt: exhausted
|
|
156
|
+
? job.runAt
|
|
157
|
+
: new Date(Date.now() + job.attempts * 60_000),
|
|
158
|
+
lastError: message.slice(0, 1000),
|
|
159
|
+
})
|
|
160
|
+
.where(eq(jobs.id, job.id));
|
|
161
|
+
result.failed += 1;
|
|
162
|
+
log.error("Job failed", {
|
|
163
|
+
jobId: job.id,
|
|
164
|
+
kind: job.kind,
|
|
165
|
+
attempt: String(job.attempts),
|
|
166
|
+
exhausted: String(exhausted),
|
|
167
|
+
error: message,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return result;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/** Jobs that exhausted their attempts, newest first — surfaced in the admin console. */
|
|
176
|
+
export async function listFailedJobs(limit = 50) {
|
|
177
|
+
return db
|
|
178
|
+
.select()
|
|
179
|
+
.from(jobs)
|
|
180
|
+
.where(eq(jobs.status, "failed"))
|
|
181
|
+
.orderBy(desc(jobs.finishedAt))
|
|
182
|
+
.limit(limit);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** Delete succeeded jobs older than the cutoff. */
|
|
186
|
+
export async function pruneJobs(before: Date): Promise<number> {
|
|
187
|
+
const deleted = await db
|
|
188
|
+
.delete(jobs)
|
|
189
|
+
.where(and(eq(jobs.status, "succeeded"), lte(jobs.finishedAt, before)))
|
|
190
|
+
.returning();
|
|
191
|
+
return deleted.length;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** The default Postgres-backed queue. */
|
|
195
|
+
export const postgresJobQueue: JobQueue = { enqueue, drain };
|
|
196
|
+
|
|
197
|
+
export { jobs } from "./db/schema";
|
|
198
|
+
export type { Job, InsertJob } from "./db/schema";
|