@sigil-dev/plugin-jobs 0.9.12 → 0.9.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -61
- package/index.ts +278 -53
- package/package.json +14 -4
- package/queue.ts +254 -43
- package/store-bun.ts +137 -0
- package/store-drizzle.ts +86 -0
- package/store.ts +117 -0
- package/worker.ts +13 -3
- package/sql.ts +0 -73
package/README.md
CHANGED
|
@@ -1,109 +1,153 @@
|
|
|
1
1
|
# @sigil-dev/plugin-jobs
|
|
2
2
|
|
|
3
|
-
Durable background jobs for [Grimoire](https://github.com/sigil-dev/grimoire)
|
|
3
|
+
Durable background jobs for [Grimoire](https://github.com/sigil-dev/grimoire): enqueue, retry, recurring schedules, and a `sigil jobs` CLI.
|
|
4
4
|
|
|
5
5
|
Opt-in. Grimoire core knows nothing about it: not installed means no queue, no table, no worker.
|
|
6
6
|
|
|
7
|
-
## Install
|
|
8
|
-
|
|
9
|
-
```sh
|
|
10
|
-
bun add @sigil-dev/plugin-jobs
|
|
11
|
-
```
|
|
12
|
-
|
|
13
7
|
## Setup
|
|
14
8
|
|
|
9
|
+
The queue lives in a database `@sigil-dev/plugin-sql` opened — `"db"` unless
|
|
10
|
+
you say otherwise — so there is nothing to connect:
|
|
11
|
+
|
|
15
12
|
```ts
|
|
16
|
-
//
|
|
17
|
-
import { Database } from "bun:sqlite";
|
|
13
|
+
// src/lib/jobs.ts
|
|
18
14
|
import { jobs } from "@sigil-dev/plugin-jobs";
|
|
19
15
|
|
|
20
|
-
const
|
|
16
|
+
export const queue = jobs({
|
|
17
|
+
inlineWorker: true, // run the worker in the server; see "Running it"
|
|
18
|
+
handlers: {
|
|
19
|
+
"send-welcome": async (p: { email: string }) => sendEmail(p.email),
|
|
20
|
+
"rankings.snapshot": async () => snapshotRankings(),
|
|
21
|
+
},
|
|
22
|
+
schedules: {
|
|
23
|
+
"rankings.snapshot": { every: "5m" },
|
|
24
|
+
},
|
|
25
|
+
});
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
```ts
|
|
29
|
+
// sigil.config.ts
|
|
30
|
+
import { adapter } from "@sigil-dev/plugin-sql/bun";
|
|
31
|
+
import { queue } from "./src/lib/jobs.ts";
|
|
21
32
|
|
|
22
33
|
export default {
|
|
23
|
-
plugins: [
|
|
24
|
-
jobs({
|
|
25
|
-
// You pass the client. This plugin does not import a driver, open
|
|
26
|
-
// a connection, or read a connection string — whoever already
|
|
27
|
-
// holds the connection has it.
|
|
28
|
-
sql: {
|
|
29
|
-
query: (sql, params = []) => Promise.resolve(db.query(sql).all(params)),
|
|
30
|
-
},
|
|
31
|
-
handlers: {
|
|
32
|
-
"send-welcome": (payload) => sendEmail(payload.email),
|
|
33
|
-
},
|
|
34
|
-
}),
|
|
35
|
-
],
|
|
34
|
+
plugins: [adapter(process.env.DATABASE_URL, { maxTotal: 12 }), queue],
|
|
36
35
|
};
|
|
37
36
|
```
|
|
38
37
|
|
|
39
|
-
`
|
|
38
|
+
`database: "userDb"` puts the queue in another plugin-sql database. The
|
|
39
|
+
database is looked up when a query runs, so the order of the plugins array
|
|
40
|
+
does not matter.
|
|
40
41
|
|
|
41
42
|
## Enqueue
|
|
42
43
|
|
|
43
|
-
|
|
44
|
-
import it wherever you need it:
|
|
44
|
+
From anywhere on the server — import the queue:
|
|
45
45
|
|
|
46
46
|
```ts
|
|
47
|
-
|
|
48
|
-
import { Database } from "bun:sqlite";
|
|
49
|
-
import { jobs } from "@sigil-dev/plugin-jobs";
|
|
47
|
+
import { queue } from "$lib/jobs";
|
|
50
48
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
handlers: { "send-welcome": (p) => sendWelcome(p.email) },
|
|
56
|
-
});
|
|
49
|
+
await queue.jobs.enqueue("send-welcome", { email: "ada@example.com" });
|
|
50
|
+
await queue.jobs.enqueue("send-welcome", { mail: "x" }); // type error: wrong payload
|
|
51
|
+
await queue.jobs.enqueue("send-welcom", {}); // type error: no such handler
|
|
52
|
+
await queue.jobs.enqueue("rankings.snapshot"); // no payload needed
|
|
57
53
|
```
|
|
58
54
|
|
|
55
|
+
In a route it is also `locals.jobs`. With plugin-sql's `perRequestTx`, a job
|
|
56
|
+
enqueued during a request is part of the request's transaction: an order
|
|
57
|
+
that rolls back takes its confirmation email with it.
|
|
58
|
+
|
|
59
|
+
Options: `{ runAt: Date, maxAttempts: number }` (default 5).
|
|
60
|
+
|
|
61
|
+
## Handlers
|
|
62
|
+
|
|
59
63
|
```ts
|
|
60
|
-
|
|
61
|
-
import { jobQueue } from "./jobqueue";
|
|
62
|
-
export default { plugins: [jobQueue] };
|
|
64
|
+
async (payload, { id, name, attempt }) => { ... }
|
|
63
65
|
```
|
|
64
66
|
|
|
65
|
-
|
|
67
|
+
`id` is stable across retries, so it can key idempotency. `attempt` is 1 on
|
|
68
|
+
the first run. A handler that throws is retried after a backoff (30s by
|
|
69
|
+
default); after `maxAttempts` it stays in the table as failed, for
|
|
70
|
+
`sigil jobs retry`.
|
|
71
|
+
|
|
72
|
+
## Schedules
|
|
66
73
|
|
|
67
74
|
```ts
|
|
68
|
-
|
|
75
|
+
schedules: { "rankings.snapshot": { every: "5m", payload: { region: "jp" } } }
|
|
76
|
+
```
|
|
69
77
|
|
|
70
|
-
|
|
71
|
-
|
|
78
|
+
`every` is milliseconds or `"30s"`, `"5m"`, `"1h"`, `"1d"`. Each schedule is
|
|
79
|
+
one row that is rescheduled when it completes, so:
|
|
72
80
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
81
|
+
- **It runs once however many workers there are.** Every worker makes sure
|
|
82
|
+
the row exists at boot; the primary key admits one, and one worker claims
|
|
83
|
+
it at a time.
|
|
84
|
+
- **A restart keeps its clock**, and a deploy does not fire every schedule
|
|
85
|
+
at once: the first run is one interval after the row is created.
|
|
86
|
+
- **A schedule that keeps failing moves on** to its next run once its
|
|
87
|
+
retries are used up, with the error kept in `last_error`.
|
|
88
|
+
- **A schedule removed from the config is removed from the table.**
|
|
80
89
|
|
|
81
|
-
|
|
90
|
+
## Running it
|
|
82
91
|
|
|
83
|
-
|
|
92
|
+
`inlineWorker: true` runs the worker inside the server. That is right for
|
|
93
|
+
one process; with `sigil start --scale full=3` every worker also works jobs,
|
|
94
|
+
which is safe (claims are race-free) but may be more than you want.
|
|
95
|
+
|
|
96
|
+
The alternative is a separate process:
|
|
84
97
|
|
|
85
98
|
```sh
|
|
86
|
-
sigil jobs work
|
|
87
|
-
sigil jobs list [--limit 20]
|
|
88
|
-
sigil jobs stats
|
|
89
|
-
sigil jobs retry
|
|
90
|
-
sigil jobs
|
|
91
|
-
sigil jobs migrate # create the jobs table
|
|
99
|
+
sigil jobs work # run the worker until stopped (drains on SIGTERM)
|
|
100
|
+
sigil jobs list # most recent jobs [--limit 20]
|
|
101
|
+
sigil jobs stats # counts by state
|
|
102
|
+
sigil jobs retry # requeue every exhausted job, or one: retry <id>
|
|
103
|
+
sigil jobs migrate # create the table (also done at boot)
|
|
92
104
|
```
|
|
93
105
|
|
|
94
|
-
The
|
|
106
|
+
The CLI opens the database from `sigil.config.ts` itself, so it works
|
|
107
|
+
without a running server.
|
|
108
|
+
|
|
109
|
+
## Other stores
|
|
110
|
+
|
|
111
|
+
Instead of `database`, pass `sql` — any object with `query(sql, params)`
|
|
112
|
+
filling `?` placeholders — for a queue that owns its connection:
|
|
113
|
+
|
|
114
|
+
```ts
|
|
115
|
+
import { bunSqlStore } from "@sigil-dev/plugin-jobs";
|
|
116
|
+
jobs({ sql: bunSqlStore(process.env.JOBS_URL!), handlers });
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
A store can declare what its dialect can do; everything defaults to false,
|
|
120
|
+
so a store that declares nothing takes the path that is correct everywhere.
|
|
121
|
+
|
|
122
|
+
| capability | true on | what it changes |
|
|
123
|
+
| --- | --- | --- |
|
|
124
|
+
| `skipLocked` | postgres, MySQL | appends `FOR UPDATE SKIP LOCKED`, so a worker does not block behind rows another holds. Throughput only. |
|
|
125
|
+
| `spaceSeparatedTimestamp` | MySQL | writes `2026-01-01 00:00:00` rather than ISO 8601, which MySQL rejects on a `TIMESTAMP` column. |
|
|
126
|
+
|
|
127
|
+
### Drizzle
|
|
128
|
+
|
|
129
|
+
```ts
|
|
130
|
+
import { drizzleStore, bunSqlRunner } from "@sigil-dev/plugin-jobs";
|
|
131
|
+
sql: drizzleStore(bunSqlRunner(client), "mysql"),
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
This takes the `Bun.SQL` connection, not your Drizzle `db`: Drizzle's
|
|
135
|
+
`execute` ignores a parameter array, and its bun-sql driver emits `$1` on
|
|
136
|
+
every dialect.
|
|
95
137
|
|
|
96
138
|
## Guarantees, precisely
|
|
97
139
|
|
|
98
140
|
**Delivery is at-least-once.** A worker can die between your side effect and the row delete, so a job may run twice. **Handlers must be idempotent.** The alternative — holding a transaction open across every handler, including every HTTP call inside one — is a worse trade than a duplicate email.
|
|
99
141
|
|
|
100
|
-
**
|
|
142
|
+
**Order is not guaranteed.** A job that fails and retries runs after jobs enqueued later. Two jobs that write the same thing should carry what they need to be applied in any order.
|
|
143
|
+
|
|
144
|
+
**Claiming is race-free without `SKIP LOCKED`.** The claim is an `UPDATE` guarded on the row being unclaimed or stale, tagged with a random token, followed by a `SELECT` of that token — so it works on MariaDB, which has no `UPDATE ... RETURNING`. Two workers racing means the loser gets an empty set, not a duplicate job.
|
|
101
145
|
|
|
102
146
|
**A stale claim is reclaimed** after 60s by default, so a hard-killed worker does not strand its jobs.
|
|
103
147
|
|
|
104
|
-
##
|
|
148
|
+
## Dialects
|
|
105
149
|
|
|
106
|
-
|
|
150
|
+
Tested against real SQLite, postgres and MariaDB.
|
|
107
151
|
|
|
108
152
|
## License
|
|
109
153
|
|
package/index.ts
CHANGED
|
@@ -1,26 +1,110 @@
|
|
|
1
1
|
import type { CliCommand, GrimoirePlugin } from "@sigil-dev/grimoire";
|
|
2
|
+
import { database } from "@sigil-dev/plugin-sql";
|
|
2
3
|
import { JobQueue } from "./queue.ts";
|
|
3
|
-
import
|
|
4
|
+
import { databaseStore } from "./store-bun.ts";
|
|
5
|
+
import {
|
|
6
|
+
everyMs,
|
|
7
|
+
type EnqueueOptions,
|
|
8
|
+
type Every,
|
|
9
|
+
type JobHandler,
|
|
10
|
+
type JobStore,
|
|
11
|
+
type PayloadOf,
|
|
12
|
+
} from "./store.ts";
|
|
4
13
|
import { JobWorker } from "./worker.ts";
|
|
5
14
|
|
|
6
15
|
export { JobQueue, SCHEMA } from "./queue.ts";
|
|
7
|
-
export type {
|
|
16
|
+
export type {
|
|
17
|
+
Capabilities,
|
|
18
|
+
EnqueueOptions,
|
|
19
|
+
Every,
|
|
20
|
+
JobContext,
|
|
21
|
+
JobHandler,
|
|
22
|
+
PayloadOf,
|
|
23
|
+
JobRow,
|
|
24
|
+
JobStats,
|
|
25
|
+
JobStore,
|
|
26
|
+
} from "./store.ts";
|
|
27
|
+
export { bunSqlStore, bunStore, databaseStore } from "./store-bun.ts";
|
|
28
|
+
export { drizzleStore, bunSqlRunner } from "./store-drizzle.ts";
|
|
29
|
+
export type { SqlRunner } from "./store-drizzle.ts";
|
|
8
30
|
export { JobWorker } from "./worker.ts";
|
|
9
31
|
export type { WorkerOptions } from "./worker.ts";
|
|
10
32
|
|
|
11
|
-
|
|
33
|
+
/** Handlers by name. */
|
|
34
|
+
export type Handlers = Record<string, JobHandler>;
|
|
35
|
+
|
|
36
|
+
export interface JobsOptions<H extends Handlers = Handlers> {
|
|
37
|
+
/**
|
|
38
|
+
* The `@sigil-dev/plugin-sql` database the queue lives in, by name — its
|
|
39
|
+
* `locals` key for `adapter()`. This is the default: with neither `sql`
|
|
40
|
+
* nor `store`, the queue uses the database named "db".
|
|
41
|
+
*
|
|
42
|
+
* Looked up when a query runs, so the order of the plugins array does
|
|
43
|
+
* not matter, and `sigil jobs work` opens it from the config.
|
|
44
|
+
* @default "db"
|
|
45
|
+
*/
|
|
46
|
+
database?: string;
|
|
47
|
+
|
|
12
48
|
/**
|
|
13
49
|
* The database client. Required, and not optional-with-a-default,
|
|
14
50
|
* because there is no sensible fallback: a queue that silently kept jobs
|
|
15
51
|
* in memory would look like it worked right up until a restart.
|
|
16
52
|
*/
|
|
17
|
-
|
|
53
|
+
/**
|
|
54
|
+
* A `JobStore`: an object with `query(sql, params)`, filling `?`
|
|
55
|
+
* placeholders, and optionally a `capabilities` block declaring what the
|
|
56
|
+
* dialect can do.
|
|
57
|
+
*
|
|
58
|
+
* The queue never opens a connection, so it cannot guess the wrong
|
|
59
|
+
* dialect — a `Bun.SQL` client, a `pg` pool, or Drizzle all work
|
|
60
|
+
* unchanged. A string used to be accepted here and opened for you; it is
|
|
61
|
+
* not any more, because working out which driver a string wanted meant
|
|
62
|
+
* guessing a dialect, and the guessing was wrong more often than not.
|
|
63
|
+
*
|
|
64
|
+
* Either this or `store` is required.
|
|
65
|
+
*/
|
|
66
|
+
sql?: JobStore;
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* A store built from the services other plugins published.
|
|
70
|
+
*
|
|
71
|
+
* This is the option to reach for when the app already registered
|
|
72
|
+
* `@sigil-dev/plugin-sql`, because the queue then runs on the connection
|
|
73
|
+
* that plugin opened instead of opening a second pool and dividing the
|
|
74
|
+
* worker budget again:
|
|
75
|
+
*
|
|
76
|
+
* const db = adapter(url, { maxTotal: 12 });
|
|
77
|
+
* const queue = jobs({
|
|
78
|
+
* store: (services) => bunStore(services.db as Bun.SQL),
|
|
79
|
+
* });
|
|
80
|
+
*
|
|
81
|
+
* Register the SQL plugin first — `services` is a snapshot of what ran
|
|
82
|
+
* before this plugin, so a `store` that does not find what it needs
|
|
83
|
+
* should throw rather than fall back to a fresh connection. The function
|
|
84
|
+
* runs inside `services()`, which is the one hook that receives them.
|
|
85
|
+
*
|
|
86
|
+
* Prefer `database`, which needs none of that. At most one of `sql`,
|
|
87
|
+
* `store` and `database`.
|
|
88
|
+
*/
|
|
89
|
+
store?: (services: Record<string, unknown>) => JobStore;
|
|
18
90
|
|
|
19
91
|
/**
|
|
20
92
|
* Named handlers the worker can run. Enqueue a name that is not here and
|
|
21
93
|
* the job fails loudly rather than being dropped.
|
|
94
|
+
*
|
|
95
|
+
* Annotate each handler's payload and `enqueue` checks it.
|
|
96
|
+
*/
|
|
97
|
+
handlers?: H;
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Recurring jobs, by handler name. One row per schedule, so several
|
|
101
|
+
* workers run it once, not once each; a restart keeps its clock.
|
|
102
|
+
*
|
|
103
|
+
* ```ts
|
|
104
|
+
* schedules: { "rankings.snapshot": { every: "5m" } }
|
|
105
|
+
* ```
|
|
22
106
|
*/
|
|
23
|
-
|
|
107
|
+
schedules?: { [K in keyof H]?: { every: Every; payload?: PayloadOf<H[K]> } };
|
|
24
108
|
|
|
25
109
|
/** Table name. @default "sigil_jobs" */
|
|
26
110
|
table?: string;
|
|
@@ -39,63 +123,161 @@ export interface JobsOptions {
|
|
|
39
123
|
inlineWorker?: boolean;
|
|
40
124
|
}
|
|
41
125
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
126
|
+
/** `enqueue`'s arguments after the name: the payload is optional when the handler takes none. */
|
|
127
|
+
type EnqueueArgs<P> = undefined extends P
|
|
128
|
+
? [payload?: P, options?: EnqueueOptions]
|
|
129
|
+
: [payload: P, options?: EnqueueOptions];
|
|
130
|
+
|
|
131
|
+
export interface JobsApi<H extends Handlers = Handlers> {
|
|
132
|
+
/** Add a job. Returns its id. */
|
|
133
|
+
enqueue<K extends keyof H & string>(name: K, ...args: EnqueueArgs<PayloadOf<H[K]>>): Promise<string>;
|
|
48
134
|
queue: JobQueue;
|
|
49
|
-
handlers:
|
|
135
|
+
handlers: H;
|
|
50
136
|
/** Register a handler after the plugin was created. */
|
|
51
137
|
register: (name: string, handler: JobHandler) => void;
|
|
52
138
|
}
|
|
53
139
|
|
|
54
140
|
/**
|
|
55
|
-
* The plugin.
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
141
|
+
* The plugin. With no `sql` or `store`, the queue lives in the plugin-sql
|
|
142
|
+
* database named by `database` ("db" by default), looked up when it is
|
|
143
|
+
* used — so a config error is the database's own "no sql database named"
|
|
144
|
+
* at boot, not a failed enqueue at 2am.
|
|
59
145
|
*/
|
|
60
|
-
export function jobs
|
|
61
|
-
|
|
146
|
+
export function jobs<H extends Handlers = Handlers>(
|
|
147
|
+
options: JobsOptions<H> = {},
|
|
148
|
+
): GrimoirePlugin & { jobs: JobsApi<H> } {
|
|
149
|
+
// The plugin does not open a connection: `sql` is an executor the caller
|
|
150
|
+
// already built, so the plugin can never guess the wrong dialect. That
|
|
151
|
+
// decision is made once, where the user made it.
|
|
152
|
+
//
|
|
153
|
+
// The queue speaks positional `?` placeholders, which is a different
|
|
154
|
+
// interface from the SQL plugin's tagged template — not a worse one. The
|
|
155
|
+
// plugin therefore does not import @sigil-dev/plugin-sql and does not
|
|
156
|
+
// have to. What it does accept is the *client* that plugin opened, via
|
|
157
|
+
// the `store` option, and a tiny adapter in this package translates
|
|
158
|
+
// between the two. Neither side has to know about the other.
|
|
159
|
+
//
|
|
160
|
+
// `store` is resolved at `services()` time, because until then there are
|
|
161
|
+
// no services to read. Everything that needs the queue — migrate, the
|
|
162
|
+
// inline worker, the api object — goes through `getQueue()`, so a
|
|
163
|
+
// config using `store` behaves identically to one using `sql`.
|
|
164
|
+
const factory = options.store;
|
|
165
|
+
const given = [options.sql, factory, options.database].filter((x) => x !== undefined);
|
|
166
|
+
if (given.length > 1) {
|
|
62
167
|
throw new Error(
|
|
63
|
-
"plugin-jobs
|
|
64
|
-
"
|
|
65
|
-
"This error is at boot rather than on the first enqueue, so a " +
|
|
66
|
-
"missing queue is a failed deploy rather than a 3am page.",
|
|
168
|
+
"plugin-jobs: pass at most one of `sql`, `store` and `database` — " +
|
|
169
|
+
"they are three ways to say where the queue lives.",
|
|
67
170
|
);
|
|
68
171
|
}
|
|
172
|
+
// The default: the plugin-sql database, looked up when a query runs.
|
|
173
|
+
const dbName = factory || options.sql ? undefined : (options.database ?? "db");
|
|
174
|
+
const provided = options.sql ?? (dbName !== undefined ? databaseStore(dbName) : undefined);
|
|
175
|
+
/** Open the database the queue lives in, when it is a plugin-sql one. */
|
|
176
|
+
const openDatabase = async () => {
|
|
177
|
+
if (dbName !== undefined) await database(dbName).open();
|
|
178
|
+
};
|
|
179
|
+
let resolved: JobStore | undefined = provided;
|
|
180
|
+
// Assigned by services() before anything reads it. Held in a mutable
|
|
181
|
+
// slot because the api object is built before the services exist.
|
|
182
|
+
let queue: JobQueue | undefined = provided ? new JobQueue(provided, options.table) : undefined;
|
|
183
|
+
const requireQueue = (): JobQueue => {
|
|
184
|
+
if (!queue) {
|
|
185
|
+
throw new Error(
|
|
186
|
+
"plugin-jobs: the queue is not available yet. With `store`, it is " +
|
|
187
|
+
"created during services() — use locals.jobs from a request, or " +
|
|
188
|
+
"pass `sql` if you need it earlier.",
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
return queue;
|
|
192
|
+
};
|
|
69
193
|
|
|
70
|
-
const
|
|
71
|
-
const
|
|
194
|
+
const handlers = { ...(options.handlers ?? {}) } as H;
|
|
195
|
+
const schedules = Object.fromEntries(
|
|
196
|
+
Object.entries(options.schedules ?? {}).flatMap(([name, spec]) => {
|
|
197
|
+
if (!spec) return [];
|
|
198
|
+
const { every, payload } = spec as { every: Every; payload?: unknown };
|
|
199
|
+
if (!(name in handlers)) {
|
|
200
|
+
throw new Error(`plugin-jobs: schedule "${name}" has no handler of that name.`);
|
|
201
|
+
}
|
|
202
|
+
return [[name, { everyMs: everyMs(every), payload }]];
|
|
203
|
+
}),
|
|
204
|
+
);
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Create the table, and start the inline worker if one was asked for.
|
|
208
|
+
*
|
|
209
|
+
* Shared by config() and services() because with `store` the queue only
|
|
210
|
+
* exists once the SQL plugin has published its client, which is strictly
|
|
211
|
+
* after config(). Doing it in one function means the `sql` path and the
|
|
212
|
+
* `store` path cannot disagree about when the table appears.
|
|
213
|
+
*
|
|
214
|
+
* The migrate is awaited, deliberately. A fire-and-forget CREATE TABLE
|
|
215
|
+
* means the first enqueue can race the table and fail with "relation
|
|
216
|
+
* does not exist" on a fresh deploy, which looks like a queue bug and is
|
|
217
|
+
* not one. Only the worker's run loop is left unawaited — it is meant to
|
|
218
|
+
* outlive this call.
|
|
219
|
+
*/
|
|
220
|
+
const migrateAndStart = async (): Promise<Record<string, unknown>> => {
|
|
221
|
+
await openDatabase();
|
|
222
|
+
if (options.autoMigrate !== false) {
|
|
223
|
+
await requireQueue().migrate();
|
|
224
|
+
}
|
|
225
|
+
await requireQueue().ensureSchedules(schedules);
|
|
226
|
+
if (options.inlineWorker) {
|
|
227
|
+
void new JobWorker(requireQueue(), handlers).run();
|
|
228
|
+
}
|
|
229
|
+
return { jobs: api };
|
|
230
|
+
};
|
|
72
231
|
|
|
73
|
-
const api: JobsApi = {
|
|
74
|
-
queue
|
|
232
|
+
const api: JobsApi<H> = {
|
|
233
|
+
get queue() {
|
|
234
|
+
return requireQueue();
|
|
235
|
+
},
|
|
75
236
|
handlers,
|
|
76
|
-
async enqueue(name, payload, opts) {
|
|
77
|
-
return
|
|
237
|
+
async enqueue(name, ...[payload, opts]) {
|
|
238
|
+
return requireQueue().enqueue(name, payload, opts);
|
|
78
239
|
},
|
|
79
240
|
register(name, handler) {
|
|
80
|
-
handlers[name] = handler;
|
|
241
|
+
(handlers as Handlers)[name] = handler;
|
|
81
242
|
},
|
|
82
243
|
};
|
|
83
244
|
|
|
84
|
-
const instance: GrimoirePlugin & { jobs: JobsApi } = {
|
|
245
|
+
const instance: GrimoirePlugin & { jobs: JobsApi<H> } = {
|
|
85
246
|
name: "plugin-jobs",
|
|
86
247
|
jobs: api,
|
|
87
248
|
|
|
88
249
|
async config(config) {
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
250
|
+
// With `store` the queue does not exist yet — it is built from
|
|
251
|
+
// the services the SQL plugin publishes, and those are only
|
|
252
|
+
// available in services(), which runs after this. So the same
|
|
253
|
+
// work happens there instead. A config using `sql` keeps
|
|
254
|
+
// migrating here, unchanged.
|
|
255
|
+
// With `database`, it runs in services() too: the database opens
|
|
256
|
+
// there, not in config(), which also runs in the coordinator.
|
|
257
|
+
if (options.sql) await migrateAndStart();
|
|
258
|
+
return config;
|
|
259
|
+
},
|
|
92
260
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
261
|
+
/**
|
|
262
|
+
* Publishes the queue as `locals.jobs`.
|
|
263
|
+
*
|
|
264
|
+
* This is what lets a route call `locals.jobs.enqueue(...)` with
|
|
265
|
+
* nothing imported and nothing exported from sigil.config.ts. The
|
|
266
|
+
* object is built once at startup, so the same queue instance is
|
|
267
|
+
* shared by every request.
|
|
268
|
+
*/
|
|
269
|
+
services(ctx) {
|
|
270
|
+
// The only hook that can see what other plugins published, and
|
|
271
|
+
// so the only place a `store` can be resolved. This runs after
|
|
272
|
+
// the SQL plugin's own services(), provided it is registered
|
|
273
|
+
// earlier in the plugins array.
|
|
274
|
+
if (factory) {
|
|
275
|
+
resolved = factory(ctx.services);
|
|
276
|
+
queue = new JobQueue(resolved, options.table);
|
|
277
|
+
return migrateAndStart();
|
|
96
278
|
}
|
|
97
|
-
|
|
98
|
-
return
|
|
279
|
+
if (dbName !== undefined) return migrateAndStart();
|
|
280
|
+
return { jobs: api };
|
|
99
281
|
},
|
|
100
282
|
|
|
101
283
|
/**
|
|
@@ -116,16 +298,44 @@ export function jobs(options: JobsOptions): GrimoirePlugin & { jobs: JobsApi } {
|
|
|
116
298
|
// before calling this, so by the time an action runs the queue
|
|
117
299
|
// exists — and if `config` threw, the CLI already reported that
|
|
118
300
|
// and never got here.
|
|
119
|
-
const connect = (): JobsApi => {
|
|
301
|
+
const connect = (): JobsApi<H> => {
|
|
120
302
|
if (!instance.jobs) {
|
|
121
303
|
throw new Error(
|
|
122
304
|
"plugin-jobs is configured but no queue is available. " +
|
|
123
305
|
"Check the `jobs` option in sigil.config.ts.",
|
|
124
306
|
);
|
|
125
307
|
}
|
|
308
|
+
// A `store` config resolves the queue from the services the
|
|
309
|
+
// SQL plugin published, which only exist inside the server's
|
|
310
|
+
// plugin lifecycle. The CLI runs config() and never
|
|
311
|
+
// services(), so say that plainly instead of reporting the
|
|
312
|
+
// queue as missing.
|
|
313
|
+
if (factory && !queue) {
|
|
314
|
+
throw new Error(
|
|
315
|
+
"plugin-jobs is configured with `store`, which reads the client " +
|
|
316
|
+
"another plugin opened. The `sigil jobs` CLI has no request " +
|
|
317
|
+
"lifecycle to publish that from — use `sql: bunSqlStore(url)` in " +
|
|
318
|
+
"a CLI-capable config, or run the queue commands against a " +
|
|
319
|
+
"server that is up.",
|
|
320
|
+
);
|
|
321
|
+
}
|
|
126
322
|
return instance.jobs;
|
|
127
323
|
};
|
|
128
324
|
|
|
325
|
+
/**
|
|
326
|
+
* Run one command against the queue: open its database, and close
|
|
327
|
+
* it after, or the open pool keeps the CLI from exiting.
|
|
328
|
+
*/
|
|
329
|
+
const oneShot = async <R>(fn: (q: JobQueue) => Promise<R>): Promise<R> => {
|
|
330
|
+
const q = connect().queue;
|
|
331
|
+
await openDatabase();
|
|
332
|
+
try {
|
|
333
|
+
return await fn(q);
|
|
334
|
+
} finally {
|
|
335
|
+
if (dbName !== undefined) await database(dbName).close();
|
|
336
|
+
}
|
|
337
|
+
};
|
|
338
|
+
|
|
129
339
|
const group = program
|
|
130
340
|
.command("jobs")
|
|
131
341
|
.description("manage the durable job queue");
|
|
@@ -137,6 +347,10 @@ export function jobs(options: JobsOptions): GrimoirePlugin & { jobs: JobsApi } {
|
|
|
137
347
|
.option("--batch <n>", "jobs per pass", (v: string) => Number(v))
|
|
138
348
|
.action(async (opts: { interval?: number; batch?: number }) => {
|
|
139
349
|
const { queue: q, handlers: h } = connect();
|
|
350
|
+
await openDatabase();
|
|
351
|
+
// The worker completes recurring jobs, so it has to know
|
|
352
|
+
// them: without the schedule a completed one is deleted.
|
|
353
|
+
await q.ensureSchedules(schedules);
|
|
140
354
|
const worker = new JobWorker(q, h, {
|
|
141
355
|
...(opts.interval ? { intervalMs: opts.interval } : {}),
|
|
142
356
|
...(opts.batch ? { batchSize: opts.batch } : {}),
|
|
@@ -163,7 +377,7 @@ export function jobs(options: JobsOptions): GrimoirePlugin & { jobs: JobsApi } {
|
|
|
163
377
|
.description("list the most recent jobs")
|
|
164
378
|
.option("--limit <n>", "how many rows", (v: string) => Number(v))
|
|
165
379
|
.action(async (opts: { limit?: number }) => {
|
|
166
|
-
const rows = await
|
|
380
|
+
const rows = await oneShot((q) => q.list(opts.limit ?? 20));
|
|
167
381
|
if (rows.length === 0) {
|
|
168
382
|
console.log("no jobs");
|
|
169
383
|
return;
|
|
@@ -186,7 +400,7 @@ export function jobs(options: JobsOptions): GrimoirePlugin & { jobs: JobsApi } {
|
|
|
186
400
|
.command("stats")
|
|
187
401
|
.description("count jobs by state")
|
|
188
402
|
.action(async () => {
|
|
189
|
-
const s = await
|
|
403
|
+
const s = await oneShot((q) => q.stats());
|
|
190
404
|
console.log(
|
|
191
405
|
`pending ${s.pending} claimed ${s.claimed} failed ${s.failed}`,
|
|
192
406
|
);
|
|
@@ -197,24 +411,25 @@ export function jobs(options: JobsOptions): GrimoirePlugin & { jobs: JobsApi } {
|
|
|
197
411
|
.argument("[id]", "job id, or all failed jobs when omitted")
|
|
198
412
|
.description("requeue a job, clearing its attempt count")
|
|
199
413
|
.action(async (id?: string) => {
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
414
|
+
await oneShot(async (q) => {
|
|
415
|
+
if (id) {
|
|
416
|
+
const ok = await q.retry(id);
|
|
417
|
+
console.log(ok ? `requeued ${id}` : `no such job: ${id}`);
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
const failed = (await q.list(1000)).filter(
|
|
421
|
+
(r) => r.attempts >= r.max_attempts,
|
|
422
|
+
);
|
|
423
|
+
for (const job of failed) await q.retry(job.id);
|
|
424
|
+
console.log(`requeued ${failed.length} failed job(s)`);
|
|
425
|
+
});
|
|
211
426
|
});
|
|
212
427
|
|
|
213
428
|
group
|
|
214
429
|
.command("migrate")
|
|
215
430
|
.description("create the jobs table if it does not exist")
|
|
216
431
|
.action(async () => {
|
|
217
|
-
await
|
|
432
|
+
await oneShot((q) => q.migrate());
|
|
218
433
|
console.log("jobs table ready");
|
|
219
434
|
});
|
|
220
435
|
},
|
|
@@ -225,3 +440,13 @@ export function jobs(options: JobsOptions): GrimoirePlugin & { jobs: JobsApi } {
|
|
|
225
440
|
|
|
226
441
|
export const plugin = jobs;
|
|
227
442
|
export default jobs;
|
|
443
|
+
|
|
444
|
+
/**
|
|
445
|
+
* Accept an executor, or fail naming what was expected.
|
|
446
|
+
*
|
|
447
|
+
* A bare string used to be accepted here. It no longer is, and a message
|
|
448
|
+
* that says only "sql is required" would send someone to the docs to
|
|
449
|
+
* discover it. A queue pointed at the wrong database is worse than a boot
|
|
450
|
+
* that fails, so this throws at construction — which is what `sigil start`
|
|
451
|
+
* runs — rather than at the first enqueue.
|
|
452
|
+
*/
|