@sigil-dev/plugin-jobs 0.9.32 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.ts +427 -414
- package/package.json +4 -4
- package/queue.ts +305 -293
- package/worker.ts +163 -134
package/index.ts
CHANGED
|
@@ -1,414 +1,427 @@
|
|
|
1
|
-
import { isAbsolute, join, relative } from "node:path";
|
|
2
|
-
import type { CliCommand, GrimoirePlugin } from "@sigil-dev/grimoire";
|
|
3
|
-
import { hasDatabase } from "@sigil-dev/plugin-sql";
|
|
4
|
-
import { adoptTables, type Tome, tomeFor } from "@sigil-dev/tome";
|
|
5
|
-
import { checkPayload, type Job, nameJob, setJobSink } from "./define.ts";
|
|
6
|
-
import { ms } from "./duration.ts";
|
|
7
|
-
import { loadManifest } from "./manifest.ts";
|
|
8
|
-
import { JobQueue } from "./queue.ts";
|
|
9
|
-
import { type JobsTable, jobsTable } from "./schema.ts";
|
|
10
|
-
import type { EnqueueOptions } from "./store.ts";
|
|
11
|
-
import { JobWorker } from "./worker.ts";
|
|
12
|
-
|
|
13
|
-
export { defineJob, type Job, type StandardSchema } from "./define.ts";
|
|
14
|
-
export { type Duration, ms } from "./duration.ts";
|
|
15
|
-
export { JobQueue } from "./queue.ts";
|
|
16
|
-
export { type JobsTable, jobsTable } from "./schema.ts";
|
|
17
|
-
export type {
|
|
18
|
-
EnqueueOptions,
|
|
19
|
-
JobContext,
|
|
20
|
-
JobRow,
|
|
21
|
-
JobStats,
|
|
22
|
-
} from "./store.ts";
|
|
23
|
-
export type { WorkerOptions } from "./worker.ts";
|
|
24
|
-
export { JobWorker } from "./worker.ts";
|
|
25
|
-
|
|
26
|
-
export interface JobsOptions<D extends string = "db"> {
|
|
27
|
-
/**
|
|
28
|
-
* The plugin-sql database whose tome holds the queue's table: its key,
|
|
29
|
-
* "db" unless you named it.
|
|
30
|
-
* @default "db"
|
|
31
|
-
*/
|
|
32
|
-
database?: D;
|
|
33
|
-
|
|
34
|
-
/**
|
|
35
|
-
* Where jobs live, one per file: `src/jobs/send-welcome.ts` is the job
|
|
36
|
-
* "send-welcome". Imported as `$jobs/send-welcome`, and server-only.
|
|
37
|
-
* @default "src/jobs"
|
|
38
|
-
*/
|
|
39
|
-
dir?: string;
|
|
40
|
-
|
|
41
|
-
/** Jobs that are not files in `dir`, by name: a package's own, or a test's. */
|
|
42
|
-
jobs?: Record<string, Job<any>>;
|
|
43
|
-
|
|
44
|
-
/** The table's name in the database. @default "sigil_jobs" */
|
|
45
|
-
table?: string;
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* Run a worker in the server process.
|
|
49
|
-
*
|
|
50
|
-
* On in development, so a queued job (a mail) runs without a second
|
|
51
|
-
* terminal. Off in production: a worker in the app means every worker
|
|
52
|
-
* scaled up is also a worker — fine for one process, surprising at
|
|
53
|
-
* three, and the reason `sigil jobs work` exists to run instead.
|
|
54
|
-
* @default true in development, false in production
|
|
55
|
-
*/
|
|
56
|
-
inlineWorker?: boolean;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
export interface JobsApi {
|
|
60
|
-
/**
|
|
61
|
-
* Queue a job by name, for when the name is data (an admin screen, a
|
|
62
|
-
* webhook). In code, import the job and call its `enqueue`: that one is
|
|
63
|
-
* typed.
|
|
64
|
-
*/
|
|
65
|
-
enqueue(
|
|
66
|
-
name: string,
|
|
67
|
-
payload?: unknown,
|
|
68
|
-
options?: EnqueueOptions,
|
|
69
|
-
): Promise<string>;
|
|
70
|
-
queue: JobQueue;
|
|
71
|
-
/** Every job, by name: the folder's and `jobs({ jobs })`'s. Filled at startup. */
|
|
72
|
-
readonly jobs: Readonly<Record<string, Job<any>>>;
|
|
73
|
-
/**
|
|
74
|
-
* False when the database is an optional adapter that was not
|
|
75
|
-
* configured: nothing runs, and `enqueue` throws rather than drop work
|
|
76
|
-
* silently.
|
|
77
|
-
*/
|
|
78
|
-
readonly enabled: boolean;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/** What `jobs()` returns: a plugin, the queue's table, and the queue. */
|
|
82
|
-
export type Jobs<D extends string = "db"> = GrimoirePlugin<{
|
|
83
|
-
jobs: JobsApi;
|
|
84
|
-
}> & {
|
|
85
|
-
jobs: JobsApi;
|
|
86
|
-
/** The queue's table. tome adds it to the schema; `sigil db generate` creates it. */
|
|
87
|
-
tables: { sigilJobs: JobsTable };
|
|
88
|
-
/** The database it is in. */
|
|
89
|
-
database: D;
|
|
90
|
-
/**
|
|
91
|
-
* Use this tome. Done for you on first use, from the tome over
|
|
92
|
-
* `database`; call it yourself to use another one, as a test does.
|
|
93
|
-
*/
|
|
94
|
-
connect(db: Tome<any, any>): void;
|
|
95
|
-
/** Find the jobs: the folder's, compiled, and the ones passed in. Done at startup. */
|
|
96
|
-
discover(): Promise<Readonly<Record<string, Job<any>>>>;
|
|
97
|
-
};
|
|
98
|
-
|
|
99
|
-
/**
|
|
100
|
-
* Durable background jobs, one per file, in a table of the app's own tome.
|
|
101
|
-
*
|
|
102
|
-
* ```ts
|
|
103
|
-
* // src/jobs/send-welcome.ts
|
|
104
|
-
* export default defineJob({
|
|
105
|
-
* payload: schema({ email: t.text().email() }),
|
|
106
|
-
* async handler({ email }) { ... },
|
|
107
|
-
* });
|
|
108
|
-
* // sigil.config.ts: tome adds the queue's table to the schema
|
|
109
|
-
* plugins: [db.plugin, jobs()]
|
|
110
|
-
* // anywhere on the server
|
|
111
|
-
* await sendWelcome.enqueue({ email });
|
|
112
|
-
* ```
|
|
113
|
-
*/
|
|
114
|
-
export function jobs<const D extends string = "db">(
|
|
115
|
-
options: JobsOptions<D> = {},
|
|
116
|
-
): Jobs<D> {
|
|
117
|
-
const key = (options.database ?? "db") as D;
|
|
118
|
-
const dirOption = options.dir ?? "src/jobs";
|
|
119
|
-
const dirOf = (root: string) =>
|
|
120
|
-
isAbsolute(dirOption) ? dirOption : join(root, dirOption);
|
|
121
|
-
const table = jobsTable(options.table ?? "sigil_jobs");
|
|
122
|
-
|
|
123
|
-
let db: Tome<any, any> | undefined;
|
|
124
|
-
function connect(found: Tome<any, any>) {
|
|
125
|
-
// tome takes it from the config; a script or test without one gets it here.
|
|
126
|
-
adoptTables(found, "plugin-jobs", { sigilJobs: table });
|
|
127
|
-
db = found;
|
|
128
|
-
}
|
|
129
|
-
// Found on first use, like auth: the schema is built from this plugin's
|
|
130
|
-
// own table, so the tome cannot exist yet when the plugin is made.
|
|
131
|
-
const conn = (): Tome<any, any> => {
|
|
132
|
-
if (!db) {
|
|
133
|
-
const found = tomeFor(key);
|
|
134
|
-
if (!found) {
|
|
135
|
-
throw new Error(
|
|
136
|
-
`plugin-jobs: no tome over the "${key}" database. Make one with ` +
|
|
137
|
-
"tome(schema) and register it: plugins: [db.plugin, jobs()].",
|
|
138
|
-
);
|
|
139
|
-
}
|
|
140
|
-
connect(found);
|
|
141
|
-
}
|
|
142
|
-
return db!;
|
|
143
|
-
};
|
|
144
|
-
const queue = new JobQueue(conn, table);
|
|
145
|
-
let dev = process.env.NODE_ENV !== "production";
|
|
146
|
-
let inline: JobWorker | undefined;
|
|
147
|
-
// An optional adapter that was not configured registers no database.
|
|
148
|
-
const enabled = () => hasDatabase(key);
|
|
149
|
-
const off = (): never => {
|
|
150
|
-
throw new Error(
|
|
151
|
-
`plugin-jobs: the queue is off — database "${key}" is not configured. ` +
|
|
152
|
-
"Check the connection URL, or guard the call with `queue.jobs.enabled`.",
|
|
153
|
-
);
|
|
154
|
-
};
|
|
155
|
-
|
|
156
|
-
let known: Record<string, Job<any>> = { ...(options.jobs ?? {}) };
|
|
157
|
-
for (const [name, job] of Object.entries(known)) nameJob(job, name);
|
|
158
|
-
let discovering: Promise<Record<string, Job<any>>> | undefined;
|
|
159
|
-
const discover = () => {
|
|
160
|
-
discovering ??= (async () => {
|
|
161
|
-
const root = process.cwd();
|
|
162
|
-
const found = await loadManifest(root, dirOf(root));
|
|
163
|
-
for (const [name, job] of Object.entries(found)) {
|
|
164
|
-
if (known[name] && known[name] !== job) {
|
|
165
|
-
throw new Error(
|
|
166
|
-
`plugin-jobs: "${name}" is both a file in ${dirOption} and in jobs({ jobs }). Keep one.`,
|
|
167
|
-
);
|
|
168
|
-
}
|
|
169
|
-
nameJob(job, name);
|
|
170
|
-
}
|
|
171
|
-
known = { ...known, ...found };
|
|
172
|
-
return known;
|
|
173
|
-
})();
|
|
174
|
-
return discovering;
|
|
175
|
-
};
|
|
176
|
-
|
|
177
|
-
const enqueueJob = async (
|
|
178
|
-
job: Job<any>,
|
|
179
|
-
payload: unknown,
|
|
180
|
-
opts: EnqueueOptions,
|
|
181
|
-
) => {
|
|
182
|
-
if (!enabled()) off();
|
|
183
|
-
const checked = await checkPayload(job, payload);
|
|
184
|
-
if (!checked.ok) {
|
|
185
|
-
throw new Error(`plugin-jobs: ${job.name}: ${checked.message}`);
|
|
186
|
-
}
|
|
187
|
-
const runAt =
|
|
188
|
-
opts.at ??
|
|
189
|
-
(opts.in !== undefined ? new Date(Date.now() + ms(opts.in)) : undefined);
|
|
190
|
-
return queue.enqueue(job.name, checked.value ?? {}, {
|
|
191
|
-
runAt,
|
|
192
|
-
maxAttempts: job.attempts,
|
|
193
|
-
});
|
|
194
|
-
};
|
|
195
|
-
setJobSink({
|
|
196
|
-
enqueue: enqueueJob,
|
|
197
|
-
discover: async () => void (await discover()),
|
|
198
|
-
});
|
|
199
|
-
|
|
200
|
-
const schedules = () =>
|
|
201
|
-
Object.fromEntries(
|
|
202
|
-
Object.entries(known)
|
|
203
|
-
.filter(([, job]) => job.every !== undefined)
|
|
204
|
-
.map(([name, job]) => [
|
|
205
|
-
name,
|
|
206
|
-
{ everyMs: job.every!, maxAttempts: job.attempts },
|
|
207
|
-
]),
|
|
208
|
-
);
|
|
209
|
-
|
|
210
|
-
const api: JobsApi = {
|
|
211
|
-
get enabled() {
|
|
212
|
-
return enabled();
|
|
213
|
-
},
|
|
214
|
-
get jobs() {
|
|
215
|
-
return known;
|
|
216
|
-
},
|
|
217
|
-
async enqueue(name, payload, opts = {}) {
|
|
218
|
-
await discover();
|
|
219
|
-
const job = known[name];
|
|
220
|
-
if (!job) throw new Error(`plugin-jobs: no job named "${name}"`);
|
|
221
|
-
return enqueueJob(job, payload, opts);
|
|
222
|
-
},
|
|
223
|
-
queue,
|
|
224
|
-
};
|
|
225
|
-
|
|
226
|
-
const self: Jobs<D> = {
|
|
227
|
-
name: "plugin-jobs",
|
|
228
|
-
// tome opens the database and checks the schema first.
|
|
229
|
-
after: ["tome"],
|
|
230
|
-
serverOnly: [dirOption],
|
|
231
|
-
tables: { sigilJobs: table },
|
|
232
|
-
database: key,
|
|
233
|
-
jobs: api,
|
|
234
|
-
connect,
|
|
235
|
-
discover,
|
|
236
|
-
|
|
237
|
-
/**
|
|
238
|
-
* `$jobs/send-welcome` imports a job. Other plugins' jobs join the
|
|
239
|
-
* known ones here: a plugin that declares `jobs: { name: job }` (mail's
|
|
240
|
-
* delivery) is run by this queue like a file in src/jobs.
|
|
241
|
-
*/
|
|
242
|
-
config(config) {
|
|
243
|
-
const seen = new Set<unknown>();
|
|
244
|
-
const visit = (p: unknown) => {
|
|
245
|
-
if (!p || typeof p !== "object" || seen.has(p) || p === self) return;
|
|
246
|
-
seen.add(p);
|
|
247
|
-
const plugin = p as {
|
|
248
|
-
name?: string;
|
|
249
|
-
jobs?: unknown;
|
|
250
|
-
includes?: readonly unknown[];
|
|
251
|
-
};
|
|
252
|
-
for (const inner of plugin.includes ?? []) visit(inner);
|
|
253
|
-
const offered = plugin.jobs;
|
|
254
|
-
if (!offered || typeof offered !== "object") return;
|
|
255
|
-
for (const [name, job] of Object.entries(offered)) {
|
|
256
|
-
if (typeof (job as Job<any>)?.handler !== "function") continue;
|
|
257
|
-
if (known[name] && known[name] !== job) {
|
|
258
|
-
throw new Error(
|
|
259
|
-
`plugin-jobs: two jobs are named "${name}" (one from ${plugin.name ?? "a plugin"}). Rename one.`,
|
|
260
|
-
);
|
|
261
|
-
}
|
|
262
|
-
nameJob(job as Job<any>, name);
|
|
263
|
-
known = { ...known, [name]: job as Job<any> };
|
|
264
|
-
}
|
|
265
|
-
};
|
|
266
|
-
for (const p of config.plugins ?? []) visit(p);
|
|
267
|
-
dev = (config as { dev?: boolean }).dev ?? dev;
|
|
268
|
-
return {
|
|
269
|
-
...config,
|
|
270
|
-
alias: { $jobs: dirOption, ...(config.alias ?? {}) },
|
|
271
|
-
};
|
|
272
|
-
},
|
|
273
|
-
|
|
274
|
-
/** Compile the folder's list with the rest of the app, so a broken job fails the build. */
|
|
275
|
-
async onBuildStart() {
|
|
276
|
-
await discover();
|
|
277
|
-
},
|
|
278
|
-
|
|
279
|
-
/**
|
|
280
|
-
* Publishes the queue as `locals.jobs`, makes the schedules exist,
|
|
281
|
-
* and starts the inline worker if one was asked for.
|
|
282
|
-
*/
|
|
283
|
-
async services() {
|
|
284
|
-
if (!enabled()) return {};
|
|
285
|
-
conn();
|
|
286
|
-
await discover();
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
.
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
const
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
);
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
);
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
.
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
1
|
+
import { isAbsolute, join, relative } from "node:path";
|
|
2
|
+
import type { CliCommand, GrimoirePlugin } from "@sigil-dev/grimoire";
|
|
3
|
+
import { hasDatabase } from "@sigil-dev/plugin-sql";
|
|
4
|
+
import { adoptTables, type Tome, tomeFor } from "@sigil-dev/tome";
|
|
5
|
+
import { checkPayload, type Job, nameJob, setJobSink } from "./define.ts";
|
|
6
|
+
import { ms } from "./duration.ts";
|
|
7
|
+
import { loadManifest } from "./manifest.ts";
|
|
8
|
+
import { isMissingTable, JobQueue } from "./queue.ts";
|
|
9
|
+
import { type JobsTable, jobsTable } from "./schema.ts";
|
|
10
|
+
import type { EnqueueOptions } from "./store.ts";
|
|
11
|
+
import { JobWorker, MISSING_TABLE } from "./worker.ts";
|
|
12
|
+
|
|
13
|
+
export { defineJob, type Job, type StandardSchema } from "./define.ts";
|
|
14
|
+
export { type Duration, ms } from "./duration.ts";
|
|
15
|
+
export { isMissingTable, JobQueue } from "./queue.ts";
|
|
16
|
+
export { type JobsTable, jobsTable } from "./schema.ts";
|
|
17
|
+
export type {
|
|
18
|
+
EnqueueOptions,
|
|
19
|
+
JobContext,
|
|
20
|
+
JobRow,
|
|
21
|
+
JobStats,
|
|
22
|
+
} from "./store.ts";
|
|
23
|
+
export type { WorkerOptions } from "./worker.ts";
|
|
24
|
+
export { JobWorker } from "./worker.ts";
|
|
25
|
+
|
|
26
|
+
export interface JobsOptions<D extends string = "db"> {
|
|
27
|
+
/**
|
|
28
|
+
* The plugin-sql database whose tome holds the queue's table: its key,
|
|
29
|
+
* "db" unless you named it.
|
|
30
|
+
* @default "db"
|
|
31
|
+
*/
|
|
32
|
+
database?: D;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Where jobs live, one per file: `src/jobs/send-welcome.ts` is the job
|
|
36
|
+
* "send-welcome". Imported as `$jobs/send-welcome`, and server-only.
|
|
37
|
+
* @default "src/jobs"
|
|
38
|
+
*/
|
|
39
|
+
dir?: string;
|
|
40
|
+
|
|
41
|
+
/** Jobs that are not files in `dir`, by name: a package's own, or a test's. */
|
|
42
|
+
jobs?: Record<string, Job<any>>;
|
|
43
|
+
|
|
44
|
+
/** The table's name in the database. @default "sigil_jobs" */
|
|
45
|
+
table?: string;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Run a worker in the server process.
|
|
49
|
+
*
|
|
50
|
+
* On in development, so a queued job (a mail) runs without a second
|
|
51
|
+
* terminal. Off in production: a worker in the app means every worker
|
|
52
|
+
* scaled up is also a worker — fine for one process, surprising at
|
|
53
|
+
* three, and the reason `sigil jobs work` exists to run instead.
|
|
54
|
+
* @default true in development, false in production
|
|
55
|
+
*/
|
|
56
|
+
inlineWorker?: boolean;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export interface JobsApi {
|
|
60
|
+
/**
|
|
61
|
+
* Queue a job by name, for when the name is data (an admin screen, a
|
|
62
|
+
* webhook). In code, import the job and call its `enqueue`: that one is
|
|
63
|
+
* typed.
|
|
64
|
+
*/
|
|
65
|
+
enqueue(
|
|
66
|
+
name: string,
|
|
67
|
+
payload?: unknown,
|
|
68
|
+
options?: EnqueueOptions,
|
|
69
|
+
): Promise<string>;
|
|
70
|
+
queue: JobQueue;
|
|
71
|
+
/** Every job, by name: the folder's and `jobs({ jobs })`'s. Filled at startup. */
|
|
72
|
+
readonly jobs: Readonly<Record<string, Job<any>>>;
|
|
73
|
+
/**
|
|
74
|
+
* False when the database is an optional adapter that was not
|
|
75
|
+
* configured: nothing runs, and `enqueue` throws rather than drop work
|
|
76
|
+
* silently.
|
|
77
|
+
*/
|
|
78
|
+
readonly enabled: boolean;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** What `jobs()` returns: a plugin, the queue's table, and the queue. */
|
|
82
|
+
export type Jobs<D extends string = "db"> = GrimoirePlugin<{
|
|
83
|
+
jobs: JobsApi;
|
|
84
|
+
}> & {
|
|
85
|
+
jobs: JobsApi;
|
|
86
|
+
/** The queue's table. tome adds it to the schema; `sigil db generate` creates it. */
|
|
87
|
+
tables: { sigilJobs: JobsTable };
|
|
88
|
+
/** The database it is in. */
|
|
89
|
+
database: D;
|
|
90
|
+
/**
|
|
91
|
+
* Use this tome. Done for you on first use, from the tome over
|
|
92
|
+
* `database`; call it yourself to use another one, as a test does.
|
|
93
|
+
*/
|
|
94
|
+
connect(db: Tome<any, any>): void;
|
|
95
|
+
/** Find the jobs: the folder's, compiled, and the ones passed in. Done at startup. */
|
|
96
|
+
discover(): Promise<Readonly<Record<string, Job<any>>>>;
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Durable background jobs, one per file, in a table of the app's own tome.
|
|
101
|
+
*
|
|
102
|
+
* ```ts
|
|
103
|
+
* // src/jobs/send-welcome.ts
|
|
104
|
+
* export default defineJob({
|
|
105
|
+
* payload: schema({ email: t.text().email() }),
|
|
106
|
+
* async handler({ email }) { ... },
|
|
107
|
+
* });
|
|
108
|
+
* // sigil.config.ts: tome adds the queue's table to the schema
|
|
109
|
+
* plugins: [db.plugin, jobs()]
|
|
110
|
+
* // anywhere on the server
|
|
111
|
+
* await sendWelcome.enqueue({ email });
|
|
112
|
+
* ```
|
|
113
|
+
*/
|
|
114
|
+
export function jobs<const D extends string = "db">(
|
|
115
|
+
options: JobsOptions<D> = {},
|
|
116
|
+
): Jobs<D> {
|
|
117
|
+
const key = (options.database ?? "db") as D;
|
|
118
|
+
const dirOption = options.dir ?? "src/jobs";
|
|
119
|
+
const dirOf = (root: string) =>
|
|
120
|
+
isAbsolute(dirOption) ? dirOption : join(root, dirOption);
|
|
121
|
+
const table = jobsTable(options.table ?? "sigil_jobs");
|
|
122
|
+
|
|
123
|
+
let db: Tome<any, any> | undefined;
|
|
124
|
+
function connect(found: Tome<any, any>) {
|
|
125
|
+
// tome takes it from the config; a script or test without one gets it here.
|
|
126
|
+
adoptTables(found, "plugin-jobs", { sigilJobs: table });
|
|
127
|
+
db = found;
|
|
128
|
+
}
|
|
129
|
+
// Found on first use, like auth: the schema is built from this plugin's
|
|
130
|
+
// own table, so the tome cannot exist yet when the plugin is made.
|
|
131
|
+
const conn = (): Tome<any, any> => {
|
|
132
|
+
if (!db) {
|
|
133
|
+
const found = tomeFor(key);
|
|
134
|
+
if (!found) {
|
|
135
|
+
throw new Error(
|
|
136
|
+
`plugin-jobs: no tome over the "${key}" database. Make one with ` +
|
|
137
|
+
"tome(schema) and register it: plugins: [db.plugin, jobs()].",
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
connect(found);
|
|
141
|
+
}
|
|
142
|
+
return db!;
|
|
143
|
+
};
|
|
144
|
+
const queue = new JobQueue(conn, table);
|
|
145
|
+
let dev = process.env.NODE_ENV !== "production";
|
|
146
|
+
let inline: JobWorker | undefined;
|
|
147
|
+
// An optional adapter that was not configured registers no database.
|
|
148
|
+
const enabled = () => hasDatabase(key);
|
|
149
|
+
const off = (): never => {
|
|
150
|
+
throw new Error(
|
|
151
|
+
`plugin-jobs: the queue is off — database "${key}" is not configured. ` +
|
|
152
|
+
"Check the connection URL, or guard the call with `queue.jobs.enabled`.",
|
|
153
|
+
);
|
|
154
|
+
};
|
|
155
|
+
|
|
156
|
+
let known: Record<string, Job<any>> = { ...(options.jobs ?? {}) };
|
|
157
|
+
for (const [name, job] of Object.entries(known)) nameJob(job, name);
|
|
158
|
+
let discovering: Promise<Record<string, Job<any>>> | undefined;
|
|
159
|
+
const discover = () => {
|
|
160
|
+
discovering ??= (async () => {
|
|
161
|
+
const root = process.cwd();
|
|
162
|
+
const found = await loadManifest(root, dirOf(root));
|
|
163
|
+
for (const [name, job] of Object.entries(found)) {
|
|
164
|
+
if (known[name] && known[name] !== job) {
|
|
165
|
+
throw new Error(
|
|
166
|
+
`plugin-jobs: "${name}" is both a file in ${dirOption} and in jobs({ jobs }). Keep one.`,
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
nameJob(job, name);
|
|
170
|
+
}
|
|
171
|
+
known = { ...known, ...found };
|
|
172
|
+
return known;
|
|
173
|
+
})();
|
|
174
|
+
return discovering;
|
|
175
|
+
};
|
|
176
|
+
|
|
177
|
+
const enqueueJob = async (
|
|
178
|
+
job: Job<any>,
|
|
179
|
+
payload: unknown,
|
|
180
|
+
opts: EnqueueOptions,
|
|
181
|
+
) => {
|
|
182
|
+
if (!enabled()) off();
|
|
183
|
+
const checked = await checkPayload(job, payload);
|
|
184
|
+
if (!checked.ok) {
|
|
185
|
+
throw new Error(`plugin-jobs: ${job.name}: ${checked.message}`);
|
|
186
|
+
}
|
|
187
|
+
const runAt =
|
|
188
|
+
opts.at ??
|
|
189
|
+
(opts.in !== undefined ? new Date(Date.now() + ms(opts.in)) : undefined);
|
|
190
|
+
return queue.enqueue(job.name, checked.value ?? {}, {
|
|
191
|
+
runAt,
|
|
192
|
+
maxAttempts: job.attempts,
|
|
193
|
+
});
|
|
194
|
+
};
|
|
195
|
+
setJobSink({
|
|
196
|
+
enqueue: enqueueJob,
|
|
197
|
+
discover: async () => void (await discover()),
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
const schedules = () =>
|
|
201
|
+
Object.fromEntries(
|
|
202
|
+
Object.entries(known)
|
|
203
|
+
.filter(([, job]) => job.every !== undefined)
|
|
204
|
+
.map(([name, job]) => [
|
|
205
|
+
name,
|
|
206
|
+
{ everyMs: job.every!, maxAttempts: job.attempts },
|
|
207
|
+
]),
|
|
208
|
+
);
|
|
209
|
+
|
|
210
|
+
const api: JobsApi = {
|
|
211
|
+
get enabled() {
|
|
212
|
+
return enabled();
|
|
213
|
+
},
|
|
214
|
+
get jobs() {
|
|
215
|
+
return known;
|
|
216
|
+
},
|
|
217
|
+
async enqueue(name, payload, opts = {}) {
|
|
218
|
+
await discover();
|
|
219
|
+
const job = known[name];
|
|
220
|
+
if (!job) throw new Error(`plugin-jobs: no job named "${name}"`);
|
|
221
|
+
return enqueueJob(job, payload, opts);
|
|
222
|
+
},
|
|
223
|
+
queue,
|
|
224
|
+
};
|
|
225
|
+
|
|
226
|
+
const self: Jobs<D> = {
|
|
227
|
+
name: "plugin-jobs",
|
|
228
|
+
// tome opens the database and checks the schema first.
|
|
229
|
+
after: ["tome"],
|
|
230
|
+
serverOnly: [dirOption],
|
|
231
|
+
tables: { sigilJobs: table },
|
|
232
|
+
database: key,
|
|
233
|
+
jobs: api,
|
|
234
|
+
connect,
|
|
235
|
+
discover,
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* `$jobs/send-welcome` imports a job. Other plugins' jobs join the
|
|
239
|
+
* known ones here: a plugin that declares `jobs: { name: job }` (mail's
|
|
240
|
+
* delivery) is run by this queue like a file in src/jobs.
|
|
241
|
+
*/
|
|
242
|
+
config(config) {
|
|
243
|
+
const seen = new Set<unknown>();
|
|
244
|
+
const visit = (p: unknown) => {
|
|
245
|
+
if (!p || typeof p !== "object" || seen.has(p) || p === self) return;
|
|
246
|
+
seen.add(p);
|
|
247
|
+
const plugin = p as {
|
|
248
|
+
name?: string;
|
|
249
|
+
jobs?: unknown;
|
|
250
|
+
includes?: readonly unknown[];
|
|
251
|
+
};
|
|
252
|
+
for (const inner of plugin.includes ?? []) visit(inner);
|
|
253
|
+
const offered = plugin.jobs;
|
|
254
|
+
if (!offered || typeof offered !== "object") return;
|
|
255
|
+
for (const [name, job] of Object.entries(offered)) {
|
|
256
|
+
if (typeof (job as Job<any>)?.handler !== "function") continue;
|
|
257
|
+
if (known[name] && known[name] !== job) {
|
|
258
|
+
throw new Error(
|
|
259
|
+
`plugin-jobs: two jobs are named "${name}" (one from ${plugin.name ?? "a plugin"}). Rename one.`,
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
nameJob(job as Job<any>, name);
|
|
263
|
+
known = { ...known, [name]: job as Job<any> };
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
for (const p of config.plugins ?? []) visit(p);
|
|
267
|
+
dev = (config as { dev?: boolean }).dev ?? dev;
|
|
268
|
+
return {
|
|
269
|
+
...config,
|
|
270
|
+
alias: { $jobs: dirOption, ...(config.alias ?? {}) },
|
|
271
|
+
};
|
|
272
|
+
},
|
|
273
|
+
|
|
274
|
+
/** Compile the folder's list with the rest of the app, so a broken job fails the build. */
|
|
275
|
+
async onBuildStart() {
|
|
276
|
+
await discover();
|
|
277
|
+
},
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Publishes the queue as `locals.jobs`, makes the schedules exist,
|
|
281
|
+
* and starts the inline worker if one was asked for.
|
|
282
|
+
*/
|
|
283
|
+
async services() {
|
|
284
|
+
if (!enabled()) return {};
|
|
285
|
+
conn();
|
|
286
|
+
await discover();
|
|
287
|
+
// before `sigil db migrate` the table isn't there: the server still
|
|
288
|
+
// starts, says what to do once, and the worker sets the schedules up
|
|
289
|
+
// when the table appears
|
|
290
|
+
const prepare = async (quiet: boolean) => {
|
|
291
|
+
try {
|
|
292
|
+
await queue.ensureSchedules(schedules());
|
|
293
|
+
return true;
|
|
294
|
+
} catch (e) {
|
|
295
|
+
if (!isMissingTable(e)) throw e;
|
|
296
|
+
if (!quiet) console.error(MISSING_TABLE);
|
|
297
|
+
return false;
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
const ready = await prepare(false);
|
|
301
|
+
if (options.inlineWorker ?? dev) {
|
|
302
|
+
inline = new JobWorker(queue, known, ready ? {} : { prepare: () => prepare(true) });
|
|
303
|
+
void inline.run();
|
|
304
|
+
}
|
|
305
|
+
return { jobs: api };
|
|
306
|
+
},
|
|
307
|
+
|
|
308
|
+
/** A restart in development stops the old worker before the next one starts. */
|
|
309
|
+
async onStop() {
|
|
310
|
+
await inline?.stop();
|
|
311
|
+
inline = undefined;
|
|
312
|
+
},
|
|
313
|
+
|
|
314
|
+
/**
|
|
315
|
+
* Registers `sigil jobs <subcommand>`.
|
|
316
|
+
*
|
|
317
|
+
* The CLI never starts the app server, so the jobs come from the
|
|
318
|
+
* folder and the config the CLI loaded.
|
|
319
|
+
*/
|
|
320
|
+
cli(program: CliCommand) {
|
|
321
|
+
/** Open the queue's database for one command, and close it after, or the pool keeps the CLI alive. */
|
|
322
|
+
const oneShot = async <R>(
|
|
323
|
+
fn: (q: JobQueue) => Promise<R>,
|
|
324
|
+
): Promise<R> => {
|
|
325
|
+
const database = conn().database;
|
|
326
|
+
await database.open();
|
|
327
|
+
try {
|
|
328
|
+
return await fn(queue);
|
|
329
|
+
} finally {
|
|
330
|
+
await database.close();
|
|
331
|
+
}
|
|
332
|
+
};
|
|
333
|
+
|
|
334
|
+
const group = program
|
|
335
|
+
.command("jobs")
|
|
336
|
+
.description("manage the durable job queue");
|
|
337
|
+
|
|
338
|
+
group
|
|
339
|
+
.command("work")
|
|
340
|
+
.description("run the job worker until stopped")
|
|
341
|
+
.option("--interval <ms>", "idle poll interval")
|
|
342
|
+
.option("--batch <n>", "jobs per pass")
|
|
343
|
+
.action((async (opts: { interval?: string; batch?: string }) => {
|
|
344
|
+
await conn().database.open();
|
|
345
|
+
await discover();
|
|
346
|
+
// The worker completes recurring jobs, so it has to know
|
|
347
|
+
// them: without the schedule a completed one is deleted.
|
|
348
|
+
await queue.ensureSchedules(schedules());
|
|
349
|
+
const worker = new JobWorker(queue, known, {
|
|
350
|
+
...(opts.interval ? { intervalMs: Number(opts.interval) } : {}),
|
|
351
|
+
...(opts.batch ? { batchSize: Number(opts.batch) } : {}),
|
|
352
|
+
});
|
|
353
|
+
// Drain before exiting so a deploy does not abandon claimed
|
|
354
|
+
// jobs. The stale-claim timeout covers a hard kill.
|
|
355
|
+
const shutdown = async () => {
|
|
356
|
+
console.log("\nstopping, draining…");
|
|
357
|
+
await worker.stop();
|
|
358
|
+
process.exit(0);
|
|
359
|
+
};
|
|
360
|
+
process.on("SIGINT", shutdown);
|
|
361
|
+
process.on("SIGTERM", shutdown);
|
|
362
|
+
const root = process.cwd();
|
|
363
|
+
console.log(
|
|
364
|
+
`sigil jobs work — ${Object.keys(known).length} job(s) from ${relative(root, dirOf(root)).replace(/\\/g, "/") || "."}`,
|
|
365
|
+
);
|
|
366
|
+
await worker.run();
|
|
367
|
+
}) as never);
|
|
368
|
+
|
|
369
|
+
group
|
|
370
|
+
.command("list")
|
|
371
|
+
.description("list the most recent jobs")
|
|
372
|
+
.option("--limit <n>", "how many rows")
|
|
373
|
+
.action((async (opts: { limit?: string }) => {
|
|
374
|
+
const rows = await oneShot((q) => q.list(Number(opts.limit ?? 20)));
|
|
375
|
+
if (rows.length === 0) {
|
|
376
|
+
console.log("no jobs");
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
console.log(["ID", "NAME", "RUN_AT", "TRIES", "ERROR"].join("\t"));
|
|
380
|
+
for (const row of rows) {
|
|
381
|
+
console.log(
|
|
382
|
+
[
|
|
383
|
+
row.id.slice(0, 8),
|
|
384
|
+
row.name,
|
|
385
|
+
row.runAt.toISOString(),
|
|
386
|
+
`${row.attempts}`,
|
|
387
|
+
row.lastError ? row.lastError.slice(0, 40) : "-",
|
|
388
|
+
].join("\t"),
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
}) as never);
|
|
392
|
+
|
|
393
|
+
group
|
|
394
|
+
.command("stats")
|
|
395
|
+
.description("count jobs by state")
|
|
396
|
+
.action((async () => {
|
|
397
|
+
const s = await oneShot((q) => q.stats());
|
|
398
|
+
console.log(
|
|
399
|
+
`pending ${s.pending} claimed ${s.claimed} failed ${s.failed}`,
|
|
400
|
+
);
|
|
401
|
+
}) as never);
|
|
402
|
+
|
|
403
|
+
group
|
|
404
|
+
.command("retry")
|
|
405
|
+
.argument("[id]", "job id, or all failed jobs when omitted")
|
|
406
|
+
.description("requeue a job, clearing its attempt count")
|
|
407
|
+
.action((async (id?: string) => {
|
|
408
|
+
await oneShot(async (q) => {
|
|
409
|
+
if (id) {
|
|
410
|
+
const ok = await q.retry(id);
|
|
411
|
+
console.log(ok ? `requeued ${id}` : `no such job: ${id}`);
|
|
412
|
+
return;
|
|
413
|
+
}
|
|
414
|
+
const failed = (await q.list(1000)).filter(
|
|
415
|
+
(r) => r.attempts >= r.maxAttempts,
|
|
416
|
+
);
|
|
417
|
+
for (const job of failed) await q.retry(job.id);
|
|
418
|
+
console.log(`requeued ${failed.length} failed job(s)`);
|
|
419
|
+
});
|
|
420
|
+
}) as never);
|
|
421
|
+
},
|
|
422
|
+
};
|
|
423
|
+
return self;
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
export const plugin = jobs;
|
|
427
|
+
export default jobs;
|