@sigil-dev/plugin-jobs 0.9.29 → 0.9.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -105
- package/define.ts +223 -0
- package/duration.ts +7 -0
- package/index.ts +275 -341
- package/manifest.ts +96 -0
- package/package.json +4 -3
- package/queue.ts +178 -281
- package/schema.ts +46 -0
- package/store.ts +20 -89
- package/worker.ts +42 -36
- package/store-bun.ts +0 -137
- package/store-drizzle.ts +0 -86
package/index.ts
CHANGED
|
@@ -1,366 +1,320 @@
|
|
|
1
|
+
import { isAbsolute, join, relative } from "node:path";
|
|
1
2
|
import type { CliCommand, GrimoirePlugin } from "@sigil-dev/grimoire";
|
|
2
|
-
import {
|
|
3
|
+
import { hasDatabase } from "@sigil-dev/plugin-sql";
|
|
4
|
+
import { adoptTables, type Tome, tomeFor } from "@sigil-dev/tome";
|
|
5
|
+
import { checkPayload, type Job, nameJob, setJobSink } from "./define.ts";
|
|
6
|
+
import { ms } from "./duration.ts";
|
|
7
|
+
import { loadManifest } from "./manifest.ts";
|
|
3
8
|
import { JobQueue } from "./queue.ts";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
everyMs,
|
|
7
|
-
type EnqueueOptions,
|
|
8
|
-
type Every,
|
|
9
|
-
type JobHandler,
|
|
10
|
-
type JobStore,
|
|
11
|
-
type PayloadOf,
|
|
12
|
-
} from "./store.ts";
|
|
9
|
+
import { type JobsTable, jobsTable } from "./schema.ts";
|
|
10
|
+
import type { EnqueueOptions } from "./store.ts";
|
|
13
11
|
import { JobWorker } from "./worker.ts";
|
|
14
12
|
|
|
15
|
-
export {
|
|
13
|
+
export { defineJob, type Job, type StandardSchema } from "./define.ts";
|
|
14
|
+
export { type Duration, ms } from "./duration.ts";
|
|
15
|
+
export { JobQueue } from "./queue.ts";
|
|
16
|
+
export { type JobsTable, jobsTable } from "./schema.ts";
|
|
16
17
|
export type {
|
|
17
|
-
Capabilities,
|
|
18
18
|
EnqueueOptions,
|
|
19
|
-
Every,
|
|
20
19
|
JobContext,
|
|
21
|
-
JobHandler,
|
|
22
|
-
PayloadOf,
|
|
23
20
|
JobRow,
|
|
24
21
|
JobStats,
|
|
25
|
-
JobStore,
|
|
26
22
|
} from "./store.ts";
|
|
27
|
-
export { bunSqlStore, bunStore, databaseStore } from "./store-bun.ts";
|
|
28
|
-
export { drizzleStore, bunSqlRunner } from "./store-drizzle.ts";
|
|
29
|
-
export type { SqlRunner } from "./store-drizzle.ts";
|
|
30
|
-
export { JobWorker } from "./worker.ts";
|
|
31
23
|
export type { WorkerOptions } from "./worker.ts";
|
|
24
|
+
export { JobWorker } from "./worker.ts";
|
|
32
25
|
|
|
33
|
-
|
|
34
|
-
export type Handlers = Record<string, JobHandler>;
|
|
35
|
-
|
|
36
|
-
export interface JobsOptions<H extends Handlers = Handlers> {
|
|
26
|
+
export interface JobsOptions<D extends string = "db"> {
|
|
37
27
|
/**
|
|
38
|
-
* The
|
|
39
|
-
*
|
|
40
|
-
* nor `store`, the queue uses the database named "db".
|
|
41
|
-
*
|
|
42
|
-
* Looked up when a query runs, so the order of the plugins array does
|
|
43
|
-
* not matter, and `sigil jobs work` opens it from the config.
|
|
28
|
+
* The plugin-sql database whose tome holds the queue's table: its key,
|
|
29
|
+
* "db" unless you named it.
|
|
44
30
|
* @default "db"
|
|
45
31
|
*/
|
|
46
|
-
database?:
|
|
47
|
-
|
|
48
|
-
/**
|
|
49
|
-
* The database client. Required, and not optional-with-a-default,
|
|
50
|
-
* because there is no sensible fallback: a queue that silently kept jobs
|
|
51
|
-
* in memory would look like it worked right up until a restart.
|
|
52
|
-
*/
|
|
53
|
-
/**
|
|
54
|
-
* A `JobStore`: an object with `query(sql, params)`, filling `?`
|
|
55
|
-
* placeholders, and optionally a `capabilities` block declaring what the
|
|
56
|
-
* dialect can do.
|
|
57
|
-
*
|
|
58
|
-
* The queue never opens a connection, so it cannot guess the wrong
|
|
59
|
-
* dialect — a `Bun.SQL` client, a `pg` pool, or Drizzle all work
|
|
60
|
-
* unchanged. A string used to be accepted here and opened for you; it is
|
|
61
|
-
* not any more, because working out which driver a string wanted meant
|
|
62
|
-
* guessing a dialect, and the guessing was wrong more often than not.
|
|
63
|
-
*
|
|
64
|
-
* Either this or `store` is required.
|
|
65
|
-
*/
|
|
66
|
-
sql?: JobStore;
|
|
67
|
-
|
|
68
|
-
/**
|
|
69
|
-
* A store built from the services other plugins published.
|
|
70
|
-
*
|
|
71
|
-
* This is the option to reach for when the app already registered
|
|
72
|
-
* `@sigil-dev/plugin-sql`, because the queue then runs on the connection
|
|
73
|
-
* that plugin opened instead of opening a second pool and dividing the
|
|
74
|
-
* worker budget again:
|
|
75
|
-
*
|
|
76
|
-
* const db = adapter(url, { maxTotal: 12 });
|
|
77
|
-
* const queue = jobs({
|
|
78
|
-
* store: (services) => bunStore(services.db as Bun.SQL),
|
|
79
|
-
* });
|
|
80
|
-
*
|
|
81
|
-
* Register the SQL plugin first — `services` is a snapshot of what ran
|
|
82
|
-
* before this plugin, so a `store` that does not find what it needs
|
|
83
|
-
* should throw rather than fall back to a fresh connection. The function
|
|
84
|
-
* runs inside `services()`, which is the one hook that receives them.
|
|
85
|
-
*
|
|
86
|
-
* Prefer `database`, which needs none of that. At most one of `sql`,
|
|
87
|
-
* `store` and `database`.
|
|
88
|
-
*/
|
|
89
|
-
store?: (services: Record<string, unknown>) => JobStore;
|
|
32
|
+
database?: D;
|
|
90
33
|
|
|
91
34
|
/**
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
* Annotate each handler's payload and `enqueue` checks it.
|
|
35
|
+
* Where jobs live, one per file: `src/jobs/send-welcome.ts` is the job
|
|
36
|
+
* "send-welcome". Imported as `$jobs/send-welcome`, and server-only.
|
|
37
|
+
* @default "src/jobs"
|
|
96
38
|
*/
|
|
97
|
-
|
|
39
|
+
dir?: string;
|
|
98
40
|
|
|
99
|
-
/**
|
|
100
|
-
|
|
101
|
-
* workers run it once, not once each; a restart keeps its clock.
|
|
102
|
-
*
|
|
103
|
-
* ```ts
|
|
104
|
-
* schedules: { "rankings.snapshot": { every: "5m" } }
|
|
105
|
-
* ```
|
|
106
|
-
*/
|
|
107
|
-
schedules?: { [K in keyof H]?: { every: Every; payload?: PayloadOf<H[K]> } };
|
|
41
|
+
/** Jobs that are not files in `dir`, by name: a package's own, or a test's. */
|
|
42
|
+
jobs?: Record<string, Job<any>>;
|
|
108
43
|
|
|
109
|
-
/**
|
|
44
|
+
/** The table's name in the database. @default "sigil_jobs" */
|
|
110
45
|
table?: string;
|
|
111
46
|
|
|
112
|
-
/** Create the table on boot. @default true */
|
|
113
|
-
autoMigrate?: boolean;
|
|
114
|
-
|
|
115
47
|
/**
|
|
116
48
|
* Run a worker in the server process.
|
|
117
49
|
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
50
|
+
* On in development, so a queued job (a mail) runs without a second
|
|
51
|
+
* terminal. Off in production: a worker in the app means every worker
|
|
52
|
+
* scaled up is also a worker — fine for one process, surprising at
|
|
53
|
+
* three, and the reason `sigil jobs work` exists to run instead.
|
|
54
|
+
* @default true in development, false in production
|
|
122
55
|
*/
|
|
123
56
|
inlineWorker?: boolean;
|
|
124
57
|
}
|
|
125
58
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
59
|
+
export interface JobsApi {
|
|
60
|
+
/**
|
|
61
|
+
* Queue a job by name, for when the name is data (an admin screen, a
|
|
62
|
+
* webhook). In code, import the job and call its `enqueue`: that one is
|
|
63
|
+
* typed.
|
|
64
|
+
*/
|
|
65
|
+
enqueue(
|
|
66
|
+
name: string,
|
|
67
|
+
payload?: unknown,
|
|
68
|
+
options?: EnqueueOptions,
|
|
69
|
+
): Promise<string>;
|
|
134
70
|
queue: JobQueue;
|
|
135
|
-
|
|
71
|
+
/** Every job, by name: the folder's and `jobs({ jobs })`'s. Filled at startup. */
|
|
72
|
+
readonly jobs: Readonly<Record<string, Job<any>>>;
|
|
136
73
|
/**
|
|
137
|
-
* False when
|
|
138
|
-
*
|
|
74
|
+
* False when the database is an optional adapter that was not
|
|
75
|
+
* configured: nothing runs, and `enqueue` throws rather than drop work
|
|
76
|
+
* silently.
|
|
139
77
|
*/
|
|
140
|
-
enabled: boolean;
|
|
141
|
-
/** Register a handler after the plugin was created. */
|
|
142
|
-
register: (name: string, handler: JobHandler) => void;
|
|
78
|
+
readonly enabled: boolean;
|
|
143
79
|
}
|
|
144
80
|
|
|
81
|
+
/** What `jobs()` returns: a plugin, the queue's table, and the queue. */
|
|
82
|
+
export type Jobs<D extends string = "db"> = GrimoirePlugin<{
|
|
83
|
+
jobs: JobsApi;
|
|
84
|
+
}> & {
|
|
85
|
+
jobs: JobsApi;
|
|
86
|
+
/** The queue's table. tome adds it to the schema; `sigil db generate` creates it. */
|
|
87
|
+
tables: { sigilJobs: JobsTable };
|
|
88
|
+
/** The database it is in. */
|
|
89
|
+
database: D;
|
|
90
|
+
/**
|
|
91
|
+
* Use this tome. Done for you on first use, from the tome over
|
|
92
|
+
* `database`; call it yourself to use another one, as a test does.
|
|
93
|
+
*/
|
|
94
|
+
connect(db: Tome<any, any>): void;
|
|
95
|
+
/** Find the jobs: the folder's, compiled, and the ones passed in. Done at startup. */
|
|
96
|
+
discover(): Promise<Readonly<Record<string, Job<any>>>>;
|
|
97
|
+
};
|
|
98
|
+
|
|
145
99
|
/**
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
100
|
+
* Durable background jobs, one per file, in a table of the app's own tome.
|
|
101
|
+
*
|
|
102
|
+
* ```ts
|
|
103
|
+
* // src/jobs/send-welcome.ts
|
|
104
|
+
* export default defineJob({
|
|
105
|
+
* payload: schema({ email: t.text().email() }),
|
|
106
|
+
* async handler({ email }) { ... },
|
|
107
|
+
* });
|
|
108
|
+
* // sigil.config.ts: tome adds the queue's table to the schema
|
|
109
|
+
* plugins: [db.plugin, jobs()]
|
|
110
|
+
* // anywhere on the server
|
|
111
|
+
* await sendWelcome.enqueue({ email });
|
|
112
|
+
* ```
|
|
150
113
|
*/
|
|
151
|
-
export function jobs<
|
|
152
|
-
options: JobsOptions<
|
|
153
|
-
):
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
// `store` is resolved at `services()` time, because until then there are
|
|
166
|
-
// no services to read. Everything that needs the queue — migrate, the
|
|
167
|
-
// inline worker, the api object — goes through `getQueue()`, so a
|
|
168
|
-
// config using `store` behaves identically to one using `sql`.
|
|
169
|
-
const factory = options.store;
|
|
170
|
-
const dbOption = options.database;
|
|
171
|
-
const disabled = typeof dbOption === "object" && dbOption.database === undefined;
|
|
172
|
-
if (disabled) {
|
|
173
|
-
const off = (): never => {
|
|
174
|
-
throw new Error(
|
|
175
|
-
`plugin-jobs: the queue is off � database "${dbOption.key}" is not configured. ` +
|
|
176
|
-
"Check the connection URL, or guard the call with `queue.jobs.enabled`.",
|
|
177
|
-
);
|
|
178
|
-
};
|
|
179
|
-
return {
|
|
180
|
-
name: "plugin-jobs (not configured)",
|
|
181
|
-
jobs: {
|
|
182
|
-
enabled: false,
|
|
183
|
-
enqueue: async () => off(),
|
|
184
|
-
get queue(): JobQueue {
|
|
185
|
-
return off();
|
|
186
|
-
},
|
|
187
|
-
handlers: { ...(options.handlers ?? {}) } as H,
|
|
188
|
-
register: () => {},
|
|
189
|
-
},
|
|
190
|
-
};
|
|
114
|
+
export function jobs<const D extends string = "db">(
|
|
115
|
+
options: JobsOptions<D> = {},
|
|
116
|
+
): Jobs<D> {
|
|
117
|
+
const key = (options.database ?? "db") as D;
|
|
118
|
+
const dirOption = options.dir ?? "src/jobs";
|
|
119
|
+
const dirOf = (root: string) =>
|
|
120
|
+
isAbsolute(dirOption) ? dirOption : join(root, dirOption);
|
|
121
|
+
const table = jobsTable(options.table ?? "sigil_jobs");
|
|
122
|
+
|
|
123
|
+
let db: Tome<any, any> | undefined;
|
|
124
|
+
function connect(found: Tome<any, any>) {
|
|
125
|
+
// tome takes it from the config; a script or test without one gets it here.
|
|
126
|
+
adoptTables(found, "plugin-jobs", { sigilJobs: table });
|
|
127
|
+
db = found;
|
|
191
128
|
}
|
|
192
|
-
|
|
193
|
-
|
|
129
|
+
// Found on first use, like auth: the schema is built from this plugin's
|
|
130
|
+
// own table, so the tome cannot exist yet when the plugin is made.
|
|
131
|
+
const conn = (): Tome<any, any> => {
|
|
132
|
+
if (!db) {
|
|
133
|
+
const found = tomeFor(key);
|
|
134
|
+
if (!found) {
|
|
135
|
+
throw new Error(
|
|
136
|
+
`plugin-jobs: no tome over the "${key}" database. Make one with ` +
|
|
137
|
+
"tome(schema) and register it: plugins: [db.plugin, jobs()].",
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
connect(found);
|
|
141
|
+
}
|
|
142
|
+
return db!;
|
|
143
|
+
};
|
|
144
|
+
const queue = new JobQueue(conn, table);
|
|
145
|
+
let dev = process.env.NODE_ENV !== "production";
|
|
146
|
+
let inline: JobWorker | undefined;
|
|
147
|
+
// An optional adapter that was not configured registers no database.
|
|
148
|
+
const enabled = () => hasDatabase(key);
|
|
149
|
+
const off = (): never => {
|
|
194
150
|
throw new Error(
|
|
195
|
-
|
|
196
|
-
"
|
|
151
|
+
`plugin-jobs: the queue is off — database "${key}" is not configured. ` +
|
|
152
|
+
"Check the connection URL, or guard the call with `queue.jobs.enabled`.",
|
|
197
153
|
);
|
|
198
|
-
}
|
|
199
|
-
// The default: the plugin-sql database, looked up when a query runs.
|
|
200
|
-
const dbName = factory || options.sql ? undefined : ((typeof dbOption === "object" ? dbOption.key : dbOption) ?? "db");
|
|
201
|
-
const provided = options.sql ?? (dbName !== undefined ? databaseStore(dbName) : undefined);
|
|
202
|
-
/** Open the database the queue lives in, when it is a plugin-sql one. */
|
|
203
|
-
const openDatabase = async () => {
|
|
204
|
-
if (dbName !== undefined) await database(dbName).open();
|
|
205
|
-
};
|
|
206
|
-
let resolved: JobStore | undefined = provided;
|
|
207
|
-
// Assigned by services() before anything reads it. Held in a mutable
|
|
208
|
-
// slot because the api object is built before the services exist.
|
|
209
|
-
let queue: JobQueue | undefined = provided ? new JobQueue(provided, options.table) : undefined;
|
|
210
|
-
const requireQueue = (): JobQueue => {
|
|
211
|
-
if (!queue) {
|
|
212
|
-
throw new Error(
|
|
213
|
-
"plugin-jobs: the queue is not available yet. With `store`, it is " +
|
|
214
|
-
"created during services() — use locals.jobs from a request, or " +
|
|
215
|
-
"pass `sql` if you need it earlier.",
|
|
216
|
-
);
|
|
217
|
-
}
|
|
218
|
-
return queue;
|
|
219
154
|
};
|
|
220
155
|
|
|
221
|
-
|
|
222
|
-
const
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
156
|
+
let known: Record<string, Job<any>> = { ...(options.jobs ?? {}) };
|
|
157
|
+
for (const [name, job] of Object.entries(known)) nameJob(job, name);
|
|
158
|
+
let discovering: Promise<Record<string, Job<any>>> | undefined;
|
|
159
|
+
const discover = () => {
|
|
160
|
+
discovering ??= (async () => {
|
|
161
|
+
const root = process.cwd();
|
|
162
|
+
const found = await loadManifest(root, dirOf(root));
|
|
163
|
+
for (const [name, job] of Object.entries(found)) {
|
|
164
|
+
if (known[name] && known[name] !== job) {
|
|
165
|
+
throw new Error(
|
|
166
|
+
`plugin-jobs: "${name}" is both a file in ${dirOption} and in jobs({ jobs }). Keep one.`,
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
nameJob(job, name);
|
|
228
170
|
}
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
171
|
+
known = { ...known, ...found };
|
|
172
|
+
return known;
|
|
173
|
+
})();
|
|
174
|
+
return discovering;
|
|
175
|
+
};
|
|
232
176
|
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
* means the first enqueue can race the table and fail with "relation
|
|
243
|
-
* does not exist" on a fresh deploy, which looks like a queue bug and is
|
|
244
|
-
* not one. Only the worker's run loop is left unawaited — it is meant to
|
|
245
|
-
* outlive this call.
|
|
246
|
-
*/
|
|
247
|
-
const migrateAndStart = async (): Promise<Record<string, unknown>> => {
|
|
248
|
-
await openDatabase();
|
|
249
|
-
if (options.autoMigrate !== false) {
|
|
250
|
-
await requireQueue().migrate();
|
|
251
|
-
}
|
|
252
|
-
await requireQueue().ensureSchedules(schedules);
|
|
253
|
-
if (options.inlineWorker) {
|
|
254
|
-
void new JobWorker(requireQueue(), handlers).run();
|
|
177
|
+
const enqueueJob = async (
|
|
178
|
+
job: Job<any>,
|
|
179
|
+
payload: unknown,
|
|
180
|
+
opts: EnqueueOptions,
|
|
181
|
+
) => {
|
|
182
|
+
if (!enabled()) off();
|
|
183
|
+
const checked = await checkPayload(job, payload);
|
|
184
|
+
if (!checked.ok) {
|
|
185
|
+
throw new Error(`plugin-jobs: ${job.name}: ${checked.message}`);
|
|
255
186
|
}
|
|
256
|
-
|
|
187
|
+
const runAt =
|
|
188
|
+
opts.at ??
|
|
189
|
+
(opts.in !== undefined ? new Date(Date.now() + ms(opts.in)) : undefined);
|
|
190
|
+
return queue.enqueue(job.name, checked.value ?? {}, {
|
|
191
|
+
runAt,
|
|
192
|
+
maxAttempts: job.attempts,
|
|
193
|
+
});
|
|
257
194
|
};
|
|
195
|
+
setJobSink({
|
|
196
|
+
enqueue: enqueueJob,
|
|
197
|
+
discover: async () => void (await discover()),
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
const schedules = () =>
|
|
201
|
+
Object.fromEntries(
|
|
202
|
+
Object.entries(known)
|
|
203
|
+
.filter(([, job]) => job.every !== undefined)
|
|
204
|
+
.map(([name, job]) => [
|
|
205
|
+
name,
|
|
206
|
+
{ everyMs: job.every!, maxAttempts: job.attempts },
|
|
207
|
+
]),
|
|
208
|
+
);
|
|
258
209
|
|
|
259
|
-
const api: JobsApi
|
|
260
|
-
enabled
|
|
261
|
-
|
|
262
|
-
return requireQueue();
|
|
210
|
+
const api: JobsApi = {
|
|
211
|
+
get enabled() {
|
|
212
|
+
return enabled();
|
|
263
213
|
},
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
return requireQueue().enqueue(name, payload, opts);
|
|
214
|
+
get jobs() {
|
|
215
|
+
return known;
|
|
267
216
|
},
|
|
268
|
-
|
|
269
|
-
(
|
|
217
|
+
async enqueue(name, payload, opts = {}) {
|
|
218
|
+
await discover();
|
|
219
|
+
const job = known[name];
|
|
220
|
+
if (!job) throw new Error(`plugin-jobs: no job named "${name}"`);
|
|
221
|
+
return enqueueJob(job, payload, opts);
|
|
270
222
|
},
|
|
223
|
+
queue,
|
|
271
224
|
};
|
|
272
225
|
|
|
273
|
-
const
|
|
226
|
+
const self: Jobs<D> = {
|
|
274
227
|
name: "plugin-jobs",
|
|
228
|
+
// tome opens the database and checks the schema first.
|
|
229
|
+
after: ["tome"],
|
|
230
|
+
serverOnly: [dirOption],
|
|
231
|
+
tables: { sigilJobs: table },
|
|
232
|
+
database: key,
|
|
275
233
|
jobs: api,
|
|
234
|
+
connect,
|
|
235
|
+
discover,
|
|
276
236
|
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
237
|
+
/**
|
|
238
|
+
* `$jobs/send-welcome` imports a job. Other plugins' jobs join the
|
|
239
|
+
* known ones here: a plugin that declares `jobs: { name: job }` (mail's
|
|
240
|
+
* delivery) is run by this queue like a file in src/jobs.
|
|
241
|
+
*/
|
|
242
|
+
config(config) {
|
|
243
|
+
const seen = new Set<unknown>();
|
|
244
|
+
const visit = (p: unknown) => {
|
|
245
|
+
if (!p || typeof p !== "object" || seen.has(p) || p === self) return;
|
|
246
|
+
seen.add(p);
|
|
247
|
+
const plugin = p as {
|
|
248
|
+
name?: string;
|
|
249
|
+
jobs?: unknown;
|
|
250
|
+
includes?: readonly unknown[];
|
|
251
|
+
};
|
|
252
|
+
for (const inner of plugin.includes ?? []) visit(inner);
|
|
253
|
+
const offered = plugin.jobs;
|
|
254
|
+
if (!offered || typeof offered !== "object") return;
|
|
255
|
+
for (const [name, job] of Object.entries(offered)) {
|
|
256
|
+
if (typeof (job as Job<any>)?.handler !== "function") continue;
|
|
257
|
+
if (known[name] && known[name] !== job) {
|
|
258
|
+
throw new Error(
|
|
259
|
+
`plugin-jobs: two jobs are named "${name}" (one from ${plugin.name ?? "a plugin"}). Rename one.`,
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
nameJob(job as Job<any>, name);
|
|
263
|
+
known = { ...known, [name]: job as Job<any> };
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
for (const p of config.plugins ?? []) visit(p);
|
|
267
|
+
dev = (config as { dev?: boolean }).dev ?? dev;
|
|
268
|
+
return {
|
|
269
|
+
...config,
|
|
270
|
+
alias: { $jobs: dirOption, ...(config.alias ?? {}) },
|
|
271
|
+
};
|
|
272
|
+
},
|
|
273
|
+
|
|
274
|
+
/** Compile the folder's list with the rest of the app, so a broken job fails the build. */
|
|
275
|
+
async onBuildStart() {
|
|
276
|
+
await discover();
|
|
287
277
|
},
|
|
288
278
|
|
|
289
279
|
/**
|
|
290
|
-
* Publishes the queue as `locals.jobs
|
|
291
|
-
*
|
|
292
|
-
* This is what lets a route call `locals.jobs.enqueue(...)` with
|
|
293
|
-
* nothing imported and nothing exported from sigil.config.ts. The
|
|
294
|
-
* object is built once at startup, so the same queue instance is
|
|
295
|
-
* shared by every request.
|
|
280
|
+
* Publishes the queue as `locals.jobs`, makes the schedules exist,
|
|
281
|
+
* and starts the inline worker if one was asked for.
|
|
296
282
|
*/
|
|
297
|
-
services(
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
if (
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
return migrateAndStart();
|
|
283
|
+
async services() {
|
|
284
|
+
if (!enabled()) return {};
|
|
285
|
+
conn();
|
|
286
|
+
await discover();
|
|
287
|
+
await queue.ensureSchedules(schedules());
|
|
288
|
+
if (options.inlineWorker ?? dev) {
|
|
289
|
+
inline = new JobWorker(queue, known);
|
|
290
|
+
void inline.run();
|
|
306
291
|
}
|
|
307
|
-
if (dbName !== undefined) return migrateAndStart();
|
|
308
292
|
return { jobs: api };
|
|
309
293
|
},
|
|
310
294
|
|
|
295
|
+
/** A restart in development stops the old worker before the next one starts. */
|
|
296
|
+
async onStop() {
|
|
297
|
+
await inline?.stop();
|
|
298
|
+
inline = undefined;
|
|
299
|
+
},
|
|
300
|
+
|
|
311
301
|
/**
|
|
312
302
|
* Registers `sigil jobs <subcommand>`.
|
|
313
303
|
*
|
|
314
|
-
* The CLI never starts the app server, so
|
|
315
|
-
*
|
|
316
|
-
* config and reads the same handler map, which is why the config is
|
|
317
|
-
* the source of truth rather than module state.
|
|
304
|
+
* The CLI never starts the app server, so the jobs come from the
|
|
305
|
+
* folder and the config the CLI loaded.
|
|
318
306
|
*/
|
|
319
|
-
cli(program
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
// before calling this, so by the time an action runs the queue
|
|
327
|
-
// exists — and if `config` threw, the CLI already reported that
|
|
328
|
-
// and never got here.
|
|
329
|
-
const connect = (): JobsApi<H> => {
|
|
330
|
-
if (!instance.jobs) {
|
|
331
|
-
throw new Error(
|
|
332
|
-
"plugin-jobs is configured but no queue is available. " +
|
|
333
|
-
"Check the `jobs` option in sigil.config.ts.",
|
|
334
|
-
);
|
|
335
|
-
}
|
|
336
|
-
// A `store` config resolves the queue from the services the
|
|
337
|
-
// SQL plugin published, which only exist inside the server's
|
|
338
|
-
// plugin lifecycle. The CLI runs config() and never
|
|
339
|
-
// services(), so say that plainly instead of reporting the
|
|
340
|
-
// queue as missing.
|
|
341
|
-
if (factory && !queue) {
|
|
342
|
-
throw new Error(
|
|
343
|
-
"plugin-jobs is configured with `store`, which reads the client " +
|
|
344
|
-
"another plugin opened. The `sigil jobs` CLI has no request " +
|
|
345
|
-
"lifecycle to publish that from — use `sql: bunSqlStore(url)` in " +
|
|
346
|
-
"a CLI-capable config, or run the queue commands against a " +
|
|
347
|
-
"server that is up.",
|
|
348
|
-
);
|
|
349
|
-
}
|
|
350
|
-
return instance.jobs;
|
|
351
|
-
};
|
|
352
|
-
|
|
353
|
-
/**
|
|
354
|
-
* Run one command against the queue: open its database, and close
|
|
355
|
-
* it after, or the open pool keeps the CLI from exiting.
|
|
356
|
-
*/
|
|
357
|
-
const oneShot = async <R>(fn: (q: JobQueue) => Promise<R>): Promise<R> => {
|
|
358
|
-
const q = connect().queue;
|
|
359
|
-
await openDatabase();
|
|
307
|
+
cli(program: CliCommand) {
|
|
308
|
+
/** Open the queue's database for one command, and close it after, or the pool keeps the CLI alive. */
|
|
309
|
+
const oneShot = async <R>(
|
|
310
|
+
fn: (q: JobQueue) => Promise<R>,
|
|
311
|
+
): Promise<R> => {
|
|
312
|
+
const database = conn().database;
|
|
313
|
+
await database.open();
|
|
360
314
|
try {
|
|
361
|
-
return await fn(
|
|
315
|
+
return await fn(queue);
|
|
362
316
|
} finally {
|
|
363
|
-
|
|
317
|
+
await database.close();
|
|
364
318
|
}
|
|
365
319
|
};
|
|
366
320
|
|
|
@@ -371,19 +325,18 @@ export function jobs<H extends Handlers = Handlers>(
|
|
|
371
325
|
group
|
|
372
326
|
.command("work")
|
|
373
327
|
.description("run the job worker until stopped")
|
|
374
|
-
.option("--interval <ms>", "idle poll interval"
|
|
375
|
-
.option("--batch <n>", "jobs per pass"
|
|
376
|
-
.action(async (opts: { interval?:
|
|
377
|
-
|
|
378
|
-
await
|
|
328
|
+
.option("--interval <ms>", "idle poll interval")
|
|
329
|
+
.option("--batch <n>", "jobs per pass")
|
|
330
|
+
.action((async (opts: { interval?: string; batch?: string }) => {
|
|
331
|
+
await conn().database.open();
|
|
332
|
+
await discover();
|
|
379
333
|
// The worker completes recurring jobs, so it has to know
|
|
380
334
|
// them: without the schedule a completed one is deleted.
|
|
381
|
-
await
|
|
382
|
-
const worker = new JobWorker(
|
|
383
|
-
...(opts.interval ? { intervalMs: opts.interval } : {}),
|
|
384
|
-
...(opts.batch ? { batchSize: opts.batch } : {}),
|
|
335
|
+
await queue.ensureSchedules(schedules());
|
|
336
|
+
const worker = new JobWorker(queue, known, {
|
|
337
|
+
...(opts.interval ? { intervalMs: Number(opts.interval) } : {}),
|
|
338
|
+
...(opts.batch ? { batchSize: Number(opts.batch) } : {}),
|
|
385
339
|
});
|
|
386
|
-
|
|
387
340
|
// Drain before exiting so a deploy does not abandon claimed
|
|
388
341
|
// jobs. The stale-claim timeout covers a hard kill.
|
|
389
342
|
const shutdown = async () => {
|
|
@@ -393,52 +346,52 @@ export function jobs<H extends Handlers = Handlers>(
|
|
|
393
346
|
};
|
|
394
347
|
process.on("SIGINT", shutdown);
|
|
395
348
|
process.on("SIGTERM", shutdown);
|
|
396
|
-
|
|
349
|
+
const root = process.cwd();
|
|
397
350
|
console.log(
|
|
398
|
-
`sigil jobs work — ${Object.keys(
|
|
351
|
+
`sigil jobs work — ${Object.keys(known).length} job(s) from ${relative(root, dirOf(root)).replace(/\\/g, "/") || "."}`,
|
|
399
352
|
);
|
|
400
353
|
await worker.run();
|
|
401
|
-
});
|
|
354
|
+
}) as never);
|
|
402
355
|
|
|
403
356
|
group
|
|
404
357
|
.command("list")
|
|
405
358
|
.description("list the most recent jobs")
|
|
406
|
-
.option("--limit <n>", "how many rows"
|
|
407
|
-
.action(async (opts: { limit?:
|
|
408
|
-
const rows = await oneShot((q) => q.list(opts.limit ?? 20));
|
|
359
|
+
.option("--limit <n>", "how many rows")
|
|
360
|
+
.action((async (opts: { limit?: string }) => {
|
|
361
|
+
const rows = await oneShot((q) => q.list(Number(opts.limit ?? 20)));
|
|
409
362
|
if (rows.length === 0) {
|
|
410
363
|
console.log("no jobs");
|
|
411
364
|
return;
|
|
412
365
|
}
|
|
413
|
-
console.log(["ID", "NAME", "RUN_AT", "TRIES", "ERROR"].join("
|
|
366
|
+
console.log(["ID", "NAME", "RUN_AT", "TRIES", "ERROR"].join("\t"));
|
|
414
367
|
for (const row of rows) {
|
|
415
368
|
console.log(
|
|
416
369
|
[
|
|
417
370
|
row.id.slice(0, 8),
|
|
418
371
|
row.name,
|
|
419
|
-
row.
|
|
372
|
+
row.runAt.toISOString(),
|
|
420
373
|
`${row.attempts}`,
|
|
421
|
-
row.
|
|
422
|
-
].join("
|
|
374
|
+
row.lastError ? row.lastError.slice(0, 40) : "-",
|
|
375
|
+
].join("\t"),
|
|
423
376
|
);
|
|
424
377
|
}
|
|
425
|
-
});
|
|
378
|
+
}) as never);
|
|
426
379
|
|
|
427
380
|
group
|
|
428
381
|
.command("stats")
|
|
429
382
|
.description("count jobs by state")
|
|
430
|
-
.action(async () => {
|
|
383
|
+
.action((async () => {
|
|
431
384
|
const s = await oneShot((q) => q.stats());
|
|
432
385
|
console.log(
|
|
433
386
|
`pending ${s.pending} claimed ${s.claimed} failed ${s.failed}`,
|
|
434
387
|
);
|
|
435
|
-
});
|
|
388
|
+
}) as never);
|
|
436
389
|
|
|
437
390
|
group
|
|
438
391
|
.command("retry")
|
|
439
392
|
.argument("[id]", "job id, or all failed jobs when omitted")
|
|
440
393
|
.description("requeue a job, clearing its attempt count")
|
|
441
|
-
.action(async (id?: string) => {
|
|
394
|
+
.action((async (id?: string) => {
|
|
442
395
|
await oneShot(async (q) => {
|
|
443
396
|
if (id) {
|
|
444
397
|
const ok = await q.retry(id);
|
|
@@ -446,35 +399,16 @@ export function jobs<H extends Handlers = Handlers>(
|
|
|
446
399
|
return;
|
|
447
400
|
}
|
|
448
401
|
const failed = (await q.list(1000)).filter(
|
|
449
|
-
(r) => r.attempts >= r.
|
|
402
|
+
(r) => r.attempts >= r.maxAttempts,
|
|
450
403
|
);
|
|
451
404
|
for (const job of failed) await q.retry(job.id);
|
|
452
405
|
console.log(`requeued ${failed.length} failed job(s)`);
|
|
453
406
|
});
|
|
454
|
-
});
|
|
455
|
-
|
|
456
|
-
group
|
|
457
|
-
.command("migrate")
|
|
458
|
-
.description("create the jobs table if it does not exist")
|
|
459
|
-
.action(async () => {
|
|
460
|
-
await oneShot((q) => q.migrate());
|
|
461
|
-
console.log("jobs table ready");
|
|
462
|
-
});
|
|
407
|
+
}) as never);
|
|
463
408
|
},
|
|
464
409
|
};
|
|
465
|
-
|
|
466
|
-
return instance;
|
|
410
|
+
return self;
|
|
467
411
|
}
|
|
468
412
|
|
|
469
413
|
export const plugin = jobs;
|
|
470
414
|
export default jobs;
|
|
471
|
-
|
|
472
|
-
/**
|
|
473
|
-
* Accept an executor, or fail naming what was expected.
|
|
474
|
-
*
|
|
475
|
-
* A bare string used to be accepted here. It no longer is, and a message
|
|
476
|
-
* that says only "sql is required" would send someone to the docs to
|
|
477
|
-
* discover it. A queue pointed at the wrong database is worse than a boot
|
|
478
|
-
* that fails, so this throws at construction — which is what `sigil start`
|
|
479
|
-
* runs — rather than at the first enqueue.
|
|
480
|
-
*/
|