@sigil-dev/plugin-jobs 0.9.12 → 0.9.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -61
- package/index.ts +278 -53
- package/package.json +14 -4
- package/queue.ts +254 -43
- package/store-bun.ts +137 -0
- package/store-drizzle.ts +86 -0
- package/store.ts +117 -0
- package/worker.ts +13 -3
- package/sql.ts +0 -73
package/package.json
CHANGED
|
@@ -4,14 +4,24 @@
|
|
|
4
4
|
"module": "index.ts",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"private": false,
|
|
7
|
-
"version": "0.9.
|
|
8
|
-
"keywords": [
|
|
9
|
-
|
|
7
|
+
"version": "0.9.13",
|
|
8
|
+
"keywords": [
|
|
9
|
+
"grimoire",
|
|
10
|
+
"sigil",
|
|
11
|
+
"jobs",
|
|
12
|
+
"queue",
|
|
13
|
+
"background"
|
|
14
|
+
],
|
|
15
|
+
"files": [
|
|
16
|
+
"*.ts",
|
|
17
|
+
"README.md"
|
|
18
|
+
],
|
|
10
19
|
"devDependencies": {
|
|
11
20
|
"@types/bun": "latest"
|
|
12
21
|
},
|
|
13
22
|
"peerDependencies": {
|
|
14
23
|
"typescript": "^5",
|
|
15
|
-
"@sigil-dev/grimoire": "0.9.
|
|
24
|
+
"@sigil-dev/grimoire": "0.9.13",
|
|
25
|
+
"@sigil-dev/plugin-sql": "0.9.13"
|
|
16
26
|
}
|
|
17
27
|
}
|
package/queue.ts
CHANGED
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type {
|
|
2
|
+
Capabilities,
|
|
3
|
+
EnqueueOptions,
|
|
4
|
+
JobRow,
|
|
5
|
+
JobStats,
|
|
6
|
+
JobStore,
|
|
7
|
+
} from "./store.ts";
|
|
2
8
|
|
|
3
9
|
/**
|
|
4
10
|
* Schema statements, kept portable.
|
|
@@ -10,14 +16,27 @@ import type { EnqueueOptions, JobRow, JobStats, SqlExecutor } from "./sql.ts";
|
|
|
10
16
|
*/
|
|
11
17
|
export const SCHEMA = `
|
|
12
18
|
CREATE TABLE IF NOT EXISTS sigil_jobs (
|
|
13
|
-
|
|
19
|
+
-- VARCHAR(64), not TEXT: MySQL refuses to index a TEXT primary key
|
|
20
|
+
-- without a prefix length ("BLOB/TEXT column 'id' used in key
|
|
21
|
+
-- specification without a key length"), so a portable id column has to
|
|
22
|
+
-- be a bounded character type. 36 is a UUID with room to spare, and it
|
|
23
|
+
-- is what the generated ids actually look like.
|
|
24
|
+
id VARCHAR(64) PRIMARY KEY,
|
|
14
25
|
name TEXT NOT NULL,
|
|
15
26
|
payload TEXT NOT NULL,
|
|
27
|
+
-- name is indexed nowhere, so TEXT is fine and keeps long handler
|
|
28
|
+
-- names from being truncated by a column width.
|
|
16
29
|
run_at TIMESTAMP NOT NULL,
|
|
17
30
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
18
31
|
max_attempts INTEGER NOT NULL DEFAULT 5,
|
|
19
32
|
last_error TEXT,
|
|
20
33
|
claimed_at TIMESTAMP,
|
|
34
|
+
-- Which claim took this row. NOT a timestamp: MySQL truncates TIMESTAMP to
|
|
35
|
+
-- whole seconds, so two workers claiming in the same second are
|
|
36
|
+
-- indistinguishable, and the stale-claim comparison then re-claims a live
|
|
37
|
+
-- row. Measured: a second worker re-claimed rows claimed milliseconds
|
|
38
|
+
-- earlier. A random token per claim is exact on all three dialects.
|
|
39
|
+
claim_token VARCHAR(64),
|
|
21
40
|
created_at TIMESTAMP NOT NULL
|
|
22
41
|
);
|
|
23
42
|
CREATE INDEX IF NOT EXISTS sigil_jobs_ready ON sigil_jobs (run_at);
|
|
@@ -33,18 +52,146 @@ CREATE INDEX IF NOT EXISTS sigil_jobs_ready ON sigil_jobs (run_at);
|
|
|
33
52
|
* held open across the entire handler, including every HTTP call inside it.
|
|
34
53
|
*/
|
|
35
54
|
export class JobQueue {
|
|
55
|
+
/** Recurring jobs: row id → interval. Set by `ensureSchedules`. */
|
|
56
|
+
private readonly schedules = new Map<string, number>();
|
|
57
|
+
|
|
36
58
|
constructor(
|
|
37
|
-
private readonly
|
|
59
|
+
private readonly store: JobStore,
|
|
38
60
|
private readonly table: string = "sigil_jobs",
|
|
39
61
|
) {}
|
|
40
62
|
|
|
41
|
-
|
|
42
|
-
|
|
63
|
+
/**
|
|
64
|
+
* Every capability defaults to false. A store that declares nothing gets
|
|
65
|
+
* the weakest path, which is correct on every dialect rather than correct
|
|
66
|
+
* on the one it was written for.
|
|
67
|
+
*
|
|
68
|
+
* Read on use rather than at construction: a store over a database that
|
|
69
|
+
* opens later only knows its dialect once it is open.
|
|
70
|
+
*/
|
|
71
|
+
private get caps(): Required<Capabilities> {
|
|
72
|
+
return {
|
|
73
|
+
skipLocked: this.store.capabilities?.skipLocked ?? false,
|
|
74
|
+
spaceSeparatedTimestamp:
|
|
75
|
+
this.store.capabilities?.spaceSeparatedTimestamp ?? false,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** The row id a schedule lives under. */
|
|
80
|
+
static scheduleId(name: string): string {
|
|
81
|
+
return `schedule:${name}`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Whether a job is a recurring one, rescheduled rather than deleted. */
|
|
85
|
+
isScheduled(id: string): boolean {
|
|
86
|
+
return this.schedules.has(id);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Make the recurring jobs exist, one row each, and remove the rows of
|
|
91
|
+
* schedules that are no longer configured.
|
|
92
|
+
*
|
|
93
|
+
* One row per schedule, rescheduled on completion, is what makes this
|
|
94
|
+
* safe with several workers: every worker runs this at boot, but a
|
|
95
|
+
* primary key admits one row, and a row is claimed by one worker at a
|
|
96
|
+
* time. A scheduler per worker firing on its own clock would run the
|
|
97
|
+
* job once per worker.
|
|
98
|
+
*
|
|
99
|
+
* An existing row keeps its `run_at`, so a restart does not reset the
|
|
100
|
+
* clock — and does not run everything the moment a deploy lands.
|
|
101
|
+
*/
|
|
102
|
+
async ensureSchedules(
|
|
103
|
+
schedules: Record<string, { everyMs: number; payload?: unknown }>,
|
|
104
|
+
): Promise<void> {
|
|
105
|
+
this.schedules.clear();
|
|
106
|
+
for (const [name, { everyMs, payload }] of Object.entries(schedules)) {
|
|
107
|
+
const id = JobQueue.scheduleId(name);
|
|
108
|
+
this.schedules.set(id, everyMs);
|
|
109
|
+
const exists = await this.store.query(
|
|
110
|
+
`SELECT id FROM ${this.table} WHERE id = ?`,
|
|
111
|
+
[id],
|
|
112
|
+
);
|
|
113
|
+
if (exists.length > 0) continue;
|
|
114
|
+
const now = new Date();
|
|
115
|
+
try {
|
|
116
|
+
await this.store.query(
|
|
117
|
+
`INSERT INTO ${this.table}
|
|
118
|
+
(id, name, payload, run_at, attempts, max_attempts, created_at)
|
|
119
|
+
VALUES (?, ?, ?, ?, 0, ?, ?)`,
|
|
120
|
+
// First run one interval from now, not at boot: a deploy
|
|
121
|
+
// should not fire every schedule at once.
|
|
122
|
+
[id, name, JSON.stringify(payload ?? {}), this.ts(new Date(now.getTime() + everyMs)), 5, this.ts(now)],
|
|
123
|
+
);
|
|
124
|
+
} catch (e) {
|
|
125
|
+
// Another worker inserted it between our SELECT and INSERT.
|
|
126
|
+
const raced = await this.store.query(
|
|
127
|
+
`SELECT id FROM ${this.table} WHERE id = ?`,
|
|
128
|
+
[id],
|
|
129
|
+
);
|
|
130
|
+
if (raced.length === 0) throw e;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const ids = [...this.schedules.keys()];
|
|
134
|
+
await this.store.query(
|
|
135
|
+
ids.length
|
|
136
|
+
? `DELETE FROM ${this.table} WHERE id LIKE 'schedule:%' AND id NOT IN (${ids.map(() => "?").join(", ")})`
|
|
137
|
+
: `DELETE FROM ${this.table} WHERE id LIKE 'schedule:%'`,
|
|
138
|
+
ids,
|
|
139
|
+
);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Move a recurring job to its next run, clearing its attempts. Used on
|
|
144
|
+
* success, and on a failure that has used up its retries: a schedule
|
|
145
|
+
* that failed five times still runs next time.
|
|
146
|
+
*/
|
|
147
|
+
async reschedule(id: string, lastError: string | null = null): Promise<void> {
|
|
148
|
+
const every = this.schedules.get(id);
|
|
149
|
+
if (every === undefined) {
|
|
150
|
+
await this.store.query(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
await this.store.query(
|
|
154
|
+
`UPDATE ${this.table}
|
|
155
|
+
SET run_at = ?, attempts = 0, last_error = ?, claimed_at = NULL, claim_token = NULL
|
|
156
|
+
WHERE id = ?`,
|
|
157
|
+
[this.ts(new Date(Date.now() + every)), lastError?.slice(0, 2000) ?? null, id],
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* A timestamp in the spelling this dialect accepts.
|
|
163
|
+
*
|
|
164
|
+
* The obvious thing — pass `Date.toISOString()` and let the driver
|
|
165
|
+
* serialise it — is what MariaDB rejects: "Incorrect datetime value:
|
|
166
|
+
* '2026-09-27T21:17:22.073Z'". The T and the Z are the problem, not the
|
|
167
|
+
* precision.
|
|
168
|
+
*/
|
|
169
|
+
private ts(date: Date): string {
|
|
170
|
+
const iso = date.toISOString();
|
|
171
|
+
return this.caps.spaceSeparatedTimestamp
|
|
172
|
+
? iso.replace("T", " ").replace("Z", "")
|
|
173
|
+
: iso;
|
|
43
174
|
}
|
|
44
175
|
|
|
45
176
|
async migrate(): Promise<void> {
|
|
46
|
-
|
|
47
|
-
|
|
177
|
+
// SCHEMA names the default table, so a custom `table` option has to be
|
|
178
|
+
// substituted here too. It used to be ignored, which produced the
|
|
179
|
+
// worst kind of failure: migrate() succeeded, then every query failed
|
|
180
|
+
// with "relation does not exist" against a name that looked correct.
|
|
181
|
+
const schema = SCHEMA.replaceAll("sigil_jobs", this.table);
|
|
182
|
+
for (const stmt of schema.split(";").map((s) => s.trim()).filter(Boolean)) {
|
|
183
|
+
try {
|
|
184
|
+
await this.store.query(stmt);
|
|
185
|
+
} catch {
|
|
186
|
+
// Two workers booting together both run CREATE TABLE IF NOT
|
|
187
|
+
// EXISTS, and postgres can fail the loser with a duplicate key
|
|
188
|
+
// on its own catalog — "IF NOT EXISTS" is checked before the
|
|
189
|
+
// insert, not atomically with it. By the time we retry the
|
|
190
|
+
// table exists, so the second attempt is a no-op; a real error
|
|
191
|
+
// fails again and surfaces.
|
|
192
|
+
await new Promise((r) => setTimeout(r, 50 + Math.random() * 100));
|
|
193
|
+
await this.store.query(stmt);
|
|
194
|
+
}
|
|
48
195
|
}
|
|
49
196
|
}
|
|
50
197
|
|
|
@@ -63,7 +210,7 @@ export class JobQueue {
|
|
|
63
210
|
const id = crypto.randomUUID();
|
|
64
211
|
const now = new Date();
|
|
65
212
|
const runAt = options.runAt ?? now;
|
|
66
|
-
await this.
|
|
213
|
+
await this.store.query(
|
|
67
214
|
`INSERT INTO ${this.table}
|
|
68
215
|
(id, name, payload, run_at, attempts, max_attempts, created_at)
|
|
69
216
|
VALUES (?, ?, ?, ?, 0, ?, ?)`,
|
|
@@ -71,9 +218,9 @@ export class JobQueue {
|
|
|
71
218
|
id,
|
|
72
219
|
name,
|
|
73
220
|
JSON.stringify(payload ?? {}),
|
|
74
|
-
|
|
221
|
+
this.ts(runAt),
|
|
75
222
|
options.maxAttempts ?? 5,
|
|
76
|
-
|
|
223
|
+
this.ts(now),
|
|
77
224
|
],
|
|
78
225
|
);
|
|
79
226
|
return id;
|
|
@@ -82,16 +229,27 @@ export class JobQueue {
|
|
|
82
229
|
/**
|
|
83
230
|
* Atomically take up to `limit` due jobs.
|
|
84
231
|
*
|
|
85
|
-
* The safety comes from the claim guard
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
232
|
+
* The safety comes from the claim guard, not from row locking: a row is
|
|
233
|
+
* only claimable when `claimed_at` is NULL or an old claim, and that
|
|
234
|
+
* check happens as part of the write. Two workers issuing this
|
|
235
|
+
* simultaneously both read the same candidate ids, but the second one's
|
|
236
|
+
* UPDATE matches nothing because the first has already set `claimed_at`.
|
|
237
|
+
* So the loser gets an empty set rather than a duplicate job — which is
|
|
238
|
+
* the whole requirement.
|
|
239
|
+
*
|
|
240
|
+
* Which shape that takes depends on the dialect:
|
|
241
|
+
*
|
|
242
|
+
* - With RETURNING, the ids are chosen and claimed in one statement, so
|
|
243
|
+
* there is no window between reading candidates and writing the
|
|
244
|
+
* claim.
|
|
245
|
+
* - Without it — MariaDB — the candidates are read first and then
|
|
246
|
+
* claimed. The window is real but harmless: correctness still comes
|
|
247
|
+
* from the `claimed_at` condition inside the UPDATE, so the loser
|
|
248
|
+
* updates zero rows and gets an empty batch rather than a duplicate.
|
|
249
|
+
* The cost is one extra round trip per claim.
|
|
92
250
|
*
|
|
93
|
-
* `SKIP LOCKED` is appended only when the
|
|
94
|
-
*
|
|
251
|
+
* `SKIP LOCKED` is appended only when the store declares it. It changes
|
|
252
|
+
* contention behaviour, not correctness.
|
|
95
253
|
*
|
|
96
254
|
* The returned rows are NOT in run_at order, and cannot be made to be:
|
|
97
255
|
* SQL does not define the order of an UPDATE's RETURNING clause. The
|
|
@@ -99,26 +257,66 @@ export class JobQueue {
|
|
|
99
257
|
* they come back. Sort in the caller if order matters.
|
|
100
258
|
*/
|
|
101
259
|
async claim(limit: number, staleMs = 60_000): Promise<JobRow[]> {
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
260
|
+
const now = this.ts(new Date());
|
|
261
|
+
const cutoff = this.ts(new Date(Date.now() - staleMs));
|
|
262
|
+
const skip = this.caps.skipLocked ? " FOR UPDATE SKIP LOCKED" : "";
|
|
263
|
+
// One token per claim pass, not per row: a single pass is a single
|
|
264
|
+
// atomic event from the point of view of "did I get these rows", and
|
|
265
|
+
// one token makes the re-read a single equality test.
|
|
266
|
+
const token = `c_${crypto.randomUUID()}`;
|
|
267
|
+
|
|
268
|
+
// The candidate batch is selected in a derived table rather than a
|
|
269
|
+
// plain `WHERE id IN (SELECT ... LIMIT n)` because MariaDB refuses
|
|
270
|
+
// the latter outright: "This version of MariaDB doesn't yet support
|
|
271
|
+
// 'LIMIT & IN/ALL/ANY/SOME subquery'". A derived table joined on id
|
|
272
|
+
// means the same thing and is accepted by all three dialects.
|
|
273
|
+
const candidates = `
|
|
274
|
+
SELECT id FROM (
|
|
107
275
|
SELECT id FROM ${this.table}
|
|
108
276
|
WHERE run_at <= ?
|
|
109
277
|
AND attempts < max_attempts
|
|
110
|
-
AND (
|
|
278
|
+
AND (claim_token IS NULL OR claimed_at < ?)
|
|
111
279
|
ORDER BY run_at
|
|
112
280
|
LIMIT ?
|
|
113
|
-
${
|
|
114
|
-
)
|
|
115
|
-
|
|
116
|
-
|
|
281
|
+
${skip}
|
|
282
|
+
) AS ready`;
|
|
283
|
+
|
|
284
|
+
const columns =
|
|
285
|
+
"id, name, payload, run_at, attempts, max_attempts, last_error, claimed_at, claim_token, created_at";
|
|
286
|
+
|
|
287
|
+
// Claim in one statement. The `claim_token IS NULL OR claimed_at < ?`
|
|
288
|
+
// guard is what makes this exclusive: two workers selecting the same
|
|
289
|
+
// candidates both issue this UPDATE, and the second one matches
|
|
290
|
+
// nothing, because the first has already set claim_token. The loser
|
|
291
|
+
// gets an empty batch rather than a duplicate job.
|
|
292
|
+
//
|
|
293
|
+
// `claim_token` is set to a fresh random value rather than
|
|
294
|
+
// claimed_at's own instant, because MySQL truncates TIMESTAMP to whole
|
|
295
|
+
// seconds. Two workers claiming in the same second would be
|
|
296
|
+
// indistinguishable, the second would match the first's rows, and
|
|
297
|
+
// both would run the same job. Measured, not theorised.
|
|
298
|
+
await this.store.query(
|
|
299
|
+
`UPDATE ${this.table}
|
|
300
|
+
SET claimed_at = ?, claim_token = ?, attempts = attempts + 1
|
|
301
|
+
WHERE id IN (${candidates})
|
|
302
|
+
AND attempts < max_attempts
|
|
303
|
+
AND (claim_token IS NULL OR claimed_at < ?)`,
|
|
304
|
+
[now, token, now, cutoff, limit, cutoff],
|
|
305
|
+
);
|
|
306
|
+
|
|
307
|
+
// Read back only the rows this pass won, by token. A dialect without
|
|
308
|
+
// RETURNING cannot tell us which ids the UPDATE matched — MariaDB
|
|
309
|
+
// returns [] even when it updated two rows — so the token is what
|
|
310
|
+
// makes the answer knowable, and it works identically on all three.
|
|
311
|
+
return this.store.query<JobRow>(
|
|
312
|
+
`SELECT ${columns} FROM ${this.table} WHERE claim_token = ?`,
|
|
313
|
+
[token],
|
|
117
314
|
);
|
|
118
315
|
}
|
|
119
316
|
|
|
120
317
|
async complete(id: string): Promise<void> {
|
|
121
|
-
|
|
318
|
+
if (this.isScheduled(id)) return this.reschedule(id);
|
|
319
|
+
await this.store.query(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
|
|
122
320
|
}
|
|
123
321
|
|
|
124
322
|
/**
|
|
@@ -131,30 +329,43 @@ export class JobQueue {
|
|
|
131
329
|
*/
|
|
132
330
|
async fail(id: string, error: unknown, backoffMs = 30_000): Promise<void> {
|
|
133
331
|
const message = error instanceof Error ? error.message : String(error);
|
|
134
|
-
const nextRun = new Date(Date.now() + backoffMs)
|
|
135
|
-
await this.
|
|
332
|
+
const nextRun = this.ts(new Date(Date.now() + backoffMs));
|
|
333
|
+
await this.store.query(
|
|
136
334
|
`UPDATE ${this.table}
|
|
137
|
-
SET last_error = ?, claimed_at = NULL, run_at = ?
|
|
335
|
+
SET last_error = ?, claimed_at = NULL, claim_token = NULL, run_at = ?
|
|
138
336
|
WHERE id = ?`,
|
|
139
337
|
[message.slice(0, 2000), nextRun, id],
|
|
140
338
|
);
|
|
141
339
|
}
|
|
142
340
|
|
|
143
|
-
/**
|
|
341
|
+
/**
|
|
342
|
+
* Put a job back on the queue immediately, for manual recovery.
|
|
343
|
+
*
|
|
344
|
+
* Reports whether the row existed. That used to come from
|
|
345
|
+
* `UPDATE ... RETURNING id`, which no dialect other than postgres and
|
|
346
|
+
* sqlite has, so a store without RETURNING has to check first. One extra
|
|
347
|
+
* query in exchange for an honest return value on every dialect —
|
|
348
|
+
* returning `true` unconditionally would tell a caller it had recovered a
|
|
349
|
+
* job that does not exist.
|
|
350
|
+
*/
|
|
144
351
|
async retry(id: string): Promise<boolean> {
|
|
145
|
-
const
|
|
352
|
+
const found = await this.store.query<{ id: string }>(
|
|
353
|
+
`SELECT id FROM ${this.table} WHERE id = ?`,
|
|
354
|
+
[id],
|
|
355
|
+
);
|
|
356
|
+
if (found.length === 0) return false;
|
|
357
|
+
await this.store.query(
|
|
146
358
|
`UPDATE ${this.table}
|
|
147
|
-
SET attempts = 0, last_error = NULL, claimed_at = NULL, run_at = ?
|
|
148
|
-
WHERE id =
|
|
149
|
-
|
|
150
|
-
[new Date().toISOString(), id],
|
|
359
|
+
SET attempts = 0, last_error = NULL, claimed_at = NULL, claim_token = NULL, run_at = ?
|
|
360
|
+
WHERE id = ?`,
|
|
361
|
+
[this.ts(new Date()), id],
|
|
151
362
|
);
|
|
152
|
-
return
|
|
363
|
+
return true;
|
|
153
364
|
}
|
|
154
365
|
|
|
155
366
|
async list(limit = 20): Promise<JobRow[]> {
|
|
156
|
-
return this.
|
|
157
|
-
`SELECT id, name, payload, run_at, attempts, max_attempts, last_error, claimed_at, created_at
|
|
367
|
+
return this.store.query<JobRow>(
|
|
368
|
+
`SELECT ${"id, name, payload, run_at, attempts, max_attempts, last_error, claimed_at, claim_token, created_at"}
|
|
158
369
|
FROM ${this.table}
|
|
159
370
|
ORDER BY created_at DESC
|
|
160
371
|
LIMIT ?`,
|
|
@@ -163,7 +374,7 @@ export class JobQueue {
|
|
|
163
374
|
}
|
|
164
375
|
|
|
165
376
|
async stats(): Promise<JobStats> {
|
|
166
|
-
const rows = await this.
|
|
377
|
+
const rows = await this.store.query<{ bucket: string; count: number }>(
|
|
167
378
|
`SELECT
|
|
168
379
|
CASE
|
|
169
380
|
WHEN attempts >= max_attempts THEN 'failed'
|
package/store-bun.ts
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { database } from "@sigil-dev/plugin-sql";
|
|
2
|
+
import type { Capabilities, JobStore } from "./store.ts";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* A JobStore over `Bun.SQL`, with the dialect capabilities detected from the
|
|
6
|
+
* DSN.
|
|
7
|
+
*
|
|
8
|
+
* The queue is dialect-agnostic on purpose. This is where the dialect
|
|
9
|
+
* knowledge lives, in one place, from a value the user already gave us.
|
|
10
|
+
*
|
|
11
|
+
* The three capabilities are read off the URL scheme, not probed at runtime,
|
|
12
|
+
* because probing means running queries at boot against a database that may
|
|
13
|
+
* not be up yet — and this plugin deliberately does not open a connection
|
|
14
|
+
* until the queue is used. A wrong guess is loud here: a missing
|
|
15
|
+
* `spaceSeparatedTimestamp` on MySQL fails the first INSERT with
|
|
16
|
+
* "Incorrect datetime value", not silently.
|
|
17
|
+
*/
|
|
18
|
+
export function bunSqlStore(url: string, extra: Partial<Capabilities> = {}): JobStore {
|
|
19
|
+
return bunStore(new Bun.SQL({ url, max: 1 }), url, extra);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* A JobStore over a `Bun.SQL` the caller already owns.
|
|
24
|
+
*
|
|
25
|
+
* This is the bridge that lets the queue run on the connection the SQL plugin
|
|
26
|
+
* opened, rather than a second pool. `bunSqlStore(url)` is for when the queue
|
|
27
|
+
* is the only thing using the database; this is for when an app already
|
|
28
|
+
* registered `@sigil-dev/plugin-sql` and the budget should be spent once:
|
|
29
|
+
*
|
|
30
|
+
* import { adapter } from "@sigil-dev/plugin-sql/bun";
|
|
31
|
+
* import { jobs, bunStore } from "@sigil-dev/plugin-jobs";
|
|
32
|
+
*
|
|
33
|
+
* const db = adapter(url, { maxTotal: 12 });
|
|
34
|
+
* const queue = jobs({
|
|
35
|
+
* // read from the plugin's own client, don't open a second pool
|
|
36
|
+
* store: (services) => bunStore(services.db as Bun.SQL),
|
|
37
|
+
* });
|
|
38
|
+
*
|
|
39
|
+
* The dialect is read from the client, which already parsed the DSN, so
|
|
40
|
+
* `url` is only a fallback for a client that does not say. Through
|
|
41
|
+
* plugin-sql's `locals.db`, a job enqueued during a `perRequestTx` request
|
|
42
|
+
* is part of that request's transaction and rolls back with it.
|
|
43
|
+
*
|
|
44
|
+
* The placeholder translation is the same one `bunSqlStore` uses, because
|
|
45
|
+
* the queue speaks `?` and postgres wants $1. It lives in this function
|
|
46
|
+
* rather than in either caller so the two cannot drift.
|
|
47
|
+
*/
|
|
48
|
+
type Dialect = "postgres" | "mysql" | "sqlite";
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Which database a `Bun.SQL` client talks to.
|
|
52
|
+
*
|
|
53
|
+
* Asked of the client first: Bun.SQL already parsed the DSN, including the
|
|
54
|
+
* spellings a scheme check misses — `postgresql://`, which Neon and Supabase
|
|
55
|
+
* hand out, and `mariadb://`. Reading the scheme alone sent a `postgresql://`
|
|
56
|
+
* client `?` placeholders.
|
|
57
|
+
*/
|
|
58
|
+
function dialectOf(db: Bun.SQL, url?: string): Dialect {
|
|
59
|
+
const fromClient = (db as unknown as { options?: { adapter?: string } })
|
|
60
|
+
.options?.adapter;
|
|
61
|
+
const raw = (fromClient ?? url?.split(":")[0] ?? "").toLowerCase();
|
|
62
|
+
if (raw === "postgres" || raw === "postgresql" || raw === "pg") return "postgres";
|
|
63
|
+
if (raw === "mysql" || raw === "mariadb") return "mysql";
|
|
64
|
+
// A bare path is a SQLite file to Bun.SQL, and so it is here.
|
|
65
|
+
return "sqlite";
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function bunStore(
|
|
69
|
+
db: Bun.SQL,
|
|
70
|
+
url?: string,
|
|
71
|
+
extra: Partial<Capabilities> = {},
|
|
72
|
+
): JobStore {
|
|
73
|
+
const dialect = dialectOf(db, url);
|
|
74
|
+
const capabilities: Capabilities = {
|
|
75
|
+
skipLocked: dialect === "postgres" || dialect === "mysql",
|
|
76
|
+
spaceSeparatedTimestamp: dialect === "mysql",
|
|
77
|
+
...extra,
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
// Two things have to be reconciled for the queue's `?` placeholders:
|
|
81
|
+
// postgres wants $1, $2, and mysql wants ? (so it passes through).
|
|
82
|
+
// Doing it here means queue.ts contains no dialect knowledge at all.
|
|
83
|
+
const placeholder = (text: string): string => {
|
|
84
|
+
if (dialect !== "postgres") return text;
|
|
85
|
+
let i = 0;
|
|
86
|
+
return text.replace(/\?/g, () => `$${++i}`);
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
return {
|
|
90
|
+
capabilities,
|
|
91
|
+
async query<T>(text: string, params: unknown[] = []): Promise<T[]> {
|
|
92
|
+
// unsafe() is required rather than query(): the queue builds its
|
|
93
|
+
// SQL as strings, so the values have to be bound separately or
|
|
94
|
+
// they are an injection waiting to happen.
|
|
95
|
+
return (await db.unsafe(placeholder(text), params as never[])) as T[];
|
|
96
|
+
},
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* A JobStore over a database `@sigil-dev/plugin-sql` registered, by name.
|
|
102
|
+
*
|
|
103
|
+
* This is what `jobs()` uses when given neither `sql` nor `store`, and it
|
|
104
|
+
* removes the three things the `store` bridge made you get right:
|
|
105
|
+
*
|
|
106
|
+
* - Order. It looks the database up when a query runs, so the jobs plugin
|
|
107
|
+
* can sit anywhere in the plugins array.
|
|
108
|
+
* - The CLI. `sigil jobs work` has no server to publish services, but it
|
|
109
|
+
* has the config, which registered the database; the command opens it.
|
|
110
|
+
* - The request transaction. Every query goes through `db()`, so a job
|
|
111
|
+
* enqueued during a `perRequestTx` request is part of it, and one
|
|
112
|
+
* enqueued anywhere else runs on the pool.
|
|
113
|
+
*/
|
|
114
|
+
export function databaseStore(name = "db", extra: Partial<Capabilities> = {}): JobStore {
|
|
115
|
+
const db = database<Bun.SQL>(name);
|
|
116
|
+
// The dialect is the client's, and there is no client until the database
|
|
117
|
+
// opens — so it is read on first use and kept.
|
|
118
|
+
let dialect: ReturnType<typeof dialectOf> | undefined;
|
|
119
|
+
const dialectNow = () => (dialect ??= dialectOf(db.pool()));
|
|
120
|
+
|
|
121
|
+
return {
|
|
122
|
+
get capabilities(): Capabilities {
|
|
123
|
+
const d = dialectNow();
|
|
124
|
+
return {
|
|
125
|
+
skipLocked: d === "postgres" || d === "mysql",
|
|
126
|
+
spaceSeparatedTimestamp: d === "mysql",
|
|
127
|
+
...extra,
|
|
128
|
+
};
|
|
129
|
+
},
|
|
130
|
+
async query<T>(text: string, params: unknown[] = []): Promise<T[]> {
|
|
131
|
+
let i = 0;
|
|
132
|
+
const sql = dialectNow() === "postgres" ? text.replace(/\?/g, () => `$${++i}`) : text;
|
|
133
|
+
const client = db.db() as Bun.SQL;
|
|
134
|
+
return (await client.unsafe(sql, params as never[])) as T[];
|
|
135
|
+
},
|
|
136
|
+
};
|
|
137
|
+
}
|
package/store-drizzle.ts
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { JobStore } from "./store.ts";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A JobStore for a project that already uses Drizzle.
|
|
5
|
+
*
|
|
6
|
+
* ## Read this before you use it, because the honest answer is "reach past
|
|
7
|
+
* ## drizzle"
|
|
8
|
+
*
|
|
9
|
+
* Measured against drizzle-orm 0.45.3 with a real Bun.SQL client, real
|
|
10
|
+
* MariaDB and real postgres:
|
|
11
|
+
*
|
|
12
|
+
* - `db.execute("SELECT ? as n", [7])` does NOT bind. The `?` reaches
|
|
13
|
+
* MariaDB as literal SQL and the server returns a syntax error. Drizzle's
|
|
14
|
+
* `execute` takes a `SQLWrapper`, not a parameter array, and passing one
|
|
15
|
+
* is silently ignored.
|
|
16
|
+
* - `db.execute(dsql`SELECT ${3}`)` works on postgres and fails on MySQL
|
|
17
|
+
* with "Unknown column '$1' in 'SELECT'". The bun-sql driver emits $1
|
|
18
|
+
* placeholders unconditionally, so the same call, on the same db object,
|
|
19
|
+
* is correct on one database and broken on another. That is a drizzle
|
|
20
|
+
* bug, and it is why the obvious implementation of this adapter does not
|
|
21
|
+
* work.
|
|
22
|
+
* - `client.unsafe("SELECT ? as n", [7])` on the underlying Bun.SQL works.
|
|
23
|
+
*
|
|
24
|
+
* So the adapter is given a function to run SQL, not a Drizzle db. Passing
|
|
25
|
+
* `db` and hoping would have produced a store that works on postgres, fails
|
|
26
|
+
* on MySQL, and fails the moment a query has a parameter — which is every
|
|
27
|
+
* query the queue runs.
|
|
28
|
+
*
|
|
29
|
+
* With a real Drizzle project, the connection is already a `Bun.SQL` (or a
|
|
30
|
+
* `pg` Pool), so pass that. Drizzle is not in the path here, and the reason
|
|
31
|
+
* is written above rather than left for someone to rediscover.
|
|
32
|
+
*
|
|
33
|
+
* If you want the queue to be a first-class Drizzle table instead — with
|
|
34
|
+
* `db.select()`, `db.insert()`, `db.update()` rather than SQL text — that is
|
|
35
|
+
* a real change to JobQueue, not an adapter: every query would be written
|
|
36
|
+
* twice and the claim guard, which is what makes the queue correct, would
|
|
37
|
+
* have to be proven in both paths.
|
|
38
|
+
*/
|
|
39
|
+
export interface SqlRunner {
|
|
40
|
+
/** Run SQL with bound parameters and return rows. */
|
|
41
|
+
run: (query: string, params: unknown[]) => Promise<unknown>;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function drizzleStore(
|
|
45
|
+
run: SqlRunner,
|
|
46
|
+
dialect: "postgres" | "mysql" | "sqlite",
|
|
47
|
+
): JobStore {
|
|
48
|
+
const capabilities = {
|
|
49
|
+
skipLocked: dialect === "postgres" || dialect === "mysql",
|
|
50
|
+
spaceSeparatedTimestamp: dialect === "mysql",
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
return {
|
|
54
|
+
capabilities,
|
|
55
|
+
async query<T>(text: string, params: unknown[] = []): Promise<T[]> {
|
|
56
|
+
// postgres wants $1..$n, MySQL and sqlite want ?. The queue emits
|
|
57
|
+
// `?` because that is the wider convention, so the rewrite happens
|
|
58
|
+
// here — once, for one dialect, rather than in every caller.
|
|
59
|
+
let i = 0;
|
|
60
|
+
const stmt =
|
|
61
|
+
dialect === "postgres" ? text.replace(/\?/g, () => `$${++i}`) : text;
|
|
62
|
+
const result = await run.run(stmt, params);
|
|
63
|
+
// Measured return shapes: a plain row array for both execute(string)
|
|
64
|
+
// and dsql.raw. The object cases below are for the node-postgres and
|
|
65
|
+
// neon drivers, which wrap rows — a DrizzleLike from those would
|
|
66
|
+
// otherwise return { rows: [...] } to the queue and every query
|
|
67
|
+
// would look like it matched nothing.
|
|
68
|
+
if (Array.isArray(result)) return result as T[];
|
|
69
|
+
if (result && typeof result === "object") {
|
|
70
|
+
const r = result as Record<string, unknown>;
|
|
71
|
+
if (Array.isArray(r.rows)) return r.rows as T[];
|
|
72
|
+
}
|
|
73
|
+
return [];
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* A runner over a `Bun.SQL` client, which is what a Drizzle bun-sql project
|
|
80
|
+
* already holds. This is the supported entry point.
|
|
81
|
+
*/
|
|
82
|
+
export function bunSqlRunner(client: {
|
|
83
|
+
unsafe(text: string, params?: unknown[]): Promise<unknown>;
|
|
84
|
+
}): SqlRunner {
|
|
85
|
+
return { run: (text, params) => client.unsafe(text, params) };
|
|
86
|
+
}
|