@sigil-dev/plugin-jobs 0.9.12 → 0.9.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +105 -61
- package/index.ts +278 -53
- package/package.json +14 -4
- package/queue.ts +254 -43
- package/store-bun.ts +137 -0
- package/store-drizzle.ts +86 -0
- package/store.ts +117 -0
- package/worker.ts +13 -3
- package/sql.ts +0 -73
package/store.ts
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The interface a job store has to satisfy, and the dialect capabilities it
|
|
3
|
+
* declares.
|
|
4
|
+
*
|
|
5
|
+
* The previous version took `query(sql, params) filling ? placeholders` and
|
|
6
|
+
* hoped. That was wrong against MariaDB in two separate ways, both found by
|
|
7
|
+
* running the queue against a real MariaDB rather than by reading it:
|
|
8
|
+
*
|
|
9
|
+
* UPDATE ... RETURNING id syntax error. MariaDB has no RETURNING, on
|
|
10
|
+
* an UPDATE or anywhere else.
|
|
11
|
+
* '2026-09-27T21:17:22.073Z' Incorrect datetime value for TIMESTAMP.
|
|
12
|
+
*
|
|
13
|
+
* Neither was a bug in the tests, because there was no test that touched
|
|
14
|
+
* MySQL. So the queue compiled, passed 21 tests, and could not claim a
|
|
15
|
+
* single job on one of the three databases it documents support for.
|
|
16
|
+
*
|
|
17
|
+
* The fix is to stop inferring. A store declares what its dialect can do,
|
|
18
|
+
* and the queue picks a strategy per capability instead of per guess:
|
|
19
|
+
*
|
|
20
|
+
* timestamp How a timestamp is spelled. Postgres and sqlite accept
|
|
21
|
+
* ISO 8601 with a T and a Z; MySQL rejects both and wants
|
|
22
|
+
* "YYYY-MM-DD HH:MM:SS.mmm". Passing an ISO string at a
|
|
23
|
+
* TIMESTAMP column is the single most portable-looking
|
|
24
|
+
* thing you can do and it does not work on MySQL.
|
|
25
|
+
* skipLocked `FOR UPDATE SKIP LOCKED` exists on postgres and MySQL.
|
|
26
|
+
* sqlite has no row locking at all. This only changes
|
|
27
|
+
* throughput — a worker that cannot skip will block behind
|
|
28
|
+
* rows another worker holds.
|
|
29
|
+
*
|
|
30
|
+
* A store that declares nothing is assumed to be the weakest thing that
|
|
31
|
+
* exists: no RETURNING, ISO timestamps, no SKIP LOCKED. Wrong there is
|
|
32
|
+
* loud rather than silent, which is the point.
|
|
33
|
+
*/
|
|
34
|
+
export interface JobStore {
|
|
35
|
+
/** Run a statement and return rows. */
|
|
36
|
+
query<T = Record<string, unknown>>(
|
|
37
|
+
sql: string,
|
|
38
|
+
params?: unknown[],
|
|
39
|
+
): Promise<T[]>;
|
|
40
|
+
|
|
41
|
+
/** What this dialect can do. Every field is optional and defaults to false. */
|
|
42
|
+
readonly capabilities?: Capabilities;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface Capabilities {
|
|
46
|
+
/** `FOR UPDATE SKIP LOCKED` is valid. @default false */
|
|
47
|
+
readonly skipLocked?: boolean;
|
|
48
|
+
/**
|
|
49
|
+
* TIMESTAMP columns need "YYYY-MM-DD HH:MM:SS" rather than ISO 8601.
|
|
50
|
+
* @default false
|
|
51
|
+
*/
|
|
52
|
+
readonly spaceSeparatedTimestamp?: boolean;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface JobRow {
|
|
56
|
+
id: string;
|
|
57
|
+
name: string;
|
|
58
|
+
payload: string;
|
|
59
|
+
run_at: string;
|
|
60
|
+
attempts: number;
|
|
61
|
+
max_attempts: number;
|
|
62
|
+
last_error: string | null;
|
|
63
|
+
claimed_at: string | null;
|
|
64
|
+
created_at: string;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** A job as it exists before it is claimed. */
|
|
68
|
+
export interface EnqueueOptions {
|
|
69
|
+
/** Delay execution. Defaults to now. */
|
|
70
|
+
runAt?: Date;
|
|
71
|
+
/** Give up after this many attempts. Defaults to 5. */
|
|
72
|
+
maxAttempts?: number;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export interface JobStats {
|
|
76
|
+
pending: number;
|
|
77
|
+
claimed: number;
|
|
78
|
+
failed: number;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** What a handler knows about the run it is in. */
|
|
82
|
+
export interface JobContext {
|
|
83
|
+
/** The job's id. Stable across retries, so it can key idempotency. */
|
|
84
|
+
id: string;
|
|
85
|
+
name: string;
|
|
86
|
+
/** 1 on the first run, 2 on the first retry, and so on. */
|
|
87
|
+
attempt: number;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* A job handler. The payload type is what `enqueue` checks against, so
|
|
92
|
+
* annotate it and the call sites are typed:
|
|
93
|
+
*
|
|
94
|
+
* ```ts
|
|
95
|
+
* handlers: { sendMail: async (p: { to: string }) => { ... } }
|
|
96
|
+
* jobs.enqueue("sendMail", { to: "a@b.c" }) // checked
|
|
97
|
+
* ```
|
|
98
|
+
*/
|
|
99
|
+
// biome-ignore lint/suspicious/noExplicitAny: the default accepts any handler, and a narrower one is inferred per name
|
|
100
|
+
export type JobHandler<P = any> = (payload: P, ctx: JobContext) => Promise<void> | void;
|
|
101
|
+
|
|
102
|
+
/** The payload a handler expects. */
|
|
103
|
+
export type PayloadOf<H> = H extends (payload: infer P, ...rest: never[]) => unknown ? P : never;
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* How often a schedule runs: milliseconds, or `"30s"`, `"5m"`, `"1h"`,
|
|
107
|
+
* `"1d"`.
|
|
108
|
+
*/
|
|
109
|
+
export type Every = number | `${number}${"ms" | "s" | "m" | "h" | "d"}`;
|
|
110
|
+
|
|
111
|
+
export function everyMs(every: Every): number {
|
|
112
|
+
if (typeof every === "number") return every;
|
|
113
|
+
const m = /^(\d+(?:\.\d+)?)(ms|s|m|h|d)$/.exec(every);
|
|
114
|
+
if (!m) throw new Error(`plugin-jobs: cannot read "${every}" as an interval`);
|
|
115
|
+
const unit = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 }[m[2] as "ms"];
|
|
116
|
+
return Number(m[1]) * unit;
|
|
117
|
+
}
|
package/worker.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { JobQueue } from "./queue.ts";
|
|
2
|
-
import type { JobHandler } from "./
|
|
2
|
+
import type { JobHandler } from "./store.ts";
|
|
3
3
|
|
|
4
4
|
export interface WorkerOptions {
|
|
5
5
|
/** How many jobs to claim per pass. */
|
|
@@ -69,10 +69,20 @@ export class JobWorker {
|
|
|
69
69
|
} catch {
|
|
70
70
|
throw new Error(`payload is not valid JSON: ${job.payload}`);
|
|
71
71
|
}
|
|
72
|
-
await handler(payload);
|
|
72
|
+
await handler(payload, { id: job.id, name: job.name, attempt: job.attempts });
|
|
73
73
|
await this.queue.complete(job.id);
|
|
74
74
|
} catch (error) {
|
|
75
|
-
|
|
75
|
+
// A recurring job that has used up its retries moves on to its
|
|
76
|
+
// next run instead of parking as failed: one bad afternoon
|
|
77
|
+
// upstream should not stop a schedule for good.
|
|
78
|
+
if (this.queue.isScheduled(job.id) && job.attempts >= job.max_attempts) {
|
|
79
|
+
await this.queue.reschedule(
|
|
80
|
+
job.id,
|
|
81
|
+
error instanceof Error ? error.message : String(error),
|
|
82
|
+
);
|
|
83
|
+
} else {
|
|
84
|
+
await this.queue.fail(job.id, error, this.backoffMs);
|
|
85
|
+
}
|
|
76
86
|
}
|
|
77
87
|
}
|
|
78
88
|
return jobs.length;
|
package/sql.ts
DELETED
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The SQL contract this plugin speaks.
|
|
3
|
-
*
|
|
4
|
-
* This is deliberately small: three methods, and the caller supplies the
|
|
5
|
-
* client. The plugin does not import a database library, does not open a
|
|
6
|
-
* connection, and does not read a connection string out of the
|
|
7
|
-
* environment. Whoever holds the connection already has it, and making
|
|
8
|
-
* them hand it over is what lets this run on whatever they already use.
|
|
9
|
-
*
|
|
10
|
-
* "SQL" here means something narrower than it sounds. The SQL this
|
|
11
|
-
* generates uses `?` placeholders and returns rows as objects, which Bun's
|
|
12
|
-
* built-in `Bun.sql` satisfies directly and most other drivers can be
|
|
13
|
-
* adapted to with a wrapper. What it does NOT do is hide dialect
|
|
14
|
-
* differences, and it cannot: `FOR UPDATE SKIP LOCKED` is how Postgres and
|
|
15
|
-
* MySQL take a row, and SQLite has no equivalent.
|
|
16
|
-
*
|
|
17
|
-
* So the concurrency guarantee is split, and the split is the point:
|
|
18
|
-
*
|
|
19
|
-
* - The claim is an atomic conditional UPDATE. Every backend that can run
|
|
20
|
-
* `UPDATE ... WHERE id IN (SELECT ... LIMIT n)` gets a correct claim
|
|
21
|
-
* with no dialect-specific locking at all, because the row's own
|
|
22
|
-
* `claimed_at` value is what makes the update exclusive. Two workers
|
|
23
|
-
* can race and exactly one wins each row.
|
|
24
|
-
* - `SKIP LOCKED` is used only when the driver says it supports it, to
|
|
25
|
-
* stop a worker from blocking behind rows another worker is holding.
|
|
26
|
-
* Without it, correctness is unchanged and throughput drops.
|
|
27
|
-
*
|
|
28
|
-
* That is why `supportsSkipLocked` exists and why the default is false.
|
|
29
|
-
* Claiming portability we have not got is worse than documenting the
|
|
30
|
-
* performance cliff.
|
|
31
|
-
*/
|
|
32
|
-
export interface SqlExecutor {
|
|
33
|
-
/** Run a query and return rows. `params` fills `?` placeholders. */
|
|
34
|
-
query<T = Record<string, unknown>>(
|
|
35
|
-
sql: string,
|
|
36
|
-
params?: unknown[],
|
|
37
|
-
): Promise<T[]>;
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Whether `FOR UPDATE SKIP LOCKED` is available. Only improves
|
|
41
|
-
* throughput; the claim is correct either way.
|
|
42
|
-
* @default false
|
|
43
|
-
*/
|
|
44
|
-
readonly supportsSkipLocked?: boolean;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
export interface JobRow {
|
|
48
|
-
id: string;
|
|
49
|
-
name: string;
|
|
50
|
-
payload: string;
|
|
51
|
-
run_at: string;
|
|
52
|
-
attempts: number;
|
|
53
|
-
max_attempts: number;
|
|
54
|
-
last_error: string | null;
|
|
55
|
-
claimed_at: string | null;
|
|
56
|
-
created_at: string;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/** A job as it exists before it is claimed. */
|
|
60
|
-
export interface EnqueueOptions {
|
|
61
|
-
/** Delay execution. Defaults to now. */
|
|
62
|
-
runAt?: Date;
|
|
63
|
-
/** Give up after this many attempts. Defaults to 5. */
|
|
64
|
-
maxAttempts?: number;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
export interface JobStats {
|
|
68
|
-
pending: number;
|
|
69
|
-
claimed: number;
|
|
70
|
-
failed: number;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
export type JobHandler = (payload: unknown) => Promise<void> | void;
|