@sigil-dev/plugin-jobs 0.9.28 → 0.9.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/store.ts CHANGED
@@ -1,75 +1,35 @@
1
+ import type { Duration } from "./duration.ts";
2
+
1
3
  /**
2
- * The interface a job store has to satisfy, and the dialect capabilities it
3
- * declares.
4
- *
5
- * The previous version took `query(sql, params) filling ? placeholders` and
6
- * hoped. That was wrong against MariaDB in two separate ways, both found by
7
- * running the queue against a real MariaDB rather than by reading it:
8
- *
9
- * UPDATE ... RETURNING id syntax error. MariaDB has no RETURNING, on
10
- * an UPDATE or anywhere else.
11
- * '2026-09-27T21:17:22.073Z' Incorrect datetime value for TIMESTAMP.
12
- *
13
- * Neither was a bug in the tests, because there was no test that touched
14
- * MySQL. So the queue compiled, passed 21 tests, and could not claim a
15
- * single job on one of the three databases it documents support for.
16
- *
17
- * The fix is to stop inferring. A store declares what its dialect can do,
18
- * and the queue picks a strategy per capability instead of per guess:
4
+ * The types a queue, a worker and a handler share.
19
5
  *
20
- * timestamp How a timestamp is spelled. Postgres and sqlite accept
21
- * ISO 8601 with a T and a Z; MySQL rejects both and wants
22
- * "YYYY-MM-DD HH:MM:SS.mmm". Passing an ISO string at a
23
- * TIMESTAMP column is the single most portable-looking
24
- * thing you can do and it does not work on MySQL.
25
- * skipLocked `FOR UPDATE SKIP LOCKED` exists on postgres and MySQL.
26
- * sqlite has no row locking at all. This only changes
27
- * throughput — a worker that cannot skip will block behind
28
- * rows another worker holds.
29
- *
30
- * A store that declares nothing is assumed to be the weakest thing that
31
- * exists: no RETURNING, ISO timestamps, no SKIP LOCKED. Wrong there is
32
- * loud rather than silent, which is the point.
6
+ * There used to be a `JobStore` interface here, so the queue could run on
7
+ * any client that could run a query. The queue is a tome table now, and
8
+ * tome knows its dialect, so there is nothing left to abstract over.
33
9
  */
34
- export interface JobStore {
35
- /** Run a statement and return rows. */
36
- query<T = Record<string, unknown>>(
37
- sql: string,
38
- params?: unknown[],
39
- ): Promise<T[]>;
40
-
41
- /** What this dialect can do. Every field is optional and defaults to false. */
42
- readonly capabilities?: Capabilities;
43
- }
44
-
45
- export interface Capabilities {
46
- /** `FOR UPDATE SKIP LOCKED` is valid. @default false */
47
- readonly skipLocked?: boolean;
48
- /**
49
- * TIMESTAMP columns need "YYYY-MM-DD HH:MM:SS" rather than ISO 8601.
50
- * @default false
51
- */
52
- readonly spaceSeparatedTimestamp?: boolean;
53
- }
54
10
 
11
+ /** A row of the jobs table. */
55
12
  export interface JobRow {
56
13
  id: string;
57
14
  name: string;
15
+ /** JSON, as text. The worker parses it, so a bad one fails only its own job. */
58
16
  payload: string;
59
- run_at: string;
17
+ runAt: Date;
18
+ /** How many times it has been claimed, the current claim included. */
60
19
  attempts: number;
61
- max_attempts: number;
62
- last_error: string | null;
63
- claimed_at: string | null;
64
- created_at: string;
20
+ maxAttempts: number;
21
+ lastError: string | null;
22
+ claimedAt: Date | null;
23
+ claimToken: string | null;
24
+ createdAt: Date;
65
25
  }
66
26
 
67
- /** A job as it exists before it is claimed. */
27
+ /** How `job.enqueue` schedules a run. */
68
28
  export interface EnqueueOptions {
69
- /** Delay execution. Defaults to now. */
70
- runAt?: Date;
71
- /** Give up after this many attempts. Defaults to 5. */
72
- maxAttempts?: number;
29
+ /** Run after this long: `"30s"`, `"1h 30m"`, or milliseconds. */
30
+ in?: Duration;
31
+ /** Run at this time. */
32
+ at?: Date;
73
33
  }
74
34
 
75
35
  export interface JobStats {
@@ -86,32 +46,3 @@ export interface JobContext {
86
46
  /** 1 on the first run, 2 on the first retry, and so on. */
87
47
  attempt: number;
88
48
  }
89
-
90
- /**
91
- * A job handler. The payload type is what `enqueue` checks against, so
92
- * annotate it and the call sites are typed:
93
- *
94
- * ```ts
95
- * handlers: { sendMail: async (p: { to: string }) => { ... } }
96
- * jobs.enqueue("sendMail", { to: "a@b.c" }) // checked
97
- * ```
98
- */
99
- // biome-ignore lint/suspicious/noExplicitAny: the default accepts any handler, and a narrower one is inferred per name
100
- export type JobHandler<P = any> = (payload: P, ctx: JobContext) => Promise<void> | void;
101
-
102
- /** The payload a handler expects. */
103
- export type PayloadOf<H> = H extends (payload: infer P, ...rest: never[]) => unknown ? P : never;
104
-
105
- /**
106
- * How often a schedule runs: milliseconds, or `"30s"`, `"5m"`, `"1h"`,
107
- * `"1d"`.
108
- */
109
- export type Every = number | `${number}${"ms" | "s" | "m" | "h" | "d"}`;
110
-
111
- export function everyMs(every: Every): number {
112
- if (typeof every === "number") return every;
113
- const m = /^(\d+(?:\.\d+)?)(ms|s|m|h|d)$/.exec(every);
114
- if (!m) throw new Error(`plugin-jobs: cannot read "${every}" as an interval`);
115
- const unit = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 }[m[2] as "ms"];
116
- return Number(m[1]) * unit;
117
- }
package/worker.ts CHANGED
@@ -1,5 +1,5 @@
1
+ import { checkPayload, type Job } from "./define.ts";
1
2
  import type { JobQueue } from "./queue.ts";
2
- import type { JobHandler } from "./store.ts";
3
3
 
4
4
  export interface WorkerOptions {
5
5
  /** How many jobs to claim per pass. */
@@ -12,12 +12,10 @@ export interface WorkerOptions {
12
12
  * jobs claimed forever.
13
13
  */
14
14
  staleMs?: number;
15
- /** Backoff after a handler throws. */
16
- backoffMs?: number;
17
15
  }
18
16
 
19
17
  /**
20
- * Polls a JobQueue and runs handlers.
18
+ * Polls a JobQueue and runs jobs.
21
19
  *
22
20
  * Nothing here is clever: claim a batch, run each, complete or fail. The
23
21
  * parts that matter are the ones that are easy to get wrong and hard to
@@ -26,6 +24,8 @@ export interface WorkerOptions {
26
24
  * - A throwing handler is caught per job, so one bad job does not stop the
27
25
  * batch. If it did, a single poisonous job would block every job behind
28
26
  * it, forever.
27
+ * - A payload the job no longer accepts (queued by an older deploy) is
28
+ * failed for good with the reason: another try cannot fix it.
29
29
  * - SIGTERM drains before exiting, so a deploy does not abandon claimed
30
30
  * work. The stale-claim timeout is the backstop when a worker is killed
31
31
  * without getting to drain at all.
@@ -34,68 +34,77 @@ export class JobWorker {
34
34
  private readonly batchSize: number;
35
35
  private readonly intervalMs: number;
36
36
  private readonly staleMs: number;
37
- private readonly backoffMs: number;
38
37
  private stopping = false;
39
38
  private running = false;
40
39
 
41
40
  constructor(
42
41
  private readonly queue: JobQueue,
43
- private readonly handlers: Record<string, JobHandler>,
42
+ private readonly jobs: Readonly<Record<string, Job<any>>>,
44
43
  opts: WorkerOptions = {},
45
44
  ) {
46
45
  this.batchSize = opts.batchSize ?? 10;
47
46
  this.intervalMs = opts.intervalMs ?? 1000;
48
47
  this.staleMs = opts.staleMs ?? 60_000;
49
- this.backoffMs = opts.backoffMs ?? 30_000;
50
48
  }
51
49
 
52
50
  /** Run one pass. Exposed so the CLI and tests can drive it directly. */
53
51
  async tick(): Promise<number> {
54
- const jobs = await this.queue.claim(this.batchSize, this.staleMs);
55
- for (const job of jobs) {
56
- const handler = this.handlers[job.name];
57
- if (!handler) {
52
+ const rows = await this.queue.claim(this.batchSize, this.staleMs);
53
+ for (const row of rows) {
54
+ const job = this.jobs[row.name];
55
+ if (!job) {
58
56
  await this.queue.fail(
59
- job.id,
60
- new Error(`no handler registered for "${job.name}"`),
61
- this.backoffMs,
57
+ row.id,
58
+ new Error(`no job named "${row.name}" (is it in src/jobs?)`),
59
+ 30_000,
62
60
  );
63
61
  continue;
64
62
  }
63
+ let raw: unknown;
65
64
  try {
66
- let payload: unknown;
67
- try {
68
- payload = JSON.parse(job.payload);
69
- } catch {
70
- throw new Error(`payload is not valid JSON: ${job.payload}`);
71
- }
72
- await handler(payload, { id: job.id, name: job.name, attempt: job.attempts });
73
- await this.queue.complete(job.id);
65
+ raw = JSON.parse(row.payload);
66
+ } catch {
67
+ await this.queue.bury(
68
+ row.id,
69
+ `payload is not valid JSON: ${row.payload}`,
70
+ );
71
+ continue;
72
+ }
73
+ const checked = await checkPayload(job, raw);
74
+ if (!checked.ok) {
75
+ await this.queue.bury(
76
+ row.id,
77
+ `payload no longer matches the job: ${checked.message}`,
78
+ );
79
+ continue;
80
+ }
81
+ try {
82
+ await job.handler(checked.value, {
83
+ id: row.id,
84
+ name: row.name,
85
+ attempt: row.attempts,
86
+ });
87
+ await this.queue.complete(row.id);
74
88
  } catch (error) {
75
- // A recurring job that has used up its retries moves on to its
76
- // next run instead of parking as failed: one bad afternoon
77
- // upstream should not stop a schedule for good.
78
- if (this.queue.isScheduled(job.id) && job.attempts >= job.max_attempts) {
89
+ if (this.queue.isScheduled(row.id) && row.attempts >= row.maxAttempts) {
79
90
  await this.queue.reschedule(
80
- job.id,
91
+ row.id,
81
92
  error instanceof Error ? error.message : String(error),
82
93
  );
83
94
  } else {
84
- await this.queue.fail(job.id, error, this.backoffMs);
95
+ await this.queue.fail(row.id, error, job.backoffAfter(row.attempts));
85
96
  }
86
97
  }
87
98
  }
88
- return jobs.length;
99
+ return rows.length;
89
100
  }
90
101
 
91
- /** Poll until stop() is called. */
102
+ /** Loop until `stop()`; resolves once the loop has exited. */
92
103
  async run(signal?: AbortSignal): Promise<void> {
93
104
  while (!this.stopping) {
94
105
  this.running = true;
95
106
  try {
96
107
  const count = await this.tick();
97
- // Only idle when there was nothing to do. Sleeping after a full
98
- // batch would cap throughput at batchSize per interval.
99
108
  if (count === 0) await sleep(this.intervalMs, signal);
100
109
  } catch (error) {
101
110
  console.error("jobs worker error:", error);
@@ -106,7 +115,7 @@ export class JobWorker {
106
115
  }
107
116
  }
108
117
 
109
- /** Stop polling and wait for the in-flight pass to finish. */
118
+ /** Finish the current pass, then stop. */
110
119
  async stop(): Promise<void> {
111
120
  this.stopping = true;
112
121
  while (this.running) await sleep(10);
@@ -116,9 +125,6 @@ export class JobWorker {
116
125
  function sleep(ms: number, signal?: AbortSignal): Promise<void> {
117
126
  return new Promise((resolve) => {
118
127
  const timer = setTimeout(resolve, ms);
119
- // A timer keeps the process alive on its own, so unref and let the
120
- // abort signal do the waking. Otherwise stop() could not be
121
- // observed until the full interval elapsed.
122
128
  (timer as unknown as { unref?: () => void }).unref?.();
123
129
  signal?.addEventListener("abort", () => {
124
130
  clearTimeout(timer);
package/store-bun.ts DELETED
@@ -1,137 +0,0 @@
1
- import { database } from "@sigil-dev/plugin-sql";
2
- import type { Capabilities, JobStore } from "./store.ts";
3
-
4
- /**
5
- * A JobStore over `Bun.SQL`, with the dialect capabilities detected from the
6
- * DSN.
7
- *
8
- * The queue is dialect-agnostic on purpose. This is where the dialect
9
- * knowledge lives, in one place, from a value the user already gave us.
10
- *
11
- * The three capabilities are read off the URL scheme, not probed at runtime,
12
- * because probing means running queries at boot against a database that may
13
- * not be up yet — and this plugin deliberately does not open a connection
14
- * until the queue is used. A wrong guess is loud here: a missing
15
- * `spaceSeparatedTimestamp` on MySQL fails the first INSERT with
16
- * "Incorrect datetime value", not silently.
17
- */
18
- export function bunSqlStore(url: string, extra: Partial<Capabilities> = {}): JobStore {
19
- return bunStore(new Bun.SQL({ url, max: 1 }), url, extra);
20
- }
21
-
22
- /**
23
- * A JobStore over a `Bun.SQL` the caller already owns.
24
- *
25
- * This is the bridge that lets the queue run on the connection the SQL plugin
26
- * opened, rather than a second pool. `bunSqlStore(url)` is for when the queue
27
- * is the only thing using the database; this is for when an app already
28
- * registered `@sigil-dev/plugin-sql` and the budget should be spent once:
29
- *
30
- * import { adapter } from "@sigil-dev/plugin-sql/bun";
31
- * import { jobs, bunStore } from "@sigil-dev/plugin-jobs";
32
- *
33
- * const db = adapter(url, { maxTotal: 12 });
34
- * const queue = jobs({
35
- * // read from the plugin's own client, don't open a second pool
36
- * store: (services) => bunStore(services.db as Bun.SQL),
37
- * });
38
- *
39
- * The dialect is read from the client, which already parsed the DSN, so
40
- * `url` is only a fallback for a client that does not say. Through
41
- * plugin-sql's `locals.db`, a job enqueued during a `perRequestTx` request
42
- * is part of that request's transaction and rolls back with it.
43
- *
44
- * The placeholder translation is the same one `bunSqlStore` uses, because
45
- * the queue speaks `?` and postgres wants $1. It lives in this function
46
- * rather than in either caller so the two cannot drift.
47
- */
48
- type Dialect = "postgres" | "mysql" | "sqlite";
49
-
50
- /**
51
- * Which database a `Bun.SQL` client talks to.
52
- *
53
- * Asked of the client first: Bun.SQL already parsed the DSN, including the
54
- * spellings a scheme check misses — `postgresql://`, which Neon and Supabase
55
- * hand out, and `mariadb://`. Reading the scheme alone sent a `postgresql://`
56
- * client `?` placeholders.
57
- */
58
- function dialectOf(db: Bun.SQL, url?: string): Dialect {
59
- const fromClient = (db as unknown as { options?: { adapter?: string } })
60
- .options?.adapter;
61
- const raw = (fromClient ?? url?.split(":")[0] ?? "").toLowerCase();
62
- if (raw === "postgres" || raw === "postgresql" || raw === "pg") return "postgres";
63
- if (raw === "mysql" || raw === "mariadb") return "mysql";
64
- // A bare path is a SQLite file to Bun.SQL, and so it is here.
65
- return "sqlite";
66
- }
67
-
68
- export function bunStore(
69
- db: Bun.SQL,
70
- url?: string,
71
- extra: Partial<Capabilities> = {},
72
- ): JobStore {
73
- const dialect = dialectOf(db, url);
74
- const capabilities: Capabilities = {
75
- skipLocked: dialect === "postgres" || dialect === "mysql",
76
- spaceSeparatedTimestamp: dialect === "mysql",
77
- ...extra,
78
- };
79
-
80
- // Two things have to be reconciled for the queue's `?` placeholders:
81
- // postgres wants $1, $2, and mysql wants ? (so it passes through).
82
- // Doing it here means queue.ts contains no dialect knowledge at all.
83
- const placeholder = (text: string): string => {
84
- if (dialect !== "postgres") return text;
85
- let i = 0;
86
- return text.replace(/\?/g, () => `$${++i}`);
87
- };
88
-
89
- return {
90
- capabilities,
91
- async query<T>(text: string, params: unknown[] = []): Promise<T[]> {
92
- // unsafe() is required rather than query(): the queue builds its
93
- // SQL as strings, so the values have to be bound separately or
94
- // they are an injection waiting to happen.
95
- return (await db.unsafe(placeholder(text), params as never[])) as T[];
96
- },
97
- };
98
- }
99
-
100
- /**
101
- * A JobStore over a database `@sigil-dev/plugin-sql` registered, by name.
102
- *
103
- * This is what `jobs()` uses when given neither `sql` nor `store`, and it
104
- * removes the three things the `store` bridge made you get right:
105
- *
106
- * - Order. It looks the database up when a query runs, so the jobs plugin
107
- * can sit anywhere in the plugins array.
108
- * - The CLI. `sigil jobs work` has no server to publish services, but it
109
- * has the config, which registered the database; the command opens it.
110
- * - The request transaction. Every query goes through `db()`, so a job
111
- * enqueued during a `perRequestTx` request is part of it, and one
112
- * enqueued anywhere else runs on the pool.
113
- */
114
- export function databaseStore(name = "db", extra: Partial<Capabilities> = {}): JobStore {
115
- const db = database<Bun.SQL>(name);
116
- // The dialect is the client's, and there is no client until the database
117
- // opens — so it is read on first use and kept.
118
- let dialect: ReturnType<typeof dialectOf> | undefined;
119
- const dialectNow = () => (dialect ??= dialectOf(db.pool()));
120
-
121
- return {
122
- get capabilities(): Capabilities {
123
- const d = dialectNow();
124
- return {
125
- skipLocked: d === "postgres" || d === "mysql",
126
- spaceSeparatedTimestamp: d === "mysql",
127
- ...extra,
128
- };
129
- },
130
- async query<T>(text: string, params: unknown[] = []): Promise<T[]> {
131
- let i = 0;
132
- const sql = dialectNow() === "postgres" ? text.replace(/\?/g, () => `$${++i}`) : text;
133
- const client = db.db() as Bun.SQL;
134
- return (await client.unsafe(sql, params as never[])) as T[];
135
- },
136
- };
137
- }
package/store-drizzle.ts DELETED
@@ -1,86 +0,0 @@
1
- import type { JobStore } from "./store.ts";
2
-
3
- /**
4
- * A JobStore for a project that already uses Drizzle.
5
- *
6
- * ## Read this before you use it, because the honest answer is "reach past
7
- * ## drizzle"
8
- *
9
- * Measured against drizzle-orm 0.45.3 with a real Bun.SQL client, real
10
- * MariaDB and real postgres:
11
- *
12
- * - `db.execute("SELECT ? as n", [7])` does NOT bind. The `?` reaches
13
- * MariaDB as literal SQL and the server returns a syntax error. Drizzle's
14
- * `execute` takes a `SQLWrapper`, not a parameter array, and passing one
15
- * is silently ignored.
16
- * - `db.execute(dsql`SELECT ${3}`)` works on postgres and fails on MySQL
17
- * with "Unknown column '$1' in 'SELECT'". The bun-sql driver emits $1
18
- * placeholders unconditionally, so the same call, on the same db object,
19
- * is correct on one database and broken on another. That is a drizzle
20
- * bug, and it is why the obvious implementation of this adapter does not
21
- * work.
22
- * - `client.unsafe("SELECT ? as n", [7])` on the underlying Bun.SQL works.
23
- *
24
- * So the adapter is given a function to run SQL, not a Drizzle db. Passing
25
- * `db` and hoping would have produced a store that works on postgres, fails
26
- * on MySQL, and fails the moment a query has a parameter — which is every
27
- * query the queue runs.
28
- *
29
- * With a real Drizzle project, the connection is already a `Bun.SQL` (or a
30
- * `pg` Pool), so pass that. Drizzle is not in the path here, and the reason
31
- * is written above rather than left for someone to rediscover.
32
- *
33
- * If you want the queue to be a first-class Drizzle table instead — with
34
- * `db.select()`, `db.insert()`, `db.update()` rather than SQL text — that is
35
- * a real change to JobQueue, not an adapter: every query would be written
36
- * twice and the claim guard, which is what makes the queue correct, would
37
- * have to be proven in both paths.
38
- */
39
- export interface SqlRunner {
40
- /** Run SQL with bound parameters and return rows. */
41
- run: (query: string, params: unknown[]) => Promise<unknown>;
42
- }
43
-
44
- export function drizzleStore(
45
- run: SqlRunner,
46
- dialect: "postgres" | "mysql" | "sqlite",
47
- ): JobStore {
48
- const capabilities = {
49
- skipLocked: dialect === "postgres" || dialect === "mysql",
50
- spaceSeparatedTimestamp: dialect === "mysql",
51
- };
52
-
53
- return {
54
- capabilities,
55
- async query<T>(text: string, params: unknown[] = []): Promise<T[]> {
56
- // postgres wants $1..$n, MySQL and sqlite want ?. The queue emits
57
- // `?` because that is the wider convention, so the rewrite happens
58
- // here — once, for one dialect, rather than in every caller.
59
- let i = 0;
60
- const stmt =
61
- dialect === "postgres" ? text.replace(/\?/g, () => `$${++i}`) : text;
62
- const result = await run.run(stmt, params);
63
- // Measured return shapes: a plain row array for both execute(string)
64
- // and dsql.raw. The object cases below are for the node-postgres and
65
- // neon drivers, which wrap rows — a DrizzleLike from those would
66
- // otherwise return { rows: [...] } to the queue and every query
67
- // would look like it matched nothing.
68
- if (Array.isArray(result)) return result as T[];
69
- if (result && typeof result === "object") {
70
- const r = result as Record<string, unknown>;
71
- if (Array.isArray(r.rows)) return r.rows as T[];
72
- }
73
- return [];
74
- },
75
- };
76
- }
77
-
78
- /**
79
- * A runner over a `Bun.SQL` client, which is what a Drizzle bun-sql project
80
- * already holds. This is the supported entry point.
81
- */
82
- export function bunSqlRunner(client: {
83
- unsafe(text: string, params?: unknown[]): Promise<unknown>;
84
- }): SqlRunner {
85
- return { run: (text, params) => client.unsafe(text, params) };
86
- }