@sigil-dev/plugin-jobs 0.9.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/README.md +110 -0
  2. package/index.ts +227 -0
  3. package/package.json +17 -0
  4. package/queue.ts +185 -0
  5. package/sql.ts +73 -0
  6. package/worker.ts +118 -0
package/README.md ADDED
@@ -0,0 +1,110 @@
1
+ # @sigil-dev/plugin-jobs
2
+
3
+ Durable background jobs for [Grimoire](https://github.com/sigil-dev/grimoire) — enqueue, retry, and a `sigil jobs` CLI.
4
+
5
+ Opt-in. Grimoire core knows nothing about it: not installed means no queue, no table, no worker.
6
+
7
+ ## Install
8
+
9
+ ```sh
10
+ bun add @sigil-dev/plugin-jobs
11
+ ```
12
+
13
+ ## Setup
14
+
15
+ ```ts
16
+ // sigil.config.ts
17
+ import { Database } from "bun:sqlite";
18
+ import { jobs } from "@sigil-dev/plugin-jobs";
19
+
20
+ const db = new Database("app.db", { create: true });
21
+
22
+ export default {
23
+ plugins: [
24
+ jobs({
25
+ // You pass the client. This plugin does not import a driver, open
26
+ // a connection, or read a connection string — whoever already
27
+ // holds the connection has it.
28
+ sql: {
29
+ query: (sql, params = []) => Promise.resolve(db.query(sql).all(params)),
30
+ },
31
+ handlers: {
32
+ "send-welcome": (payload) => sendEmail(payload.email),
33
+ },
34
+ }),
35
+ ],
36
+ };
37
+ ```
38
+
39
+ `jobs()` throws at construction if `sql` is missing, so a queue you forgot to configure is a failed deploy rather than a 3am page.
40
+
41
+ ## Enqueue
42
+
43
+ The API hangs off the plugin instance, so put it in its own module and
44
+ import it wherever you need it:
45
+
46
+ ```ts
47
+ // jobqueue.ts
48
+ import { Database } from "bun:sqlite";
49
+ import { jobs } from "@sigil-dev/plugin-jobs";
50
+
51
+ const db = new Database("app.db", { create: true });
52
+
53
+ export const jobQueue = jobs({
54
+ sql: { query: (sql, params = []) => Promise.resolve(db.query(sql).all(params)) },
55
+ handlers: { "send-welcome": (p) => sendWelcome(p.email) },
56
+ });
57
+ ```
58
+
59
+ ```ts
60
+ // sigil.config.ts
61
+ import { jobQueue } from "./jobqueue";
62
+ export default { plugins: [jobQueue] };
63
+ ```
64
+
65
+ Then enqueue from anywhere — a load, an action, a request hook:
66
+
67
+ ```ts
68
+ import { jobQueue } from "./jobqueue";
69
+
70
+ await jobQueue.jobs.enqueue("send-welcome", { email: "ada@example.com" });
71
+ await jobQueue.jobs.enqueue("rebuild-index", {}, { maxAttempts: 1 });
72
+
73
+ // deferred, via runAt
74
+ await jobQueue.jobs.enqueue(
75
+ "settle",
76
+ { invoiceId },
77
+ { runAt: new Date(Date.now() + 5 * 60_000) },
78
+ );
79
+ ```
80
+
81
+ Register a handler after boot with `jobQueue.jobs.register(name, fn)`.
82
+
83
+ ## CLI
84
+
85
+ ```sh
86
+ sigil jobs work # run the worker until stopped
87
+ sigil jobs list [--limit 20] # most recent jobs
88
+ sigil jobs stats # counts by state
89
+ sigil jobs retry <id> # requeue one job, clearing its attempts
90
+ sigil jobs retry # requeue every exhausted job
91
+ sigil jobs migrate # create the jobs table
92
+ ```
93
+
94
+ The worker is a separate process by default. `inlineWorker: true` exists for a single-process deploy — note that an inline worker means every scaled worker is also a worker, which is surprising at three.
95
+
96
+ ## Guarantees, precisely
97
+
98
+ **Delivery is at-least-once.** A worker can die between your side effect and the row delete, so a job may run twice. **Handlers must be idempotent.** The alternative — holding a transaction open across every handler, including every HTTP call inside one — is a worse trade than a duplicate email.
99
+
100
+ **Claiming is race-free without `SKIP LOCKED`.** The claim guard lives inside the `UPDATE`: a row is only claimable when `claimed_at` is null or stale, and that check is part of the write. Two workers racing means the loser gets an empty set, not a duplicate job. `SKIP LOCKED` is appended only when your driver advertises it, and changes contention behaviour, not correctness — a backend without it is slower, never wrong.
101
+
102
+ **A stale claim is reclaimed** after 60s by default, so a hard-killed worker does not strand its jobs.
103
+
104
+ ## SQL requirements
105
+
106
+ Portable SQL only. Verified against SQLite; the claim path uses no dialect-specific syntax, so Postgres and MySQL work without a driver-specific branch. `SKIP LOCKED` is used when available and is never load-bearing.
107
+
108
+ ## License
109
+
110
+ MIT
package/index.ts ADDED
@@ -0,0 +1,227 @@
1
+ import type { CliCommand, GrimoirePlugin } from "@sigil-dev/grimoire";
2
+ import { JobQueue } from "./queue.ts";
3
+ import type { JobHandler, SqlExecutor } from "./sql.ts";
4
+ import { JobWorker } from "./worker.ts";
5
+
6
+ export { JobQueue, SCHEMA } from "./queue.ts";
7
+ export type { EnqueueOptions, JobHandler, JobRow, JobStats, SqlExecutor } from "./sql.ts";
8
+ export { JobWorker } from "./worker.ts";
9
+ export type { WorkerOptions } from "./worker.ts";
10
+
11
+ export interface JobsOptions {
12
+ /**
13
+ * The database client. Required, and not optional-with-a-default,
14
+ * because there is no sensible fallback: a queue that silently kept jobs
15
+ * in memory would look like it worked right up until a restart.
16
+ */
17
+ sql: SqlExecutor;
18
+
19
+ /**
20
+ * Named handlers the worker can run. Enqueue a name that is not here and
21
+ * the job fails loudly rather than being dropped.
22
+ */
23
+ handlers?: Record<string, JobHandler>;
24
+
25
+ /** Table name. @default "sigil_jobs" */
26
+ table?: string;
27
+
28
+ /** Create the table on boot. @default true */
29
+ autoMigrate?: boolean;
30
+
31
+ /**
32
+ * Run a worker in the server process.
33
+ *
34
+ * Off by default. A worker in the app means every worker scaled up is
35
+ * also a worker — fine for one process, surprising at three, and the
36
+ * reason `sigil jobs work` exists as a separate process to run instead.
37
+ * @default false
38
+ */
39
+ inlineWorker?: boolean;
40
+ }
41
+
42
+ export interface JobsApi {
43
+ enqueue: (
44
+ name: string,
45
+ payload?: unknown,
46
+ options?: { runAt?: Date; maxAttempts?: number },
47
+ ) => Promise<string>;
48
+ queue: JobQueue;
49
+ handlers: Record<string, JobHandler>;
50
+ /** Register a handler after the plugin was created. */
51
+ register: (name: string, handler: JobHandler) => void;
52
+ }
53
+
54
+ /**
55
+ * The plugin. A factory rather than a bare object because `sql` is
56
+ * required: a plugin instantiated with no options could only be a
57
+ * misconfigured one, and failing at `sigil start` is far better than
58
+ * failing on the first enqueue at 2am.
59
+ */
60
+ export function jobs(options: JobsOptions): GrimoirePlugin & { jobs: JobsApi } {
61
+ if (!options?.sql) {
62
+ throw new Error(
63
+ "plugin-jobs requires a `sql` option. " +
64
+ "Pass a database client: jobs({ sql, handlers }). " +
65
+ "This error is at boot rather than on the first enqueue, so a " +
66
+ "missing queue is a failed deploy rather than a 3am page.",
67
+ );
68
+ }
69
+
70
+ const queue = new JobQueue(options.sql, options.table);
71
+ const handlers = { ...(options.handlers ?? {}) };
72
+
73
+ const api: JobsApi = {
74
+ queue,
75
+ handlers,
76
+ async enqueue(name, payload, opts) {
77
+ return queue.enqueue(name, payload, opts);
78
+ },
79
+ register(name, handler) {
80
+ handlers[name] = handler;
81
+ },
82
+ };
83
+
84
+ const instance: GrimoirePlugin & { jobs: JobsApi } = {
85
+ name: "plugin-jobs",
86
+ jobs: api,
87
+
88
+ async config(config) {
89
+ if (options.autoMigrate !== false) {
90
+ await queue.migrate();
91
+ }
92
+
93
+ if (options.inlineWorker) {
94
+ const worker = new JobWorker(queue, handlers);
95
+ void worker.run();
96
+ }
97
+
98
+ return config;
99
+ },
100
+
101
+ /**
102
+ * Registers `sigil jobs <subcommand>`.
103
+ *
104
+ * The CLI never starts the app server, so handlers cannot be
105
+ * recovered from a running process — the worker loads the user's
106
+ * config and reads the same handler map, which is why the config is
107
+ * the source of truth rather than module state.
108
+ */
109
+ cli(program, _context) {
110
+ // Resolved lazily so `sigil --help` and unrelated commands never
111
+ // need a database, and so the error names the real problem instead
112
+ // of surfacing as a failed import.
113
+ //
114
+ // This reads the queue the plugin's own `config` hook built. The
115
+ // CLI loads sigil.config.ts and runs plugin config transforms
116
+ // before calling this, so by the time an action runs the queue
117
+ // exists — and if `config` threw, the CLI already reported that
118
+ // and never got here.
119
+ const connect = (): JobsApi => {
120
+ if (!instance.jobs) {
121
+ throw new Error(
122
+ "plugin-jobs is configured but no queue is available. " +
123
+ "Check the `jobs` option in sigil.config.ts.",
124
+ );
125
+ }
126
+ return instance.jobs;
127
+ };
128
+
129
+ const group = program
130
+ .command("jobs")
131
+ .description("manage the durable job queue");
132
+
133
+ group
134
+ .command("work")
135
+ .description("run the job worker until stopped")
136
+ .option("--interval <ms>", "idle poll interval", (v: string) => Number(v))
137
+ .option("--batch <n>", "jobs per pass", (v: string) => Number(v))
138
+ .action(async (opts: { interval?: number; batch?: number }) => {
139
+ const { queue: q, handlers: h } = connect();
140
+ const worker = new JobWorker(q, h, {
141
+ ...(opts.interval ? { intervalMs: opts.interval } : {}),
142
+ ...(opts.batch ? { batchSize: opts.batch } : {}),
143
+ });
144
+
145
+ // Drain before exiting so a deploy does not abandon claimed
146
+ // jobs. The stale-claim timeout covers a hard kill.
147
+ const shutdown = async () => {
148
+ console.log("\nstopping, draining…");
149
+ await worker.stop();
150
+ process.exit(0);
151
+ };
152
+ process.on("SIGINT", shutdown);
153
+ process.on("SIGTERM", shutdown);
154
+
155
+ console.log(
156
+ `sigil jobs work — ${Object.keys(h).length} handler(s) registered`,
157
+ );
158
+ await worker.run();
159
+ });
160
+
161
+ group
162
+ .command("list")
163
+ .description("list the most recent jobs")
164
+ .option("--limit <n>", "how many rows", (v: string) => Number(v))
165
+ .action(async (opts: { limit?: number }) => {
166
+ const rows = await connect().queue.list(opts.limit ?? 20);
167
+ if (rows.length === 0) {
168
+ console.log("no jobs");
169
+ return;
170
+ }
171
+ console.log(["ID", "NAME", "RUN_AT", "TRIES", "ERROR"].join(" "));
172
+ for (const row of rows) {
173
+ console.log(
174
+ [
175
+ row.id.slice(0, 8),
176
+ row.name,
177
+ row.run_at,
178
+ `${row.attempts}`,
179
+ row.last_error ? row.last_error.slice(0, 40) : "-",
180
+ ].join(" "),
181
+ );
182
+ }
183
+ });
184
+
185
+ group
186
+ .command("stats")
187
+ .description("count jobs by state")
188
+ .action(async () => {
189
+ const s = await connect().queue.stats();
190
+ console.log(
191
+ `pending ${s.pending} claimed ${s.claimed} failed ${s.failed}`,
192
+ );
193
+ });
194
+
195
+ group
196
+ .command("retry")
197
+ .argument("[id]", "job id, or all failed jobs when omitted")
198
+ .description("requeue a job, clearing its attempt count")
199
+ .action(async (id?: string) => {
200
+ const q = connect().queue;
201
+ if (id) {
202
+ const ok = await q.retry(id);
203
+ console.log(ok ? `requeued ${id}` : `no such job: ${id}`);
204
+ return;
205
+ }
206
+ const failed = (await q.list(1000)).filter(
207
+ (r) => r.attempts >= r.max_attempts,
208
+ );
209
+ for (const job of failed) await q.retry(job.id);
210
+ console.log(`requeued ${failed.length} failed job(s)`);
211
+ });
212
+
213
+ group
214
+ .command("migrate")
215
+ .description("create the jobs table if it does not exist")
216
+ .action(async () => {
217
+ await connect().queue.migrate();
218
+ console.log("jobs table ready");
219
+ });
220
+ },
221
+ };
222
+
223
+ return instance;
224
+ }
225
+
226
+ export const plugin = jobs;
227
+ export default jobs;
package/package.json ADDED
@@ -0,0 +1,17 @@
1
+ {
2
+ "name": "@sigil-dev/plugin-jobs",
3
+ "description": "Durable background jobs for Grimoire: enqueue, retry, and a `sigil jobs` CLI",
4
+ "module": "index.ts",
5
+ "type": "module",
6
+ "private": false,
7
+ "version": "0.9.12",
8
+ "keywords": ["grimoire", "sigil", "jobs", "queue", "background"],
9
+ "files": ["index.ts", "queue.ts", "worker.ts", "sql.ts", "README.md"],
10
+ "devDependencies": {
11
+ "@types/bun": "latest"
12
+ },
13
+ "peerDependencies": {
14
+ "typescript": "^5",
15
+ "@sigil-dev/grimoire": "0.9.12"
16
+ }
17
+ }
package/queue.ts ADDED
@@ -0,0 +1,185 @@
1
+ import type { EnqueueOptions, JobRow, JobStats, SqlExecutor } from "./sql.ts";
2
+
3
+ /**
4
+ * Schema statements, kept portable.
5
+ *
6
+ * Deliberately plain: no dialect, no generated columns, no `jsonb`. The
7
+ * payload is TEXT holding JSON rather than a JSON column type, because the
8
+ * json type is spelled differently everywhere and nothing here needs to
9
+ * query inside the payload.
10
+ */
11
+ export const SCHEMA = `
12
+ CREATE TABLE IF NOT EXISTS sigil_jobs (
13
+ id TEXT PRIMARY KEY,
14
+ name TEXT NOT NULL,
15
+ payload TEXT NOT NULL,
16
+ run_at TIMESTAMP NOT NULL,
17
+ attempts INTEGER NOT NULL DEFAULT 0,
18
+ max_attempts INTEGER NOT NULL DEFAULT 5,
19
+ last_error TEXT,
20
+ claimed_at TIMESTAMP,
21
+ created_at TIMESTAMP NOT NULL
22
+ );
23
+ CREATE INDEX IF NOT EXISTS sigil_jobs_ready ON sigil_jobs (run_at);
24
+ `;
25
+
26
+ /**
27
+ * A durable queue over any client that can run a query and return rows.
28
+ *
29
+ * Delivery is at-least-once, and that is not a caveat to be apologetic
30
+ * about — it is the guarantee being bought. A worker can die between doing
31
+ * the side effect and deleting the row, so a handler must be idempotent.
32
+ * The alternative is exactly-once, which in practice means transactions
33
+ * held open across the entire handler, including every HTTP call inside it.
34
+ */
35
+ export class JobQueue {
36
+ constructor(
37
+ private readonly sql: SqlExecutor,
38
+ private readonly table: string = "sigil_jobs",
39
+ ) {}
40
+
41
+ private get skipLocked(): string {
42
+ return this.sql.supportsSkipLocked ? " FOR UPDATE SKIP LOCKED" : "";
43
+ }
44
+
45
+ async migrate(): Promise<void> {
46
+ for (const stmt of SCHEMA.split(";").map((s) => s.trim()).filter(Boolean)) {
47
+ await this.sql.query(stmt);
48
+ }
49
+ }
50
+
51
+ /**
52
+ * Add a job.
53
+ *
54
+ * The id is generated here rather than by the database so a retry of
55
+ * the enqueue is recognisable, and so a caller who retries cannot
56
+ * accidentally create a second job for the same logical work.
57
+ */
58
+ async enqueue(
59
+ name: string,
60
+ payload: unknown = {},
61
+ options: EnqueueOptions = {},
62
+ ): Promise<string> {
63
+ const id = crypto.randomUUID();
64
+ const now = new Date();
65
+ const runAt = options.runAt ?? now;
66
+ await this.sql.query(
67
+ `INSERT INTO ${this.table}
68
+ (id, name, payload, run_at, attempts, max_attempts, created_at)
69
+ VALUES (?, ?, ?, ?, 0, ?, ?)`,
70
+ [
71
+ id,
72
+ name,
73
+ JSON.stringify(payload ?? {}),
74
+ runAt.toISOString(),
75
+ options.maxAttempts ?? 5,
76
+ now.toISOString(),
77
+ ],
78
+ );
79
+ return id;
80
+ }
81
+
82
+ /**
83
+ * Atomically take up to `limit` due jobs.
84
+ *
85
+ * The safety comes from the claim guard inside the UPDATE, not from row
86
+ * locking: a row is only claimable when `claimed_at` is NULL or an old
87
+ * claim, and that check happens as part of the write. Two workers
88
+ * issuing this simultaneously both read the same candidate ids, but the
89
+ * second one's UPDATE matches nothing because the first has already set
90
+ * `claimed_at`. So the loser gets an empty set rather than a duplicate
91
+ * job — which is the whole requirement.
92
+ *
93
+ * `SKIP LOCKED` is appended only when the driver advertises support. It
94
+ * changes contention behaviour, not correctness.
95
+ *
96
+ * The returned rows are NOT in run_at order, and cannot be made to be:
97
+ * SQL does not define the order of an UPDATE's RETURNING clause. The
98
+ * inner ORDER BY decides which `limit` rows are taken, not the order
99
+ * they come back. Sort in the caller if order matters.
100
+ */
101
+ async claim(limit: number, staleMs = 60_000): Promise<JobRow[]> {
102
+ const cutoff = new Date(Date.now() - staleMs).toISOString();
103
+ return this.sql.query<JobRow>(
104
+ `UPDATE ${this.table}
105
+ SET claimed_at = ?, attempts = attempts + 1
106
+ WHERE id IN (
107
+ SELECT id FROM ${this.table}
108
+ WHERE run_at <= ?
109
+ AND attempts < max_attempts
110
+ AND (claimed_at IS NULL OR claimed_at < ?)
111
+ ORDER BY run_at
112
+ LIMIT ?
113
+ ${this.skipLocked}
114
+ )
115
+ RETURNING id, name, payload, run_at, attempts, max_attempts, last_error, claimed_at, created_at`,
116
+ [new Date().toISOString(), new Date().toISOString(), cutoff, limit],
117
+ );
118
+ }
119
+
120
+ async complete(id: string): Promise<void> {
121
+ await this.sql.query(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
122
+ }
123
+
124
+ /**
125
+ * Record a failure.
126
+ *
127
+ * The row stays, because retry is the point. `claimed_at` is cleared so
128
+ * the next pass can take it, and `run_at` is pushed into the future for
129
+ * a failed job so a persistently broken handler does not spin — a
130
+ * crash-looping job would otherwise burn a worker in a tight loop.
131
+ */
132
+ async fail(id: string, error: unknown, backoffMs = 30_000): Promise<void> {
133
+ const message = error instanceof Error ? error.message : String(error);
134
+ const nextRun = new Date(Date.now() + backoffMs).toISOString();
135
+ await this.sql.query(
136
+ `UPDATE ${this.table}
137
+ SET last_error = ?, claimed_at = NULL, run_at = ?
138
+ WHERE id = ?`,
139
+ [message.slice(0, 2000), nextRun, id],
140
+ );
141
+ }
142
+
143
+ /** Put a job back on the queue immediately, for manual recovery. */
144
+ async retry(id: string): Promise<boolean> {
145
+ const rows = await this.sql.query<JobRow>(
146
+ `UPDATE ${this.table}
147
+ SET attempts = 0, last_error = NULL, claimed_at = NULL, run_at = ?
148
+ WHERE id = ?
149
+ RETURNING id`,
150
+ [new Date().toISOString(), id],
151
+ );
152
+ return rows.length > 0;
153
+ }
154
+
155
+ async list(limit = 20): Promise<JobRow[]> {
156
+ return this.sql.query<JobRow>(
157
+ `SELECT id, name, payload, run_at, attempts, max_attempts, last_error, claimed_at, created_at
158
+ FROM ${this.table}
159
+ ORDER BY created_at DESC
160
+ LIMIT ?`,
161
+ [limit],
162
+ );
163
+ }
164
+
165
+ async stats(): Promise<JobStats> {
166
+ const rows = await this.sql.query<{ bucket: string; count: number }>(
167
+ `SELECT
168
+ CASE
169
+ WHEN attempts >= max_attempts THEN 'failed'
170
+ WHEN claimed_at IS NOT NULL THEN 'claimed'
171
+ ELSE 'pending'
172
+ END AS bucket,
173
+ COUNT(*) AS count
174
+ FROM ${this.table}
175
+ GROUP BY bucket`,
176
+ );
177
+ const stats: JobStats = { pending: 0, claimed: 0, failed: 0 };
178
+ for (const row of rows) {
179
+ if (row.bucket === "failed") stats.failed = Number(row.count);
180
+ else if (row.bucket === "claimed") stats.claimed = Number(row.count);
181
+ else stats.pending = Number(row.count);
182
+ }
183
+ return stats;
184
+ }
185
+ }
package/sql.ts ADDED
@@ -0,0 +1,73 @@
1
+ /**
2
+ * The SQL contract this plugin speaks.
3
+ *
4
+ * This is deliberately small: three methods, and the caller supplies the
5
+ * client. The plugin does not import a database library, does not open a
6
+ * connection, and does not read a connection string out of the
7
+ * environment. Whoever holds the connection already has it, and making
8
+ * them hand it over is what lets this run on whatever they already use.
9
+ *
10
+ * "SQL" here means something narrower than it sounds. The SQL this
11
+ * generates uses `?` placeholders and returns rows as objects, which Bun's
12
+ * built-in `Bun.sql` satisfies directly and most other drivers can be
13
+ * adapted to with a wrapper. What it does NOT do is hide dialect
14
+ * differences, and it cannot: `FOR UPDATE SKIP LOCKED` is how Postgres and
15
+ * MySQL take a row, and SQLite has no equivalent.
16
+ *
17
+ * So the concurrency guarantee is split, and the split is the point:
18
+ *
19
+ * - The claim is an atomic conditional UPDATE. Every backend that can run
20
+ * `UPDATE ... WHERE id IN (SELECT ... LIMIT n)` gets a correct claim
21
+ * with no dialect-specific locking at all, because the row's own
22
+ * `claimed_at` value is what makes the update exclusive. Two workers
23
+ * can race and exactly one wins each row.
24
+ * - `SKIP LOCKED` is used only when the driver says it supports it, to
25
+ * stop a worker from blocking behind rows another worker is holding.
26
+ * Without it, correctness is unchanged and throughput drops.
27
+ *
28
+ * That is why `supportsSkipLocked` exists and why the default is false.
29
+ * Claiming portability we have not got is worse than documenting the
30
+ * performance cliff.
31
+ */
32
+ export interface SqlExecutor {
33
+ /** Run a query and return rows. `params` fills `?` placeholders. */
34
+ query<T = Record<string, unknown>>(
35
+ sql: string,
36
+ params?: unknown[],
37
+ ): Promise<T[]>;
38
+
39
+ /**
40
+ * Whether `FOR UPDATE SKIP LOCKED` is available. Only improves
41
+ * throughput; the claim is correct either way.
42
+ * @default false
43
+ */
44
+ readonly supportsSkipLocked?: boolean;
45
+ }
46
+
47
+ export interface JobRow {
48
+ id: string;
49
+ name: string;
50
+ payload: string;
51
+ run_at: string;
52
+ attempts: number;
53
+ max_attempts: number;
54
+ last_error: string | null;
55
+ claimed_at: string | null;
56
+ created_at: string;
57
+ }
58
+
59
+ /** A job as it exists before it is claimed. */
60
+ export interface EnqueueOptions {
61
+ /** Delay execution. Defaults to now. */
62
+ runAt?: Date;
63
+ /** Give up after this many attempts. Defaults to 5. */
64
+ maxAttempts?: number;
65
+ }
66
+
67
+ export interface JobStats {
68
+ pending: number;
69
+ claimed: number;
70
+ failed: number;
71
+ }
72
+
73
+ export type JobHandler = (payload: unknown) => Promise<void> | void;
package/worker.ts ADDED
@@ -0,0 +1,118 @@
1
+ import type { JobQueue } from "./queue.ts";
2
+ import type { JobHandler } from "./sql.ts";
3
+
4
+ export interface WorkerOptions {
5
+ /** How many jobs to claim per pass. */
6
+ batchSize?: number;
7
+ /** Milliseconds between passes when the queue is empty. */
8
+ intervalMs?: number;
9
+ /**
10
+ * A claim older than this is treated as abandoned — the worker that took
11
+ * it died without releasing. Without this, one crashed worker leaves its
12
+ * jobs claimed forever.
13
+ */
14
+ staleMs?: number;
15
+ /** Backoff after a handler throws. */
16
+ backoffMs?: number;
17
+ }
18
+
19
+ /**
20
+ * Polls a JobQueue and runs handlers.
21
+ *
22
+ * Nothing here is clever: claim a batch, run each, complete or fail. The
23
+ * parts that matter are the ones that are easy to get wrong and hard to
24
+ * notice:
25
+ *
26
+ * - A throwing handler is caught per job, so one bad job does not stop the
27
+ * batch. If it did, a single poisonous job would block every job behind
28
+ * it, forever.
29
+ * - SIGTERM drains before exiting, so a deploy does not abandon claimed
30
+ * work. The stale-claim timeout is the backstop when a worker is killed
31
+ * without getting to drain at all.
32
+ */
33
+ export class JobWorker {
34
+ private readonly batchSize: number;
35
+ private readonly intervalMs: number;
36
+ private readonly staleMs: number;
37
+ private readonly backoffMs: number;
38
+ private stopping = false;
39
+ private running = false;
40
+
41
+ constructor(
42
+ private readonly queue: JobQueue,
43
+ private readonly handlers: Record<string, JobHandler>,
44
+ opts: WorkerOptions = {},
45
+ ) {
46
+ this.batchSize = opts.batchSize ?? 10;
47
+ this.intervalMs = opts.intervalMs ?? 1000;
48
+ this.staleMs = opts.staleMs ?? 60_000;
49
+ this.backoffMs = opts.backoffMs ?? 30_000;
50
+ }
51
+
52
+ /** Run one pass. Exposed so the CLI and tests can drive it directly. */
53
+ async tick(): Promise<number> {
54
+ const jobs = await this.queue.claim(this.batchSize, this.staleMs);
55
+ for (const job of jobs) {
56
+ const handler = this.handlers[job.name];
57
+ if (!handler) {
58
+ await this.queue.fail(
59
+ job.id,
60
+ new Error(`no handler registered for "${job.name}"`),
61
+ this.backoffMs,
62
+ );
63
+ continue;
64
+ }
65
+ try {
66
+ let payload: unknown;
67
+ try {
68
+ payload = JSON.parse(job.payload);
69
+ } catch {
70
+ throw new Error(`payload is not valid JSON: ${job.payload}`);
71
+ }
72
+ await handler(payload);
73
+ await this.queue.complete(job.id);
74
+ } catch (error) {
75
+ await this.queue.fail(job.id, error, this.backoffMs);
76
+ }
77
+ }
78
+ return jobs.length;
79
+ }
80
+
81
+ /** Poll until stop() is called. */
82
+ async run(signal?: AbortSignal): Promise<void> {
83
+ while (!this.stopping) {
84
+ this.running = true;
85
+ try {
86
+ const count = await this.tick();
87
+ // Only idle when there was nothing to do. Sleeping after a full
88
+ // batch would cap throughput at batchSize per interval.
89
+ if (count === 0) await sleep(this.intervalMs, signal);
90
+ } catch (error) {
91
+ console.error("jobs worker error:", error);
92
+ await sleep(this.intervalMs, signal);
93
+ } finally {
94
+ this.running = false;
95
+ }
96
+ }
97
+ }
98
+
99
+ /** Stop polling and wait for the in-flight pass to finish. */
100
+ async stop(): Promise<void> {
101
+ this.stopping = true;
102
+ while (this.running) await sleep(10);
103
+ }
104
+ }
105
+
106
+ function sleep(ms: number, signal?: AbortSignal): Promise<void> {
107
+ return new Promise((resolve) => {
108
+ const timer = setTimeout(resolve, ms);
109
+ // A timer keeps the process alive on its own, so unref and let the
110
+ // abort signal do the waking. Otherwise stop() could not be
111
+ // observed until the full interval elapsed.
112
+ (timer as unknown as { unref?: () => void }).unref?.();
113
+ signal?.addEventListener("abort", () => {
114
+ clearTimeout(timer);
115
+ resolve();
116
+ });
117
+ });
118
+ }