@sigil-dev/plugin-jobs 0.9.33 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/index.ts +427 -414
  2. package/package.json +4 -4
  3. package/queue.ts +305 -293
  4. package/worker.ts +163 -134
package/worker.ts CHANGED
@@ -1,134 +1,163 @@
1
- import { checkPayload, type Job } from "./define.ts";
2
- import type { JobQueue } from "./queue.ts";
3
-
4
- export interface WorkerOptions {
5
- /** How many jobs to claim per pass. */
6
- batchSize?: number;
7
- /** Milliseconds between passes when the queue is empty. */
8
- intervalMs?: number;
9
- /**
10
- * A claim older than this is treated as abandoned — the worker that took
11
- * it died without releasing. Without this, one crashed worker leaves its
12
- * jobs claimed forever.
13
- */
14
- staleMs?: number;
15
- }
16
-
17
- /**
18
- * Polls a JobQueue and runs jobs.
19
- *
20
- * Nothing here is clever: claim a batch, run each, complete or fail. The
21
- * parts that matter are the ones that are easy to get wrong and hard to
22
- * notice:
23
- *
24
- * - A throwing handler is caught per job, so one bad job does not stop the
25
- * batch. If it did, a single poisonous job would block every job behind
26
- * it, forever.
27
- * - A payload the job no longer accepts (queued by an older deploy) is
28
- * failed for good with the reason: another try cannot fix it.
29
- * - SIGTERM drains before exiting, so a deploy does not abandon claimed
30
- * work. The stale-claim timeout is the backstop when a worker is killed
31
- * without getting to drain at all.
32
- */
33
- export class JobWorker {
34
- private readonly batchSize: number;
35
- private readonly intervalMs: number;
36
- private readonly staleMs: number;
37
- private stopping = false;
38
- private running = false;
39
-
40
- constructor(
41
- private readonly queue: JobQueue,
42
- private readonly jobs: Readonly<Record<string, Job<any>>>,
43
- opts: WorkerOptions = {},
44
- ) {
45
- this.batchSize = opts.batchSize ?? 10;
46
- this.intervalMs = opts.intervalMs ?? 1000;
47
- this.staleMs = opts.staleMs ?? 60_000;
48
- }
49
-
50
- /** Run one pass. Exposed so the CLI and tests can drive it directly. */
51
- async tick(): Promise<number> {
52
- const rows = await this.queue.claim(this.batchSize, this.staleMs);
53
- for (const row of rows) {
54
- const job = this.jobs[row.name];
55
- if (!job) {
56
- await this.queue.fail(
57
- row.id,
58
- new Error(`no job named "${row.name}" (is it in src/jobs?)`),
59
- 30_000,
60
- );
61
- continue;
62
- }
63
- let raw: unknown;
64
- try {
65
- raw = JSON.parse(row.payload);
66
- } catch {
67
- await this.queue.bury(
68
- row.id,
69
- `payload is not valid JSON: ${row.payload}`,
70
- );
71
- continue;
72
- }
73
- const checked = await checkPayload(job, raw);
74
- if (!checked.ok) {
75
- await this.queue.bury(
76
- row.id,
77
- `payload no longer matches the job: ${checked.message}`,
78
- );
79
- continue;
80
- }
81
- try {
82
- await job.handler(checked.value, {
83
- id: row.id,
84
- name: row.name,
85
- attempt: row.attempts,
86
- });
87
- await this.queue.complete(row.id);
88
- } catch (error) {
89
- if (this.queue.isScheduled(row.id) && row.attempts >= row.maxAttempts) {
90
- await this.queue.reschedule(
91
- row.id,
92
- error instanceof Error ? error.message : String(error),
93
- );
94
- } else {
95
- await this.queue.fail(row.id, error, job.backoffAfter(row.attempts));
96
- }
97
- }
98
- }
99
- return rows.length;
100
- }
101
-
102
- /** Loop until `stop()`; resolves once the loop has exited. */
103
- async run(signal?: AbortSignal): Promise<void> {
104
- while (!this.stopping) {
105
- this.running = true;
106
- try {
107
- const count = await this.tick();
108
- if (count === 0) await sleep(this.intervalMs, signal);
109
- } catch (error) {
110
- console.error("jobs worker error:", error);
111
- await sleep(this.intervalMs, signal);
112
- } finally {
113
- this.running = false;
114
- }
115
- }
116
- }
117
-
118
- /** Finish the current pass, then stop. */
119
- async stop(): Promise<void> {
120
- this.stopping = true;
121
- while (this.running) await sleep(10);
122
- }
123
- }
124
-
125
- function sleep(ms: number, signal?: AbortSignal): Promise<void> {
126
- return new Promise((resolve) => {
127
- const timer = setTimeout(resolve, ms);
128
- (timer as unknown as { unref?: () => void }).unref?.();
129
- signal?.addEventListener("abort", () => {
130
- clearTimeout(timer);
131
- resolve();
132
- });
133
- });
134
- }
1
+ import { checkPayload, type Job } from "./define.ts";
2
+ import { isMissingTable, type JobQueue } from "./queue.ts";
3
+
4
+ export interface WorkerOptions {
5
+ /** How many jobs to claim per pass. */
6
+ batchSize?: number;
7
+ /** Milliseconds between passes when the queue is empty. */
8
+ intervalMs?: number;
9
+ /**
10
+ * A claim older than this is treated as abandoned — the worker that took
11
+ * it died without releasing. Without this, one crashed worker leaves its
12
+ * jobs claimed forever.
13
+ */
14
+ staleMs?: number;
15
+ /**
16
+ * Run before the first pass, and again before each pass until it returns
17
+ * true: setting the queue up (the schedules) when the table may not
18
+ * exist yet.
19
+ */
20
+ prepare?: () => Promise<boolean>;
21
+ }
22
+
23
+ /**
24
+ * Polls a JobQueue and runs jobs.
25
+ *
26
+ * Nothing here is clever: claim a batch, run each, complete or fail. The
27
+ * parts that matter are the ones that are easy to get wrong and hard to
28
+ * notice:
29
+ *
30
+ * - A throwing handler is caught per job, so one bad job does not stop the
31
+ * batch. If it did, a single poisonous job would block every job behind
32
+ * it, forever.
33
+ * - A payload the job no longer accepts (queued by an older deploy) is
34
+ * failed for good with the reason: another try cannot fix it.
35
+ * - SIGTERM drains before exiting, so a deploy does not abandon claimed
36
+ * work. The stale-claim timeout is the backstop when a worker is killed
37
+ * without getting to drain at all.
38
+ */
39
+ export class JobWorker {
40
+ private readonly batchSize: number;
41
+ private readonly intervalMs: number;
42
+ private readonly staleMs: number;
43
+ private stopping = false;
44
+ private running = false;
45
+ private prepared: boolean;
46
+ private readonly prepare?: () => Promise<boolean>;
47
+ /** Said once: the table is missing. Said again only after it came back and went. */
48
+ private warnedMissing = false;
49
+
50
+ constructor(
51
+ private readonly queue: JobQueue,
52
+ private readonly jobs: Readonly<Record<string, Job<any>>>,
53
+ opts: WorkerOptions = {},
54
+ ) {
55
+ this.batchSize = opts.batchSize ?? 10;
56
+ this.intervalMs = opts.intervalMs ?? 1000;
57
+ this.staleMs = opts.staleMs ?? 60_000;
58
+ this.prepare = opts.prepare;
59
+ this.prepared = !opts.prepare;
60
+ }
61
+
62
+ /** Run one pass. Exposed so the CLI and tests can drive it directly. */
63
+ async tick(): Promise<number> {
64
+ const rows = await this.queue.claim(this.batchSize, this.staleMs);
65
+ for (const row of rows) {
66
+ const job = this.jobs[row.name];
67
+ if (!job) {
68
+ await this.queue.fail(
69
+ row.id,
70
+ new Error(`no job named "${row.name}" (is it in src/jobs?)`),
71
+ 30_000,
72
+ );
73
+ continue;
74
+ }
75
+ let raw: unknown;
76
+ try {
77
+ raw = JSON.parse(row.payload);
78
+ } catch {
79
+ await this.queue.bury(
80
+ row.id,
81
+ `payload is not valid JSON: ${row.payload}`,
82
+ );
83
+ continue;
84
+ }
85
+ const checked = await checkPayload(job, raw);
86
+ if (!checked.ok) {
87
+ await this.queue.bury(
88
+ row.id,
89
+ `payload no longer matches the job: ${checked.message}`,
90
+ );
91
+ continue;
92
+ }
93
+ try {
94
+ await job.handler(checked.value, {
95
+ id: row.id,
96
+ name: row.name,
97
+ attempt: row.attempts,
98
+ });
99
+ await this.queue.complete(row.id);
100
+ } catch (error) {
101
+ if (this.queue.isScheduled(row.id) && row.attempts >= row.maxAttempts) {
102
+ await this.queue.reschedule(
103
+ row.id,
104
+ error instanceof Error ? error.message : String(error),
105
+ );
106
+ } else {
107
+ await this.queue.fail(row.id, error, job.backoffAfter(row.attempts));
108
+ }
109
+ }
110
+ }
111
+ return rows.length;
112
+ }
113
+
114
+ /** Loop until `stop()`; resolves once the loop has exited. */
115
+ async run(signal?: AbortSignal): Promise<void> {
116
+ while (!this.stopping) {
117
+ this.running = true;
118
+ try {
119
+ if (!this.prepared) {
120
+ this.prepared = await this.prepare!();
121
+ // the table isn't there yet: look again in a while, not every second
122
+ if (!this.prepared) {
123
+ await sleep(this.intervalMs * 5, signal);
124
+ continue;
125
+ }
126
+ }
127
+ const count = await this.tick();
128
+ this.warnedMissing = false;
129
+ if (count === 0) await sleep(this.intervalMs, signal);
130
+ } catch (error) {
131
+ if (isMissingTable(error)) {
132
+ if (!this.warnedMissing) console.error(MISSING_TABLE);
133
+ this.warnedMissing = true;
134
+ await sleep(this.intervalMs * 5, signal);
135
+ } else {
136
+ console.error("jobs worker error:", error);
137
+ await sleep(this.intervalMs, signal);
138
+ }
139
+ } finally {
140
+ this.running = false;
141
+ }
142
+ }
143
+ }
144
+
145
+ /** Finish the current pass, then stop. */
146
+ async stop(): Promise<void> {
147
+ this.stopping = true;
148
+ while (this.running) await sleep(10);
149
+ }
150
+ }
151
+
152
+ export const MISSING_TABLE = "plugin-jobs: the queue's table doesn't exist yet. Run `sigil db migrate`; jobs start running once it's there.";
153
+
154
+ function sleep(ms: number, signal?: AbortSignal): Promise<void> {
155
+ return new Promise((resolve) => {
156
+ const timer = setTimeout(resolve, ms);
157
+ (timer as unknown as { unref?: () => void }).unref?.();
158
+ signal?.addEventListener("abort", () => {
159
+ clearTimeout(timer);
160
+ resolve();
161
+ });
162
+ });
163
+ }