@sigil-dev/plugin-jobs 0.9.20 → 0.9.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +159 -154
- package/index.ts +480 -452
- package/package.json +3 -3
- package/queue.ts +396 -396
- package/store-bun.ts +137 -137
- package/store-drizzle.ts +86 -86
- package/store.ts +117 -117
- package/worker.ts +128 -128
package/README.md
CHANGED
|
@@ -1,154 +1,159 @@
|
|
|
1
|
-
# @sigil-dev/plugin-jobs
|
|
2
|
-
|
|
3
|
-
Durable background jobs for [Grimoire](https://sigil.cane1712.dev/docs): enqueue, retry, recurring schedules, and a `sigil jobs` CLI.
|
|
4
|
-
|
|
5
|
-
Opt-in. Grimoire core knows nothing about it: not installed means no queue, no table, no worker.
|
|
6
|
-
|
|
7
|
-
## Setup
|
|
8
|
-
|
|
9
|
-
The queue lives in a database `@sigil-dev/plugin-sql` opened — `"db"` unless
|
|
10
|
-
you say otherwise — so there is nothing to connect:
|
|
11
|
-
|
|
12
|
-
```ts
|
|
13
|
-
// src/lib/jobs.ts
|
|
14
|
-
import { jobs } from "@sigil-dev/plugin-jobs";
|
|
15
|
-
|
|
16
|
-
export const queue = jobs({
|
|
17
|
-
inlineWorker: true, // run the worker in the server; see "Running it"
|
|
18
|
-
handlers: {
|
|
19
|
-
"send-welcome": async (p: { email: string }) => sendEmail(p.email),
|
|
20
|
-
"rankings.snapshot": async () => snapshotRankings(),
|
|
21
|
-
},
|
|
22
|
-
schedules: {
|
|
23
|
-
"rankings.snapshot": { every: "5m" },
|
|
24
|
-
},
|
|
25
|
-
});
|
|
26
|
-
```
|
|
27
|
-
|
|
28
|
-
```ts
|
|
29
|
-
// sigil.config.ts
|
|
30
|
-
import { adapter } from "@sigil-dev/plugin-sql/bun";
|
|
31
|
-
import { queue } from "./src/lib/jobs.ts";
|
|
32
|
-
|
|
33
|
-
export default {
|
|
34
|
-
plugins: [adapter(process.env.DATABASE_URL, { maxTotal: 12 }), queue],
|
|
35
|
-
};
|
|
36
|
-
```
|
|
37
|
-
|
|
38
|
-
`database: "userDb"` puts the queue in another plugin-sql database.
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
- **
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
1
|
+
# @sigil-dev/plugin-jobs
|
|
2
|
+
|
|
3
|
+
Durable background jobs for [Grimoire](https://sigil.cane1712.dev/docs): enqueue, retry, recurring schedules, and a `sigil jobs` CLI.
|
|
4
|
+
|
|
5
|
+
Opt-in. Grimoire core knows nothing about it: not installed means no queue, no table, no worker.
|
|
6
|
+
|
|
7
|
+
## Setup
|
|
8
|
+
|
|
9
|
+
The queue lives in a database `@sigil-dev/plugin-sql` opened — `"db"` unless
|
|
10
|
+
you say otherwise — so there is nothing to connect:
|
|
11
|
+
|
|
12
|
+
```ts
|
|
13
|
+
// src/lib/jobs.ts
|
|
14
|
+
import { jobs } from "@sigil-dev/plugin-jobs";
|
|
15
|
+
|
|
16
|
+
export const queue = jobs({
|
|
17
|
+
inlineWorker: true, // run the worker in the server; see "Running it"
|
|
18
|
+
handlers: {
|
|
19
|
+
"send-welcome": async (p: { email: string }) => sendEmail(p.email),
|
|
20
|
+
"rankings.snapshot": async () => snapshotRankings(),
|
|
21
|
+
},
|
|
22
|
+
schedules: {
|
|
23
|
+
"rankings.snapshot": { every: "5m" },
|
|
24
|
+
},
|
|
25
|
+
});
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
```ts
|
|
29
|
+
// sigil.config.ts
|
|
30
|
+
import { adapter } from "@sigil-dev/plugin-sql/bun";
|
|
31
|
+
import { queue } from "./src/lib/jobs.ts";
|
|
32
|
+
|
|
33
|
+
export default {
|
|
34
|
+
plugins: [adapter(process.env.DATABASE_URL, { maxTotal: 12 }), queue],
|
|
35
|
+
};
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
`database: "userDb"` puts the queue in another plugin-sql database. Prefer
|
|
39
|
+
passing the adapter itself � `database: userDb`, where
|
|
40
|
+
`const userDb = adapter({ key: "userDb", url, optional: true })`. A typo is then
|
|
41
|
+
a compile error, and if `userDb` is optional and was not configured the queue
|
|
42
|
+
registers nothing: `queue.jobs.enabled` is `false` and `enqueue` throws, so the
|
|
43
|
+
config needs no conditional. The
|
|
44
|
+
database is looked up when a query runs, so the order of the plugins array
|
|
45
|
+
does not matter.
|
|
46
|
+
|
|
47
|
+
## Enqueue
|
|
48
|
+
|
|
49
|
+
From anywhere on the server — import the queue:
|
|
50
|
+
|
|
51
|
+
```ts
|
|
52
|
+
import { queue } from "$lib/jobs";
|
|
53
|
+
|
|
54
|
+
await queue.jobs.enqueue("send-welcome", { email: "ada@example.com" });
|
|
55
|
+
await queue.jobs.enqueue("send-welcome", { mail: "x" }); // type error: wrong payload
|
|
56
|
+
await queue.jobs.enqueue("send-welcom", {}); // type error: no such handler
|
|
57
|
+
await queue.jobs.enqueue("rankings.snapshot"); // no payload needed
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
In a route it is also `locals.jobs`. With plugin-sql's `perRequestTx`, a job
|
|
61
|
+
enqueued during a request is part of the request's transaction: an order
|
|
62
|
+
that rolls back takes its confirmation email with it.
|
|
63
|
+
|
|
64
|
+
Options: `{ runAt: Date, maxAttempts: number }` (default 5).
|
|
65
|
+
|
|
66
|
+
## Handlers
|
|
67
|
+
|
|
68
|
+
```ts
|
|
69
|
+
async (payload, { id, name, attempt }) => { ... }
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
`id` is stable across retries, so it can key idempotency. `attempt` is 1 on
|
|
73
|
+
the first run. A handler that throws is retried after a backoff (30s by
|
|
74
|
+
default); after `maxAttempts` it stays in the table as failed, for
|
|
75
|
+
`sigil jobs retry`.
|
|
76
|
+
|
|
77
|
+
## Schedules
|
|
78
|
+
|
|
79
|
+
```ts
|
|
80
|
+
schedules: { "rankings.snapshot": { every: "5m", payload: { region: "jp" } } }
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`every` is milliseconds or `"30s"`, `"5m"`, `"1h"`, `"1d"`. Each schedule is
|
|
84
|
+
one row that is rescheduled when it completes, so:
|
|
85
|
+
|
|
86
|
+
- **It runs once however many workers there are.** Every worker makes sure
|
|
87
|
+
the row exists at boot; the primary key admits one, and one worker claims
|
|
88
|
+
it at a time.
|
|
89
|
+
- **A restart keeps its clock**, and a deploy does not fire every schedule
|
|
90
|
+
at once: the first run is one interval after the row is created.
|
|
91
|
+
- **A schedule that keeps failing moves on** to its next run once its
|
|
92
|
+
retries are used up, with the error kept in `last_error`.
|
|
93
|
+
- **A schedule removed from the config is removed from the table.**
|
|
94
|
+
|
|
95
|
+
## Running it
|
|
96
|
+
|
|
97
|
+
`inlineWorker: true` runs the worker inside the server. That is right for
|
|
98
|
+
one process; with `sigil start --scale full=3` every worker also works jobs,
|
|
99
|
+
which is safe (claims are race-free) but may be more than you want.
|
|
100
|
+
|
|
101
|
+
The alternative is a separate process:
|
|
102
|
+
|
|
103
|
+
```sh
|
|
104
|
+
sigil jobs work # run the worker until stopped (drains on SIGTERM)
|
|
105
|
+
sigil jobs list # most recent jobs [--limit 20]
|
|
106
|
+
sigil jobs stats # counts by state
|
|
107
|
+
sigil jobs retry # requeue every exhausted job, or one: retry <id>
|
|
108
|
+
sigil jobs migrate # create the table (also done at boot)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The CLI opens the database from `sigil.config.ts` itself, so it works
|
|
112
|
+
without a running server.
|
|
113
|
+
|
|
114
|
+
## Other stores
|
|
115
|
+
|
|
116
|
+
Instead of `database`, pass `sql` — any object with `query(sql, params)`
|
|
117
|
+
filling `?` placeholders — for a queue that owns its connection:
|
|
118
|
+
|
|
119
|
+
```ts
|
|
120
|
+
import { bunSqlStore } from "@sigil-dev/plugin-jobs";
|
|
121
|
+
jobs({ sql: bunSqlStore(process.env.JOBS_URL!), handlers });
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
A store can declare what its dialect can do; everything defaults to false,
|
|
125
|
+
so a store that declares nothing takes the path that is correct everywhere.
|
|
126
|
+
|
|
127
|
+
| capability | true on | what it changes |
|
|
128
|
+
| --- | --- | --- |
|
|
129
|
+
| `skipLocked` | postgres, MySQL | appends `FOR UPDATE SKIP LOCKED`, so a worker does not block behind rows another holds. Throughput only. |
|
|
130
|
+
| `spaceSeparatedTimestamp` | MySQL | writes `2026-01-01 00:00:00` rather than ISO 8601, which MySQL rejects on a `TIMESTAMP` column. |
|
|
131
|
+
|
|
132
|
+
### Drizzle
|
|
133
|
+
|
|
134
|
+
```ts
|
|
135
|
+
import { drizzleStore, bunSqlRunner } from "@sigil-dev/plugin-jobs";
|
|
136
|
+
sql: drizzleStore(bunSqlRunner(client), "mysql"),
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
This takes the `Bun.SQL` connection, not your Drizzle `db`: Drizzle's
|
|
140
|
+
`execute` ignores a parameter array, and its bun-sql driver emits `$1` on
|
|
141
|
+
every dialect.
|
|
142
|
+
|
|
143
|
+
## Guarantees, precisely
|
|
144
|
+
|
|
145
|
+
**Delivery is at-least-once.** A worker can die between your side effect and the row delete, so a job may run twice. **Handlers must be idempotent.** The alternative — holding a transaction open across every handler, including every HTTP call inside one — is a worse trade than a duplicate email.
|
|
146
|
+
|
|
147
|
+
**Order is not guaranteed.** A job that fails and retries runs after jobs enqueued later. Two jobs that write the same thing should carry what they need to be applied in any order.
|
|
148
|
+
|
|
149
|
+
**Claiming is race-free without `SKIP LOCKED`.** The claim is an `UPDATE` guarded on the row being unclaimed or stale, tagged with a random token, followed by a `SELECT` of that token — so it works on MariaDB, which has no `UPDATE ... RETURNING`. Two workers racing means the loser gets an empty set, not a duplicate job.
|
|
150
|
+
|
|
151
|
+
**A stale claim is reclaimed** after 60s by default, so a hard-killed worker does not strand its jobs.
|
|
152
|
+
|
|
153
|
+
## Dialects
|
|
154
|
+
|
|
155
|
+
Tested against real SQLite, postgres and MariaDB.
|
|
156
|
+
|
|
157
|
+
## License
|
|
158
|
+
|
|
159
|
+
MIT
|