@warlock.js/queue 5.24.0 → 5.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/cjs/index.cjs +10 -2
- package/cjs/index.cjs.map +1 -1
- package/esm/define-job.mjs.map +1 -1
- package/esm/queue-connector.mjs.map +1 -1
- package/esm/queue-manager.d.mts +5 -2
- package/esm/queue-manager.d.mts.map +1 -1
- package/esm/queue-manager.mjs +10 -2
- package/esm/queue-manager.mjs.map +1 -1
- package/llms-full.txt +229 -213
- package/package.json +3 -3
- package/skills/configure-queue/SKILL.md +59 -59
- package/skills/define-jobs/SKILL.md +67 -67
- package/skills/manage-failed-jobs/SKILL.md +68 -68
- package/skills/queue-notifications/SKILL.md +35 -35
package/llms-full.txt
CHANGED
|
@@ -6,192 +6,208 @@
|
|
|
6
6
|
|
|
7
7
|
## configure-queue `@warlock.js/queue/configure-queue/SKILL.md`
|
|
8
8
|
|
|
9
|
-
---
|
|
10
|
-
name: configure-queue
|
|
11
|
-
description: 'Configure `@warlock.js/queue`: the declarative `src/config/queue.ts` (`QueueConfig` — `connection`, `prefix`, `defaultQueue`, `defaultJobOptions`, `workers: { enabled, concurrency, shutdownTimeout }`), registering `queueConnector()` in `warlock.config.ts > connectors`, running a dispatch-only process, and graceful shutdown. Programmatic `setQueueConfig` / `startWorkers` / `closeQueue` for scripts and tests. Triggers: `QueueConfig`, `queueConnector`, `setQueueConfig`, `startWorkers`, `closeQueue`, `workers.enabled`, `shutdownTimeout`; "configure the queue", "connect BullMQ to Redis", "disable workers in the web process", "graceful shutdown of jobs". Skip: writing jobs — `@warlock.js/queue/define-jobs/SKILL.md`.'
|
|
12
|
-
---
|
|
13
|
-
|
|
14
|
-
# Configure the queue
|
|
15
|
-
|
|
16
|
-
Redis is required. The config is declarative; the connector applies it.
|
|
17
|
-
|
|
18
|
-
```ts title="src/config/queue.ts"
|
|
19
|
-
import type { QueueConfig } from "@warlock.js/queue";
|
|
20
|
-
|
|
21
|
-
const queueConfig: QueueConfig = {
|
|
22
|
-
connection: { host: "127.0.0.1", port: 6379 }, // any BullMQ connection option, or { url }
|
|
23
|
-
prefix: "my-app", // Redis key prefix, default "warlock"
|
|
24
|
-
defaultJobOptions: { attempts: 3, backoff: { type: "exponential", delay: 1000 } },
|
|
25
|
-
workers: {
|
|
26
|
-
enabled: true, // default true
|
|
27
|
-
concurrency: 5, // jobs in parallel per queue, default 1
|
|
28
|
-
shutdownTimeout: 30_000, // ms to wait for active jobs, default 30000
|
|
29
|
-
},
|
|
30
|
-
};
|
|
31
|
-
|
|
32
|
-
export default queueConfig;
|
|
33
|
-
```
|
|
34
|
-
|
|
35
|
-
```ts title="warlock.config.ts"
|
|
36
|
-
import { defineConfig } from "@warlock.js/core";
|
|
37
|
-
import { queueConnector } from "@warlock.js/queue";
|
|
38
|
-
|
|
39
|
-
export default defineConfig({
|
|
40
|
-
connectors: [queueConnector()],
|
|
41
|
-
});
|
|
42
|
-
```
|
|
43
|
-
|
|
44
|
-
## What the connector does
|
|
45
|
-
|
|
46
|
-
- Starts in the **late** phase — after app code is imported — so every `defineJob` has registered. One worker per queue that has a job.
|
|
47
|
-
- No `queue` config → logs a warning and does nothing.
|
|
48
|
-
- On SIGINT/SIGTERM: workers stop taking jobs, active jobs get up to `shutdownTimeout` ms, then Redis connections close. A job still running after that is retried by another worker once its lock expires.
|
|
49
|
-
|
|
50
|
-
## Dispatch-only process
|
|
51
|
-
|
|
52
|
-
Set `workers.enabled: false`. `dispatch()` still works; nothing is processed in that process. Run the workers elsewhere with the same job definitions:
|
|
53
|
-
|
|
54
|
-
```ts
|
|
55
|
-
import { setQueueConfig, startWorkers, closeQueue } from "@warlock.js/queue";
|
|
56
|
-
import "./jobs"; // modules that call defineJob
|
|
57
|
-
|
|
58
|
-
setQueueConfig({ connection: { host: "127.0.0.1", port: 6379 } });
|
|
59
|
-
await startWorkers();
|
|
60
|
-
process.on("SIGTERM", () => closeQueue());
|
|
61
|
-
```
|
|
62
|
-
|
|
63
|
-
Every process must use the same `prefix`, or they will not see each other's jobs.
|
|
64
|
-
|
|
65
|
-
## Dashboard
|
|
66
|
-
|
|
67
|
-
`QueueConfig` also takes a `dashboard: { enabled, path, middleware }` block — see [`manage-failed-jobs`](../manage-failed-jobs/SKILL.md#dashboard-optional) for the `warlock add bull-board` setup and the production guard (`QueueDashboardUnguardedError`).
|
|
9
|
+
---
|
|
10
|
+
name: configure-queue
|
|
11
|
+
description: 'Configure `@warlock.js/queue`: the declarative `src/config/queue.ts` (`QueueConfig` — `connection`, `prefix`, `defaultQueue`, `defaultJobOptions`, `workers: { enabled, concurrency, shutdownTimeout }`), registering `queueConnector()` in `warlock.config.ts > connectors`, running a dispatch-only process, and graceful shutdown. Programmatic `setQueueConfig` / `startWorkers` / `closeQueue` for scripts and tests. Triggers: `QueueConfig`, `queueConnector`, `setQueueConfig`, `startWorkers`, `closeQueue`, `workers.enabled`, `shutdownTimeout`; "configure the queue", "connect BullMQ to Redis", "disable workers in the web process", "graceful shutdown of jobs". Skip: writing jobs — `@warlock.js/queue/define-jobs/SKILL.md`.'
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
# Configure the queue
|
|
15
|
+
|
|
16
|
+
Redis is required. The config is declarative; the connector applies it.
|
|
17
|
+
|
|
18
|
+
```ts title="src/config/queue.ts"
|
|
19
|
+
import type { QueueConfig } from "@warlock.js/queue";
|
|
20
|
+
|
|
21
|
+
const queueConfig: QueueConfig = {
|
|
22
|
+
connection: { host: "127.0.0.1", port: 6379 }, // any BullMQ connection option, or { url }
|
|
23
|
+
prefix: "my-app", // Redis key prefix, default "warlock"
|
|
24
|
+
defaultJobOptions: { attempts: 3, backoff: { type: "exponential", delay: 1000 } },
|
|
25
|
+
workers: {
|
|
26
|
+
enabled: true, // default true
|
|
27
|
+
concurrency: 5, // jobs in parallel per queue, default 1
|
|
28
|
+
shutdownTimeout: 30_000, // ms to wait for active jobs, default 30000
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
export default queueConfig;
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
```ts title="warlock.config.ts"
|
|
36
|
+
import { defineConfig } from "@warlock.js/core";
|
|
37
|
+
import { queueConnector } from "@warlock.js/queue";
|
|
38
|
+
|
|
39
|
+
export default defineConfig({
|
|
40
|
+
connectors: [queueConnector()],
|
|
41
|
+
});
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## What the connector does
|
|
45
|
+
|
|
46
|
+
- Starts in the **late** phase — after app code is imported — so every `defineJob` has registered. One worker per queue that has a job.
|
|
47
|
+
- No `queue` config → logs a warning and does nothing.
|
|
48
|
+
- On SIGINT/SIGTERM: workers stop taking jobs, active jobs get up to `shutdownTimeout` ms, then Redis connections close. A job still running after that is retried by another worker once its lock expires.
|
|
49
|
+
|
|
50
|
+
## Dispatch-only process
|
|
51
|
+
|
|
52
|
+
Set `workers.enabled: false`. `dispatch()` still works; nothing is processed in that process. Run the workers elsewhere with the same job definitions:
|
|
53
|
+
|
|
54
|
+
```ts
|
|
55
|
+
import { setQueueConfig, startWorkers, closeQueue } from "@warlock.js/queue";
|
|
56
|
+
import "./jobs"; // modules that call defineJob
|
|
57
|
+
|
|
58
|
+
setQueueConfig({ connection: { host: "127.0.0.1", port: 6379 } });
|
|
59
|
+
await startWorkers();
|
|
60
|
+
process.on("SIGTERM", () => closeQueue());
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Every process must use the same `prefix`, or they will not see each other's jobs.
|
|
64
|
+
|
|
65
|
+
## Dashboard
|
|
66
|
+
|
|
67
|
+
`QueueConfig` also takes a `dashboard: { enabled, path, middleware }` block — see [`manage-failed-jobs`](../manage-failed-jobs/SKILL.md#dashboard-optional) for the `warlock add bull-board` setup and the production guard (`QueueDashboardUnguardedError`).
|
|
68
68
|
|
|
69
69
|
|
|
70
70
|
## define-jobs `@warlock.js/queue/define-jobs/SKILL.md`
|
|
71
71
|
|
|
72
|
-
---
|
|
73
|
-
name: define-jobs
|
|
74
|
-
description: 'Define and dispatch background jobs with `@warlock.js/queue`: `defineJob({ name, queue?, attempts?, backoff?, removeOnComplete?, removeOnFail?, handle(payload, ctx) })` returns a typed job; `job.dispatch(payload, { delay, priority, jobId, attempts, backoff })`; the handler context (`id`, `attempt`, `maxAttempts`, `progress()`, `log()`); reading a job with `job.find(id)`. Triggers: `defineJob`, `.dispatch(`, `ctx.progress`, `JobContext`, `DispatchOptions`, `backoff`, `attempts`, `priority`, `jobId`; "run this in the background", "retry with backoff", "delay a job", "job progress", "idempotent dispatch". Skip: config and workers — `@warlock.js/queue/configure-queue/SKILL.md`; failed jobs — `@warlock.js/queue/manage-failed-jobs/SKILL.md`.'
|
|
75
|
-
---
|
|
76
|
-
|
|
77
|
-
# Define and dispatch jobs
|
|
78
|
-
|
|
79
|
-
```ts title="src/app/invoices/jobs/send-invoice.job.ts"
|
|
80
|
-
import { defineJob } from "@warlock.js/queue";
|
|
81
|
-
|
|
82
|
-
export const sendInvoice = defineJob({
|
|
83
|
-
name: "invoices.send", // unique across the app
|
|
84
|
-
attempts: 5, // total tries, default 1
|
|
85
|
-
backoff: { type: "exponential", delay: 2000 }, // or a number = fixed ms
|
|
86
|
-
async handle(payload: { invoiceId: string }, ctx) {
|
|
87
|
-
await ctx.log(`attempt ${ctx.attempt} of ${ctx.maxAttempts}`);
|
|
88
|
-
await ctx.progress(50);
|
|
89
|
-
// throw to fail this attempt; it is retried until attempts run out
|
|
90
|
-
return { sent: true }; // stored as the job result
|
|
91
|
-
},
|
|
92
|
-
});
|
|
93
|
-
```
|
|
94
|
-
|
|
95
|
-
The job module must be imported by the process that runs workers — in a Warlock app, anything under `src/app` that is loaded at boot.
|
|
96
|
-
|
|
97
|
-
## Dispatch
|
|
98
|
-
|
|
99
|
-
```ts
|
|
100
|
-
const { id } = await sendInvoice.dispatch({ invoiceId: "42" });
|
|
101
|
-
|
|
102
|
-
await sendInvoice.dispatch({ invoiceId: "43" }, {
|
|
103
|
-
delay: "10m", // ms number or "500ms" | "30s" | "10m" | "2h" | "1d"
|
|
104
|
-
priority: 1, // 1 runs first; larger numbers later
|
|
105
|
-
jobId: "invoice:43", // a second dispatch with a live id is ignored
|
|
106
|
-
});
|
|
107
|
-
```
|
|
108
|
-
|
|
109
|
-
Option precedence: dispatch options > `defineJob` > `queue.defaultJobOptions`.
|
|
110
|
-
|
|
111
|
-
## Read a job
|
|
112
|
-
|
|
113
|
-
```ts
|
|
114
|
-
const job = await sendInvoice.find(id);
|
|
115
|
-
// { id, name, queue, state, payload, progress, attemptsMade, result, failedReason, createdAt, finishedAt }
|
|
116
|
-
```
|
|
117
|
-
|
|
118
|
-
##
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
72
|
+
---
|
|
73
|
+
name: define-jobs
|
|
74
|
+
description: 'Define and dispatch background jobs with `@warlock.js/queue`: `defineJob({ name, queue?, attempts?, backoff?, removeOnComplete?, removeOnFail?, handle(payload, ctx) })` returns a typed job; `job.dispatch(payload, { delay, priority, jobId, attempts, backoff })`; the handler context (`id`, `attempt`, `maxAttempts`, `progress()`, `log()`); reading a job with `job.find(id)`. Triggers: `defineJob`, `.dispatch(`, `ctx.progress`, `JobContext`, `DispatchOptions`, `backoff`, `attempts`, `priority`, `jobId`; "run this in the background", "retry with backoff", "delay a job", "job progress", "idempotent dispatch". Skip: config and workers — `@warlock.js/queue/configure-queue/SKILL.md`; failed jobs — `@warlock.js/queue/manage-failed-jobs/SKILL.md`.'
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
# Define and dispatch jobs
|
|
78
|
+
|
|
79
|
+
```ts title="src/app/invoices/jobs/send-invoice.job.ts"
|
|
80
|
+
import { defineJob } from "@warlock.js/queue";
|
|
81
|
+
|
|
82
|
+
export const sendInvoice = defineJob({
|
|
83
|
+
name: "invoices.send", // unique across the app
|
|
84
|
+
attempts: 5, // total tries, default 1
|
|
85
|
+
backoff: { type: "exponential", delay: 2000 }, // or a number = fixed ms
|
|
86
|
+
async handle(payload: { invoiceId: string }, ctx) {
|
|
87
|
+
await ctx.log(`attempt ${ctx.attempt} of ${ctx.maxAttempts}`);
|
|
88
|
+
await ctx.progress(50);
|
|
89
|
+
// throw to fail this attempt; it is retried until attempts run out
|
|
90
|
+
return { sent: true }; // stored as the job result
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
The job module must be imported by the process that runs workers — in a Warlock app, anything under `src/app` that is loaded at boot.
|
|
96
|
+
|
|
97
|
+
## Dispatch
|
|
98
|
+
|
|
99
|
+
```ts
|
|
100
|
+
const { id } = await sendInvoice.dispatch({ invoiceId: "42" });
|
|
101
|
+
|
|
102
|
+
await sendInvoice.dispatch({ invoiceId: "43" }, {
|
|
103
|
+
delay: "10m", // ms number or "500ms" | "30s" | "10m" | "2h" | "1d"
|
|
104
|
+
priority: 1, // 1 runs first; larger numbers later
|
|
105
|
+
jobId: "invoice:43", // a second dispatch with a live id is ignored
|
|
106
|
+
});
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Option precedence: dispatch options > `defineJob` > `queue.defaultJobOptions`.
|
|
110
|
+
|
|
111
|
+
## Read a job
|
|
112
|
+
|
|
113
|
+
```ts
|
|
114
|
+
const job = await sendInvoice.find(id);
|
|
115
|
+
// { id, name, queue, state, payload, progress, attemptsMade, result, failedReason, createdAt, finishedAt }
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## Carry request context into jobs
|
|
119
|
+
|
|
120
|
+
```ts
|
|
121
|
+
setQueueContext(
|
|
122
|
+
defineQueueContext({
|
|
123
|
+
capture: () => tenantStorage.getStore(), // runs inside dispatch()
|
|
124
|
+
restore: (tenant, run) => tenantStorage.run(tenant, run), // wraps handle()
|
|
125
|
+
}),
|
|
126
|
+
);
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
- `undefined` from `capture` carries nothing; the value must be JSON-safe. Read it in a handler as `ctx.context` (`unknown`).
|
|
130
|
+
- Jobs dispatched from a restored handler capture the same context.
|
|
131
|
+
- `defineJob({ context: false })` opts a job out.
|
|
132
|
+
- Throw `UnrecoverableJobError` from a handler or `restore` to fail the job without retries.
|
|
133
|
+
|
|
134
|
+
## Rules
|
|
135
|
+
|
|
136
|
+
- A job name with no handler in the worker process fails at once, without retries.
|
|
137
|
+
- Payloads are stored as JSON: pass ids, not model instances.
|
|
138
|
+
- Defining the same name again replaces the handler (this is what keeps dev reloads working), so keep names unique.
|
|
123
139
|
|
|
124
140
|
|
|
125
141
|
## manage-failed-jobs `@warlock.js/queue/manage-failed-jobs/SKILL.md`
|
|
126
142
|
|
|
127
|
-
---
|
|
128
|
-
name: manage-failed-jobs
|
|
129
|
-
description: 'Inspect and retry failed `@warlock.js/queue` jobs: `failedJobs({ queue, start, end })` returns `FailedJob[]` (`id`, `name`, `payload`, `attemptsMade`, `failedReason`, `stacktrace`, `failedAt`, `retry()`), `retryFailedJob(id, { queue })` (throws `FailedJobNotFoundError`), and the bull-board UI — `warlock add bull-board` (requires: ["queue"], so it installs queue first when missing) writes a `dashboard: { enabled, path, middleware }` block to `src/config/queue.ts`; a guard middleware is required in production or boot throws `QueueDashboardUnguardedError`. Manual `queueDashboard(server, { basePath, queues, middleware })` also available — needs `@bull-board/api` + `@bull-board/fastify`, loaded only on call, missing ones throw `QueueDashboardDependencyError`. Triggers: `failedJobs`, `retryFailedJob`, `queueDashboard`, `bull-board`, `dashboard.enabled`, `QueueDashboardUnguardedError`; "list failed jobs", "retry a failed job", "queue dashboard", "job admin UI". Skip: defining retries — `@warlock.js/queue/define-jobs/SKILL.md`.'
|
|
130
|
-
---
|
|
131
|
-
|
|
132
|
-
# Failed jobs
|
|
133
|
-
|
|
134
|
-
A job is failed once it has used every attempt, or failed in a way that cannot be retried (for example, no handler for its name). Failed jobs are kept unless you set `removeOnFail`.
|
|
135
|
-
|
|
136
|
-
```ts
|
|
137
|
-
import { failedJobs, retryFailedJob } from "@warlock.js/queue";
|
|
138
|
-
|
|
139
|
-
const failed = await failedJobs({ queue: "default", start: 0, end: 49 }); // newest first
|
|
140
|
-
|
|
141
|
-
for (const job of failed) {
|
|
142
|
-
console.log(job.name, job.failedReason, job.attemptsMade);
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
await failed[0]?.retry(); // back to waiting
|
|
146
|
-
await retryFailedJob("invoice:43"); // by id; throws FailedJobNotFoundError if not failed
|
|
147
|
-
```
|
|
148
|
-
|
|
149
|
-
## Dashboard (optional)
|
|
150
|
-
|
|
151
|
-
```sh
|
|
152
|
-
warlock add bull-board
|
|
153
|
-
```
|
|
154
|
-
|
|
155
|
-
This installs `@bull-board/api` + `@bull-board/fastify` and writes a `dashboard` block to `src/config/queue.ts`. It `requires: ["queue"]`, so it adds the `queue` feature first automatically when it isn't installed yet — no need to run `warlock add queue` yourself first:
|
|
156
|
-
|
|
157
|
-
```ts title="src/config/queue.ts"
|
|
158
|
-
import type { QueueConfig } from "@warlock.js/queue";
|
|
159
|
-
import { middleware } from "@warlock.js/core";
|
|
160
|
-
import { authMiddleware } from "@warlock.js/auth";
|
|
161
|
-
|
|
162
|
-
const queueConfig: QueueConfig = {
|
|
163
|
-
connection: { host: "127.0.0.1", port: 6379 },
|
|
164
|
-
dashboard: {
|
|
165
|
-
enabled: true,
|
|
166
|
-
path: "/admin/queues", // default
|
|
167
|
-
middleware: [authMiddleware("admin")], // guards every dashboard route
|
|
168
|
-
},
|
|
169
|
-
};
|
|
170
|
-
|
|
171
|
-
export default queueConfig;
|
|
172
|
-
```
|
|
173
|
-
|
|
174
|
-
- The dashboard can retry and delete jobs, so it must be guarded. In production, an empty `middleware` list throws `QueueDashboardUnguardedError` at boot instead of mounting exposed. Outside production it logs a warning and mounts anyway.
|
|
175
|
-
- `queueConnector()` mounts the dashboard at boot from this config — no extra wiring needed.
|
|
176
|
-
|
|
177
|
-
### Advanced: manual `queueDashboard()`
|
|
178
|
-
|
|
179
|
-
For mounting outside the config-driven path (a custom server, a non-standard boot sequence), call `queueDashboard` yourself:
|
|
180
|
-
|
|
181
|
-
```ts
|
|
182
|
-
import { getHttpServer } from "@warlock.js/core";
|
|
183
|
-
import { queueDashboard } from "@warlock.js/queue";
|
|
184
|
-
import { authMiddleware } from "@warlock.js/auth";
|
|
185
|
-
|
|
186
|
-
await queueDashboard(getHttpServer(), {
|
|
187
|
-
basePath: "/admin/queues",
|
|
188
|
-
middleware: [authMiddleware("admin")],
|
|
189
|
-
});
|
|
190
|
-
```
|
|
191
|
-
|
|
192
|
-
- Call it before the HTTP server starts listening.
|
|
193
|
-
- Shows every queue that has a job, plus the default queue, unless you pass `queues`.
|
|
194
|
-
- It has no authentication of its own — pass `middleware` yourself; this path does not enforce the production guard that the `dashboard` config does.
|
|
143
|
+
---
|
|
144
|
+
name: manage-failed-jobs
|
|
145
|
+
description: 'Inspect and retry failed `@warlock.js/queue` jobs: `failedJobs({ queue, start, end })` returns `FailedJob[]` (`id`, `name`, `payload`, `attemptsMade`, `failedReason`, `stacktrace`, `failedAt`, `retry()`), `retryFailedJob(id, { queue })` (throws `FailedJobNotFoundError`), and the bull-board UI — `warlock add bull-board` (requires: ["queue"], so it installs queue first when missing) writes a `dashboard: { enabled, path, middleware }` block to `src/config/queue.ts`; a guard middleware is required in production or boot throws `QueueDashboardUnguardedError`. Manual `queueDashboard(server, { basePath, queues, middleware })` also available — needs `@bull-board/api` + `@bull-board/fastify`, loaded only on call, missing ones throw `QueueDashboardDependencyError`. Triggers: `failedJobs`, `retryFailedJob`, `queueDashboard`, `bull-board`, `dashboard.enabled`, `QueueDashboardUnguardedError`; "list failed jobs", "retry a failed job", "queue dashboard", "job admin UI". Skip: defining retries — `@warlock.js/queue/define-jobs/SKILL.md`.'
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
# Failed jobs
|
|
149
|
+
|
|
150
|
+
A job is failed once it has used every attempt, or failed in a way that cannot be retried (for example, no handler for its name). Failed jobs are kept unless you set `removeOnFail`.
|
|
151
|
+
|
|
152
|
+
```ts
|
|
153
|
+
import { failedJobs, retryFailedJob } from "@warlock.js/queue";
|
|
154
|
+
|
|
155
|
+
const failed = await failedJobs({ queue: "default", start: 0, end: 49 }); // newest first
|
|
156
|
+
|
|
157
|
+
for (const job of failed) {
|
|
158
|
+
console.log(job.name, job.failedReason, job.attemptsMade);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
await failed[0]?.retry(); // back to waiting
|
|
162
|
+
await retryFailedJob("invoice:43"); // by id; throws FailedJobNotFoundError if not failed
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## Dashboard (optional)
|
|
166
|
+
|
|
167
|
+
```sh
|
|
168
|
+
warlock add bull-board
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
This installs `@bull-board/api` + `@bull-board/fastify` and writes a `dashboard` block to `src/config/queue.ts`. It `requires: ["queue"]`, so it adds the `queue` feature first automatically when it isn't installed yet — no need to run `warlock add queue` yourself first:
|
|
172
|
+
|
|
173
|
+
```ts title="src/config/queue.ts"
|
|
174
|
+
import type { QueueConfig } from "@warlock.js/queue";
|
|
175
|
+
import { middleware } from "@warlock.js/core";
|
|
176
|
+
import { authMiddleware } from "@warlock.js/auth";
|
|
177
|
+
|
|
178
|
+
const queueConfig: QueueConfig = {
|
|
179
|
+
connection: { host: "127.0.0.1", port: 6379 },
|
|
180
|
+
dashboard: {
|
|
181
|
+
enabled: true,
|
|
182
|
+
path: "/admin/queues", // default
|
|
183
|
+
middleware: [authMiddleware("admin")], // guards every dashboard route
|
|
184
|
+
},
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
export default queueConfig;
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
- The dashboard can retry and delete jobs, so it must be guarded. In production, an empty `middleware` list throws `QueueDashboardUnguardedError` at boot instead of mounting exposed. Outside production it logs a warning and mounts anyway.
|
|
191
|
+
- `queueConnector()` mounts the dashboard at boot from this config — no extra wiring needed.
|
|
192
|
+
|
|
193
|
+
### Advanced: manual `queueDashboard()`
|
|
194
|
+
|
|
195
|
+
For mounting outside the config-driven path (a custom server, a non-standard boot sequence), call `queueDashboard` yourself:
|
|
196
|
+
|
|
197
|
+
```ts
|
|
198
|
+
import { getHttpServer } from "@warlock.js/core";
|
|
199
|
+
import { queueDashboard } from "@warlock.js/queue";
|
|
200
|
+
import { authMiddleware } from "@warlock.js/auth";
|
|
201
|
+
|
|
202
|
+
await queueDashboard(getHttpServer(), {
|
|
203
|
+
basePath: "/admin/queues",
|
|
204
|
+
middleware: [authMiddleware("admin")],
|
|
205
|
+
});
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
- Call it before the HTTP server starts listening.
|
|
209
|
+
- Shows every queue that has a job, plus the default queue, unless you pass `queues`.
|
|
210
|
+
- It has no authentication of its own — pass `middleware` yourself; this path does not enforce the production guard that the `dashboard` config does.
|
|
195
211
|
|
|
196
212
|
|
|
197
213
|
## overview `@warlock.js/queue/overview/SKILL.md`
|
|
@@ -226,40 +242,40 @@ Durable background jobs. Jobs are stored in **Redis** (required), retried on fai
|
|
|
226
242
|
|
|
227
243
|
## queue-notifications `@warlock.js/queue/queue-notifications/SKILL.md`
|
|
228
244
|
|
|
229
|
-
---
|
|
230
|
-
name: queue-notifications
|
|
231
|
-
description: 'Send `@warlock.js/notifications` `.queue()` deliveries through BullMQ with `bullmqQueue({ queue?, attempts?, backoff? })` from `@warlock.js/notifications` — put it in the `queue` slot of `src/config/notifications.ts`; the queue workers deliver. Honours `SendOptions.delay` (number = seconds, or "10m"), retries a failing `channel.send`, and fails at once for a channel missing from the worker config. Triggers: `bullmqQueue`, `NotificationConfig.queue`, `QueuePackageNotInstalledError`; "queue notifications with BullMQ", "retry notification delivery", "delayed notification". Skip: the herald backend — `@warlock.js/notifications/queue-notifications/SKILL.md`.'
|
|
232
|
-
---
|
|
233
|
-
|
|
234
|
-
# Queue notifications with BullMQ
|
|
235
|
-
|
|
236
|
-
```ts title="src/config/notifications.ts"
|
|
237
|
-
import { type NotificationConfig, bullmqQueue, mailChannel } from "@warlock.js/notifications";
|
|
238
|
-
|
|
239
|
-
const config: NotificationConfig = {
|
|
240
|
-
channels: { mail: mailChannel() },
|
|
241
|
-
queue: bullmqQueue({ attempts: 3, backoff: { type: "exponential", delay: 5000 } }),
|
|
242
|
-
};
|
|
243
|
-
|
|
244
|
-
export default config;
|
|
245
|
-
```
|
|
246
|
-
|
|
247
|
-
`bullmqQueue` ships inside `@warlock.js/notifications` itself and lazy-loads `@warlock.js/queue` (an OPTIONAL peer) the first time `.queue()` runs — install it with `warlock add queue` if it isn't there yet, otherwise the first `.queue()` throws `QueuePackageNotInstalledError`.
|
|
248
|
-
|
|
249
|
-
Also configure the queue itself (`configure-queue`). No separate notifications worker is needed: the delivery is an ordinary job, run by the queue workers.
|
|
250
|
-
|
|
251
|
-
```ts
|
|
252
|
-
await orderShipped.queue(user, { order }); // enqueued, delivered by a worker
|
|
253
|
-
await orderShipped.queue(user, { order }, { delay: "10m" }); // delivered in 10 minutes
|
|
254
|
-
```
|
|
255
|
-
|
|
256
|
-
## Behaviour
|
|
257
|
-
|
|
258
|
-
- The job carries the rendered payload and resolved route; the worker looks the channel up by name in its own notifications config and calls `channel.send`.
|
|
259
|
-
- `delay`: a number is seconds (notifications' convention); a string is a duration.
|
|
260
|
-
- `channel.send` throws → retried per `attempts` / `backoff`, then listed by `failedJobs()`.
|
|
261
|
-
- Channel not configured in the worker → fails at once, no retries.
|
|
262
|
-
- Deliveries run under the job name `warlock.notifications.deliver`.
|
|
263
|
-
|
|
245
|
+
---
|
|
246
|
+
name: queue-notifications
|
|
247
|
+
description: 'Send `@warlock.js/notifications` `.queue()` deliveries through BullMQ with `bullmqQueue({ queue?, attempts?, backoff? })` from `@warlock.js/notifications` — put it in the `queue` slot of `src/config/notifications.ts`; the queue workers deliver. Honours `SendOptions.delay` (number = seconds, or "10m"), retries a failing `channel.send`, and fails at once for a channel missing from the worker config. Triggers: `bullmqQueue`, `NotificationConfig.queue`, `QueuePackageNotInstalledError`; "queue notifications with BullMQ", "retry notification delivery", "delayed notification". Skip: the herald backend — `@warlock.js/notifications/queue-notifications/SKILL.md`.'
|
|
248
|
+
---
|
|
249
|
+
|
|
250
|
+
# Queue notifications with BullMQ
|
|
251
|
+
|
|
252
|
+
```ts title="src/config/notifications.ts"
|
|
253
|
+
import { type NotificationConfig, bullmqQueue, mailChannel } from "@warlock.js/notifications";
|
|
254
|
+
|
|
255
|
+
const config: NotificationConfig = {
|
|
256
|
+
channels: { mail: mailChannel() },
|
|
257
|
+
queue: bullmqQueue({ attempts: 3, backoff: { type: "exponential", delay: 5000 } }),
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
export default config;
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
`bullmqQueue` ships inside `@warlock.js/notifications` itself and lazy-loads `@warlock.js/queue` (an OPTIONAL peer) the first time `.queue()` runs — install it with `warlock add queue` if it isn't there yet, otherwise the first `.queue()` throws `QueuePackageNotInstalledError`.
|
|
264
|
+
|
|
265
|
+
Also configure the queue itself (`configure-queue`). No separate notifications worker is needed: the delivery is an ordinary job, run by the queue workers.
|
|
266
|
+
|
|
267
|
+
```ts
|
|
268
|
+
await orderShipped.queue(user, { order }); // enqueued, delivered by a worker
|
|
269
|
+
await orderShipped.queue(user, { order }, { delay: "10m" }); // delivered in 10 minutes
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
## Behaviour
|
|
273
|
+
|
|
274
|
+
- The job carries the rendered payload and resolved route; the worker looks the channel up by name in its own notifications config and calls `channel.send`.
|
|
275
|
+
- `delay`: a number is seconds (notifications' convention); a string is a duration.
|
|
276
|
+
- `channel.send` throws → retried per `attempts` / `backoff`, then listed by `failedJobs()`.
|
|
277
|
+
- Channel not configured in the worker → fails at once, no retries.
|
|
278
|
+
- Deliveries run under the job name `warlock.notifications.deliver`.
|
|
279
|
+
|
|
264
280
|
|
|
265
281
|
|
package/package.json
CHANGED
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
"peerDependencies": {
|
|
12
12
|
"@bull-board/api": "^9.0.0",
|
|
13
13
|
"@bull-board/fastify": "^9.0.0",
|
|
14
|
-
"@warlock.js/core": "5.
|
|
15
|
-
"@warlock.js/logger": "5.
|
|
14
|
+
"@warlock.js/core": "5.26.0",
|
|
15
|
+
"@warlock.js/logger": "5.26.0"
|
|
16
16
|
},
|
|
17
17
|
"peerDependenciesMeta": {
|
|
18
18
|
"@bull-board/api": {
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
],
|
|
38
38
|
"author": "hassanzohdy",
|
|
39
39
|
"license": "MIT",
|
|
40
|
-
"version": "5.
|
|
40
|
+
"version": "5.26.0",
|
|
41
41
|
"main": "./cjs/index.cjs",
|
|
42
42
|
"module": "./esm/index.mjs",
|
|
43
43
|
"types": "./esm/index.d.mts",
|
|
@@ -1,59 +1,59 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: configure-queue
|
|
3
|
-
description: 'Configure `@warlock.js/queue`: the declarative `src/config/queue.ts` (`QueueConfig` — `connection`, `prefix`, `defaultQueue`, `defaultJobOptions`, `workers: { enabled, concurrency, shutdownTimeout }`), registering `queueConnector()` in `warlock.config.ts > connectors`, running a dispatch-only process, and graceful shutdown. Programmatic `setQueueConfig` / `startWorkers` / `closeQueue` for scripts and tests. Triggers: `QueueConfig`, `queueConnector`, `setQueueConfig`, `startWorkers`, `closeQueue`, `workers.enabled`, `shutdownTimeout`; "configure the queue", "connect BullMQ to Redis", "disable workers in the web process", "graceful shutdown of jobs". Skip: writing jobs — `@warlock.js/queue/define-jobs/SKILL.md`.'
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
# Configure the queue
|
|
7
|
-
|
|
8
|
-
Redis is required. The config is declarative; the connector applies it.
|
|
9
|
-
|
|
10
|
-
```ts title="src/config/queue.ts"
|
|
11
|
-
import type { QueueConfig } from "@warlock.js/queue";
|
|
12
|
-
|
|
13
|
-
const queueConfig: QueueConfig = {
|
|
14
|
-
connection: { host: "127.0.0.1", port: 6379 }, // any BullMQ connection option, or { url }
|
|
15
|
-
prefix: "my-app", // Redis key prefix, default "warlock"
|
|
16
|
-
defaultJobOptions: { attempts: 3, backoff: { type: "exponential", delay: 1000 } },
|
|
17
|
-
workers: {
|
|
18
|
-
enabled: true, // default true
|
|
19
|
-
concurrency: 5, // jobs in parallel per queue, default 1
|
|
20
|
-
shutdownTimeout: 30_000, // ms to wait for active jobs, default 30000
|
|
21
|
-
},
|
|
22
|
-
};
|
|
23
|
-
|
|
24
|
-
export default queueConfig;
|
|
25
|
-
```
|
|
26
|
-
|
|
27
|
-
```ts title="warlock.config.ts"
|
|
28
|
-
import { defineConfig } from "@warlock.js/core";
|
|
29
|
-
import { queueConnector } from "@warlock.js/queue";
|
|
30
|
-
|
|
31
|
-
export default defineConfig({
|
|
32
|
-
connectors: [queueConnector()],
|
|
33
|
-
});
|
|
34
|
-
```
|
|
35
|
-
|
|
36
|
-
## What the connector does
|
|
37
|
-
|
|
38
|
-
- Starts in the **late** phase — after app code is imported — so every `defineJob` has registered. One worker per queue that has a job.
|
|
39
|
-
- No `queue` config → logs a warning and does nothing.
|
|
40
|
-
- On SIGINT/SIGTERM: workers stop taking jobs, active jobs get up to `shutdownTimeout` ms, then Redis connections close. A job still running after that is retried by another worker once its lock expires.
|
|
41
|
-
|
|
42
|
-
## Dispatch-only process
|
|
43
|
-
|
|
44
|
-
Set `workers.enabled: false`. `dispatch()` still works; nothing is processed in that process. Run the workers elsewhere with the same job definitions:
|
|
45
|
-
|
|
46
|
-
```ts
|
|
47
|
-
import { setQueueConfig, startWorkers, closeQueue } from "@warlock.js/queue";
|
|
48
|
-
import "./jobs"; // modules that call defineJob
|
|
49
|
-
|
|
50
|
-
setQueueConfig({ connection: { host: "127.0.0.1", port: 6379 } });
|
|
51
|
-
await startWorkers();
|
|
52
|
-
process.on("SIGTERM", () => closeQueue());
|
|
53
|
-
```
|
|
54
|
-
|
|
55
|
-
Every process must use the same `prefix`, or they will not see each other's jobs.
|
|
56
|
-
|
|
57
|
-
## Dashboard
|
|
58
|
-
|
|
59
|
-
`QueueConfig` also takes a `dashboard: { enabled, path, middleware }` block — see [`manage-failed-jobs`](../manage-failed-jobs/SKILL.md#dashboard-optional) for the `warlock add bull-board` setup and the production guard (`QueueDashboardUnguardedError`).
|
|
1
|
+
---
|
|
2
|
+
name: configure-queue
|
|
3
|
+
description: 'Configure `@warlock.js/queue`: the declarative `src/config/queue.ts` (`QueueConfig` — `connection`, `prefix`, `defaultQueue`, `defaultJobOptions`, `workers: { enabled, concurrency, shutdownTimeout }`), registering `queueConnector()` in `warlock.config.ts > connectors`, running a dispatch-only process, and graceful shutdown. Programmatic `setQueueConfig` / `startWorkers` / `closeQueue` for scripts and tests. Triggers: `QueueConfig`, `queueConnector`, `setQueueConfig`, `startWorkers`, `closeQueue`, `workers.enabled`, `shutdownTimeout`; "configure the queue", "connect BullMQ to Redis", "disable workers in the web process", "graceful shutdown of jobs". Skip: writing jobs — `@warlock.js/queue/define-jobs/SKILL.md`.'
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Configure the queue
|
|
7
|
+
|
|
8
|
+
Redis is required. The config is declarative; the connector applies it.
|
|
9
|
+
|
|
10
|
+
```ts title="src/config/queue.ts"
|
|
11
|
+
import type { QueueConfig } from "@warlock.js/queue";
|
|
12
|
+
|
|
13
|
+
const queueConfig: QueueConfig = {
|
|
14
|
+
connection: { host: "127.0.0.1", port: 6379 }, // any BullMQ connection option, or { url }
|
|
15
|
+
prefix: "my-app", // Redis key prefix, default "warlock"
|
|
16
|
+
defaultJobOptions: { attempts: 3, backoff: { type: "exponential", delay: 1000 } },
|
|
17
|
+
workers: {
|
|
18
|
+
enabled: true, // default true
|
|
19
|
+
concurrency: 5, // jobs in parallel per queue, default 1
|
|
20
|
+
shutdownTimeout: 30_000, // ms to wait for active jobs, default 30000
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export default queueConfig;
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
```ts title="warlock.config.ts"
|
|
28
|
+
import { defineConfig } from "@warlock.js/core";
|
|
29
|
+
import { queueConnector } from "@warlock.js/queue";
|
|
30
|
+
|
|
31
|
+
export default defineConfig({
|
|
32
|
+
connectors: [queueConnector()],
|
|
33
|
+
});
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## What the connector does
|
|
37
|
+
|
|
38
|
+
- Starts in the **late** phase — after app code is imported — so every `defineJob` has registered. One worker per queue that has a job.
|
|
39
|
+
- No `queue` config → logs a warning and does nothing.
|
|
40
|
+
- On SIGINT/SIGTERM: workers stop taking jobs, active jobs get up to `shutdownTimeout` ms, then Redis connections close. A job still running after that is retried by another worker once its lock expires.
|
|
41
|
+
|
|
42
|
+
## Dispatch-only process
|
|
43
|
+
|
|
44
|
+
Set `workers.enabled: false`. `dispatch()` still works; nothing is processed in that process. Run the workers elsewhere with the same job definitions:
|
|
45
|
+
|
|
46
|
+
```ts
|
|
47
|
+
import { setQueueConfig, startWorkers, closeQueue } from "@warlock.js/queue";
|
|
48
|
+
import "./jobs"; // modules that call defineJob
|
|
49
|
+
|
|
50
|
+
setQueueConfig({ connection: { host: "127.0.0.1", port: 6379 } });
|
|
51
|
+
await startWorkers();
|
|
52
|
+
process.on("SIGTERM", () => closeQueue());
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Every process must use the same `prefix`, or they will not see each other's jobs.
|
|
56
|
+
|
|
57
|
+
## Dashboard
|
|
58
|
+
|
|
59
|
+
`QueueConfig` also takes a `dashboard: { enabled, path, middleware }` block — see [`manage-failed-jobs`](../manage-failed-jobs/SKILL.md#dashboard-optional) for the `warlock add bull-board` setup and the production guard (`QueueDashboardUnguardedError`).
|