@basaltkit/queue 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -5
- package/dist/driver.d.ts +87 -0
- package/dist/driver.js +31 -1
- package/dist/drivers/bullmq.d.ts +12 -1
- package/dist/drivers/bullmq.js +45 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +82 -2
- package/dist/manager.d.ts +14 -1
- package/dist/manager.js +15 -0
- package/package.json +7 -3
package/README.md
CHANGED
|
@@ -195,17 +195,35 @@ If a job reaches a worker that hasn't registered it, `UnknownJobError` is thrown
|
|
|
195
195
|
|
|
196
196
|
### CLI commands (`basalt queue:*`)
|
|
197
197
|
|
|
198
|
-
Registering `queuePlugin` also wires
|
|
198
|
+
Registering `queuePlugin` also wires four CLI commands (run via the `@basaltkit/cli` runner):
|
|
199
199
|
|
|
200
200
|
```bash
|
|
201
201
|
basalt queue:work --queue=default --concurrency=5 # run a worker until Ctrl+C
|
|
202
202
|
basalt queue:stats --queue=billing # waiting/active/completed/failed/delayed
|
|
203
203
|
basalt queue:retry --queue=billing --limit=100 # re-enqueue failed jobs
|
|
204
|
+
basalt queue:jobs --queue=billing --states=failed # list individual jobs
|
|
204
205
|
```
|
|
205
206
|
|
|
206
|
-
`queue:
|
|
207
|
-
|
|
208
|
-
|
|
207
|
+
`queue:jobs` flags:
|
|
208
|
+
|
|
209
|
+
| Flag | Default | Description |
|
|
210
|
+
|---|---|---|
|
|
211
|
+
| `--queue` | `default` | Queue to inspect. |
|
|
212
|
+
| `--states` | `completed,failed,waiting,active` | Comma-separated states from `waiting`, `active`, `completed`, `failed`, `delayed`. An unknown state is rejected with the valid list, never silently dropped. |
|
|
213
|
+
| `--limit` | `20` | Maximum rows in **total** (newest first), capped at `1000`. |
|
|
214
|
+
| `--payload` | off | Also print each job's payload (truncated). **Off by default: payloads can contain personal data.** |
|
|
215
|
+
|
|
216
|
+
```
|
|
217
|
+
id name state attempts age
|
|
218
|
+
1042 email.welcome completed 1 3s
|
|
219
|
+
1041 email.welcome failed 3 2m
|
|
220
|
+
2 job(s) on "billing" (completed, failed, waiting, active, limit 20). Payloads hidden — add --payload to include them.
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
`queue:stats`, `queue:retry` and `queue:jobs` need a driver that can introspect job
|
|
224
|
+
state — the **BullMQ** driver (a Redis `connection`). With the inline `sync` driver,
|
|
225
|
+
or a broker driver that cannot read a job without consuming it (RabbitMQ, SQS, Kafka),
|
|
226
|
+
they report the operation as unsupported rather than guessing.
|
|
209
227
|
|
|
210
228
|
### Manual use without a plugin (e.g. in tests)
|
|
211
229
|
|
|
@@ -300,6 +318,7 @@ const manager = app.container.get(QUEUE) // get the QueueManager from the contai
|
|
|
300
318
|
| `work` | `(queue = 'default', { concurrency? }?) => void` | Starts a worker for the queue (no-op on the sync driver). |
|
|
301
319
|
| `stats` | `(queue = 'default') => Promise<QueueStats \| undefined>` | Job counts per state, or `undefined` when the driver can't introspect (sync). |
|
|
302
320
|
| `retryFailed` | `(queue = 'default', { limit? }?) => Promise<number \| undefined>` | Re-enqueues failed jobs; returns the count, or `undefined` when the driver doesn't support it. |
|
|
321
|
+
| `list` | `(queue = 'default', options?: ListJobsOptions) => Promise<JobSummary[] \| undefined>` | Lists individual jobs, newest first, payload already unwrapped from the dispatch envelope. `undefined` when the driver can't list. |
|
|
303
322
|
| `close` | `() => Promise<void>` | Closes workers and connections. |
|
|
304
323
|
|
|
305
324
|
`QueueManagerOptions`:
|
|
@@ -311,6 +330,60 @@ const manager = app.container.get(QUEUE) // get the QueueManager from the contai
|
|
|
311
330
|
| `removeOnComplete` | `JobRetention` | driver default | Default retention for completed jobs. |
|
|
312
331
|
| `removeOnFail` | `JobRetention` | driver default | Default retention for failed jobs. |
|
|
313
332
|
|
|
333
|
+
### Listing jobs — `manager.list(queue?, options?)`
|
|
334
|
+
|
|
335
|
+
The supported way to see *which* jobs are on a queue, without reaching into the
|
|
336
|
+
broker's own client (which re-couples your app to one backend):
|
|
337
|
+
|
|
338
|
+
```ts
|
|
339
|
+
import { QUEUE } from '@basaltkit/queue'
|
|
340
|
+
|
|
341
|
+
const jobs = await app.container.get(QUEUE).list('billing', { states: ['failed'], limit: 10 })
|
|
342
|
+
if (!jobs) {
|
|
343
|
+
// the active driver cannot list — handle it, don't assume an empty queue
|
|
344
|
+
} else {
|
|
345
|
+
for (const job of jobs) console.log(job.id, job.name, job.state, job.payload)
|
|
346
|
+
}
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
`ListJobsOptions`:
|
|
350
|
+
|
|
351
|
+
| Field | Type | Default | Description |
|
|
352
|
+
|---|---|---|---|
|
|
353
|
+
| `states` | `JobState[]` | `['completed', 'failed', 'waiting', 'active']` | Which states to look in. `JobState` = `'waiting' \| 'active' \| 'completed' \| 'failed' \| 'delayed'` — the same vocabulary as `QueueStats`. |
|
|
354
|
+
| `limit` | `number` | `20` (max `1000`) | Maximum jobs returned in **total**, newest first — not per state. Each state is read up to `limit` and the merged result is truncated, so one busy state can't starve the others. |
|
|
355
|
+
|
|
356
|
+
**Why `completed` and `failed` are in the default.** A worker drains `waiting` in
|
|
357
|
+
milliseconds, so a healthy queue shows `waiting: 0, active: 0` almost always.
|
|
358
|
+
Defaulting to only those would answer "did my job run?" with an empty list on a
|
|
359
|
+
queue that is working perfectly. `delayed` is *not* in the default — it is a
|
|
360
|
+
separate question; ask for it explicitly.
|
|
361
|
+
|
|
362
|
+
`JobSummary` (driver-neutral — never the backend's own job object):
|
|
363
|
+
|
|
364
|
+
| Field | Type | Description |
|
|
365
|
+
|---|---|---|
|
|
366
|
+
| `id` | `string` | Backend-assigned job id. |
|
|
367
|
+
| `name` | `string` | The job name from `defineJob({ name })`. |
|
|
368
|
+
| `state` | `JobState` | Where the job was found. |
|
|
369
|
+
| `attemptsMade` | `number` | Attempts so far (`0` before the first run). |
|
|
370
|
+
| `timestamp` | `number` | Creation time, epoch ms. |
|
|
371
|
+
| `payload` | `unknown` | **Your** payload — already unwrapped from the `{ payload, context }` dispatch envelope. |
|
|
372
|
+
| `context` | `RequestContext \| undefined` | The request context captured at dispatch (`requestId`, `tenantId`, …), when there was one. |
|
|
373
|
+
| `failedReason` | `string \| undefined` | Why it failed — only for `state: 'failed'`. |
|
|
374
|
+
|
|
375
|
+
**A `JobSummary` carries the job's payload, so it can carry personal data.** Treat
|
|
376
|
+
the result like the records it came from: never log it wholesale, and if you expose
|
|
377
|
+
it over HTTP, require authentication (`meta: { auth: true }`), authorize it per
|
|
378
|
+
tenant, and put raw payloads behind an explicit opt-in. `basalt queue:jobs` follows
|
|
379
|
+
the same rule — it hides payloads unless you pass `--payload`.
|
|
380
|
+
|
|
381
|
+
`readJobEnvelope(data)` is exported for custom drivers implementing `list`: it
|
|
382
|
+
opens the `{ payload, context }` envelope defensively, treating anything that
|
|
383
|
+
isn't one (an older producer, a hand-written job) as the payload itself.
|
|
384
|
+
`DEFAULT_LIST_STATES`, `DEFAULT_LIST_LIMIT` and `MAX_LIST_LIMIT` are exported so
|
|
385
|
+
a custom driver defaults the same way BullMQ does.
|
|
386
|
+
|
|
314
387
|
### `queuedOn<T>(bus, manager, event, handler, options?): () => void`
|
|
315
388
|
|
|
316
389
|
Creates the event→job bridge. Returns the subscription cancel function.
|
|
@@ -327,7 +400,20 @@ Creates the event→job bridge. Returns the subscription cancel function.
|
|
|
327
400
|
|
|
328
401
|
- **`class SyncQueueDriver`** — runs inline on `dispatch`, honors `attempts` (immediate retry). Public property `executed: { queue, jobName, attempts }[]` with the execution history (capped at 1000 entries). For testing and dev without Redis.
|
|
329
402
|
- **`class BullmqQueueDriver`** — production over Redis; see the options table below.
|
|
330
|
-
- **`interface QueueDriver`** (Advanced) — contract for custom drivers: `setExecutor(executor)`, `add(queue, jobName, data, options: AddJobOptions)`, `startWorker(queue, { concurrency? })`, optional `stats(queue)` / `retryFailed(queue, { limit? })`, `close()`, plus the optional `name` and `capabilities` fields. Helper types: `AddJobOptions`, `JobExecutor`, `QueueStats`, `DriverCapabilities`.
|
|
403
|
+
- **`interface QueueDriver`** (Advanced) — contract for custom drivers: `setExecutor(executor)`, `add(queue, jobName, data, options: AddJobOptions)`, `startWorker(queue, { concurrency? })`, optional `stats(queue)` / `retryFailed(queue, { limit? })` / `list(queue, options)`, `close()`, plus the optional `name` and `capabilities` fields. Helper types: `AddJobOptions`, `JobExecutor`, `QueueStats`, `DriverCapabilities`, `JobState`, `JobSummary`, `JobEnvelope`, `ListJobsOptions`.
|
|
404
|
+
|
|
405
|
+
The three optional methods are a deliberate pattern: a driver **omits** what its
|
|
406
|
+
backend cannot do, `QueueManager` returns `undefined`, and the caller gets an
|
|
407
|
+
honest "unsupported" instead of a guess. Omit `list` rather than implement it
|
|
408
|
+
with a destructive read — merely *looking* at a queue must never change it.
|
|
409
|
+
|
|
410
|
+
| Driver | `stats` | `retryFailed` | `list` | Why |
|
|
411
|
+
|---|:---:|:---:|:---:|---|
|
|
412
|
+
| `bullmq` | ✅ | ✅ | ✅ | Redis keeps jobs; reading them is non-destructive. |
|
|
413
|
+
| `sync` | ❌ | ❌ | ❌ | Runs inline and stores nothing. |
|
|
414
|
+
| [`rabbitmq`](https://www.npmjs.com/package/@basaltkit/queue-rabbitmq) | ❌ | ❌ | ❌ | AMQP has no non-destructive read: `basic.get`/consume hide the message from real workers and mark it redelivered. |
|
|
415
|
+
| [`sqs`](https://www.npmjs.com/package/@basaltkit/queue-sqs) | ❌ | ❌ | ❌ | `ReceiveMessage` starts the visibility timeout and bumps `ApproximateReceiveCount` — peeking could redrive jobs to the DLQ. |
|
|
416
|
+
| [`kafka`](https://www.npmjs.com/package/@basaltkit/queue-kafka) | ❌ | ❌ | ❌ | Reading is non-destructive, but a log has no per-message state — any job states would be invented. |
|
|
331
417
|
|
|
332
418
|
#### `new BullmqQueueDriver(options: BullmqDriverOptions)`
|
|
333
419
|
|
|
@@ -441,6 +527,13 @@ The `onUnsupported` policy caught a dispatch option the active driver can't hono
|
|
|
441
527
|
**A job failed for good and nothing was logged.**
|
|
442
528
|
Only the BullMQ driver reports exhausted jobs, via `onJobFailed` (default `console.error`). If you replaced it with a no-op, you removed the only signal. On the broker drivers, inspect the dead-letter destination (`q.dead`, `<topic>.dead`, `<queue>-dead`).
|
|
443
529
|
|
|
530
|
+
**How do I see what's actually on the queue?**
|
|
531
|
+
`basalt queue:stats` for the counts and `basalt queue:jobs` for the individual jobs
|
|
532
|
+
(or `QUEUE`'s `manager.stats()` / `manager.list()` in code). Don't open the broker's
|
|
533
|
+
own client — that couples your app to one backend and hands you the raw dispatch
|
|
534
|
+
envelope instead of your payload. On a driver that can't introspect, both return
|
|
535
|
+
`undefined` / print "Not supported" — an honest gap, not an empty queue.
|
|
536
|
+
|
|
444
537
|
**Do I need Redis to run the tests?**
|
|
445
538
|
No. Without `connection`, the plugin uses `SyncQueueDriver`. You can also instantiate the driver directly and inspect `driver.executed`.
|
|
446
539
|
|
package/dist/driver.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { RequestContext } from '@basaltkit/core';
|
|
1
2
|
/** Driver-neutral retention: `true`/`false`, a count, or `{ ageMs, count }`. */
|
|
2
3
|
export type RetentionOption = boolean | number | {
|
|
3
4
|
ageMs?: number;
|
|
@@ -42,6 +43,80 @@ export interface QueueStats {
|
|
|
42
43
|
failed: number;
|
|
43
44
|
delayed: number;
|
|
44
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* Driver-neutral job state — deliberately the same vocabulary as
|
|
48
|
+
* {@link QueueStats}, so `stats()` and `list()` describe the same lifecycle.
|
|
49
|
+
* Backend-specific states are mapped onto these by the driver.
|
|
50
|
+
*/
|
|
51
|
+
export type JobState = 'waiting' | 'active' | 'completed' | 'failed' | 'delayed';
|
|
52
|
+
/**
|
|
53
|
+
* States {@link QueueDriver.list} returns when the caller picks none.
|
|
54
|
+
*
|
|
55
|
+
* `completed` and `failed` come FIRST and are included on purpose: a worker
|
|
56
|
+
* drains `waiting` in milliseconds, so a healthy queue has `waiting: 0,
|
|
57
|
+
* active: 0` nearly always. Defaulting to only those would answer "did my job
|
|
58
|
+
* run?" with an empty list on a queue that is working perfectly — the classic
|
|
59
|
+
* false alarm. `delayed` is left out of the default (it is a deliberate,
|
|
60
|
+
* separate question); ask for it explicitly.
|
|
61
|
+
*/
|
|
62
|
+
export declare const DEFAULT_LIST_STATES: readonly JobState[];
|
|
63
|
+
/** Default `limit` for {@link QueueDriver.list} — a screenful, not a dump. */
|
|
64
|
+
export declare const DEFAULT_LIST_LIMIT = 20;
|
|
65
|
+
/** Hard ceiling on `limit`, so an inspection call can't turn into a full scan. */
|
|
66
|
+
export declare const MAX_LIST_LIMIT = 1000;
|
|
67
|
+
export interface ListJobsOptions {
|
|
68
|
+
/** States to look in. Default {@link DEFAULT_LIST_STATES}. */
|
|
69
|
+
states?: JobState[] | undefined;
|
|
70
|
+
/**
|
|
71
|
+
* Maximum jobs returned in TOTAL (newest first), not per state. Default
|
|
72
|
+
* {@link DEFAULT_LIST_LIMIT}, capped at {@link MAX_LIST_LIMIT}.
|
|
73
|
+
*/
|
|
74
|
+
limit?: number | undefined;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* A driver-neutral view of ONE job. Drivers must return this — never their
|
|
78
|
+
* backend's own job object (a BullMQ `Job`, an amqplib message, …). Leaking
|
|
79
|
+
* that would invert the coupling this API exists to remove: the app would be
|
|
80
|
+
* back to writing broker-specific code, just through a Basalt method.
|
|
81
|
+
*/
|
|
82
|
+
export interface JobSummary {
|
|
83
|
+
/** Backend-assigned job id, as a string. */
|
|
84
|
+
id: string;
|
|
85
|
+
/** The job name — what `defineJob({ name })` declared. */
|
|
86
|
+
name: string;
|
|
87
|
+
/** Which state the job was found in. */
|
|
88
|
+
state: JobState;
|
|
89
|
+
/** Attempts made so far (0 before the first run). */
|
|
90
|
+
attemptsMade: number;
|
|
91
|
+
/** When the job was created, epoch milliseconds. */
|
|
92
|
+
timestamp: number;
|
|
93
|
+
/** The APP's payload — unwrapped from the dispatch envelope. */
|
|
94
|
+
payload: unknown;
|
|
95
|
+
/** The request context captured at dispatch (requestId, tenantId, …), if any. */
|
|
96
|
+
context?: RequestContext | undefined;
|
|
97
|
+
/** Why it failed — only meaningful for `state: 'failed'`. */
|
|
98
|
+
failedReason?: string | undefined;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* The wire shape `QueueManager.dispatch` hands to {@link QueueDriver.add} as
|
|
102
|
+
* `data`: the app's payload plus a snapshot of the request context, so the
|
|
103
|
+
* context survives the hop to the worker. Drivers store it opaquely; only
|
|
104
|
+
* `list()` needs to open it — via {@link readJobEnvelope}.
|
|
105
|
+
*/
|
|
106
|
+
export interface JobEnvelope {
|
|
107
|
+
payload: unknown;
|
|
108
|
+
context?: RequestContext | undefined;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Opens a dispatch envelope for {@link JobSummary}. Defensive on purpose: a
|
|
112
|
+
* queue can also hold data written by an older version or by a non-Basalt
|
|
113
|
+
* producer, and an inspection command must never throw on it — anything that
|
|
114
|
+
* is not an envelope is reported as the payload itself.
|
|
115
|
+
*/
|
|
116
|
+
export declare function readJobEnvelope(data: unknown): {
|
|
117
|
+
payload: unknown;
|
|
118
|
+
context?: RequestContext | undefined;
|
|
119
|
+
};
|
|
45
120
|
/** Queue driver contract. BullMQ in production; sync in tests/dev. */
|
|
46
121
|
export interface QueueDriver {
|
|
47
122
|
/** Short identifier used in diagnostics (e.g. 'bullmq', 'sync'). */
|
|
@@ -68,5 +143,17 @@ export interface QueueDriver {
|
|
|
68
143
|
retryFailed?(queue: string, options?: {
|
|
69
144
|
limit?: number;
|
|
70
145
|
}): Promise<number>;
|
|
146
|
+
/**
|
|
147
|
+
* Optional: list individual jobs (`basalt queue:jobs`) as driver-neutral
|
|
148
|
+
* {@link JobSummary} objects — newest first, `payload` unwrapped from the
|
|
149
|
+
* dispatch envelope via {@link readJobEnvelope}.
|
|
150
|
+
*
|
|
151
|
+
* Omit it when the backend cannot read a job WITHOUT consuming it: faking
|
|
152
|
+
* the capability with a destructive read (an AMQP `basic.get`, an SQS
|
|
153
|
+
* `ReceiveMessage`) would make merely *looking* at a queue change it. The
|
|
154
|
+
* QueueManager then returns `undefined` and the CLI reports the operation as
|
|
155
|
+
* unsupported — an honest gap beats a dangerous guess.
|
|
156
|
+
*/
|
|
157
|
+
list?(queue: string, options?: ListJobsOptions): Promise<JobSummary[]>;
|
|
71
158
|
close(): Promise<void>;
|
|
72
159
|
}
|
package/dist/driver.js
CHANGED
|
@@ -1 +1,31 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* States {@link QueueDriver.list} returns when the caller picks none.
|
|
3
|
+
*
|
|
4
|
+
* `completed` and `failed` come FIRST and are included on purpose: a worker
|
|
5
|
+
* drains `waiting` in milliseconds, so a healthy queue has `waiting: 0,
|
|
6
|
+
* active: 0` nearly always. Defaulting to only those would answer "did my job
|
|
7
|
+
* run?" with an empty list on a queue that is working perfectly — the classic
|
|
8
|
+
* false alarm. `delayed` is left out of the default (it is a deliberate,
|
|
9
|
+
* separate question); ask for it explicitly.
|
|
10
|
+
*/
|
|
11
|
+
export const DEFAULT_LIST_STATES = ['completed', 'failed', 'waiting', 'active'];
|
|
12
|
+
/** Default `limit` for {@link QueueDriver.list} — a screenful, not a dump. */
|
|
13
|
+
export const DEFAULT_LIST_LIMIT = 20;
|
|
14
|
+
/** Hard ceiling on `limit`, so an inspection call can't turn into a full scan. */
|
|
15
|
+
export const MAX_LIST_LIMIT = 1000;
|
|
16
|
+
/**
|
|
17
|
+
* Opens a dispatch envelope for {@link JobSummary}. Defensive on purpose: a
|
|
18
|
+
* queue can also hold data written by an older version or by a non-Basalt
|
|
19
|
+
* producer, and an inspection command must never throw on it — anything that
|
|
20
|
+
* is not an envelope is reported as the payload itself.
|
|
21
|
+
*/
|
|
22
|
+
export function readJobEnvelope(data) {
|
|
23
|
+
if (typeof data === 'object' && data !== null && 'payload' in data) {
|
|
24
|
+
const envelope = data;
|
|
25
|
+
return {
|
|
26
|
+
payload: envelope.payload,
|
|
27
|
+
...(envelope.context !== undefined ? { context: envelope.context } : {}),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
return { payload: data };
|
|
31
|
+
}
|
package/dist/drivers/bullmq.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type ConnectionOptions } from 'bullmq';
|
|
2
|
-
import type
|
|
2
|
+
import { type AddJobOptions, type JobExecutor, type JobSummary, type ListJobsOptions, type QueueDriver, type QueueStats } from '../driver.js';
|
|
3
3
|
export interface BullmqDriverOptions {
|
|
4
4
|
/** Redis URL (redis://... or rediss://...) or ioredis connection options. */
|
|
5
5
|
connection: string | ConnectionOptions;
|
|
@@ -48,6 +48,17 @@ export declare class BullmqQueueDriver implements QueueDriver {
|
|
|
48
48
|
retryFailed(queue: string, options?: {
|
|
49
49
|
limit?: number;
|
|
50
50
|
}): Promise<number>;
|
|
51
|
+
/**
|
|
52
|
+
* Reads jobs WITHOUT consuming them — Redis keeps finished jobs (subject to
|
|
53
|
+
* retention), so inspection here is non-destructive, which is why BullMQ can
|
|
54
|
+
* offer `list` at all.
|
|
55
|
+
*
|
|
56
|
+
* One `getJobs` per state, each capped at `limit`: that keeps every job's
|
|
57
|
+
* state known without an extra `job.getState()` round-trip per job, and
|
|
58
|
+
* stops one busy state (usually `completed`) from starving the others. The
|
|
59
|
+
* merged result is sorted newest-first and truncated to `limit`.
|
|
60
|
+
*/
|
|
61
|
+
list(queue: string, options?: ListJobsOptions): Promise<JobSummary[]>;
|
|
51
62
|
close(): Promise<void>;
|
|
52
63
|
private queue;
|
|
53
64
|
}
|
package/dist/drivers/bullmq.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Queue, Worker } from 'bullmq';
|
|
2
|
+
import { DEFAULT_LIST_LIMIT, DEFAULT_LIST_STATES, MAX_LIST_LIMIT, readJobEnvelope, } from '../driver.js';
|
|
2
3
|
/** Map the driver-neutral retention to BullMQ's (age in seconds), falling back to a default. */
|
|
3
4
|
function toBullRetention(retention, fallback) {
|
|
4
5
|
if (retention === undefined)
|
|
@@ -84,6 +85,32 @@ export class BullmqQueueDriver {
|
|
|
84
85
|
}
|
|
85
86
|
return retried;
|
|
86
87
|
}
|
|
88
|
+
/**
|
|
89
|
+
* Reads jobs WITHOUT consuming them — Redis keeps finished jobs (subject to
|
|
90
|
+
* retention), so inspection here is non-destructive, which is why BullMQ can
|
|
91
|
+
* offer `list` at all.
|
|
92
|
+
*
|
|
93
|
+
* One `getJobs` per state, each capped at `limit`: that keeps every job's
|
|
94
|
+
* state known without an extra `job.getState()` round-trip per job, and
|
|
95
|
+
* stops one busy state (usually `completed`) from starving the others. The
|
|
96
|
+
* merged result is sorted newest-first and truncated to `limit`.
|
|
97
|
+
*/
|
|
98
|
+
async list(queue, options = {}) {
|
|
99
|
+
const requested = options.states?.length ? options.states : DEFAULT_LIST_STATES;
|
|
100
|
+
const states = [...new Set(requested)];
|
|
101
|
+
const limit = Math.min(Math.max(1, Math.trunc(options.limit ?? DEFAULT_LIST_LIMIT)), MAX_LIST_LIMIT);
|
|
102
|
+
const summaries = [];
|
|
103
|
+
for (const state of states) {
|
|
104
|
+
const jobs = await this.queue(queue).getJobs([state], 0, limit - 1);
|
|
105
|
+
for (const job of jobs) {
|
|
106
|
+
// getJobs can yield holes when a job is removed mid-read (retention).
|
|
107
|
+
if (job)
|
|
108
|
+
summaries.push(toSummary(job, state));
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
summaries.sort((a, b) => b.timestamp - a.timestamp);
|
|
112
|
+
return summaries.slice(0, limit);
|
|
113
|
+
}
|
|
87
114
|
async close() {
|
|
88
115
|
await Promise.all(this.workers.map((worker) => worker.close()));
|
|
89
116
|
await Promise.all([...this.queues.values()].map((queue) => queue.close()));
|
|
@@ -98,6 +125,24 @@ export class BullmqQueueDriver {
|
|
|
98
125
|
return queue;
|
|
99
126
|
}
|
|
100
127
|
}
|
|
128
|
+
/**
|
|
129
|
+
* BullMQ `Job` → driver-neutral {@link JobSummary}. The BullMQ object never
|
|
130
|
+
* escapes the driver, and `job.data` (the dispatch envelope) is opened so the
|
|
131
|
+
* caller sees its own payload.
|
|
132
|
+
*/
|
|
133
|
+
function toSummary(job, state) {
|
|
134
|
+
const { payload, context } = readJobEnvelope(job.data);
|
|
135
|
+
return {
|
|
136
|
+
id: String(job.id ?? ''),
|
|
137
|
+
name: job.name,
|
|
138
|
+
state,
|
|
139
|
+
attemptsMade: job.attemptsMade ?? 0,
|
|
140
|
+
timestamp: job.timestamp ?? 0,
|
|
141
|
+
payload,
|
|
142
|
+
...(context !== undefined ? { context } : {}),
|
|
143
|
+
...(job.failedReason ? { failedReason: job.failedReason } : {}),
|
|
144
|
+
};
|
|
145
|
+
}
|
|
101
146
|
function parseRedisUrl(url) {
|
|
102
147
|
const parsed = new URL(url);
|
|
103
148
|
return {
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { type QueueDriver } from './driver.js';
|
|
1
2
|
import { type BullmqDriverOptions } from './drivers/bullmq.js';
|
|
2
|
-
import type { QueueDriver } from './driver.js';
|
|
3
3
|
import type { JobDefinition, JobRetention } from './job.js';
|
|
4
4
|
import { QueueManager, type UnsupportedPolicy } from './manager.js';
|
|
5
5
|
export { defineJob, JobValidationError, JobNotRegisteredError, type JobDefinition, type JobSchema, type JobBackoff, type JobRetention, type DispatchOptions, } from './job.js';
|
|
@@ -7,7 +7,7 @@ export { QueueManager, UnknownJobError, UnsupportedJobOptionError, type Unsuppor
|
|
|
7
7
|
export { queuedOn, type QueuedListenerOptions } from './bridge.js';
|
|
8
8
|
export { SyncQueueDriver } from './drivers/sync.js';
|
|
9
9
|
export { BullmqQueueDriver, type BullmqDriverOptions } from './drivers/bullmq.js';
|
|
10
|
-
export type
|
|
10
|
+
export { readJobEnvelope, DEFAULT_LIST_STATES, DEFAULT_LIST_LIMIT, MAX_LIST_LIMIT, type QueueDriver, type QueueStats, type AddJobOptions, type JobExecutor, type DriverCapabilities, type JobState, type JobSummary, type JobEnvelope, type ListJobsOptions, } from './driver.js';
|
|
11
11
|
export declare const QUEUE: import("@basaltkit/core").Token<QueueManager>;
|
|
12
12
|
export interface QueuePluginOptions {
|
|
13
13
|
/** Jobs known to this process (producer and/or worker). */
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createToken, definePlugin, ensureMetadata } from '@basaltkit/core';
|
|
2
|
+
import { DEFAULT_LIST_LIMIT, DEFAULT_LIST_STATES, } from './driver.js';
|
|
2
3
|
import { BullmqQueueDriver } from './drivers/bullmq.js';
|
|
3
4
|
import { SyncQueueDriver } from './drivers/sync.js';
|
|
4
5
|
import { QueueManager } from './manager.js';
|
|
@@ -7,6 +8,7 @@ export { QueueManager, UnknownJobError, UnsupportedJobOptionError, } from './man
|
|
|
7
8
|
export { queuedOn } from './bridge.js';
|
|
8
9
|
export { SyncQueueDriver } from './drivers/sync.js';
|
|
9
10
|
export { BullmqQueueDriver } from './drivers/bullmq.js';
|
|
11
|
+
export { readJobEnvelope, DEFAULT_LIST_STATES, DEFAULT_LIST_LIMIT, MAX_LIST_LIMIT, } from './driver.js';
|
|
10
12
|
export const QUEUE = createToken('queue');
|
|
11
13
|
export function queuePlugin(options = {}) {
|
|
12
14
|
return definePlugin({
|
|
@@ -58,14 +60,52 @@ export function queuePlugin(options = {}) {
|
|
|
58
60
|
},
|
|
59
61
|
});
|
|
60
62
|
}
|
|
63
|
+
/** The states `queue:jobs --states` accepts — the driver-neutral vocabulary. */
|
|
64
|
+
const LIST_STATES = ['waiting', 'active', 'completed', 'failed', 'delayed'];
|
|
65
|
+
/** Parses `--states failed,waiting`. Throws on an unknown state rather than silently dropping it. */
|
|
66
|
+
function parseStates(raw) {
|
|
67
|
+
if (typeof raw !== 'string' || raw.trim() === '')
|
|
68
|
+
return undefined;
|
|
69
|
+
const states = raw.split(',').map((part) => part.trim()).filter(Boolean);
|
|
70
|
+
const unknown = states.filter((state) => !LIST_STATES.includes(state));
|
|
71
|
+
if (unknown.length > 0) {
|
|
72
|
+
throw new Error(`Unknown job state(s): ${unknown.join(', ')}. Valid states: ${LIST_STATES.join(', ')}.`);
|
|
73
|
+
}
|
|
74
|
+
return states;
|
|
75
|
+
}
|
|
76
|
+
/** Compact age for the CLI table ('3s', '12m', '4h', '2d') — friendlier than an epoch. */
|
|
77
|
+
function formatAge(timestamp, now) {
|
|
78
|
+
const seconds = Math.max(0, Math.round((now - timestamp) / 1000));
|
|
79
|
+
if (seconds < 60)
|
|
80
|
+
return `${seconds}s`;
|
|
81
|
+
if (seconds < 3600)
|
|
82
|
+
return `${Math.round(seconds / 60)}m`;
|
|
83
|
+
if (seconds < 86_400)
|
|
84
|
+
return `${Math.round(seconds / 3600)}h`;
|
|
85
|
+
return `${Math.round(seconds / 86_400)}d`;
|
|
86
|
+
}
|
|
87
|
+
/** One-line, length-capped rendering of an arbitrary value for a table cell. */
|
|
88
|
+
function preview(value, max = 120) {
|
|
89
|
+
let text;
|
|
90
|
+
try {
|
|
91
|
+
text = typeof value === 'string' ? value : JSON.stringify(value);
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
text = String(value);
|
|
95
|
+
}
|
|
96
|
+
text = (text ?? String(value)).replace(/\s+/g, ' ');
|
|
97
|
+
return text.length > max ? `${text.slice(0, max - 1)}…` : text;
|
|
98
|
+
}
|
|
61
99
|
/**
|
|
62
|
-
* Registers `queue:work`, `queue:stats` and `queue:
|
|
63
|
-
* bucket. Commands resolve the manager lazily, so they work with whatever driver
|
|
100
|
+
* Registers `queue:work`, `queue:stats`, `queue:retry` and `queue:jobs` into the
|
|
101
|
+
* CLI command bucket. Commands resolve the manager lazily, so they work with whatever driver
|
|
64
102
|
* the app configured. Registered structurally to avoid a hard @basaltkit/cli dep.
|
|
65
103
|
*/
|
|
66
104
|
function registerQueueCommands(container) {
|
|
67
105
|
const manager = () => container.get(QUEUE);
|
|
68
106
|
const unsupported = 'Not supported by the active queue driver — the inline sync driver keeps no job state. Use the BullMQ driver (a Redis `connection`).';
|
|
107
|
+
const unsupportedList = 'Not supported by the active queue driver — listing jobs needs a backend that can read a job WITHOUT consuming it. ' +
|
|
108
|
+
'Use the BullMQ driver (a Redis `connection`); the sync driver keeps no job state, and the RabbitMQ/SQS/Kafka drivers cannot list non-destructively.';
|
|
69
109
|
ensureMetadata(container).add('commands', {
|
|
70
110
|
name: 'queue:work',
|
|
71
111
|
description: 'Run a worker that processes jobs for a queue (Ctrl+C to stop)',
|
|
@@ -104,4 +144,44 @@ function registerQueueCommands(container) {
|
|
|
104
144
|
io.log(`Re-enqueued ${retried} failed job(s) on "${queue}".`);
|
|
105
145
|
},
|
|
106
146
|
});
|
|
147
|
+
ensureMetadata(container).add('commands', {
|
|
148
|
+
name: 'queue:jobs',
|
|
149
|
+
description: 'List individual jobs on a queue (id/name/state/attempts); --payload also prints their data',
|
|
150
|
+
async handle({ io, flags, }) {
|
|
151
|
+
const queue = typeof flags['queue'] === 'string' ? flags['queue'] : 'default';
|
|
152
|
+
const states = parseStates(flags['states']);
|
|
153
|
+
const limit = typeof flags['limit'] === 'string' ? Number(flags['limit']) : undefined;
|
|
154
|
+
const jobs = await manager().list(queue, {
|
|
155
|
+
...(states !== undefined ? { states } : {}),
|
|
156
|
+
...(limit !== undefined ? { limit } : {}),
|
|
157
|
+
});
|
|
158
|
+
if (!jobs) {
|
|
159
|
+
io.log(unsupportedList);
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
const asked = (states ?? DEFAULT_LIST_STATES).join(', ');
|
|
163
|
+
if (jobs.length === 0) {
|
|
164
|
+
io.log(`No jobs on "${queue}" in ${asked}.`);
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
// Payloads can hold personal data, so they are NOT printed by default —
|
|
168
|
+
// `--payload` is the deliberate opt-in (same posture as the docs' warning).
|
|
169
|
+
const withPayload = flags['payload'] === true || flags['payload'] === 'true';
|
|
170
|
+
const now = Date.now();
|
|
171
|
+
const anyFailed = jobs.some((job) => job.failedReason !== undefined);
|
|
172
|
+
io.table(jobs.map((job) => ({
|
|
173
|
+
id: job.id,
|
|
174
|
+
name: job.name,
|
|
175
|
+
state: job.state,
|
|
176
|
+
attempts: job.attemptsMade,
|
|
177
|
+
age: formatAge(job.timestamp, now),
|
|
178
|
+
...(anyFailed ? { reason: job.failedReason ? preview(job.failedReason, 60) : '' } : {}),
|
|
179
|
+
...(withPayload ? { payload: preview(job.payload) } : {}),
|
|
180
|
+
})));
|
|
181
|
+
io.log(`${jobs.length} job(s) on "${queue}" (${asked}, limit ${limit ?? DEFAULT_LIST_LIMIT}).` +
|
|
182
|
+
(withPayload
|
|
183
|
+
? ' Payloads shown — they can contain personal data.'
|
|
184
|
+
: ' Payloads hidden — add --payload to include them.'));
|
|
185
|
+
},
|
|
186
|
+
});
|
|
107
187
|
}
|
package/dist/manager.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { BasaltError } from '@basaltkit/core';
|
|
2
|
-
import type { QueueDriver, QueueStats } from './driver.js';
|
|
2
|
+
import type { JobSummary, ListJobsOptions, QueueDriver, QueueStats } from './driver.js';
|
|
3
3
|
import { type DispatchOptions, type JobDefinition, type JobDispatcher, type JobRetention } from './job.js';
|
|
4
4
|
export declare class UnknownJobError extends BasaltError {
|
|
5
5
|
constructor(job: string);
|
|
@@ -55,6 +55,19 @@ export declare class QueueManager implements JobDispatcher {
|
|
|
55
55
|
retryFailed(queue?: string, options?: {
|
|
56
56
|
limit?: number;
|
|
57
57
|
}): Promise<number | undefined>;
|
|
58
|
+
/**
|
|
59
|
+
* Lists individual jobs on the queue — newest first, with each job's own
|
|
60
|
+
* payload already unwrapped from the dispatch envelope. Returns `undefined`
|
|
61
|
+
* when the driver can't list (the sync driver keeps no state; a broker that
|
|
62
|
+
* cannot read a message without consuming it deliberately omits `list`).
|
|
63
|
+
*
|
|
64
|
+
* This is the supported alternative to reaching around the framework into
|
|
65
|
+
* the broker's own client, which couples the app to one backend.
|
|
66
|
+
*
|
|
67
|
+
* A summary carries the job's payload, so it can carry personal data —
|
|
68
|
+
* treat the result as sensitive (see the queues guide).
|
|
69
|
+
*/
|
|
70
|
+
list(queue?: string, options?: ListJobsOptions): Promise<JobSummary[] | undefined>;
|
|
58
71
|
close(): Promise<void>;
|
|
59
72
|
/** Executes a job received from the driver: validates, restores the context, runs the handler. */
|
|
60
73
|
private execute;
|
package/dist/manager.js
CHANGED
|
@@ -128,6 +128,21 @@ export class QueueManager {
|
|
|
128
128
|
async retryFailed(queue = 'default', options = {}) {
|
|
129
129
|
return this.driver.retryFailed?.(queue, options);
|
|
130
130
|
}
|
|
131
|
+
/**
|
|
132
|
+
* Lists individual jobs on the queue — newest first, with each job's own
|
|
133
|
+
* payload already unwrapped from the dispatch envelope. Returns `undefined`
|
|
134
|
+
* when the driver can't list (the sync driver keeps no state; a broker that
|
|
135
|
+
* cannot read a message without consuming it deliberately omits `list`).
|
|
136
|
+
*
|
|
137
|
+
* This is the supported alternative to reaching around the framework into
|
|
138
|
+
* the broker's own client, which couples the app to one backend.
|
|
139
|
+
*
|
|
140
|
+
* A summary carries the job's payload, so it can carry personal data —
|
|
141
|
+
* treat the result as sensitive (see the queues guide).
|
|
142
|
+
*/
|
|
143
|
+
async list(queue = 'default', options = {}) {
|
|
144
|
+
return this.driver.list?.(queue, options);
|
|
145
|
+
}
|
|
131
146
|
async close() {
|
|
132
147
|
await this.driver.close();
|
|
133
148
|
}
|
package/package.json
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@basaltkit/queue",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.0",
|
|
4
|
+
"engines": {
|
|
5
|
+
"node": ">=22.5.0"
|
|
6
|
+
},
|
|
4
7
|
"description": "Basalt queues on top of BullMQ: declarative jobs with Zod payloads, context propagation (tenant/requestId) to workers and a sync driver for tests.",
|
|
5
8
|
"license": "MIT",
|
|
6
9
|
"type": "module",
|
|
10
|
+
"sideEffects": false,
|
|
7
11
|
"exports": {
|
|
8
12
|
".": {
|
|
9
13
|
"types": "./dist/index.d.ts",
|
|
@@ -15,8 +19,8 @@
|
|
|
15
19
|
],
|
|
16
20
|
"dependencies": {
|
|
17
21
|
"bullmq": "^6.2.1",
|
|
18
|
-
"@basaltkit/
|
|
19
|
-
"@basaltkit/
|
|
22
|
+
"@basaltkit/core": "^1.3.1",
|
|
23
|
+
"@basaltkit/events": "^1.1.1"
|
|
20
24
|
},
|
|
21
25
|
"devDependencies": {
|
|
22
26
|
"@types/node": "^26.3.0",
|