@qkitt/queue 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +723 -0
- package/dist/config/config-freeze.util.d.ts +7 -0
- package/dist/config/from-config.d.ts +33 -0
- package/dist/config/index.d.ts +4 -0
- package/dist/config/parse.util.d.ts +12 -0
- package/dist/config/store-resolve.util.d.ts +11 -0
- package/dist/config/types.d.ts +205 -0
- package/dist/config/validate.d.ts +36 -0
- package/dist/events/index.d.ts +48 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +1552 -0
- package/dist/persist/index.d.ts +3 -0
- package/dist/persist/json-codec.util.d.ts +13 -0
- package/dist/persist/memory.d.ts +14 -0
- package/dist/persist/web-storage-access.util.d.ts +10 -0
- package/dist/persist/web-storage.d.ts +59 -0
- package/dist/queue/core/forward.util.d.ts +21 -0
- package/dist/queue/core/layers.util.d.ts +13 -0
- package/dist/queue/core/queue.d.ts +66 -0
- package/dist/queue/index.d.ts +7 -0
- package/dist/queue/persist/create-id.util.d.ts +10 -0
- package/dist/queue/persist/hydrate-gate.util.d.ts +9 -0
- package/dist/queue/persist/persist.support.d.ts +19 -0
- package/dist/queue/persist/persist.types.d.ts +23 -0
- package/dist/queue/persist/row-ids.util.d.ts +14 -0
- package/dist/queue/persist/with-row-persist.d.ts +72 -0
- package/dist/queue/persist/with-snapshot-persist.d.ts +45 -0
- package/dist/queue/persist/write-chain.util.d.ts +12 -0
- package/dist/queue/worker/with-worker.d.ts +56 -0
- package/dist/router/index.d.ts +3 -0
- package/dist/router/match.util.d.ts +24 -0
- package/dist/router/router.d.ts +107 -0
- package/dist/worker/index.d.ts +4 -0
- package/dist/worker/pipeline.d.ts +22 -0
- package/dist/worker/retry.d.ts +31 -0
- package/dist/worker/types.d.ts +5 -0
- package/package.json +64 -0
package/README.md
ADDED
|
@@ -0,0 +1,723 @@
|
|
|
1
|
+
# @qkitt/queue
|
|
2
|
+
|
|
3
|
+
A small, typed **FIFO queue toolkit** for TypeScript: compose queues with workers, retries, pipelines, topic routing, and optional persistence.
|
|
4
|
+
|
|
5
|
+
Zero runtime dependencies. Event-driven. Designed to stay readable and stackable.
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- **Typed FIFO queue** — `enqueue` / `dequeue` / `peek` with lifecycle events
|
|
10
|
+
- **Workers** — process items with concurrency, start/stop, and idle detection
|
|
11
|
+
- **Retries** — wrap any worker with backoff and `shouldRetry`
|
|
12
|
+
- **Pipelines** — chain steps so each step’s output feeds the next
|
|
13
|
+
- **Topic router** — MQTT/AMQP-style patterns (`*`, `#`) into queues
|
|
14
|
+
- **Persistence** — snapshot or row-level stores (memory, `localStorage` / `sessionStorage`)
|
|
15
|
+
- **JS config** — declare queues, persistence, router bindings, and imported workers in one module (JSON subset still supported)
|
|
16
|
+
- **Events everywhere** — typed `on` / `once` / `off` / `emit` with `expand()` for composition
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
npm install @qkitt/queue
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
**Requirements:** Node.js **18+** (see `engines`). **ESM only** — `"type": "module"`, no CommonJS/`require` build.
|
|
25
|
+
|
|
26
|
+
Zero runtime dependencies (TypeScript + Vitest + tsup are dev-only). Published entry is compiled ESM + declarations under `dist/` (`import` / `types`). Tree-shaking friendly (`sideEffects: false`).
|
|
27
|
+
|
|
28
|
+
```ts
|
|
29
|
+
import {
|
|
30
|
+
buildQueue,
|
|
31
|
+
withWorker,
|
|
32
|
+
pipeline,
|
|
33
|
+
withRetry,
|
|
34
|
+
buildRouter,
|
|
35
|
+
withRowPersist,
|
|
36
|
+
withSnapshotPersist,
|
|
37
|
+
createMemoryRowStore,
|
|
38
|
+
createMemorySnapshotStore,
|
|
39
|
+
createLocalStorageRowStore,
|
|
40
|
+
createLocalStorageSnapshotStore,
|
|
41
|
+
buildFromConfig,
|
|
42
|
+
buildFromJson,
|
|
43
|
+
defineConfig,
|
|
44
|
+
} from '@qkitt/queue'
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
## Quick start
|
|
48
|
+
|
|
49
|
+
```ts
|
|
50
|
+
import { buildQueue, withWorker } from '@qkitt/queue'
|
|
51
|
+
|
|
52
|
+
type Job = { id: string; url: string }
|
|
53
|
+
|
|
54
|
+
const queue = withWorker<Job, Response>(
|
|
55
|
+
buildQueue<Job>(),
|
|
56
|
+
async (job) => fetch(job.url),
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
queue.on('worker:completed', ({ item, result }) => {
|
|
60
|
+
console.log(item.id, result.status)
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
queue.on('worker:failed', ({ item, error }) => {
|
|
64
|
+
console.error(item.id, error)
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
queue.enqueue({ id: '1', url: 'https://example.com' })
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Queue
|
|
71
|
+
|
|
72
|
+
```ts
|
|
73
|
+
import { buildQueue, QueueFullError } from '@qkitt/queue'
|
|
74
|
+
|
|
75
|
+
const queue = buildQueue<number>()
|
|
76
|
+
|
|
77
|
+
queue.enqueue(1)
|
|
78
|
+
queue.enqueue(2)
|
|
79
|
+
|
|
80
|
+
queue.peek() // 1 (does not remove)
|
|
81
|
+
queue.size() // 2
|
|
82
|
+
queue.dequeue() // 1
|
|
83
|
+
queue.toArray() // [2]
|
|
84
|
+
queue.isEmpty() // false
|
|
85
|
+
queue.clear() // removes remaining items
|
|
86
|
+
|
|
87
|
+
// Optional capacity (backpressure): enqueue throws when full
|
|
88
|
+
const bounded = buildQueue<number>({ maxSize: 100 })
|
|
89
|
+
try {
|
|
90
|
+
bounded.enqueue(1)
|
|
91
|
+
} catch (error) {
|
|
92
|
+
if (error instanceof QueueFullError) {
|
|
93
|
+
// drop, wait, or reject the producer
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### Queue events
|
|
99
|
+
|
|
100
|
+
| Event | Payload | When |
|
|
101
|
+
| --- | --- | --- |
|
|
102
|
+
| `queue:enqueued` | `{ item, size }` | After an item is added |
|
|
103
|
+
| `queue:dequeued` | `{ item, size }` | After an item is removed |
|
|
104
|
+
| `queue:emptied` | `undefined` | When the last item is dequeued |
|
|
105
|
+
| `queue:cleared` | `{ removed }` | After `clear()` removes items |
|
|
106
|
+
|
|
107
|
+
```ts
|
|
108
|
+
queue.on('queue:enqueued', ({ item, size }) => {
|
|
109
|
+
console.log('added', item, 'size=', size)
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
const unsubscribe = queue.on('queue:emptied', () => {
|
|
113
|
+
console.log('drained')
|
|
114
|
+
})
|
|
115
|
+
// later
|
|
116
|
+
unsubscribe()
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
## Workers
|
|
120
|
+
|
|
121
|
+
`withWorker` dequeues items and runs your async function. By default it **auto-starts** and processes with **concurrency 1**.
|
|
122
|
+
|
|
123
|
+
```ts
|
|
124
|
+
import { buildQueue, withWorker } from '@qkitt/queue'
|
|
125
|
+
|
|
126
|
+
const queue = withWorker(
|
|
127
|
+
buildQueue<string>(),
|
|
128
|
+
async (name) => `hello ${name}`,
|
|
129
|
+
{ concurrency: 3, autoStart: true },
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
queue.on('worker:started', ({ item }) => console.log('start', item))
|
|
133
|
+
queue.on('worker:completed', ({ item, result }) => console.log(item, '→', result))
|
|
134
|
+
queue.on('worker:failed', ({ item, error }) => console.error(item, error))
|
|
135
|
+
queue.on('worker:idle', () => console.log('nothing left to do'))
|
|
136
|
+
|
|
137
|
+
queue.enqueue('a')
|
|
138
|
+
queue.enqueue('b')
|
|
139
|
+
|
|
140
|
+
queue.stop() // stop taking new items (in-flight still finish)
|
|
141
|
+
queue.start() // resume
|
|
142
|
+
queue.isRunning() // boolean
|
|
143
|
+
queue.isProcessing() // any active work?
|
|
144
|
+
queue.activeCount() // how many in flight
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
### Manual start
|
|
148
|
+
|
|
149
|
+
```ts
|
|
150
|
+
const queue = withWorker(buildQueue<number>(), async (n) => n * 2, {
|
|
151
|
+
autoStart: false,
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
queue.enqueue(1)
|
|
155
|
+
queue.enqueue(2)
|
|
156
|
+
// still queued until:
|
|
157
|
+
queue.start()
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### Worker events
|
|
161
|
+
|
|
162
|
+
| Event | Payload | When |
|
|
163
|
+
| --- | --- | --- |
|
|
164
|
+
| `worker:started` | `{ item }` | Just before the worker runs |
|
|
165
|
+
| `worker:completed` | `{ item, result }` | Worker resolved |
|
|
166
|
+
| `worker:failed` | `{ item, error }` | Worker threw/rejected |
|
|
167
|
+
| `worker:idle` | `undefined` | No in-flight work and queue empty |
|
|
168
|
+
|
|
169
|
+
Failed items are **not** re-queued automatically — use `withRetry` or handle `worker:failed`.
|
|
170
|
+
|
|
171
|
+
## Retry
|
|
172
|
+
|
|
173
|
+
```ts
|
|
174
|
+
import { buildQueue, withRetry, withWorker } from '@qkitt/queue'
|
|
175
|
+
|
|
176
|
+
const worker = withRetry(
|
|
177
|
+
async (job: { url: string }) => {
|
|
178
|
+
const res = await fetch(job.url)
|
|
179
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`)
|
|
180
|
+
return res.json()
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
retries: 3, // total attempts = retries + 1
|
|
184
|
+
delay: (attempt) => 100 * 2 ** (attempt - 1), // exponential backoff
|
|
185
|
+
shouldRetry: (error) => !(error instanceof TypeError), // optional filter
|
|
186
|
+
},
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
const queue = withWorker(buildQueue<{ url: string }>(), worker)
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Shorthand for retries only:
|
|
193
|
+
|
|
194
|
+
```ts
|
|
195
|
+
const worker = withRetry(async (n: number) => callApi(n), 2)
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
When all attempts fail, the worker throws a `RetryExhaustedError` class instance:
|
|
199
|
+
|
|
200
|
+
- `instanceof RetryExhaustedError` works across normal module boundaries
|
|
201
|
+
- `attempts` — how many tries ran
|
|
202
|
+
- `cause` — last underlying error
|
|
203
|
+
|
|
204
|
+
## Pipeline
|
|
205
|
+
|
|
206
|
+
Compose steps so the output of step *n* becomes the input of step *n+1*:
|
|
207
|
+
|
|
208
|
+
```ts
|
|
209
|
+
import { buildQueue, pipeline, withWorker } from '@qkitt/queue'
|
|
210
|
+
|
|
211
|
+
const worker = pipeline(
|
|
212
|
+
async (id: string) => fetchUser(id),
|
|
213
|
+
async (user) => enrich(user),
|
|
214
|
+
async (enriched) => save(enriched),
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
const queue = withWorker(buildQueue<string>(), worker)
|
|
218
|
+
queue.enqueue('user-42')
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
Combine with retry:
|
|
222
|
+
|
|
223
|
+
```ts
|
|
224
|
+
const worker = withRetry(
|
|
225
|
+
pipeline(
|
|
226
|
+
async (job: Job) => validate(job),
|
|
227
|
+
async (job) => deliver(job),
|
|
228
|
+
),
|
|
229
|
+
{ retries: 2, delay: 250 },
|
|
230
|
+
)
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
## Topic router
|
|
234
|
+
|
|
235
|
+
Publish on concrete topics; bind queues to patterns (MQTT/AMQP style):
|
|
236
|
+
|
|
237
|
+
| Pattern | Matches |
|
|
238
|
+
| --- | --- |
|
|
239
|
+
| `orders.created` | Exact topic only |
|
|
240
|
+
| `orders.*` | One segment (`orders.created`, not `orders.a.b`) |
|
|
241
|
+
| `orders.#` | Zero or more trailing segments |
|
|
242
|
+
| `#` | Everything |
|
|
243
|
+
|
|
244
|
+
```ts
|
|
245
|
+
import {
|
|
246
|
+
buildQueue,
|
|
247
|
+
buildRouter,
|
|
248
|
+
withWorker,
|
|
249
|
+
type RouteMessage,
|
|
250
|
+
} from '@qkitt/queue'
|
|
251
|
+
|
|
252
|
+
type Order = { id: number; total: number }
|
|
253
|
+
|
|
254
|
+
const router = buildRouter()
|
|
255
|
+
const created = buildQueue<RouteMessage<Order>>()
|
|
256
|
+
const allOrders = buildQueue<RouteMessage>()
|
|
257
|
+
|
|
258
|
+
router.bind('orders.created', created)
|
|
259
|
+
router.bind('orders.#', allOrders)
|
|
260
|
+
|
|
261
|
+
// Optional: process routed messages with a worker
|
|
262
|
+
const createdWorker = withWorker(created, async ({ topic, data }) => {
|
|
263
|
+
console.log(topic, data.id, data.total)
|
|
264
|
+
})
|
|
265
|
+
|
|
266
|
+
router.publish('orders.created', { id: 1, total: 42 })
|
|
267
|
+
// → both queues receive { topic: 'orders.created', data: { id: 1, total: 42 } }
|
|
268
|
+
|
|
269
|
+
router.publish('orders.shipped', { id: 1, carrier: 'ups' })
|
|
270
|
+
// → only `orders.#` matches
|
|
271
|
+
|
|
272
|
+
const unbind = router.bind('jobs.*', buildQueue())
|
|
273
|
+
unbind() // remove this binding only
|
|
274
|
+
|
|
275
|
+
router.on('router:unmatched', ({ topic, data, delivered }) => {
|
|
276
|
+
console.warn('no route for', topic, data, 'sink=', delivered)
|
|
277
|
+
})
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
### Unrouted (unmatched) publishes
|
|
281
|
+
|
|
282
|
+
When nothing binds, the router tracks the miss and can optionally park the message:
|
|
283
|
+
|
|
284
|
+
```ts
|
|
285
|
+
const unrouted = buildQueue<RouteMessage>()
|
|
286
|
+
const router = buildRouter({ unmatchedTarget: unrouted })
|
|
287
|
+
|
|
288
|
+
router.publish('no.binding', { id: 1 })
|
|
289
|
+
// → publish returns 0
|
|
290
|
+
// → unrouted receives { topic: 'no.binding', data: { id: 1 } }
|
|
291
|
+
// → router:unmatched with { delivered: true }
|
|
292
|
+
|
|
293
|
+
router.unmatchedCount() // how many unrouted since last clear
|
|
294
|
+
router.lastUnmatched() // { topic, data } | undefined
|
|
295
|
+
router.clearUnmatched() // reset stats only (does not drain the sink queue)
|
|
296
|
+
router.setUnmatchedTarget(unrouted) // attach / replace / clear (`undefined`)
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
In config, point at a named queue (not a pattern bind):
|
|
300
|
+
|
|
301
|
+
```ts
|
|
302
|
+
router: {
|
|
303
|
+
bindings: [{ pattern: 'mail.#', queue: 'mail' }],
|
|
304
|
+
unmatchedQueue: 'unrouted', // must exist under queues
|
|
305
|
+
}
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
The unmatched sink does **not** count as a match (`publish` still returns `0`).
|
|
309
|
+
|
|
310
|
+
### Router events
|
|
311
|
+
|
|
312
|
+
| Event | Payload |
|
|
313
|
+
| --- | --- |
|
|
314
|
+
| `router:bound` | `{ pattern }` |
|
|
315
|
+
| `router:unbound` | `{ pattern, removed }` |
|
|
316
|
+
| `router:published` | `{ topic, data, matched }` |
|
|
317
|
+
| `router:unmatched` | `{ topic, data, delivered }` — `delivered` if the unmatched sink accepted it |
|
|
318
|
+
| `router:error` | `{ operation, error, topic?, pattern? }` — `operation` is `publish` \| `bind` \| `unmatched` |
|
|
319
|
+
|
|
320
|
+
## Persistence
|
|
321
|
+
|
|
322
|
+
Two strategies:
|
|
323
|
+
|
|
324
|
+
1. **Snapshot** — rewrite the whole queue on change (simple backends)
|
|
325
|
+
2. **Row** — insert/remove per item (DB / per-key storage)
|
|
326
|
+
|
|
327
|
+
### Snapshot (memory)
|
|
328
|
+
|
|
329
|
+
```ts
|
|
330
|
+
import {
|
|
331
|
+
buildQueue,
|
|
332
|
+
withSnapshotPersist,
|
|
333
|
+
createMemorySnapshotStore,
|
|
334
|
+
} from '@qkitt/queue'
|
|
335
|
+
|
|
336
|
+
const store = createMemorySnapshotStore<string>()
|
|
337
|
+
const queue = withSnapshotPersist(buildQueue<string>(), store)
|
|
338
|
+
|
|
339
|
+
await queue.hydrate() // load from store into memory
|
|
340
|
+
queue.enqueue('a') // auto-saves by default
|
|
341
|
+
await queue.persist() // manual save (also used when autoSave is false)
|
|
342
|
+
await queue.flush() // wait for pending auto-saves
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
### Row (memory)
|
|
346
|
+
|
|
347
|
+
```ts
|
|
348
|
+
import {
|
|
349
|
+
buildQueue,
|
|
350
|
+
withRowPersist,
|
|
351
|
+
createMemoryRowStore,
|
|
352
|
+
} from '@qkitt/queue'
|
|
353
|
+
|
|
354
|
+
const store = createMemoryRowStore<string>()
|
|
355
|
+
const queue = withRowPersist(buildQueue<string>(), store)
|
|
356
|
+
// Optional: override default nanoid-style ids
|
|
357
|
+
// withRowPersist(buildQueue<string>(), store, { createId: () => crypto.randomUUID() })
|
|
358
|
+
|
|
359
|
+
await queue.hydrate()
|
|
360
|
+
queue.enqueue('job-1')
|
|
361
|
+
await queue.flush() // wait for async store insert
|
|
362
|
+
queue.rowIds() // stable ids aligned with toArray()
|
|
363
|
+
queue.dequeue()
|
|
364
|
+
await queue.flush()
|
|
365
|
+
```
|
|
366
|
+
|
|
367
|
+
### Durability notes
|
|
368
|
+
|
|
369
|
+
Both persist wrappers keep **sync** `enqueue` / `dequeue` / `clear` and apply the same composition model: they **override** mutation methods, serialize store I/O on a write chain, and use a silent `replaceAll` during `hydrate` (no mid-hydrate worker drain). After hydrate completes they emit one `queue:enqueued` so a stacked worker pumps with store removes/saves enabled. Concurrent mutations during `hydrate` throw.
|
|
370
|
+
|
|
371
|
+
| | Snapshot | Row |
|
|
372
|
+
| --- | --- | --- |
|
|
373
|
+
| Memory vs store | Snapshot rewrites the full list | Optimistic memory; store insert/remove per op |
|
|
374
|
+
| Failed write | `persist:error` on save; memory unchanged by save failure | Failed **insert** rolls that row back out of memory (no clear/enqueue noise); failed remove/clear emit error only (call `hydrate` to resync) |
|
|
375
|
+
| Wait for I/O | `flush()` or `persist()` | `flush()` |
|
|
376
|
+
| Hydrate | Flushes pending saves, then loads | Flushes pending writes, then loads |
|
|
377
|
+
|
|
378
|
+
**Composition (required):** worker must be **outer** so `dequeue` hits the persist override:
|
|
379
|
+
|
|
380
|
+
```ts
|
|
381
|
+
// correct
|
|
382
|
+
withWorker(withRowPersist(buildQueue(), store), worker)
|
|
383
|
+
|
|
384
|
+
// wrong — throws from withRowPersist / withSnapshotPersist if a worker is already attached
|
|
385
|
+
withRowPersist(withWorker(buildQueue(), worker), store)
|
|
386
|
+
```
|
|
387
|
+
|
|
388
|
+
### Browser `localStorage`
|
|
389
|
+
|
|
390
|
+
```ts
|
|
391
|
+
import {
|
|
392
|
+
buildQueue,
|
|
393
|
+
withSnapshotPersist,
|
|
394
|
+
withRowPersist,
|
|
395
|
+
createLocalStorageSnapshotStore,
|
|
396
|
+
createLocalStorageRowStore,
|
|
397
|
+
} from '@qkitt/queue'
|
|
398
|
+
|
|
399
|
+
// Whole queue as one JSON array
|
|
400
|
+
const snapQueue = withSnapshotPersist(
|
|
401
|
+
buildQueue<{ id: string }>(),
|
|
402
|
+
createLocalStorageSnapshotStore('my-app:queue'),
|
|
403
|
+
)
|
|
404
|
+
await snapQueue.hydrate()
|
|
405
|
+
|
|
406
|
+
// One key per row + order list
|
|
407
|
+
const rowQueue = withRowPersist(
|
|
408
|
+
buildQueue<{ id: string }>(),
|
|
409
|
+
createLocalStorageRowStore('my-app:jobs'),
|
|
410
|
+
)
|
|
411
|
+
await rowQueue.hydrate()
|
|
412
|
+
```
|
|
413
|
+
|
|
414
|
+
`sessionStorage` helpers: `createSessionStorageSnapshotStore`, `createSessionStorageRowStore`.
|
|
415
|
+
|
|
416
|
+
**Web Storage limits:** not multi-tab safe and not transactional. Snapshot save is a single key write (last tab wins). Row ops touch multiple keys (`order` + per-row); a quota error or crash mid-op can leave order and rows inconsistent. Prefer a single owning tab, or a server/DB store when durability is shared across tabs or processes.
|
|
417
|
+
|
|
418
|
+
Custom stores implement either:
|
|
419
|
+
|
|
420
|
+
```ts
|
|
421
|
+
type SnapshotStore<T> = {
|
|
422
|
+
load: () => readonly T[] | Promise<readonly T[]>
|
|
423
|
+
save: (items: readonly T[]) => void | Promise<void>
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
type RowStore<T> = {
|
|
427
|
+
loadAll: () => readonly { id: string; item: T }[] | Promise<...>
|
|
428
|
+
insert: (record: { id: string; item: T }) => void | Promise<void>
|
|
429
|
+
remove: (id: string) => void | Promise<void>
|
|
430
|
+
clear: () => void | Promise<void>
|
|
431
|
+
}
|
|
432
|
+
```
|
|
433
|
+
|
|
434
|
+
### Persist events
|
|
435
|
+
|
|
436
|
+
**Snapshot:** `persist:loaded`, `persist:saved`, `persist:error`
|
|
437
|
+
**Row:** `persist:loaded`, `persist:inserted`, `persist:removed`, `persist:cleared`, `persist:error`
|
|
438
|
+
|
|
439
|
+
```ts
|
|
440
|
+
queue.on('persist:error', ({ operation, error }) => {
|
|
441
|
+
console.error(operation, error)
|
|
442
|
+
})
|
|
443
|
+
|
|
444
|
+
// Ensure durable writes finished (both strategies expose flush)
|
|
445
|
+
await queue.flush()
|
|
446
|
+
```
|
|
447
|
+
|
|
448
|
+
## Config (one object: stores + queues)
|
|
449
|
+
|
|
450
|
+
A **single config** has two sections:
|
|
451
|
+
|
|
452
|
+
1. **`stores`** — named adapters (built-in factories or your own `SnapshotStore` / `RowStore`)
|
|
453
|
+
2. **`queues`** — wire queues to store **names**, optional workers, plus router bindings
|
|
454
|
+
|
|
455
|
+
Custom storage = implement the interface and register it. You never extend a hardcoded store list.
|
|
456
|
+
|
|
457
|
+
### JS config (recommended)
|
|
458
|
+
|
|
459
|
+
```ts
|
|
460
|
+
// queue.config.ts
|
|
461
|
+
import { defineConfig } from '@qkitt/queue'
|
|
462
|
+
import { handleMail } from './workers/mail'
|
|
463
|
+
import { createRedisRowStore } from './stores/redis' // your impl of RowStore
|
|
464
|
+
|
|
465
|
+
export default defineConfig({
|
|
466
|
+
stores: {
|
|
467
|
+
// built-in adapter
|
|
468
|
+
mailDisk: {
|
|
469
|
+
adapter: 'localStorage',
|
|
470
|
+
strategy: 'row',
|
|
471
|
+
key: 'mail',
|
|
472
|
+
},
|
|
473
|
+
// custom adapter (JS only)
|
|
474
|
+
redis: {
|
|
475
|
+
strategy: 'row',
|
|
476
|
+
impl: createRedisRowStore('queue:mail'),
|
|
477
|
+
},
|
|
478
|
+
},
|
|
479
|
+
queues: {
|
|
480
|
+
mail: {
|
|
481
|
+
maxSize: 1000, // optional backpressure (QueueFullError when full)
|
|
482
|
+
persist: { store: 'mailDisk' }, // reference by name
|
|
483
|
+
worker: { run: handleMail, concurrency: 2 },
|
|
484
|
+
},
|
|
485
|
+
scratch: {},
|
|
486
|
+
// optional unmatched sink (see router.unmatchedQueue)
|
|
487
|
+
unrouted: {},
|
|
488
|
+
},
|
|
489
|
+
router: {
|
|
490
|
+
bindings: [{ pattern: 'mail.#', queue: 'mail' }],
|
|
491
|
+
unmatchedQueue: 'unrouted',
|
|
492
|
+
},
|
|
493
|
+
// defaults to true when any queue has persist
|
|
494
|
+
hydrate: true,
|
|
495
|
+
})
|
|
496
|
+
```
|
|
497
|
+
|
|
498
|
+
```ts
|
|
499
|
+
// app.ts
|
|
500
|
+
import { buildFromConfig } from '@qkitt/queue'
|
|
501
|
+
import config from './queue.config'
|
|
502
|
+
|
|
503
|
+
const system = await buildFromConfig(config)
|
|
504
|
+
|
|
505
|
+
system.router!.publish('mail.send', { to: 'a@b.c', body: 'hi' })
|
|
506
|
+
// system.queues.mail is already a worker queue (start/stop/…)
|
|
507
|
+
// system.stores.mailDisk is the resolved store instance
|
|
508
|
+
|
|
509
|
+
await system.flushAll()
|
|
510
|
+
```
|
|
511
|
+
|
|
512
|
+
Build order: **resolve stores → queue → persist → worker → router → hydrate**.
|
|
513
|
+
|
|
514
|
+
Composition rules enforced by the library:
|
|
515
|
+
|
|
516
|
+
- Persist must wrap the **bare** queue; worker is always **outer**.
|
|
517
|
+
- Only **one** persist layer per queue (row **or** snapshot, not both).
|
|
518
|
+
|
|
519
|
+
|
|
520
|
+
### Data-only JSON (built-in adapters only)
|
|
521
|
+
|
|
522
|
+
```json
|
|
523
|
+
{
|
|
524
|
+
"stores": {
|
|
525
|
+
"ordersMem": { "adapter": "memory", "strategy": "snapshot" },
|
|
526
|
+
"auditDisk": {
|
|
527
|
+
"adapter": "localStorage",
|
|
528
|
+
"strategy": "row",
|
|
529
|
+
"key": "app:audit"
|
|
530
|
+
}
|
|
531
|
+
},
|
|
532
|
+
"queues": {
|
|
533
|
+
"orders": { "persist": { "store": "ordersMem", "autoSave": true } },
|
|
534
|
+
"audit": { "persist": { "store": "auditDisk" } }
|
|
535
|
+
},
|
|
536
|
+
"router": {
|
|
537
|
+
"bindings": [
|
|
538
|
+
{ "pattern": "orders.#", "queue": "orders" },
|
|
539
|
+
{ "pattern": "orders.created", "queue": "audit" }
|
|
540
|
+
]
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
```
|
|
544
|
+
|
|
545
|
+
```ts
|
|
546
|
+
const system = await buildFromJson(jsonText, { storage: myWebStorage })
|
|
547
|
+
```
|
|
548
|
+
|
|
549
|
+
| Field | Meaning |
|
|
550
|
+
| --- | --- |
|
|
551
|
+
| `stores.<name>.adapter` | Built-in: `"memory"` \| `"localStorage"` \| `"sessionStorage"` |
|
|
552
|
+
| `stores.<name>.strategy` | `"snapshot"` or `"row"` (required) |
|
|
553
|
+
| `stores.<name>.key` | Required for web adapters |
|
|
554
|
+
| `stores.<name>.impl` | **JS only** — your `SnapshotStore` / `RowStore` instance |
|
|
555
|
+
| `queues.<name>` | Named queue. `{}` = plain in-memory |
|
|
556
|
+
| `queues.<name>.maxSize` | Optional capacity; enqueue throws `QueueFullError` when full |
|
|
557
|
+
| `queues.<name>.persist.store` | **Name** of an entry in `stores` |
|
|
558
|
+
| `queues.<name>.persist.autoSave` | Snapshot only; default `true` |
|
|
559
|
+
| `queues.<name>.worker` | **JS only** — a function, or `{ run, concurrency?, autoStart? }` |
|
|
560
|
+
| `router.bindings` | `{ pattern, queue }` → bind pattern to a named queue |
|
|
561
|
+
| `hydrate` | Load persisted queues after build (default `true` if any queue has `persist`) |
|
|
562
|
+
|
|
563
|
+
### Helpers
|
|
564
|
+
|
|
565
|
+
| API | Role |
|
|
566
|
+
| --- | --- |
|
|
567
|
+
| `defineConfig(config)` | Typed JS helper; validates and keeps function / store refs |
|
|
568
|
+
| `buildFromConfig(config, options?)` | Resolve stores + build queues / workers / router |
|
|
569
|
+
| `buildFromJson(json, options?)` | Data-only JSON → build (rejects `worker` / `impl`) |
|
|
570
|
+
| `validateSystemConfig(value)` | Data-only validation |
|
|
571
|
+
| `validateJsConfig(config)` | JS validation (allows workers + custom stores) |
|
|
572
|
+
| `parseSystemConfig(json)` | Parse + data-only validate without building |
|
|
573
|
+
|
|
574
|
+
Build options:
|
|
575
|
+
|
|
576
|
+
- `storage` — inject `localStorage` / `sessionStorage` (or a mock) for built-in web adapters
|
|
577
|
+
|
|
578
|
+
Returned system:
|
|
579
|
+
|
|
580
|
+
```ts
|
|
581
|
+
system.stores // resolved store instances by name
|
|
582
|
+
system.queues // named queues (+ worker controls / hydrate / flush when configured)
|
|
583
|
+
system.router // present when config.router was set
|
|
584
|
+
system.hydrateAll()
|
|
585
|
+
system.flushAll()
|
|
586
|
+
system.config // shallow-frozen config (workers / impls preserved)
|
|
587
|
+
```
|
|
588
|
+
|
|
589
|
+
## Putting it together
|
|
590
|
+
|
|
591
|
+
Route jobs into a durable, concurrent worker with retries:
|
|
592
|
+
|
|
593
|
+
```ts
|
|
594
|
+
import {
|
|
595
|
+
buildQueue,
|
|
596
|
+
withWorker,
|
|
597
|
+
withRowPersist,
|
|
598
|
+
pipeline,
|
|
599
|
+
withRetry,
|
|
600
|
+
buildRouter,
|
|
601
|
+
createMemoryRowStore,
|
|
602
|
+
type RouteMessage,
|
|
603
|
+
} from '@qkitt/queue'
|
|
604
|
+
|
|
605
|
+
type EmailJob = { to: string; body: string }
|
|
606
|
+
|
|
607
|
+
const router = buildRouter()
|
|
608
|
+
const store = createMemoryRowStore<RouteMessage<EmailJob>>()
|
|
609
|
+
|
|
610
|
+
const base = withRowPersist(
|
|
611
|
+
buildQueue<RouteMessage<EmailJob>>(),
|
|
612
|
+
store,
|
|
613
|
+
)
|
|
614
|
+
await base.hydrate()
|
|
615
|
+
|
|
616
|
+
const worker = withRetry(
|
|
617
|
+
pipeline(
|
|
618
|
+
async (msg: RouteMessage<EmailJob>) => {
|
|
619
|
+
if (!msg.data.to.includes('@')) throw new Error('bad recipient')
|
|
620
|
+
return msg
|
|
621
|
+
},
|
|
622
|
+
async (msg) => {
|
|
623
|
+
await sendEmail(msg.data)
|
|
624
|
+
return msg.data.to
|
|
625
|
+
},
|
|
626
|
+
),
|
|
627
|
+
{ retries: 3, delay: (n) => 50 * n },
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
const queue = withWorker(base, worker, { concurrency: 2 })
|
|
631
|
+
|
|
632
|
+
router.bind('mail.send', queue)
|
|
633
|
+
router.bind('mail.#', queue) // same queue can have multiple bindings
|
|
634
|
+
|
|
635
|
+
router.publish('mail.send', { to: 'you@example.com', body: 'hi' })
|
|
636
|
+
|
|
637
|
+
queue.on('worker:completed', ({ result }) => console.log('sent to', result))
|
|
638
|
+
queue.on('worker:failed', ({ error }) => console.error(error))
|
|
639
|
+
queue.on('worker:idle', () => console.log('inbox empty'))
|
|
640
|
+
```
|
|
641
|
+
|
|
642
|
+
## Design notes
|
|
643
|
+
|
|
644
|
+
| Concern | Approach |
|
|
645
|
+
| --- | --- |
|
|
646
|
+
| Composition | Decorator stack: bare queue → persist (optional) → worker (optional) |
|
|
647
|
+
| Sync API | `enqueue` / `dequeue` stay sync; store I/O is async on a serialized write chain |
|
|
648
|
+
| Events | Typed emitter; listener errors are isolated so pumps keep running |
|
|
649
|
+
| Routing | MQTT/AMQP-style patterns; unmatched sink does not count as a match |
|
|
650
|
+
| Config | Named stores + named queues; workers/`impl` are JS-only; JSON is data-only |
|
|
651
|
+
| Packaging | Zero runtime deps; ESM-only npm package; `npm run build` emits ESM + `.d.ts` to `dist/` |
|
|
652
|
+
|
|
653
|
+
## Source layout
|
|
654
|
+
|
|
655
|
+
Consumers always import `@qkitt/queue` (root only). The tree under `src/` mirrors composition layers:
|
|
656
|
+
|
|
657
|
+
| Folder | Meaning |
|
|
658
|
+
| --- | --- |
|
|
659
|
+
| `queue/core` | FIFO queue (`buildQueue`) |
|
|
660
|
+
| `queue/worker` | Attach a processor (`withWorker`) |
|
|
661
|
+
| `queue/persist` | Durable queue decorators + store contracts |
|
|
662
|
+
| `persist` | Storage backends (memory, Web Storage) |
|
|
663
|
+
| `worker` | Processor helpers (`pipeline`, `withRetry`) — not queue decoration |
|
|
664
|
+
| `router` | Topic routing |
|
|
665
|
+
| `config` | Declarative system build |
|
|
666
|
+
| `events` | Typed emitter |
|
|
667
|
+
|
|
668
|
+
**Name disambiguation:** `src/worker` = functions that process items; `src/queue/worker` = the decorator that runs them on a queue. `src/persist` = store adapters; `src/queue/persist` = queue decorators that use those stores.
|
|
669
|
+
|
|
670
|
+
## Naming convention
|
|
671
|
+
|
|
672
|
+
Files follow **`<concept>.<role>.ts`** with a closed role set (exactly **one** role suffix):
|
|
673
|
+
|
|
674
|
+
| Role | Meaning | Example |
|
|
675
|
+
| --- | --- | --- |
|
|
676
|
+
| *(none)* | Primary implementation of that concept | `queue.ts`, `with-worker.ts`, `pipeline.ts` |
|
|
677
|
+
| `.util` | Pure / small helper | `write-chain.util.ts`, `match.util.ts` |
|
|
678
|
+
| `.support` | Shared glue for a feature | `persist.support.ts` |
|
|
679
|
+
| `.types` | Shared contracts only | `persist.types.ts` |
|
|
680
|
+
| `.test` | Co-located tests | `queue.test.ts` |
|
|
681
|
+
|
|
682
|
+
Rules:
|
|
683
|
+
|
|
684
|
+
- **Folders** answer *which product area / composition layer* (`core/`, `persist/`).
|
|
685
|
+
- **Role suffix** answers *kind* (product vs helper vs types). Do not invent a second role word (e.g. not `web-storage.access.util.ts`).
|
|
686
|
+
- **Mechanism words live in the concept**, hyphenated: `write-chain.util.ts`, `web-storage-access.util.ts`, `json-codec.util.ts` — not `write.chain.ts` or `json.codec.util.ts`.
|
|
687
|
+
- Untagged file in a folder ≈ the thing users think of for that area.
|
|
688
|
+
- Prefer folders for area + closed role suffixes for kind; avoid deep `utils/` trees unless util count grows large.
|
|
689
|
+
- Extract `.types` when shared across modules or folders; keep types co-located when only one module uses them.
|
|
690
|
+
|
|
691
|
+
## API map
|
|
692
|
+
|
|
693
|
+
Public surface (via `@qkitt/queue` / `src/index.ts`):
|
|
694
|
+
|
|
695
|
+
| Area | Exports |
|
|
696
|
+
| --- | --- |
|
|
697
|
+
| Queue | `buildQueue`, `QueueFullError`, `Queue`, `QueueEvents`, `BuildQueueOptions` |
|
|
698
|
+
| Worker | `withWorker`, `QueueWithWorker`, `WorkerEvents`, `WithWorkerOptions` |
|
|
699
|
+
| Snapshot persist | `withSnapshotPersist`, `SnapshotStore`, `SnapshotPersistOptions` |
|
|
700
|
+
| Row persist | `withRowPersist`, `RowStore`, `RowRecord`, `createId`, `RowPersistOptions` |
|
|
701
|
+
| Worker helpers | `pipeline`, `withRetry`, `RetryExhaustedError`, `WorkerFn`, `StepFn` |
|
|
702
|
+
| Router | `buildRouter`, `RouteMessage`, `matchTopic`, `isValidTopic`, `isValidPattern`, … |
|
|
703
|
+
| Persist stores | memory + Web Storage factories, `StorageCodecError`, `WebStorageLike` |
|
|
704
|
+
| Config | `defineConfig`, `buildFromConfig`, `buildFromJson`, `parseSystemConfig`, `validate*`, `SystemConfig`, … |
|
|
705
|
+
| Events | `buildEventEmitter`, `createTypedEmit`, `EventEmitter`, `MergeEventMaps` |
|
|
706
|
+
|
|
707
|
+
Internals (`forward.util`, `hydrate-gate.util`, `write-chain.util`, `row-ids.util`, codecs) are not part of the stable public contract.
|
|
708
|
+
|
|
709
|
+
## Development
|
|
710
|
+
|
|
711
|
+
```bash
|
|
712
|
+
npm test # vitest
|
|
713
|
+
npm run typecheck # tsc --noEmit
|
|
714
|
+
npm run build # tsup (ESM) + tsc declarations → dist/
|
|
715
|
+
npm run pack:check # npm pack --dry-run (what would publish)
|
|
716
|
+
npm run release:check # typecheck + test + build + pack:check
|
|
717
|
+
```
|
|
718
|
+
|
|
719
|
+
See [CHANGELOG.md](./CHANGELOG.md) for release history.
|
|
720
|
+
|
|
721
|
+
## License
|
|
722
|
+
|
|
723
|
+
[ISC](./LICENSE)
|