@bumail/queue 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -6
- package/dist/chunks/index-vg971acr.js.map +2 -2
- package/dist/chunks/index-y3n1w2az.js +18 -0
- package/dist/chunks/index-y3n1w2az.js.map +10 -0
- package/dist/errors.d.ts +2 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/index.js +73 -23
- package/dist/index.js.map +6 -5
- package/dist/masked.d.ts +7 -0
- package/dist/masked.d.ts.map +1 -0
- package/dist/postgres/connect.d.ts.map +1 -1
- package/dist/postgres/index.js +4 -13
- package/dist/postgres/index.js.map +3 -3
- package/dist/queue/deliver.d.ts +7 -0
- package/dist/queue/deliver.d.ts.map +1 -1
- package/dist/queue/lease.d.ts +41 -0
- package/dist/queue/lease.d.ts.map +1 -0
- package/dist/queue/worker.d.ts.map +1 -1
- package/dist/redis/connect.d.ts +25 -0
- package/dist/redis/connect.d.ts.map +1 -0
- package/dist/redis/index.d.ts +3 -0
- package/dist/redis/index.d.ts.map +1 -0
- package/dist/redis/index.js +430 -0
- package/dist/redis/index.js.map +14 -0
- package/dist/redis/items.d.ts +13 -0
- package/dist/redis/items.d.ts.map +1 -0
- package/dist/redis/options.d.ts +47 -0
- package/dist/redis/options.d.ts.map +1 -0
- package/dist/redis/run.d.ts +12 -0
- package/dist/redis/run.d.ts.map +1 -0
- package/dist/redis/scripts.d.ts +60 -0
- package/dist/redis/scripts.d.ts.map +1 -0
- package/dist/redis/store.d.ts +30 -0
- package/dist/redis/store.d.ts.map +1 -0
- package/docs/README.md +1 -1
- package/docs/guide.md +185 -6
- package/docs/roadmap.md +30 -7
- package/docs/troubleshooting.md +229 -13
- package/package.json +9 -3
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Lua scripts a Redis store runs: each operation that writes is one
|
|
3
|
+
* script, so Redis runs it whole, with no other command between its
|
|
4
|
+
* steps. The fixed keys come as `KEYS`; an item's own keys are its
|
|
5
|
+
* prefix (`ARGV[1]`), `item:` or `message:`, and its id.
|
|
6
|
+
*
|
|
7
|
+
* An item is a hash, `<prefix>item:<id>`; its message a string,
|
|
8
|
+
* `<prefix>message:<id>`, written from the bytes given, as they are. Its
|
|
9
|
+
* member in the sorted sets is its sequence number, sixteen digits, `:`,
|
|
10
|
+
* and its id: Redis orders members of one score by their bytes, so items
|
|
11
|
+
* equally due come oldest first. Times travel as the strings JavaScript
|
|
12
|
+
* writes for them and are only compared here (`tonumber`), never written
|
|
13
|
+
* back from Lua, so none loses a digit.
|
|
14
|
+
*/
|
|
15
|
+
/** A script, and the SHA-1 `EVALSHA` names it by. */
|
|
16
|
+
export interface Script {
|
|
17
|
+
readonly source: string;
|
|
18
|
+
readonly sha: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* KEYS schema; ARGV prefix, version. Writes the version on a new queue;
|
|
22
|
+
* returns the one found.
|
|
23
|
+
*/
|
|
24
|
+
export declare const SCHEMA: Script;
|
|
25
|
+
/**
|
|
26
|
+
* KEYS items, ready, seq; ARGV prefix, id, from, recipients, size,
|
|
27
|
+
* created, message, max (or ''). `{'full', n}` when `max` items are
|
|
28
|
+
* there already, else `{'ok'}`.
|
|
29
|
+
*/
|
|
30
|
+
export declare const ADD: Script;
|
|
31
|
+
/**
|
|
32
|
+
* KEYS ready, leases, items; ARGV prefix, owner, now, expiry. The
|
|
33
|
+
* earliest due, then the oldest, of the items no lease holds and of those
|
|
34
|
+
* whose lease expired, leased to the owner; its fields, or false. The
|
|
35
|
+
* expired leases are read whole: there are as many as the leases a
|
|
36
|
+
* crashed or stalled worker left behind, and the leased items an admin
|
|
37
|
+
* moved later, until they are due.
|
|
38
|
+
*/
|
|
39
|
+
export declare const CLAIM: Script;
|
|
40
|
+
/** KEYS leases; ARGV prefix, id, owner, expiry. 1 while the owner holds the lease, else 0. */
|
|
41
|
+
export declare const RENEW: Script;
|
|
42
|
+
/**
|
|
43
|
+
* KEYS items, ready, leases; ARGV prefix, id, owner, rev, then 'drop',
|
|
44
|
+
* or 'record' and recipients, next, attempts, delay. 0 when the owner
|
|
45
|
+
* does not hold the lease; -1 when another outcome was recorded since
|
|
46
|
+
* the item was read at `rev`, to read it again; 1 once recorded, the
|
|
47
|
+
* lease let go of, or the item and its message dropped.
|
|
48
|
+
*/
|
|
49
|
+
export declare const COMPLETE: Script;
|
|
50
|
+
/**
|
|
51
|
+
* KEYS items, ready, leases; ARGV prefix, id, at, owner (or ''). With an
|
|
52
|
+
* owner, only while it holds the lease, which it lets go of; without,
|
|
53
|
+
* the lease stays. 1 when moved, else 0.
|
|
54
|
+
*/
|
|
55
|
+
export declare const RESCHEDULE: Script;
|
|
56
|
+
/** KEYS items, ready, leases; ARGV prefix, id. The item's fields as it stood, or false; dropped with its message. */
|
|
57
|
+
export declare const CANCEL: Script;
|
|
58
|
+
/** KEYS items; ARGV prefix, first, last. A page of items, each its fields, in one read. */
|
|
59
|
+
export declare const LIST: Script;
|
|
60
|
+
//# sourceMappingURL=scripts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scripts.d.ts","sourceRoot":"","sources":["../../src/redis/scripts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAuBH,qDAAqD;AACrD,MAAM,WAAW,MAAM;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACrB;AAQD;;;GAGG;AACH,eAAO,MAAM,MAAM,QAKjB,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,GAAG,QAcd,CAAC;AAEH;;;;;;;GAOG;AACH,eAAO,MAAM,KAAK,QAyBhB,CAAC;AAEH,8FAA8F;AAC9F,eAAO,MAAM,KAAK,QAMhB,CAAC;AAEH;;;;;;GAMG;AACH,eAAO,MAAM,QAAQ,QAmBnB,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,UAAU,QAerB,CAAC;AAEH,qHAAqH;AACrH,eAAO,MAAM,MAAM,QAUjB,CAAC;AAEH,2FAA2F;AAC3F,eAAO,MAAM,IAAI,QAMf,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { QueueStore } from '../contract/queue-store';
|
|
2
|
+
import type { AddOptions, AttemptResult, ClaimRequest, NewQueueItem, QueueItem, QueueListOptions } from '../contract/types';
|
|
3
|
+
import type { RedisQueueStoreOptions } from './options';
|
|
4
|
+
/**
|
|
5
|
+
* A `QueueStore` on Redis through Bun's own `Bun.RedisClient`: several
|
|
6
|
+
* instances, on one machine or many, share one queue. Every operation
|
|
7
|
+
* that writes is one Lua script, which Redis runs whole, so two
|
|
8
|
+
* instances never take the same item. The due items are in sorted sets,
|
|
9
|
+
* each item a hash, its message a string written byte for byte. One
|
|
10
|
+
* Redis, or a primary with replicas — not Redis Cluster.
|
|
11
|
+
*/
|
|
12
|
+
export declare class RedisQueueStore implements QueueStore {
|
|
13
|
+
#private;
|
|
14
|
+
private constructor();
|
|
15
|
+
/** Checks the options; connects to nothing until the first call. What is wrong is `INVALID`. */
|
|
16
|
+
static open(options: RedisQueueStoreOptions): RedisQueueStore;
|
|
17
|
+
/** Closes the client the store opened for a URL, never one it was given; closing twice is fine. */
|
|
18
|
+
close(): Promise<void>;
|
|
19
|
+
add(item: NewQueueItem, options?: AddOptions): Promise<QueueItem>;
|
|
20
|
+
get(id: string): Promise<QueueItem | undefined>;
|
|
21
|
+
list(options?: QueueListOptions): Promise<QueueItem[]>;
|
|
22
|
+
count(): Promise<number>;
|
|
23
|
+
readMessage(id: string): Promise<Uint8Array | undefined>;
|
|
24
|
+
claim(request: ClaimRequest): Promise<QueueItem | undefined>;
|
|
25
|
+
renew(id: string, owner: string, expiresAt: number): Promise<boolean>;
|
|
26
|
+
complete(id: string, owner: string, result: AttemptResult): Promise<QueueItem | undefined>;
|
|
27
|
+
reschedule(id: string, at: number, owner?: string): Promise<boolean>;
|
|
28
|
+
cancel(id: string): Promise<QueueItem | undefined>;
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/redis/store.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,KAAK,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,SAAS,EACT,gBAAgB,EAChB,MAAM,mBAAmB,CAAC;AAK3B,OAAO,KAAK,EAAoB,sBAAsB,EAAE,MAAM,WAAW,CAAC;AAa1E;;;;;;;GAOG;AACH,qBAAa,eAAgB,YAAW,UAAU;;IAOjD,OAAO;IAMP,gGAAgG;IAChG,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,GAAG,eAAe;IAI7D,mGAAmG;IAC7F,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAmCtB,GAAG,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,SAAS,CAAC;IA2BrE,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;IAS/C,IAAI,CAAC,OAAO,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IAatD,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAKxB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAOxD,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;IAc5D,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAarE,QAAQ,CACb,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,aAAa,GACnB,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;IA2B3B,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAepE,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC;CAQxD"}
|
package/docs/README.md
CHANGED
|
@@ -5,6 +5,6 @@ long one.
|
|
|
5
5
|
|
|
6
6
|
| Page | Read it when |
|
|
7
7
|
| --- | --- |
|
|
8
|
-
| [Guide](guide.md) | enqueuing, choosing a route (MX, a smarthost, per domain), tuning the retry schedule and the DSNs, running several workers or instances on PostgreSQL, listening to events, testing with a fake clock and sender, or writing a store of your own |
|
|
8
|
+
| [Guide](guide.md) | enqueuing, choosing a route (MX, a smarthost, per domain), tuning the retry schedule and the DSNs, running several workers or instances on PostgreSQL or Redis, listening to events, testing with a fake clock and sender, or writing a store of your own |
|
|
9
9
|
| [Troubleshooting](troubleshooting.md) | the queue or a store threw a `QueueError`, the `error` event fired, or mail stays deferred |
|
|
10
10
|
| [Roadmap](roadmap.md) | wondering what is coming, and what is not planned |
|
package/docs/guide.md
CHANGED
|
@@ -14,6 +14,7 @@ workers share it.
|
|
|
14
14
|
- [Admin](#admin)
|
|
15
15
|
- [The stores](#the-stores)
|
|
16
16
|
- [PostgreSQL](#postgresql)
|
|
17
|
+
- [Redis](#redis)
|
|
17
18
|
- [Testing](#testing)
|
|
18
19
|
- [Writing a store](#writing-a-store)
|
|
19
20
|
- [Options](#options)
|
|
@@ -236,7 +237,8 @@ is written as `utf-8; j\x{F6}rg@…`, RFC 6533), and every field is folded.
|
|
|
236
237
|
## Several workers, leases and stopping
|
|
237
238
|
|
|
238
239
|
Any number of queues may share one store, in one process or several —
|
|
239
|
-
or, with [PostgreSQL](#postgresql), on several
|
|
240
|
+
or, with [PostgreSQL](#postgresql) or [Redis](#redis), on several
|
|
241
|
+
machines:
|
|
240
242
|
|
|
241
243
|
```ts
|
|
242
244
|
const store = SqliteQueueStore.open({ directory: '/var/lib/bumail/queue' });
|
|
@@ -254,9 +256,17 @@ queue.start();
|
|
|
254
256
|
stops them.
|
|
255
257
|
- **A crashed worker** loses its items when their leases expire: another
|
|
256
258
|
worker claims them then. A worker whose lease was taken meanwhile
|
|
257
|
-
records nothing, and says so on the `error` event (`LEASE_LOST`)
|
|
258
|
-
|
|
259
|
-
|
|
259
|
+
records nothing, and says so on the `error` event (`LEASE_LOST`) — also
|
|
260
|
+
when that worker has since finished the item and dropped it, the
|
|
261
|
+
message then sent twice. An item cancelled while it was delivered, its
|
|
262
|
+
lease still holding, has its outcomes told by the events, with no DSN
|
|
263
|
+
and no error. Once the lease's expiry has passed, a worker cannot tell
|
|
264
|
+
a cancel from an item another worker finished: it reports `LEASE_LOST`,
|
|
265
|
+
lost or cancelled, and tells no outcome. One gap remains when the
|
|
266
|
+
instances' clocks are out of step: an instance whose clock runs ahead
|
|
267
|
+
can take the item while this worker's clock says the lease holds, and
|
|
268
|
+
if it finishes the item between two renewals, the attempt reads as a
|
|
269
|
+
cancel. Keep the machines on NTP.
|
|
260
270
|
- **`stop()`** claims nothing more, lets every session under way end and
|
|
261
271
|
records its outcome, and gives back what was claimed but not started —
|
|
262
272
|
due at once, no attempt counted — so another worker can take it.
|
|
@@ -354,7 +364,7 @@ store.close(); // closing twice is fine
|
|
|
354
364
|
each item is claimed once; a writer waits up to `busyTimeout`
|
|
355
365
|
milliseconds for another.
|
|
356
366
|
Not on a network file system: SQLite's locks do not hold there. For
|
|
357
|
-
several machines, use [PostgreSQL](#postgresql).
|
|
367
|
+
several machines, use [PostgreSQL](#postgresql) or [Redis](#redis).
|
|
358
368
|
- **Messages in a table of their own**, `messages`, dropped with their
|
|
359
369
|
item in the same transaction. A queue holds a message for days at most
|
|
360
370
|
and never shares one between items, so content addressing (as
|
|
@@ -368,6 +378,10 @@ store.close(); // closing twice is fine
|
|
|
368
378
|
On PostgreSQL, for instances on several machines: see
|
|
369
379
|
[PostgreSQL](#postgresql).
|
|
370
380
|
|
|
381
|
+
### `RedisQueueStore`
|
|
382
|
+
|
|
383
|
+
On Redis, for instances on several machines: see [Redis](#redis).
|
|
384
|
+
|
|
371
385
|
## PostgreSQL
|
|
372
386
|
|
|
373
387
|
```ts
|
|
@@ -485,6 +499,167 @@ Raise `max` only when a busy worker waits on the pool, and keep every
|
|
|
485
499
|
instance's pool, and your application's, within the server's
|
|
486
500
|
`max_connections` (100 by default): four instances of `max: 10` take 40.
|
|
487
501
|
|
|
502
|
+
## Redis
|
|
503
|
+
|
|
504
|
+
```ts
|
|
505
|
+
import { createQueue } from '@bumail/queue';
|
|
506
|
+
import { RedisQueueStore } from '@bumail/queue/redis';
|
|
507
|
+
|
|
508
|
+
const client = new Bun.RedisClient(Bun.env['REDIS_URL']);
|
|
509
|
+
const store = RedisQueueStore.open({ client, keyPrefix: 'mail:queue:' });
|
|
510
|
+
const queue = createQueue({ store, hostname, resolver, owner: `${host}-${process.pid}` });
|
|
511
|
+
queue.start();
|
|
512
|
+
|
|
513
|
+
process.on('SIGTERM', async () => {
|
|
514
|
+
await queue.stop();
|
|
515
|
+
client.close(); // yours: the store never closes a client it was given
|
|
516
|
+
});
|
|
517
|
+
```
|
|
518
|
+
|
|
519
|
+
`@bumail/queue/redis` runs on Bun's own `Bun.RedisClient` (`Bun.redis`
|
|
520
|
+
is its default instance): there is no driver to install and no peer to
|
|
521
|
+
add. Every instance of your server opens a store on the same Redis, and
|
|
522
|
+
they share one queue. It is held to the same contract specs as the other
|
|
523
|
+
stores, and to the same specs of two instances delivering every item
|
|
524
|
+
exactly once, against `redis:7`.
|
|
525
|
+
|
|
526
|
+
### The client
|
|
527
|
+
|
|
528
|
+
- **`client`** is a `Bun.RedisClient` of yours — you set its options and
|
|
529
|
+
close it — or **`url`** a `redis://` (`rediss://` for TLS, or the other
|
|
530
|
+
schemes `Bun.RedisClient` takes), for which the store opens a client
|
|
531
|
+
with Bun's defaults and closes it in `close()`. One or the other, never
|
|
532
|
+
both.
|
|
533
|
+
- It is typed by its shape, `RedisQueueClient` (`send`, `getBuffer` and
|
|
534
|
+
`close`), so the declarations need nothing of `@types/bun`; a
|
|
535
|
+
`Bun.RedisClient` fits it.
|
|
536
|
+
- **Nothing connects at `open`**: a wrong option is refused there, and a
|
|
537
|
+
server out of reach on the first call. With Bun's defaults a client
|
|
538
|
+
reconnects on its own, up to 20 times, and holds the commands sent
|
|
539
|
+
while it is disconnected until Redis is back (a `PING` sent during a
|
|
540
|
+
5-second stop answered once Redis started again). A command already
|
|
541
|
+
sent when the connection drops, or sent before the client noticed,
|
|
542
|
+
fails instead (`Connection closed`): a renewal tries again on its own,
|
|
543
|
+
and a claim or an outcome is told on `error` (see Troubleshooting).
|
|
544
|
+
- One client is one connection, pipelined: every call of one store goes
|
|
545
|
+
through it, so there is no pool to size.
|
|
546
|
+
|
|
547
|
+
### The keys
|
|
548
|
+
|
|
549
|
+
Every key starts with `keyPrefix` (`bumail:queue:` by default):
|
|
550
|
+
|
|
551
|
+
| key | type | what it holds |
|
|
552
|
+
| --- | --- | --- |
|
|
553
|
+
| `<prefix>item:<id>` | hash | an item: `id`, `from`, `recipients` (JSON, each recipient's state), `size`, `created`, `next`, `attempts`, `delay`, while leased `owner` and `expires`, its `member` in the sorted sets and `rev`, how many outcomes were recorded |
|
|
554
|
+
| `<prefix>message:<id>` | string | the message as enqueued, whole, byte for byte |
|
|
555
|
+
| `<prefix>items` | sorted set | every item, scored by its next attempt: `list` and `count` |
|
|
556
|
+
| `<prefix>ready` | sorted set | the items no lease holds, scored by their next attempt |
|
|
557
|
+
| `<prefix>leases` | sorted set | the leased items, scored by when their leases expire |
|
|
558
|
+
| `<prefix>seq` | string | the counter that orders items equally due |
|
|
559
|
+
| `<prefix>schema` | string | the layout version |
|
|
560
|
+
|
|
561
|
+
A member of the sorted sets is the item's sequence number, sixteen
|
|
562
|
+
digits, then `:` and its id, so items equally due come oldest first.
|
|
563
|
+
Times are kept as the strings JavaScript writes for them, and read back
|
|
564
|
+
exactly. The prefix is lowercase letters, digits, `_`, `:`, `.` and
|
|
565
|
+
`-`, starting with a letter, 40 characters at most: no `{`, so no
|
|
566
|
+
Cluster hash tag, and no glob character, so `SCAN 0 MATCH <prefix>*`
|
|
567
|
+
finds the queue's keys and nothing else. A `<prefix>schema` that holds
|
|
568
|
+
anything but a layout version is refused at the first call: the prefix
|
|
569
|
+
is another application's. Give each queue its own
|
|
570
|
+
prefix to keep several in one Redis database. An id the store did not
|
|
571
|
+
make (`crypto.randomUUID()`'s form) is one no item has: no id a caller
|
|
572
|
+
gives reaches another key.
|
|
573
|
+
|
|
574
|
+
### Several instances and leases
|
|
575
|
+
|
|
576
|
+
Every operation that writes is one Lua script, run by `EVALSHA` (by
|
|
577
|
+
`EVAL` when Redis no longer has it, after a restart, a failover or a
|
|
578
|
+
`SCRIPT FLUSH`): Redis runs a script whole, with no command of another
|
|
579
|
+
instance between its steps.
|
|
580
|
+
|
|
581
|
+
- **A claim** takes the first item of `ready` due by `now`, and every
|
|
582
|
+
lease of `leases` expired by `now`, keeps the earliest due of them,
|
|
583
|
+
then the oldest, and moves it from `ready` to `leases` with its new
|
|
584
|
+
expiry, in one script. Two instances never take the same item. An
|
|
585
|
+
expired lease is found at once, not by a sweep; there are only as many
|
|
586
|
+
as a crashed or stalled instance left.
|
|
587
|
+
- **`complete`** reads the item, applies the outcome in JavaScript, and
|
|
588
|
+
records it with a script that checks the owner still holds the lease
|
|
589
|
+
and that no other outcome was recorded since (`rev`); otherwise it
|
|
590
|
+
reads again, at most 8 times, then records nothing (`LEASE_LOST`), as
|
|
591
|
+
it does when the lease was lost.
|
|
592
|
+
- **`renew`**, **`reschedule`** and **`cancel`** are a script each, and
|
|
593
|
+
an `add` with `limits.maxItems` counts and adds in one, so two
|
|
594
|
+
instances never both take the last place.
|
|
595
|
+
- **The clocks** of the instances must agree, as on PostgreSQL: the store
|
|
596
|
+
keeps no clock, and Redis's is never read.
|
|
597
|
+
|
|
598
|
+
### Deploying Redis
|
|
599
|
+
|
|
600
|
+
- **One Redis, or a primary with replicas — not Redis Cluster.** A script
|
|
601
|
+
reaches each item's keys, which are not all named up front, so the
|
|
602
|
+
queue's keys must live on one node. Point the store at the primary: a
|
|
603
|
+
replica refuses writes (`READONLY`). After a failover, the URL must
|
|
604
|
+
reach the new primary — a provider's endpoint that follows it, say.
|
|
605
|
+
- **Durability is Redis's, and weaker than PostgreSQL's.** Redis
|
|
606
|
+
acknowledges a write once it is in memory:
|
|
607
|
+
- with `appendonly yes` and `appendfsync always`, a write is on disk
|
|
608
|
+
before it is acknowledged, as on `bun:sqlite` or PostgreSQL;
|
|
609
|
+
- with `appendfsync everysec` (Redis's default once AOF is on), a
|
|
610
|
+
crash of the server loses up to about a second of acknowledged
|
|
611
|
+
writes;
|
|
612
|
+
- with snapshots only (`save`, AOF off), it loses everything since the
|
|
613
|
+
last snapshot, minutes of it;
|
|
614
|
+
- **replication is asynchronous**: a primary that fails before its
|
|
615
|
+
replica has a write loses that write when the replica is promoted,
|
|
616
|
+
whatever the fsync setting.
|
|
617
|
+
|
|
618
|
+
A lost write is an enqueued message gone (the client was told it was
|
|
619
|
+
queued), an outcome forgotten (a recipient delivered and sent the
|
|
620
|
+
message again), or a lease given twice. Use `appendfsync always`
|
|
621
|
+
where a lost message matters, and PostgreSQL where it must survive a
|
|
622
|
+
failover.
|
|
623
|
+
- **`maxmemory-policy noeviction`.** Every message lives in memory, whole,
|
|
624
|
+
until its item is done. Under any other policy a full Redis evicts
|
|
625
|
+
queue keys — an item, or its message — to make room; with
|
|
626
|
+
`noeviction` it refuses the write (`OOM command not allowed`), and
|
|
627
|
+
`enqueue` rejects with that `RedisError`. An item whose hash is gone
|
|
628
|
+
all the same (evicted, or deleted by hand) is dropped by the next
|
|
629
|
+
claim that meets it, its message with it, and never leased. Bound the queue with `limits.maxItems` and
|
|
630
|
+
`limits.maxMessageSize`, and size `maxmemory` for both.
|
|
631
|
+
- **An ACL user** needs the commands the store sends and the ones its
|
|
632
|
+
scripts call, on its prefix's keys (`~<keyPrefix>*`; here the
|
|
633
|
+
example's `mail:queue:`), and nothing else:
|
|
634
|
+
|
|
635
|
+
```
|
|
636
|
+
ACL SETUSER bumail-worker on >secret resetkeys ~mail:queue:* -@all +evalsha +eval +get +set +del +incr +hgetall +hget +hset +hdel +hincrby +zadd +zrem +zrange +zrangebyscore +zcard
|
|
637
|
+
```
|
|
638
|
+
|
|
639
|
+
### What `Bun.redis` does with bytes and text
|
|
640
|
+
|
|
641
|
+
Measured on Bun 1.4.2 against Redis 7:
|
|
642
|
+
|
|
643
|
+
- An argument may be a `Uint8Array`: `send` writes it as it is, every
|
|
644
|
+
byte value, to `EVAL` and `EVALSHA` too. That is how a message is
|
|
645
|
+
written, through the script that adds its item.
|
|
646
|
+
- A bulk string in a reply is decoded as UTF-8 into a JavaScript string,
|
|
647
|
+
so bytes that are not UTF-8 do not come back from `send`. A message is
|
|
648
|
+
read with `getBuffer`, which keeps the bytes.
|
|
649
|
+
- A string argument is encoded as UTF-8, and a lone surrogate becomes
|
|
650
|
+
U+FFFD without a word: the contract's checks refuse a lone surrogate
|
|
651
|
+
(and a NUL, which Redis could keep) before anything is sent, so what is
|
|
652
|
+
read back is always what was given.
|
|
653
|
+
- `HGETALL` sent directly comes back as an object; the same from a script
|
|
654
|
+
as a flat list of names and values. A script's `false` is `null`.
|
|
655
|
+
- An error reply rejects with a `RedisError` whose message is Redis's
|
|
656
|
+
(`NOSCRIPT No matching script…`); none of them repeats the URL.
|
|
657
|
+
- A server out of reach: with Bun's defaults a call waits for 20
|
|
658
|
+
reconnections, about half a minute, then rejects with `Max
|
|
659
|
+
reconnection attempts reached`; a URL that is not one is not refused by
|
|
660
|
+
`new Bun.RedisClient`, only by its first call, so the store checks the
|
|
661
|
+
scheme itself.
|
|
662
|
+
|
|
488
663
|
## Testing
|
|
489
664
|
|
|
490
665
|
Inject the clock and the sender, and call `deliverDue()` by hand: no
|
|
@@ -549,12 +724,16 @@ codes, and keeps its promises:
|
|
|
549
724
|
dropped with its message.
|
|
550
725
|
- Times are given by the caller; the store keeps no clock.
|
|
551
726
|
- Nothing it returns is shared with what it keeps.
|
|
727
|
+
- `reschedule` with an owner refuses an empty owner as `INVALID`, even
|
|
728
|
+
for an id no item has. An id a store cannot keep (one holding a NUL or
|
|
729
|
+
a lone surrogate) may instead be answered `false`, unknown, before the
|
|
730
|
+
owner is checked.
|
|
552
731
|
- Text it keeps holds no NUL and no lone surrogate: the contract's
|
|
553
732
|
checks refuse them (`INVALID`) in what is added, claimed and recorded,
|
|
554
733
|
since PostgreSQL cannot keep them, and an id holding one is unknown.
|
|
555
734
|
The queue never gives a store such text: it cleans every reply.
|
|
556
735
|
|
|
557
|
-
The contract's specs, `describeQueueStore`, which
|
|
736
|
+
The contract's specs, `describeQueueStore`, which every store here runs,
|
|
558
737
|
are internal for now: a store written outside this package cannot run
|
|
559
738
|
them yet.
|
|
560
739
|
|
package/docs/roadmap.md
CHANGED
|
@@ -6,19 +6,17 @@ only number on it.
|
|
|
6
6
|
|
|
7
7
|
## Now
|
|
8
8
|
|
|
9
|
-
Nothing in progress: the
|
|
10
|
-
taken. The
|
|
9
|
+
Nothing in progress: the MongoDB store, first under Next, is the next
|
|
10
|
+
one taken. The Redis store is merged (see Shipped).
|
|
11
11
|
|
|
12
12
|
## Next
|
|
13
13
|
|
|
14
14
|
More answers to the `QueueStore` contract, built for several instances of
|
|
15
15
|
a server sharing one queue — each claim atomic across them, each lease
|
|
16
16
|
taken back from an instance that crashed — and each held to the same
|
|
17
|
-
contract specs as the memory, `bun:sqlite
|
|
18
|
-
adds a dependency.
|
|
17
|
+
contract specs as the memory, `bun:sqlite`, PostgreSQL and Redis
|
|
18
|
+
stores. None adds a dependency.
|
|
19
19
|
|
|
20
|
-
- **A Redis store**, on Bun's own `Bun.redis` — the due items in a sorted
|
|
21
|
-
set, the claim and the lease in one script.
|
|
22
20
|
- **A MongoDB store** — the contract answered structurally, against a
|
|
23
21
|
collection of the shape a MongoDB driver gives, so the package peers on
|
|
24
22
|
no driver: the application passes in its own collection; the claim a
|
|
@@ -55,7 +53,7 @@ Then:
|
|
|
55
53
|
|
|
56
54
|
- **A runtime dependency.** The memory store needs none, the `bun:sqlite`
|
|
57
55
|
store uses Bun's own SQLite, the PostgreSQL store Bun's own `Bun.sql`,
|
|
58
|
-
the Redis store
|
|
56
|
+
the Redis store Bun's own `Bun.redis`, and the MongoDB store will take
|
|
59
57
|
the application's own collection, typed by its shape.
|
|
60
58
|
- **Relaying on its own.** The queue sends what the app enqueues; who may
|
|
61
59
|
send is decided before, by the SMTP server's AUTH or the app. No option
|
|
@@ -67,6 +65,31 @@ Then:
|
|
|
67
65
|
|
|
68
66
|
### Unreleased — merged, not yet published
|
|
69
67
|
|
|
68
|
+
- **A Redis store**, `@bumail/queue/redis`, on Bun's own `Bun.redis`,
|
|
69
|
+
for instances of a server on several machines sharing one queue:
|
|
70
|
+
`RedisQueueStore.open({ client })` takes a `Bun.RedisClient`, or
|
|
71
|
+
`{ url }` a `redis://` URL, and needs no driver. Every operation that
|
|
72
|
+
writes is one Lua script, which Redis runs whole: two instances never
|
|
73
|
+
take the same item, a crashed instance's leases expire and its items
|
|
74
|
+
are claimed again, and `maxItems` holds across instances. The due
|
|
75
|
+
items are in sorted sets, each item a hash, its message written byte
|
|
76
|
+
for byte, every key under a `keyPrefix`. Held to the same contract
|
|
77
|
+
specs as the other stores, and to specs of two instances delivering
|
|
78
|
+
every item exactly once. One Redis, or a primary with replicas, not
|
|
79
|
+
Cluster; its durability is Redis's, which the guide spells out.
|
|
80
|
+
- **A lease lost to an instance that finished the item is reported** —
|
|
81
|
+
an instance that stalled past its lease, while another claimed the
|
|
82
|
+
item, delivered it and dropped it, no longer takes the item's absence
|
|
83
|
+
for a cancel: it reports `LEASE_LOST` on the `error` event, tells no
|
|
84
|
+
outcome and sends no DSN, so the operator learns the message may have
|
|
85
|
+
gone out twice. With no record of who dropped an item, a lease that
|
|
86
|
+
expired, or a renewal refused while the item was still there, is
|
|
87
|
+
reported as lost or cancelled; a cancel under a lease that
|
|
88
|
+
still held is told as before, and no longer draws a `LEASE_LOST` from
|
|
89
|
+
a renewal that finds the item gone.
|
|
90
|
+
|
|
91
|
+
### 0.2.0
|
|
92
|
+
|
|
70
93
|
- **A PostgreSQL store**, `@bumail/queue/postgres`, on Bun's own
|
|
71
94
|
`Bun.sql`, for instances of a server on several machines sharing one
|
|
72
95
|
queue: `PostgresQueueStore.open({ sql })` takes a `Bun.SQL` client or a
|