@bumail/queue 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -6
- package/dist/chunks/index-vg971acr.js.map +2 -2
- package/dist/chunks/index-y3n1w2az.js +18 -0
- package/dist/chunks/index-y3n1w2az.js.map +10 -0
- package/dist/errors.d.ts +2 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/index.js +73 -23
- package/dist/index.js.map +6 -5
- package/dist/masked.d.ts +7 -0
- package/dist/masked.d.ts.map +1 -0
- package/dist/postgres/connect.d.ts.map +1 -1
- package/dist/postgres/index.js +4 -13
- package/dist/postgres/index.js.map +3 -3
- package/dist/queue/deliver.d.ts +7 -0
- package/dist/queue/deliver.d.ts.map +1 -1
- package/dist/queue/lease.d.ts +41 -0
- package/dist/queue/lease.d.ts.map +1 -0
- package/dist/queue/worker.d.ts.map +1 -1
- package/dist/redis/connect.d.ts +25 -0
- package/dist/redis/connect.d.ts.map +1 -0
- package/dist/redis/index.d.ts +3 -0
- package/dist/redis/index.d.ts.map +1 -0
- package/dist/redis/index.js +430 -0
- package/dist/redis/index.js.map +14 -0
- package/dist/redis/items.d.ts +13 -0
- package/dist/redis/items.d.ts.map +1 -0
- package/dist/redis/options.d.ts +47 -0
- package/dist/redis/options.d.ts.map +1 -0
- package/dist/redis/run.d.ts +12 -0
- package/dist/redis/run.d.ts.map +1 -0
- package/dist/redis/scripts.d.ts +60 -0
- package/dist/redis/scripts.d.ts.map +1 -0
- package/dist/redis/store.d.ts +30 -0
- package/dist/redis/store.d.ts.map +1 -0
- package/docs/README.md +1 -1
- package/docs/guide.md +185 -6
- package/docs/roadmap.md +30 -7
- package/docs/troubleshooting.md +229 -13
- package/package.json +9 -3
package/README.md
CHANGED
|
@@ -4,9 +4,9 @@ The outbound queue of a mail server: it keeps every message your server
|
|
|
4
4
|
sends to another one, delivers it through `@bumail/smtp/client`, retries
|
|
5
5
|
what failed for now, and sends a delivery status notification (RFC 3464)
|
|
6
6
|
back when it gives up. Each recipient has its own state, the queue
|
|
7
|
-
survives a restart on `bun:sqlite
|
|
8
|
-
on one machine, or on many with PostgreSQL — can
|
|
9
|
-
runtime dependency: only peers.
|
|
7
|
+
survives a restart on `bun:sqlite`, PostgreSQL or Redis, and several
|
|
8
|
+
workers — on one machine, or on many with PostgreSQL or Redis — can
|
|
9
|
+
share one queue. No runtime dependency: only peers.
|
|
10
10
|
|
|
11
11
|
## Install
|
|
12
12
|
|
|
@@ -27,6 +27,7 @@ queue that only uses a smarthost needs none.
|
|
|
27
27
|
| `@bumail/queue/memory` | `MemoryQueueStore`: in memory, lost on a restart — for specs and trials |
|
|
28
28
|
| `@bumail/queue/sqlite` | `SqliteQueueStore`: on disk with `bun:sqlite`, shared by the processes of one machine |
|
|
29
29
|
| `@bumail/queue/postgres` | `PostgresQueueStore`: on PostgreSQL through Bun's own `Bun.sql`, shared by instances on several machines |
|
|
30
|
+
| `@bumail/queue/redis` | `RedisQueueStore`: on Redis through Bun's own `Bun.redis`, shared by instances on several machines |
|
|
30
31
|
|
|
31
32
|
## Usage
|
|
32
33
|
|
|
@@ -175,6 +176,38 @@ waits on one another is taking. An instance that crashes loses its items
|
|
|
175
176
|
when their leases expire, as with `bun:sqlite`. No driver to install:
|
|
176
177
|
`Bun.sql` is Bun's.
|
|
177
178
|
|
|
179
|
+
## Redis, for several machines
|
|
180
|
+
|
|
181
|
+
```ts
|
|
182
|
+
import { nodeResolver } from '@bumail/dns';
|
|
183
|
+
import { createQueue } from '@bumail/queue';
|
|
184
|
+
import { RedisQueueStore } from '@bumail/queue/redis';
|
|
185
|
+
|
|
186
|
+
// In each instance, on the same Redis. A Bun.RedisClient of your own works too:
|
|
187
|
+
// RedisQueueStore.open({ client: new Bun.RedisClient(url) }).
|
|
188
|
+
const store = RedisQueueStore.open({
|
|
189
|
+
url: Bun.env['REDIS_URL'] ?? 'redis://localhost:6379',
|
|
190
|
+
keyPrefix: 'bumail:queue:', // the default: bumail:queue:items, …:item:<id>, …
|
|
191
|
+
});
|
|
192
|
+
const queue = createQueue({ store, hostname: 'mail.example.net', resolver: nodeResolver() });
|
|
193
|
+
queue.start();
|
|
194
|
+
|
|
195
|
+
process.on('SIGTERM', async () => {
|
|
196
|
+
await queue.stop();
|
|
197
|
+
await store.close(); // closes the client it opened for the URL, never yours
|
|
198
|
+
});
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
Every operation that writes is one Lua script, which Redis runs whole:
|
|
202
|
+
two instances never take the same item, and a crashed instance's items
|
|
203
|
+
are claimed again once their leases expire. The message is kept byte for
|
|
204
|
+
byte. One Redis, or a primary with replicas — not Redis Cluster. Redis
|
|
205
|
+
acknowledges a write once it is in memory: with `appendfsync everysec`
|
|
206
|
+
a crash loses up to a second of writes, and a failover to a replica can
|
|
207
|
+
lose acknowledged ones, so keep `appendonly yes`, `appendfsync always`
|
|
208
|
+
and `maxmemory-policy noeviction` where a lost message matters (the
|
|
209
|
+
guide's Redis section). No driver to install: `Bun.redis` is Bun's.
|
|
210
|
+
|
|
178
211
|
## Events and admin
|
|
179
212
|
|
|
180
213
|
```ts
|
|
@@ -243,8 +276,8 @@ control character in it.
|
|
|
243
276
|
|
|
244
277
|
## Traps
|
|
245
278
|
|
|
246
|
-
- **Bun only.** The stores use `bun:sqlite` and `Bun.
|
|
247
|
-
runs on Bun 1.4.2 or later, not on Node.
|
|
279
|
+
- **Bun only.** The stores use `bun:sqlite`, `Bun.sql` and `Bun.redis`,
|
|
280
|
+
so the package runs on Bun 1.4.2 or later, not on Node.
|
|
248
281
|
- **It sends what you enqueue, to anyone.** The queue is not a relay
|
|
249
282
|
policy: enqueue only what an authenticated user submitted, or what your
|
|
250
283
|
own server writes, never what an unauthenticated client handed you.
|
|
@@ -264,12 +297,13 @@ control character in it.
|
|
|
264
297
|
| `MemoryQueueStore` | from `@bumail/queue/memory` |
|
|
265
298
|
| `SqliteQueueStore`, `SqliteQueueStoreOptions` | from `@bumail/queue/sqlite`: `SqliteQueueStore.open({ directory, busyTimeout? })`, `close()` |
|
|
266
299
|
| `PostgresQueueStore`, `PostgresQueueStoreOptions`, `PostgresClient`, `PostgresQueryable` | from `@bumail/queue/postgres`: `PostgresQueueStore.open({ sql, tablePrefix? })`, `migrate()`, `close()`; `sql` a `Bun.SQL` client or a `postgres://` URL |
|
|
300
|
+
| `RedisQueueStore`, `RedisQueueStoreOptions`, `RedisQueueClientOptions`, `RedisQueueUrlOptions`, `RedisQueueClient` | from `@bumail/queue/redis`: `RedisQueueStore.open({ client \| url, keyPrefix? })`, `close()`; `client` a `Bun.RedisClient`, `url` a `redis://` URL |
|
|
267
301
|
|
|
268
302
|
## Documentation
|
|
269
303
|
|
|
270
304
|
These pages ship in the package, under `docs/`.
|
|
271
305
|
|
|
272
306
|
- [Index](https://github.com/softistx/bumail/blob/develop/packages/queue/docs/README.md): the pages, and when to read each.
|
|
273
|
-
- [Guide](https://github.com/softistx/bumail/blob/develop/packages/queue/docs/guide.md): enqueuing, delivery and routing, the retry schedule, DSNs, several workers and leases, events and admin, the stores (PostgreSQL included), testing, and writing a store of your own.
|
|
307
|
+
- [Guide](https://github.com/softistx/bumail/blob/develop/packages/queue/docs/guide.md): enqueuing, delivery and routing, the retry schedule, DSNs, several workers and leases, events and admin, the stores (PostgreSQL and Redis included), testing, and writing a store of your own.
|
|
274
308
|
- [Troubleshooting](https://github.com/softistx/bumail/blob/develop/packages/queue/docs/troubleshooting.md): every `QueueError`, and what to do about it.
|
|
275
309
|
- [Roadmap](https://github.com/softistx/bumail/blob/develop/packages/queue/docs/roadmap.md): what is coming, and what is not planned.
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/errors.ts", "../src/text.ts", "../src/contract/checks.ts"],
|
|
4
4
|
"sourcesContent": [
|
|
5
|
-
"/**\n * Why the queue or a store refused a call. `INVALID`: an option or an\n * argument it cannot take. `MESSAGE_TOO_BIG`, `TOO_MANY_RECIPIENTS` and\n * `QUEUE_FULL`: a limit `enqueue` enforces. `CLOSED`: a store used after\n * `close()`. `LEASE_LOST`: given to the `error` event when another worker\n * took an item before its outcome was recorded.\n */\nexport type QueueErrorCode =\n\t| 'INVALID'\n\t| 'MESSAGE_TOO_BIG'\n\t| 'TOO_MANY_RECIPIENTS'\n\t| 'QUEUE_FULL'\n\t| 'CLOSED'\n\t| 'LEASE_LOST';\n\n/** Thrown by the queue and by every store, whichever answers the contract, with the same codes. */\nexport class QueueError extends Error {\n\toverride readonly name = 'QueueError';\n\treadonly code: QueueErrorCode;\n\n\tconstructor(code: QueueErrorCode, message: string) {\n\t\tsuper(message);\n\t\tthis.code = code;\n\t}\n}\n\n/** `INVALID`, for an option or an argument. */\nexport const invalid = (message: string) => new QueueError('INVALID', message);\n",
|
|
5
|
+
"/**\n * Why the queue or a store refused a call. `INVALID`: an option or an\n * argument it cannot take. `MESSAGE_TOO_BIG`, `TOO_MANY_RECIPIENTS` and\n * `QUEUE_FULL`: a limit `enqueue` enforces. `CLOSED`: a store used after\n * `close()`. `LEASE_LOST`: given to the `error` event when another worker\n * took an item before its outcome was recorded — or, once the lease's\n * expiry passed, when the item is gone: lost or cancelled.\n */\nexport type QueueErrorCode =\n\t| 'INVALID'\n\t| 'MESSAGE_TOO_BIG'\n\t| 'TOO_MANY_RECIPIENTS'\n\t| 'QUEUE_FULL'\n\t| 'CLOSED'\n\t| 'LEASE_LOST';\n\n/** Thrown by the queue and by every store, whichever answers the contract, with the same codes. */\nexport class QueueError extends Error {\n\toverride readonly name = 'QueueError';\n\treadonly code: QueueErrorCode;\n\n\tconstructor(code: QueueErrorCode, message: string) {\n\t\tsuper(message);\n\t\tthis.code = code;\n\t}\n}\n\n/** `INVALID`, for an option or an argument. */\nexport const invalid = (message: string) => new QueueError('INVALID', message);\n",
|
|
6
6
|
"/** Text from the other side, made safe to keep and to write in a header field. */\n\n/** C0, DEL and C1: every control character, CR, LF and TAB included. */\nconst isControl = (code: number) =>\n\tcode < 0x20 || (code >= 0x7f && code <= 0x9f);\n\n/** Whether the text holds a control character. */\nexport function hasControl(text: string): boolean {\n\tfor (let i = 0; i < text.length; i++) {\n\t\tif (isControl(text.charCodeAt(i))) return true;\n\t}\n\treturn false;\n}\n\n/** Each control character replaced by `by`. */\nexport function replaceControls(text: string, by: string): string {\n\tlet out = '';\n\tfor (let i = 0; i < text.length; i++) {\n\t\tout += isControl(text.charCodeAt(i)) ? by : (text[i] as string);\n\t}\n\treturn out;\n}\n\n/**\n * Every control character (CR and LF included) as a space, a lone\n * surrogate as U+FFFD, runs of space as one, cut at `max` characters with\n * `...`, never inside a surrogate pair: what a reply leaves in an item,\n * an event and a DSN.\n */\nexport function cleanText(text: string, max: number): string {\n\tconst clean = replaceControls(text.toWellFormed(), ' ')\n\t\t.replace(/ {2,}/g, ' ')\n\t\t.trim();\n\treturn clean.length > max ? `${cut(clean, max - 3)}...` : clean;\n}\n\n/**\n * The first `length` UTF-16 units, never ending inside a surrogate pair:\n * a lone surrogate is text no store can keep (PostgreSQL's `jsonb` refuses\n * it), nor any UTF-8 encode.\n */\nfunction cut(text: string, length: number): string {\n\tconst head = text.slice(0, length);\n\tconst last = head.charCodeAt(head.length - 1);\n\treturn last >= 0xd800 && last <= 0xdbff ? head.slice(0, -1) : head;\n}\n\n/**\n * Text every store can keep: no NUL (PostgreSQL's `text` and `jsonb` hold\n * none) and no lone surrogate. `cleanText`'s output always is.\n */\nexport const isStorable = (text: string): boolean =>\n\ttext.isWellFormed() && !text.includes('\\0');\n\n/** `cleanText`, then anything not printable ASCII as `?`: for a `message/delivery-status` field. */\nexport const asciiText = (text: string, max: number): string =>\n\tcleanText(text, max).replace(/[^ -~]/g, '?');\n",
|
|
7
7
|
"import { invalid } from '../errors';\nimport { isStorable } from '../text';\nimport type {\n\tAttemptResult,\n\tClaimRequest,\n\tDiagnostic,\n\tNewQueueItem,\n\tQueueItem,\n\tQueueListOptions,\n\tRecipientState,\n\tRecipientUpdate,\n} from './types';\n\n/** What every store checks the same way, so each refuses the same input with the same error. */\n\nconst isTime = (value: unknown): value is number =>\n\ttypeof value === 'number' && Number.isFinite(value);\n\n/** A time in milliseconds since the epoch, or `INVALID` naming it. */\nexport function checkTime(name: string, value: unknown): number {\n\tif (!isTime(value)) throw invalid(`${name} must be a finite number`);\n\treturn value;\n}\n\n/**\n * Text a store keeps: refused, the same way by every store, when it holds\n * a NUL or a lone surrogate, which PostgreSQL cannot keep and no UTF-8\n * encode can carry.\n */\nexport function checkStorable(name: string, text: string): void {\n\tif (!isStorable(text)) {\n\t\tthrow invalid(\n\t\t\t`${name} holds a NUL or a lone surrogate, which a store cannot keep`,\n\t\t);\n\t}\n}\n\n/** A lease owner: a non-empty string. */\nexport function checkOwner(owner: unknown): string {\n\tif (typeof owner !== 'string' || owner === '') {\n\t\tthrow invalid('owner must be a non-empty string');\n\t}\n\tcheckStorable('owner', owner);\n\treturn owner;\n}\n\n/** Every text of a recipient's outcome. */\nfunction checkUpdate(update: RecipientUpdate): void {\n\tcheckStorable('A recipient', update.address);\n\tconst reply = update.reply as Partial<Diagnostic> | undefined;\n\tif (reply === undefined) return;\n\tif (\n\t\ttypeof reply !== 'object' ||\n\t\treply === null ||\n\t\ttypeof reply.text !== 'string'\n\t) {\n\t\tthrow invalid('A reply is an object with a text');\n\t}\n\tcheckStorable('reply.text', reply.text);\n\tfor (const name of ['status', 'host'] as const) {\n\t\tconst value = reply[name];\n\t\tif (value === undefined) continue;\n\t\tif (typeof value !== 'string')\n\t\t\tthrow invalid(`reply.${name} must be a string`);\n\t\tcheckStorable(`reply.${name}`, value);\n\t}\n}\n\nexport function checkNewItem(item: NewQueueItem): void {\n\tif (typeof item !== 'object' || item === null) {\n\t\tthrow invalid('A new item is an object');\n\t}\n\tif (typeof item.from !== 'string') throw invalid('from must be a string');\n\tcheckStorable('from', item.from);\n\tconst to = item.to as unknown;\n\tif (\n\t\t!Array.isArray(to) ||\n\t\tto.length === 0 ||\n\t\t!to.every((a) => typeof a === 'string' && a !== '')\n\t) {\n\t\tthrow invalid('to must be a non-empty array of addresses');\n\t}\n\tfor (const address of to as string[]) checkStorable('A recipient', address);\n\tif (!(item.message instanceof Uint8Array)) {\n\t\tthrow invalid('message must be a Uint8Array');\n\t}\n\tcheckTime('createdAt', item.createdAt);\n}\n\n/** `maxItems`: a positive integer, or nothing. */\nexport function checkMaxItems(max: unknown): number | undefined {\n\tif (max === undefined) return undefined;\n\tif (typeof max !== 'number' || !Number.isInteger(max) || max < 1) {\n\t\tthrow invalid(`maxItems must be an integer of at least 1, not ${max}`);\n\t}\n\treturn max;\n}\n\nexport function checkClaim(request: ClaimRequest): void {\n\tif (typeof request !== 'object' || request === null) {\n\t\tthrow invalid('A claim is an object');\n\t}\n\tcheckOwner(request.owner);\n\tcheckTime('now', request.now);\n\tif (!isTime(request.leaseMs) || request.leaseMs <= 0) {\n\t\tthrow invalid('leaseMs must be a positive number');\n\t}\n}\n\nexport function checkResult(result: AttemptResult): void {\n\tif (typeof result !== 'object' || result === null) {\n\t\tthrow invalid('An attempt result is an object');\n\t}\n\tcheckTime('now', result.now);\n\tcheckTime('nextAttemptAt', result.nextAttemptAt);\n\tif (!Number.isInteger(result.attempts) || result.attempts < 0) {\n\t\tthrow invalid('attempts must be an integer of at least 0');\n\t}\n\tif (typeof result.delayNotified !== 'boolean') {\n\t\tthrow invalid('delayNotified must be true or false');\n\t}\n\tconst statuses = ['delivered', 'deferred', 'failed'];\n\tif (\n\t\t!Array.isArray(result.recipients) ||\n\t\t!result.recipients.every(\n\t\t\t(u) => typeof u?.address === 'string' && statuses.includes(u.status),\n\t\t)\n\t) {\n\t\tthrow invalid(\n\t\t\t'recipients must be an array of { address, status: delivered, deferred or failed }',\n\t\t);\n\t}\n\tfor (const update of result.recipients) checkUpdate(update);\n}\n\n/** The offset and limit of a list: integers, the limit 1 to 1000. */\nexport function checkList(options: QueueListOptions = {}): {\n\toffset: number;\n\tlimit: number;\n} {\n\tconst offset = options.offset ?? 0;\n\tconst limit = options.limit ?? 100;\n\tif (!Number.isInteger(offset) || offset < 0) {\n\t\tthrow invalid(`offset must be an integer of at least 0, not ${offset}`);\n\t}\n\tif (!Number.isInteger(limit) || limit < 1 || limit > 1000) {\n\t\tthrow invalid(`limit must be an integer from 1 to 1000, not ${limit}`);\n\t}\n\treturn { offset, limit };\n}\n\n/** Every recipient is `delivered` or `failed`. */\nexport const isDone = (item: Pick<QueueItem, 'recipients'>): boolean =>\n\titem.recipients.every(\n\t\t(r) => r.status === 'delivered' || r.status === 'failed',\n\t);\n\n/**\n * The item once an attempt is recorded: its recipients' outcomes, a final\n * state never changed again, the schedule, and no lease.\n */\nexport function applyAttempt(\n\titem: QueueItem,\n\tresult: AttemptResult,\n): QueueItem {\n\tconst updates = new Map(result.recipients.map((u) => [u.address, u]));\n\tconst recipients = item.recipients.map((r): RecipientState => {\n\t\tconst update = updates.get(r.address);\n\t\tif (!update || r.status === 'delivered' || r.status === 'failed') return r;\n\t\treturn {\n\t\t\taddress: r.address,\n\t\t\tstatus: update.status,\n\t\t\t...(update.reply ? { reply: update.reply } : {}),\n\t\t\tupdatedAt: result.now,\n\t\t};\n\t});\n\tconst { lease: _, ...rest } = item;\n\treturn {\n\t\t...rest,\n\t\trecipients,\n\t\tnextAttemptAt: result.nextAttemptAt,\n\t\tattempts: result.attempts,\n\t\tdelayNotified: result.delayNotified,\n\t};\n}\n\n/** A new item as a store first keeps it. */\nexport function newItem(id: string, item: NewQueueItem): QueueItem {\n\treturn {\n\t\tid,\n\t\tfrom: item.from,\n\t\trecipients: item.to.map((address) => ({ address, status: 'pending' })),\n\t\tsize: item.message.length,\n\t\tcreatedAt: item.createdAt,\n\t\tnextAttemptAt: item.createdAt,\n\t\tattempts: 0,\n\t\tdelayNotified: false,\n\t};\n}\n\n/** Whether a lease, if any, leaves the item free to claim at `now`. */\nexport const claimable = (item: QueueItem, now: number): boolean =>\n\titem.nextAttemptAt <= now &&\n\t(item.lease === undefined || item.lease.expiresAt <= now);\n"
|
|
8
8
|
],
|
|
9
|
-
"mappings": ";
|
|
9
|
+
"mappings": ";AAiBO,MAAM,oBAAmB,MAAM;AAAA,EACnB,OAAO;AAAA,EAChB;AAAA,EAET,WAAW,CAAC,MAAsB,SAAiB;AAAA,IAClD,MAAM,OAAO;AAAA,IACb,KAAK,OAAO;AAAA;AAEd;AAGO,IAAM,UAAU,CAAC,YAAoB,IAAI,YAAW,WAAW,OAAO;;;ACzB7E,IAAM,YAAY,CAAC,SAClB,OAAO,MAAS,QAAQ,OAAQ,QAAQ;AAGlC,SAAS,UAAU,CAAC,MAAuB;AAAA,EACjD,SAAS,IAAI,EAAG,IAAI,KAAK,QAAQ,KAAK;AAAA,IACrC,IAAI,UAAU,KAAK,WAAW,CAAC,CAAC;AAAA,MAAG,OAAO;AAAA,EAC3C;AAAA,EACA,OAAO;AAAA;AAID,SAAS,eAAe,CAAC,MAAc,IAAoB;AAAA,EACjE,IAAI,MAAM;AAAA,EACV,SAAS,IAAI,EAAG,IAAI,KAAK,QAAQ,KAAK;AAAA,IACrC,OAAO,UAAU,KAAK,WAAW,CAAC,CAAC,IAAI,KAAM,KAAK;AAAA,EACnD;AAAA,EACA,OAAO;AAAA;AASD,SAAS,SAAS,CAAC,MAAc,KAAqB;AAAA,EAC5D,MAAM,QAAQ,gBAAgB,KAAK,aAAa,GAAG,GAAG,EACpD,QAAQ,UAAU,GAAG,EACrB,KAAK;AAAA,EACP,OAAO,MAAM,SAAS,MAAM,GAAG,IAAI,OAAO,MAAM,CAAC,SAAS;AAAA;AAQ3D,SAAS,GAAG,CAAC,MAAc,QAAwB;AAAA,EAClD,MAAM,OAAO,KAAK,MAAM,GAAG,MAAM;AAAA,EACjC,MAAM,OAAO,KAAK,WAAW,KAAK,SAAS,CAAC;AAAA,EAC5C,OAAO,QAAQ,SAAU,QAAQ,QAAS,KAAK,MAAM,GAAG,EAAE,IAAI;AAAA;AAOxD,IAAM,aAAa,CAAC,SAC1B,KAAK,aAAa,KAAK,CAAC,KAAK,SAAS,MAAI;AAGpC,IAAM,YAAY,CAAC,MAAc,QACvC,UAAU,MAAM,GAAG,EAAE,QAAQ,WAAW,GAAG;;;ACzC5C,IAAM,SAAS,CAAC,UACf,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAG5C,SAAS,SAAS,CAAC,MAAc,OAAwB;AAAA,EAC/D,IAAI,CAAC,OAAO,KAAK;AAAA,IAAG,MAAM,QAAQ,GAAG,8BAA8B;AAAA,EACnE,OAAO;AAAA;AAQD,SAAS,aAAa,CAAC,MAAc,MAAoB;AAAA,EAC/D,IAAI,CAAC,WAAW,IAAI,GAAG;AAAA,IACtB,MAAM,QACL,GAAG,iEACJ;AAAA,EACD;AAAA;AAIM,SAAS,UAAU,CAAC,OAAwB;AAAA,EAClD,IAAI,OAAO,UAAU,YAAY,UAAU,IAAI;AAAA,IAC9C,MAAM,QAAQ,kCAAkC;AAAA,EACjD;AAAA,EACA,cAAc,SAAS,KAAK;AAAA,EAC5B,OAAO;AAAA;AAIR,SAAS,WAAW,CAAC,QAA+B;AAAA,EACnD,cAAc,eAAe,OAAO,OAAO;AAAA,EAC3C,MAAM,QAAQ,OAAO;AAAA,EACrB,IAAI,UAAU;AAAA,IAAW;AAAA,EACzB,IACC,OAAO,UAAU,YACjB,UAAU,QACV,OAAO,MAAM,SAAS,UACrB;AAAA,IACD,MAAM,QAAQ,kCAAkC;AAAA,EACjD;AAAA,EACA,cAAc,cAAc,MAAM,IAAI;AAAA,EACtC,WAAW,QAAQ,CAAC,UAAU,MAAM,GAAY;AAAA,IAC/C,MAAM,QAAQ,MAAM;AAAA,IACpB,IAAI,UAAU;AAAA,MAAW;AAAA,IACzB,IAAI,OAAO,UAAU;AAAA,MACpB,MAAM,QAAQ,SAAS,uBAAuB;AAAA,IAC/C,cAAc,SAAS,QAAQ,KAAK;AAAA,EACrC;AAAA;AAGM,SAAS,YAAY,CAAC,MAA0B;AAAA,EACtD,IAAI,OAAO,SAAS,YAAY,SAAS,MAAM;AAAA,IAC9C,MAAM,QAAQ,yBAAyB;AAAA,EACxC;AAAA,EACA,IAAI,OAAO,KAAK,SAAS;AAAA,IAAU,MAAM,QAAQ,uBAAuB;AAAA,EACxE,cAAc,QAAQ,KAAK,IAAI;AAAA,EAC/B,MAAM,KAAK,KAAK;AAAA,EAChB,IACC,CAAC,MAAM,QAAQ,EAAE,KACjB,GAAG,WAAW,KACd,CAAC,GAAG,MAAM,CAAC,MAAM,OAAO,MAAM,YAAY,MAAM,EAAE,GACjD;AAAA,IACD,MAAM,QAAQ,2CAA2C;AAAA,EAC1D;AAAA,EACA,WAAW,WAAW;AAAA,IAAgB,cAAc,eAAe,OAAO;AAAA,EAC1E,IAAI,EAAE,KAAK,mBAAmB,aAAa;AAAA,IAC1C,MAAM,QAAQ,8BAA8B;AAAA,EAC7C;AAAA,EACA,UAAU,aAAa,KAAK,SAAS;AAAA;AAI/B,SAAS,aAAa,CAAC,KAAkC;AAAA,EAC/D,IAAI,QAAQ;AAAA,IAAW;AAAA,EACvB,IAAI,OAAO,QAAQ,YAAY,CAAC,OAAO,UAAU,GAAG,KAAK,MAAM,GAAG;AAAA,IACjE,MAAM,QAAQ,kDAAkD,KAAK;AAAA,EACtE;AAAA,EACA,OAAO;AAAA;AAGD,SAAS,UAAU,CAAC,SAA6B;AAAA,EACvD,IAAI,OAAO,YAAY,YAAY,YAAY,MAAM;AAAA,IACpD,MAAM,QAAQ,sBAAsB;AAAA,EACrC;AAAA,EACA,WAAW,QAAQ,KAAK;AAAA,EACxB,UAAU,OAAO,QAAQ,GAAG;AAAA,EAC5B,IAAI,CAAC,OAAO,QAAQ,OAAO,KAAK,QAAQ,WAAW,GAAG;AAAA,IACrD,MAAM,QAAQ,mCAAmC;AAAA,EAClD;AAAA;AAGM,SAAS,WAAW,CAAC,QAA6B;AAAA,EACxD,IAAI,OAAO,WAAW,YAAY,WAAW,MAAM;AAAA,IAClD,MAAM,QAAQ,gCAAgC;AAAA,EAC/C;AAAA,EACA,UAAU,OAAO,OAAO,GAAG;AAAA,EAC3B,UAAU,iBAAiB,OAAO,aAAa;AAAA,EAC/C,IAAI,CAAC,OAAO,UAAU,OAAO,QAAQ,KAAK,OAAO,WAAW,GAAG;AAAA,IAC9D,MAAM,QAAQ,2CAA2C;AAAA,EAC1D;AAAA,EACA,IAAI,OAAO,OAAO,kBAAkB,WAAW;AAAA,IAC9C,MAAM,QAAQ,qCAAqC;AAAA,EACpD;AAAA,EACA,MAAM,WAAW,CAAC,aAAa,YAAY,QAAQ;AAAA,EACnD,IACC,CAAC,MAAM,QAAQ,OAAO,UAAU,KAChC,CAAC,OAAO,WAAW,MAClB,CAAC,MAAM,OAAO,GAAG,YAAY,YAAY,SAAS,SAAS,EAAE,MAAM,CACpE,GACC;AAAA,IACD,MAAM,QACL,mFACD;AAAA,EACD;AAAA,EACA,WAAW,UAAU,OAAO;AAAA,IAAY,YAAY,MAAM;AAAA;AAIpD,SAAS,SAAS,CAAC,UAA4B,CAAC,GAGrD;AAAA,EACD,MAAM,SAAS,QAAQ,UAAU;AAAA,EACjC,MAAM,QAAQ,QAAQ,SAAS;AAAA,EAC/B,IAAI,CAAC,OAAO,UAAU,MAAM,KAAK,SAAS,GAAG;AAAA,IAC5C,MAAM,QAAQ,gDAAgD,QAAQ;AAAA,EACvE;AAAA,EACA,IAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,KAAK,QAAQ,MAAM;AAAA,IAC1D,MAAM,QAAQ,gDAAgD,OAAO;AAAA,EACtE;AAAA,EACA,OAAO,EAAE,QAAQ,MAAM;AAAA;AAIjB,IAAM,SAAS,CAAC,SACtB,KAAK,WAAW,MACf,CAAC,MAAM,EAAE,WAAW,eAAe,EAAE,WAAW,QACjD;AAMM,SAAS,YAAY,CAC3B,MACA,QACY;AAAA,EACZ,MAAM,UAAU,IAAI,IAAI,OAAO,WAAW,IAAI,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;AAAA,EACpE,MAAM,aAAa,KAAK,WAAW,IAAI,CAAC,MAAsB;AAAA,IAC7D,MAAM,SAAS,QAAQ,IAAI,EAAE,OAAO;AAAA,IACpC,IAAI,CAAC,UAAU,EAAE,WAAW,eAAe,EAAE,WAAW;AAAA,MAAU,OAAO;AAAA,IACzE,OAAO;AAAA,MACN,SAAS,EAAE;AAAA,MACX,QAAQ,OAAO;AAAA,SACX,OAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,IAAI,CAAC;AAAA,MAC9C,WAAW,OAAO;AAAA,IACnB;AAAA,GACA;AAAA,EACD,QAAQ,OAAO,MAAM,SAAS;AAAA,EAC9B,OAAO;AAAA,OACH;AAAA,IACH;AAAA,IACA,eAAe,OAAO;AAAA,IACtB,UAAU,OAAO;AAAA,IACjB,eAAe,OAAO;AAAA,EACvB;AAAA;AAIM,SAAS,OAAO,CAAC,IAAY,MAA+B;AAAA,EAClE,OAAO;AAAA,IACN;AAAA,IACA,MAAM,KAAK;AAAA,IACX,YAAY,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,QAAQ,UAAU,EAAE;AAAA,IACrE,MAAM,KAAK,QAAQ;AAAA,IACnB,WAAW,KAAK;AAAA,IAChB,eAAe,KAAK;AAAA,IACpB,UAAU;AAAA,IACV,eAAe;AAAA,EAChB;AAAA;AAIM,IAAM,YAAY,CAAC,MAAiB,QAC1C,KAAK,iBAAiB,QACrB,KAAK,UAAU,aAAa,KAAK,MAAM,aAAa;",
|
|
10
10
|
"debugId": "81614FFC0DB6330C64756E2164756E21",
|
|
11
11
|
"names": []
|
|
12
12
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// src/masked.ts
|
|
2
|
+
function masked(text, raw) {
|
|
3
|
+
let decoded = raw;
|
|
4
|
+
try {
|
|
5
|
+
decoded = decodeURIComponent(raw);
|
|
6
|
+
} catch {}
|
|
7
|
+
let out = text;
|
|
8
|
+
for (const password of [raw, decoded]) {
|
|
9
|
+
if (password !== "")
|
|
10
|
+
out = out.replaceAll(password, "…");
|
|
11
|
+
}
|
|
12
|
+
return out;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export { masked };
|
|
16
|
+
|
|
17
|
+
//# debugId=64BFD310901EABF064756E2164756E21
|
|
18
|
+
//# sourceMappingURL=index-y3n1w2az.js.map
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/masked.ts"],
|
|
4
|
+
"sourcesContent": [
|
|
5
|
+
"/**\n * `text` with a URL's password masked, as written in the URL and decoded,\n * if it decodes: what a store opening a URL repeats of the client's\n * reason, which may name it.\n */\nexport function masked(text: string, raw: string): string {\n\tlet decoded = raw;\n\ttry {\n\t\tdecoded = decodeURIComponent(raw);\n\t} catch {\n\t\t// Not percent-encoding a client reads either: the raw form is what it would repeat.\n\t}\n\tlet out = text;\n\tfor (const password of [raw, decoded]) {\n\t\tif (password !== '') out = out.replaceAll(password, '…');\n\t}\n\treturn out;\n}\n"
|
|
6
|
+
],
|
|
7
|
+
"mappings": ";AAKO,SAAS,MAAM,CAAC,MAAc,KAAqB;AAAA,EACzD,IAAI,UAAU;AAAA,EACd,IAAI;AAAA,IACH,UAAU,mBAAmB,GAAG;AAAA,IAC/B,MAAM;AAAA,EAGR,IAAI,MAAM;AAAA,EACV,WAAW,YAAY,CAAC,KAAK,OAAO,GAAG;AAAA,IACtC,IAAI,aAAa;AAAA,MAAI,MAAM,IAAI,WAAW,UAAU,GAAG;AAAA,EACxD;AAAA,EACA,OAAO;AAAA;",
|
|
8
|
+
"debugId": "64BFD310901EABF064756E2164756E21",
|
|
9
|
+
"names": []
|
|
10
|
+
}
|
package/dist/errors.d.ts
CHANGED
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
* argument it cannot take. `MESSAGE_TOO_BIG`, `TOO_MANY_RECIPIENTS` and
|
|
4
4
|
* `QUEUE_FULL`: a limit `enqueue` enforces. `CLOSED`: a store used after
|
|
5
5
|
* `close()`. `LEASE_LOST`: given to the `error` event when another worker
|
|
6
|
-
* took an item before its outcome was recorded
|
|
6
|
+
* took an item before its outcome was recorded — or, once the lease's
|
|
7
|
+
* expiry passed, when the item is gone: lost or cancelled.
|
|
7
8
|
*/
|
|
8
9
|
export type QueueErrorCode = 'INVALID' | 'MESSAGE_TOO_BIG' | 'TOO_MANY_RECIPIENTS' | 'QUEUE_FULL' | 'CLOSED' | 'LEASE_LOST';
|
|
9
10
|
/** Thrown by the queue and by every store, whichever answers the contract, with the same codes. */
|
package/dist/errors.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,MAAM,MAAM,cAAc,GACvB,SAAS,GACT,iBAAiB,GACjB,qBAAqB,GACrB,YAAY,GACZ,QAAQ,GACR,YAAY,CAAC;AAEhB,mGAAmG;AACnG,qBAAa,UAAW,SAAQ,KAAK;IACpC,SAAkB,IAAI,gBAAgB;IACtC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;gBAElB,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM;CAIjD;AAED,+CAA+C;AAC/C,eAAO,MAAM,OAAO,GAAI,SAAS,MAAM,eAAuC,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -714,6 +714,7 @@ function settle(item, updates, skipped, now, settings) {
|
|
|
714
714
|
}
|
|
715
715
|
|
|
716
716
|
// src/queue/deliver.ts
|
|
717
|
+
var goneAfter = (lapse, id) => new QueueError2("LEASE_LOST", lapse === "taken" ? `The lease on ${id} was lost while it was delivered, and the item is gone: finished by the worker that took it, the message then sent twice, or cancelled` : `The lease on ${id} expired before its outcome was recorded, and the item is gone: lost to another worker that finished it, the message then sent twice, or cancelled`);
|
|
717
718
|
function groupsOf(item) {
|
|
718
719
|
const groups = new Map;
|
|
719
720
|
for (const r of item.recipients) {
|
|
@@ -806,6 +807,11 @@ async function deliverItem(ctx, item) {
|
|
|
806
807
|
const after = await store.complete(item.id, settings.owner, settled.result);
|
|
807
808
|
ctx.recorded();
|
|
808
809
|
if (!after && await store.get(item.id) === undefined) {
|
|
810
|
+
const lapse = await ctx.leaseLapsed(now);
|
|
811
|
+
if (lapse) {
|
|
812
|
+
events.emit("error", { error: goneAfter(lapse, item.id), id: item.id });
|
|
813
|
+
return;
|
|
814
|
+
}
|
|
809
815
|
emitOutcomes(events, item, settled);
|
|
810
816
|
return;
|
|
811
817
|
}
|
|
@@ -821,6 +827,57 @@ async function deliverItem(ctx, item) {
|
|
|
821
827
|
}
|
|
822
828
|
}
|
|
823
829
|
|
|
830
|
+
// src/queue/lease.ts
|
|
831
|
+
function leaseOf(ctx, item) {
|
|
832
|
+
const { settings } = ctx;
|
|
833
|
+
return {
|
|
834
|
+
item,
|
|
835
|
+
done: false,
|
|
836
|
+
expiresAt: item.lease?.expiresAt ?? settings.now() + settings.leaseMs,
|
|
837
|
+
taken: false,
|
|
838
|
+
looking: Promise.resolve()
|
|
839
|
+
};
|
|
840
|
+
}
|
|
841
|
+
async function lookAfterRefusal(ctx, lease) {
|
|
842
|
+
const { id } = lease.item;
|
|
843
|
+
if (await ctx.store.get(id) === undefined)
|
|
844
|
+
return;
|
|
845
|
+
lease.taken = true;
|
|
846
|
+
const error = new QueueError2("LEASE_LOST", `The lease on ${id} was lost while it was delivered`);
|
|
847
|
+
ctx.events.emit("error", { error, id });
|
|
848
|
+
}
|
|
849
|
+
async function renew(ctx, lease) {
|
|
850
|
+
const { settings, store, events } = ctx;
|
|
851
|
+
const { id } = lease.item;
|
|
852
|
+
const next = settings.now() + settings.leaseMs;
|
|
853
|
+
let held;
|
|
854
|
+
try {
|
|
855
|
+
held = await store.renew(id, settings.owner, next);
|
|
856
|
+
} catch (error) {
|
|
857
|
+
if (!lease.done)
|
|
858
|
+
events.emit("error", { error, id });
|
|
859
|
+
return !lease.done;
|
|
860
|
+
}
|
|
861
|
+
if (held)
|
|
862
|
+
lease.expiresAt = Math.max(lease.expiresAt, next);
|
|
863
|
+
if (lease.done)
|
|
864
|
+
return false;
|
|
865
|
+
if (held)
|
|
866
|
+
return true;
|
|
867
|
+
lease.done = true;
|
|
868
|
+
lease.looking = lookAfterRefusal(ctx, lease).catch((error) => {
|
|
869
|
+
events.emit("error", { error, id });
|
|
870
|
+
});
|
|
871
|
+
return false;
|
|
872
|
+
}
|
|
873
|
+
async function lapsed(lease, now) {
|
|
874
|
+
await lease.looking;
|
|
875
|
+
if (lease.taken)
|
|
876
|
+
return "taken";
|
|
877
|
+
return lease.expiresAt <= now ? "expired" : undefined;
|
|
878
|
+
}
|
|
879
|
+
var settled = (lease) => lease.looking;
|
|
880
|
+
|
|
824
881
|
// src/queue/limiter.ts
|
|
825
882
|
class Limiter {
|
|
826
883
|
#size;
|
|
@@ -888,42 +945,35 @@ class Worker {
|
|
|
888
945
|
this.#slots = new Limiter(ctx.settings.concurrency);
|
|
889
946
|
this.#domains = new KeyedLimiter(ctx.settings.perDomain);
|
|
890
947
|
}
|
|
891
|
-
#renewing(
|
|
892
|
-
const { settings, store, events } = this.#ctx;
|
|
893
|
-
let done = false;
|
|
894
|
-
const stop = () => {
|
|
895
|
-
done = true;
|
|
896
|
-
clearInterval(timer);
|
|
897
|
-
};
|
|
948
|
+
#renewing(lease) {
|
|
898
949
|
const timer = setInterval(() => {
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
return;
|
|
903
|
-
stop();
|
|
904
|
-
const error = new QueueError2("LEASE_LOST", `The lease on ${item.id} was lost while it was delivered`);
|
|
905
|
-
events.emit("error", { error, id: item.id });
|
|
906
|
-
}, (error) => {
|
|
907
|
-
if (!done)
|
|
908
|
-
events.emit("error", { error, id: item.id });
|
|
950
|
+
renew(this.#ctx, lease).then((more) => {
|
|
951
|
+
if (!more)
|
|
952
|
+
clearInterval(timer);
|
|
909
953
|
});
|
|
910
|
-
}, settings.leaseMs / 3);
|
|
954
|
+
}, this.#ctx.settings.leaseMs / 3);
|
|
911
955
|
timer.unref?.();
|
|
912
|
-
return
|
|
956
|
+
return () => {
|
|
957
|
+
lease.done = true;
|
|
958
|
+
clearInterval(timer);
|
|
959
|
+
};
|
|
913
960
|
}
|
|
914
961
|
async#deliver(item) {
|
|
915
|
-
const
|
|
962
|
+
const lease = leaseOf(this.#ctx, item);
|
|
963
|
+
const stop = this.#renewing(lease);
|
|
916
964
|
try {
|
|
917
965
|
await deliverItem({
|
|
918
966
|
...this.#ctx,
|
|
919
967
|
domains: this.#domains,
|
|
920
968
|
stopping: () => this.#stopping,
|
|
921
|
-
recorded:
|
|
969
|
+
recorded: stop,
|
|
970
|
+
leaseLapsed: (now) => lapsed(lease, now)
|
|
922
971
|
}, item);
|
|
923
972
|
} catch (error) {
|
|
924
973
|
this.#ctx.events.emit("error", { error, id: item.id });
|
|
925
974
|
} finally {
|
|
926
|
-
|
|
975
|
+
stop();
|
|
976
|
+
await settled(lease);
|
|
927
977
|
}
|
|
928
978
|
}
|
|
929
979
|
#claim() {
|
|
@@ -1066,5 +1116,5 @@ export {
|
|
|
1066
1116
|
createQueue
|
|
1067
1117
|
};
|
|
1068
1118
|
|
|
1069
|
-
//# debugId=
|
|
1119
|
+
//# debugId=FF4E5F691337408064756E2164756E21
|
|
1070
1120
|
//# sourceMappingURL=index.js.map
|