@softistx/janus-webhooks-redis 0.0.0-stage → 0.12.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +203 -2
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +431 -0
- package/dist/index.js.map +12 -0
- package/dist/queue.d.ts +30 -0
- package/dist/queue.d.ts.map +1 -0
- package/dist/replies.d.ts +28 -0
- package/dist/replies.d.ts.map +1 -0
- package/dist/scripts.d.ts +58 -0
- package/dist/scripts.d.ts.map +1 -0
- package/docs/README.md +12 -0
- package/docs/guide/wiring.md +162 -0
- package/docs/roadmap.md +63 -0
- package/docs/troubleshooting.md +339 -0
- package/package.json +65 -5
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/queue.ts", "../src/replies.ts", "../src/scripts.ts"],
|
|
4
|
+
"sourcesContent": [
|
|
5
|
+
"import { randomUUID } from 'node:crypto';\nimport { StoreFailure } from '@softistx/janus';\nimport type {\n\tFailure,\n\tSendableEvent,\n\tWebhookQueue,\n} from '@softistx/janus-webhooks';\nimport type { RedisConnection } from '@softistx/redis';\nimport type { RedisClient } from 'bun';\nimport {\n\tlimitOf,\n\ttype Method,\n\tstampOf,\n\tstatusOf,\n\ttoCount,\n\ttoDeliveries,\n\ttoHeld,\n} from './replies';\nimport {\n\tCLAIM_DELIVERIES,\n\tCLAIM_ORPHANED_DELIVERIES,\n\tDELETE_DELIVERY,\n\tEXTEND_LEASE,\n\tINSERT_DELIVERIES,\n\tSCHEDULE_RETRY,\n} from './scripts';\n\n/** What the queue takes besides the connection. */\nexport interface RedisWebhookQueueOptions {\n\t/**\n\t * What every key starts with — `janus:webhooks:` unless you say\n\t * otherwise. Every process that shares deliveries passes the same one;\n\t * two applications sharing one Redis each take their own.\n\t */\n\treadonly prefix?: string;\n}\n\n/**\n * The durable queue of `@softistx/janus-webhooks`, over one Redis, on\n * `@softistx/redis`'s connection: deliveries wait in Redis, so a retry waiting\n * when a process stops is sent by the next one.\n *\n * ```ts\n * const redis = await connectRedis(process.env.REDIS_URL);\n * const listener = webhooks({ endpoints, queue: createRedisWebhookQueue(redis) });\n * ```\n *\n * **Every method is one Lua script**, atomic as the port asks: two claims\n * running at once — in one process or in twenty — never answer one\n * delivery. Every time is the caller's, never Redis's clock.\n *\n * It creates nothing and needs no schema: the keys are made by the first\n * insert, and a delivery's keys go with its delete.\n */\nexport function createRedisWebhookQueue(\n\tredis: RedisConnection,\n\toptions: RedisWebhookQueueOptions = {},\n): WebhookQueue {\n\tconst run = runner(\n\t\tscriptsOver(redis.client),\n\t\toptions.prefix ?? 'janus:webhooks:',\n\t);\n\n\treturn {\n\t\tinsertDeliveries: (event, endpoints, dueAt) =>\n\t\t\trun(\n\t\t\t\t'insertDeliveries',\n\t\t\t\tINSERT_DELIVERIES,\n\t\t\t\tinsertArgsOf(event, endpoints, dueAt),\n\t\t\t\ttoCount,\n\t\t\t),\n\n\t\t// A lease per claim, minted here: every write after it names it.\n\t\tclaimDeliveries: (endpoints, now, leaseUntil, limit) =>\n\t\t\trun(\n\t\t\t\t'claimDeliveries',\n\t\t\t\tCLAIM_DELIVERIES,\n\t\t\t\t(operation) => [\n\t\t\t\t\tstampOf(now, operation, 'now'),\n\t\t\t\t\t...claimArgsOf(leaseUntil, limit, operation),\n\t\t\t\t\t...endpoints,\n\t\t\t\t],\n\t\t\t\ttoDeliveries,\n\t\t\t),\n\n\t\tclaimOrphanedDeliveries: (known, dueBefore, leaseUntil, limit) =>\n\t\t\trun(\n\t\t\t\t'claimOrphanedDeliveries',\n\t\t\t\tCLAIM_ORPHANED_DELIVERIES,\n\t\t\t\t(operation) => [\n\t\t\t\t\tstampOf(dueBefore, operation, 'dueBefore'),\n\t\t\t\t\t...claimArgsOf(leaseUntil, limit, operation),\n\t\t\t\t\t...known,\n\t\t\t\t],\n\t\t\t\ttoDeliveries,\n\t\t\t),\n\n\t\textendLease: (id, lease, until) =>\n\t\t\trun(\n\t\t\t\t'extendLease',\n\t\t\t\tEXTEND_LEASE,\n\t\t\t\t(operation) => [id, lease, stampOf(until, operation, 'until')],\n\t\t\t\ttoHeld,\n\t\t\t),\n\n\t\tscheduleRetry: (id, lease, dueAt, failed) =>\n\t\t\trun(\n\t\t\t\t'scheduleRetry',\n\t\t\t\tSCHEDULE_RETRY,\n\t\t\t\tretryArgsOf(id, lease, dueAt, failed),\n\t\t\t\ttoHeld,\n\t\t\t),\n\n\t\tdeleteDelivery: (id, lease) =>\n\t\t\trun('deleteDelivery', DELETE_DELIVERY, () => [id, lease], toHeld),\n\t};\n}\n\n/**\n * Runs `operation`'s script over the arguments `argsOf` builds — which\n * refuses, before any I/O, what the script would store and no claim could\n * read back — and decodes its reply.\n */\ntype Run = <T>(\n\toperation: Method,\n\tscript: string,\n\targsOf: (operation: Method) => readonly string[],\n\tdecode: (reply: unknown, operation: Method) => T,\n) => Promise<T>;\n\n/** A runner whose every script gets `prefix` as its first argument. */\nfunction runner(evaluate: Evaluate, prefix: string): Run {\n\treturn async (operation, script, argsOf, decode) => {\n\t\tconst args = argsOf(operation);\n\t\treturn decode(\n\t\t\tawait evaluate(operation, script, [prefix, ...args]),\n\t\t\toperation,\n\t\t);\n\t};\n}\n\n/**\n * An event's fields — a user event's five and its provider (`''` when it has\n * none), or a generic event's id, type, date and its `data` as JSON — when\n * its deliveries fall due, then the endpoints.\n */\nfunction insertArgsOf(\n\tevent: SendableEvent,\n\tendpoints: readonly string[],\n\tdueAt: Date,\n): (operation: Method) => readonly string[] {\n\treturn (operation) => [\n\t\tevent.id,\n\t\tevent.type,\n\t\tstampOf(event.occurredAt, operation, 'event.occurredAt'),\n\t\t'data' in event ? '' : event.userId,\n\t\t'data' in event ? '' : event.userType,\n\t\t'data' in event ? '' : (event.provider ?? ''),\n\t\t'data' in event ? dataOf(event.data, operation) : '',\n\t\tstampOf(dueAt, operation, 'dueAt'),\n\t\t...endpoints,\n\t];\n}\n\n/**\n * A generic event's `data` as the JSON it is stored as — or a `TypeError`,\n * before any I/O, for what no claim could read back.\n */\nfunction dataOf(data: unknown, operation: Method): string {\n\tlet json: string | undefined;\n\ttry {\n\t\tjson = JSON.stringify(data);\n\t} catch {\n\t\tjson = undefined;\n\t}\n\tif (json === undefined || !json.startsWith('{')) {\n\t\tthrow new TypeError(\n\t\t\t`webhookQueue.${operation}: event.data is a JSON object`,\n\t\t);\n\t}\n\treturn json;\n}\n\n/**\n * What both claims take after the instant they claim as of: the lease's\n * end, the limit, and the lease itself, minted here.\n */\nfunction claimArgsOf(\n\tleaseUntil: Date,\n\tlimit: number,\n\toperation: Method,\n): readonly string[] {\n\treturn [\n\t\tstampOf(leaseUntil, operation, 'leaseUntil'),\n\t\tlimitOf(limit, operation),\n\t\trandomUUID(),\n\t];\n}\n\n/** The delivery and its lease, when it falls due again, and what failed. */\nfunction retryArgsOf(\n\tid: string,\n\tlease: string,\n\tdueAt: Date,\n\tfailed: Failure,\n): (operation: Method) => readonly string[] {\n\treturn (operation) => [\n\t\tid,\n\t\tlease,\n\t\tstampOf(dueAt, operation, 'dueAt'),\n\t\tstatusOf(failed.status, operation),\n\t\tfailed.error ?? '',\n\t];\n}\n\n/** Runs a script, and makes every rejection one the port allows. */\ntype Evaluate = (\n\toperation: Method,\n\tscript: string,\n\targs: readonly string[],\n) => Promise<unknown>;\n\n/**\n * Each script is sent once by its SHA — `EVALSHA` — and in full only when\n * Redis answers `NOSCRIPT`: after a restart, a failover, or a `SCRIPT FLUSH`.\n * Every other error is a failure, and nothing is ever answered as an absence.\n */\nfunction scriptsOver(client: RedisClient): Evaluate {\n\tconst shas = new Map<string, string>();\n\tconst shaOf = (script: string) => {\n\t\tlet sha = shas.get(script);\n\t\tif (sha === undefined) {\n\t\t\tsha = new Bun.CryptoHasher('sha1').update(script).digest('hex');\n\t\t\tshas.set(script, sha);\n\t\t}\n\t\treturn sha;\n\t};\n\n\treturn async (operation, script, args) => {\n\t\ttry {\n\t\t\ttry {\n\t\t\t\treturn await client.send('EVALSHA', [shaOf(script), '0', ...args]);\n\t\t\t} catch (error) {\n\t\t\t\tif (!isNoScript(error)) throw error;\n\t\t\t\treturn await client.send('EVAL', [script, '0', ...args]);\n\t\t\t}\n\t\t} catch (error) {\n\t\t\tthrow new StoreFailure(\n\t\t\t\t`webhookQueue.${operation}: the queue could not answer`,\n\t\t\t\t{ operation, cause: error },\n\t\t\t);\n\t\t}\n\t};\n}\n\nfunction isNoScript(error: unknown): boolean {\n\treturn error instanceof Error && error.message.startsWith('NOSCRIPT');\n}\n",
|
|
6
|
+
"import {\n\tStoreFailure,\n\ttype UserEvent,\n\ttype UserEventType,\n} from '@softistx/janus';\nimport type {\n\tFailure,\n\tQueuedDelivery,\n\tWebhookEvent,\n\tWebhookQueue,\n} from '@softistx/janus-webhooks';\n\n/** A method of the queue port: what a failure names. */\nexport type Method = keyof WebhookQueue & string;\n\n/**\n * What a script answers, read back into the port's deliveries. **A reply\n * this adapter did not write is a failure, never an absence**: a key of the\n * prefix changed by hand, or written by another version, rejects with\n * `StoreFailure` — naming the method it failed, as every other failure does.\n *\n * Dates travel as milliseconds since the epoch, and `null` as `''`.\n */\n\nexport function stamp(at: Date): string {\n\treturn String(at.getTime());\n}\n\n/**\n * A time the caller passed, as milliseconds — or a `TypeError`, before any\n * I/O, for an Invalid Date: `stamp` would write `'NaN'`, which no claim could\n * read back. A date before 1970 is a negative number, and reads back.\n */\nexport function stampOf(at: Date, operation: Method, name: string): string {\n\tif (!(at instanceof Date) || Number.isNaN(at.getTime())) {\n\t\tthrow new TypeError(`webhookQueue.${operation}: ${name} is a valid Date`);\n\t}\n\treturn stamp(at);\n}\n\n/** A claim's `limit`, or a `TypeError`: a whole number of deliveries, 0 or more. */\nexport function limitOf(limit: number, operation: Method): string {\n\tif (!Number.isSafeInteger(limit) || limit < 0) {\n\t\tthrow new TypeError(\n\t\t\t`webhookQueue.${operation}: limit is a whole number of deliveries, 0 or more`,\n\t\t);\n\t}\n\treturn String(limit);\n}\n\n/** A failure's `status`, or a `TypeError`: `null`, or a whole number. */\nexport function statusOf(status: number | null, operation: Method): string {\n\tif (status === null) return '';\n\tif (!Number.isSafeInteger(status) || status < 0) {\n\t\tthrow new TypeError(\n\t\t\t`webhookQueue.${operation}: failed.status is null or a whole number`,\n\t\t);\n\t}\n\treturn String(status);\n}\n\n/** `[[id, field, value, …], …]` as the deliveries a claim took. */\nexport function toDeliveries(\n\treply: unknown,\n\toperation: Method,\n): QueuedDelivery[] {\n\tif (!Array.isArray(reply)) throw unreadable(operation, 'a list');\n\treturn reply.map((one) => toDelivery(one, operation));\n}\n\n/** `1` as `true`, `0` as `false`: what a write guarded by a lease answers. */\nexport function toHeld(reply: unknown, operation: Method): boolean {\n\tif (reply !== 0 && reply !== 1) throw unreadable(operation, '0 or 1');\n\treturn reply === 1;\n}\n\nexport function toCount(reply: unknown, operation: Method): number {\n\tif (!Number.isSafeInteger(reply) || (reply as number) < 0) {\n\t\tthrow unreadable(operation, 'a count');\n\t}\n\treturn reply as number;\n}\n\nfunction toDelivery(reply: unknown, operation: Method): QueuedDelivery {\n\tif (!Array.isArray(reply) || typeof reply[0] !== 'string') {\n\t\tthrow unreadable(operation, 'a delivery');\n\t}\n\tconst [id, ...rest] = reply as [string, ...unknown[]];\n\tconst read = readerOf(rest, operation);\n\tconst lease = read.text('lease');\n\tif (lease === '') throw unreadable(operation, 'a delivery claimed');\n\tconst event = read.has('data')\n\t\t? ({\n\t\t\t\tid: read.text('eventId'),\n\t\t\t\ttype: read.name('type'),\n\t\t\t\toccurredAt: read.date('occurredAt'),\n\t\t\t\tdata: read.json('data'),\n\t\t\t} satisfies WebhookEvent)\n\t\t: ({\n\t\t\t\tid: read.text('eventId'),\n\t\t\t\ttype: read.type('type'),\n\t\t\t\toccurredAt: read.date('occurredAt'),\n\t\t\t\tuserId: read.text('userId'),\n\t\t\t\tuserType: read.text('userType'),\n\t\t\t\t...read.optionalProvider(),\n\t\t\t} satisfies UserEvent);\n\treturn {\n\t\tid,\n\t\tevent,\n\t\tendpoint: read.text('endpoint'),\n\t\tattempts: read.count('attempts'),\n\t\tfailed: read.failure(),\n\t\tlease,\n\t};\n}\n\n/**\n * Every user event type, once: `satisfies` fails to compile the day\n * `@softistx/janus` adds another, which a queue must then read back too.\n */\nconst TYPES = {\n\t'user.created': true,\n\t'user.emailVerified': true,\n\t'user.passwordReset': true,\n\t'user.passwordChanged': true,\n\t'user.emailChanged': true,\n\t'user.secondFactorEnabled': true,\n\t'user.secondFactorDisabled': true,\n\t'user.recoveryCodesRegenerated': true,\n\t'user.recoveryCodeUsed': true,\n\t'user.newDeviceSignedIn': true,\n\t'user.avatarChanged': true,\n\t'user.avatarRemoved': true,\n\t'user.socialLinked': true,\n\t'user.socialUnlinked': true,\n\t'user.passkeyAdded': true,\n\t'user.passkeyRemoved': true,\n\t'user.passkeyCloneSuspected': true,\n\t'user.apiKeyCreated': true,\n\t'user.apiKeyRevoked': true,\n\t'user.deleted': true,\n} as const satisfies Record<UserEventType, true>;\n\n/** The fields of `[field, value, …]`, as Redis answers a hash from a script. */\nfunction readerOf(reply: readonly unknown[], operation: Method) {\n\tconst fields = new Map<string, string>();\n\tfor (let i = 0; i + 1 < reply.length; i += 2) {\n\t\tfields.set(String(reply[i]), String(reply[i + 1]));\n\t}\n\tconst text = (name: string): string => {\n\t\tconst value = fields.get(name);\n\t\tif (value === undefined) {\n\t\t\tthrow unreadable(operation, `a hash with \\`${name}\\``);\n\t\t}\n\t\treturn value;\n\t};\n\t/** Only what this adapter writes: digits. `Number()` would take '' and '1e1'. */\n\tconst digits = (name: string, what: string): number => {\n\t\tconst value = text(name);\n\t\tconst read = /^(0|[1-9]\\d*)$/.test(value) ? Number(value) : Number.NaN;\n\t\tif (!Number.isSafeInteger(read)) throw unreadable(operation, what);\n\t\treturn read;\n\t};\n\treturn {\n\t\ttext,\n\t\thas: (name: string) => fields.has(name),\n\t\t/** A generic event's type: any name, as written. */\n\t\tname: (name: string): string => {\n\t\t\tconst value = text(name);\n\t\t\tif (value === '') throw unreadable(operation, 'an event type');\n\t\t\treturn value;\n\t\t},\n\t\t/** A generic event's `data`: the JSON object an insert wrote. */\n\t\tjson: (name: string): Record<string, unknown> => {\n\t\t\tlet value: unknown;\n\t\t\ttry {\n\t\t\t\tvalue = JSON.parse(text(name));\n\t\t\t} catch {\n\t\t\t\tvalue = null;\n\t\t\t}\n\t\t\tif (typeof value !== 'object' || value === null || Array.isArray(value)) {\n\t\t\t\tthrow unreadable(operation, `a JSON object in \\`${name}\\``);\n\t\t\t}\n\t\t\treturn value as Record<string, unknown>;\n\t\t},\n\t\tcount: (name: string) => digits(name, `a count in \\`${name}\\``),\n\t\t/** As `@softistx/janus-redis` reads one: before 1970 is negative; '' and 'NaN' are not dates. */\n\t\tdate: (name: string): Date => {\n\t\t\tconst value = text(name);\n\t\t\tconst at = new Date(Number(value));\n\t\t\tif (value === '' || Number.isNaN(at.getTime())) {\n\t\t\t\tthrow unreadable(operation, `a date in \\`${name}\\``);\n\t\t\t}\n\t\t\treturn at;\n\t\t},\n\t\ttype: (name: string): UserEventType => {\n\t\t\tconst value = text(name);\n\t\t\tif (!Object.hasOwn(TYPES, value))\n\t\t\t\tthrow unreadable(operation, 'a user event type');\n\t\t\treturn value as UserEventType;\n\t\t},\n\t\t/** The provider a social event was written with; nothing for any other event. */\n\t\toptionalProvider: (): { provider?: string } => {\n\t\t\tconst provider = fields.get('provider');\n\t\t\treturn provider === undefined || provider === '' ? {} : { provider };\n\t\t},\n\t\t/** `null` before any attempt failed: then neither field is written. */\n\t\tfailure: (): Failure | null => {\n\t\t\tif (!fields.has('status') && !fields.has('error')) return null;\n\t\t\tconst status = text('status');\n\t\t\tconst error = text('error');\n\t\t\treturn {\n\t\t\t\tstatus: status === '' ? null : digits('status', 'a status in `status`'),\n\t\t\t\terror: error === '' ? null : error,\n\t\t\t};\n\t\t},\n\t};\n}\n\nfunction unreadable(operation: Method, what: string): StoreFailure {\n\treturn new StoreFailure(\n\t\t`webhookQueue.${operation}: a reply that is not ${what} — a key under the prefix this adapter did not write`,\n\t\t{ operation },\n\t);\n}\n",
|
|
7
|
+
"/**\n * The Lua scripts, **one per port method**. A script is atomic: Redis runs\n * nothing else while it does, which is what the port asks of every method —\n * and what makes a claim one step, so two claims running at once never\n * answer one delivery.\n *\n * The layout, every key built inside the script from `ARGV[1]`, the prefix:\n *\n * | Key | Type | Holds |\n * | --- | --- | --- |\n * | `{p}delivery:{id}` | hash | `eventId type occurredAt endpoint attempts lease`, then `userId userType` — plus `provider` on a social event — for a user event, or `data` (JSON) for a generic one; and `status error` once an attempt failed |\n * | `{p}due:{endpoint}` | sorted set | one member per delivery id, scored by when it is due — or, while claimed, when its lease ends |\n * | `{p}endpoints` | set | the endpoint ids with deliveries waiting: what an orphan claim walks |\n *\n * A key's name depends on the arguments — a delivery's key on its endpoint —\n * so the keys are not declared up front. That is why Redis Cluster is not\n * supported: one script may touch keys of more than one slot.\n *\n * Dates travel as milliseconds since the epoch, passed in by the caller —\n * never Redis's `TIME` — and `null` as `''`. An `error` of `''` is read as\n * `null` too: a failure is named by its class, never the empty string.\n */\n\n/**\n * Claims up to `remaining` deliveries of one endpoint due at `now`, earliest\n * first, into `out`, and answers how many it may still claim. A member\n * whose hash is gone — a key deleted by hand — is dropped on the way, and\n * the walk goes on past it.\n *\n * `seen` holds the ids this call already claimed: a lease that ends at `now`\n * or earlier leaves a delivery in range, and the walk past a stale member\n * reads the range again — it must not answer that delivery twice.\n */\nconst CLAIM_FROM = `\nlocal function claimFrom(p, endpoint, now, leaseUntil, lease, remaining, out, seen)\n\tlocal due = p .. 'due:' .. endpoint\n\tlocal here = 0\n\tlocal stale\n\trepeat\n\t\tstale = false\n\t\tlocal ids = redis.call('ZRANGEBYSCORE', due, '-inf', now, 'LIMIT', 0, remaining + here)\n\t\tfor _, id in ipairs(ids) do\n\t\t\tlocal key = p .. 'delivery:' .. id\n\t\t\tif seen[id] or remaining <= 0 then\n\t\t\t\t-- claimed by this call already, or the limit is reached\n\t\t\telseif redis.call('EXISTS', key) == 0 then\n\t\t\t\tredis.call('ZREM', due, id)\n\t\t\t\tstale = true\n\t\t\telse\n\t\t\t\tseen[id] = true\n\t\t\t\there = here + 1\n\t\t\t\tredis.call('HINCRBY', key, 'attempts', 1)\n\t\t\t\tredis.call('HSET', key, 'lease', lease)\n\t\t\t\tredis.call('ZADD', due, leaseUntil, id)\n\t\t\t\tlocal fields = redis.call('HGETALL', key)\n\t\t\t\ttable.insert(fields, 1, id)\n\t\t\t\tout[#out + 1] = fields\n\t\t\t\tremaining = remaining - 1\n\t\t\tend\n\t\tend\n\tuntil remaining <= 0 or not stale\n\tif redis.call('ZCARD', due) == 0 then\n\t\tredis.call('SREM', p .. 'endpoints', endpoint)\n\tend\n\treturn remaining\nend\n`;\n\n/**\n * The delivery `id`'s key, when `lease` is the one it holds; `nil`\n * otherwise. A released lease is `''`, which no claim ever hands out.\n */\nconst LEASED = `\nlocal function leased(p, id, lease)\n\tlocal key = p .. 'delivery:' .. id\n\tif lease == '' or redis.call('HGET', key, 'lease') ~= lease then return nil end\n\treturn key\nend\n`;\n\n/**\n * `ARGV`: prefix, eventId, type, occurredAt, userId, userType, provider\n * (`''` when the event has none), data (the JSON of a generic event's\n * `data`, `''` for a user event), dueAt, then the endpoints. Answers how many deliveries were new. A delivery already\n * held — waiting, claimed or waiting for a retry — is kept as it is.\n *\n * **All or none**: Redis does not roll a script back when a command in it\n * fails — a key of the wrong type, a permission refused — so the inserts run\n * under `pcall`, each write recording how to undo it, and a failure undoes\n * every write before it, in reverse, before the script fails.\n */\nexport const INSERT_DELIVERIES = `\nlocal p, eventId, eventType, dueAt = ARGV[1], ARGV[2], ARGV[3], ARGV[9]\nlocal undo = {}\nlocal function write(rollback, ...)\n\tlocal reply = redis.call(...)\n\tundo[#undo + 1] = rollback\n\treturn reply\nend\nlocal inserted = 0\nlocal ok, failure = pcall(function()\n\tlocal endpoints = p .. 'endpoints'\n\tfor i = 10, #ARGV do\n\t\tlocal endpoint = ARGV[i]\n\t\tlocal id = eventId .. ':' .. eventType .. ':' .. endpoint\n\t\tlocal key = p .. 'delivery:' .. id\n\t\tif redis.call('EXISTS', key) == 0 then\n\t\t\tlocal due = p .. 'due:' .. endpoint\n\t\t\twrite({ 'ZREM', due, id }, 'ZADD', due, dueAt, id)\n\t\t\tif redis.call('SISMEMBER', endpoints, endpoint) == 0 then\n\t\t\t\twrite({ 'SREM', endpoints, endpoint }, 'SADD', endpoints, endpoint)\n\t\t\tend\n\t\t\tlocal fields = {\n\t\t\t\t'eventId', eventId, 'type', eventType, 'occurredAt', ARGV[4],\n\t\t\t\t'endpoint', endpoint, 'attempts', '0', 'lease', '' }\n\t\t\tif ARGV[8] ~= '' then\n\t\t\t\tfields[#fields + 1] = 'data'\n\t\t\t\tfields[#fields + 1] = ARGV[8]\n\t\t\telse\n\t\t\t\tfields[#fields + 1] = 'userId'\n\t\t\t\tfields[#fields + 1] = ARGV[5]\n\t\t\t\tfields[#fields + 1] = 'userType'\n\t\t\t\tfields[#fields + 1] = ARGV[6]\n\t\t\tend\n\t\t\twrite({ 'DEL', key }, 'HSET', key, unpack(fields))\n\t\t\tif ARGV[7] ~= '' then redis.call('HSET', key, 'provider', ARGV[7]) end\n\t\t\tinserted = inserted + 1\n\t\tend\n\tend\nend)\nif ok then return inserted end\nfor i = #undo, 1, -1 do redis.pcall(unpack(undo[i])) end\nif type(failure) == 'table' and failure.err then\n\treturn redis.error_reply(failure.err)\nend\nreturn redis.error_reply(tostring(failure))\n`;\n\n/**\n * `ARGV`: prefix, now, leaseUntil, limit, lease, then the endpoints in the\n * order to walk them. Answers `[[id, field, value, …], …]`.\n */\nexport const CLAIM_DELIVERIES = `${CLAIM_FROM}\nlocal p, now, leaseUntil, lease = ARGV[1], ARGV[2], ARGV[3], ARGV[5]\nlocal remaining = tonumber(ARGV[4])\nlocal out, seen, walked = {}, {}, {}\nfor i = 6, #ARGV do\n\tif remaining <= 0 then break end\n\tlocal endpoint = ARGV[i]\n\t-- An endpoint named twice is walked once, as the memory queue does.\n\tif not walked[endpoint] then\n\t\twalked[endpoint] = true\n\t\tremaining = claimFrom(p, endpoint, now, leaseUntil, lease, remaining, out, seen)\n\tend\nend\nreturn out\n`;\n\n/**\n * `ARGV`: prefix, dueBefore, leaseUntil, limit, lease, then the endpoints\n * known. The same claim, over every endpoint with deliveries that is not\n * known, in the order of their ids.\n */\nexport const CLAIM_ORPHANED_DELIVERIES = `${CLAIM_FROM}\nlocal p, dueBefore, leaseUntil, lease = ARGV[1], ARGV[2], ARGV[3], ARGV[5]\nlocal remaining = tonumber(ARGV[4])\nlocal known = {}\nfor i = 6, #ARGV do known[ARGV[i]] = true end\nlocal orphaned = {}\nfor _, endpoint in ipairs(redis.call('SMEMBERS', p .. 'endpoints')) do\n\tif not known[endpoint] then orphaned[#orphaned + 1] = endpoint end\nend\ntable.sort(orphaned)\nlocal out, seen = {}, {}\nfor _, endpoint in ipairs(orphaned) do\n\tif remaining <= 0 then break end\n\tremaining = claimFrom(p, endpoint, dueBefore, leaseUntil, lease, remaining, out, seen)\nend\nreturn out\n`;\n\n/** `ARGV`: prefix, id, lease, until. `1` when extended, `0` when not held. */\nexport const EXTEND_LEASE = `${LEASED}\nlocal p, id = ARGV[1], ARGV[2]\nlocal key = leased(p, id, ARGV[3])\nif not key then return 0 end\nlocal endpoint = redis.call('HGET', key, 'endpoint')\nredis.call('ZADD', p .. 'due:' .. endpoint, ARGV[4], id)\nreturn 1\n`;\n\n/**\n * `ARGV`: prefix, id, lease, dueAt, status, error — `''` for `null`. `1`\n * when scheduled and the lease released, `0` when not held.\n */\nexport const SCHEDULE_RETRY = `${LEASED}\nlocal p, id = ARGV[1], ARGV[2]\nlocal key = leased(p, id, ARGV[3])\nif not key then return 0 end\n-- The endpoint is read before the first write. A refusal of the ZADD after\n-- the HSET leaves the lease released and the failure stored, with the\n-- delivery due when the old lease ends: retried then, earlier than scheduled.\nlocal endpoint = redis.call('HGET', key, 'endpoint')\nredis.call('HSET', key, 'status', ARGV[5], 'error', ARGV[6], 'lease', '')\nredis.call('ZADD', p .. 'due:' .. endpoint, ARGV[4], id)\nreturn 1\n`;\n\n/**\n * `ARGV`: prefix, id, lease. `1` when removed, `0` when not held or gone.\n * The endpoint leaves the set of endpoints with its last delivery.\n */\nexport const DELETE_DELIVERY = `${LEASED}\nlocal p, id = ARGV[1], ARGV[2]\nlocal key = leased(p, id, ARGV[3])\nif not key then return 0 end\nlocal endpoint = redis.call('HGET', key, 'endpoint')\nlocal due = p .. 'due:' .. endpoint\n-- The hash first: a member left without it is dropped by the next claim,\n-- where a hash left without its member would wait for ever.\nredis.call('DEL', key)\nredis.call('ZREM', due, id)\nif redis.call('ZCARD', due) == 0 then\n\tredis.call('SREM', p .. 'endpoints', endpoint)\nend\nreturn 1\n`;\n"
|
|
8
|
+
],
|
|
9
|
+
"mappings": ";AAAA;AACA,yBAAS;;;ACDT;AAAA;AAAA;AAwBO,SAAS,KAAK,CAAC,IAAkB;AAAA,EACvC,OAAO,OAAO,GAAG,QAAQ,CAAC;AAAA;AAQpB,SAAS,OAAO,CAAC,IAAU,WAAmB,MAAsB;AAAA,EAC1E,IAAI,EAAE,cAAc,SAAS,OAAO,MAAM,GAAG,QAAQ,CAAC,GAAG;AAAA,IACxD,MAAM,IAAI,UAAU,gBAAgB,cAAc,sBAAsB;AAAA,EACzE;AAAA,EACA,OAAO,MAAM,EAAE;AAAA;AAIT,SAAS,OAAO,CAAC,OAAe,WAA2B;AAAA,EACjE,IAAI,CAAC,OAAO,cAAc,KAAK,KAAK,QAAQ,GAAG;AAAA,IAC9C,MAAM,IAAI,UACT,gBAAgB,6DACjB;AAAA,EACD;AAAA,EACA,OAAO,OAAO,KAAK;AAAA;AAIb,SAAS,QAAQ,CAAC,QAAuB,WAA2B;AAAA,EAC1E,IAAI,WAAW;AAAA,IAAM,OAAO;AAAA,EAC5B,IAAI,CAAC,OAAO,cAAc,MAAM,KAAK,SAAS,GAAG;AAAA,IAChD,MAAM,IAAI,UACT,gBAAgB,oDACjB;AAAA,EACD;AAAA,EACA,OAAO,OAAO,MAAM;AAAA;AAId,SAAS,YAAY,CAC3B,OACA,WACmB;AAAA,EACnB,IAAI,CAAC,MAAM,QAAQ,KAAK;AAAA,IAAG,MAAM,WAAW,WAAW,QAAQ;AAAA,EAC/D,OAAO,MAAM,IAAI,CAAC,QAAQ,WAAW,KAAK,SAAS,CAAC;AAAA;AAI9C,SAAS,MAAM,CAAC,OAAgB,WAA4B;AAAA,EAClE,IAAI,UAAU,KAAK,UAAU;AAAA,IAAG,MAAM,WAAW,WAAW,QAAQ;AAAA,EACpE,OAAO,UAAU;AAAA;AAGX,SAAS,OAAO,CAAC,OAAgB,WAA2B;AAAA,EAClE,IAAI,CAAC,OAAO,cAAc,KAAK,KAAM,QAAmB,GAAG;AAAA,IAC1D,MAAM,WAAW,WAAW,SAAS;AAAA,EACtC;AAAA,EACA,OAAO;AAAA;AAGR,SAAS,UAAU,CAAC,OAAgB,WAAmC;AAAA,EACtE,IAAI,CAAC,MAAM,QAAQ,KAAK,KAAK,OAAO,MAAM,OAAO,UAAU;AAAA,IAC1D,MAAM,WAAW,WAAW,YAAY;AAAA,EACzC;AAAA,EACA,OAAO,OAAO,QAAQ;AAAA,EACtB,MAAM,OAAO,SAAS,MAAM,SAAS;AAAA,EACrC,MAAM,QAAQ,KAAK,KAAK,OAAO;AAAA,EAC/B,IAAI,UAAU;AAAA,IAAI,MAAM,WAAW,WAAW,oBAAoB;AAAA,EAClE,MAAM,QAAQ,KAAK,IAAI,MAAM,IACzB;AAAA,IACD,IAAI,KAAK,KAAK,SAAS;AAAA,IACvB,MAAM,KAAK,KAAK,MAAM;AAAA,IACtB,YAAY,KAAK,KAAK,YAAY;AAAA,IAClC,MAAM,KAAK,KAAK,MAAM;AAAA,EACvB,IACE;AAAA,IACD,IAAI,KAAK,KAAK,SAAS;AAAA,IACvB,MAAM,KAAK,KAAK,MAAM;AAAA,IACtB,YAAY,KAAK,KAAK,YAAY;AAAA,IAClC,QAAQ,KAAK,KAAK,QAAQ;AAAA,IAC1B,UAAU,KAAK,KAAK,UAAU;AAAA,OAC3B,KAAK,iBAAiB;AAAA,EAC1B;AAAA,EACF,OAAO;AAAA,IACN;AAAA,IACA;AAAA,IACA,UAAU,KAAK,KAAK,UAAU;AAAA,IAC9B,UAAU,KAAK,MAAM,UAAU;AAAA,IAC/B,QAAQ,KAAK,QAAQ;AAAA,IACrB;AAAA,EACD;AAAA;AAOD,IAAM,QAAQ;AAAA,EACb,gBAAgB;AAAA,EAChB,sBAAsB;AAAA,EACtB,sBAAsB;AAAA,EACtB,wBAAwB;AAAA,EACxB,qBAAqB;AAAA,EACrB,4BAA4B;AAAA,EAC5B,6BAA6B;AAAA,EAC7B,iCAAiC;AAAA,EACjC,yBAAyB;AAAA,EACzB,0BAA0B;AAAA,EAC1B,sBAAsB;AAAA,EACtB,sBAAsB;AAAA,EACtB,qBAAqB;AAAA,EACrB,uBAAuB;AAAA,EACvB,qBAAqB;AAAA,EACrB,uBAAuB;AAAA,EACvB,8BAA8B;AAAA,EAC9B,sBAAsB;AAAA,EACtB,sBAAsB;AAAA,EACtB,gBAAgB;AACjB;AAGA,SAAS,QAAQ,CAAC,OAA2B,WAAmB;AAAA,EAC/D,MAAM,SAAS,IAAI;AAAA,EACnB,SAAS,IAAI,EAAG,IAAI,IAAI,MAAM,QAAQ,KAAK,GAAG;AAAA,IAC7C,OAAO,IAAI,OAAO,MAAM,EAAE,GAAG,OAAO,MAAM,IAAI,EAAE,CAAC;AAAA,EAClD;AAAA,EACA,MAAM,OAAO,CAAC,SAAyB;AAAA,IACtC,MAAM,QAAQ,OAAO,IAAI,IAAI;AAAA,IAC7B,IAAI,UAAU,WAAW;AAAA,MACxB,MAAM,WAAW,WAAW,iBAAiB,QAAQ;AAAA,IACtD;AAAA,IACA,OAAO;AAAA;AAAA,EAGR,MAAM,SAAS,CAAC,MAAc,SAAyB;AAAA,IACtD,MAAM,QAAQ,KAAK,IAAI;AAAA,IACvB,MAAM,OAAO,iBAAiB,KAAK,KAAK,IAAI,OAAO,KAAK,IAAI,OAAO;AAAA,IACnE,IAAI,CAAC,OAAO,cAAc,IAAI;AAAA,MAAG,MAAM,WAAW,WAAW,IAAI;AAAA,IACjE,OAAO;AAAA;AAAA,EAER,OAAO;AAAA,IACN;AAAA,IACA,KAAK,CAAC,SAAiB,OAAO,IAAI,IAAI;AAAA,IAEtC,MAAM,CAAC,SAAyB;AAAA,MAC/B,MAAM,QAAQ,KAAK,IAAI;AAAA,MACvB,IAAI,UAAU;AAAA,QAAI,MAAM,WAAW,WAAW,eAAe;AAAA,MAC7D,OAAO;AAAA;AAAA,IAGR,MAAM,CAAC,SAA0C;AAAA,MAChD,IAAI;AAAA,MACJ,IAAI;AAAA,QACH,QAAQ,KAAK,MAAM,KAAK,IAAI,CAAC;AAAA,QAC5B,MAAM;AAAA,QACP,QAAQ;AAAA;AAAA,MAET,IAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,MAAM,QAAQ,KAAK,GAAG;AAAA,QACxE,MAAM,WAAW,WAAW,sBAAsB,QAAQ;AAAA,MAC3D;AAAA,MACA,OAAO;AAAA;AAAA,IAER,OAAO,CAAC,SAAiB,OAAO,MAAM,gBAAgB,QAAQ;AAAA,IAE9D,MAAM,CAAC,SAAuB;AAAA,MAC7B,MAAM,QAAQ,KAAK,IAAI;AAAA,MACvB,MAAM,KAAK,IAAI,KAAK,OAAO,KAAK,CAAC;AAAA,MACjC,IAAI,UAAU,MAAM,OAAO,MAAM,GAAG,QAAQ,CAAC,GAAG;AAAA,QAC/C,MAAM,WAAW,WAAW,eAAe,QAAQ;AAAA,MACpD;AAAA,MACA,OAAO;AAAA;AAAA,IAER,MAAM,CAAC,SAAgC;AAAA,MACtC,MAAM,QAAQ,KAAK,IAAI;AAAA,MACvB,IAAI,CAAC,OAAO,OAAO,OAAO,KAAK;AAAA,QAC9B,MAAM,WAAW,WAAW,mBAAmB;AAAA,MAChD,OAAO;AAAA;AAAA,IAGR,kBAAkB,MAA6B;AAAA,MAC9C,MAAM,WAAW,OAAO,IAAI,UAAU;AAAA,MACtC,OAAO,aAAa,aAAa,aAAa,KAAK,CAAC,IAAI,EAAE,SAAS;AAAA;AAAA,IAGpE,SAAS,MAAsB;AAAA,MAC9B,IAAI,CAAC,OAAO,IAAI,QAAQ,KAAK,CAAC,OAAO,IAAI,OAAO;AAAA,QAAG,OAAO;AAAA,MAC1D,MAAM,SAAS,KAAK,QAAQ;AAAA,MAC5B,MAAM,QAAQ,KAAK,OAAO;AAAA,MAC1B,OAAO;AAAA,QACN,QAAQ,WAAW,KAAK,OAAO,OAAO,UAAU,sBAAsB;AAAA,QACtE,OAAO,UAAU,KAAK,OAAO;AAAA,MAC9B;AAAA;AAAA,EAEF;AAAA;AAGD,SAAS,UAAU,CAAC,WAAmB,MAA4B;AAAA,EAClE,OAAO,IAAI,aACV,gBAAgB,kCAAkC,4DAClD,EAAE,UAAU,CACb;AAAA;;;AC9LD,IAAM,aAAa;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAuCnB,IAAM,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmBR,IAAM,oBAAoB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmD1B,IAAM,mBAAmB,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAqB5B,IAAM,4BAA4B,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAmBrC,IAAM,eAAe,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAaxB,IAAM,iBAAiB,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiB1B,IAAM,kBAAkB,GAAG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;AF9J3B,SAAS,uBAAuB,CACtC,OACA,UAAoC,CAAC,GACtB;AAAA,EACf,MAAM,MAAM,OACX,YAAY,MAAM,MAAM,GACxB,QAAQ,UAAU,iBACnB;AAAA,EAEA,OAAO;AAAA,IACN,kBAAkB,CAAC,OAAO,WAAW,UACpC,IACC,oBACA,mBACA,aAAa,OAAO,WAAW,KAAK,GACpC,OACD;AAAA,IAGD,iBAAiB,CAAC,WAAW,KAAK,YAAY,UAC7C,IACC,mBACA,kBACA,CAAC,cAAc;AAAA,MACd,QAAQ,KAAK,WAAW,KAAK;AAAA,MAC7B,GAAG,YAAY,YAAY,OAAO,SAAS;AAAA,MAC3C,GAAG;AAAA,IACJ,GACA,YACD;AAAA,IAED,yBAAyB,CAAC,OAAO,WAAW,YAAY,UACvD,IACC,2BACA,2BACA,CAAC,cAAc;AAAA,MACd,QAAQ,WAAW,WAAW,WAAW;AAAA,MACzC,GAAG,YAAY,YAAY,OAAO,SAAS;AAAA,MAC3C,GAAG;AAAA,IACJ,GACA,YACD;AAAA,IAED,aAAa,CAAC,IAAI,OAAO,UACxB,IACC,eACA,cACA,CAAC,cAAc,CAAC,IAAI,OAAO,QAAQ,OAAO,WAAW,OAAO,CAAC,GAC7D,MACD;AAAA,IAED,eAAe,CAAC,IAAI,OAAO,OAAO,WACjC,IACC,iBACA,gBACA,YAAY,IAAI,OAAO,OAAO,MAAM,GACpC,MACD;AAAA,IAED,gBAAgB,CAAC,IAAI,UACpB,IAAI,kBAAkB,iBAAiB,MAAM,CAAC,IAAI,KAAK,GAAG,MAAM;AAAA,EAClE;AAAA;AAgBD,SAAS,MAAM,CAAC,UAAoB,QAAqB;AAAA,EACxD,OAAO,OAAO,WAAW,QAAQ,QAAQ,WAAW;AAAA,IACnD,MAAM,OAAO,OAAO,SAAS;AAAA,IAC7B,OAAO,OACN,MAAM,SAAS,WAAW,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC,GACnD,SACD;AAAA;AAAA;AASF,SAAS,YAAY,CACpB,OACA,WACA,OAC2C;AAAA,EAC3C,OAAO,CAAC,cAAc;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,MAAM,YAAY,WAAW,kBAAkB;AAAA,IACvD,UAAU,QAAQ,KAAK,MAAM;AAAA,IAC7B,UAAU,QAAQ,KAAK,MAAM;AAAA,IAC7B,UAAU,QAAQ,KAAM,MAAM,YAAY;AAAA,IAC1C,UAAU,QAAQ,OAAO,MAAM,MAAM,SAAS,IAAI;AAAA,IAClD,QAAQ,OAAO,WAAW,OAAO;AAAA,IACjC,GAAG;AAAA,EACJ;AAAA;AAOD,SAAS,MAAM,CAAC,MAAe,WAA2B;AAAA,EACzD,IAAI;AAAA,EACJ,IAAI;AAAA,IACH,OAAO,KAAK,UAAU,IAAI;AAAA,IACzB,MAAM;AAAA,IACP,OAAO;AAAA;AAAA,EAER,IAAI,SAAS,aAAa,CAAC,KAAK,WAAW,GAAG,GAAG;AAAA,IAChD,MAAM,IAAI,UACT,gBAAgB,wCACjB;AAAA,EACD;AAAA,EACA,OAAO;AAAA;AAOR,SAAS,WAAW,CACnB,YACA,OACA,WACoB;AAAA,EACpB,OAAO;AAAA,IACN,QAAQ,YAAY,WAAW,YAAY;AAAA,IAC3C,QAAQ,OAAO,SAAS;AAAA,IACxB,WAAW;AAAA,EACZ;AAAA;AAID,SAAS,WAAW,CACnB,IACA,OACA,OACA,QAC2C;AAAA,EAC3C,OAAO,CAAC,cAAc;AAAA,IACrB;AAAA,IACA;AAAA,IACA,QAAQ,OAAO,WAAW,OAAO;AAAA,IACjC,SAAS,OAAO,QAAQ,SAAS;AAAA,IACjC,OAAO,SAAS;AAAA,EACjB;AAAA;AAeD,SAAS,WAAW,CAAC,QAA+B;AAAA,EACnD,MAAM,OAAO,IAAI;AAAA,EACjB,MAAM,QAAQ,CAAC,WAAmB;AAAA,IACjC,IAAI,MAAM,KAAK,IAAI,MAAM;AAAA,IACzB,IAAI,QAAQ,WAAW;AAAA,MACtB,MAAM,IAAI,IAAI,aAAa,MAAM,EAAE,OAAO,MAAM,EAAE,OAAO,KAAK;AAAA,MAC9D,KAAK,IAAI,QAAQ,GAAG;AAAA,IACrB;AAAA,IACA,OAAO;AAAA;AAAA,EAGR,OAAO,OAAO,WAAW,QAAQ,SAAS;AAAA,IACzC,IAAI;AAAA,MACH,IAAI;AAAA,QACH,OAAO,MAAM,OAAO,KAAK,WAAW,CAAC,MAAM,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC;AAAA,QAChE,OAAO,OAAO;AAAA,QACf,IAAI,CAAC,WAAW,KAAK;AAAA,UAAG,MAAM;AAAA,QAC9B,OAAO,MAAM,OAAO,KAAK,QAAQ,CAAC,QAAQ,KAAK,GAAG,IAAI,CAAC;AAAA;AAAA,MAEvD,OAAO,OAAO;AAAA,MACf,MAAM,IAAI,cACT,gBAAgB,yCAChB,EAAE,WAAW,OAAO,MAAM,CAC3B;AAAA;AAAA;AAAA;AAKH,SAAS,UAAU,CAAC,OAAyB;AAAA,EAC5C,OAAO,iBAAiB,SAAS,MAAM,QAAQ,WAAW,UAAU;AAAA;",
|
|
10
|
+
"debugId": "A2BC8E2E4D2633C864756E2164756E21",
|
|
11
|
+
"names": []
|
|
12
|
+
}
|
package/dist/queue.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { WebhookQueue } from '@softistx/janus-webhooks';
|
|
2
|
+
import type { RedisConnection } from '@softistx/redis';
|
|
3
|
+
/** What the queue takes besides the connection. */
|
|
4
|
+
export interface RedisWebhookQueueOptions {
|
|
5
|
+
/**
|
|
6
|
+
* What every key starts with — `janus:webhooks:` unless you say
|
|
7
|
+
* otherwise. Every process that shares deliveries passes the same one;
|
|
8
|
+
* two applications sharing one Redis each take their own.
|
|
9
|
+
*/
|
|
10
|
+
readonly prefix?: string;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* The durable queue of `@softistx/janus-webhooks`, over one Redis, on
|
|
14
|
+
* `@softistx/redis`'s connection: deliveries wait in Redis, so a retry waiting
|
|
15
|
+
* when a process stops is sent by the next one.
|
|
16
|
+
*
|
|
17
|
+
* ```ts
|
|
18
|
+
* const redis = await connectRedis(process.env.REDIS_URL);
|
|
19
|
+
* const listener = webhooks({ endpoints, queue: createRedisWebhookQueue(redis) });
|
|
20
|
+
* ```
|
|
21
|
+
*
|
|
22
|
+
* **Every method is one Lua script**, atomic as the port asks: two claims
|
|
23
|
+
* running at once — in one process or in twenty — never answer one
|
|
24
|
+
* delivery. Every time is the caller's, never Redis's clock.
|
|
25
|
+
*
|
|
26
|
+
* It creates nothing and needs no schema: the keys are made by the first
|
|
27
|
+
* insert, and a delivery's keys go with its delete.
|
|
28
|
+
*/
|
|
29
|
+
export declare function createRedisWebhookQueue(redis: RedisConnection, options?: RedisWebhookQueueOptions): WebhookQueue;
|
|
30
|
+
//# sourceMappingURL=queue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAGX,YAAY,EACZ,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAoBvD,mDAAmD;AACnD,MAAM,WAAW,wBAAwB;IACxC;;;;OAIG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,uBAAuB,CACtC,KAAK,EAAE,eAAe,EACtB,OAAO,GAAE,wBAA6B,GACpC,YAAY,CA2Dd"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { QueuedDelivery, WebhookQueue } from '@softistx/janus-webhooks';
|
|
2
|
+
/** A method of the queue port: what a failure names. */
|
|
3
|
+
export type Method = keyof WebhookQueue & string;
|
|
4
|
+
/**
|
|
5
|
+
* What a script answers, read back into the port's deliveries. **A reply
|
|
6
|
+
* this adapter did not write is a failure, never an absence**: a key of the
|
|
7
|
+
* prefix changed by hand, or written by another version, rejects with
|
|
8
|
+
* `StoreFailure` — naming the method it failed, as every other failure does.
|
|
9
|
+
*
|
|
10
|
+
* Dates travel as milliseconds since the epoch, and `null` as `''`.
|
|
11
|
+
*/
|
|
12
|
+
export declare function stamp(at: Date): string;
|
|
13
|
+
/**
|
|
14
|
+
* A time the caller passed, as milliseconds — or a `TypeError`, before any
|
|
15
|
+
* I/O, for an Invalid Date: `stamp` would write `'NaN'`, which no claim could
|
|
16
|
+
* read back. A date before 1970 is a negative number, and reads back.
|
|
17
|
+
*/
|
|
18
|
+
export declare function stampOf(at: Date, operation: Method, name: string): string;
|
|
19
|
+
/** A claim's `limit`, or a `TypeError`: a whole number of deliveries, 0 or more. */
|
|
20
|
+
export declare function limitOf(limit: number, operation: Method): string;
|
|
21
|
+
/** A failure's `status`, or a `TypeError`: `null`, or a whole number. */
|
|
22
|
+
export declare function statusOf(status: number | null, operation: Method): string;
|
|
23
|
+
/** `[[id, field, value, …], …]` as the deliveries a claim took. */
|
|
24
|
+
export declare function toDeliveries(reply: unknown, operation: Method): QueuedDelivery[];
|
|
25
|
+
/** `1` as `true`, `0` as `false`: what a write guarded by a lease answers. */
|
|
26
|
+
export declare function toHeld(reply: unknown, operation: Method): boolean;
|
|
27
|
+
export declare function toCount(reply: unknown, operation: Method): number;
|
|
28
|
+
//# sourceMappingURL=replies.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"replies.d.ts","sourceRoot":"","sources":["../src/replies.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAEX,cAAc,EAEd,YAAY,EACZ,MAAM,0BAA0B,CAAC;AAElC,wDAAwD;AACxD,MAAM,MAAM,MAAM,GAAG,MAAM,YAAY,GAAG,MAAM,CAAC;AAEjD;;;;;;;GAOG;AAEH,wBAAgB,KAAK,CAAC,EAAE,EAAE,IAAI,GAAG,MAAM,CAEtC;AAED;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAKzE;AAED,oFAAoF;AACpF,wBAAgB,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAOhE;AAED,yEAAyE;AACzE,wBAAgB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAQzE;AAED,mEAAmE;AACnE,wBAAgB,YAAY,CAC3B,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,GACf,cAAc,EAAE,CAGlB;AAED,8EAA8E;AAC9E,wBAAgB,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAGjE;AAED,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAKjE"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Lua scripts, **one per port method**. A script is atomic: Redis runs
|
|
3
|
+
* nothing else while it does, which is what the port asks of every method —
|
|
4
|
+
* and what makes a claim one step, so two claims running at once never
|
|
5
|
+
* answer one delivery.
|
|
6
|
+
*
|
|
7
|
+
* The layout, every key built inside the script from `ARGV[1]`, the prefix:
|
|
8
|
+
*
|
|
9
|
+
* | Key | Type | Holds |
|
|
10
|
+
* | --- | --- | --- |
|
|
11
|
+
* | `{p}delivery:{id}` | hash | `eventId type occurredAt endpoint attempts lease`, then `userId userType` — plus `provider` on a social event — for a user event, or `data` (JSON) for a generic one; and `status error` once an attempt failed |
|
|
12
|
+
* | `{p}due:{endpoint}` | sorted set | one member per delivery id, scored by when it is due — or, while claimed, when its lease ends |
|
|
13
|
+
* | `{p}endpoints` | set | the endpoint ids with deliveries waiting: what an orphan claim walks |
|
|
14
|
+
*
|
|
15
|
+
* A key's name depends on the arguments — a delivery's key on its endpoint —
|
|
16
|
+
* so the keys are not declared up front. That is why Redis Cluster is not
|
|
17
|
+
* supported: one script may touch keys of more than one slot.
|
|
18
|
+
*
|
|
19
|
+
* Dates travel as milliseconds since the epoch, passed in by the caller —
|
|
20
|
+
* never Redis's `TIME` — and `null` as `''`. An `error` of `''` is read as
|
|
21
|
+
* `null` too: a failure is named by its class, never the empty string.
|
|
22
|
+
*/
|
|
23
|
+
/**
|
|
24
|
+
* `ARGV`: prefix, eventId, type, occurredAt, userId, userType, provider
|
|
25
|
+
* (`''` when the event has none), data (the JSON of a generic event's
|
|
26
|
+
* `data`, `''` for a user event), dueAt, then the endpoints. Answers how many deliveries were new. A delivery already
|
|
27
|
+
* held — waiting, claimed or waiting for a retry — is kept as it is.
|
|
28
|
+
*
|
|
29
|
+
* **All or none**: Redis does not roll a script back when a command in it
|
|
30
|
+
* fails — a key of the wrong type, a permission refused — so the inserts run
|
|
31
|
+
* under `pcall`, each write recording how to undo it, and a failure undoes
|
|
32
|
+
* every write before it, in reverse, before the script fails.
|
|
33
|
+
*/
|
|
34
|
+
export declare const INSERT_DELIVERIES = "\nlocal p, eventId, eventType, dueAt = ARGV[1], ARGV[2], ARGV[3], ARGV[9]\nlocal undo = {}\nlocal function write(rollback, ...)\n\tlocal reply = redis.call(...)\n\tundo[#undo + 1] = rollback\n\treturn reply\nend\nlocal inserted = 0\nlocal ok, failure = pcall(function()\n\tlocal endpoints = p .. 'endpoints'\n\tfor i = 10, #ARGV do\n\t\tlocal endpoint = ARGV[i]\n\t\tlocal id = eventId .. ':' .. eventType .. ':' .. endpoint\n\t\tlocal key = p .. 'delivery:' .. id\n\t\tif redis.call('EXISTS', key) == 0 then\n\t\t\tlocal due = p .. 'due:' .. endpoint\n\t\t\twrite({ 'ZREM', due, id }, 'ZADD', due, dueAt, id)\n\t\t\tif redis.call('SISMEMBER', endpoints, endpoint) == 0 then\n\t\t\t\twrite({ 'SREM', endpoints, endpoint }, 'SADD', endpoints, endpoint)\n\t\t\tend\n\t\t\tlocal fields = {\n\t\t\t\t'eventId', eventId, 'type', eventType, 'occurredAt', ARGV[4],\n\t\t\t\t'endpoint', endpoint, 'attempts', '0', 'lease', '' }\n\t\t\tif ARGV[8] ~= '' then\n\t\t\t\tfields[#fields + 1] = 'data'\n\t\t\t\tfields[#fields + 1] = ARGV[8]\n\t\t\telse\n\t\t\t\tfields[#fields + 1] = 'userId'\n\t\t\t\tfields[#fields + 1] = ARGV[5]\n\t\t\t\tfields[#fields + 1] = 'userType'\n\t\t\t\tfields[#fields + 1] = ARGV[6]\n\t\t\tend\n\t\t\twrite({ 'DEL', key }, 'HSET', key, unpack(fields))\n\t\t\tif ARGV[7] ~= '' then redis.call('HSET', key, 'provider', ARGV[7]) end\n\t\t\tinserted = inserted + 1\n\t\tend\n\tend\nend)\nif ok then return inserted end\nfor i = #undo, 1, -1 do redis.pcall(unpack(undo[i])) end\nif type(failure) == 'table' and failure.err then\n\treturn redis.error_reply(failure.err)\nend\nreturn redis.error_reply(tostring(failure))\n";
|
|
35
|
+
/**
|
|
36
|
+
* `ARGV`: prefix, now, leaseUntil, limit, lease, then the endpoints in the
|
|
37
|
+
* order to walk them. Answers `[[id, field, value, …], …]`.
|
|
38
|
+
*/
|
|
39
|
+
export declare const CLAIM_DELIVERIES = "\nlocal function claimFrom(p, endpoint, now, leaseUntil, lease, remaining, out, seen)\n\tlocal due = p .. 'due:' .. endpoint\n\tlocal here = 0\n\tlocal stale\n\trepeat\n\t\tstale = false\n\t\tlocal ids = redis.call('ZRANGEBYSCORE', due, '-inf', now, 'LIMIT', 0, remaining + here)\n\t\tfor _, id in ipairs(ids) do\n\t\t\tlocal key = p .. 'delivery:' .. id\n\t\t\tif seen[id] or remaining <= 0 then\n\t\t\t\t-- claimed by this call already, or the limit is reached\n\t\t\telseif redis.call('EXISTS', key) == 0 then\n\t\t\t\tredis.call('ZREM', due, id)\n\t\t\t\tstale = true\n\t\t\telse\n\t\t\t\tseen[id] = true\n\t\t\t\there = here + 1\n\t\t\t\tredis.call('HINCRBY', key, 'attempts', 1)\n\t\t\t\tredis.call('HSET', key, 'lease', lease)\n\t\t\t\tredis.call('ZADD', due, leaseUntil, id)\n\t\t\t\tlocal fields = redis.call('HGETALL', key)\n\t\t\t\ttable.insert(fields, 1, id)\n\t\t\t\tout[#out + 1] = fields\n\t\t\t\tremaining = remaining - 1\n\t\t\tend\n\t\tend\n\tuntil remaining <= 0 or not stale\n\tif redis.call('ZCARD', due) == 0 then\n\t\tredis.call('SREM', p .. 'endpoints', endpoint)\n\tend\n\treturn remaining\nend\n\nlocal p, now, leaseUntil, lease = ARGV[1], ARGV[2], ARGV[3], ARGV[5]\nlocal remaining = tonumber(ARGV[4])\nlocal out, seen, walked = {}, {}, {}\nfor i = 6, #ARGV do\n\tif remaining <= 0 then break end\n\tlocal endpoint = ARGV[i]\n\t-- An endpoint named twice is walked once, as the memory queue does.\n\tif not walked[endpoint] then\n\t\twalked[endpoint] = true\n\t\tremaining = claimFrom(p, endpoint, now, leaseUntil, lease, remaining, out, seen)\n\tend\nend\nreturn out\n";
|
|
40
|
+
/**
|
|
41
|
+
* `ARGV`: prefix, dueBefore, leaseUntil, limit, lease, then the endpoints
|
|
42
|
+
* known. The same claim, over every endpoint with deliveries that is not
|
|
43
|
+
* known, in the order of their ids.
|
|
44
|
+
*/
|
|
45
|
+
export declare const CLAIM_ORPHANED_DELIVERIES = "\nlocal function claimFrom(p, endpoint, now, leaseUntil, lease, remaining, out, seen)\n\tlocal due = p .. 'due:' .. endpoint\n\tlocal here = 0\n\tlocal stale\n\trepeat\n\t\tstale = false\n\t\tlocal ids = redis.call('ZRANGEBYSCORE', due, '-inf', now, 'LIMIT', 0, remaining + here)\n\t\tfor _, id in ipairs(ids) do\n\t\t\tlocal key = p .. 'delivery:' .. id\n\t\t\tif seen[id] or remaining <= 0 then\n\t\t\t\t-- claimed by this call already, or the limit is reached\n\t\t\telseif redis.call('EXISTS', key) == 0 then\n\t\t\t\tredis.call('ZREM', due, id)\n\t\t\t\tstale = true\n\t\t\telse\n\t\t\t\tseen[id] = true\n\t\t\t\there = here + 1\n\t\t\t\tredis.call('HINCRBY', key, 'attempts', 1)\n\t\t\t\tredis.call('HSET', key, 'lease', lease)\n\t\t\t\tredis.call('ZADD', due, leaseUntil, id)\n\t\t\t\tlocal fields = redis.call('HGETALL', key)\n\t\t\t\ttable.insert(fields, 1, id)\n\t\t\t\tout[#out + 1] = fields\n\t\t\t\tremaining = remaining - 1\n\t\t\tend\n\t\tend\n\tuntil remaining <= 0 or not stale\n\tif redis.call('ZCARD', due) == 0 then\n\t\tredis.call('SREM', p .. 'endpoints', endpoint)\n\tend\n\treturn remaining\nend\n\nlocal p, dueBefore, leaseUntil, lease = ARGV[1], ARGV[2], ARGV[3], ARGV[5]\nlocal remaining = tonumber(ARGV[4])\nlocal known = {}\nfor i = 6, #ARGV do known[ARGV[i]] = true end\nlocal orphaned = {}\nfor _, endpoint in ipairs(redis.call('SMEMBERS', p .. 'endpoints')) do\n\tif not known[endpoint] then orphaned[#orphaned + 1] = endpoint end\nend\ntable.sort(orphaned)\nlocal out, seen = {}, {}\nfor _, endpoint in ipairs(orphaned) do\n\tif remaining <= 0 then break end\n\tremaining = claimFrom(p, endpoint, dueBefore, leaseUntil, lease, remaining, out, seen)\nend\nreturn out\n";
|
|
46
|
+
/** `ARGV`: prefix, id, lease, until. `1` when extended, `0` when not held. */
|
|
47
|
+
export declare const EXTEND_LEASE = "\nlocal function leased(p, id, lease)\n\tlocal key = p .. 'delivery:' .. id\n\tif lease == '' or redis.call('HGET', key, 'lease') ~= lease then return nil end\n\treturn key\nend\n\nlocal p, id = ARGV[1], ARGV[2]\nlocal key = leased(p, id, ARGV[3])\nif not key then return 0 end\nlocal endpoint = redis.call('HGET', key, 'endpoint')\nredis.call('ZADD', p .. 'due:' .. endpoint, ARGV[4], id)\nreturn 1\n";
|
|
48
|
+
/**
|
|
49
|
+
* `ARGV`: prefix, id, lease, dueAt, status, error — `''` for `null`. `1`
|
|
50
|
+
* when scheduled and the lease released, `0` when not held.
|
|
51
|
+
*/
|
|
52
|
+
export declare const SCHEDULE_RETRY = "\nlocal function leased(p, id, lease)\n\tlocal key = p .. 'delivery:' .. id\n\tif lease == '' or redis.call('HGET', key, 'lease') ~= lease then return nil end\n\treturn key\nend\n\nlocal p, id = ARGV[1], ARGV[2]\nlocal key = leased(p, id, ARGV[3])\nif not key then return 0 end\n-- The endpoint is read before the first write. A refusal of the ZADD after\n-- the HSET leaves the lease released and the failure stored, with the\n-- delivery due when the old lease ends: retried then, earlier than scheduled.\nlocal endpoint = redis.call('HGET', key, 'endpoint')\nredis.call('HSET', key, 'status', ARGV[5], 'error', ARGV[6], 'lease', '')\nredis.call('ZADD', p .. 'due:' .. endpoint, ARGV[4], id)\nreturn 1\n";
|
|
53
|
+
/**
|
|
54
|
+
* `ARGV`: prefix, id, lease. `1` when removed, `0` when not held or gone.
|
|
55
|
+
* The endpoint leaves the set of endpoints with its last delivery.
|
|
56
|
+
*/
|
|
57
|
+
export declare const DELETE_DELIVERY = "\nlocal function leased(p, id, lease)\n\tlocal key = p .. 'delivery:' .. id\n\tif lease == '' or redis.call('HGET', key, 'lease') ~= lease then return nil end\n\treturn key\nend\n\nlocal p, id = ARGV[1], ARGV[2]\nlocal key = leased(p, id, ARGV[3])\nif not key then return 0 end\nlocal endpoint = redis.call('HGET', key, 'endpoint')\nlocal due = p .. 'due:' .. endpoint\n-- The hash first: a member left without it is dropped by the next claim,\n-- where a hash left without its member would wait for ever.\nredis.call('DEL', key)\nredis.call('ZREM', due, id)\nif redis.call('ZCARD', due) == 0 then\n\tredis.call('SREM', p .. 'endpoints', endpoint)\nend\nreturn 1\n";
|
|
58
|
+
//# sourceMappingURL=scripts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scripts.d.ts","sourceRoot":"","sources":["../src/scripts.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AA2DH;;;;;;;;;;GAUG;AACH,eAAO,MAAM,iBAAiB,mmDA6C7B,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,gBAAgB,ikDAc5B,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,yqDAgBrC,CAAC;AAEF,8EAA8E;AAC9E,eAAO,MAAM,YAAY,sZAOxB,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,cAAc,ssBAW1B,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,eAAe,6pBAc3B,CAAC"}
|
package/docs/README.md
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# @softistx/janus-webhooks-redis — documentation
|
|
2
|
+
|
|
3
|
+
The [README](../README.md) shows the whole wiring in one example; these pages
|
|
4
|
+
give the detail. The words — delivery, claim, lease, endpoint id, orphan —
|
|
5
|
+
are `@softistx/janus-webhooks`'s, defined in
|
|
6
|
+
[its index](https://github.com/softistx/nxgt-janus/blob/develop/packages/janus-webhooks/docs/README.md#words).
|
|
7
|
+
|
|
8
|
+
| Page | Read it when |
|
|
9
|
+
| --- | --- |
|
|
10
|
+
| [Wiring the queue to Redis](guide/wiring.md) | You are passing the queue to `webhooks()`, choosing a prefix and the connection's options, configuring Redis, reading what Redis holds and how each method stays atomic, or handling what a failure throws |
|
|
11
|
+
| [Troubleshooting](troubleshooting.md) | You have an error message and want its cause and its fix |
|
|
12
|
+
| [Roadmap](roadmap.md) | You want to know what is coming, what shipped, and what is deliberately not planned |
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# Wiring the queue to Redis
|
|
2
|
+
|
|
3
|
+
This page covers:
|
|
4
|
+
- passing the Redis queue to `webhooks()`, in every process;
|
|
5
|
+
- the prefix, and the connection options that matter;
|
|
6
|
+
- what Redis must be configured with;
|
|
7
|
+
- what Redis holds, and how each method stays atomic;
|
|
8
|
+
- what a failure looks like.
|
|
9
|
+
|
|
10
|
+
How a queue is used — inserts, claims, leases, orphans — is
|
|
11
|
+
`@softistx/janus-webhooks`'s, in its
|
|
12
|
+
[queues guide](https://github.com/softistx/nxgt-janus/blob/develop/packages/janus-webhooks/docs/guide/queues.md).
|
|
13
|
+
|
|
14
|
+
## In every process
|
|
15
|
+
|
|
16
|
+
`createRedisWebhookQueue(redis)` answers a `WebhookQueue`. Pass it to
|
|
17
|
+
`webhooks({ queue })` in **every process** of the application, over the same
|
|
18
|
+
Redis and with the same prefix: they then share the deliveries, and what one
|
|
19
|
+
leaves waiting — a retry, a request a crash cut short — another sends.
|
|
20
|
+
|
|
21
|
+
```ts
|
|
22
|
+
import { webhooks } from '@softistx/janus-webhooks';
|
|
23
|
+
import { createRedisWebhookQueue } from '@softistx/janus-webhooks-redis';
|
|
24
|
+
import { connectRedis } from '@softistx/redis';
|
|
25
|
+
|
|
26
|
+
const redis = await connectRedis(process.env.REDIS_URL ?? 'redis://localhost:6379', {
|
|
27
|
+
enableOfflineQueue: false,
|
|
28
|
+
});
|
|
29
|
+
const crmSecret = process.env.CRM_WEBHOOK_SECRET;
|
|
30
|
+
const searchSecret = process.env.SEARCH_WEBHOOK_SECRET;
|
|
31
|
+
if (!crmSecret || !searchSecret) throw new Error('a webhook secret is not set');
|
|
32
|
+
|
|
33
|
+
export const listener = webhooks({
|
|
34
|
+
endpoints: [
|
|
35
|
+
{ id: 'crm', url: 'https://crm.example.com/hooks/janus', secrets: [crmSecret] },
|
|
36
|
+
{ id: 'search', url: 'https://search.example.com/hooks', secrets: [searchSecret], types: ['user.deleted'] },
|
|
37
|
+
],
|
|
38
|
+
queue: createRedisWebhookQueue(redis),
|
|
39
|
+
});
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
**Give each endpoint an `id`** from the start. The queue holds a delivery
|
|
43
|
+
under its endpoint's id, never its URL; without an `id`, the id is a hash of
|
|
44
|
+
the URL, and changing the URL leaves the deliveries waiting to an endpoint no
|
|
45
|
+
process knows — given up as `endpointRemoved` after `orphanGrace`.
|
|
46
|
+
|
|
47
|
+
## `createRedisWebhookQueue(redis, options?)`
|
|
48
|
+
|
|
49
|
+
```ts
|
|
50
|
+
function createRedisWebhookQueue(
|
|
51
|
+
redis: RedisConnection, // what connectRedis returns
|
|
52
|
+
options?: { prefix?: string }, // 'janus:webhooks:' by default
|
|
53
|
+
): WebhookQueue;
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
It connects to nothing and creates nothing: the keys are made by the first
|
|
57
|
+
insert, and Redis needs no schema. `redis` is `@softistx/redis`'s connection,
|
|
58
|
+
whose client is Bun's `RedisClient`, so this runs on Bun.
|
|
59
|
+
|
|
60
|
+
`prefix` starts every key. Every process of one application passes the same;
|
|
61
|
+
two applications sharing one Redis — or the partitions of one, a region, a
|
|
62
|
+
customer's own deployment — each take their own, as do the tests of one
|
|
63
|
+
application:
|
|
64
|
+
|
|
65
|
+
```ts
|
|
66
|
+
createRedisWebhookQueue(redis, { prefix: 'clinic:webhooks:' });
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
It may share a Redis, and a connection, with `@softistx/janus-redis`: that one's
|
|
70
|
+
keys start with `janus:` and hold sessions and tokens; a session key never
|
|
71
|
+
starts with `janus:webhooks:`.
|
|
72
|
+
|
|
73
|
+
## The connection's options
|
|
74
|
+
|
|
75
|
+
`connectRedis(uri, options)` takes Bun's `RedisOptions`. One of them decides
|
|
76
|
+
how an outage looks to your users — because the listener **awaits the
|
|
77
|
+
insert**, so every flow that sends an event waits for Redis:
|
|
78
|
+
|
|
79
|
+
| `enableOfflineQueue` | While Redis is unreachable | What the flow sees |
|
|
80
|
+
| --- | --- | --- |
|
|
81
|
+
| `true`, Bun's default | Every command waits while the client reconnects, then fails | `signUp` waits about **31 s**, then succeeds; the event is reported as `JANUS_EVENT_FAILED` |
|
|
82
|
+
| `false` | Every command fails at once | `signUp` succeeds at once; the event is reported as `JANUS_EVENT_FAILED` |
|
|
83
|
+
|
|
84
|
+
Either way the flow is not failed: `janus` reports a listener that rejects
|
|
85
|
+
and answers. The claims fail too, and `@softistx/janus-webhooks` warns once per
|
|
86
|
+
outage with `JANUS_WEBHOOK_QUEUE_FAILED`; what waits in Redis is sent when it
|
|
87
|
+
answers again.
|
|
88
|
+
|
|
89
|
+
`connectRedis` shares one client per URI, and refuses a second connection to
|
|
90
|
+
the same URI with other options. If your application already connects to this
|
|
91
|
+
Redis, pass the same options everywhere, or give the queue a URI of its own,
|
|
92
|
+
for example another database number: `redis://localhost:6379/2`.
|
|
93
|
+
|
|
94
|
+
## What Redis must be configured with
|
|
95
|
+
|
|
96
|
+
| Setting | Why |
|
|
97
|
+
| --- | --- |
|
|
98
|
+
| `maxmemory-policy noeviction` | An evicted key is a delivery lost without a report. With `noeviction`, a full Redis refuses the insert with `OOM`, which is reported |
|
|
99
|
+
| AOF (`appendonly yes`, `appendfsync everysec`) or RDB snapshots | Without persistence, a restart of Redis loses every delivery waiting in it. With AOF `everysec`, a crash loses at most a second of inserts |
|
|
100
|
+
| A single Redis, or a primary with replicas — **not Cluster** | A script touches a delivery's key and its endpoint's, which may hash to different slots |
|
|
101
|
+
| Redis 7.0 or later, or Valkey | Tested on Redis 7.0, 7.4 and Valkey 7.2; the floor `@softistx/janus-redis` needs, so one Redis serves both |
|
|
102
|
+
|
|
103
|
+
A Redis user restricted by ACL needs `+@scripting` and the commands the
|
|
104
|
+
scripts run — `HSET`, `HGET`, `HGETALL`, `HINCRBY`, `EXISTS`, `DEL`, `ZADD`,
|
|
105
|
+
`ZREM`, `ZRANGEBYSCORE`, `ZCARD`, `SADD`, `SREM`, `SISMEMBER`, `SMEMBERS` — on
|
|
106
|
+
the keys `~janus:webhooks:*`, or `~<your prefix>*`.
|
|
107
|
+
|
|
108
|
+
## What Redis holds, and for how long
|
|
109
|
+
|
|
110
|
+
| Key | Holds |
|
|
111
|
+
| --- | --- |
|
|
112
|
+
| `<prefix>delivery:<event id>:<type>:<endpoint id>` | the delivery, as a hash: the event's fields — five for a user event (and `provider`, on a social event), `userId` and `userType` among them, and for a generic event its `data`, as JSON, in their place — the endpoint id, `attempts`, the claim's `lease`, and the last failure's `status` and `error` |
|
|
113
|
+
| `<prefix>due:<endpoint id>` | the endpoint's deliveries, as a sorted set scored by when each is due, or when its lease ends while claimed |
|
|
114
|
+
| `<prefix>endpoints` | the endpoint ids with deliveries waiting, as a set: what the orphan claim walks |
|
|
115
|
+
|
|
116
|
+
**No URL and no secret is stored**: the queue holds the endpoint's id, and
|
|
117
|
+
the URL and secrets are read from the running configuration at each attempt.
|
|
118
|
+
No e-mail address and no session id either: `@softistx/janus-webhooks` drops the
|
|
119
|
+
`formerEmail` of a `user.emailChanged`, and the `sessionId` of a
|
|
120
|
+
`user.newDeviceSignedIn`, before they reach the queue.
|
|
121
|
+
Dates are milliseconds since the epoch, passed in by `@softistx/janus-webhooks` —
|
|
122
|
+
never Redis's clock.
|
|
123
|
+
|
|
124
|
+
Nothing expires by itself: a delivery stays until it is delivered or given
|
|
125
|
+
up, and its keys go with its last delete — an empty queue holds nothing.
|
|
126
|
+
What waits is bounded by what your endpoints refuse — at most eight attempts
|
|
127
|
+
over about 28 hours with the default schedule — and by `orphanGrace` for an
|
|
128
|
+
endpoint removed.
|
|
129
|
+
|
|
130
|
+
A claim reads each endpoint's sorted set from its earliest delivery, so it
|
|
131
|
+
costs the deliveries it claims, not the deliveries waiting.
|
|
132
|
+
|
|
133
|
+
## One Lua script per method
|
|
134
|
+
|
|
135
|
+
Redis runs nothing else while a script runs, so of twenty claims at once, no
|
|
136
|
+
two answer one delivery. Redis does not roll a script back when a command in
|
|
137
|
+
it fails half-way — a permission refused, a key of the wrong type — so **the
|
|
138
|
+
insert undoes its own writes** before it fails, and is all or none. The
|
|
139
|
+
others are not undone; what each leaves is in
|
|
140
|
+
[troubleshooting](../troubleshooting.md#store_failed-webhookqueuemethod-the-queue-could-not-answer).
|
|
141
|
+
|
|
142
|
+
The scripts are sent by SHA (`EVALSHA`), and in full only when Redis has
|
|
143
|
+
forgotten them after a restart, a failover or a `SCRIPT FLUSH`.
|
|
144
|
+
|
|
145
|
+
## What a failure looks like
|
|
146
|
+
|
|
147
|
+
The adapter defines **no error class**. A rejection is `@softistx/janus`'s
|
|
148
|
+
`StoreFailure`, with `operation` the method that failed — or, for an
|
|
149
|
+
argument the queue could not read back, a `TypeError` before anything is
|
|
150
|
+
sent:
|
|
151
|
+
|
|
152
|
+
| What happens | What you get |
|
|
153
|
+
| --- | --- |
|
|
154
|
+
| Redis is unreachable, the connection closed, a timeout | `webhookQueue.<method>: the queue could not answer`; `cause` is Bun's `RedisError` |
|
|
155
|
+
| Redis refuses a command: `NOPERM`, `OOM`, `READONLY` on a replica, `WRONGTYPE` | the same; `cause` carries Redis's reply. An insert refused half-way is undone first; the other scripts are not — see [troubleshooting](../troubleshooting.md#store_failed-webhookqueuemethod-the-queue-could-not-answer) |
|
|
156
|
+
| A key under the prefix this adapter did not write | `webhookQueue.<method>: a reply that is not … — a key under the prefix this adapter did not write`, with no `cause`: never read as nothing due |
|
|
157
|
+
| An Invalid Date, a `limit` that is not a whole number 0 or more, a `failed.status` that is not `null` or a whole number | a `TypeError` — `webhookQueue.<method>: <name> is a valid Date`, `… limit is a whole number of deliveries, 0 or more`, `… failed.status is null or a whole number` — before any I/O; `webhooks()` never passes one |
|
|
158
|
+
|
|
159
|
+
Nothing answers `[]` or `false` for an error: a claim that answered "nothing
|
|
160
|
+
due" during an outage would hold every delivery back without a word.
|
|
161
|
+
`@softistx/janus-webhooks` turns these into its warnings; the adapter writes
|
|
162
|
+
none of its own.
|
package/docs/roadmap.md
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# Roadmap
|
|
2
|
+
|
|
3
|
+
Where `@softistx/janus-webhooks-redis` is heading. It shows a direction, not a
|
|
4
|
+
commitment: there are no dates here, and the only number is the version
|
|
5
|
+
something shipped in.
|
|
6
|
+
|
|
7
|
+
## Now
|
|
8
|
+
|
|
9
|
+
Nothing yet.
|
|
10
|
+
|
|
11
|
+
## Next
|
|
12
|
+
|
|
13
|
+
Nothing yet.
|
|
14
|
+
|
|
15
|
+
## Later
|
|
16
|
+
|
|
17
|
+
Nothing yet.
|
|
18
|
+
|
|
19
|
+
## Not planned
|
|
20
|
+
|
|
21
|
+
- **Redis Cluster.** A script touches a delivery's key and its endpoint's,
|
|
22
|
+
which may live in different slots.
|
|
23
|
+
- **Expiring deliveries in Redis.** A delivery stays until it is delivered or
|
|
24
|
+
given up: an expiry would drop it without a report, which is what the
|
|
25
|
+
queue exists to prevent.
|
|
26
|
+
- **Clients other than Bun's.** `@softistx/redis`'s connection is Bun's
|
|
27
|
+
`RedisClient`, so this runs on Bun.
|
|
28
|
+
- **Error classes of its own.** The adapter throws `@softistx/janus`'s
|
|
29
|
+
`StoreFailure`, which is why `@softistx/janus` is a required peer and never a
|
|
30
|
+
dependency.
|
|
31
|
+
|
|
32
|
+
## Shipped
|
|
33
|
+
|
|
34
|
+
- **Generic events persisted** (0.12.0) — a delivery of an event of any type, with its `data` as JSON, is written and read back like a user event's, for `@softistx/janus-webhooks` 0.13.0's `webhooks<WebhookEvent>()`; `prefix` lets several applications share one Redis. Upgrade every process sharing a queue before the first one is sent. [#893](https://github.com/softistx/nxgt-janus/issues/893)
|
|
35
|
+
- **API key events** (0.11.0) — reads back `user.apiKeyCreated` and `user.apiKeyRevoked` deliveries; upgrade every process before the first one is sent.
|
|
36
|
+
|
|
37
|
+
Newest first; from the first release on, the package's CHANGELOG holds every one.
|
|
38
|
+
|
|
39
|
+
- **Moved to `@softistx` on GitHub Packages** (0.7.0) — formerly
|
|
40
|
+
the `@nxgt` scope on npm; same names under the new scope, published privately.
|
|
41
|
+
|
|
42
|
+
- **A sign-in from a new device, v0.5.0** — a delivery of
|
|
43
|
+
`user.newDeviceSignedIn`, from `@softistx/janus` 0.17, is written and read
|
|
44
|
+
back like the other ten. No session id is stored: `@softistx/janus-webhooks`
|
|
45
|
+
drops the event's `sessionId` before the queue. A 0.4.x process sharing
|
|
46
|
+
the queue cannot read one back: upgrade every process, with each
|
|
47
|
+
endpoint's `types` limited meanwhile.
|
|
48
|
+
|
|
49
|
+
- **The change events, v0.4.0** — a delivery of `user.passwordChanged` or
|
|
50
|
+
`user.emailChanged`, from `@softistx/janus` 0.14, is written and read back like
|
|
51
|
+
the other eight. No address is stored: `@softistx/janus-webhooks` drops an
|
|
52
|
+
e-mail change's `formerEmail` before the queue.
|
|
53
|
+
|
|
54
|
+
- **The recovery codes' events, v0.3.0** — a delivery of
|
|
55
|
+
`user.recoveryCodesRegenerated` or `user.recoveryCodeUsed`, from
|
|
56
|
+
`@softistx/janus` 0.10, is written and read back like the other six.
|
|
57
|
+
- **The second factor's events, v0.2.0** — a delivery of
|
|
58
|
+
`user.secondFactorEnabled` or `user.secondFactorDisabled`, from
|
|
59
|
+
`@softistx/janus` 0.9, is written and read back like the other four.
|
|
60
|
+
- **The first release, v0.1.0.** The Redis queue for `@softistx/janus-webhooks`: every
|
|
61
|
+
method one Lua script, claims atomic across processes, an insert all or
|
|
62
|
+
none, and the `@softistx/janus-webhooks/conformance` suite passed against Redis
|
|
63
|
+
7.4 with the server's own faults.
|