@brickkit/redis-world 0.15.0 → 0.16.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/internal.d.ts +1 -1
- package/dist/internal.js +1 -1
- package/dist/keys-CqKp_Amk.js.map +1 -1
- package/dist/{world-783VJfKL.js → world-Cac56z4M.js} +19 -2
- package/dist/world-Cac56z4M.js.map +1 -0
- package/dist/{world-BDW_evdb.d.ts → world-DCbuc04m.d.ts} +11 -2
- package/dist/world-DCbuc04m.d.ts.map +1 -0
- package/package.json +10 -10
- package/dist/world-783VJfKL.js.map +0 -1
- package/dist/world-BDW_evdb.d.ts.map +0 -1
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { _ as RunRetention, a as QueueStats, f as RedisWorldConfig, g as KeepRuns, t as RedisBackedWorld } from "./world-
|
|
2
|
-
import
|
|
1
|
+
import { _ as RunRetention, a as QueueStats, f as RedisWorldConfig, g as KeepRuns, t as RedisBackedWorld } from "./world-DCbuc04m.js";
|
|
2
|
+
import "ioredis";
|
|
3
3
|
/** `config` is laid over `configFromEnv()`: an option left out comes from the
|
|
4
4
|
* `WORKFLOW_REDIS_*` variables, then the defaults. */
|
|
5
5
|
export declare function createWorld(config?: Partial<RedisWorldConfig>): RedisBackedWorld;
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { n as checkInteger, r as configFromEnv, t as keys } from "./keys-CqKp_Amk.js";
|
|
2
|
-
import { i as debug, n as createWorldQueue, r as retryDelayMs, t as createRedisBackedWorld } from "./world-
|
|
2
|
+
import { i as debug, n as createWorldQueue, r as retryDelayMs, t as createRedisBackedWorld } from "./world-Cac56z4M.js";
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
4
|
import { setTimeout } from "node:timers/promises";
|
|
5
5
|
import { MessageId, ValidQueueName } from "@workflow/world";
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","names":["delay"],"sources":["../src/queue.ts","../src/index.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { setTimeout as delay } from \"node:timers/promises\";\nimport { MessageId, ValidQueueName } from \"@workflow/world\";\nimport type { Redis } from \"ioredis\";\nimport {\n checkInteger,\n type RedisWorldConfig,\n type ResolvedConfig,\n} from \"./config.js\";\nimport { keys } from \"./keys.js\";\nimport { debug } from \"./util.js\";\nimport {\n createWorldQueue,\n retryDelayMs,\n type Deliver,\n type Delivery,\n type QueueBackend,\n type WorldQueue,\n type WorldQueueOptions,\n} from \"./world-queue.js\";\n\nexport type { QueueStats } from \"./world-queue.js\";\n\ninterface Envelope extends Delivery {\n dedupKey: string;\n headers: Record<string, string>;\n}\n\n// Derived from the upstream reliable Redis list queue. Every transition is a\n// single script: a process crash cannot leave a message between structures.\nconst ENQUEUE = `\n if ARGV[1] ~= '' then\n local existing = redis.call('GET', KEYS[1])\n if existing then return existing end\n redis.call('SET', KEYS[1], ARGV[2])\n if tonumber(ARGV[3]) > 0 then redis.call('PEXPIRE', KEYS[1], ARGV[3]) end\n end\n if tonumber(ARGV[5]) > 0 then\n redis.call('ZADD', KEYS[3], ARGV[5], ARGV[4])\n else\n redis.call('LPUSH', KEYS[2], ARGV[4])\n end\n return ARGV[2]\n`;\nconst RESERVE = `\n if redis.call('EXISTS', KEYS[3]) == 0 then return nil end\n local raw = redis.call('LINDEX', KEYS[1], -1)\n if not raw then return nil end\n local job = cjson.decode(raw)\n job.attempt = job.attempt + 1\n job.deliveryCount = job.deliveryCount + 1\n local claim = cjson.encode(job)\n redis.call('RPOP', KEYS[1])\n redis.call('LPUSH', KEYS[2], claim)\n return claim\n`;\nconst ACK = `\n if redis.call('EXISTS', KEYS[3]) == 0 then return 0 end\n if redis.call('LREM', KEYS[1], 1, ARGV[1]) == 0 then return 0 end\n if ARGV[2] ~= '' and redis.call('GET', KEYS[2]) == ARGV[3] then\n redis.call('DEL', KEYS[2])\n end\n return 1\n`;\nconst DEFER = `\n if redis.call('EXISTS', KEYS[3]) == 0 then return 0 end\n if redis.call('LREM', KEYS[1], 1, ARGV[1]) == 0 then return 0 end\n redis.call('ZADD', KEYS[2], ARGV[3], ARGV[2])\n return 1\n`;\nconst PROMOTE = `\n local due = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[1], 'LIMIT', 0, 100)\n for _, raw in ipairs(due) do\n redis.call('LPUSH', KEYS[2], raw)\n redis.call('ZREM', KEYS[1], raw)\n end\n return #due\n`;\nconst RECLAIM = `\n if redis.call('EXISTS', KEYS[3]) == 1 then return 0 end\n local moved = 0\n for i = 1, 100 do\n local raw = redis.call('RPOPLPUSH', KEYS[1], KEYS[2])\n if not raw then break end\n moved = moved + 1\n end\n if redis.call('LLEN', KEYS[1]) == 0 then redis.call('SREM', KEYS[4], ARGV[1]) end\n return moved\n`;\nconst RENEW = `\n if redis.call('EXISTS', KEYS[1]) == 0 then return 0 end\n redis.call('PEXPIRE', KEYS[1], ARGV[1])\n return 1\n`;\nconst REGISTER = `\n redis.call('SET', KEYS[1], '1', 'PX', ARGV[2])\n redis.call('SADD', KEYS[2], ARGV[1])\n`;\n// ARGV[1] .. owner is the owner's processing list: `keys().queue.processing(\"\")`.\nconst STATS = `\n local active = 0\n for _, owner in ipairs(redis.call('SMEMBERS', KEYS[3])) do\n active = active + redis.call('LLEN', ARGV[1] .. owner)\n end\n return {redis.call('LLEN', KEYS[1]), active, redis.call('ZCARD', KEYS[2])}\n`;\n\nfunction parseEnvelope(raw: string): Envelope {\n const value: unknown = JSON.parse(raw);\n if (\n value === null ||\n typeof value !== \"object\" ||\n !(\"messageId\" in value) ||\n !(\"queueName\" in value) ||\n !(\"message\" in value) ||\n typeof value.message !== \"string\" ||\n !(\"attempt\" in value) ||\n typeof value.attempt !== \"number\" ||\n !(\"deliveryCount\" in value) ||\n typeof value.deliveryCount !== \"number\" ||\n !(\"createdAt\" in value) ||\n typeof value.createdAt !== \"number\" ||\n !(\"dedupKey\" in value) ||\n typeof value.dedupKey !== \"string\" ||\n !(\"headers\" in value) ||\n value.headers === null ||\n typeof value.headers !== \"object\"\n ) {\n throw new TypeError(\"Invalid Redis queue envelope\");\n }\n const headers: Record<string, string> = {};\n for (const [key, header] of Object.entries(value.headers)) {\n if (typeof header !== \"string\") throw new TypeError(\"Invalid queue header\");\n headers[key] = header;\n }\n return {\n messageId: MessageId.parse(value.messageId),\n queueName: ValidQueueName.parse(value.queueName),\n message: value.message,\n attempt: value.attempt,\n deliveryCount: value.deliveryCount,\n createdAt: value.createdAt,\n dedupKey: value.dedupKey,\n headers,\n ...(\"route\" in value &&\n typeof value.route === \"string\" && { route: value.route }),\n };\n}\n\nexport function createQueue(\n redis: Redis,\n config: ResolvedConfig<RedisWorldConfig>,\n options: WorldQueueOptions = {},\n): WorldQueue {\n const queueKeys = keys(config.keyPrefix).queue;\n const { ready, delayed, workers: owners, processing, heartbeat } = queueKeys;\n const heartbeatTtlMs = config.heartbeatTtlMs ?? 90_000;\n checkInteger(\"heartbeatTtlMs\", heartbeatTtlMs, 300);\n const { queueConcurrency: concurrency, maxAttempts, maxBackoffMs } = config;\n let closed = false;\n let startPromise: Promise<void> | undefined;\n let loop: Promise<void> | undefined;\n let owner: string | undefined;\n const inflight = new Set<Promise<void>>();\n const lostOwners = new Set<string>();\n const abort = new AbortController();\n const sleep = () =>\n delay(50, undefined, { signal: abort.signal }).catch(() => {});\n\n async function enqueue(\n message: Parameters<QueueBackend[\"enqueue\"]>[0],\n opts: Parameters<QueueBackend[\"enqueue\"]>[1],\n ): Promise<MessageId> {\n const dedupKey = opts.idempotencyKey\n ? queueKeys.idempotent(opts.idempotencyKey)\n : \"\";\n const envelope: Envelope = {\n messageId: message.messageId,\n queueName: message.queueName,\n message: message.message,\n attempt: 0,\n deliveryCount: 0,\n createdAt: Date.now(),\n dedupKey,\n headers: message.headers ?? {},\n ...(message.route && { route: message.route }),\n };\n const delayMs = Math.max(0, (opts.delaySeconds ?? 0) * 1000);\n if (!Number.isFinite(delayMs))\n throw new RangeError(\"delaySeconds must be finite\");\n const result = await redis.eval(\n ENQUEUE,\n 3,\n dedupKey || ready,\n ready,\n delayed,\n dedupKey,\n message.messageId,\n config.idempotencyTtlMs ?? 0,\n JSON.stringify(envelope),\n delayMs > 0 ? Date.now() + delayMs : 0,\n );\n return MessageId.parse(result);\n }\n\n async function settle(\n raw: string,\n claimOwner: string,\n deliver: Deliver,\n ): Promise<void> {\n try {\n const job = parseEnvelope(raw);\n let seconds: number | undefined;\n let failed = false;\n try {\n // Reservation persists the attempt before delivery, so a crash\n // redelivery reports one more.\n seconds = await deliver(job);\n } catch (error) {\n failed = true;\n debug(\n `Delivery ${job.messageId} attempt ${job.attempt} failed:`,\n error,\n );\n seconds =\n (job.attempt >= maxAttempts\n ? maxBackoffMs\n : retryDelayMs(job.attempt, config)) / 1000;\n if (job.attempt === maxAttempts)\n console.error(\n `Queue message ${job.messageId} exhausted maxAttempts; retaining it for capped retries`,\n );\n }\n if (seconds === undefined) {\n await redis.eval(\n ACK,\n 3,\n processing(claimOwner),\n job.dedupKey || ready,\n heartbeat(claimOwner),\n raw,\n job.dedupKey,\n job.messageId,\n );\n } else {\n // Only successful reschedules preserve the attempt. A crash always\n // leaves the reserved attempt in Redis, so reclaim counts as failure.\n const next = failed ? job : { ...job, attempt: job.attempt - 1 };\n await redis.eval(\n DEFER,\n 3,\n processing(claimOwner),\n delayed,\n heartbeat(claimOwner),\n raw,\n JSON.stringify(next),\n Date.now() + seconds * 1000,\n );\n }\n } catch (error) {\n // Keep the claim for reclaim on connection loss or malformed data.\n lostOwners.add(claimOwner);\n console.error(\"Queue settlement error:\", error);\n }\n }\n\n async function register(): Promise<string> {\n const next = randomUUID();\n await redis.eval(\n REGISTER,\n 2,\n heartbeat(next),\n owners,\n next,\n heartbeatTtlMs,\n );\n return next;\n }\n\n async function run(deliver: Deliver): Promise<void> {\n let renewedAt = Date.now();\n let maintainedAt = 0;\n let failing = false;\n while (!closed) {\n try {\n for (const lost of lostOwners) {\n await redis.del(heartbeat(lost));\n lostOwners.delete(lost);\n if (owner === lost) owner = undefined;\n }\n if (!owner || Date.now() - renewedAt >= heartbeatTtlMs / 3) {\n // Never resurrect an expired lease: its processing list may already\n // belong to the reclaimer. A fresh UUID fences all old settlements.\n if (\n !owner ||\n (await redis.eval(RENEW, 1, heartbeat(owner), heartbeatTtlMs)) !== 1\n )\n owner = await register();\n renewedAt = Date.now();\n }\n if (Date.now() - maintainedAt >= 100) {\n await redis.eval(PROMOTE, 2, delayed, ready, Date.now());\n for (const stale of await redis.smembers(owners)) {\n if (stale !== owner)\n await redis.eval(\n RECLAIM,\n 4,\n processing(stale),\n ready,\n heartbeat(stale),\n owners,\n stale,\n );\n }\n maintainedAt = Date.now();\n }\n for (\n let reserved = 0;\n reserved < concurrency &&\n !closed &&\n owner &&\n inflight.size < concurrency;\n reserved++\n ) {\n const claimOwner = owner;\n const raw = await redis.eval(\n RESERVE,\n 3,\n ready,\n processing(claimOwner),\n heartbeat(claimOwner),\n );\n if (typeof raw !== \"string\") break;\n const task = settle(raw, claimOwner, deliver);\n inflight.add(task);\n void task.finally(() => inflight.delete(task));\n }\n failing = false;\n } catch (error) {\n // Warn once per outage: a queue that lost Redis otherwise looks idle.\n if (!closed && !failing) console.warn(\"Queue polling failed:\", error);\n else if (!closed) debug(\"Queue polling error:\", error);\n failing = true;\n }\n await sleep();\n }\n }\n\n const backend: QueueBackend = {\n enqueue,\n async getQueueStats() {\n const counts = await redis.eval(\n STATS,\n 3,\n ready,\n delayed,\n owners,\n processing(\"\"),\n );\n if (\n !Array.isArray(counts) ||\n counts.length !== 3 ||\n counts.some((c) => typeof c !== \"number\")\n )\n throw new TypeError(\"Invalid queue statistics\");\n return {\n waiting: Number(counts[0]),\n active: Number(counts[1]),\n delayed: Number(counts[2]),\n completed: 0,\n failed: 0,\n };\n },\n start(deliver) {\n if (closed) return Promise.reject(new Error(\"Queue is closed\"));\n startPromise ??= (async () => {\n owner = await register();\n loop = run(deliver);\n })().catch((error) => {\n startPromise = undefined;\n throw error;\n });\n return startPromise;\n },\n async close() {\n if (closed) return;\n closed = true;\n abort.abort();\n await startPromise;\n await loop;\n // While handlers drain, keep the lease alive. close() must not permit a\n // live long-running handler to be reclaimed merely because shutdown began.\n const timer = setInterval(\n () => {\n if (owner)\n void redis\n .eval(RENEW, 1, heartbeat(owner), heartbeatTtlMs)\n .catch((error) => debug(\"Queue close heartbeat:\", error));\n },\n Math.floor(heartbeatTtlMs / 3),\n );\n try {\n await Promise.all(inflight);\n } finally {\n clearInterval(timer);\n }\n if (owner) {\n await redis.del(heartbeat(owner));\n await redis.eval(\n RECLAIM,\n 4,\n processing(owner),\n ready,\n heartbeat(owner),\n owners,\n owner,\n );\n }\n },\n };\n return createWorldQueue(backend, config, { ...options, debug });\n}\n","import { configFromEnv, type RedisWorldConfig } from \"./config.js\";\nimport { createQueue } from \"./queue.js\";\nimport { debug } from \"./util.js\";\nimport { createRedisBackedWorld, type RedisBackedWorld } from \"./world.js\";\n\n/** `config` is laid over `configFromEnv()`: an option left out comes from the\n * `WORKFLOW_REDIS_*` variables, then the defaults. */\nexport function createWorld(\n config: Partial<RedisWorldConfig> = {},\n): RedisBackedWorld {\n return createRedisBackedWorld(config, {\n env: configFromEnv(),\n createQueue,\n label: \"world-redis\",\n debug,\n });\n}\n\nexport type { RedisWorldConfig } from \"./config.js\";\nexport type { QueueStats } from \"./queue.js\";\nexport type { KeepRuns, RunRetention } from \"./reaper.js\";\n"],"mappings":";;;;;;AA8BA,MAAM,UAAU;;;;;;;;;;;;;;AAchB,MAAM,UAAU;;;;;;;;;;;;AAYhB,MAAM,MAAM;;;;;;;;AAQZ,MAAM,QAAQ;;;;;;AAMd,MAAM,UAAU;;;;;;;;AAQhB,MAAM,UAAU;;;;;;;;;;;AAWhB,MAAM,QAAQ;;;;;AAKd,MAAM,WAAW;;;;AAKjB,MAAM,QAAQ;;;;;;;AAQd,SAAS,cAAc,KAAuB;CAC5C,MAAM,QAAiB,KAAK,MAAM,GAAG;CACrC,IACE,UAAU,QACV,OAAO,UAAU,YACjB,EAAE,eAAe,UACjB,EAAE,eAAe,UACjB,EAAE,aAAa,UACf,OAAO,MAAM,YAAY,YACzB,EAAE,aAAa,UACf,OAAO,MAAM,YAAY,YACzB,EAAE,mBAAmB,UACrB,OAAO,MAAM,kBAAkB,YAC/B,EAAE,eAAe,UACjB,OAAO,MAAM,cAAc,YAC3B,EAAE,cAAc,UAChB,OAAO,MAAM,aAAa,YAC1B,EAAE,aAAa,UACf,MAAM,YAAY,QAClB,OAAO,MAAM,YAAY,UAEzB,MAAM,IAAI,UAAU,8BAA8B;CAEpD,MAAM,UAAkC,CAAC;CACzC,KAAK,MAAM,CAAC,KAAK,WAAW,OAAO,QAAQ,MAAM,OAAO,GAAG;EACzD,IAAI,OAAO,WAAW,UAAU,MAAM,IAAI,UAAU,sBAAsB;EAC1E,QAAQ,OAAO;CACjB;CACA,OAAO;EACL,WAAW,UAAU,MAAM,MAAM,SAAS;EAC1C,WAAW,eAAe,MAAM,MAAM,SAAS;EAC/C,SAAS,MAAM;EACf,SAAS,MAAM;EACf,eAAe,MAAM;EACrB,WAAW,MAAM;EACjB,UAAU,MAAM;EAChB;EACA,GAAI,WAAW,SACb,OAAO,MAAM,UAAU,YAAY,EAAE,OAAO,MAAM,MAAM;CAC5D;AACF;AAEA,SAAgB,YACd,OACA,QACA,UAA6B,CAAC,GAClB;CACZ,MAAM,YAAY,KAAK,OAAO,SAAS,CAAC,CAAC;CACzC,MAAM,EAAE,OAAO,SAAS,SAAS,QAAQ,YAAY,cAAc;CACnE,MAAM,iBAAiB,OAAO,kBAAkB;CAChD,aAAa,kBAAkB,gBAAgB,GAAG;CAClD,MAAM,EAAE,kBAAkB,aAAa,aAAa,iBAAiB;CACrE,IAAI,SAAS;CACb,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,MAAM,2BAAW,IAAI,IAAmB;CACxC,MAAM,6BAAa,IAAI,IAAY;CACnC,MAAM,QAAQ,IAAI,gBAAgB;CAClC,MAAM,cACJA,WAAM,IAAI,KAAA,GAAW,EAAE,QAAQ,MAAM,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAE/D,eAAe,QACb,SACA,MACoB;EACpB,MAAM,WAAW,KAAK,iBAClB,UAAU,WAAW,KAAK,cAAc,IACxC;EACJ,MAAM,WAAqB;GACzB,WAAW,QAAQ;GACnB,WAAW,QAAQ;GACnB,SAAS,QAAQ;GACjB,SAAS;GACT,eAAe;GACf,WAAW,KAAK,IAAI;GACpB;GACA,SAAS,QAAQ,WAAW,CAAC;GAC7B,GAAI,QAAQ,SAAS,EAAE,OAAO,QAAQ,MAAM;EAC9C;EACA,MAAM,UAAU,KAAK,IAAI,IAAI,KAAK,gBAAgB,KAAK,GAAI;EAC3D,IAAI,CAAC,OAAO,SAAS,OAAO,GAC1B,MAAM,IAAI,WAAW,6BAA6B;EACpD,MAAM,SAAS,MAAM,MAAM,KACzB,SACA,GACA,YAAY,OACZ,OACA,SACA,UACA,QAAQ,WACR,OAAO,oBAAoB,GAC3B,KAAK,UAAU,QAAQ,GACvB,UAAU,IAAI,KAAK,IAAI,IAAI,UAAU,CACvC;EACA,OAAO,UAAU,MAAM,MAAM;CAC/B;CAEA,eAAe,OACb,KACA,YACA,SACe;EACf,IAAI;GACF,MAAM,MAAM,cAAc,GAAG;GAC7B,IAAI;GACJ,IAAI,SAAS;GACb,IAAI;IAGF,UAAU,MAAM,QAAQ,GAAG;GAC7B,SAAS,OAAO;IACd,SAAS;IACT,MACE,YAAY,IAAI,UAAU,WAAW,IAAI,QAAQ,WACjD,KACF;IACA,WACG,IAAI,WAAW,cACZ,eACA,aAAa,IAAI,SAAS,MAAM,KAAK;IAC3C,IAAI,IAAI,YAAY,aAClB,QAAQ,MACN,iBAAiB,IAAI,UAAU,wDACjC;GACJ;GACA,IAAI,YAAY,KAAA,GACd,MAAM,MAAM,KACV,KACA,GACA,WAAW,UAAU,GACrB,IAAI,YAAY,OAChB,UAAU,UAAU,GACpB,KACA,IAAI,UACJ,IAAI,SACN;QACK;IAGL,MAAM,OAAO,SAAS,MAAM;KAAE,GAAG;KAAK,SAAS,IAAI,UAAU;IAAE;IAC/D,MAAM,MAAM,KACV,OACA,GACA,WAAW,UAAU,GACrB,SACA,UAAU,UAAU,GACpB,KACA,KAAK,UAAU,IAAI,GACnB,KAAK,IAAI,IAAI,UAAU,GACzB;GACF;EACF,SAAS,OAAO;GAEd,WAAW,IAAI,UAAU;GACzB,QAAQ,MAAM,2BAA2B,KAAK;EAChD;CACF;CAEA,eAAe,WAA4B;EACzC,MAAM,OAAO,WAAW;EACxB,MAAM,MAAM,KACV,UACA,GACA,UAAU,IAAI,GACd,QACA,MACA,cACF;EACA,OAAO;CACT;CAEA,eAAe,IAAI,SAAiC;EAClD,IAAI,YAAY,KAAK,IAAI;EACzB,IAAI,eAAe;EACnB,IAAI,UAAU;EACd,OAAO,CAAC,QAAQ;GACd,IAAI;IACF,KAAK,MAAM,QAAQ,YAAY;KAC7B,MAAM,MAAM,IAAI,UAAU,IAAI,CAAC;KAC/B,WAAW,OAAO,IAAI;KACtB,IAAI,UAAU,MAAM,QAAQ,KAAA;IAC9B;IACA,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,aAAa,iBAAiB,GAAG;KAG1D,IACE,CAAC,SACA,MAAM,MAAM,KAAK,OAAO,GAAG,UAAU,KAAK,GAAG,cAAc,MAAO,GAEnE,QAAQ,MAAM,SAAS;KACzB,YAAY,KAAK,IAAI;IACvB;IACA,IAAI,KAAK,IAAI,IAAI,gBAAgB,KAAK;KACpC,MAAM,MAAM,KAAK,SAAS,GAAG,SAAS,OAAO,KAAK,IAAI,CAAC;KACvD,KAAK,MAAM,SAAS,MAAM,MAAM,SAAS,MAAM,GAC7C,IAAI,UAAU,OACZ,MAAM,MAAM,KACV,SACA,GACA,WAAW,KAAK,GAChB,OACA,UAAU,KAAK,GACf,QACA,KACF;KAEJ,eAAe,KAAK,IAAI;IAC1B;IACA,KACE,IAAI,WAAW,GACf,WAAW,eACX,CAAC,UACD,SACA,SAAS,OAAO,aAChB,YACA;KACA,MAAM,aAAa;KACnB,MAAM,MAAM,MAAM,MAAM,KACtB,SACA,GACA,OACA,WAAW,UAAU,GACrB,UAAU,UAAU,CACtB;KACA,IAAI,OAAO,QAAQ,UAAU;KAC7B,MAAM,OAAO,OAAO,KAAK,YAAY,OAAO;KAC5C,SAAS,IAAI,IAAI;KACjB,KAAU,cAAc,SAAS,OAAO,IAAI,CAAC;IAC/C;IACA,UAAU;GACZ,SAAS,OAAO;IAEd,IAAI,CAAC,UAAU,CAAC,SAAS,QAAQ,KAAK,yBAAyB,KAAK;SAC/D,IAAI,CAAC,QAAQ,MAAM,wBAAwB,KAAK;IACrD,UAAU;GACZ;GACA,MAAM,MAAM;EACd;CACF;CA0EA,OAAO,iBAAiB;EAvEtB;EACA,MAAM,gBAAgB;GACpB,MAAM,SAAS,MAAM,MAAM,KACzB,OACA,GACA,OACA,SACA,QACA,WAAW,EAAE,CACf;GACA,IACE,CAAC,MAAM,QAAQ,MAAM,KACrB,OAAO,WAAW,KAClB,OAAO,MAAM,MAAM,OAAO,MAAM,QAAQ,GAExC,MAAM,IAAI,UAAU,0BAA0B;GAChD,OAAO;IACL,SAAS,OAAO,OAAO,EAAE;IACzB,QAAQ,OAAO,OAAO,EAAE;IACxB,SAAS,OAAO,OAAO,EAAE;IACzB,WAAW;IACX,QAAQ;GACV;EACF;EACA,MAAM,SAAS;GACb,IAAI,QAAQ,OAAO,QAAQ,uBAAO,IAAI,MAAM,iBAAiB,CAAC;GAC9D,kBAAkB,YAAY;IAC5B,QAAQ,MAAM,SAAS;IACvB,OAAO,IAAI,OAAO;GACpB,EAAA,CAAG,CAAC,CAAC,OAAO,UAAU;IACpB,eAAe,KAAA;IACf,MAAM;GACR,CAAC;GACD,OAAO;EACT;EACA,MAAM,QAAQ;GACZ,IAAI,QAAQ;GACZ,SAAS;GACT,MAAM,MAAM;GACZ,MAAM;GACN,MAAM;GAGN,MAAM,QAAQ,kBACN;IACJ,IAAI,OACF,MACG,KAAK,OAAO,GAAG,UAAU,KAAK,GAAG,cAAc,CAAC,CAChD,OAAO,UAAU,MAAM,0BAA0B,KAAK,CAAC;GAC9D,GACA,KAAK,MAAM,iBAAiB,CAAC,CAC/B;GACA,IAAI;IACF,MAAM,QAAQ,IAAI,QAAQ;GAC5B,UAAU;IACR,cAAc,KAAK;GACrB;GACA,IAAI,OAAO;IACT,MAAM,MAAM,IAAI,UAAU,KAAK,CAAC;IAChC,MAAM,MAAM,KACV,SACA,GACA,WAAW,KAAK,GAChB,OACA,UAAU,KAAK,GACf,QACA,KACF;GACF;EACF;CAE4B,GAAG,QAAQ;EAAE,GAAG;EAAS;CAAM,CAAC;AAChE;;;;AC9ZA,SAAgB,YACd,SAAoC,CAAC,GACnB;CAClB,OAAO,uBAAuB,QAAQ;EACpC,KAAK,cAAc;EACnB;EACA,OAAO;EACP;CACF,CAAC;AACH"}
|
|
1
|
+
{"version":3,"file":"index.js","names":["delay"],"sources":["../src/queue.ts","../src/index.ts"],"sourcesContent":["import { randomUUID } from \"node:crypto\";\nimport { setTimeout as delay } from \"node:timers/promises\";\nimport { MessageId, ValidQueueName } from \"@workflow/world\";\nimport type { Redis } from \"ioredis\";\nimport {\n checkInteger,\n type RedisWorldConfig,\n type ResolvedConfig,\n} from \"./config.js\";\nimport { keys } from \"./keys.js\";\nimport { debug } from \"./util.js\";\nimport {\n createWorldQueue,\n retryDelayMs,\n type Deliver,\n type Delivery,\n type QueueBackend,\n type WorldQueue,\n type WorldQueueOptions,\n} from \"./world-queue.js\";\n\nexport type { QueueStats } from \"./world-queue.js\";\n\ninterface Envelope extends Delivery {\n dedupKey: string;\n headers: Record<string, string>;\n}\n\n// Derived from the upstream reliable Redis list queue. Every transition is a\n// single script: a process crash cannot leave a message between structures.\nconst ENQUEUE = `\n if ARGV[1] ~= '' then\n local existing = redis.call('GET', KEYS[1])\n if existing then return existing end\n redis.call('SET', KEYS[1], ARGV[2])\n if tonumber(ARGV[3]) > 0 then redis.call('PEXPIRE', KEYS[1], ARGV[3]) end\n end\n if tonumber(ARGV[5]) > 0 then\n redis.call('ZADD', KEYS[3], ARGV[5], ARGV[4])\n else\n redis.call('LPUSH', KEYS[2], ARGV[4])\n end\n return ARGV[2]\n`;\nconst RESERVE = `\n if redis.call('EXISTS', KEYS[3]) == 0 then return nil end\n local raw = redis.call('LINDEX', KEYS[1], -1)\n if not raw then return nil end\n local job = cjson.decode(raw)\n job.attempt = job.attempt + 1\n job.deliveryCount = job.deliveryCount + 1\n local claim = cjson.encode(job)\n redis.call('RPOP', KEYS[1])\n redis.call('LPUSH', KEYS[2], claim)\n return claim\n`;\nconst ACK = `\n if redis.call('EXISTS', KEYS[3]) == 0 then return 0 end\n if redis.call('LREM', KEYS[1], 1, ARGV[1]) == 0 then return 0 end\n if ARGV[2] ~= '' and redis.call('GET', KEYS[2]) == ARGV[3] then\n redis.call('DEL', KEYS[2])\n end\n return 1\n`;\nconst DEFER = `\n if redis.call('EXISTS', KEYS[3]) == 0 then return 0 end\n if redis.call('LREM', KEYS[1], 1, ARGV[1]) == 0 then return 0 end\n redis.call('ZADD', KEYS[2], ARGV[3], ARGV[2])\n return 1\n`;\nconst PROMOTE = `\n local due = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[1], 'LIMIT', 0, 100)\n for _, raw in ipairs(due) do\n redis.call('LPUSH', KEYS[2], raw)\n redis.call('ZREM', KEYS[1], raw)\n end\n return #due\n`;\nconst RECLAIM = `\n if redis.call('EXISTS', KEYS[3]) == 1 then return 0 end\n local moved = 0\n for i = 1, 100 do\n local raw = redis.call('RPOPLPUSH', KEYS[1], KEYS[2])\n if not raw then break end\n moved = moved + 1\n end\n if redis.call('LLEN', KEYS[1]) == 0 then redis.call('SREM', KEYS[4], ARGV[1]) end\n return moved\n`;\nconst RENEW = `\n if redis.call('EXISTS', KEYS[1]) == 0 then return 0 end\n redis.call('PEXPIRE', KEYS[1], ARGV[1])\n return 1\n`;\nconst REGISTER = `\n redis.call('SET', KEYS[1], '1', 'PX', ARGV[2])\n redis.call('SADD', KEYS[2], ARGV[1])\n`;\n// ARGV[1] .. owner is the owner's processing list: `keys().queue.processing(\"\")`.\nconst STATS = `\n local active = 0\n for _, owner in ipairs(redis.call('SMEMBERS', KEYS[3])) do\n active = active + redis.call('LLEN', ARGV[1] .. owner)\n end\n return {redis.call('LLEN', KEYS[1]), active, redis.call('ZCARD', KEYS[2])}\n`;\n\nfunction parseEnvelope(raw: string): Envelope {\n const value: unknown = JSON.parse(raw);\n if (\n value === null ||\n typeof value !== \"object\" ||\n !(\"messageId\" in value) ||\n !(\"queueName\" in value) ||\n !(\"message\" in value) ||\n typeof value.message !== \"string\" ||\n !(\"attempt\" in value) ||\n typeof value.attempt !== \"number\" ||\n !(\"deliveryCount\" in value) ||\n typeof value.deliveryCount !== \"number\" ||\n !(\"createdAt\" in value) ||\n typeof value.createdAt !== \"number\" ||\n !(\"dedupKey\" in value) ||\n typeof value.dedupKey !== \"string\" ||\n !(\"headers\" in value) ||\n value.headers === null ||\n typeof value.headers !== \"object\"\n ) {\n throw new TypeError(\"Invalid Redis queue envelope\");\n }\n const headers: Record<string, string> = {};\n for (const [key, header] of Object.entries(value.headers)) {\n if (typeof header !== \"string\") throw new TypeError(\"Invalid queue header\");\n headers[key] = header;\n }\n return {\n messageId: MessageId.parse(value.messageId),\n queueName: ValidQueueName.parse(value.queueName),\n message: value.message,\n attempt: value.attempt,\n deliveryCount: value.deliveryCount,\n createdAt: value.createdAt,\n dedupKey: value.dedupKey,\n headers,\n ...(\"route\" in value &&\n typeof value.route === \"string\" && { route: value.route }),\n };\n}\n\nexport function createQueue(\n redis: Redis,\n config: ResolvedConfig<RedisWorldConfig>,\n options: WorldQueueOptions = {},\n): WorldQueue {\n const queueKeys = keys(config.keyPrefix).queue;\n const { ready, delayed, workers: owners, processing, heartbeat } = queueKeys;\n const heartbeatTtlMs = config.heartbeatTtlMs ?? 90_000;\n checkInteger(\"heartbeatTtlMs\", heartbeatTtlMs, 300);\n const { queueConcurrency: concurrency, maxAttempts, maxBackoffMs } = config;\n let closed = false;\n let startPromise: Promise<void> | undefined;\n let loop: Promise<void> | undefined;\n let owner: string | undefined;\n const inflight = new Set<Promise<void>>();\n const lostOwners = new Set<string>();\n const abort = new AbortController();\n const sleep = () =>\n delay(50, undefined, { signal: abort.signal }).catch(() => {});\n\n async function enqueue(\n message: Parameters<QueueBackend[\"enqueue\"]>[0],\n opts: Parameters<QueueBackend[\"enqueue\"]>[1],\n ): Promise<MessageId> {\n const dedupKey = opts.idempotencyKey\n ? queueKeys.idempotent(opts.idempotencyKey)\n : \"\";\n const envelope: Envelope = {\n messageId: message.messageId,\n queueName: message.queueName,\n message: message.message,\n attempt: 0,\n deliveryCount: 0,\n createdAt: Date.now(),\n dedupKey,\n headers: message.headers ?? {},\n ...(message.route && { route: message.route }),\n };\n const delayMs = Math.max(0, (opts.delaySeconds ?? 0) * 1000);\n if (!Number.isFinite(delayMs))\n throw new RangeError(\"delaySeconds must be finite\");\n const result = await redis.eval(\n ENQUEUE,\n 3,\n dedupKey || ready,\n ready,\n delayed,\n dedupKey,\n message.messageId,\n config.idempotencyTtlMs ?? 0,\n JSON.stringify(envelope),\n delayMs > 0 ? Date.now() + delayMs : 0,\n );\n return MessageId.parse(result);\n }\n\n async function settle(\n raw: string,\n claimOwner: string,\n deliver: Deliver,\n ): Promise<void> {\n try {\n const job = parseEnvelope(raw);\n let seconds: number | undefined;\n let failed = false;\n try {\n // Reservation persists the attempt before delivery, so a crash\n // redelivery reports one more.\n seconds = await deliver(job);\n } catch (error) {\n failed = true;\n debug(\n `Delivery ${job.messageId} attempt ${job.attempt} failed:`,\n error,\n );\n seconds =\n (job.attempt >= maxAttempts\n ? maxBackoffMs\n : retryDelayMs(job.attempt, config)) / 1000;\n if (job.attempt === maxAttempts)\n console.error(\n `Queue message ${job.messageId} exhausted maxAttempts; retaining it for capped retries`,\n );\n }\n if (seconds === undefined) {\n await redis.eval(\n ACK,\n 3,\n processing(claimOwner),\n job.dedupKey || ready,\n heartbeat(claimOwner),\n raw,\n job.dedupKey,\n job.messageId,\n );\n } else {\n // Only successful reschedules preserve the attempt. A crash always\n // leaves the reserved attempt in Redis, so reclaim counts as failure.\n const next = failed ? job : { ...job, attempt: job.attempt - 1 };\n await redis.eval(\n DEFER,\n 3,\n processing(claimOwner),\n delayed,\n heartbeat(claimOwner),\n raw,\n JSON.stringify(next),\n Date.now() + seconds * 1000,\n );\n }\n } catch (error) {\n // Keep the claim for reclaim on connection loss or malformed data.\n lostOwners.add(claimOwner);\n console.error(\"Queue settlement error:\", error);\n }\n }\n\n async function register(): Promise<string> {\n const next = randomUUID();\n await redis.eval(\n REGISTER,\n 2,\n heartbeat(next),\n owners,\n next,\n heartbeatTtlMs,\n );\n return next;\n }\n\n async function run(deliver: Deliver): Promise<void> {\n let renewedAt = Date.now();\n let maintainedAt = 0;\n let failing = false;\n while (!closed) {\n try {\n for (const lost of lostOwners) {\n await redis.del(heartbeat(lost));\n lostOwners.delete(lost);\n if (owner === lost) owner = undefined;\n }\n if (!owner || Date.now() - renewedAt >= heartbeatTtlMs / 3) {\n // Never resurrect an expired lease: its processing list may already\n // belong to the reclaimer. A fresh UUID fences all old settlements.\n if (\n !owner ||\n (await redis.eval(RENEW, 1, heartbeat(owner), heartbeatTtlMs)) !== 1\n )\n owner = await register();\n renewedAt = Date.now();\n }\n if (Date.now() - maintainedAt >= 100) {\n await redis.eval(PROMOTE, 2, delayed, ready, Date.now());\n for (const stale of await redis.smembers(owners)) {\n if (stale !== owner)\n await redis.eval(\n RECLAIM,\n 4,\n processing(stale),\n ready,\n heartbeat(stale),\n owners,\n stale,\n );\n }\n maintainedAt = Date.now();\n }\n for (\n let reserved = 0;\n reserved < concurrency &&\n !closed &&\n owner &&\n inflight.size < concurrency;\n reserved++\n ) {\n const claimOwner = owner;\n const raw = await redis.eval(\n RESERVE,\n 3,\n ready,\n processing(claimOwner),\n heartbeat(claimOwner),\n );\n if (typeof raw !== \"string\") break;\n const task = settle(raw, claimOwner, deliver);\n inflight.add(task);\n void task.finally(() => inflight.delete(task));\n }\n failing = false;\n } catch (error) {\n // Warn once per outage: a queue that lost Redis otherwise looks idle.\n if (!closed && !failing) console.warn(\"Queue polling failed:\", error);\n else if (!closed) debug(\"Queue polling error:\", error);\n failing = true;\n }\n await sleep();\n }\n }\n\n const backend: QueueBackend = {\n enqueue,\n async getQueueStats() {\n const counts = await redis.eval(\n STATS,\n 3,\n ready,\n delayed,\n owners,\n processing(\"\"),\n );\n if (\n !Array.isArray(counts) ||\n counts.length !== 3 ||\n counts.some((c) => typeof c !== \"number\")\n )\n throw new TypeError(\"Invalid queue statistics\");\n return {\n waiting: Number(counts[0]),\n active: Number(counts[1]),\n delayed: Number(counts[2]),\n completed: 0,\n failed: 0,\n };\n },\n start(deliver) {\n if (closed) return Promise.reject(new Error(\"Queue is closed\"));\n startPromise ??= (async () => {\n owner = await register();\n loop = run(deliver);\n })().catch((error) => {\n startPromise = undefined;\n throw error;\n });\n return startPromise;\n },\n async close() {\n if (closed) return;\n closed = true;\n abort.abort();\n await startPromise;\n await loop;\n // While handlers drain, keep the lease alive. close() must not permit a\n // live long-running handler to be reclaimed merely because shutdown began.\n const timer = setInterval(\n () => {\n if (owner)\n void redis\n .eval(RENEW, 1, heartbeat(owner), heartbeatTtlMs)\n .catch((error) => debug(\"Queue close heartbeat:\", error));\n },\n Math.floor(heartbeatTtlMs / 3),\n );\n try {\n await Promise.all(inflight);\n } finally {\n clearInterval(timer);\n }\n if (owner) {\n await redis.del(heartbeat(owner));\n await redis.eval(\n RECLAIM,\n 4,\n processing(owner),\n ready,\n heartbeat(owner),\n owners,\n owner,\n );\n }\n },\n };\n return createWorldQueue(backend, config, { ...options, debug });\n}\n","import { configFromEnv, type RedisWorldConfig } from \"./config.js\";\nimport { createQueue } from \"./queue.js\";\nimport { debug } from \"./util.js\";\nimport { createRedisBackedWorld, type RedisBackedWorld } from \"./world.js\";\n\n/** `config` is laid over `configFromEnv()`: an option left out comes from the\n * `WORKFLOW_REDIS_*` variables, then the defaults. */\nexport function createWorld(\n config: Partial<RedisWorldConfig> = {},\n): RedisBackedWorld {\n return createRedisBackedWorld(config, {\n env: configFromEnv(),\n createQueue,\n label: \"world-redis\",\n debug,\n });\n}\n\nexport type { RedisWorldConfig } from \"./config.js\";\nexport type { QueueStats } from \"./queue.js\";\nexport type { KeepRuns, RunRetention } from \"./reaper.js\";\n"],"mappings":";;;;;;AA8BA,MAAM,UAAU;;;;;;;;;;;;;;AAchB,MAAM,UAAU;;;;;;;;;;;;AAYhB,MAAM,MAAM;;;;;;;;AAQZ,MAAM,QAAQ;;;;;;AAMd,MAAM,UAAU;;;;;;;;AAQhB,MAAM,UAAU;;;;;;;;;;;AAWhB,MAAM,QAAQ;;;;;AAKd,MAAM,WAAW;;;;AAKjB,MAAM,QAAQ;;;;;;;AAQd,SAAS,cAAc,KAAuB;CAC5C,MAAM,QAAiB,KAAK,MAAM,GAAG;CACrC,IACE,UAAU,QACV,OAAO,UAAU,YACjB,EAAE,eAAe,UACjB,EAAE,eAAe,UACjB,EAAE,aAAa,UACf,OAAO,MAAM,YAAY,YACzB,EAAE,aAAa,UACf,OAAO,MAAM,YAAY,YACzB,EAAE,mBAAmB,UACrB,OAAO,MAAM,kBAAkB,YAC/B,EAAE,eAAe,UACjB,OAAO,MAAM,cAAc,YAC3B,EAAE,cAAc,UAChB,OAAO,MAAM,aAAa,YAC1B,EAAE,aAAa,UACf,MAAM,YAAY,QAClB,OAAO,MAAM,YAAY,UAEzB,MAAM,IAAI,UAAU,8BAA8B;CAEpD,MAAM,UAAkC,CAAC;CACzC,KAAK,MAAM,CAAC,KAAK,WAAW,OAAO,QAAQ,MAAM,OAAO,GAAG;EACzD,IAAI,OAAO,WAAW,UAAU,MAAM,IAAI,UAAU,sBAAsB;EAC1E,QAAQ,OAAO;CACjB;CACA,OAAO;EACL,WAAW,UAAU,MAAM,MAAM,SAAS;EAC1C,WAAW,eAAe,MAAM,MAAM,SAAS;EAC/C,SAAS,MAAM;EACf,SAAS,MAAM;EACf,eAAe,MAAM;EACrB,WAAW,MAAM;EACjB,UAAU,MAAM;EAChB;EACA,GAAI,WAAW,SACb,OAAO,MAAM,UAAU,YAAY,EAAE,OAAO,MAAM,MAAM;CAC5D;AACF;AAEA,SAAgB,YACd,OACA,QACA,UAA6B,CAAC,GAClB;CACZ,MAAM,YAAY,KAAK,OAAO,SAAS,CAAC,CAAC;CACzC,MAAM,EAAE,OAAO,SAAS,SAAS,QAAQ,YAAY,cAAc;CACnE,MAAM,iBAAiB,OAAO,kBAAkB;CAChD,aAAa,kBAAkB,gBAAgB,GAAG;CAClD,MAAM,EAAE,kBAAkB,aAAa,aAAa,iBAAiB;CACrE,IAAI,SAAS;CACb,IAAI;CACJ,IAAI;CACJ,IAAI;CACJ,MAAM,2BAAW,IAAI,IAAmB;CACxC,MAAM,6BAAa,IAAI,IAAY;CACnC,MAAM,QAAQ,IAAI,gBAAgB;CAClC,MAAM,cACJA,WAAM,IAAI,KAAA,GAAW,EAAE,QAAQ,MAAM,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;CAE/D,eAAe,QACb,SACA,MACoB;EACpB,MAAM,WAAW,KAAK,iBAClB,UAAU,WAAW,KAAK,cAAc,IACxC;EACJ,MAAM,WAAqB;GACzB,WAAW,QAAQ;GACnB,WAAW,QAAQ;GACnB,SAAS,QAAQ;GACjB,SAAS;GACT,eAAe;GACf,WAAW,KAAK,IAAI;GACpB;GACA,SAAS,QAAQ,WAAW,CAAC;GAC7B,GAAI,QAAQ,SAAS,EAAE,OAAO,QAAQ,MAAM;EAC9C;EACA,MAAM,UAAU,KAAK,IAAI,IAAI,KAAK,gBAAgB,KAAK,GAAI;EAC3D,IAAI,CAAC,OAAO,SAAS,OAAO,GAC1B,MAAM,IAAI,WAAW,6BAA6B;EACpD,MAAM,SAAS,MAAM,MAAM,KACzB,SACA,GACA,YAAY,OACZ,OACA,SACA,UACA,QAAQ,WACR,OAAO,oBAAoB,GAC3B,KAAK,UAAU,QAAQ,GACvB,UAAU,IAAI,KAAK,IAAI,IAAI,UAAU,CACvC;EACA,OAAO,UAAU,MAAM,MAAM;CAC/B;CAEA,eAAe,OACb,KACA,YACA,SACe;EACf,IAAI;GACF,MAAM,MAAM,cAAc,GAAG;GAC7B,IAAI;GACJ,IAAI,SAAS;GACb,IAAI;IAGF,UAAU,MAAM,QAAQ,GAAG;GAC7B,SAAS,OAAO;IACd,SAAS;IACT,MACE,YAAY,IAAI,UAAU,WAAW,IAAI,QAAQ,WACjD,KACF;IACA,WACG,IAAI,WAAW,cACZ,eACA,aAAa,IAAI,SAAS,MAAM,KAAK;IAC3C,IAAI,IAAI,YAAY,aAClB,QAAQ,MACN,iBAAiB,IAAI,UAAU,wDACjC;GACJ;GACA,IAAI,YAAY,KAAA,GACd,MAAM,MAAM,KACV,KACA,GACA,WAAW,UAAU,GACrB,IAAI,YAAY,OAChB,UAAU,UAAU,GACpB,KACA,IAAI,UACJ,IAAI,SACN;QACK;IAGL,MAAM,OAAO,SAAS,MAAM;KAAE,GAAG;KAAK,SAAS,IAAI,UAAU;IAAE;IAC/D,MAAM,MAAM,KACV,OACA,GACA,WAAW,UAAU,GACrB,SACA,UAAU,UAAU,GACpB,KACA,KAAK,UAAU,IAAI,GACnB,KAAK,IAAI,IAAI,UAAU,GACzB;GACF;EACF,SAAS,OAAO;GAEd,WAAW,IAAI,UAAU;GACzB,QAAQ,MAAM,2BAA2B,KAAK;EAChD;CACF;CAEA,eAAe,WAA4B;EACzC,MAAM,OAAO,WAAW;EACxB,MAAM,MAAM,KACV,UACA,GACA,UAAU,IAAI,GACd,QACA,MACA,cACF;EACA,OAAO;CACT;CAEA,eAAe,IAAI,SAAiC;EAClD,IAAI,YAAY,KAAK,IAAI;EACzB,IAAI,eAAe;EACnB,IAAI,UAAU;EACd,OAAO,CAAC,QAAQ;GACd,IAAI;IACF,KAAK,MAAM,QAAQ,YAAY;KAC7B,MAAM,MAAM,IAAI,UAAU,IAAI,CAAC;KAC/B,WAAW,OAAO,IAAI;KACtB,IAAI,UAAU,MAAM,QAAQ,KAAA;IAC9B;IACA,IAAI,CAAC,SAAS,KAAK,IAAI,IAAI,aAAa,iBAAiB,GAAG;KAG1D,IACE,CAAC,SACA,MAAM,MAAM,KAAK,OAAO,GAAG,UAAU,KAAK,GAAG,cAAc,MAAO,GAEnE,QAAQ,MAAM,SAAS;KACzB,YAAY,KAAK,IAAI;IACvB;IACA,IAAI,KAAK,IAAI,IAAI,gBAAgB,KAAK;KACpC,MAAM,MAAM,KAAK,SAAS,GAAG,SAAS,OAAO,KAAK,IAAI,CAAC;KACvD,KAAK,MAAM,SAAS,MAAM,MAAM,SAAS,MAAM,GAC7C,IAAI,UAAU,OACZ,MAAM,MAAM,KACV,SACA,GACA,WAAW,KAAK,GAChB,OACA,UAAU,KAAK,GACf,QACA,KACF;KAEJ,eAAe,KAAK,IAAI;IAC1B;IACA,KACE,IAAI,WAAW,GACf,WAAW,eACX,CAAC,UACD,SACA,SAAS,OAAO,aAChB,YACA;KACA,MAAM,aAAa;KACnB,MAAM,MAAM,MAAM,MAAM,KACtB,SACA,GACA,OACA,WAAW,UAAU,GACrB,UAAU,UAAU,CACtB;KACA,IAAI,OAAO,QAAQ,UAAU;KAC7B,MAAM,OAAO,OAAO,KAAK,YAAY,OAAO;KAC5C,SAAS,IAAI,IAAI;KACZ,KAAK,cAAc,SAAS,OAAO,IAAI,CAAC;IAC/C;IACA,UAAU;GACZ,SAAS,OAAO;IAEd,IAAI,CAAC,UAAU,CAAC,SAAS,QAAQ,KAAK,yBAAyB,KAAK;SAC/D,IAAI,CAAC,QAAQ,MAAM,wBAAwB,KAAK;IACrD,UAAU;GACZ;GACA,MAAM,MAAM;EACd;CACF;CA0EA,OAAO,iBAAiB;EAvEtB;EACA,MAAM,gBAAgB;GACpB,MAAM,SAAS,MAAM,MAAM,KACzB,OACA,GACA,OACA,SACA,QACA,WAAW,EAAE,CACf;GACA,IACE,CAAC,MAAM,QAAQ,MAAM,KACrB,OAAO,WAAW,KAClB,OAAO,MAAM,MAAM,OAAO,MAAM,QAAQ,GAExC,MAAM,IAAI,UAAU,0BAA0B;GAChD,OAAO;IACL,SAAS,OAAO,OAAO,EAAE;IACzB,QAAQ,OAAO,OAAO,EAAE;IACxB,SAAS,OAAO,OAAO,EAAE;IACzB,WAAW;IACX,QAAQ;GACV;EACF;EACA,MAAM,SAAS;GACb,IAAI,QAAQ,OAAO,QAAQ,uBAAO,IAAI,MAAM,iBAAiB,CAAC;GAC9D,kBAAkB,YAAY;IAC5B,QAAQ,MAAM,SAAS;IACvB,OAAO,IAAI,OAAO;GACpB,EAAA,CAAG,CAAC,CAAC,OAAO,UAAU;IACpB,eAAe,KAAA;IACf,MAAM;GACR,CAAC;GACD,OAAO;EACT;EACA,MAAM,QAAQ;GACZ,IAAI,QAAQ;GACZ,SAAS;GACT,MAAM,MAAM;GACZ,MAAM;GACN,MAAM;GAGN,MAAM,QAAQ,kBACN;IACJ,IAAI,OACG,MACF,KAAK,OAAO,GAAG,UAAU,KAAK,GAAG,cAAc,CAAC,CAChD,OAAO,UAAU,MAAM,0BAA0B,KAAK,CAAC;GAC9D,GACA,KAAK,MAAM,iBAAiB,CAAC,CAC/B;GACA,IAAI;IACF,MAAM,QAAQ,IAAI,QAAQ;GAC5B,UAAU;IACR,cAAc,KAAK;GACrB;GACA,IAAI,OAAO;IACT,MAAM,MAAM,IAAI,UAAU,KAAK,CAAC;IAChC,MAAM,MAAM,KACV,SACA,GACA,WAAW,KAAK,GAChB,OACA,UAAU,KAAK,GACf,QACA,KACF;GACF;EACF;CAE4B,GAAG,QAAQ;EAAE,GAAG;EAAS;CAAM,CAAC;AAChE;;;;AC9ZA,SAAgB,YACd,SAAoC,CAAC,GACnB;CAClB,OAAO,uBAAuB,QAAQ;EACpC,KAAK,cAAc;EACnB;EACA,OAAO;EACP;CACF,CAAC;AACH"}
|
package/dist/internal.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { _ as RunRetention, a as QueueStats, c as WorldQueueOptions, d as RedisBackedWorldConfig, g as KeepRuns, h as resolveConfig, i as QueueBackend, l as createWorldQueue, m as configFromEnv, n as createRedisBackedWorld, o as QueuedMessage, p as ResolvedConfig, r as Deliver, s as WorldQueue, t as RedisBackedWorld, u as retryDelayMs } from "./world-
|
|
1
|
+
import { _ as RunRetention, a as QueueStats, c as WorldQueueOptions, d as RedisBackedWorldConfig, g as KeepRuns, h as resolveConfig, i as QueueBackend, l as createWorldQueue, m as configFromEnv, n as createRedisBackedWorld, o as QueuedMessage, p as ResolvedConfig, r as Deliver, s as WorldQueue, t as RedisBackedWorld, u as retryDelayMs } from "./world-DCbuc04m.js";
|
|
2
2
|
import { setupRedis } from "./cli.js";
|
|
3
3
|
/**
|
|
4
4
|
* Creates a namespaced debug logger.
|
package/dist/internal.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import { i as resolveConfig, r as configFromEnv } from "./keys-CqKp_Amk.js";
|
|
2
2
|
import { t as setupRedis } from "./cli-BEHfAdNK.js";
|
|
3
|
-
import { a as createDebugLogger, n as createWorldQueue, r as retryDelayMs, t as createRedisBackedWorld } from "./world-
|
|
3
|
+
import { a as createDebugLogger, n as createWorldQueue, r as retryDelayMs, t as createRedisBackedWorld } from "./world-Cac56z4M.js";
|
|
4
4
|
export { configFromEnv, createDebugLogger, createRedisBackedWorld, createWorldQueue, resolveConfig, retryDelayMs, setupRedis };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"keys-CqKp_Amk.js","names":[],"sources":["../src/config.ts","../src/keys.ts"],"sourcesContent":["import type { RedisOptions } from \"ioredis\";\nimport type { RunRetention } from \"./reaper.js\";\n\n/** The options every Redis-backed world shares, whatever its queue. */\nexport interface RedisBackedWorldConfig {\n /**\n * Redis connection URL or connection options\n * Examples:\n * - 'redis://localhost:6379'\n * - 'rediss://username:password@host:6380/0'\n * - { host: 'localhost', port: 6379 }\n */\n redis: string | RedisOptions;\n\n /**\n * Number of concurrent workers processing jobs\n * Default: 10\n */\n queueConcurrency?: number;\n\n /**\n * Optional key prefix for all Redis keys\n * Useful for multi-tenancy or namespace isolation\n * Default: 'workflow:'\n */\n keyPrefix?: string;\n\n /**\n * Backoff strategy for retries: 'fixed' replays at the same interval,\n * 'exponential' doubles the delay on each attempt.\n * Default: 'exponential'\n */\n backoffType?: \"fixed\" | \"exponential\";\n\n /**\n * Base delay for backoff (ms). For exponential backoff this is multiplied\n * by 2^(attempt - 1), up to `maxBackoffMs`.\n * Default: 1000\n */\n backoffDelayMs?: number;\n\n /**\n * The longest a retry is delayed (ms). With the default number of attempts an\n * uncapped exponential backoff would wait hours between late attempts.\n * Default: 30000 (30s)\n */\n maxBackoffMs?: number;\n\n /**\n * Base URL the queue worker uses to dispatch jobs back to the user's HTTP\n * server, which must serve `world.createQueueHandler(...)` at\n * `/.well-known/workflow/v1/flow` (steps travel on the same route). A route\n * registered with `world.registerRoute`, or an in-process handler registered\n * with `world.registerHandler`, takes precedence.\n * Default: process.env.WORKFLOW_BASE_URL || `http://localhost:${process.env.PORT ?? 3000}`\n */\n baseUrl?: string;\n\n /**\n * Per-job HTTP request timeout (ms). Long-running workflow steps may need a\n * higher value.\n * Default: 300_000 (5 minutes)\n */\n httpTimeoutMs?: number;\n\n /** Maximum events a run may accumulate, reported as `EventResult.maxEvents`.\n * Default: `WORKFLOW_MAX_EVENTS` || 25_000 */\n maxEventsPerRun?: number;\n\n /**\n * Batch commands issued in the same event-loop tick into one write. A win\n * once several runs are in flight (~1.75x at 256 concurrent) and a marginal\n * loss for a single serial caller.\n * Default: true\n */\n enableAutoPipelining?: boolean;\n\n /**\n * How long finished runs are kept, per terminal status: `age` in seconds,\n * `count` of newest runs to keep. A reaper on each worker deletes the run, its event log, steps, hooks\n * and streams once it passes. A status without a policy is kept forever.\n * Default: undefined (nothing expires)\n *\n * @example\n * retention: {\n * completed: { age: 24 * 60 * 60, count: 1000 },\n * failed: { age: 7 * 24 * 60 * 60 },\n * }\n */\n retention?: RunRetention;\n\n /**\n * Delivery budget per message (see each world for what happens past it).\n * At least 49: the runtime records a run as failed (`MAX_DELIVERIES_EXCEEDED`)\n * on delivery 49, and a message retired earlier strands its run in `running`.\n * Default: 73\n */\n maxAttempts?: number;\n\n /**\n * Optional TTL (ms) for queue deduplication when `idempotencyKey`\n * is supplied to `queue()`. By default no TTL is applied: the dedup key\n * lives until the job is acknowledged, so duplicate enqueues are dropped\n * for the entire lifetime of the pending/delayed job (the semantics the\n * workflow runtime relies on when re-enqueuing pending steps on replay).\n * Only set this if you need the dedup guard to expire on wall-clock time;\n * it must be much larger than the longest expected queue delay/backlog,\n * otherwise duplicate step executions become possible.\n * Default: undefined (dedup key released on job completion)\n */\n idempotencyTtlMs?: number;\n}\n\nexport interface RedisWorldConfig extends RedisBackedWorldConfig {\n /** Worker liveness TTL in milliseconds. Default 90000; minimum 300.\n * Set above the longest event-loop pause. Renewed every TTL / 3. */\n heartbeatTtlMs?: number;\n\n /**\n * Delivery budget before retries continue at maxBackoffMs. Messages are never\n * silently retired. At least 49. Default: 73\n */\n maxAttempts?: number;\n}\n\n/**\n * The config the `WORKFLOW_REDIS_*` variables give. `createWorld` lays the\n * config it is passed over this, so an option left out still comes from env.\n */\nexport function configFromEnv() {\n return {\n redis:\n process.env.WORKFLOW_REDIS_URL ||\n process.env.REDIS_URL ||\n \"redis://localhost:6379\",\n queueConcurrency:\n Number.parseInt(\n process.env.WORKFLOW_REDIS_WORKER_CONCURRENCY ?? \"\",\n 10,\n ) || undefined,\n keyPrefix: process.env.WORKFLOW_REDIS_KEY_PREFIX || undefined,\n };\n}\n\nconst DEFAULTS = {\n queueConcurrency: 10,\n keyPrefix: \"workflow:\",\n backoffType: \"exponential\",\n backoffDelayMs: 1000,\n maxBackoffMs: 30_000,\n httpTimeoutMs: 300_000,\n maxAttempts: 73,\n enableAutoPipelining: true,\n} as const;\n\n/** A world config with every shared default filled in. */\nexport type ResolvedConfig<C extends RedisBackedWorldConfig> = C & {\n -readonly [K in keyof typeof DEFAULTS]-?: NonNullable<\n RedisBackedWorldConfig[K]\n >;\n};\n\n/** Fills the shared defaults and checks the shared limits, for every queue. */\nexport function resolveConfig<C extends RedisBackedWorldConfig>(\n config: C,\n): ResolvedConfig<C> {\n const resolved: Record<string, unknown> = { ...(config as object) };\n for (const [key, value] of Object.entries(DEFAULTS))\n if (resolved[key] === undefined || resolved[key] === \"\")\n resolved[key] = value;\n for (const [name, minimum] of [\n [\"queueConcurrency\", 1],\n [\"maxAttempts\", 49],\n [\"maxBackoffMs\", 1],\n [\"backoffDelayMs\", 0],\n [\"httpTimeoutMs\", 1],\n [\"idempotencyTtlMs\", 1],\n ] as const)\n checkInteger(name, resolved[name], minimum);\n return resolved as unknown as ResolvedConfig<C>;\n}\n\n/** Throws unless `value` is an integer from `minimum` to Node's timer limit; `undefined` passes. */\nexport function checkInteger(name: string, value: unknown, minimum: number) {\n if (\n value !== undefined &&\n (typeof value !== \"number\" ||\n !Number.isSafeInteger(value) ||\n value < minimum ||\n value > 2_147_483_647)\n )\n throw new RangeError(\n `${name} must be an integer between ${minimum} and 2147483647`,\n );\n}\n","/**\n * The world's Redis key layout, the one place storage, streams, the queue and\n * the reaper build key names from. A key added here that belongs to a run must\n * also go in `ownedByRun`, or the reaper leaks it.\n */\nexport function keys(prefix: string) {\n const queue = `${prefix}queue:flows`;\n const k = {\n runKey: (id: string) => `${prefix}run:${id}`,\n runAttributesKey: (id: string) => `${prefix}run:attrs:${id}`,\n // `<status>|<specVersion>`, read by scripts that must check status atomically.\n runMetaKey: (id: string) => `${prefix}run:meta:${id}`,\n runsIndexKey: () => `${prefix}runs:index`,\n runsByNameKey: (name: string) => `${prefix}runs:by_name:${name}`,\n runsByStatusKey: (status: string) => `${prefix}runs:by_status:${status}`,\n\n eventsKey: (runId: string) => `${prefix}events:data:${runId}`,\n // A correlation id is only unique within its run.\n eventsByCorrelationKey: (runId: string, correlationId: string) =>\n `${prefix}events:by_correlation:${runId}:${correlationId}`,\n resumeClaimKey: (runId: string, resumeId: string) =>\n `${prefix}events:resume:${runId}:${resumeId}`,\n\n stepKey: (runId: string, stepId: string) =>\n `${prefix}step:${runId}:${stepId}`,\n stepsIndexKey: (runId: string) => `${prefix}steps:by_run:${runId}`,\n // `<status>|<attempt>`, kept beside the step so a script can check it atomically.\n stepStatusKey: (runId: string, stepId: string) =>\n `${prefix}step:status:${runId}:${stepId}`,\n\n hookKey: (hookId: string) => `${prefix}hook:${hookId}`,\n hooksByTokenKey: (token: string) => `${prefix}hooks:by_token:${token}`,\n hooksIndexKey: (runId: string) => `${prefix}hooks:by_run:${runId}`,\n hooksTokenMapKey: (runId: string) => `${prefix}hooks:tokens:${runId}`,\n\n // Run IDs (`wrun_<ulid>`) never contain ':', so the first ':' after the run\n // ID always starts the stream name, even if the name contains colons itself.\n streamKey: (runId: string, name: string) =>\n `${prefix}stream:${runId}:${name}`,\n streamsByRunKey: (runId: string) => `${prefix}streams:by_run:${runId}`,\n streamChannel: `${prefix}stream_notify`,\n\n reaperLockKey: `${prefix}reaper:lock`,\n /** The list queue's keys. */\n queue: {\n ready: queue,\n delayed: `${queue}:delayed`,\n workers: `${queue}:workers`,\n processing: (owner: string) => `${queue}:processing:${owner}`,\n heartbeat: (owner: string) => `${queue}:heartbeat:${owner}`,\n idempotent: (key: string) => `${queue}:idempotent:${key}`,\n /** Every queue key, for SCAN/KEYS. */\n pattern: `${prefix}queue:*`,\n },\n\n /**\n * Every key a run owns except its run, meta and index entries, which the\n * reaper deletes last. Resume claims are not indexed: scan `resumeClaimKey(runId, \"*\")`.\n */\n ownedByRun(\n runId: string,\n ids: {\n correlationIds: Iterable<string>;\n stepIds: string[];\n hookIds: string[];\n streamNames: string[];\n },\n ): string[] {\n return [\n k.eventsKey(runId),\n k.runAttributesKey(runId),\n k.stepsIndexKey(runId),\n k.hooksIndexKey(runId),\n k.hooksTokenMapKey(runId),\n k.streamsByRunKey(runId),\n ...[...ids.correlationIds].map((id) =>\n k.eventsByCorrelationKey(runId, id),\n ),\n ...ids.stepIds.flatMap((id) => [\n k.stepKey(runId, id),\n k.stepStatusKey(runId, id),\n ]),\n ...ids.hookIds.map(k.hookKey),\n ...ids.streamNames.map((name) => k.streamKey(runId, name)),\n ];\n },\n };\n return k;\n}\n"],"mappings":";;;;;AAiIA,SAAgB,gBAAgB;CAC9B,OAAO;EACL,OACE,QAAQ,IAAI,sBACZ,QAAQ,IAAI,aACZ;EACF,kBACE,OAAO,SACL,QAAQ,IAAI,qCAAqC,IACjD,EACF,KAAK,KAAA;EACP,WAAW,QAAQ,IAAI,6BAA6B,KAAA;CACtD;AACF;AAEA,MAAM,WAAW;CACf,kBAAkB;CAClB,WAAW;CACX,aAAa;CACb,gBAAgB;CAChB,cAAc;CACd,eAAe;CACf,aAAa;CACb,sBAAsB;AACxB;;AAUA,SAAgB,cACd,QACmB;CACnB,MAAM,WAAoC,EAAE,GAAI,OAAkB;CAClE,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,QAAQ,GAChD,IAAI,SAAS,SAAS,KAAA,KAAa,SAAS,SAAS,IACnD,SAAS,OAAO;CACpB,KAAK,MAAM,CAAC,MAAM,YAAY;EAC5B,CAAC,oBAAoB,CAAC;EACtB,CAAC,eAAe,EAAE;EAClB,CAAC,gBAAgB,CAAC;EAClB,CAAC,kBAAkB,CAAC;EACpB,CAAC,iBAAiB,CAAC;EACnB,CAAC,oBAAoB,CAAC;CACxB,GACE,aAAa,MAAM,SAAS,OAAO,OAAO;CAC5C,OAAO;AACT;;AAGA,SAAgB,aAAa,MAAc,OAAgB,SAAiB;CAC1E,IACE,UAAU,KAAA,MACT,OAAO,UAAU,YAChB,CAAC,OAAO,cAAc,KAAK,KAC3B,QAAQ,WACR,QAAQ,aAEV,MAAM,IAAI,WACR,GAAG,KAAK,8BAA8B,QAAQ,gBAChD;AACJ;;;;;;;AC7LA,SAAgB,KAAK,QAAgB;CACnC,MAAM,QAAQ,GAAG,OAAO;CACxB,MAAM,IAAI;EACR,SAAS,OAAe,GAAG,OAAO,MAAM;EACxC,mBAAmB,OAAe,GAAG,OAAO,YAAY;EAExD,aAAa,OAAe,GAAG,OAAO,WAAW;EACjD,oBAAoB,GAAG,OAAO;EAC9B,gBAAgB,SAAiB,GAAG,OAAO,eAAe;EAC1D,kBAAkB,WAAmB,GAAG,OAAO,iBAAiB;EAEhE,YAAY,UAAkB,GAAG,OAAO,cAAc;EAEtD,yBAAyB,OAAe,kBACtC,GAAG,OAAO,wBAAwB,MAAM,GAAG;EAC7C,iBAAiB,OAAe,aAC9B,GAAG,OAAO,gBAAgB,MAAM,GAAG;EAErC,UAAU,OAAe,WACvB,GAAG,OAAO,OAAO,MAAM,GAAG;EAC5B,gBAAgB,UAAkB,GAAG,OAAO,eAAe;EAE3D,gBAAgB,OAAe,WAC7B,GAAG,OAAO,cAAc,MAAM,GAAG;EAEnC,UAAU,WAAmB,GAAG,OAAO,OAAO;EAC9C,kBAAkB,UAAkB,GAAG,OAAO,iBAAiB;EAC/D,gBAAgB,UAAkB,GAAG,OAAO,eAAe;EAC3D,mBAAmB,UAAkB,GAAG,OAAO,eAAe;EAI9D,YAAY,OAAe,SACzB,GAAG,OAAO,SAAS,MAAM,GAAG;EAC9B,kBAAkB,UAAkB,GAAG,OAAO,iBAAiB;EAC/D,eAAe,GAAG,OAAO;EAEzB,eAAe,GAAG,OAAO;;EAEzB,OAAO;GACL,OAAO;GACP,SAAS,GAAG,MAAM;GAClB,SAAS,GAAG,MAAM;GAClB,aAAa,UAAkB,GAAG,MAAM,cAAc;GACtD,YAAY,UAAkB,GAAG,MAAM,aAAa;GACpD,aAAa,QAAgB,GAAG,MAAM,cAAc;;GAEpD,SAAS,GAAG,OAAO;EACrB;;;;;EAMA,WACE,OACA,KAMU;GACV,OAAO;IACL,EAAE,UAAU,KAAK;IACjB,EAAE,iBAAiB,KAAK;IACxB,EAAE,cAAc,KAAK;IACrB,EAAE,cAAc,KAAK;IACrB,EAAE,iBAAiB,KAAK;IACxB,EAAE,gBAAgB,KAAK;IACvB,GAAG,CAAC,GAAG,IAAI,cAAc,CAAC,CAAC,KAAK,OAC9B,EAAE,uBAAuB,OAAO,EAAE,CACpC;IACA,GAAG,IAAI,QAAQ,SAAS,OAAO,CAC7B,EAAE,QAAQ,OAAO,EAAE,GACnB,EAAE,cAAc,OAAO,EAAE,CAC3B,CAAC;IACD,GAAG,IAAI,QAAQ,IAAI,EAAE,OAAO;IAC5B,GAAG,IAAI,YAAY,KAAK,SAAS,EAAE,UAAU,OAAO,IAAI,CAAC;GAC3D;EACF;CACF;CACA,OAAO;AACT"}
|
|
1
|
+
{"version":3,"file":"keys-CqKp_Amk.js","names":[],"sources":["../src/config.ts","../src/keys.ts"],"sourcesContent":["import type { RedisOptions } from \"ioredis\";\nimport type { RunRetention } from \"./reaper.js\";\n\n/** The options every Redis-backed world shares, whatever its queue. */\nexport interface RedisBackedWorldConfig {\n /**\n * Redis connection URL or connection options\n * Examples:\n * - 'redis://localhost:6379'\n * - 'rediss://username:password@host:6380/0'\n * - { host: 'localhost', port: 6379 }\n */\n redis: string | RedisOptions;\n\n /**\n * Number of concurrent workers processing jobs\n * Default: 10\n */\n queueConcurrency?: number;\n\n /**\n * Optional key prefix for all Redis keys\n * Useful for multi-tenancy or namespace isolation\n * Default: 'workflow:'\n */\n keyPrefix?: string;\n\n /**\n * Backoff strategy for retries: 'fixed' replays at the same interval,\n * 'exponential' doubles the delay on each attempt.\n * Default: 'exponential'\n */\n backoffType?: \"fixed\" | \"exponential\";\n\n /**\n * Base delay for backoff (ms). For exponential backoff this is multiplied\n * by 2^(attempt - 1), up to `maxBackoffMs`.\n * Default: 1000\n */\n backoffDelayMs?: number;\n\n /**\n * The longest a retry is delayed (ms). With the default number of attempts an\n * uncapped exponential backoff would wait hours between late attempts.\n * Default: 30000 (30s)\n */\n maxBackoffMs?: number;\n\n /**\n * Base URL the queue worker uses to dispatch jobs back to the user's HTTP\n * server, which must serve `world.createQueueHandler(...)` at\n * `/.well-known/workflow/v1/flow` (steps travel on the same route). A route\n * registered with `world.registerRoute`, or an in-process handler registered\n * with `world.registerHandler`, takes precedence.\n * Default: process.env.WORKFLOW_BASE_URL || `http://localhost:${process.env.PORT ?? 3000}`\n */\n baseUrl?: string;\n\n /**\n * Per-job HTTP request timeout (ms). Long-running workflow steps may need a\n * higher value.\n * Default: 300_000 (5 minutes)\n */\n httpTimeoutMs?: number;\n\n /**\n * The key every delivery to the flow route carries, so the Workflow SDK\n * refuses a request that did not come from this world's queue. The same\n * value on every process of the app: the app's own and each worker's.\n * Generate one with\n * `node -e \"console.log(crypto.randomBytes(32).toString('base64url'))\"`.\n * Default: process.env.WORKFLOW_QUEUE_AUTH_KEY. Required.\n */\n queueAuthKey?: string;\n\n /** Maximum events a run may accumulate, reported as `EventResult.maxEvents`.\n * Default: `WORKFLOW_MAX_EVENTS` || 25_000 */\n maxEventsPerRun?: number;\n\n /**\n * Batch commands issued in the same event-loop tick into one write. A win\n * once several runs are in flight (~1.75x at 256 concurrent) and a marginal\n * loss for a single serial caller.\n * Default: true\n */\n enableAutoPipelining?: boolean;\n\n /**\n * How long finished runs are kept, per terminal status: `age` in seconds,\n * `count` of newest runs to keep. A reaper on each worker deletes the run, its event log, steps, hooks\n * and streams once it passes. A status without a policy is kept forever.\n * Default: undefined (nothing expires)\n *\n * @example\n * retention: {\n * completed: { age: 24 * 60 * 60, count: 1000 },\n * failed: { age: 7 * 24 * 60 * 60 },\n * }\n */\n retention?: RunRetention;\n\n /**\n * Delivery budget per message (see each world for what happens past it).\n * At least 49: the runtime records a run as failed (`MAX_DELIVERIES_EXCEEDED`)\n * on delivery 49, and a message retired earlier strands its run in `running`.\n * Default: 73\n */\n maxAttempts?: number;\n\n /**\n * Optional TTL (ms) for queue deduplication when `idempotencyKey`\n * is supplied to `queue()`. By default no TTL is applied: the dedup key\n * lives until the job is acknowledged, so duplicate enqueues are dropped\n * for the entire lifetime of the pending/delayed job (the semantics the\n * workflow runtime relies on when re-enqueuing pending steps on replay).\n * Only set this if you need the dedup guard to expire on wall-clock time;\n * it must be much larger than the longest expected queue delay/backlog,\n * otherwise duplicate step executions become possible.\n * Default: undefined (dedup key released on job completion)\n */\n idempotencyTtlMs?: number;\n}\n\nexport interface RedisWorldConfig extends RedisBackedWorldConfig {\n /** Worker liveness TTL in milliseconds. Default 90000; minimum 300.\n * Set above the longest event-loop pause. Renewed every TTL / 3. */\n heartbeatTtlMs?: number;\n\n /**\n * Delivery budget before retries continue at maxBackoffMs. Messages are never\n * silently retired. At least 49. Default: 73\n */\n maxAttempts?: number;\n}\n\n/**\n * The config the `WORKFLOW_REDIS_*` variables give. `createWorld` lays the\n * config it is passed over this, so an option left out still comes from env.\n */\nexport function configFromEnv() {\n return {\n redis:\n process.env.WORKFLOW_REDIS_URL ||\n process.env.REDIS_URL ||\n \"redis://localhost:6379\",\n queueConcurrency:\n Number.parseInt(\n process.env.WORKFLOW_REDIS_WORKER_CONCURRENCY ?? \"\",\n 10,\n ) || undefined,\n keyPrefix: process.env.WORKFLOW_REDIS_KEY_PREFIX || undefined,\n };\n}\n\nconst DEFAULTS = {\n queueConcurrency: 10,\n keyPrefix: \"workflow:\",\n backoffType: \"exponential\",\n backoffDelayMs: 1000,\n maxBackoffMs: 30_000,\n httpTimeoutMs: 300_000,\n maxAttempts: 73,\n enableAutoPipelining: true,\n} as const;\n\n/** A world config with every shared default filled in. */\nexport type ResolvedConfig<C extends RedisBackedWorldConfig> = C & {\n -readonly [K in keyof typeof DEFAULTS]-?: NonNullable<\n RedisBackedWorldConfig[K]\n >;\n};\n\n/** Fills the shared defaults and checks the shared limits, for every queue. */\nexport function resolveConfig<C extends RedisBackedWorldConfig>(\n config: C,\n): ResolvedConfig<C> {\n const resolved: Record<string, unknown> = { ...(config as object) };\n for (const [key, value] of Object.entries(DEFAULTS))\n if (resolved[key] === undefined || resolved[key] === \"\")\n resolved[key] = value;\n for (const [name, minimum] of [\n [\"queueConcurrency\", 1],\n [\"maxAttempts\", 49],\n [\"maxBackoffMs\", 1],\n [\"backoffDelayMs\", 0],\n [\"httpTimeoutMs\", 1],\n [\"idempotencyTtlMs\", 1],\n ] as const)\n checkInteger(name, resolved[name], minimum);\n return resolved as unknown as ResolvedConfig<C>;\n}\n\n/** Throws unless `value` is an integer from `minimum` to Node's timer limit; `undefined` passes. */\nexport function checkInteger(name: string, value: unknown, minimum: number) {\n if (\n value !== undefined &&\n (typeof value !== \"number\" ||\n !Number.isSafeInteger(value) ||\n value < minimum ||\n value > 2_147_483_647)\n )\n throw new RangeError(\n `${name} must be an integer between ${minimum} and 2147483647`,\n );\n}\n","/**\n * The world's Redis key layout, the one place storage, streams, the queue and\n * the reaper build key names from. A key added here that belongs to a run must\n * also go in `ownedByRun`, or the reaper leaks it.\n */\nexport function keys(prefix: string) {\n const queue = `${prefix}queue:flows`;\n const k = {\n runKey: (id: string) => `${prefix}run:${id}`,\n runAttributesKey: (id: string) => `${prefix}run:attrs:${id}`,\n // `<status>|<specVersion>`, read by scripts that must check status atomically.\n runMetaKey: (id: string) => `${prefix}run:meta:${id}`,\n runsIndexKey: () => `${prefix}runs:index`,\n runsByNameKey: (name: string) => `${prefix}runs:by_name:${name}`,\n runsByStatusKey: (status: string) => `${prefix}runs:by_status:${status}`,\n\n eventsKey: (runId: string) => `${prefix}events:data:${runId}`,\n // A correlation id is only unique within its run.\n eventsByCorrelationKey: (runId: string, correlationId: string) =>\n `${prefix}events:by_correlation:${runId}:${correlationId}`,\n resumeClaimKey: (runId: string, resumeId: string) =>\n `${prefix}events:resume:${runId}:${resumeId}`,\n\n stepKey: (runId: string, stepId: string) =>\n `${prefix}step:${runId}:${stepId}`,\n stepsIndexKey: (runId: string) => `${prefix}steps:by_run:${runId}`,\n // `<status>|<attempt>`, kept beside the step so a script can check it atomically.\n stepStatusKey: (runId: string, stepId: string) =>\n `${prefix}step:status:${runId}:${stepId}`,\n\n hookKey: (hookId: string) => `${prefix}hook:${hookId}`,\n hooksByTokenKey: (token: string) => `${prefix}hooks:by_token:${token}`,\n hooksIndexKey: (runId: string) => `${prefix}hooks:by_run:${runId}`,\n hooksTokenMapKey: (runId: string) => `${prefix}hooks:tokens:${runId}`,\n\n // Run IDs (`wrun_<ulid>`) never contain ':', so the first ':' after the run\n // ID always starts the stream name, even if the name contains colons itself.\n streamKey: (runId: string, name: string) =>\n `${prefix}stream:${runId}:${name}`,\n streamsByRunKey: (runId: string) => `${prefix}streams:by_run:${runId}`,\n streamChannel: `${prefix}stream_notify`,\n\n reaperLockKey: `${prefix}reaper:lock`,\n /** The list queue's keys. */\n queue: {\n ready: queue,\n delayed: `${queue}:delayed`,\n workers: `${queue}:workers`,\n processing: (owner: string) => `${queue}:processing:${owner}`,\n heartbeat: (owner: string) => `${queue}:heartbeat:${owner}`,\n idempotent: (key: string) => `${queue}:idempotent:${key}`,\n /** Every queue key, for SCAN/KEYS. */\n pattern: `${prefix}queue:*`,\n },\n\n /**\n * Every key a run owns except its run, meta and index entries, which the\n * reaper deletes last. Resume claims are not indexed: scan `resumeClaimKey(runId, \"*\")`.\n */\n ownedByRun(\n runId: string,\n ids: {\n correlationIds: Iterable<string>;\n stepIds: string[];\n hookIds: string[];\n streamNames: string[];\n },\n ): string[] {\n return [\n k.eventsKey(runId),\n k.runAttributesKey(runId),\n k.stepsIndexKey(runId),\n k.hooksIndexKey(runId),\n k.hooksTokenMapKey(runId),\n k.streamsByRunKey(runId),\n ...[...ids.correlationIds].map((id) =>\n k.eventsByCorrelationKey(runId, id),\n ),\n ...ids.stepIds.flatMap((id) => [\n k.stepKey(runId, id),\n k.stepStatusKey(runId, id),\n ]),\n ...ids.hookIds.map(k.hookKey),\n ...ids.streamNames.map((name) => k.streamKey(runId, name)),\n ];\n },\n };\n return k;\n}\n"],"mappings":";;;;;AA2IA,SAAgB,gBAAgB;CAC9B,OAAO;EACL,OACE,QAAQ,IAAI,sBACZ,QAAQ,IAAI,aACZ;EACF,kBACE,OAAO,SACL,QAAQ,IAAI,qCAAqC,IACjD,EACF,KAAK,KAAA;EACP,WAAW,QAAQ,IAAI,6BAA6B,KAAA;CACtD;AACF;AAEA,MAAM,WAAW;CACf,kBAAkB;CAClB,WAAW;CACX,aAAa;CACb,gBAAgB;CAChB,cAAc;CACd,eAAe;CACf,aAAa;CACb,sBAAsB;AACxB;;AAUA,SAAgB,cACd,QACmB;CACnB,MAAM,WAAoC,EAAE,GAAI,OAAkB;CAClE,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,QAAQ,GAChD,IAAI,SAAS,SAAS,KAAA,KAAa,SAAS,SAAS,IACnD,SAAS,OAAO;CACpB,KAAK,MAAM,CAAC,MAAM,YAAY;EAC5B,CAAC,oBAAoB,CAAC;EACtB,CAAC,eAAe,EAAE;EAClB,CAAC,gBAAgB,CAAC;EAClB,CAAC,kBAAkB,CAAC;EACpB,CAAC,iBAAiB,CAAC;EACnB,CAAC,oBAAoB,CAAC;CACxB,GACE,aAAa,MAAM,SAAS,OAAO,OAAO;CAC5C,OAAO;AACT;;AAGA,SAAgB,aAAa,MAAc,OAAgB,SAAiB;CAC1E,IACE,UAAU,KAAA,MACT,OAAO,UAAU,YAChB,CAAC,OAAO,cAAc,KAAK,KAC3B,QAAQ,WACR,QAAQ,aAEV,MAAM,IAAI,WACR,GAAG,KAAK,8BAA8B,QAAQ,gBAChD;AACJ;;;;;;;ACvMA,SAAgB,KAAK,QAAgB;CACnC,MAAM,QAAQ,GAAG,OAAO;CACxB,MAAM,IAAI;EACR,SAAS,OAAe,GAAG,OAAO,MAAM;EACxC,mBAAmB,OAAe,GAAG,OAAO,YAAY;EAExD,aAAa,OAAe,GAAG,OAAO,WAAW;EACjD,oBAAoB,GAAG,OAAO;EAC9B,gBAAgB,SAAiB,GAAG,OAAO,eAAe;EAC1D,kBAAkB,WAAmB,GAAG,OAAO,iBAAiB;EAEhE,YAAY,UAAkB,GAAG,OAAO,cAAc;EAEtD,yBAAyB,OAAe,kBACtC,GAAG,OAAO,wBAAwB,MAAM,GAAG;EAC7C,iBAAiB,OAAe,aAC9B,GAAG,OAAO,gBAAgB,MAAM,GAAG;EAErC,UAAU,OAAe,WACvB,GAAG,OAAO,OAAO,MAAM,GAAG;EAC5B,gBAAgB,UAAkB,GAAG,OAAO,eAAe;EAE3D,gBAAgB,OAAe,WAC7B,GAAG,OAAO,cAAc,MAAM,GAAG;EAEnC,UAAU,WAAmB,GAAG,OAAO,OAAO;EAC9C,kBAAkB,UAAkB,GAAG,OAAO,iBAAiB;EAC/D,gBAAgB,UAAkB,GAAG,OAAO,eAAe;EAC3D,mBAAmB,UAAkB,GAAG,OAAO,eAAe;EAI9D,YAAY,OAAe,SACzB,GAAG,OAAO,SAAS,MAAM,GAAG;EAC9B,kBAAkB,UAAkB,GAAG,OAAO,iBAAiB;EAC/D,eAAe,GAAG,OAAO;EAEzB,eAAe,GAAG,OAAO;;EAEzB,OAAO;GACL,OAAO;GACP,SAAS,GAAG,MAAM;GAClB,SAAS,GAAG,MAAM;GAClB,aAAa,UAAkB,GAAG,MAAM,cAAc;GACtD,YAAY,UAAkB,GAAG,MAAM,aAAa;GACpD,aAAa,QAAgB,GAAG,MAAM,cAAc;;GAEpD,SAAS,GAAG,OAAO;EACrB;;;;;EAMA,WACE,OACA,KAMU;GACV,OAAO;IACL,EAAE,UAAU,KAAK;IACjB,EAAE,iBAAiB,KAAK;IACxB,EAAE,cAAc,KAAK;IACrB,EAAE,cAAc,KAAK;IACrB,EAAE,iBAAiB,KAAK;IACxB,EAAE,gBAAgB,KAAK;IACvB,GAAG,CAAC,GAAG,IAAI,cAAc,CAAC,CAAC,KAAK,OAC9B,EAAE,uBAAuB,OAAO,EAAE,CACpC;IACA,GAAG,IAAI,QAAQ,SAAS,OAAO,CAC7B,EAAE,QAAQ,OAAO,EAAE,GACnB,EAAE,cAAc,OAAO,EAAE,CAC3B,CAAC;IACD,GAAG,IAAI,QAAQ,IAAI,EAAE,OAAO;IAC5B,GAAG,IAAI,YAAY,KAAK,SAAS,EAAE,UAAU,OAAO,IAAI,CAAC;GAC3D;EACF;CACF;CACA,OAAO;AACT"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { i as resolveConfig, t as keys } from "./keys-CqKp_Amk.js";
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { EventSchema, FIRST_EVENT_SLOT, HookSchema, MessageId, StepSchema, WorkflowRunSchema, entityResolveData, eventIdToSlot, isLegacySpecVersion, mintedSpecVersion, parseQueueName, parseQueueRoute, reenqueueActiveRuns, requiresNewerWorld, slotToEventId, stripEventDataRefs, validateAttributeChanges } from "@workflow/world";
|
|
4
|
-
import { createAppRouteUrl, createWorkflowBaseUrl, createWorkflowUrl, globalSingleton } from "@workflow/utils";
|
|
4
|
+
import { QUEUE_AUTH_HEADER, createAppRouteUrl, createWorkflowBaseUrl, createWorkflowUrl, globalSingleton, isQueueAuthRefusal, isWellFormedQueueAuthKey, setQueueAuthKeySource } from "@workflow/utils";
|
|
5
5
|
import { encodeTime, monotonicFactory, ulid } from "ulid";
|
|
6
6
|
import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError } from "@workflow/errors";
|
|
7
7
|
import { Redis } from "ioredis";
|
|
@@ -182,6 +182,13 @@ function initialMessageKey(message) {
|
|
|
182
182
|
return typeof runId === "string" && runInput !== void 0 ? `start:${runId}` : void 0;
|
|
183
183
|
}
|
|
184
184
|
// src/world-queue.ts
|
|
185
|
+
const GENERATE_KEY = `node -e "console.log(crypto.randomBytes(32).toString('base64url'))"`;
|
|
186
|
+
/** @throws Unless the config or `WORKFLOW_QUEUE_AUTH_KEY` holds a well-formed key. */
|
|
187
|
+
function queueAuthKeyOf(config) {
|
|
188
|
+
const key = config.queueAuthKey ?? process.env.WORKFLOW_QUEUE_AUTH_KEY;
|
|
189
|
+
if (isWellFormedQueueAuthKey(key)) return key;
|
|
190
|
+
throw new Error(key ? `WORKFLOW_QUEUE_AUTH_KEY is not a queue auth key (43 characters, base64url). Generate one with: ${GENERATE_KEY}` : `Set WORKFLOW_QUEUE_AUTH_KEY to the same key on every process of the app, so the flow route accepts only this world's deliveries. Generate one with: ${GENERATE_KEY}`);
|
|
191
|
+
}
|
|
185
192
|
/** A response without a valid `{ timeoutSeconds }` acknowledges the message. */
|
|
186
193
|
function parseReschedule(text) {
|
|
187
194
|
try {
|
|
@@ -196,6 +203,12 @@ function parseReschedule(text) {
|
|
|
196
203
|
*/
|
|
197
204
|
function createWorldQueue(backend, config, options) {
|
|
198
205
|
const { debug } = options;
|
|
206
|
+
const queueAuthKey = queueAuthKeyOf(config);
|
|
207
|
+
const keySourceId = `@brickkit/redis-world:${crypto.randomUUID()}`;
|
|
208
|
+
setQueueAuthKeySource(keySourceId, {
|
|
209
|
+
current: () => queueAuthKey,
|
|
210
|
+
reload: async () => queueAuthKey
|
|
211
|
+
});
|
|
199
212
|
const generateMessageId = monotonicFactory();
|
|
200
213
|
const { httpTimeoutMs } = config;
|
|
201
214
|
const directHandlers = /* @__PURE__ */ new Map();
|
|
@@ -226,6 +239,7 @@ function createWorldQueue(backend, config, options) {
|
|
|
226
239
|
const { prefix } = parseQueueName(job.queueName);
|
|
227
240
|
const headers = new Headers(job.headers);
|
|
228
241
|
headers.set("content-type", "application/json");
|
|
242
|
+
headers.set(QUEUE_AUTH_HEADER, queueAuthKey);
|
|
229
243
|
headers.set("x-vqs-queue-name", job.queueName);
|
|
230
244
|
headers.set("x-vqs-message-id", job.messageId);
|
|
231
245
|
headers.set("x-vqs-message-attempt", String(job.attempt));
|
|
@@ -251,6 +265,7 @@ function createWorldQueue(backend, config, options) {
|
|
|
251
265
|
});
|
|
252
266
|
}
|
|
253
267
|
const text = await response.text();
|
|
268
|
+
if (isQueueAuthRefusal(response)) throw new Error("The flow route refused this delivery's queue key: give every process of the app the same WORKFLOW_QUEUE_AUTH_KEY");
|
|
254
269
|
if (!response.ok) throw new Error(`HTTP ${response.status}: ${text}`);
|
|
255
270
|
return parseReschedule(text);
|
|
256
271
|
};
|
|
@@ -283,6 +298,7 @@ function createWorldQueue(backend, config, options) {
|
|
|
283
298
|
};
|
|
284
299
|
};
|
|
285
300
|
return {
|
|
301
|
+
queueAuthMode: "loopback",
|
|
286
302
|
queue,
|
|
287
303
|
createQueueHandler,
|
|
288
304
|
getDeploymentId: async () => "redis",
|
|
@@ -297,6 +313,7 @@ function createWorldQueue(backend, config, options) {
|
|
|
297
313
|
async close() {
|
|
298
314
|
closed = true;
|
|
299
315
|
await backend.close();
|
|
316
|
+
setQueueAuthKeySource(keySourceId, void 0);
|
|
300
317
|
}
|
|
301
318
|
};
|
|
302
319
|
}
|
|
@@ -2449,4 +2466,4 @@ function createRedisBackedWorld(config, world) {
|
|
|
2449
2466
|
}
|
|
2450
2467
|
export { createDebugLogger as a, debug as i, createWorldQueue as n, retryDelayMs as r, createRedisBackedWorld as t };
|
|
2451
2468
|
|
|
2452
|
-
//# sourceMappingURL=world-
|
|
2469
|
+
//# sourceMappingURL=world-Cac56z4M.js.map
|