@brickkit/redis-world 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/setup.js +0 -2
- package/dist/cli-BEHfAdNK.js +65 -0
- package/dist/cli-BEHfAdNK.js.map +1 -0
- package/dist/cli.d.ts +10 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1 -49
- package/dist/index.d.ts +5 -119
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +45 -234
- package/dist/index.js.map +1 -1
- package/dist/internal.d.ts +5 -81
- package/dist/internal.d.ts.map +1 -1
- package/dist/internal.js +4 -3
- package/dist/keys-CqKp_Amk.js +104 -0
- package/dist/keys-CqKp_Amk.js.map +1 -0
- package/dist/{shared-Bhvu4aZA.js → world-783VJfKL.js} +354 -247
- package/dist/world-783VJfKL.js.map +1 -0
- package/dist/world-BDW_evdb.d.ts +242 -0
- package/dist/world-BDW_evdb.d.ts.map +1 -0
- package/package.json +1 -1
- package/dist/cli.js.map +0 -1
- package/dist/env-DMwDVTiZ.js +0 -18
- package/dist/env-DMwDVTiZ.js.map +0 -1
- package/dist/reaper-SOJDwnBw.d.ts +0 -45
- package/dist/reaper-SOJDwnBw.d.ts.map +0 -1
- package/dist/shared-Bhvu4aZA.js.map +0 -1
package/bin/setup.js
CHANGED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { i as resolveConfig, r as configFromEnv, t as keys } from "./keys-CqKp_Amk.js";
|
|
2
|
+
import { Redis } from "ioredis";
|
|
3
|
+
// src/internal/env.ts
|
|
4
|
+
/**
|
|
5
|
+
* Load an env file when present without making a `.env` file mandatory.
|
|
6
|
+
*
|
|
7
|
+
* Node 24 provides the parser natively, so published worlds do not need a
|
|
8
|
+
* runtime dependency solely to load local development configuration.
|
|
9
|
+
*/
|
|
10
|
+
function loadOptionalEnvFile(path) {
|
|
11
|
+
try {
|
|
12
|
+
if (path === void 0) process.loadEnvFile();
|
|
13
|
+
else process.loadEnvFile(path);
|
|
14
|
+
} catch (error) {
|
|
15
|
+
if (error.code !== "ENOENT") throw error;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
// src/cli.ts
|
|
19
|
+
async function setupRedis(queue = (keyPrefix) => ({
|
|
20
|
+
binName: "brickkit-redis-world-setup",
|
|
21
|
+
description: `Redis queue: ${keys(keyPrefix).queue.ready}, processing lists and delayed set`
|
|
22
|
+
})) {
|
|
23
|
+
loadOptionalEnvFile();
|
|
24
|
+
const { redis: connectionString, keyPrefix } = resolveConfig(configFromEnv());
|
|
25
|
+
const { binName, description, keyPattern } = queue(keyPrefix);
|
|
26
|
+
console.log("Setting up Redis for Workflow...");
|
|
27
|
+
console.log(`Connection: ${connectionString.replace(/^(\w+:\/\/)([^@]+)@/, "$1[redacted]@")}`);
|
|
28
|
+
console.log(`Key Prefix: ${keyPrefix}`);
|
|
29
|
+
let redis;
|
|
30
|
+
let exitCode = 0;
|
|
31
|
+
try {
|
|
32
|
+
redis = new Redis(connectionString);
|
|
33
|
+
await redis.ping();
|
|
34
|
+
console.log("Successfully connected to Redis!");
|
|
35
|
+
const version = (await redis.info("server")).match(/redis_version:([^\r\n]+)/)?.[1];
|
|
36
|
+
if (version) console.log(`Redis version: ${version}`);
|
|
37
|
+
const existingKeys = await redis.keys(`${keyPrefix}*`);
|
|
38
|
+
if (keyPattern) existingKeys.push(...await redis.keys(keyPattern));
|
|
39
|
+
if (existingKeys.length > 0) {
|
|
40
|
+
console.log(`\nWarning: found ${existingKeys.length} existing workflow keys with prefix "${keyPrefix}"`);
|
|
41
|
+
console.log("\nTo flush existing workflow data, run with FLUSH=true environment variable:");
|
|
42
|
+
console.log(` FLUSH=true npx ${binName}`);
|
|
43
|
+
if (process.env.FLUSH === "true") {
|
|
44
|
+
console.log("\nFlushing existing workflow data...");
|
|
45
|
+
const pipeline = redis.pipeline();
|
|
46
|
+
for (const key of existingKeys) pipeline.del(key);
|
|
47
|
+
await pipeline.exec();
|
|
48
|
+
console.log(`Deleted ${existingKeys.length} keys`);
|
|
49
|
+
}
|
|
50
|
+
} else console.log(`\nNo existing workflow data found (clean slate)`);
|
|
51
|
+
console.log("\nRedis is ready for Workflow DevKit!");
|
|
52
|
+
console.log("\nThe following Redis data structures will be used:");
|
|
53
|
+
console.log(` - Runs, event logs, steps, hooks and streams: ${keyPrefix}*`);
|
|
54
|
+
console.log(` - ${description}`);
|
|
55
|
+
} catch (error) {
|
|
56
|
+
exitCode = 1;
|
|
57
|
+
console.error("Failed to setup Redis:", error);
|
|
58
|
+
} finally {
|
|
59
|
+
await redis?.quit().catch(() => {});
|
|
60
|
+
}
|
|
61
|
+
process.exit(exitCode);
|
|
62
|
+
}
|
|
63
|
+
export { setupRedis as t };
|
|
64
|
+
|
|
65
|
+
//# sourceMappingURL=cli-BEHfAdNK.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli-BEHfAdNK.js","names":[],"sources":["../src/internal/env.ts","../src/cli.ts"],"sourcesContent":["/**\n * Load an env file when present without making a `.env` file mandatory.\n *\n * Node 24 provides the parser natively, so published worlds do not need a\n * runtime dependency solely to load local development configuration.\n */\nexport function loadOptionalEnvFile(path?: string): void {\n try {\n if (path === undefined) {\n process.loadEnvFile();\n } else {\n process.loadEnvFile(path);\n }\n } catch (error: unknown) {\n if ((error as NodeJS.ErrnoException).code !== \"ENOENT\") {\n throw error;\n }\n }\n}\n","import { Redis } from \"ioredis\";\nimport { configFromEnv, resolveConfig } from \"./config.js\";\nimport { loadOptionalEnvFile } from \"./internal/env.js\";\nimport { keys } from \"./keys.js\";\n\n/** What the setup CLI says about, and flushes for, a world's queue. */\nexport interface SetupQueue {\n /** The bin users run, for the FLUSH hint. */\n binName: string;\n /** One line describing the queue's keys. */\n description: string;\n /** Queue keys outside `keyPrefix`, flushed with it. */\n keyPattern?: string;\n}\n\nasync function setupRedis(\n queue: (keyPrefix: string) => SetupQueue = (keyPrefix) => ({\n binName: \"brickkit-redis-world-setup\",\n description: `Redis queue: ${keys(keyPrefix).queue.ready}, processing lists and delayed set`,\n }),\n) {\n loadOptionalEnvFile();\n\n const { redis: connectionString, keyPrefix } = resolveConfig(configFromEnv());\n const { binName, description, keyPattern } = queue(keyPrefix);\n\n console.log(\"Setting up Redis for Workflow...\");\n console.log(\n `Connection: ${connectionString.replace(/^(\\w+:\\/\\/)([^@]+)@/, \"$1[redacted]@\")}`,\n );\n console.log(`Key Prefix: ${keyPrefix}`);\n\n let redis: Redis | undefined;\n let exitCode = 0;\n\n try {\n redis = new Redis(connectionString);\n\n await redis.ping();\n console.log(\"Successfully connected to Redis!\");\n\n const info = await redis.info(\"server\");\n const version = info.match(/redis_version:([^\\r\\n]+)/)?.[1];\n if (version) {\n console.log(`Redis version: ${version}`);\n }\n\n const existingKeys = await redis.keys(`${keyPrefix}*`);\n if (keyPattern) existingKeys.push(...(await redis.keys(keyPattern)));\n if (existingKeys.length > 0) {\n console.log(\n `\\nWarning: found ${existingKeys.length} existing workflow keys with prefix \"${keyPrefix}\"`,\n );\n console.log(\n \"\\nTo flush existing workflow data, run with FLUSH=true environment variable:\",\n );\n console.log(` FLUSH=true npx ${binName}`);\n\n if (process.env.FLUSH === \"true\") {\n console.log(\"\\nFlushing existing workflow data...\");\n const pipeline = redis.pipeline();\n for (const key of existingKeys) {\n pipeline.del(key);\n }\n await pipeline.exec();\n console.log(`Deleted ${existingKeys.length} keys`);\n }\n } else {\n console.log(`\\nNo existing workflow data found (clean slate)`);\n }\n\n console.log(\"\\nRedis is ready for Workflow DevKit!\");\n console.log(\"\\nThe following Redis data structures will be used:\");\n console.log(\n ` - Runs, event logs, steps, hooks and streams: ${keyPrefix}*`,\n );\n console.log(` - ${description}`);\n } catch (error) {\n exitCode = 1;\n console.error(\"Failed to setup Redis:\", error);\n } finally {\n await redis?.quit().catch(() => {});\n }\n\n process.exit(exitCode);\n}\n\nexport { setupRedis };\n"],"mappings":";;;;;;;;;AAMA,SAAgB,oBAAoB,MAAqB;CACvD,IAAI;EACF,IAAI,SAAS,KAAA,GACX,QAAQ,YAAY;OAEpB,QAAQ,YAAY,IAAI;CAE5B,SAAS,OAAgB;EACvB,IAAK,MAAgC,SAAS,UAC5C,MAAM;CAEV;AACF;;ACHA,eAAe,WACb,SAA4C,eAAe;CACzD,SAAS;CACT,aAAa,gBAAgB,KAAK,SAAS,CAAC,CAAC,MAAM,MAAM;AAC3D,IACA;CACA,oBAAoB;CAEpB,MAAM,EAAE,OAAO,kBAAkB,cAAc,cAAc,cAAc,CAAC;CAC5E,MAAM,EAAE,SAAS,aAAa,eAAe,MAAM,SAAS;CAE5D,QAAQ,IAAI,kCAAkC;CAC9C,QAAQ,IACN,eAAe,iBAAiB,QAAQ,uBAAuB,eAAe,GAChF;CACA,QAAQ,IAAI,eAAe,WAAW;CAEtC,IAAI;CACJ,IAAI,WAAW;CAEf,IAAI;EACF,QAAQ,IAAI,MAAM,gBAAgB;EAElC,MAAM,MAAM,KAAK;EACjB,QAAQ,IAAI,kCAAkC;EAG9C,MAAM,WAAU,MADG,MAAM,KAAK,QAAQ,EAAA,CACjB,MAAM,0BAA0B,CAAC,GAAG;EACzD,IAAI,SACF,QAAQ,IAAI,kBAAkB,SAAS;EAGzC,MAAM,eAAe,MAAM,MAAM,KAAK,GAAG,UAAU,EAAE;EACrD,IAAI,YAAY,aAAa,KAAK,GAAI,MAAM,MAAM,KAAK,UAAU,CAAE;EACnE,IAAI,aAAa,SAAS,GAAG;GAC3B,QAAQ,IACN,oBAAoB,aAAa,OAAO,uCAAuC,UAAU,EAC3F;GACA,QAAQ,IACN,8EACF;GACA,QAAQ,IAAI,oBAAoB,SAAS;GAEzC,IAAI,QAAQ,IAAI,UAAU,QAAQ;IAChC,QAAQ,IAAI,sCAAsC;IAClD,MAAM,WAAW,MAAM,SAAS;IAChC,KAAK,MAAM,OAAO,cAChB,SAAS,IAAI,GAAG;IAElB,MAAM,SAAS,KAAK;IACpB,QAAQ,IAAI,WAAW,aAAa,OAAO,MAAM;GACnD;EACF,OACE,QAAQ,IAAI,iDAAiD;EAG/D,QAAQ,IAAI,uCAAuC;EACnD,QAAQ,IAAI,qDAAqD;EACjE,QAAQ,IACN,mDAAmD,UAAU,EAC/D;EACA,QAAQ,IAAI,OAAO,aAAa;CAClC,SAAS,OAAO;EACd,WAAW;EACX,QAAQ,MAAM,0BAA0B,KAAK;CAC/C,UAAU;EACR,MAAM,OAAO,KAAK,CAAC,CAAC,YAAY,CAAC,CAAC;CACpC;CAEA,QAAQ,KAAK,QAAQ;AACvB"}
|
package/dist/cli.d.ts
CHANGED
|
@@ -1,3 +1,12 @@
|
|
|
1
|
-
|
|
1
|
+
/** What the setup CLI says about, and flushes for, a world's queue. */
|
|
2
|
+
export interface SetupQueue {
|
|
3
|
+
/** The bin users run, for the FLUSH hint. */
|
|
4
|
+
binName: string;
|
|
5
|
+
/** One line describing the queue's keys. */
|
|
6
|
+
description: string;
|
|
7
|
+
/** Queue keys outside `keyPrefix`, flushed with it. */
|
|
8
|
+
keyPattern?: string;
|
|
9
|
+
}
|
|
10
|
+
declare function setupRedis(queue?: (keyPrefix: string) => SetupQueue): Promise<void>;
|
|
2
11
|
export { setupRedis };
|
|
3
12
|
//# sourceMappingURL=cli.d.ts.map
|
package/dist/cli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.d.ts","names":[],"sources":["../src/cli.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"cli.d.ts","names":[],"sources":["../src/cli.ts"],"mappings":";iBAMiB;;EAEf;;EAEA;;EAEA;;iBAGa,WACb,SAAQ,sBAAsB,aAG5B"}
|
package/dist/cli.js
CHANGED
|
@@ -1,49 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { Redis } from "ioredis";
|
|
3
|
-
// src/cli.ts
|
|
4
|
-
async function setupRedis() {
|
|
5
|
-
loadOptionalEnvFile();
|
|
6
|
-
const connectionString = process.env.WORKFLOW_REDIS_URL || process.env.REDIS_URL || "redis://localhost:6379";
|
|
7
|
-
const keyPrefix = process.env.WORKFLOW_REDIS_KEY_PREFIX || "workflow:";
|
|
8
|
-
console.log("Setting up Redis for Workflow...");
|
|
9
|
-
console.log(`Connection: ${connectionString.replace(/^(\w+:\/\/)([^@]+)@/, "$1[redacted]@")}`);
|
|
10
|
-
console.log(`Key Prefix: ${keyPrefix}`);
|
|
11
|
-
let redis;
|
|
12
|
-
let exitCode = 0;
|
|
13
|
-
try {
|
|
14
|
-
redis = new Redis(connectionString);
|
|
15
|
-
await redis.ping();
|
|
16
|
-
console.log("Successfully connected to Redis!");
|
|
17
|
-
const version = (await redis.info("server")).match(/redis_version:([^\r\n]+)/)?.[1];
|
|
18
|
-
if (version) console.log(`Redis version: ${version}`);
|
|
19
|
-
const existingKeys = await redis.keys(`${keyPrefix}*`);
|
|
20
|
-
if (existingKeys.length > 0) {
|
|
21
|
-
console.log(`\nWarning: found ${existingKeys.length} existing workflow keys with prefix "${keyPrefix}"`);
|
|
22
|
-
console.log("\nTo flush existing workflow data, run with FLUSH=true environment variable:");
|
|
23
|
-
console.log(" FLUSH=true npx brickkit-redis-world-setup");
|
|
24
|
-
if (process.env.FLUSH === "true") {
|
|
25
|
-
console.log("\nFlushing existing workflow data...");
|
|
26
|
-
const pipeline = redis.pipeline();
|
|
27
|
-
for (const key of existingKeys) pipeline.del(key);
|
|
28
|
-
await pipeline.exec();
|
|
29
|
-
console.log(`Deleted ${existingKeys.length} keys`);
|
|
30
|
-
}
|
|
31
|
-
} else console.log(`\nNo existing workflow data found (clean slate)`);
|
|
32
|
-
console.log("\nRedis is ready for Workflow DevKit!");
|
|
33
|
-
console.log("\nThe following Redis data structures will be used:");
|
|
34
|
-
console.log(` - Hashes: ${keyPrefix}run:*, ${keyPrefix}step:*, ${keyPrefix}event:*, ${keyPrefix}hook:*`);
|
|
35
|
-
console.log(` - Sorted Sets: ${keyPrefix}runs:*, ${keyPrefix}steps:*, ${keyPrefix}events:*`);
|
|
36
|
-
console.log(` - Streams: ${keyPrefix}stream:*`);
|
|
37
|
-
console.log(` - Redis queue: ${keyPrefix}queue:flows, processing lists and delayed set`);
|
|
38
|
-
} catch (error) {
|
|
39
|
-
exitCode = 1;
|
|
40
|
-
console.error("Failed to setup Redis:", error);
|
|
41
|
-
} finally {
|
|
42
|
-
await redis?.quit().catch(() => {});
|
|
43
|
-
}
|
|
44
|
-
process.exit(exitCode);
|
|
45
|
-
}
|
|
46
|
-
if (import.meta.url === `file://${process.argv[1]}`) setupRedis();
|
|
47
|
-
export { setupRedis };
|
|
48
|
-
|
|
49
|
-
//# sourceMappingURL=cli.js.map
|
|
1
|
+
export { t as setupRedis } from "./cli-BEHfAdNK.js";
|
package/dist/index.d.ts
CHANGED
|
@@ -1,121 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
* Redis connection URL or connection options
|
|
7
|
-
* Examples:
|
|
8
|
-
* - 'redis://localhost:6379'
|
|
9
|
-
* - 'rediss://username:password@host:6380/0'
|
|
10
|
-
* - { host: 'localhost', port: 6379 }
|
|
11
|
-
*/
|
|
12
|
-
redis: string | RedisOptions;
|
|
13
|
-
/**
|
|
14
|
-
* Number of concurrent workers processing jobs
|
|
15
|
-
* Default: 10
|
|
16
|
-
*/
|
|
17
|
-
queueConcurrency?: number;
|
|
18
|
-
/** Worker liveness TTL in milliseconds. Default 90000; minimum 300.
|
|
19
|
-
* Set above the longest event-loop pause. Renewed every TTL / 3. */
|
|
20
|
-
heartbeatTtlMs?: number;
|
|
21
|
-
/**
|
|
22
|
-
* Optional key prefix for all Redis keys
|
|
23
|
-
* Useful for multi-tenancy or namespace isolation
|
|
24
|
-
* Default: 'workflow:'
|
|
25
|
-
*/
|
|
26
|
-
keyPrefix?: string;
|
|
27
|
-
/**
|
|
28
|
-
* Delivery budget before retries continue at maxBackoffMs. Messages are never silently retired.
|
|
29
|
-
*
|
|
30
|
-
* Keep it above the runtime's own delivery ceiling (48). The runtime records a
|
|
31
|
-
* run as failed (`MAX_DELIVERIES_EXCEEDED`) on delivery 49; a job retired
|
|
32
|
-
* earlier strands its run in `running`, with nothing to redeliver it until a
|
|
33
|
-
* process calls `start()` again.
|
|
34
|
-
* Default: 73
|
|
35
|
-
*/
|
|
36
|
-
maxAttempts?: number;
|
|
37
|
-
/**
|
|
38
|
-
* Backoff strategy for retries: 'fixed' replays at the same interval,
|
|
39
|
-
* 'exponential' doubles the delay on each attempt.
|
|
40
|
-
* Default: 'exponential'
|
|
41
|
-
*/
|
|
42
|
-
backoffType?: "fixed" | "exponential";
|
|
43
|
-
/**
|
|
44
|
-
* Base delay for backoff (ms). For exponential backoff this is multiplied
|
|
45
|
-
* by 2^(attempt - 1), up to `maxBackoffMs`.
|
|
46
|
-
* Default: 1000
|
|
47
|
-
*/
|
|
48
|
-
backoffDelayMs?: number;
|
|
49
|
-
/**
|
|
50
|
-
* The longest a retry is delayed (ms). With the default number of attempts an
|
|
51
|
-
* uncapped exponential backoff would wait hours between late attempts.
|
|
52
|
-
* Default: 30000 (30s)
|
|
53
|
-
*/
|
|
54
|
-
maxBackoffMs?: number;
|
|
55
|
-
/**
|
|
56
|
-
* Base URL the worker uses to dispatch jobs back to the user's HTTP
|
|
57
|
-
* server, which must serve `world.createQueueHandler(...)` at
|
|
58
|
-
* `/.well-known/workflow/v1/flow` (steps travel on the same route). A route
|
|
59
|
-
* registered with `world.registerRoute`, or an in-process handler registered
|
|
60
|
-
* with `world.registerHandler`, takes precedence.
|
|
61
|
-
* Default: process.env.WORKFLOW_BASE_URL || `http://localhost:${process.env.PORT ?? 3000}`
|
|
62
|
-
*/
|
|
63
|
-
baseUrl?: string;
|
|
64
|
-
/**
|
|
65
|
-
* Per-job HTTP request timeout (ms). Long-running workflow steps may need a
|
|
66
|
-
* higher value.
|
|
67
|
-
* Default: 300_000 (5 minutes)
|
|
68
|
-
*/
|
|
69
|
-
httpTimeoutMs?: number;
|
|
70
|
-
/**
|
|
71
|
-
* Optional TTL (ms) for queue deduplication when `idempotencyKey`
|
|
72
|
-
* is supplied to `queue()`. By default no TTL is applied: the dedup key
|
|
73
|
-
* lives until the job is acknowledged, so duplicate enqueues are dropped
|
|
74
|
-
* for the entire lifetime of the pending/delayed job (the semantics the
|
|
75
|
-
* workflow runtime relies on when re-enqueuing pending steps on replay).
|
|
76
|
-
* Only set this if you need the dedup guard to expire on wall-clock time;
|
|
77
|
-
* it must be much larger than the longest expected queue delay/backlog,
|
|
78
|
-
* otherwise duplicate step executions become possible.
|
|
79
|
-
* Default: undefined (dedup key released on job completion)
|
|
80
|
-
*/
|
|
81
|
-
idempotencyTtlMs?: number;
|
|
82
|
-
/** Maximum events a run may accumulate, reported as `EventResult.maxEvents`.
|
|
83
|
-
* Default: `WORKFLOW_MAX_EVENTS` || 25_000 */
|
|
84
|
-
maxEventsPerRun?: number;
|
|
85
|
-
/**
|
|
86
|
-
* Batch commands issued in the same event-loop tick into one write. A win
|
|
87
|
-
* once several runs are in flight (~1.75x at 256 concurrent) and a marginal
|
|
88
|
-
* loss for a single serial caller.
|
|
89
|
-
* Default: true
|
|
90
|
-
*/
|
|
91
|
-
enableAutoPipelining?: boolean;
|
|
92
|
-
/**
|
|
93
|
-
* How long finished runs are kept, per terminal status: `age` in seconds,
|
|
94
|
-
* `count` of newest runs to keep. A reaper on each worker deletes the run, its event log, steps, hooks
|
|
95
|
-
* and streams once it passes. A status without a policy is kept forever.
|
|
96
|
-
* Default: undefined (nothing expires)
|
|
97
|
-
*
|
|
98
|
-
* @example
|
|
99
|
-
* retention: {
|
|
100
|
-
* completed: { age: 24 * 60 * 60, count: 1000 },
|
|
101
|
-
* failed: { age: 7 * 24 * 60 * 60 },
|
|
102
|
-
* }
|
|
103
|
-
*/
|
|
104
|
-
retention?: RunRetention;
|
|
105
|
-
}
|
|
106
|
-
interface QueueStats {
|
|
107
|
-
waiting: number;
|
|
108
|
-
active: number;
|
|
109
|
-
delayed: number;
|
|
110
|
-
/** Completed and failed delivery records are not retained. */
|
|
111
|
-
completed: number;
|
|
112
|
-
failed: number;
|
|
113
|
-
}
|
|
114
|
-
export declare function createWorld(config?: RedisWorldConfig): World & {
|
|
115
|
-
start(): Promise<void>;
|
|
116
|
-
getQueueStats(): Promise<QueueStats>;
|
|
117
|
-
/** Run one retention sweep now; returns the runs deleted. 0 without `retention`. */
|
|
118
|
-
sweepRetention(): Promise<number>;
|
|
119
|
-
};
|
|
1
|
+
import { _ as RunRetention, a as QueueStats, f as RedisWorldConfig, g as KeepRuns, t as RedisBackedWorld } from "./world-BDW_evdb.js";
|
|
2
|
+
import { Redis } from "ioredis";
|
|
3
|
+
/** `config` is laid over `configFromEnv()`: an option left out comes from the
|
|
4
|
+
* `WORKFLOW_REDIS_*` variables, then the defaults. */
|
|
5
|
+
export declare function createWorld(config?: Partial<RedisWorldConfig>): RedisBackedWorld;
|
|
120
6
|
export type { KeepRuns, QueueStats, RedisWorldConfig, RunRetention };
|
|
121
7
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","names":[],"sources":["../src/
|
|
1
|
+
{"version":3,"file":"index.d.ts","names":[],"sources":["../src/index.ts"],"mappings":";;;;wBAOgB,YACd,SAAQ,QAAQ,oBACf"}
|
package/dist/index.js
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { WorkflowRunNotFoundError } from "@workflow/errors";
|
|
4
|
-
import { Redis } from "ioredis";
|
|
1
|
+
import { n as checkInteger, r as configFromEnv, t as keys } from "./keys-CqKp_Amk.js";
|
|
2
|
+
import { i as debug, n as createWorldQueue, r as retryDelayMs, t as createRedisBackedWorld } from "./world-783VJfKL.js";
|
|
5
3
|
import { randomUUID } from "node:crypto";
|
|
6
4
|
import { setTimeout } from "node:timers/promises";
|
|
7
|
-
import {
|
|
8
|
-
import { monotonicFactory } from "ulid";
|
|
5
|
+
import { MessageId, ValidQueueName } from "@workflow/world";
|
|
9
6
|
// src/queue.ts
|
|
10
7
|
const ENQUEUE = `
|
|
11
8
|
if ARGV[1] ~= '' then
|
|
@@ -78,7 +75,7 @@ const REGISTER = `
|
|
|
78
75
|
const STATS = `
|
|
79
76
|
local active = 0
|
|
80
77
|
for _, owner in ipairs(redis.call('SMEMBERS', KEYS[3])) do
|
|
81
|
-
active = active + redis.call('LLEN', ARGV[1] ..
|
|
78
|
+
active = active + redis.call('LLEN', ARGV[1] .. owner)
|
|
82
79
|
end
|
|
83
80
|
return {redis.call('LLEN', KEYS[1]), active, redis.call('ZCARD', KEYS[2])}
|
|
84
81
|
`;
|
|
@@ -102,75 +99,12 @@ function parseEnvelope(raw) {
|
|
|
102
99
|
..."route" in value && typeof value.route === "string" && { route: value.route }
|
|
103
100
|
};
|
|
104
101
|
}
|
|
105
|
-
function serializeQueueMessage(message) {
|
|
106
|
-
return hasBinary(message) ? JSON.stringify(message, uint8ArrayReplacer) : JSON.stringify(message);
|
|
107
|
-
}
|
|
108
|
-
function queueMessageReviver(_key, value) {
|
|
109
|
-
if (value !== null && typeof value === "object") {
|
|
110
|
-
const tagged = value;
|
|
111
|
-
if (tagged.__type === "Uint8Array" && typeof tagged.data === "string") return new Uint8Array(Buffer.from(tagged.data, "base64"));
|
|
112
|
-
}
|
|
113
|
-
return value;
|
|
114
|
-
}
|
|
115
|
-
function deserializeQueueMessage(text) {
|
|
116
|
-
return text.includes("\"__type\"") ? JSON.parse(text, queueMessageReviver) : JSON.parse(text);
|
|
117
|
-
}
|
|
118
|
-
function resolveBaseUrl(config) {
|
|
119
|
-
return config.baseUrl || process.env.WORKFLOW_BASE_URL || `http://localhost:${process.env.PORT ?? "3000"}`;
|
|
120
|
-
}
|
|
121
102
|
function createQueue(redis, config, options = {}) {
|
|
122
|
-
const
|
|
123
|
-
const
|
|
124
|
-
const ready = root;
|
|
125
|
-
const delayed = `${root}:delayed`;
|
|
126
|
-
const owners = `${root}:workers`;
|
|
127
|
-
const processing = (owner) => `${root}:processing:${owner}`;
|
|
128
|
-
const heartbeat = (owner) => `${root}:heartbeat:${owner}`;
|
|
103
|
+
const queueKeys = keys(config.keyPrefix).queue;
|
|
104
|
+
const { ready, delayed, workers: owners, processing, heartbeat } = queueKeys;
|
|
129
105
|
const heartbeatTtlMs = config.heartbeatTtlMs ?? 9e4;
|
|
130
|
-
|
|
131
|
-
const maxAttempts = config
|
|
132
|
-
const maxBackoffMs = config.maxBackoffMs ?? 3e4;
|
|
133
|
-
const backoffDelayMs = config.backoffDelayMs ?? 1e3;
|
|
134
|
-
const httpTimeoutMs = config.httpTimeoutMs ?? 3e5;
|
|
135
|
-
for (const [name, value, minimum] of [
|
|
136
|
-
[
|
|
137
|
-
"heartbeatTtlMs",
|
|
138
|
-
heartbeatTtlMs,
|
|
139
|
-
300
|
|
140
|
-
],
|
|
141
|
-
[
|
|
142
|
-
"queueConcurrency",
|
|
143
|
-
concurrency,
|
|
144
|
-
1
|
|
145
|
-
],
|
|
146
|
-
[
|
|
147
|
-
"maxAttempts",
|
|
148
|
-
maxAttempts,
|
|
149
|
-
49
|
|
150
|
-
],
|
|
151
|
-
[
|
|
152
|
-
"maxBackoffMs",
|
|
153
|
-
maxBackoffMs,
|
|
154
|
-
1
|
|
155
|
-
],
|
|
156
|
-
[
|
|
157
|
-
"backoffDelayMs",
|
|
158
|
-
backoffDelayMs,
|
|
159
|
-
0
|
|
160
|
-
],
|
|
161
|
-
[
|
|
162
|
-
"httpTimeoutMs",
|
|
163
|
-
httpTimeoutMs,
|
|
164
|
-
1
|
|
165
|
-
],
|
|
166
|
-
...config.idempotencyTtlMs === void 0 ? [] : [[
|
|
167
|
-
"idempotencyTtlMs",
|
|
168
|
-
config.idempotencyTtlMs,
|
|
169
|
-
1
|
|
170
|
-
]]
|
|
171
|
-
]) if (typeof value !== "number" || typeof minimum !== "number" || !Number.isSafeInteger(value) || value < minimum || value > 2147483647) throw new RangeError(`${name} must be an integer between ${minimum} and 2147483647`);
|
|
172
|
-
const directHandlers = /* @__PURE__ */ new Map();
|
|
173
|
-
const routes = /* @__PURE__ */ new Map();
|
|
106
|
+
checkInteger("heartbeatTtlMs", heartbeatTtlMs, 300);
|
|
107
|
+
const { queueConcurrency: concurrency, maxAttempts, maxBackoffMs } = config;
|
|
174
108
|
let closed = false;
|
|
175
109
|
let startPromise;
|
|
176
110
|
let loop;
|
|
@@ -179,112 +113,35 @@ function createQueue(redis, config, options = {}) {
|
|
|
179
113
|
const lostOwners = /* @__PURE__ */ new Set();
|
|
180
114
|
const abort = new AbortController();
|
|
181
115
|
const sleep = () => setTimeout(50, void 0, { signal: abort.signal }).catch(() => {});
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
const { prefix } = parseQueueName(queueName);
|
|
185
|
-
const messageId = MessageId.parse(`msg_${generateMessageId()}`);
|
|
186
|
-
const idempotencyKey = opts?.idempotencyKey ?? initialMessageKey(message);
|
|
187
|
-
if (!opts?.idempotencyKey && idempotencyKey && await options.runHasStarted?.(message.runId)) {
|
|
188
|
-
debug("run already started, not queueing its first message again", { runId: message.runId });
|
|
189
|
-
return { messageId };
|
|
190
|
-
}
|
|
191
|
-
const dedupKey = idempotencyKey ? `${root}:idempotent:${idempotencyKey}` : "";
|
|
116
|
+
async function enqueue(message, opts) {
|
|
117
|
+
const dedupKey = opts.idempotencyKey ? queueKeys.idempotent(opts.idempotencyKey) : "";
|
|
192
118
|
const envelope = {
|
|
193
|
-
messageId,
|
|
194
|
-
queueName,
|
|
195
|
-
message:
|
|
119
|
+
messageId: message.messageId,
|
|
120
|
+
queueName: message.queueName,
|
|
121
|
+
message: message.message,
|
|
196
122
|
attempt: 0,
|
|
197
123
|
deliveryCount: 0,
|
|
198
124
|
createdAt: Date.now(),
|
|
199
125
|
dedupKey,
|
|
200
|
-
headers:
|
|
201
|
-
...
|
|
126
|
+
headers: message.headers ?? {},
|
|
127
|
+
...message.route && { route: message.route }
|
|
202
128
|
};
|
|
203
|
-
const delayMs = Math.max(0, (opts
|
|
129
|
+
const delayMs = Math.max(0, (opts.delaySeconds ?? 0) * 1e3);
|
|
204
130
|
if (!Number.isFinite(delayMs)) throw new RangeError("delaySeconds must be finite");
|
|
205
|
-
const result = await redis.eval(ENQUEUE, 3, dedupKey || ready, ready, delayed, dedupKey, messageId, config.idempotencyTtlMs ?? 0, JSON.stringify(envelope), delayMs > 0 ? Date.now() + delayMs : 0);
|
|
206
|
-
return
|
|
207
|
-
};
|
|
208
|
-
function parseReschedule(text) {
|
|
209
|
-
try {
|
|
210
|
-
const seconds = JSON.parse(text)?.timeoutSeconds;
|
|
211
|
-
if (typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0) return seconds;
|
|
212
|
-
} catch {}
|
|
131
|
+
const result = await redis.eval(ENQUEUE, 3, dedupKey || ready, ready, delayed, dedupKey, message.messageId, config.idempotencyTtlMs ?? 0, JSON.stringify(envelope), delayMs > 0 ? Date.now() + delayMs : 0);
|
|
132
|
+
return MessageId.parse(result);
|
|
213
133
|
}
|
|
214
|
-
async function
|
|
215
|
-
const { prefix: queuePrefix } = parseQueueName(job.queueName);
|
|
216
|
-
const messageId = job.messageId;
|
|
217
|
-
const headers = new Headers(job.headers);
|
|
218
|
-
headers.set("content-type", "application/json");
|
|
219
|
-
headers.set("x-vqs-queue-name", job.queueName);
|
|
220
|
-
headers.set("x-vqs-message-id", messageId);
|
|
221
|
-
headers.set("x-vqs-message-attempt", String(job.attempt));
|
|
222
|
-
headers.set("x-vqs-message-delivery-count", String(job.deliveryCount));
|
|
223
|
-
headers.set("x-vqs-message-created-at", new Date(job.createdAt).toISOString());
|
|
224
|
-
const deliveryRoute = routes.get(queuePrefix) ?? job.route;
|
|
225
|
-
const directHandler = directHandlers.get(queuePrefix);
|
|
226
|
-
let response;
|
|
227
|
-
if (directHandler) {
|
|
228
|
-
const localBase = createWorkflowBaseUrl("http://localhost");
|
|
229
|
-
response = await directHandler(new Request(deliveryRoute ? createAppRouteUrl(localBase, deliveryRoute) : createWorkflowUrl(localBase, { type: "flow" }), {
|
|
230
|
-
method: "POST",
|
|
231
|
-
headers,
|
|
232
|
-
body: job.message
|
|
233
|
-
}));
|
|
234
|
-
} else {
|
|
235
|
-
const baseUrl = resolveBaseUrl(config);
|
|
236
|
-
response = await fetch(deliveryRoute ? createAppRouteUrl(baseUrl, deliveryRoute) : createWorkflowUrl(baseUrl, { type: "flow" }), {
|
|
237
|
-
method: "POST",
|
|
238
|
-
headers,
|
|
239
|
-
body: job.message,
|
|
240
|
-
signal: AbortSignal.timeout(httpTimeoutMs)
|
|
241
|
-
});
|
|
242
|
-
}
|
|
243
|
-
const text = await response.text();
|
|
244
|
-
if (!response.ok) throw new Error(`HTTP ${response.status}: ${text}`);
|
|
245
|
-
return parseReschedule(text);
|
|
246
|
-
}
|
|
247
|
-
const createQueueHandler = (queueNamePrefix, handler) => {
|
|
248
|
-
return async (req) => {
|
|
249
|
-
const queueName = req.headers.get("x-vqs-queue-name");
|
|
250
|
-
const messageId = req.headers.get("x-vqs-message-id");
|
|
251
|
-
const attemptStr = req.headers.get("x-vqs-message-attempt");
|
|
252
|
-
const deliveryCountStr = req.headers.get("x-vqs-message-delivery-count");
|
|
253
|
-
const createdAtStr = req.headers.get("x-vqs-message-created-at");
|
|
254
|
-
if (!queueName || !messageId || !attemptStr || !req.body) return Response.json({ error: "Missing required headers or body" }, { status: 400 });
|
|
255
|
-
if (!queueName.startsWith(queueNamePrefix)) return Response.json({ error: "Unhandled queue" }, { status: 400 });
|
|
256
|
-
const attempt = Number.parseInt(attemptStr, 10);
|
|
257
|
-
const deliveryCount = deliveryCountStr ? Number.parseInt(deliveryCountStr, 10) : attempt;
|
|
258
|
-
const createdAt = createdAtStr ? new Date(createdAtStr) : void 0;
|
|
259
|
-
try {
|
|
260
|
-
const timeoutSeconds = (await handler(deserializeQueueMessage(await req.text()), {
|
|
261
|
-
attempt,
|
|
262
|
-
deliveryCount,
|
|
263
|
-
...createdAt && !Number.isNaN(createdAt.getTime()) && { createdAt },
|
|
264
|
-
queueName,
|
|
265
|
-
messageId
|
|
266
|
-
}))?.timeoutSeconds;
|
|
267
|
-
if (typeof timeoutSeconds === "number" && Number.isFinite(timeoutSeconds)) return Response.json({ timeoutSeconds: Math.max(0, timeoutSeconds) });
|
|
268
|
-
return Response.json({ ok: true });
|
|
269
|
-
} catch (error) {
|
|
270
|
-
debug("queue handler error:", error);
|
|
271
|
-
return Response.json({ error: String(error) }, { status: 500 });
|
|
272
|
-
}
|
|
273
|
-
};
|
|
274
|
-
};
|
|
275
|
-
const getDeploymentId = async () => "redis";
|
|
276
|
-
async function settle(raw, claimOwner) {
|
|
134
|
+
async function settle(raw, claimOwner, deliver) {
|
|
277
135
|
try {
|
|
278
136
|
const job = parseEnvelope(raw);
|
|
279
137
|
let seconds;
|
|
280
138
|
let failed = false;
|
|
281
139
|
try {
|
|
282
|
-
seconds = await
|
|
140
|
+
seconds = await deliver(job);
|
|
283
141
|
} catch (error) {
|
|
284
142
|
failed = true;
|
|
285
143
|
debug(`Delivery ${job.messageId} attempt ${job.attempt} failed:`, error);
|
|
286
|
-
|
|
287
|
-
seconds = (job.attempt >= maxAttempts ? maxBackoffMs : Math.min(maxBackoffMs, curve)) / 1e3;
|
|
144
|
+
seconds = (job.attempt >= maxAttempts ? maxBackoffMs : retryDelayMs(job.attempt, config)) / 1e3;
|
|
288
145
|
if (job.attempt === maxAttempts) console.error(`Queue message ${job.messageId} exhausted maxAttempts; retaining it for capped retries`);
|
|
289
146
|
}
|
|
290
147
|
if (seconds === void 0) await redis.eval(ACK, 3, processing(claimOwner), job.dedupKey || ready, heartbeat(claimOwner), raw, job.dedupKey, job.messageId);
|
|
@@ -305,9 +162,10 @@ function createQueue(redis, config, options = {}) {
|
|
|
305
162
|
await redis.eval(REGISTER, 2, heartbeat(next), owners, next, heartbeatTtlMs);
|
|
306
163
|
return next;
|
|
307
164
|
}
|
|
308
|
-
async function run() {
|
|
165
|
+
async function run(deliver) {
|
|
309
166
|
let renewedAt = Date.now();
|
|
310
167
|
let maintainedAt = 0;
|
|
168
|
+
let failing = false;
|
|
311
169
|
while (!closed) {
|
|
312
170
|
try {
|
|
313
171
|
for (const lost of lostOwners) {
|
|
@@ -328,28 +186,23 @@ function createQueue(redis, config, options = {}) {
|
|
|
328
186
|
const claimOwner = owner;
|
|
329
187
|
const raw = await redis.eval(RESERVE, 3, ready, processing(claimOwner), heartbeat(claimOwner));
|
|
330
188
|
if (typeof raw !== "string") break;
|
|
331
|
-
const task = settle(raw, claimOwner);
|
|
189
|
+
const task = settle(raw, claimOwner, deliver);
|
|
332
190
|
inflight.add(task);
|
|
333
191
|
task.finally(() => inflight.delete(task));
|
|
334
192
|
}
|
|
193
|
+
failing = false;
|
|
335
194
|
} catch (error) {
|
|
336
|
-
if (!closed)
|
|
195
|
+
if (!closed && !failing) console.warn("Queue polling failed:", error);
|
|
196
|
+
else if (!closed) debug("Queue polling error:", error);
|
|
197
|
+
failing = true;
|
|
337
198
|
}
|
|
338
199
|
await sleep();
|
|
339
200
|
}
|
|
340
201
|
}
|
|
341
|
-
return {
|
|
342
|
-
|
|
343
|
-
createQueueHandler,
|
|
344
|
-
getDeploymentId,
|
|
345
|
-
registerHandler(prefix, handler) {
|
|
346
|
-
directHandlers.set(prefix, handler);
|
|
347
|
-
},
|
|
348
|
-
registerRoute(prefix, path) {
|
|
349
|
-
routes.set(prefix, parseQueueRoute(path));
|
|
350
|
-
},
|
|
202
|
+
return createWorldQueue({
|
|
203
|
+
enqueue,
|
|
351
204
|
async getQueueStats() {
|
|
352
|
-
const counts = await redis.eval(STATS, 3, ready, delayed, owners,
|
|
205
|
+
const counts = await redis.eval(STATS, 3, ready, delayed, owners, processing(""));
|
|
353
206
|
if (!Array.isArray(counts) || counts.length !== 3 || counts.some((c) => typeof c !== "number")) throw new TypeError("Invalid queue statistics");
|
|
354
207
|
return {
|
|
355
208
|
waiting: Number(counts[0]),
|
|
@@ -359,11 +212,11 @@ function createQueue(redis, config, options = {}) {
|
|
|
359
212
|
failed: 0
|
|
360
213
|
};
|
|
361
214
|
},
|
|
362
|
-
start() {
|
|
215
|
+
start(deliver) {
|
|
363
216
|
if (closed) return Promise.reject(/* @__PURE__ */ new Error("Queue is closed"));
|
|
364
217
|
startPromise ??= (async () => {
|
|
365
218
|
owner = await register();
|
|
366
|
-
loop = run();
|
|
219
|
+
loop = run(deliver);
|
|
367
220
|
})().catch((error) => {
|
|
368
221
|
startPromise = void 0;
|
|
369
222
|
throw error;
|
|
@@ -389,63 +242,21 @@ function createQueue(redis, config, options = {}) {
|
|
|
389
242
|
await redis.eval(RECLAIM, 4, processing(owner), ready, heartbeat(owner), owners, owner);
|
|
390
243
|
}
|
|
391
244
|
}
|
|
392
|
-
}
|
|
245
|
+
}, config, {
|
|
246
|
+
...options,
|
|
247
|
+
debug
|
|
248
|
+
});
|
|
393
249
|
}
|
|
394
250
|
// src/index.ts
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
...config.redis
|
|
251
|
+
/** `config` is laid over `configFromEnv()`: an option left out comes from the
|
|
252
|
+
* `WORKFLOW_REDIS_*` variables, then the defaults. */
|
|
253
|
+
function createWorld(config = {}) {
|
|
254
|
+
return createRedisBackedWorld(config, {
|
|
255
|
+
env: configFromEnv(),
|
|
256
|
+
createQueue,
|
|
257
|
+
label: "world-redis",
|
|
258
|
+
debug
|
|
404
259
|
});
|
|
405
|
-
const keyPrefix = config.keyPrefix || "workflow:";
|
|
406
|
-
validateRetention(config.retention);
|
|
407
|
-
const reaper = hasRetention(config.retention) ? createReaper({
|
|
408
|
-
redis,
|
|
409
|
-
keyPrefix,
|
|
410
|
-
retention: config.retention
|
|
411
|
-
}) : void 0;
|
|
412
|
-
const storage = createStorage(redis, keyPrefix, config.maxEventsPerRun);
|
|
413
|
-
const queue = createQueue(redis, config, { runHasStarted: async (runId) => {
|
|
414
|
-
try {
|
|
415
|
-
return (await storage.runs.get(runId, { resolveData: "none" })).status !== "pending";
|
|
416
|
-
} catch (error) {
|
|
417
|
-
if (WorkflowRunNotFoundError.is(error)) return false;
|
|
418
|
-
throw error;
|
|
419
|
-
}
|
|
420
|
-
} });
|
|
421
|
-
const streamer = createStreamer({
|
|
422
|
-
redis,
|
|
423
|
-
keyPrefix
|
|
424
|
-
});
|
|
425
|
-
return {
|
|
426
|
-
...storage,
|
|
427
|
-
...streamer,
|
|
428
|
-
...queue,
|
|
429
|
-
specVersion: mintedSpecVersion(),
|
|
430
|
-
createRunId,
|
|
431
|
-
async start() {
|
|
432
|
-
await queue.start();
|
|
433
|
-
reaper?.start();
|
|
434
|
-
await reenqueueActiveRuns(storage.runs, (queueName, message, opts) => queue.queue(queueName, message, {
|
|
435
|
-
...opts,
|
|
436
|
-
idempotencyKey: `recover:${message.runId}`
|
|
437
|
-
}), "world-redis");
|
|
438
|
-
},
|
|
439
|
-
async sweepRetention() {
|
|
440
|
-
return reaper ? reaper.sweep() : 0;
|
|
441
|
-
},
|
|
442
|
-
async close() {
|
|
443
|
-
await reaper?.stop();
|
|
444
|
-
await queue.close();
|
|
445
|
-
await streamer.close();
|
|
446
|
-
await redis.quit();
|
|
447
|
-
}
|
|
448
|
-
};
|
|
449
260
|
}
|
|
450
261
|
export { createWorld };
|
|
451
262
|
|