@brickkit/redis-world 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/setup.js +0 -2
- package/dist/cli-BEHfAdNK.js +65 -0
- package/dist/cli-BEHfAdNK.js.map +1 -0
- package/dist/cli.d.ts +10 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1 -49
- package/dist/index.d.ts +5 -119
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +45 -234
- package/dist/index.js.map +1 -1
- package/dist/internal.d.ts +5 -81
- package/dist/internal.d.ts.map +1 -1
- package/dist/internal.js +4 -3
- package/dist/keys-CqKp_Amk.js +104 -0
- package/dist/keys-CqKp_Amk.js.map +1 -0
- package/dist/{shared-Bhvu4aZA.js → world-783VJfKL.js} +354 -247
- package/dist/world-783VJfKL.js.map +1 -0
- package/dist/world-BDW_evdb.d.ts +242 -0
- package/dist/world-BDW_evdb.d.ts.map +1 -0
- package/package.json +1 -1
- package/dist/cli.js.map +0 -1
- package/dist/env-DMwDVTiZ.js +0 -18
- package/dist/env-DMwDVTiZ.js.map +0 -1
- package/dist/reaper-SOJDwnBw.d.ts +0 -45
- package/dist/reaper-SOJDwnBw.d.ts.map +0 -1
- package/dist/shared-Bhvu4aZA.js.map +0 -1
|
@@ -1,73 +1,19 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError } from "@workflow/errors";
|
|
1
|
+
import { i as resolveConfig, t as keys } from "./keys-CqKp_Amk.js";
|
|
3
2
|
import { createHash } from "node:crypto";
|
|
4
|
-
import {
|
|
3
|
+
import { EventSchema, FIRST_EVENT_SLOT, HookSchema, MessageId, StepSchema, WorkflowRunSchema, entityResolveData, eventIdToSlot, isLegacySpecVersion, mintedSpecVersion, parseQueueName, parseQueueRoute, reenqueueActiveRuns, requiresNewerWorld, slotToEventId, stripEventDataRefs, validateAttributeChanges } from "@workflow/world";
|
|
4
|
+
import { createAppRouteUrl, createWorkflowBaseUrl, createWorkflowUrl, globalSingleton } from "@workflow/utils";
|
|
5
5
|
import { encodeTime, monotonicFactory, ulid } from "ulid";
|
|
6
|
+
import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError } from "@workflow/errors";
|
|
7
|
+
import { Redis } from "ioredis";
|
|
6
8
|
import { EventEmitter } from "node:events";
|
|
7
|
-
// src/run-id.ts
|
|
8
|
-
const CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
|
|
9
|
-
const MAX_TIME = 2 ** 48 - 1;
|
|
10
|
-
const MAX_KEY_LENGTH = 1024;
|
|
11
|
-
function keyedRandomness(key) {
|
|
12
|
-
const digest = createHash("sha256").update(key).digest();
|
|
13
|
-
let bits = 0n;
|
|
14
|
-
for (let index = 0; index < 10; index++) bits = bits << 8n | BigInt(digest[index]);
|
|
15
|
-
let out = "";
|
|
16
|
-
for (let group = 15; group >= 0; group--) out += CROCKFORD[Number(bits >> BigInt(group * 5) & 31n)];
|
|
17
|
-
return out;
|
|
18
|
-
}
|
|
19
|
-
/**
|
|
20
|
-
* The run id the SDK asks the world for when a run starts. Without options it is
|
|
21
|
-
* a fresh ULID. With `runKey` and `runKeyTime` it is the same ULID every time, so
|
|
22
|
-
* starting the same run twice creates one run: the second start finds it already
|
|
23
|
-
* there.
|
|
24
|
-
*
|
|
25
|
-
* The id is a real ULID. Its time is `runKeyTime`, not the clock, because the SDK
|
|
26
|
-
* seeds a workflow's clock from the time in its run id. Pass the time the run is
|
|
27
|
-
* for, such as when the event it handles happened, and the same time on every
|
|
28
|
-
* attempt.
|
|
29
|
-
*
|
|
30
|
-
* @param options - The options given to `start()`. `runKey` is a non-empty string
|
|
31
|
-
* that names the run (at most 1024 characters), and `runKeyTime` is milliseconds
|
|
32
|
-
* since the epoch.
|
|
33
|
-
* @returns The id without its `wrun_` prefix, which the SDK adds.
|
|
34
|
-
* @throws When `runKey` is not a usable string, or comes without a valid `runKeyTime`.
|
|
35
|
-
*/
|
|
36
|
-
function createRunId(options) {
|
|
37
|
-
const key = options?.runKey;
|
|
38
|
-
if (key === void 0) return ulid();
|
|
39
|
-
if (typeof key !== "string" || !key || key.length > MAX_KEY_LENGTH) throw new TypeError(`runKey must be a non-empty string of at most ${MAX_KEY_LENGTH} characters`);
|
|
40
|
-
const time = options?.runKeyTime;
|
|
41
|
-
if (typeof time !== "number" || !Number.isInteger(time) || time < 0 || time > MAX_TIME) throw new TypeError("runKey needs runKeyTime, the time the run is for in milliseconds since the epoch, so the run id stays a valid ULID with a true time");
|
|
42
|
-
return encodeTime(time, 10) + keyedRandomness(key);
|
|
43
|
-
}
|
|
44
|
-
/**
|
|
45
|
-
* The idempotency key for the message that starts a run, which is the only
|
|
46
|
-
* message that carries the run's input. `start()` sends no key of its own, so a
|
|
47
|
-
* second start of the same run would otherwise queue a second first delivery.
|
|
48
|
-
*
|
|
49
|
-
* The key only covers a first message that is still queued or running. Once the
|
|
50
|
-
* run has started, the queue does not enqueue the message at all (see
|
|
51
|
-
* `runHasStarted` in `createQueue`): the runtime runs a run's first step from the
|
|
52
|
-
* first message, in parallel with `run_created`, so a late second one would run
|
|
53
|
-
* that step's code again.
|
|
54
|
-
*
|
|
55
|
-
* @param message - The queue payload.
|
|
56
|
-
* @returns A key for a run's first message, and `undefined` for every other message.
|
|
57
|
-
*/
|
|
58
|
-
function initialMessageKey(message) {
|
|
59
|
-
const { runId, runInput } = message ?? {};
|
|
60
|
-
return typeof runId === "string" && runInput !== void 0 ? `start:${runId}` : void 0;
|
|
61
|
-
}
|
|
62
9
|
// src/internal/debug.ts
|
|
63
10
|
/**
|
|
64
11
|
* Creates a namespaced debug logger.
|
|
65
12
|
* Logs to stderr to avoid corrupting CLI JSON output on stdout.
|
|
66
13
|
*
|
|
67
14
|
* Control via WORKFLOW_DEBUG environment variable:
|
|
68
|
-
* WORKFLOW_DEBUG=1
|
|
69
|
-
* WORKFLOW_DEBUG=
|
|
70
|
-
* WORKFLOW_DEBUG=mysql-world,redis-world // Multiple namespaces
|
|
15
|
+
* WORKFLOW_DEBUG=1 // Enable all
|
|
16
|
+
* WORKFLOW_DEBUG=redis-world // Named namespaces, comma-separated
|
|
71
17
|
*/
|
|
72
18
|
function createDebugLogger(namespace) {
|
|
73
19
|
return (...args) => {
|
|
@@ -79,40 +25,8 @@ function createDebugLogger(namespace) {
|
|
|
79
25
|
process.stderr.write(`${prefix} ${message}\n`);
|
|
80
26
|
};
|
|
81
27
|
}
|
|
82
|
-
// src/
|
|
83
|
-
|
|
84
|
-
promise = Promise.resolve();
|
|
85
|
-
andThen(fn) {
|
|
86
|
-
this.promise = this.promise.then(() => fn(), () => fn());
|
|
87
|
-
return this.promise;
|
|
88
|
-
}
|
|
89
|
-
};
|
|
90
|
-
function compact(obj) {
|
|
91
|
-
const value = {};
|
|
92
|
-
for (const key in obj) if (obj[key] !== null) value[key] = obj[key];
|
|
93
|
-
else value[key] = void 0;
|
|
94
|
-
return value;
|
|
95
|
-
}
|
|
96
|
-
/**
|
|
97
|
-
* Reference counting helper for managing resource lifecycle
|
|
98
|
-
*/
|
|
99
|
-
var Rc = class {
|
|
100
|
-
count = 0;
|
|
101
|
-
disposeFn;
|
|
102
|
-
constructor(disposeFn) {
|
|
103
|
-
this.disposeFn = disposeFn;
|
|
104
|
-
}
|
|
105
|
-
inc() {
|
|
106
|
-
this.count++;
|
|
107
|
-
}
|
|
108
|
-
async dec() {
|
|
109
|
-
this.count--;
|
|
110
|
-
if (this.count === 0 && this.disposeFn) await this.disposeFn();
|
|
111
|
-
}
|
|
112
|
-
getCount() {
|
|
113
|
-
return this.count;
|
|
114
|
-
}
|
|
115
|
-
};
|
|
28
|
+
// src/util.ts
|
|
29
|
+
const debug = createDebugLogger("redis-world");
|
|
116
30
|
// src/internal/serialization.ts
|
|
117
31
|
/** Standard Date fields used across all worlds */
|
|
118
32
|
const DATE_FIELDS = /* @__PURE__ */ new Set([
|
|
@@ -143,16 +57,16 @@ function uint8ArrayReplacer(_key, value) {
|
|
|
143
57
|
};
|
|
144
58
|
return value;
|
|
145
59
|
}
|
|
146
|
-
/**
|
|
147
|
-
|
|
148
|
-
* (`{ __type: 'Uint8Array', data: '<base64>' }`) and ISO date strings.
|
|
149
|
-
*/
|
|
150
|
-
function uint8ArrayReviver(key, value) {
|
|
60
|
+
/** The Uint8Array a tagged object (`{ __type: 'Uint8Array', data: '<base64>' }`) encodes. */
|
|
61
|
+
function taggedBinary(value) {
|
|
151
62
|
if (value && typeof value === "object") {
|
|
152
63
|
const obj = value;
|
|
153
64
|
if (obj.__type === "Uint8Array" && typeof obj.data === "string") return new Uint8Array(Buffer.from(obj.data, "base64"));
|
|
154
65
|
}
|
|
155
|
-
|
|
66
|
+
}
|
|
67
|
+
/** JSON reviver that decodes tagged Uint8Array objects and ISO date strings. */
|
|
68
|
+
function uint8ArrayReviver(key, value) {
|
|
69
|
+
return taggedBinary(value) ?? dateReviver(key, value);
|
|
156
70
|
}
|
|
157
71
|
/** The tag `uint8ArrayReviver` decodes. Absent, only dates need reviving. */
|
|
158
72
|
const BINARY_TAG = "\"__type\"";
|
|
@@ -205,8 +119,192 @@ function parse(text) {
|
|
|
205
119
|
if (text.includes(BINARY_TAG)) return JSON.parse(text, uint8ArrayReviver);
|
|
206
120
|
return reviveDatesDeep(JSON.parse(text));
|
|
207
121
|
}
|
|
208
|
-
|
|
209
|
-
|
|
122
|
+
/**
|
|
123
|
+
* Parse a queue message: tagged Uint8Arrays only. Queue payloads are the
|
|
124
|
+
* runtime's own, so a field that happens to be named like a date stays a string.
|
|
125
|
+
*/
|
|
126
|
+
function parseQueueMessage(text) {
|
|
127
|
+
return text.includes(BINARY_TAG) ? JSON.parse(text, (_key, value) => taggedBinary(value) ?? value) : JSON.parse(text);
|
|
128
|
+
}
|
|
129
|
+
// src/run-id.ts
|
|
130
|
+
const CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
|
|
131
|
+
const MAX_TIME = 2 ** 48 - 1;
|
|
132
|
+
const MAX_KEY_LENGTH = 1024;
|
|
133
|
+
function keyedRandomness(key) {
|
|
134
|
+
const digest = createHash("sha256").update(key).digest();
|
|
135
|
+
let bits = 0n;
|
|
136
|
+
for (let index = 0; index < 10; index++) bits = bits << 8n | BigInt(digest[index]);
|
|
137
|
+
let out = "";
|
|
138
|
+
for (let group = 15; group >= 0; group--) out += CROCKFORD[Number(bits >> BigInt(group * 5) & 31n)];
|
|
139
|
+
return out;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* The run id the SDK asks the world for when a run starts. Without options it is
|
|
143
|
+
* a fresh ULID. With `runKey` and `runKeyTime` it is the same ULID every time, so
|
|
144
|
+
* starting the same run twice creates one run: the second start finds it already
|
|
145
|
+
* there.
|
|
146
|
+
*
|
|
147
|
+
* The id is a real ULID. Its time is `runKeyTime`, not the clock, because the SDK
|
|
148
|
+
* seeds a workflow's clock from the time in its run id. Pass the time the run is
|
|
149
|
+
* for, such as when the event it handles happened, and the same time on every
|
|
150
|
+
* attempt.
|
|
151
|
+
*
|
|
152
|
+
* @param options - The options given to `start()`. `runKey` is a non-empty string
|
|
153
|
+
* that names the run (at most 1024 characters), and `runKeyTime` is milliseconds
|
|
154
|
+
* since the epoch.
|
|
155
|
+
* @returns The id without its `wrun_` prefix, which the SDK adds.
|
|
156
|
+
* @throws When `runKey` is not a usable string, or comes without a valid `runKeyTime`.
|
|
157
|
+
*/
|
|
158
|
+
function createRunId(options) {
|
|
159
|
+
const key = options?.runKey;
|
|
160
|
+
if (key === void 0) return ulid();
|
|
161
|
+
if (typeof key !== "string" || !key || key.length > MAX_KEY_LENGTH) throw new TypeError(`runKey must be a non-empty string of at most ${MAX_KEY_LENGTH} characters`);
|
|
162
|
+
const time = options?.runKeyTime;
|
|
163
|
+
if (typeof time !== "number" || !Number.isInteger(time) || time < 0 || time > MAX_TIME) throw new TypeError("runKey needs runKeyTime, the time the run is for in milliseconds since the epoch, so the run id stays a valid ULID with a true time");
|
|
164
|
+
return encodeTime(time, 10) + keyedRandomness(key);
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* The idempotency key for the message that starts a run, which is the only
|
|
168
|
+
* message that carries the run's input. `start()` sends no key of its own, so a
|
|
169
|
+
* second start of the same run would otherwise queue a second first delivery.
|
|
170
|
+
*
|
|
171
|
+
* The key only covers a first message that is still queued or running. Once the
|
|
172
|
+
* run has started, the queue does not enqueue the message at all (see
|
|
173
|
+
* `runHasStarted` in `createWorldQueue`): the runtime runs a run's first step from the
|
|
174
|
+
* first message, in parallel with `run_created`, so a late second one would run
|
|
175
|
+
* that step's code again.
|
|
176
|
+
*
|
|
177
|
+
* @param message - The queue payload.
|
|
178
|
+
* @returns A key for a run's first message, and `undefined` for every other message.
|
|
179
|
+
*/
|
|
180
|
+
function initialMessageKey(message) {
|
|
181
|
+
const { runId, runInput } = message ?? {};
|
|
182
|
+
return typeof runId === "string" && runInput !== void 0 ? `start:${runId}` : void 0;
|
|
183
|
+
}
|
|
184
|
+
// src/world-queue.ts
|
|
185
|
+
/** A response without a valid `{ timeoutSeconds }` acknowledges the message. */
|
|
186
|
+
function parseReschedule(text) {
|
|
187
|
+
try {
|
|
188
|
+
const seconds = JSON.parse(text)?.timeoutSeconds;
|
|
189
|
+
if (typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0) return seconds;
|
|
190
|
+
} catch {}
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* The Workflow SDK `Queue` on top of a backend: message ids, the first-message
|
|
194
|
+
* guard, the payload transport, HTTP or in-process delivery with the `x-vqs-*`
|
|
195
|
+
* headers, and the handler that answers those deliveries.
|
|
196
|
+
*/
|
|
197
|
+
function createWorldQueue(backend, config, options) {
|
|
198
|
+
const { debug } = options;
|
|
199
|
+
const generateMessageId = monotonicFactory();
|
|
200
|
+
const { httpTimeoutMs } = config;
|
|
201
|
+
const directHandlers = /* @__PURE__ */ new Map();
|
|
202
|
+
const routes = /* @__PURE__ */ new Map();
|
|
203
|
+
let closed = false;
|
|
204
|
+
const queue = async (queueName, message, opts) => {
|
|
205
|
+
if (closed) throw new Error("Queue is closed");
|
|
206
|
+
const { prefix } = parseQueueName(queueName);
|
|
207
|
+
const messageId = MessageId.parse(`msg_${generateMessageId()}`);
|
|
208
|
+
const idempotencyKey = opts?.idempotencyKey ?? initialMessageKey(message);
|
|
209
|
+
if (!opts?.idempotencyKey && idempotencyKey && await options.runHasStarted?.(message.runId)) {
|
|
210
|
+
debug("run already started, not queueing its first message again", { runId: message.runId });
|
|
211
|
+
return { messageId };
|
|
212
|
+
}
|
|
213
|
+
const route = routes.get(prefix);
|
|
214
|
+
return { messageId: await backend.enqueue({
|
|
215
|
+
messageId,
|
|
216
|
+
queueName,
|
|
217
|
+
message: stringify(message),
|
|
218
|
+
...opts?.headers && { headers: opts.headers },
|
|
219
|
+
...route && { route }
|
|
220
|
+
}, {
|
|
221
|
+
idempotencyKey,
|
|
222
|
+
delaySeconds: opts?.delaySeconds
|
|
223
|
+
}) };
|
|
224
|
+
};
|
|
225
|
+
const deliver = async (job) => {
|
|
226
|
+
const { prefix } = parseQueueName(job.queueName);
|
|
227
|
+
const headers = new Headers(job.headers);
|
|
228
|
+
headers.set("content-type", "application/json");
|
|
229
|
+
headers.set("x-vqs-queue-name", job.queueName);
|
|
230
|
+
headers.set("x-vqs-message-id", job.messageId);
|
|
231
|
+
headers.set("x-vqs-message-attempt", String(job.attempt));
|
|
232
|
+
headers.set("x-vqs-message-delivery-count", String(job.deliveryCount));
|
|
233
|
+
headers.set("x-vqs-message-created-at", new Date(job.createdAt).toISOString());
|
|
234
|
+
const deliveryRoute = routes.get(prefix) ?? job.route;
|
|
235
|
+
const directHandler = directHandlers.get(prefix);
|
|
236
|
+
let response;
|
|
237
|
+
if (directHandler) {
|
|
238
|
+
const localBase = createWorkflowBaseUrl("http://localhost");
|
|
239
|
+
response = await directHandler(new Request(deliveryRoute ? createAppRouteUrl(localBase, deliveryRoute) : createWorkflowUrl(localBase, { type: "flow" }), {
|
|
240
|
+
method: "POST",
|
|
241
|
+
headers,
|
|
242
|
+
body: job.message
|
|
243
|
+
}));
|
|
244
|
+
} else {
|
|
245
|
+
const baseUrl = config.baseUrl || process.env.WORKFLOW_BASE_URL || `http://localhost:${process.env.PORT ?? "3000"}`;
|
|
246
|
+
response = await fetch(deliveryRoute ? createAppRouteUrl(baseUrl, deliveryRoute) : createWorkflowUrl(baseUrl, { type: "flow" }), {
|
|
247
|
+
method: "POST",
|
|
248
|
+
headers,
|
|
249
|
+
body: job.message,
|
|
250
|
+
signal: AbortSignal.timeout(httpTimeoutMs)
|
|
251
|
+
});
|
|
252
|
+
}
|
|
253
|
+
const text = await response.text();
|
|
254
|
+
if (!response.ok) throw new Error(`HTTP ${response.status}: ${text}`);
|
|
255
|
+
return parseReschedule(text);
|
|
256
|
+
};
|
|
257
|
+
const createQueueHandler = (queueNamePrefix, handler) => {
|
|
258
|
+
return async (req) => {
|
|
259
|
+
const queueName = req.headers.get("x-vqs-queue-name");
|
|
260
|
+
const messageId = req.headers.get("x-vqs-message-id");
|
|
261
|
+
const attemptStr = req.headers.get("x-vqs-message-attempt");
|
|
262
|
+
const deliveryCountStr = req.headers.get("x-vqs-message-delivery-count");
|
|
263
|
+
const createdAtStr = req.headers.get("x-vqs-message-created-at");
|
|
264
|
+
if (!queueName || !messageId || !attemptStr || !req.body) return Response.json({ error: "Missing required headers or body" }, { status: 400 });
|
|
265
|
+
if (!queueName.startsWith(queueNamePrefix)) return Response.json({ error: "Unhandled queue" }, { status: 400 });
|
|
266
|
+
const attempt = Number.parseInt(attemptStr, 10);
|
|
267
|
+
const deliveryCount = deliveryCountStr ? Number.parseInt(deliveryCountStr, 10) : attempt;
|
|
268
|
+
const createdAt = createdAtStr ? new Date(createdAtStr) : void 0;
|
|
269
|
+
try {
|
|
270
|
+
const timeoutSeconds = (await handler(parseQueueMessage(await req.text()), {
|
|
271
|
+
attempt,
|
|
272
|
+
deliveryCount,
|
|
273
|
+
...createdAt && !Number.isNaN(createdAt.getTime()) && { createdAt },
|
|
274
|
+
queueName,
|
|
275
|
+
messageId
|
|
276
|
+
}))?.timeoutSeconds;
|
|
277
|
+
if (typeof timeoutSeconds === "number" && Number.isFinite(timeoutSeconds)) return Response.json({ timeoutSeconds: Math.max(0, timeoutSeconds) });
|
|
278
|
+
return Response.json({ ok: true });
|
|
279
|
+
} catch (error) {
|
|
280
|
+
debug("queue handler error:", error);
|
|
281
|
+
return Response.json({ error: String(error) }, { status: 500 });
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
};
|
|
285
|
+
return {
|
|
286
|
+
queue,
|
|
287
|
+
createQueueHandler,
|
|
288
|
+
getDeploymentId: async () => "redis",
|
|
289
|
+
registerHandler(prefix, handler) {
|
|
290
|
+
directHandlers.set(prefix, handler);
|
|
291
|
+
},
|
|
292
|
+
registerRoute(prefix, path) {
|
|
293
|
+
routes.set(prefix, parseQueueRoute(path));
|
|
294
|
+
},
|
|
295
|
+
getQueueStats: () => backend.getQueueStats(),
|
|
296
|
+
start: () => backend.start(deliver),
|
|
297
|
+
async close() {
|
|
298
|
+
closed = true;
|
|
299
|
+
await backend.close();
|
|
300
|
+
}
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
/** The delay before retrying a failed delivery: the configured curve, capped. */
|
|
304
|
+
function retryDelayMs(attempt, config) {
|
|
305
|
+
const delay = config.backoffType === "fixed" ? config.backoffDelayMs : config.backoffDelayMs * 2 ** Math.min(30, Math.max(0, attempt - 1));
|
|
306
|
+
return Math.min(config.maxBackoffMs, delay);
|
|
307
|
+
}
|
|
210
308
|
// src/reaper.ts
|
|
211
309
|
const TERMINAL = [
|
|
212
310
|
"completed",
|
|
@@ -242,11 +340,11 @@ function hasRetention(retention) {
|
|
|
242
340
|
* finishes the job.
|
|
243
341
|
*/
|
|
244
342
|
function createReaper(options) {
|
|
245
|
-
const { redis, keyPrefix, retention
|
|
343
|
+
const { redis, keyPrefix, retention } = options;
|
|
246
344
|
const intervalMs = retention.sweepIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
|
|
247
345
|
const batchSize = retention.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
248
|
-
const
|
|
249
|
-
const byStatusKey =
|
|
346
|
+
const k = keys(keyPrefix);
|
|
347
|
+
const byStatusKey = k.runsByStatusKey;
|
|
250
348
|
let timer;
|
|
251
349
|
let running = Promise.resolve();
|
|
252
350
|
/** Runs past retention, oldest first: too old, or beyond the newest `count`. */
|
|
@@ -282,17 +380,17 @@ function createReaper(options) {
|
|
|
282
380
|
}
|
|
283
381
|
/** Delete one run if it is still finished. Returns whether it was deleted. */
|
|
284
382
|
async function deleteRun(runId, listedAs) {
|
|
285
|
-
const metaKey =
|
|
383
|
+
const metaKey = k.runMetaKey(runId);
|
|
286
384
|
const meta = await redis.get(metaKey);
|
|
287
385
|
const status = meta?.split("|")[0];
|
|
288
386
|
if (!status || !TERMINAL.includes(status)) {
|
|
289
387
|
if (!meta) await redis.zrem(byStatusKey(listedAs), runId);
|
|
290
388
|
return false;
|
|
291
389
|
}
|
|
292
|
-
const runKey =
|
|
390
|
+
const runKey = k.runKey(runId);
|
|
293
391
|
const runJson = await redis.get(runKey);
|
|
294
392
|
const workflowName = runJson ? JSON.parse(runJson).workflowName : void 0;
|
|
295
|
-
const eventsKey =
|
|
393
|
+
const eventsKey = k.eventsKey(runId);
|
|
296
394
|
const correlationIds = /* @__PURE__ */ new Set();
|
|
297
395
|
let hasResume = false;
|
|
298
396
|
for (const raw of await redis.hvals(eventsKey)) try {
|
|
@@ -300,35 +398,29 @@ function createReaper(options) {
|
|
|
300
398
|
if (event.correlationId) correlationIds.add(event.correlationId);
|
|
301
399
|
if (event.eventType === "hook_received") hasResume = true;
|
|
302
400
|
} catch {}
|
|
303
|
-
const stepIds = await redis.zrange(
|
|
304
|
-
const hookIds = await redis.zrange(
|
|
305
|
-
const streamNames = await redis.smembers(
|
|
306
|
-
const keys =
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
...[...correlationIds].map((id) => `${keyPrefix}events:by_correlation:${runId}:${id}`),
|
|
314
|
-
...stepIds.flatMap((id) => [`${keyPrefix}step:${runId}:${id}`, `${keyPrefix}step:status:${runId}:${id}`]),
|
|
315
|
-
...hookIds.map((id) => `${keyPrefix}hook:${id}`),
|
|
316
|
-
...streamNames.map((name) => `${keyPrefix}stream:${runId}:${name}`)
|
|
317
|
-
];
|
|
318
|
-
if (hasResume) keys.push(...await scan(`${keyPrefix}events:resume:${runId}:*`));
|
|
401
|
+
const stepIds = await redis.zrange(k.stepsIndexKey(runId), "0", "-1");
|
|
402
|
+
const hookIds = await redis.zrange(k.hooksIndexKey(runId), "0", "-1");
|
|
403
|
+
const streamNames = await redis.smembers(k.streamsByRunKey(runId));
|
|
404
|
+
const keys = k.ownedByRun(runId, {
|
|
405
|
+
correlationIds,
|
|
406
|
+
stepIds,
|
|
407
|
+
hookIds,
|
|
408
|
+
streamNames
|
|
409
|
+
});
|
|
410
|
+
if (hasResume) keys.push(...await scan(k.resumeClaimKey(runId, "*")));
|
|
319
411
|
await unlink(keys);
|
|
320
|
-
const tx = redis.multi().zrem(
|
|
321
|
-
if (workflowName) tx.zrem(
|
|
412
|
+
const tx = redis.multi().zrem(k.runsIndexKey(), runId).zrem(byStatusKey(status), runId).unlink(metaKey, runKey);
|
|
413
|
+
if (workflowName) tx.zrem(k.runsByNameKey(workflowName), runId);
|
|
322
414
|
await tx.exec();
|
|
323
415
|
return true;
|
|
324
416
|
}
|
|
325
417
|
async function sweep() {
|
|
326
|
-
if (await redis.set(
|
|
418
|
+
if (await redis.set(k.reaperLockKey, "1", "PX", Math.max(1e3, Math.floor(intervalMs / 2)), "NX") !== "OK") return 0;
|
|
327
419
|
let deleted = 0;
|
|
328
420
|
for (const status of TERMINAL) for (const runId of await expired(status)) try {
|
|
329
421
|
if (await deleteRun(runId, status)) deleted++;
|
|
330
422
|
} catch (err) {
|
|
331
|
-
|
|
423
|
+
console.warn(`Retention: could not delete run ${runId}:`, err);
|
|
332
424
|
}
|
|
333
425
|
return deleted;
|
|
334
426
|
}
|
|
@@ -337,7 +429,7 @@ function createReaper(options) {
|
|
|
337
429
|
start() {
|
|
338
430
|
if (timer) return;
|
|
339
431
|
const tick = () => {
|
|
340
|
-
running = sweep().catch((err) =>
|
|
432
|
+
running = sweep().catch((err) => console.warn("Retention sweep failed:", err));
|
|
341
433
|
};
|
|
342
434
|
timer = setInterval(tick, intervalMs);
|
|
343
435
|
timer.unref();
|
|
@@ -349,6 +441,13 @@ function createReaper(options) {
|
|
|
349
441
|
}
|
|
350
442
|
};
|
|
351
443
|
}
|
|
444
|
+
// src/internal/util.ts
|
|
445
|
+
function compact(obj) {
|
|
446
|
+
const value = {};
|
|
447
|
+
for (const key in obj) if (obj[key] !== null) value[key] = obj[key];
|
|
448
|
+
else value[key] = void 0;
|
|
449
|
+
return value;
|
|
450
|
+
}
|
|
352
451
|
// src/storage.ts
|
|
353
452
|
/** Per-run event ceiling reported to core, mirroring the Vercel World. */
|
|
354
453
|
const DEFAULT_MAX_EVENTS_PER_RUN = 25e3;
|
|
@@ -424,18 +523,6 @@ local function isTerminal(status)
|
|
|
424
523
|
return status == 'completed' or status == 'failed' or status == 'cancelled'
|
|
425
524
|
end
|
|
426
525
|
`;
|
|
427
|
-
/** Stringify an object with Uint8Array support. Delegates to the shared
|
|
428
|
-
* helper, which tags binary as base64; the previous number-array encoding
|
|
429
|
-
* cost ~450x more CPU to re-parse and ~2.7x more storage at 2MB. */
|
|
430
|
-
function stringifyWithUint8Array(obj) {
|
|
431
|
-
return stringify(obj);
|
|
432
|
-
}
|
|
433
|
-
/** Parse JSON with Uint8Array and Date support. The shared reviver decodes
|
|
434
|
-
* both the base64 tag and the legacy number-array tag, so rows already in
|
|
435
|
-
* Redis keep reading back correctly. */
|
|
436
|
-
function parseWithUint8Array(json) {
|
|
437
|
-
return parse(json);
|
|
438
|
-
}
|
|
439
526
|
/**
|
|
440
527
|
* Filter hook data based on resolveData parameter
|
|
441
528
|
*/
|
|
@@ -997,11 +1084,7 @@ function parseRunMeta(value) {
|
|
|
997
1084
|
*/
|
|
998
1085
|
function createRunsStorage(config) {
|
|
999
1086
|
const { redis, keyPrefix } = config;
|
|
1000
|
-
const runKey = (
|
|
1001
|
-
const runAttributesKey = (id) => `${keyPrefix}run:attrs:${id}`;
|
|
1002
|
-
const runsIndexKey = () => `${keyPrefix}runs:index`;
|
|
1003
|
-
const runsByNameKey = (name) => `${keyPrefix}runs:by_name:${name}`;
|
|
1004
|
-
const runsByStatusKey = (status) => `${keyPrefix}runs:by_status:${status}`;
|
|
1087
|
+
const { runKey, runAttributesKey, runsIndexKey, runsByNameKey, runsByStatusKey } = keys(keyPrefix);
|
|
1005
1088
|
/** The statuses a list filter allows, or undefined for any. `status` takes one
|
|
1006
1089
|
* status or an array; an empty array matches nothing, unlike omitting it. */
|
|
1007
1090
|
function allowedStatuses(params) {
|
|
@@ -1020,7 +1103,7 @@ function createRunsStorage(config) {
|
|
|
1020
1103
|
for (let i = 0; i + 1 < (results?.length ?? 0); i += 2) {
|
|
1021
1104
|
const result = results[i];
|
|
1022
1105
|
if (!result?.[1]) continue;
|
|
1023
|
-
const run =
|
|
1106
|
+
const run = parse(result[1]);
|
|
1024
1107
|
run.attributes = results[i + 1]?.[1] ?? {};
|
|
1025
1108
|
const statuses = allowedStatuses(params);
|
|
1026
1109
|
const statusMatches = !statuses || statuses.includes(run.status);
|
|
@@ -1038,7 +1121,7 @@ function createRunsStorage(config) {
|
|
|
1038
1121
|
const replies = await redis.pipeline().get(runKey(id)).hgetall(runAttributesKey(id)).exec();
|
|
1039
1122
|
const data = replies?.[0]?.[1];
|
|
1040
1123
|
if (!data) throw new WorkflowRunNotFoundError(id);
|
|
1041
|
-
const run =
|
|
1124
|
+
const run = parse(data);
|
|
1042
1125
|
run.attributes = replies?.[1]?.[1] ?? {};
|
|
1043
1126
|
return filterRunData(WorkflowRunSchema.parse(compact(run)), params?.resolveData ?? "all");
|
|
1044
1127
|
}),
|
|
@@ -1093,22 +1176,7 @@ function createEventsStorage(config) {
|
|
|
1093
1176
|
const scripts = withScripts(redis);
|
|
1094
1177
|
const ulid = monotonicFactory();
|
|
1095
1178
|
const maxEventsPerRun = resolveMaxEventsPerRun(config.maxEventsPerRun);
|
|
1096
|
-
const eventsKey = (
|
|
1097
|
-
const eventsByCorrelationKey = (runId, correlationId) => `${keyPrefix}events:by_correlation:${runId}:${correlationId}`;
|
|
1098
|
-
const resumeClaimKey = (runId, resumeId) => `${keyPrefix}events:resume:${runId}:${resumeId}`;
|
|
1099
|
-
const runKey = (id) => `${keyPrefix}run:${id}`;
|
|
1100
|
-
const runAttributesKey = (id) => `${keyPrefix}run:attrs:${id}`;
|
|
1101
|
-
const runMetaKey = (id) => `${keyPrefix}run:meta:${id}`;
|
|
1102
|
-
const runsIndexKey = () => `${keyPrefix}runs:index`;
|
|
1103
|
-
const runsByNameKey = (name) => `${keyPrefix}runs:by_name:${name}`;
|
|
1104
|
-
const runsByStatusKey = (status) => `${keyPrefix}runs:by_status:${status}`;
|
|
1105
|
-
const stepKey = (runId, stepId) => `${keyPrefix}step:${runId}:${stepId}`;
|
|
1106
|
-
const stepsIndexKey = (runId) => `${keyPrefix}steps:by_run:${runId}`;
|
|
1107
|
-
const stepStatusKey = (runId, stepId) => `${keyPrefix}step:status:${runId}:${stepId}`;
|
|
1108
|
-
const hookKey = (hookId) => `${keyPrefix}hook:${hookId}`;
|
|
1109
|
-
const hooksByTokenKey = (token) => `${keyPrefix}hooks:by_token:${token}`;
|
|
1110
|
-
const hooksIndexKey = (runId) => `${keyPrefix}hooks:by_run:${runId}`;
|
|
1111
|
-
const hooksTokenMapKey = (runId) => `${keyPrefix}hooks:tokens:${runId}`;
|
|
1179
|
+
const { eventsKey, eventsByCorrelationKey, resumeClaimKey, runKey, runAttributesKey, runMetaKey, runsIndexKey, runsByNameKey, runsByStatusKey, stepKey, stepsIndexKey, stepStatusKey, hookKey, hooksByTokenKey, hooksIndexKey, hooksTokenMapKey } = keys(keyPrefix);
|
|
1112
1180
|
/** Events per `HMGET`; keeps a long log from building one huge command. */
|
|
1113
1181
|
const READ_CHUNK = 500;
|
|
1114
1182
|
/** The most events an inline delta or skipped-slot report carries; the
|
|
@@ -1127,7 +1195,7 @@ function createEventsStorage(config) {
|
|
|
1127
1195
|
/** Serialize an event for a script: `eventId` first, holding the placeholder. */
|
|
1128
1196
|
function serializeEvent(event) {
|
|
1129
1197
|
const { eventId: _eventId, ...rest } = event;
|
|
1130
|
-
return
|
|
1198
|
+
return stringify({
|
|
1131
1199
|
eventId: EVENT_ID_PLACEHOLDER,
|
|
1132
1200
|
...rest
|
|
1133
1201
|
});
|
|
@@ -1137,7 +1205,7 @@ function createEventsStorage(config) {
|
|
|
1137
1205
|
const events = [];
|
|
1138
1206
|
for (let i = 0; i < eventIds.length; i += READ_CHUNK) {
|
|
1139
1207
|
const values = await redis.hmget(eventsKey(runId), ...eventIds.slice(i, i + READ_CHUNK));
|
|
1140
|
-
for (const value of values) if (value) events.push(EventSchema.parse(compact(
|
|
1208
|
+
for (const value of values) if (value) events.push(EventSchema.parse(compact(parse(value))));
|
|
1141
1209
|
}
|
|
1142
1210
|
return events;
|
|
1143
1211
|
}
|
|
@@ -1155,7 +1223,7 @@ function createEventsStorage(config) {
|
|
|
1155
1223
|
* written.
|
|
1156
1224
|
*/
|
|
1157
1225
|
async function transitionRun(runId, oldStatus, newStatus, updatedRun, at, event, releaseHooks) {
|
|
1158
|
-
const result = await scripts.wfTransitionRun(runKey(runId), runsByStatusKey(oldStatus), runsByStatusKey(newStatus), runMetaKey(runId), eventsKey(runId), hooksIndexKey(runId), hooksTokenMapKey(runId),
|
|
1226
|
+
const result = await scripts.wfTransitionRun(runKey(runId), runsByStatusKey(oldStatus), runsByStatusKey(newStatus), runMetaKey(runId), eventsKey(runId), hooksIndexKey(runId), hooksTokenMapKey(runId), stringify(updatedRun), runId, at.getTime().toString(), runMetaValue(newStatus, updatedRun.specVersion), oldStatus, serializeEvent(event), releaseHooks ? "1" : "0", hookKey(""), hooksByTokenKey(""));
|
|
1159
1227
|
if (result[0] === 1) return {
|
|
1160
1228
|
kind: "ok",
|
|
1161
1229
|
eventId: result[1]
|
|
@@ -1266,7 +1334,7 @@ function createEventsStorage(config) {
|
|
|
1266
1334
|
async function loadRunBody() {
|
|
1267
1335
|
const runData = await redis.get(runKey(effectiveRunId));
|
|
1268
1336
|
if (runData) {
|
|
1269
|
-
existingRun =
|
|
1337
|
+
existingRun = parse(runData);
|
|
1270
1338
|
currentRun = {
|
|
1271
1339
|
status: existingRun.status,
|
|
1272
1340
|
specVersion: existingRun.specVersion
|
|
@@ -1305,7 +1373,7 @@ function createEventsStorage(config) {
|
|
|
1305
1373
|
createdAt: now,
|
|
1306
1374
|
updatedAt: now
|
|
1307
1375
|
};
|
|
1308
|
-
const result = await scripts.wfCreateRunWithEvent(runKey(effectiveRunId), runsIndexKey(), runsByNameKey(runInputData.workflowName), runsByStatusKey("pending"), runMetaKey(effectiveRunId), eventsKey(effectiveRunId), runAttributesKey(effectiveRunId),
|
|
1376
|
+
const result = await scripts.wfCreateRunWithEvent(runKey(effectiveRunId), runsIndexKey(), runsByNameKey(runInputData.workflowName), runsByStatusKey("pending"), runMetaKey(effectiveRunId), eventsKey(effectiveRunId), runAttributesKey(effectiveRunId), stringify(newRun), effectiveRunId, now.getTime().toString(), runMetaValue("pending", effectiveSpecVersion), serializeEvent({
|
|
1309
1377
|
eventType: "run_created",
|
|
1310
1378
|
eventData: {
|
|
1311
1379
|
deploymentId: runInputData.deploymentId,
|
|
@@ -1327,7 +1395,7 @@ function createEventsStorage(config) {
|
|
|
1327
1395
|
};
|
|
1328
1396
|
existingRun = newRun;
|
|
1329
1397
|
} else {
|
|
1330
|
-
const parsed =
|
|
1398
|
+
const parsed = parse(result[1]);
|
|
1331
1399
|
currentRun = {
|
|
1332
1400
|
status: parsed.status,
|
|
1333
1401
|
specVersion: parsed.specVersion
|
|
@@ -1364,7 +1432,7 @@ function createEventsStorage(config) {
|
|
|
1364
1432
|
const lazyStepStart = data.eventType === "step_started" && typeof data.eventData?.stepName === "string" && data.eventData?.input !== void 0;
|
|
1365
1433
|
if (["step_started", "step_retrying"].includes(data.eventType) && data.correlationId) {
|
|
1366
1434
|
const stepData = await redis.get(stepKey(effectiveRunId, data.correlationId));
|
|
1367
|
-
if (stepData) validatedStep =
|
|
1435
|
+
if (stepData) validatedStep = parse(stepData);
|
|
1368
1436
|
if (!validatedStep && !lazyStepStart) throw new WorkflowWorldError(`Step "${data.correlationId}" not found`, { status: 404 });
|
|
1369
1437
|
if (lazyStepStart && validatedStep) throw new EntityConflictError(`Step "${data.correlationId}" already created`);
|
|
1370
1438
|
if (validatedStep) {
|
|
@@ -1380,7 +1448,7 @@ function createEventsStorage(config) {
|
|
|
1380
1448
|
if (hookEventsRequiringExistence.includes(data.eventType) && data.correlationId && !isResumeRetryCandidate) {
|
|
1381
1449
|
const existingHook = await redis.get(hookKey(data.correlationId));
|
|
1382
1450
|
if (!existingHook) throw new HookNotFoundError(data.correlationId);
|
|
1383
|
-
validatedHook =
|
|
1451
|
+
validatedHook = parse(existingHook);
|
|
1384
1452
|
}
|
|
1385
1453
|
if (data.eventType === "run_created") {
|
|
1386
1454
|
const eventData = data.eventData;
|
|
@@ -1408,7 +1476,7 @@ function createEventsStorage(config) {
|
|
|
1408
1476
|
updatedAt: now
|
|
1409
1477
|
};
|
|
1410
1478
|
const event = buildEvent({ createdAt: now });
|
|
1411
|
-
const result = await scripts.wfCreateRunWithEvent(runKey(effectiveRunId), runsIndexKey(), runsByNameKey(eventData.workflowName), runsByStatusKey("pending"), runMetaKey(effectiveRunId), eventsKey(effectiveRunId), runAttributesKey(effectiveRunId),
|
|
1479
|
+
const result = await scripts.wfCreateRunWithEvent(runKey(effectiveRunId), runsIndexKey(), runsByNameKey(eventData.workflowName), runsByStatusKey("pending"), runMetaKey(effectiveRunId), eventsKey(effectiveRunId), runAttributesKey(effectiveRunId), stringify(newRun), effectiveRunId, now.getTime().toString(), runMetaValue("pending", effectiveSpecVersion), serializeEvent(event), ...flattenAttributes(seeds));
|
|
1412
1480
|
if (result[0] === 0) throw new EntityConflictError(`Workflow run "${effectiveRunId}" already exists`);
|
|
1413
1481
|
run = WorkflowRunSchema.parse({
|
|
1414
1482
|
...compact(newRun),
|
|
@@ -1514,7 +1582,7 @@ function createEventsStorage(config) {
|
|
|
1514
1582
|
updatedAt: now
|
|
1515
1583
|
};
|
|
1516
1584
|
const event = buildEvent({ createdAt: now });
|
|
1517
|
-
const result = await scripts.wfCreateStepWithEvent(stepKey(effectiveRunId, data.correlationId), stepsIndexKey(effectiveRunId), eventsKey(effectiveRunId), eventsByCorrelationKey(effectiveRunId, data.correlationId), stepStatusKey(effectiveRunId, data.correlationId), runMetaKey(effectiveRunId),
|
|
1585
|
+
const result = await scripts.wfCreateStepWithEvent(stepKey(effectiveRunId, data.correlationId), stepsIndexKey(effectiveRunId), eventsKey(effectiveRunId), eventsByCorrelationKey(effectiveRunId, data.correlationId), stepStatusKey(effectiveRunId, data.correlationId), runMetaKey(effectiveRunId), stringify(newStep), data.correlationId, now.getTime().toString(), serializeEvent(event), "pending|0");
|
|
1518
1586
|
if (result[0] === -6) throw new EntityConflictError(`Cannot create new entities on run in terminal state "${result[1]}"`);
|
|
1519
1587
|
if (result[0] === 0) throw new EntityConflictError(`Step "${data.correlationId}" already exists`);
|
|
1520
1588
|
step = StepSchema.parse(compact(newStep));
|
|
@@ -1561,7 +1629,7 @@ function createEventsStorage(config) {
|
|
|
1561
1629
|
createdAt: now,
|
|
1562
1630
|
eventData: startedEventData
|
|
1563
1631
|
});
|
|
1564
|
-
const result = await scripts.wfCreateStepStarted(stepKey(effectiveRunId, data.correlationId), stepsIndexKey(effectiveRunId), eventsKey(effectiveRunId), eventsByCorrelationKey(effectiveRunId, data.correlationId), stepStatusKey(effectiveRunId, data.correlationId), runMetaKey(effectiveRunId),
|
|
1632
|
+
const result = await scripts.wfCreateStepStarted(stepKey(effectiveRunId, data.correlationId), stepsIndexKey(effectiveRunId), eventsKey(effectiveRunId), eventsByCorrelationKey(effectiveRunId, data.correlationId), stepStatusKey(effectiveRunId, data.correlationId), runMetaKey(effectiveRunId), stringify(startedStep), data.correlationId, now.getTime().toString(), serializeEvent(createdEvent), serializeEvent(startedEvent), "running|1");
|
|
1565
1633
|
if (result[0] === -6) throw new RunExpiredError(`Cannot start step on run in terminal state "${result[1]}"`);
|
|
1566
1634
|
if (result[0] === 0) throw new EntityConflictError(`Step "${data.correlationId}" already created`);
|
|
1567
1635
|
step = StepSchema.parse(compact(startedStep));
|
|
@@ -1587,7 +1655,7 @@ function createEventsStorage(config) {
|
|
|
1587
1655
|
updatedAt: now
|
|
1588
1656
|
};
|
|
1589
1657
|
pendingEntityKey = stepKey(effectiveRunId, data.correlationId);
|
|
1590
|
-
pendingEntityJson =
|
|
1658
|
+
pendingEntityJson = stringify(updatedStep);
|
|
1591
1659
|
pendingStep = {
|
|
1592
1660
|
key: stepStatusKey(effectiveRunId, data.correlationId),
|
|
1593
1661
|
status: "running",
|
|
@@ -1601,7 +1669,7 @@ function createEventsStorage(config) {
|
|
|
1601
1669
|
const eventData = data.eventData;
|
|
1602
1670
|
const existingData = await redis.get(stepKey(effectiveRunId, data.correlationId));
|
|
1603
1671
|
if (existingData) {
|
|
1604
|
-
const existing =
|
|
1672
|
+
const existing = parse(existingData);
|
|
1605
1673
|
if (isTerminal(existing.status)) throw new EntityConflictError(`Cannot modify step in terminal state "${existing.status}"`);
|
|
1606
1674
|
const updatedStep = {
|
|
1607
1675
|
...existing,
|
|
@@ -1611,7 +1679,7 @@ function createEventsStorage(config) {
|
|
|
1611
1679
|
updatedAt: now
|
|
1612
1680
|
};
|
|
1613
1681
|
pendingEntityKey = stepKey(effectiveRunId, data.correlationId);
|
|
1614
|
-
pendingEntityJson =
|
|
1682
|
+
pendingEntityJson = stringify(updatedStep);
|
|
1615
1683
|
pendingStep = {
|
|
1616
1684
|
key: stepStatusKey(effectiveRunId, data.correlationId),
|
|
1617
1685
|
status: "completed",
|
|
@@ -1624,7 +1692,7 @@ function createEventsStorage(config) {
|
|
|
1624
1692
|
const eventData = data.eventData;
|
|
1625
1693
|
const existingData = await redis.get(stepKey(effectiveRunId, data.correlationId));
|
|
1626
1694
|
if (existingData) {
|
|
1627
|
-
const existing =
|
|
1695
|
+
const existing = parse(existingData);
|
|
1628
1696
|
if (isTerminal(existing.status)) throw new EntityConflictError(`Cannot modify step in terminal state "${existing.status}"`);
|
|
1629
1697
|
const updatedStep = {
|
|
1630
1698
|
...existing,
|
|
@@ -1634,7 +1702,7 @@ function createEventsStorage(config) {
|
|
|
1634
1702
|
updatedAt: now
|
|
1635
1703
|
};
|
|
1636
1704
|
pendingEntityKey = stepKey(effectiveRunId, data.correlationId);
|
|
1637
|
-
pendingEntityJson =
|
|
1705
|
+
pendingEntityJson = stringify(updatedStep);
|
|
1638
1706
|
pendingStep = {
|
|
1639
1707
|
key: stepStatusKey(effectiveRunId, data.correlationId),
|
|
1640
1708
|
status: "failed",
|
|
@@ -1655,7 +1723,7 @@ function createEventsStorage(config) {
|
|
|
1655
1723
|
updatedAt: now
|
|
1656
1724
|
};
|
|
1657
1725
|
pendingEntityKey = stepKey(effectiveRunId, data.correlationId);
|
|
1658
|
-
pendingEntityJson =
|
|
1726
|
+
pendingEntityJson = stringify(updatedStep);
|
|
1659
1727
|
pendingStep = {
|
|
1660
1728
|
key: stepStatusKey(effectiveRunId, data.correlationId),
|
|
1661
1729
|
status: "pending",
|
|
@@ -1680,13 +1748,13 @@ function createEventsStorage(config) {
|
|
|
1680
1748
|
...eventData.isSystem !== void 0 && { isSystem: eventData.isSystem }
|
|
1681
1749
|
};
|
|
1682
1750
|
const event = buildEvent({ createdAt: now });
|
|
1683
|
-
const result = await scripts.wfCreateHookWithEvent(hookKey(data.correlationId), hooksByTokenKey(eventData.token), hooksIndexKey(effectiveRunId), hooksTokenMapKey(effectiveRunId), eventsKey(effectiveRunId), eventsByCorrelationKey(effectiveRunId, data.correlationId), runMetaKey(effectiveRunId),
|
|
1751
|
+
const result = await scripts.wfCreateHookWithEvent(hookKey(data.correlationId), hooksByTokenKey(eventData.token), hooksIndexKey(effectiveRunId), hooksTokenMapKey(effectiveRunId), eventsKey(effectiveRunId), eventsByCorrelationKey(effectiveRunId, data.correlationId), runMetaKey(effectiveRunId), stringify(newHook), data.correlationId, now.getTime().toString(), eventData.token, serializeEvent(event), hookKey(""));
|
|
1684
1752
|
if (result[0] === -6) throw new EntityConflictError(`Cannot create new entities on run in terminal state "${result[1]}"`);
|
|
1685
1753
|
if (result[0] === 2) {
|
|
1686
1754
|
const owningHookId = result[1];
|
|
1687
1755
|
let conflictingRunId;
|
|
1688
1756
|
const owningHookData = await redis.get(hookKey(owningHookId));
|
|
1689
|
-
if (owningHookData) conflictingRunId =
|
|
1757
|
+
if (owningHookData) conflictingRunId = parse(owningHookData).runId;
|
|
1690
1758
|
const stored = await appendEvent(effectiveRunId, {
|
|
1691
1759
|
eventType: "hook_conflict",
|
|
1692
1760
|
correlationId: data.correlationId,
|
|
@@ -1824,7 +1892,7 @@ function createEventsStorage(config) {
|
|
|
1824
1892
|
async get(runId, eventId, params) {
|
|
1825
1893
|
const data = await redis.hget(eventsKey(runId), eventId);
|
|
1826
1894
|
if (!data) throw new WorkflowWorldError(`Event not found: ${eventId}`, { status: 404 });
|
|
1827
|
-
const event = EventSchema.parse(compact(
|
|
1895
|
+
const event = EventSchema.parse(compact(parse(data)));
|
|
1828
1896
|
return stripEventDataRefs(event, params?.resolveData ?? "all");
|
|
1829
1897
|
},
|
|
1830
1898
|
async list(params) {
|
|
@@ -1888,12 +1956,11 @@ function createEventsStorage(config) {
|
|
|
1888
1956
|
*/
|
|
1889
1957
|
function createStepsStorage(config) {
|
|
1890
1958
|
const { redis, keyPrefix } = config;
|
|
1891
|
-
const stepKey = (
|
|
1892
|
-
const stepsIndexKey = (runId) => `${keyPrefix}steps:by_run:${runId}`;
|
|
1959
|
+
const { stepKey, stepsIndexKey } = keys(keyPrefix);
|
|
1893
1960
|
async function getStepData(key, stepId, params) {
|
|
1894
1961
|
const data = await redis.get(key);
|
|
1895
1962
|
if (!data) throw new WorkflowWorldError(`Step not found: ${stepId}`, { status: 404 });
|
|
1896
|
-
const step =
|
|
1963
|
+
const step = parse(data);
|
|
1897
1964
|
return filterStepData(StepSchema.parse(compact(step)), params?.resolveData ?? "all");
|
|
1898
1965
|
}
|
|
1899
1966
|
return {
|
|
@@ -1908,7 +1975,7 @@ function createStepsStorage(config) {
|
|
|
1908
1975
|
const resolveData = params?.resolveData ?? "all";
|
|
1909
1976
|
const steps = [];
|
|
1910
1977
|
for (const [i, result] of (results ?? []).entries()) if (result?.[1]) {
|
|
1911
|
-
const step =
|
|
1978
|
+
const step = parse(result[1]);
|
|
1912
1979
|
const parsed = StepSchema.parse(compact(step));
|
|
1913
1980
|
steps.push({
|
|
1914
1981
|
step: filterStepData(parsed, resolveData),
|
|
@@ -1930,17 +1997,15 @@ function createStepsStorage(config) {
|
|
|
1930
1997
|
*/
|
|
1931
1998
|
function createHooksStorage(config) {
|
|
1932
1999
|
const { redis, keyPrefix } = config;
|
|
1933
|
-
const hookKeyFn = (
|
|
1934
|
-
const hooksByTokenKey = (token) => `${keyPrefix}hooks:by_token:${token}`;
|
|
1935
|
-
const hooksIndexKey = (runId) => `${keyPrefix}hooks:by_run:${runId}`;
|
|
2000
|
+
const { hookKey: hookKeyFn, hooksByTokenKey, hooksIndexKey, runsByStatusKey } = keys(keyPrefix);
|
|
1936
2001
|
async function liveHooksIndexKeys() {
|
|
1937
|
-
return (await Promise.all(["pending", "running"].map((status) => redis.zrange(
|
|
2002
|
+
return (await Promise.all(["pending", "running"].map((status) => redis.zrange(runsByStatusKey(status), "0", "-1")))).flat().map(hooksIndexKey);
|
|
1938
2003
|
}
|
|
1939
2004
|
return {
|
|
1940
2005
|
async get(hookId, params) {
|
|
1941
2006
|
const data = await redis.get(hookKeyFn(hookId));
|
|
1942
2007
|
if (!data) throw new HookNotFoundError(hookId);
|
|
1943
|
-
const hook =
|
|
2008
|
+
const hook = parse(data);
|
|
1944
2009
|
return filterHookData(HookSchema.parse(compact(hook)), params?.resolveData ?? "all");
|
|
1945
2010
|
},
|
|
1946
2011
|
async getByToken(token, params) {
|
|
@@ -1957,7 +2022,7 @@ function createHooksStorage(config) {
|
|
|
1957
2022
|
const results = await pipeline.exec();
|
|
1958
2023
|
const hooks = [];
|
|
1959
2024
|
for (const [i, result] of (results ?? []).entries()) if (result?.[1]) {
|
|
1960
|
-
const hook =
|
|
2025
|
+
const hook = parse(result[1]);
|
|
1961
2026
|
const parsed = HookSchema.parse(compact(hook));
|
|
1962
2027
|
hooks.push({
|
|
1963
2028
|
hook: filterHookData(parsed, params?.resolveData ?? "all"),
|
|
@@ -2067,6 +2132,14 @@ const LUA_APPEND_AND_PUBLISH = `
|
|
|
2067
2132
|
redis.call('SADD', runIndexKey, streamId)
|
|
2068
2133
|
return 1
|
|
2069
2134
|
`;
|
|
2135
|
+
/** An XRANGE entry, by the field order `LUA_APPEND_AND_PUBLISH` writes: data, then eof. */
|
|
2136
|
+
function decodeEntry([id, fields]) {
|
|
2137
|
+
return {
|
|
2138
|
+
id,
|
|
2139
|
+
data: Buffer.from(fields[1], "base64"),
|
|
2140
|
+
eof: fields[3] === "true"
|
|
2141
|
+
};
|
|
2142
|
+
}
|
|
2070
2143
|
/**
|
|
2071
2144
|
* Create a streamer implementation using Redis Streams.
|
|
2072
2145
|
*
|
|
@@ -2083,33 +2156,22 @@ const LUA_APPEND_AND_PUBLISH = `
|
|
|
2083
2156
|
function createStreamer(config) {
|
|
2084
2157
|
const { redis, keyPrefix } = config;
|
|
2085
2158
|
const events = new EventEmitter();
|
|
2086
|
-
const
|
|
2087
|
-
const withStreamMutex =
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
};
|
|
2096
|
-
mutexes.set(key, entry);
|
|
2097
|
-
}
|
|
2098
|
-
entry.rc.inc();
|
|
2099
|
-
try {
|
|
2100
|
-
return await entry.mutex.andThen(fn);
|
|
2101
|
-
} finally {
|
|
2102
|
-
await entry.rc.dec();
|
|
2103
|
-
}
|
|
2159
|
+
const tails = /* @__PURE__ */ new Map();
|
|
2160
|
+
const withStreamMutex = (key, fn) => {
|
|
2161
|
+
const run = (tails.get(key) ?? Promise.resolve()).then(fn, fn);
|
|
2162
|
+
const tail = run.catch(() => {});
|
|
2163
|
+
tails.set(key, tail);
|
|
2164
|
+
tail.then(() => {
|
|
2165
|
+
if (tails.get(key) === tail) tails.delete(key);
|
|
2166
|
+
});
|
|
2167
|
+
return run;
|
|
2104
2168
|
};
|
|
2105
2169
|
redis.defineCommand("wfStreamAppend", {
|
|
2106
2170
|
numberOfKeys: 2,
|
|
2107
2171
|
lua: LUA_APPEND_AND_PUBLISH
|
|
2108
2172
|
});
|
|
2109
2173
|
const scripted = redis;
|
|
2110
|
-
const streamKey
|
|
2111
|
-
const streamsByRunKey = (runId) => `${keyPrefix}streams:by_run:${runId}`;
|
|
2112
|
-
const STREAM_CHANNEL = `${keyPrefix}stream_notify`;
|
|
2174
|
+
const { streamKey, streamsByRunKey, streamChannel: STREAM_CHANNEL } = keys(keyPrefix);
|
|
2113
2175
|
const emitterKey = (runId, name) => `strm:${runId}:${name}`;
|
|
2114
2176
|
const subscriber = redis.duplicate();
|
|
2115
2177
|
const subscriptionReady = subscriber.subscribe(STREAM_CHANNEL).then(() => {});
|
|
@@ -2200,13 +2262,11 @@ function createStreamer(config) {
|
|
|
2200
2262
|
do {
|
|
2201
2263
|
again = false;
|
|
2202
2264
|
const entries = await redis.xrange(streamKey(runId, name), lastEntryId === "" ? "-" : `(${lastEntryId}`, "+");
|
|
2203
|
-
for (const
|
|
2204
|
-
id,
|
|
2205
|
-
data: Buffer.from(fields[1], "base64"),
|
|
2206
|
-
eof: fields[3] === "true"
|
|
2207
|
-
});
|
|
2265
|
+
for (const entry of entries) enqueue(decodeEntry(entry));
|
|
2208
2266
|
} while (again && !cleanedUp);
|
|
2209
|
-
} catch
|
|
2267
|
+
} catch (error) {
|
|
2268
|
+
debug("Stream catch-up failed:", error);
|
|
2269
|
+
} finally {
|
|
2210
2270
|
catchingUp = false;
|
|
2211
2271
|
}
|
|
2212
2272
|
}
|
|
@@ -2228,20 +2288,10 @@ function createStreamer(config) {
|
|
|
2228
2288
|
cleanups.push(() => {
|
|
2229
2289
|
events.off(key, onData);
|
|
2230
2290
|
});
|
|
2231
|
-
const
|
|
2291
|
+
const parsedChunks = (await redis.xrange(streamKey(runId, name), "-", "+").catch((err) => {
|
|
2232
2292
|
cleanup();
|
|
2233
2293
|
throw err;
|
|
2234
|
-
});
|
|
2235
|
-
const parsedChunks = [];
|
|
2236
|
-
for (const [id, fields] of chunks) {
|
|
2237
|
-
const data = Buffer.from(fields[1], "base64");
|
|
2238
|
-
const eof = fields[3] === "true";
|
|
2239
|
-
parsedChunks.push({
|
|
2240
|
-
id,
|
|
2241
|
-
data,
|
|
2242
|
-
eof
|
|
2243
|
-
});
|
|
2244
|
-
}
|
|
2294
|
+
})).map(decodeEntry);
|
|
2245
2295
|
if (offset < 0) {
|
|
2246
2296
|
const firstEof = parsedChunks.findIndex((chunk) => chunk.eof);
|
|
2247
2297
|
const dataCount = firstEof === -1 ? parsedChunks.length : firstEof;
|
|
@@ -2285,8 +2335,9 @@ function createStreamer(config) {
|
|
|
2285
2335
|
let done = false;
|
|
2286
2336
|
let hasMore = false;
|
|
2287
2337
|
let lastId = null;
|
|
2288
|
-
for (const
|
|
2289
|
-
|
|
2338
|
+
for (const entry of entries) {
|
|
2339
|
+
const { id, data: bytes, eof } = decodeEntry(entry);
|
|
2340
|
+
if (eof) {
|
|
2290
2341
|
done = true;
|
|
2291
2342
|
break;
|
|
2292
2343
|
}
|
|
@@ -2296,7 +2347,7 @@ function createStreamer(config) {
|
|
|
2296
2347
|
}
|
|
2297
2348
|
data.push({
|
|
2298
2349
|
index: dataIndex,
|
|
2299
|
-
data: new Uint8Array(
|
|
2350
|
+
data: new Uint8Array(bytes)
|
|
2300
2351
|
});
|
|
2301
2352
|
dataIndex++;
|
|
2302
2353
|
lastId = id;
|
|
@@ -2326,20 +2377,76 @@ function createStreamer(config) {
|
|
|
2326
2377
|
}
|
|
2327
2378
|
};
|
|
2328
2379
|
}
|
|
2329
|
-
// src/
|
|
2330
|
-
|
|
2331
|
-
|
|
2380
|
+
// src/world.ts
|
|
2381
|
+
/**
|
|
2382
|
+
* A world on Redis: storage, streams and retention are the same for every
|
|
2383
|
+
* queue; `createQueue` supplies the queue. `config` is laid over `env`, so an
|
|
2384
|
+
* option the caller leaves out still comes from the environment.
|
|
2385
|
+
*/
|
|
2386
|
+
function createRedisBackedWorld(config, world) {
|
|
2387
|
+
const merged = { ...world.env };
|
|
2388
|
+
for (const [key, value] of Object.entries(config)) if (value !== void 0) merged[key] = value;
|
|
2389
|
+
const cfg = resolveConfig(merged);
|
|
2390
|
+
const autoPipelining = cfg.enableAutoPipelining;
|
|
2391
|
+
validateRetention(cfg.retention);
|
|
2392
|
+
const redis = typeof cfg.redis === "string" ? new Redis(cfg.redis, { enableAutoPipelining: autoPipelining }) : new Redis({
|
|
2393
|
+
enableAutoPipelining: autoPipelining,
|
|
2394
|
+
...cfg.redis
|
|
2395
|
+
});
|
|
2396
|
+
const { keyPrefix } = cfg;
|
|
2397
|
+
const reaper = hasRetention(cfg.retention) ? createReaper({
|
|
2332
2398
|
redis,
|
|
2333
2399
|
keyPrefix,
|
|
2334
|
-
|
|
2400
|
+
retention: cfg.retention
|
|
2401
|
+
}) : void 0;
|
|
2402
|
+
const storageConfig = {
|
|
2403
|
+
redis,
|
|
2404
|
+
keyPrefix,
|
|
2405
|
+
maxEventsPerRun: cfg.maxEventsPerRun
|
|
2406
|
+
};
|
|
2407
|
+
const storage = {
|
|
2408
|
+
runs: createRunsStorage(storageConfig),
|
|
2409
|
+
events: createEventsStorage(storageConfig),
|
|
2410
|
+
hooks: createHooksStorage(storageConfig),
|
|
2411
|
+
steps: createStepsStorage(storageConfig)
|
|
2335
2412
|
};
|
|
2413
|
+
const queue = world.createQueue(redis, cfg, { runHasStarted: async (runId) => {
|
|
2414
|
+
try {
|
|
2415
|
+
return (await storage.runs.get(runId, { resolveData: "none" })).status !== "pending";
|
|
2416
|
+
} catch (error) {
|
|
2417
|
+
if (WorkflowRunNotFoundError.is(error)) return false;
|
|
2418
|
+
throw error;
|
|
2419
|
+
}
|
|
2420
|
+
} });
|
|
2421
|
+
const streamer = createStreamer({
|
|
2422
|
+
redis,
|
|
2423
|
+
keyPrefix
|
|
2424
|
+
});
|
|
2336
2425
|
return {
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2426
|
+
...storage,
|
|
2427
|
+
...streamer,
|
|
2428
|
+
...queue,
|
|
2429
|
+
specVersion: mintedSpecVersion(),
|
|
2430
|
+
createRunId,
|
|
2431
|
+
async start() {
|
|
2432
|
+
await queue.start();
|
|
2433
|
+
reaper?.start();
|
|
2434
|
+
await reenqueueActiveRuns(storage.runs, (queueName, message, opts) => queue.queue(queueName, message, {
|
|
2435
|
+
...opts,
|
|
2436
|
+
idempotencyKey: `recover:${message.runId}`
|
|
2437
|
+
}), world.label);
|
|
2438
|
+
},
|
|
2439
|
+
async sweepRetention() {
|
|
2440
|
+
return reaper ? reaper.sweep() : 0;
|
|
2441
|
+
},
|
|
2442
|
+
async close() {
|
|
2443
|
+
await reaper?.stop();
|
|
2444
|
+
await queue.close();
|
|
2445
|
+
await streamer.close();
|
|
2446
|
+
await redis.quit();
|
|
2447
|
+
}
|
|
2341
2448
|
};
|
|
2342
2449
|
}
|
|
2343
|
-
export {
|
|
2450
|
+
export { createDebugLogger as a, debug as i, createWorldQueue as n, retryDelayMs as r, createRedisBackedWorld as t };
|
|
2344
2451
|
|
|
2345
|
-
//# sourceMappingURL=
|
|
2452
|
+
//# sourceMappingURL=world-783VJfKL.js.map
|