@brickkit/redis-world 0.0.0-stage → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +17 -0
- package/README.md +108 -2
- package/bin/setup.js +12 -0
- package/dist/cli.d.ts +5 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +65 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +148 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +2797 -0
- package/dist/index.js.map +1 -0
- package/package.json +67 -4
package/dist/index.js
ADDED
|
@@ -0,0 +1,2797 @@
|
|
|
1
|
+
import { EventSchema, FIRST_EVENT_SLOT, HookSchema, MessageId, StepSchema, ValidQueueName, WorkflowRunSchema, entityResolveData, eventIdToSlot, isLegacySpecVersion, mintedSpecVersion, parseQueueName, parseQueueRoute, reenqueueActiveRuns, requiresNewerWorld, slotToEventId, stripEventDataRefs, validateAttributeChanges } from "@workflow/world";
|
|
2
|
+
import { EntityConflictError, HookNotFoundError, RunExpiredError, RunNotSupportedError, TooEarlyError, WorkflowRunNotFoundError, WorkflowWorldError } from "@workflow/errors";
|
|
3
|
+
import { Redis } from "ioredis";
|
|
4
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
5
|
+
import { setTimeout } from "node:timers/promises";
|
|
6
|
+
import { createAppRouteUrl, createWorkflowBaseUrl, createWorkflowUrl, globalSingleton } from "@workflow/utils";
|
|
7
|
+
import { encodeTime, monotonicFactory, ulid } from "ulid";
|
|
8
|
+
import { EventEmitter } from "node:events";
|
|
9
|
+
//#region src/run-id.ts
|
|
10
|
+
const CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
|
|
11
|
+
const MAX_TIME = 2 ** 48 - 1;
|
|
12
|
+
const MAX_KEY_LENGTH = 1024;
|
|
13
|
+
function keyedRandomness(key) {
|
|
14
|
+
const digest = createHash("sha256").update(key).digest();
|
|
15
|
+
let bits = 0n;
|
|
16
|
+
for (let index = 0; index < 10; index++) bits = bits << 8n | BigInt(digest[index]);
|
|
17
|
+
let out = "";
|
|
18
|
+
for (let group = 15; group >= 0; group--) out += CROCKFORD[Number(bits >> BigInt(group * 5) & 31n)];
|
|
19
|
+
return out;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* The run id the SDK asks the world for when a run starts. Without options it is
|
|
23
|
+
* a fresh ULID. With `runKey` and `runKeyTime` it is the same ULID every time, so
|
|
24
|
+
* starting the same run twice creates one run: the second start finds it already
|
|
25
|
+
* there.
|
|
26
|
+
*
|
|
27
|
+
* The id is a real ULID. Its time is `runKeyTime`, not the clock, because the SDK
|
|
28
|
+
* seeds a workflow's clock from the time in its run id. Pass the time the run is
|
|
29
|
+
* for, such as when the event it handles happened, and the same time on every
|
|
30
|
+
* attempt.
|
|
31
|
+
*
|
|
32
|
+
* @param options - The options given to `start()`. `runKey` is a non-empty string
|
|
33
|
+
* that names the run (at most 1024 characters), and `runKeyTime` is milliseconds
|
|
34
|
+
* since the epoch.
|
|
35
|
+
* @returns The id without its `wrun_` prefix, which the SDK adds.
|
|
36
|
+
* @throws When `runKey` is not a usable string, or comes without a valid `runKeyTime`.
|
|
37
|
+
*/
|
|
38
|
+
function createRunId(options) {
|
|
39
|
+
const key = options?.runKey;
|
|
40
|
+
if (key === void 0) return ulid();
|
|
41
|
+
if (typeof key !== "string" || !key || key.length > MAX_KEY_LENGTH) throw new TypeError(`runKey must be a non-empty string of at most ${MAX_KEY_LENGTH} characters`);
|
|
42
|
+
const time = options?.runKeyTime;
|
|
43
|
+
if (typeof time !== "number" || !Number.isInteger(time) || time < 0 || time > MAX_TIME) throw new TypeError("runKey needs runKeyTime, the time the run is for in milliseconds since the epoch, so the run id stays a valid ULID with a true time");
|
|
44
|
+
return encodeTime(time, 10) + keyedRandomness(key);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* The idempotency key for the message that starts a run, which is the only
|
|
48
|
+
* message that carries the run's input. `start()` sends no key of its own, so a
|
|
49
|
+
* second start of the same run would otherwise queue a second first delivery.
|
|
50
|
+
*
|
|
51
|
+
* The key only covers a first message that is still queued or running. Once the
|
|
52
|
+
* run has started, the queue does not enqueue the message at all (see
|
|
53
|
+
* `runHasStarted` in `createQueue`): the runtime runs a run's first step from the
|
|
54
|
+
* first message, in parallel with `run_created`, so a late second one would run
|
|
55
|
+
* that step's code again.
|
|
56
|
+
*
|
|
57
|
+
* @param message - The queue payload.
|
|
58
|
+
* @returns A key for a run's first message, and `undefined` for every other message.
|
|
59
|
+
*/
|
|
60
|
+
function initialMessageKey(message) {
|
|
61
|
+
const { runId, runInput } = message ?? {};
|
|
62
|
+
return typeof runId === "string" && runInput !== void 0 ? `start:${runId}` : void 0;
|
|
63
|
+
}
|
|
64
|
+
//#endregion
|
|
65
|
+
//#region src/internal/debug.ts
|
|
66
|
+
/**
|
|
67
|
+
* Creates a namespaced debug logger.
|
|
68
|
+
* Logs to stderr to avoid corrupting CLI JSON output on stdout.
|
|
69
|
+
*
|
|
70
|
+
* Control via WORKFLOW_DEBUG environment variable:
|
|
71
|
+
* WORKFLOW_DEBUG=1 // Enable all
|
|
72
|
+
* WORKFLOW_DEBUG=mysql-world // Specific namespace
|
|
73
|
+
* WORKFLOW_DEBUG=mysql-world,redis-world // Multiple namespaces
|
|
74
|
+
*/
|
|
75
|
+
function createDebugLogger(namespace) {
|
|
76
|
+
return (...args) => {
|
|
77
|
+
const debug = process.env.WORKFLOW_DEBUG;
|
|
78
|
+
if (!debug) return;
|
|
79
|
+
if (!(debug === "1" || debug === "true" || debug === "*" || debug.split(",").some((ns) => ns.trim() === namespace))) return;
|
|
80
|
+
const prefix = `[${(/* @__PURE__ */ new Date()).toISOString()}] [${namespace}]`;
|
|
81
|
+
const message = args.map((arg) => typeof arg === "object" ? JSON.stringify(arg, null, 2) : String(arg)).join(" ");
|
|
82
|
+
process.stderr.write(`${prefix} ${message}\n`);
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
//#endregion
|
|
86
|
+
//#region src/internal/util.ts
|
|
87
|
+
var Mutex = class {
|
|
88
|
+
promise = Promise.resolve();
|
|
89
|
+
andThen(fn) {
|
|
90
|
+
this.promise = this.promise.then(() => fn(), () => fn());
|
|
91
|
+
return this.promise;
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
function compact(obj) {
|
|
95
|
+
const value = {};
|
|
96
|
+
for (const key in obj) if (obj[key] !== null) value[key] = obj[key];
|
|
97
|
+
else value[key] = void 0;
|
|
98
|
+
return value;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Reference counting helper for managing resource lifecycle
|
|
102
|
+
*/
|
|
103
|
+
var Rc = class {
|
|
104
|
+
count = 0;
|
|
105
|
+
disposeFn;
|
|
106
|
+
constructor(disposeFn) {
|
|
107
|
+
this.disposeFn = disposeFn;
|
|
108
|
+
}
|
|
109
|
+
inc() {
|
|
110
|
+
this.count++;
|
|
111
|
+
}
|
|
112
|
+
async dec() {
|
|
113
|
+
this.count--;
|
|
114
|
+
if (this.count === 0 && this.disposeFn) await this.disposeFn();
|
|
115
|
+
}
|
|
116
|
+
getCount() {
|
|
117
|
+
return this.count;
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
//#endregion
|
|
121
|
+
//#region src/internal/serialization.ts
|
|
122
|
+
/** Standard Date fields used across all worlds */
|
|
123
|
+
const DATE_FIELDS = /* @__PURE__ */ new Set([
|
|
124
|
+
"createdAt",
|
|
125
|
+
"updatedAt",
|
|
126
|
+
"startedAt",
|
|
127
|
+
"completedAt",
|
|
128
|
+
"retryAfter"
|
|
129
|
+
]);
|
|
130
|
+
/**
|
|
131
|
+
* JSON reviver that converts ISO date strings to Date objects.
|
|
132
|
+
* Use with JSON.parse(json, dateReviver).
|
|
133
|
+
*/
|
|
134
|
+
function dateReviver(key, value) {
|
|
135
|
+
if (DATE_FIELDS.has(key) && typeof value === "string") {
|
|
136
|
+
const date = new Date(value);
|
|
137
|
+
return Number.isNaN(date.getTime()) ? value : date;
|
|
138
|
+
}
|
|
139
|
+
return value;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* JSON replacer that encodes Uint8Array as a tagged object with base64 data.
|
|
143
|
+
*/
|
|
144
|
+
function uint8ArrayReplacer(_key, value) {
|
|
145
|
+
if (value instanceof Uint8Array) return {
|
|
146
|
+
__type: "Uint8Array",
|
|
147
|
+
data: Buffer.from(value).toString("base64")
|
|
148
|
+
};
|
|
149
|
+
return value;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* JSON reviver that decodes tagged Uint8Array objects
|
|
153
|
+
* (`{ __type: 'Uint8Array', data: '<base64>' }`) and ISO date strings.
|
|
154
|
+
*/
|
|
155
|
+
function uint8ArrayReviver(key, value) {
|
|
156
|
+
if (value && typeof value === "object") {
|
|
157
|
+
const obj = value;
|
|
158
|
+
if (obj.__type === "Uint8Array" && typeof obj.data === "string") return new Uint8Array(Buffer.from(obj.data, "base64"));
|
|
159
|
+
}
|
|
160
|
+
return dateReviver(key, value);
|
|
161
|
+
}
|
|
162
|
+
/** The tag `uint8ArrayReviver` decodes. Absent, only dates need reviving. */
|
|
163
|
+
const BINARY_TAG = "\"__type\"";
|
|
164
|
+
/** Revive `DATE_FIELDS` in place at any depth — the same reach as
|
|
165
|
+
* `dateReviver` under `JSON.parse`, just without the per-node JSON interop. */
|
|
166
|
+
function reviveDatesDeep(value) {
|
|
167
|
+
if (value === null || typeof value !== "object") return value;
|
|
168
|
+
if (Array.isArray(value)) {
|
|
169
|
+
for (let i = 0; i < value.length; i++) value[i] = reviveDatesDeep(value[i]);
|
|
170
|
+
return value;
|
|
171
|
+
}
|
|
172
|
+
const obj = value;
|
|
173
|
+
for (const key in obj) {
|
|
174
|
+
const child = obj[key];
|
|
175
|
+
if (DATE_FIELDS.has(key) && typeof child === "string") {
|
|
176
|
+
const date = new Date(child);
|
|
177
|
+
if (!Number.isNaN(date.getTime())) {
|
|
178
|
+
obj[key] = date;
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
obj[key] = reviveDatesDeep(child);
|
|
183
|
+
}
|
|
184
|
+
return obj;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Is there a `Uint8Array` anywhere in the graph?
|
|
188
|
+
*
|
|
189
|
+
* Must stay exact: `JSON.stringify` turns a bare `Uint8Array` into
|
|
190
|
+
* `{"0":1,...}` without complaint, so a miss is data corruption, not a slow
|
|
191
|
+
* path. The walk allocates nothing, which is why it beats the replacer over
|
|
192
|
+
* the same nodes.
|
|
193
|
+
*/
|
|
194
|
+
function hasBinary(value) {
|
|
195
|
+
if (value === null || typeof value !== "object") return false;
|
|
196
|
+
if (value instanceof Uint8Array) return true;
|
|
197
|
+
if (Array.isArray(value)) {
|
|
198
|
+
for (let i = 0; i < value.length; i++) if (hasBinary(value[i])) return true;
|
|
199
|
+
return false;
|
|
200
|
+
}
|
|
201
|
+
for (const key in value) if (hasBinary(value[key])) return true;
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
/** Stringify with Uint8Array support. */
|
|
205
|
+
function stringify(value) {
|
|
206
|
+
return hasBinary(value) ? JSON.stringify(value, uint8ArrayReplacer) : JSON.stringify(value);
|
|
207
|
+
}
|
|
208
|
+
/** Parse with Uint8Array and Date support. */
|
|
209
|
+
function parse(text) {
|
|
210
|
+
if (text.includes(BINARY_TAG)) return JSON.parse(text, uint8ArrayReviver);
|
|
211
|
+
return reviveDatesDeep(JSON.parse(text));
|
|
212
|
+
}
|
|
213
|
+
//#endregion
|
|
214
|
+
//#region src/util.ts
|
|
215
|
+
const debug = createDebugLogger("redis-world");
|
|
216
|
+
//#endregion
|
|
217
|
+
//#region src/queue.ts
|
|
218
|
+
const ENQUEUE = `
|
|
219
|
+
if ARGV[1] ~= '' then
|
|
220
|
+
local existing = redis.call('GET', KEYS[1])
|
|
221
|
+
if existing then return existing end
|
|
222
|
+
redis.call('SET', KEYS[1], ARGV[2])
|
|
223
|
+
if tonumber(ARGV[3]) > 0 then redis.call('PEXPIRE', KEYS[1], ARGV[3]) end
|
|
224
|
+
end
|
|
225
|
+
if tonumber(ARGV[5]) > 0 then
|
|
226
|
+
redis.call('ZADD', KEYS[3], ARGV[5], ARGV[4])
|
|
227
|
+
else
|
|
228
|
+
redis.call('LPUSH', KEYS[2], ARGV[4])
|
|
229
|
+
end
|
|
230
|
+
return ARGV[2]
|
|
231
|
+
`;
|
|
232
|
+
const RESERVE = `
|
|
233
|
+
if redis.call('EXISTS', KEYS[3]) == 0 then return nil end
|
|
234
|
+
local raw = redis.call('LINDEX', KEYS[1], -1)
|
|
235
|
+
if not raw then return nil end
|
|
236
|
+
local job = cjson.decode(raw)
|
|
237
|
+
job.attempt = job.attempt + 1
|
|
238
|
+
job.deliveryCount = job.deliveryCount + 1
|
|
239
|
+
local claim = cjson.encode(job)
|
|
240
|
+
redis.call('RPOP', KEYS[1])
|
|
241
|
+
redis.call('LPUSH', KEYS[2], claim)
|
|
242
|
+
return claim
|
|
243
|
+
`;
|
|
244
|
+
const ACK = `
|
|
245
|
+
if redis.call('EXISTS', KEYS[3]) == 0 then return 0 end
|
|
246
|
+
if redis.call('LREM', KEYS[1], 1, ARGV[1]) == 0 then return 0 end
|
|
247
|
+
if ARGV[2] ~= '' and redis.call('GET', KEYS[2]) == ARGV[3] then
|
|
248
|
+
redis.call('DEL', KEYS[2])
|
|
249
|
+
end
|
|
250
|
+
return 1
|
|
251
|
+
`;
|
|
252
|
+
const DEFER = `
|
|
253
|
+
if redis.call('EXISTS', KEYS[3]) == 0 then return 0 end
|
|
254
|
+
if redis.call('LREM', KEYS[1], 1, ARGV[1]) == 0 then return 0 end
|
|
255
|
+
redis.call('ZADD', KEYS[2], ARGV[3], ARGV[2])
|
|
256
|
+
return 1
|
|
257
|
+
`;
|
|
258
|
+
const PROMOTE = `
|
|
259
|
+
local due = redis.call('ZRANGEBYSCORE', KEYS[1], '-inf', ARGV[1], 'LIMIT', 0, 100)
|
|
260
|
+
for _, raw in ipairs(due) do
|
|
261
|
+
redis.call('LPUSH', KEYS[2], raw)
|
|
262
|
+
redis.call('ZREM', KEYS[1], raw)
|
|
263
|
+
end
|
|
264
|
+
return #due
|
|
265
|
+
`;
|
|
266
|
+
const RECLAIM = `
|
|
267
|
+
if redis.call('EXISTS', KEYS[3]) == 1 then return 0 end
|
|
268
|
+
local moved = 0
|
|
269
|
+
for i = 1, 100 do
|
|
270
|
+
local raw = redis.call('RPOPLPUSH', KEYS[1], KEYS[2])
|
|
271
|
+
if not raw then break end
|
|
272
|
+
moved = moved + 1
|
|
273
|
+
end
|
|
274
|
+
if redis.call('LLEN', KEYS[1]) == 0 then redis.call('SREM', KEYS[4], ARGV[1]) end
|
|
275
|
+
return moved
|
|
276
|
+
`;
|
|
277
|
+
const RENEW = `
|
|
278
|
+
if redis.call('EXISTS', KEYS[1]) == 0 then return 0 end
|
|
279
|
+
redis.call('PEXPIRE', KEYS[1], ARGV[1])
|
|
280
|
+
return 1
|
|
281
|
+
`;
|
|
282
|
+
const REGISTER = `
|
|
283
|
+
redis.call('SET', KEYS[1], '1', 'PX', ARGV[2])
|
|
284
|
+
redis.call('SADD', KEYS[2], ARGV[1])
|
|
285
|
+
`;
|
|
286
|
+
const STATS = `
|
|
287
|
+
local active = 0
|
|
288
|
+
for _, owner in ipairs(redis.call('SMEMBERS', KEYS[3])) do
|
|
289
|
+
active = active + redis.call('LLEN', ARGV[1] .. ':processing:' .. owner)
|
|
290
|
+
end
|
|
291
|
+
return {redis.call('LLEN', KEYS[1]), active, redis.call('ZCARD', KEYS[2])}
|
|
292
|
+
`;
|
|
293
|
+
function parseEnvelope(raw) {
|
|
294
|
+
const value = JSON.parse(raw);
|
|
295
|
+
if (value === null || typeof value !== "object" || !("messageId" in value) || !("queueName" in value) || !("message" in value) || typeof value.message !== "string" || !("attempt" in value) || typeof value.attempt !== "number" || !("deliveryCount" in value) || typeof value.deliveryCount !== "number" || !("createdAt" in value) || typeof value.createdAt !== "number" || !("dedupKey" in value) || typeof value.dedupKey !== "string" || !("headers" in value) || value.headers === null || typeof value.headers !== "object") throw new TypeError("Invalid Redis queue envelope");
|
|
296
|
+
const headers = {};
|
|
297
|
+
for (const [key, header] of Object.entries(value.headers)) {
|
|
298
|
+
if (typeof header !== "string") throw new TypeError("Invalid queue header");
|
|
299
|
+
headers[key] = header;
|
|
300
|
+
}
|
|
301
|
+
return {
|
|
302
|
+
messageId: MessageId.parse(value.messageId),
|
|
303
|
+
queueName: ValidQueueName.parse(value.queueName),
|
|
304
|
+
message: value.message,
|
|
305
|
+
attempt: value.attempt,
|
|
306
|
+
deliveryCount: value.deliveryCount,
|
|
307
|
+
createdAt: value.createdAt,
|
|
308
|
+
dedupKey: value.dedupKey,
|
|
309
|
+
headers,
|
|
310
|
+
..."route" in value && typeof value.route === "string" && { route: value.route }
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
function serializeQueueMessage(message) {
|
|
314
|
+
return hasBinary(message) ? JSON.stringify(message, uint8ArrayReplacer) : JSON.stringify(message);
|
|
315
|
+
}
|
|
316
|
+
function queueMessageReviver(_key, value) {
|
|
317
|
+
if (value !== null && typeof value === "object") {
|
|
318
|
+
const tagged = value;
|
|
319
|
+
if (tagged.__type === "Uint8Array" && typeof tagged.data === "string") return new Uint8Array(Buffer.from(tagged.data, "base64"));
|
|
320
|
+
}
|
|
321
|
+
return value;
|
|
322
|
+
}
|
|
323
|
+
function deserializeQueueMessage(text) {
|
|
324
|
+
return text.includes("\"__type\"") ? JSON.parse(text, queueMessageReviver) : JSON.parse(text);
|
|
325
|
+
}
|
|
326
|
+
function resolveBaseUrl(config) {
|
|
327
|
+
return config.baseUrl || process.env.WORKFLOW_BASE_URL || `http://localhost:${process.env.PORT ?? "3000"}`;
|
|
328
|
+
}
|
|
329
|
+
function createQueue(redis, config, options = {}) {
|
|
330
|
+
const generateMessageId = monotonicFactory();
|
|
331
|
+
const root = `${config.keyPrefix || "workflow:"}queue:flows`;
|
|
332
|
+
const ready = root;
|
|
333
|
+
const delayed = `${root}:delayed`;
|
|
334
|
+
const owners = `${root}:workers`;
|
|
335
|
+
const processing = (owner) => `${root}:processing:${owner}`;
|
|
336
|
+
const heartbeat = (owner) => `${root}:heartbeat:${owner}`;
|
|
337
|
+
const heartbeatTtlMs = config.heartbeatTtlMs ?? 9e4;
|
|
338
|
+
const concurrency = config.queueConcurrency ?? 10;
|
|
339
|
+
const maxAttempts = config.maxAttempts ?? 73;
|
|
340
|
+
const maxBackoffMs = config.maxBackoffMs ?? 3e4;
|
|
341
|
+
const backoffDelayMs = config.backoffDelayMs ?? 1e3;
|
|
342
|
+
const httpTimeoutMs = config.httpTimeoutMs ?? 3e5;
|
|
343
|
+
for (const [name, value, minimum] of [
|
|
344
|
+
[
|
|
345
|
+
"heartbeatTtlMs",
|
|
346
|
+
heartbeatTtlMs,
|
|
347
|
+
300
|
|
348
|
+
],
|
|
349
|
+
[
|
|
350
|
+
"queueConcurrency",
|
|
351
|
+
concurrency,
|
|
352
|
+
1
|
|
353
|
+
],
|
|
354
|
+
[
|
|
355
|
+
"maxAttempts",
|
|
356
|
+
maxAttempts,
|
|
357
|
+
49
|
|
358
|
+
],
|
|
359
|
+
[
|
|
360
|
+
"maxBackoffMs",
|
|
361
|
+
maxBackoffMs,
|
|
362
|
+
1
|
|
363
|
+
],
|
|
364
|
+
[
|
|
365
|
+
"backoffDelayMs",
|
|
366
|
+
backoffDelayMs,
|
|
367
|
+
0
|
|
368
|
+
],
|
|
369
|
+
[
|
|
370
|
+
"httpTimeoutMs",
|
|
371
|
+
httpTimeoutMs,
|
|
372
|
+
1
|
|
373
|
+
],
|
|
374
|
+
...config.idempotencyTtlMs === void 0 ? [] : [[
|
|
375
|
+
"idempotencyTtlMs",
|
|
376
|
+
config.idempotencyTtlMs,
|
|
377
|
+
1
|
|
378
|
+
]]
|
|
379
|
+
]) if (typeof value !== "number" || typeof minimum !== "number" || !Number.isSafeInteger(value) || value < minimum || value > 2147483647) throw new RangeError(`${name} must be an integer between ${minimum} and 2147483647`);
|
|
380
|
+
const directHandlers = /* @__PURE__ */ new Map();
|
|
381
|
+
const routes = /* @__PURE__ */ new Map();
|
|
382
|
+
let closed = false;
|
|
383
|
+
let startPromise;
|
|
384
|
+
let loop;
|
|
385
|
+
let owner;
|
|
386
|
+
const inflight = /* @__PURE__ */ new Set();
|
|
387
|
+
const lostOwners = /* @__PURE__ */ new Set();
|
|
388
|
+
const abort = new AbortController();
|
|
389
|
+
const sleep = () => setTimeout(50, void 0, { signal: abort.signal }).catch(() => {});
|
|
390
|
+
const queue = async (queueName, message, opts) => {
|
|
391
|
+
if (closed) throw new Error("Queue is closed");
|
|
392
|
+
const { prefix } = parseQueueName(queueName);
|
|
393
|
+
const messageId = MessageId.parse(`msg_${generateMessageId()}`);
|
|
394
|
+
const idempotencyKey = opts?.idempotencyKey ?? initialMessageKey(message);
|
|
395
|
+
if (!opts?.idempotencyKey && idempotencyKey && await options.runHasStarted?.(message.runId)) {
|
|
396
|
+
debug("run already started, not queueing its first message again", { runId: message.runId });
|
|
397
|
+
return { messageId };
|
|
398
|
+
}
|
|
399
|
+
const dedupKey = idempotencyKey ? `${root}:idempotent:${idempotencyKey}` : "";
|
|
400
|
+
const envelope = {
|
|
401
|
+
messageId,
|
|
402
|
+
queueName,
|
|
403
|
+
message: serializeQueueMessage(message),
|
|
404
|
+
attempt: 0,
|
|
405
|
+
deliveryCount: 0,
|
|
406
|
+
createdAt: Date.now(),
|
|
407
|
+
dedupKey,
|
|
408
|
+
headers: opts?.headers ?? {},
|
|
409
|
+
...routes.has(prefix) && { route: routes.get(prefix) }
|
|
410
|
+
};
|
|
411
|
+
const delayMs = Math.max(0, (opts?.delaySeconds ?? 0) * 1e3);
|
|
412
|
+
if (!Number.isFinite(delayMs)) throw new RangeError("delaySeconds must be finite");
|
|
413
|
+
const result = await redis.eval(ENQUEUE, 3, dedupKey || ready, ready, delayed, dedupKey, messageId, config.idempotencyTtlMs ?? 0, JSON.stringify(envelope), delayMs > 0 ? Date.now() + delayMs : 0);
|
|
414
|
+
return { messageId: MessageId.parse(result) };
|
|
415
|
+
};
|
|
416
|
+
function parseReschedule(text) {
|
|
417
|
+
try {
|
|
418
|
+
const seconds = JSON.parse(text)?.timeoutSeconds;
|
|
419
|
+
if (typeof seconds === "number" && Number.isFinite(seconds) && seconds >= 0) return seconds;
|
|
420
|
+
} catch {}
|
|
421
|
+
}
|
|
422
|
+
async function dispatch(job) {
|
|
423
|
+
const { prefix: queuePrefix } = parseQueueName(job.queueName);
|
|
424
|
+
const messageId = job.messageId;
|
|
425
|
+
const headers = new Headers(job.headers);
|
|
426
|
+
headers.set("content-type", "application/json");
|
|
427
|
+
headers.set("x-vqs-queue-name", job.queueName);
|
|
428
|
+
headers.set("x-vqs-message-id", messageId);
|
|
429
|
+
headers.set("x-vqs-message-attempt", String(job.attempt));
|
|
430
|
+
headers.set("x-vqs-message-delivery-count", String(job.deliveryCount));
|
|
431
|
+
headers.set("x-vqs-message-created-at", new Date(job.createdAt).toISOString());
|
|
432
|
+
const deliveryRoute = routes.get(queuePrefix) ?? job.route;
|
|
433
|
+
const directHandler = directHandlers.get(queuePrefix);
|
|
434
|
+
let response;
|
|
435
|
+
if (directHandler) {
|
|
436
|
+
const localBase = createWorkflowBaseUrl("http://localhost");
|
|
437
|
+
response = await directHandler(new Request(deliveryRoute ? createAppRouteUrl(localBase, deliveryRoute) : createWorkflowUrl(localBase, { type: "flow" }), {
|
|
438
|
+
method: "POST",
|
|
439
|
+
headers,
|
|
440
|
+
body: job.message
|
|
441
|
+
}));
|
|
442
|
+
} else {
|
|
443
|
+
const baseUrl = resolveBaseUrl(config);
|
|
444
|
+
response = await fetch(deliveryRoute ? createAppRouteUrl(baseUrl, deliveryRoute) : createWorkflowUrl(baseUrl, { type: "flow" }), {
|
|
445
|
+
method: "POST",
|
|
446
|
+
headers,
|
|
447
|
+
body: job.message,
|
|
448
|
+
signal: AbortSignal.timeout(httpTimeoutMs)
|
|
449
|
+
});
|
|
450
|
+
}
|
|
451
|
+
const text = await response.text();
|
|
452
|
+
if (!response.ok) throw new Error(`HTTP ${response.status}: ${text}`);
|
|
453
|
+
return parseReschedule(text);
|
|
454
|
+
}
|
|
455
|
+
const createQueueHandler = (queueNamePrefix, handler) => {
|
|
456
|
+
return async (req) => {
|
|
457
|
+
const queueName = req.headers.get("x-vqs-queue-name");
|
|
458
|
+
const messageId = req.headers.get("x-vqs-message-id");
|
|
459
|
+
const attemptStr = req.headers.get("x-vqs-message-attempt");
|
|
460
|
+
const deliveryCountStr = req.headers.get("x-vqs-message-delivery-count");
|
|
461
|
+
const createdAtStr = req.headers.get("x-vqs-message-created-at");
|
|
462
|
+
if (!queueName || !messageId || !attemptStr || !req.body) return Response.json({ error: "Missing required headers or body" }, { status: 400 });
|
|
463
|
+
if (!queueName.startsWith(queueNamePrefix)) return Response.json({ error: "Unhandled queue" }, { status: 400 });
|
|
464
|
+
const attempt = Number.parseInt(attemptStr, 10);
|
|
465
|
+
const deliveryCount = deliveryCountStr ? Number.parseInt(deliveryCountStr, 10) : attempt;
|
|
466
|
+
const createdAt = createdAtStr ? new Date(createdAtStr) : void 0;
|
|
467
|
+
try {
|
|
468
|
+
const timeoutSeconds = (await handler(deserializeQueueMessage(await req.text()), {
|
|
469
|
+
attempt,
|
|
470
|
+
deliveryCount,
|
|
471
|
+
...createdAt && !Number.isNaN(createdAt.getTime()) && { createdAt },
|
|
472
|
+
queueName,
|
|
473
|
+
messageId
|
|
474
|
+
}))?.timeoutSeconds;
|
|
475
|
+
if (typeof timeoutSeconds === "number" && Number.isFinite(timeoutSeconds)) return Response.json({ timeoutSeconds: Math.max(0, timeoutSeconds) });
|
|
476
|
+
return Response.json({ ok: true });
|
|
477
|
+
} catch (error) {
|
|
478
|
+
debug("queue handler error:", error);
|
|
479
|
+
return Response.json({ error: String(error) }, { status: 500 });
|
|
480
|
+
}
|
|
481
|
+
};
|
|
482
|
+
};
|
|
483
|
+
const getDeploymentId = async () => "redis";
|
|
484
|
+
async function settle(raw, claimOwner) {
|
|
485
|
+
try {
|
|
486
|
+
const job = parseEnvelope(raw);
|
|
487
|
+
let seconds;
|
|
488
|
+
let failed = false;
|
|
489
|
+
try {
|
|
490
|
+
seconds = await dispatch(job);
|
|
491
|
+
} catch (error) {
|
|
492
|
+
failed = true;
|
|
493
|
+
debug(`Delivery ${job.messageId} attempt ${job.attempt} failed:`, error);
|
|
494
|
+
const curve = config.backoffType === "fixed" ? backoffDelayMs : backoffDelayMs * 2 ** Math.min(30, job.attempt - 1);
|
|
495
|
+
seconds = (job.attempt >= maxAttempts ? maxBackoffMs : Math.min(maxBackoffMs, curve)) / 1e3;
|
|
496
|
+
if (job.attempt === maxAttempts) console.error(`Queue message ${job.messageId} exhausted maxAttempts; retaining it for capped retries`);
|
|
497
|
+
}
|
|
498
|
+
if (seconds === void 0) await redis.eval(ACK, 3, processing(claimOwner), job.dedupKey || ready, heartbeat(claimOwner), raw, job.dedupKey, job.messageId);
|
|
499
|
+
else {
|
|
500
|
+
const next = failed ? job : {
|
|
501
|
+
...job,
|
|
502
|
+
attempt: job.attempt - 1
|
|
503
|
+
};
|
|
504
|
+
await redis.eval(DEFER, 3, processing(claimOwner), delayed, heartbeat(claimOwner), raw, JSON.stringify(next), Date.now() + seconds * 1e3);
|
|
505
|
+
}
|
|
506
|
+
} catch (error) {
|
|
507
|
+
lostOwners.add(claimOwner);
|
|
508
|
+
console.error("Queue settlement error:", error);
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
async function register() {
|
|
512
|
+
const next = randomUUID();
|
|
513
|
+
await redis.eval(REGISTER, 2, heartbeat(next), owners, next, heartbeatTtlMs);
|
|
514
|
+
return next;
|
|
515
|
+
}
|
|
516
|
+
async function run() {
|
|
517
|
+
let renewedAt = Date.now();
|
|
518
|
+
let maintainedAt = 0;
|
|
519
|
+
while (!closed) {
|
|
520
|
+
try {
|
|
521
|
+
for (const lost of lostOwners) {
|
|
522
|
+
await redis.del(heartbeat(lost));
|
|
523
|
+
lostOwners.delete(lost);
|
|
524
|
+
if (owner === lost) owner = void 0;
|
|
525
|
+
}
|
|
526
|
+
if (!owner || Date.now() - renewedAt >= heartbeatTtlMs / 3) {
|
|
527
|
+
if (!owner || await redis.eval(RENEW, 1, heartbeat(owner), heartbeatTtlMs) !== 1) owner = await register();
|
|
528
|
+
renewedAt = Date.now();
|
|
529
|
+
}
|
|
530
|
+
if (Date.now() - maintainedAt >= 100) {
|
|
531
|
+
await redis.eval(PROMOTE, 2, delayed, ready, Date.now());
|
|
532
|
+
for (const stale of await redis.smembers(owners)) if (stale !== owner) await redis.eval(RECLAIM, 4, processing(stale), ready, heartbeat(stale), owners, stale);
|
|
533
|
+
maintainedAt = Date.now();
|
|
534
|
+
}
|
|
535
|
+
for (let reserved = 0; reserved < concurrency && !closed && owner && inflight.size < concurrency; reserved++) {
|
|
536
|
+
const claimOwner = owner;
|
|
537
|
+
const raw = await redis.eval(RESERVE, 3, ready, processing(claimOwner), heartbeat(claimOwner));
|
|
538
|
+
if (typeof raw !== "string") break;
|
|
539
|
+
const task = settle(raw, claimOwner);
|
|
540
|
+
inflight.add(task);
|
|
541
|
+
task.finally(() => inflight.delete(task));
|
|
542
|
+
}
|
|
543
|
+
} catch (error) {
|
|
544
|
+
if (!closed) debug("Queue polling error:", error);
|
|
545
|
+
}
|
|
546
|
+
await sleep();
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
return {
|
|
550
|
+
queue,
|
|
551
|
+
createQueueHandler,
|
|
552
|
+
getDeploymentId,
|
|
553
|
+
registerHandler(prefix, handler) {
|
|
554
|
+
directHandlers.set(prefix, handler);
|
|
555
|
+
},
|
|
556
|
+
registerRoute(prefix, path) {
|
|
557
|
+
routes.set(prefix, parseQueueRoute(path));
|
|
558
|
+
},
|
|
559
|
+
async getQueueStats() {
|
|
560
|
+
const counts = await redis.eval(STATS, 3, ready, delayed, owners, root);
|
|
561
|
+
if (!Array.isArray(counts) || counts.length !== 3 || counts.some((c) => typeof c !== "number")) throw new TypeError("Invalid queue statistics");
|
|
562
|
+
return {
|
|
563
|
+
waiting: Number(counts[0]),
|
|
564
|
+
active: Number(counts[1]),
|
|
565
|
+
delayed: Number(counts[2]),
|
|
566
|
+
completed: 0,
|
|
567
|
+
failed: 0
|
|
568
|
+
};
|
|
569
|
+
},
|
|
570
|
+
start() {
|
|
571
|
+
if (closed) return Promise.reject(/* @__PURE__ */ new Error("Queue is closed"));
|
|
572
|
+
startPromise ??= (async () => {
|
|
573
|
+
owner = await register();
|
|
574
|
+
loop = run();
|
|
575
|
+
})().catch((error) => {
|
|
576
|
+
startPromise = void 0;
|
|
577
|
+
throw error;
|
|
578
|
+
});
|
|
579
|
+
return startPromise;
|
|
580
|
+
},
|
|
581
|
+
async close() {
|
|
582
|
+
if (closed) return;
|
|
583
|
+
closed = true;
|
|
584
|
+
abort.abort();
|
|
585
|
+
await startPromise;
|
|
586
|
+
await loop;
|
|
587
|
+
const timer = setInterval(() => {
|
|
588
|
+
if (owner) redis.eval(RENEW, 1, heartbeat(owner), heartbeatTtlMs).catch((error) => debug("Queue close heartbeat:", error));
|
|
589
|
+
}, Math.floor(heartbeatTtlMs / 3));
|
|
590
|
+
try {
|
|
591
|
+
await Promise.all(inflight);
|
|
592
|
+
} finally {
|
|
593
|
+
clearInterval(timer);
|
|
594
|
+
}
|
|
595
|
+
if (owner) {
|
|
596
|
+
await redis.del(heartbeat(owner));
|
|
597
|
+
await redis.eval(RECLAIM, 4, processing(owner), ready, heartbeat(owner), owners, owner);
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
//#endregion
|
|
603
|
+
//#region src/storage.ts
|
|
604
|
+
/** Per-run event ceiling reported to core, mirroring the Vercel World. */
|
|
605
|
+
const DEFAULT_MAX_EVENTS_PER_RUN = 25e3;
|
|
606
|
+
/** Resolve the per-run event ceiling surfaced as `EventResult.maxEvents`:
|
|
607
|
+
* explicit config, then `WORKFLOW_MAX_EVENTS`, then the default. A
|
|
608
|
+
* non-positive configured value throws rather than being ignored. */
|
|
609
|
+
function resolveMaxEventsPerRun(configured) {
|
|
610
|
+
if (configured !== void 0) {
|
|
611
|
+
if (!Number.isInteger(configured) || configured <= 0) throw new TypeError(`maxEventsPerRun must be a positive integer, received ${String(configured)}`);
|
|
612
|
+
return configured;
|
|
613
|
+
}
|
|
614
|
+
const raw = process.env.WORKFLOW_MAX_EVENTS;
|
|
615
|
+
const parsed = raw !== void 0 ? Number(raw) : NaN;
|
|
616
|
+
return Number.isInteger(parsed) && parsed > 0 ? parsed : DEFAULT_MAX_EVENTS_PER_RUN;
|
|
617
|
+
}
|
|
618
|
+
/** Events that mutate the run, so validation needs its full body. Everything
|
|
619
|
+
* else only reads status/specVersion and is served from the run meta key. */
|
|
620
|
+
const RUN_LIFECYCLE_EVENT_TYPES = /* @__PURE__ */ new Set([
|
|
621
|
+
"run_started",
|
|
622
|
+
"run_completed",
|
|
623
|
+
"run_failed",
|
|
624
|
+
"run_cancelled"
|
|
625
|
+
]);
|
|
626
|
+
/** Sentinel from the event scripts when a creation-dedup check rejects. */
|
|
627
|
+
const LUA_DUPLICATE_CREATION = -8;
|
|
628
|
+
/**
|
|
629
|
+
* Stand-in for the event id inside the event JSON the caller hands to a script.
|
|
630
|
+
* The id is a log position (`evnt_` + the 1-based slot), and a position may only
|
|
631
|
+
* be chosen at the commit (see `LUA_APPEND_EVENT`), so the script writes it into
|
|
632
|
+
* the JSON itself. `serializeEvent` always emits `eventId` as the first key, so
|
|
633
|
+
* the first match is ours whatever the payload holds. A constant, not a random
|
|
634
|
+
* value: two bundled copies of this module can share one connection's scripts.
|
|
635
|
+
*/
|
|
636
|
+
const EVENT_ID_PLACEHOLDER = "__brick_event_id__";
|
|
637
|
+
/**
|
|
638
|
+
* Lua helper shared by every script that appends to a run's event log.
|
|
639
|
+
*
|
|
640
|
+
* A run's events live in one hash, `eventId -> JSON`. The log is dense and
|
|
641
|
+
* 1-based, and a script runs atomically, so the next slot is the hash length
|
|
642
|
+
* plus one and no other writer can interleave between reading it and writing the
|
|
643
|
+
* event. That is the whole race story: the slot is chosen at the commit, in the
|
|
644
|
+
* same operation that appends, and a writer that is behind simply lands on the
|
|
645
|
+
* next free slot (the caller reports what it skipped).
|
|
646
|
+
*
|
|
647
|
+
* `appendEvent(eventsKey, correlationKey, json)` returns the new event id.
|
|
648
|
+
* `correlationKey` may be nil. The id is written into `json` in place of the
|
|
649
|
+
* placeholder, so what is stored is exactly what the reader parses.
|
|
650
|
+
*/
|
|
651
|
+
const LUA_APPEND_EVENT = `
|
|
652
|
+
local function appendEvent(eventsKey, correlationKey, json)
|
|
653
|
+
local slot = redis.call('HLEN', eventsKey) + 1
|
|
654
|
+
local digits = string.format('%d', slot)
|
|
655
|
+
local eventId = 'evnt_' .. string.rep('0', 26 - #digits) .. digits
|
|
656
|
+
local from, to = string.find(json, '${EVENT_ID_PLACEHOLDER}', 1, true)
|
|
657
|
+
json = string.sub(json, 1, from - 1) .. eventId .. string.sub(json, to + 1)
|
|
658
|
+
redis.call('HSET', eventsKey, eventId, json)
|
|
659
|
+
if correlationKey then
|
|
660
|
+
redis.call('ZADD', correlationKey, slot, eventId)
|
|
661
|
+
end
|
|
662
|
+
return eventId
|
|
663
|
+
end
|
|
664
|
+
|
|
665
|
+
-- A run's status from its meta key ('<status>|<specVersion>'), or false when absent.
|
|
666
|
+
local function runStatus(metaKey)
|
|
667
|
+
local meta = redis.call('GET', metaKey)
|
|
668
|
+
if not meta then
|
|
669
|
+
return false
|
|
670
|
+
end
|
|
671
|
+
return string.match(meta, '^([^|]*)')
|
|
672
|
+
end
|
|
673
|
+
|
|
674
|
+
local function isTerminal(status)
|
|
675
|
+
return status == 'completed' or status == 'failed' or status == 'cancelled'
|
|
676
|
+
end
|
|
677
|
+
`;
|
|
678
|
+
/** Stringify an object with Uint8Array support. Delegates to the shared
|
|
679
|
+
* helper, which tags binary as base64; the previous number-array encoding
|
|
680
|
+
* cost ~450x more CPU to re-parse and ~2.7x more storage at 2MB. */
|
|
681
|
+
function stringifyWithUint8Array(obj) {
|
|
682
|
+
return stringify(obj);
|
|
683
|
+
}
|
|
684
|
+
/** Parse JSON with Uint8Array and Date support. The shared reviver decodes
|
|
685
|
+
* both the base64 tag and the legacy number-array tag, so rows already in
|
|
686
|
+
* Redis keep reading back correctly. */
|
|
687
|
+
function parseWithUint8Array(json) {
|
|
688
|
+
return parse(json);
|
|
689
|
+
}
|
|
690
|
+
/**
|
|
691
|
+
* Filter hook data based on resolveData parameter
|
|
692
|
+
*/
|
|
693
|
+
function filterHookData(hook, resolveData) {
|
|
694
|
+
if (resolveData === "none" && "metadata" in hook) {
|
|
695
|
+
const { metadata: _, ...rest } = hook;
|
|
696
|
+
return {
|
|
697
|
+
metadata: void 0,
|
|
698
|
+
...rest
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
return hook;
|
|
702
|
+
}
|
|
703
|
+
function filterStepData(step, resolveData) {
|
|
704
|
+
if (resolveData === "none") {
|
|
705
|
+
const { input: _, output: __, ...rest } = step;
|
|
706
|
+
return {
|
|
707
|
+
input: void 0,
|
|
708
|
+
output: void 0,
|
|
709
|
+
...rest
|
|
710
|
+
};
|
|
711
|
+
}
|
|
712
|
+
return step;
|
|
713
|
+
}
|
|
714
|
+
function filterRunData(run, resolveData) {
|
|
715
|
+
if (resolveData === "none") {
|
|
716
|
+
const { input: _, output: __, ...rest } = run;
|
|
717
|
+
return {
|
|
718
|
+
input: void 0,
|
|
719
|
+
output: void 0,
|
|
720
|
+
...rest
|
|
721
|
+
};
|
|
722
|
+
}
|
|
723
|
+
return run;
|
|
724
|
+
}
|
|
725
|
+
/**
|
|
726
|
+
* Read up to `want` members of one or more sorted-set indexes, in score order,
|
|
727
|
+
* resuming after `cursor`. With several indexes the members are merged into one
|
|
728
|
+
* ordering, which is how a listing across statuses avoids scanning an index of
|
|
729
|
+
* every run there has ever been.
|
|
730
|
+
*
|
|
731
|
+
* A cursor is the last member's `<score>|<id>`, not its rank: a rank means
|
|
732
|
+
* nothing once that member has left the index (a run that changed status, a
|
|
733
|
+
* hook that was disposed), and paging would fail halfway through a listing. A
|
|
734
|
+
* position by score and id stays valid wherever the member went. Members that
|
|
735
|
+
* share a score are ordered by id, as Redis orders them.
|
|
736
|
+
*/
|
|
737
|
+
async function pageIndex(redis, indexKeys, cursor, want, order = "desc") {
|
|
738
|
+
let after;
|
|
739
|
+
if (cursor) {
|
|
740
|
+
const sep = cursor.indexOf("|");
|
|
741
|
+
const score = sep > 0 ? Number(cursor.slice(0, sep)) : NaN;
|
|
742
|
+
if (Number.isNaN(score)) throw new WorkflowWorldError(`Invalid pagination cursor "${cursor}"`, { status: 400 });
|
|
743
|
+
after = {
|
|
744
|
+
score,
|
|
745
|
+
id: cursor.slice(sep + 1)
|
|
746
|
+
};
|
|
747
|
+
}
|
|
748
|
+
const bound = after ? String(after.score) : order === "desc" ? "+inf" : "-inf";
|
|
749
|
+
const readOne = async (indexKey) => {
|
|
750
|
+
const out = [];
|
|
751
|
+
const batch = Math.max(want, 50);
|
|
752
|
+
let offset = 0;
|
|
753
|
+
while (out.length < want) {
|
|
754
|
+
const raw = order === "desc" ? await redis.zrevrangebyscore(indexKey, bound, "-inf", "WITHSCORES", "LIMIT", offset, batch) : await redis.zrangebyscore(indexKey, bound, "+inf", "WITHSCORES", "LIMIT", offset, batch);
|
|
755
|
+
for (let i = 0; i < raw.length && out.length < want; i += 2) {
|
|
756
|
+
const id = raw[i];
|
|
757
|
+
const score = Number(raw[i + 1]);
|
|
758
|
+
if (after && score === after.score && (order === "desc" ? id >= after.id : id <= after.id)) continue;
|
|
759
|
+
out.push({
|
|
760
|
+
id,
|
|
761
|
+
score,
|
|
762
|
+
cursor: `${raw[i + 1]}|${id}`
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
offset += raw.length / 2;
|
|
766
|
+
if (raw.length / 2 < batch) break;
|
|
767
|
+
}
|
|
768
|
+
return out;
|
|
769
|
+
};
|
|
770
|
+
const keys = Array.isArray(indexKeys) ? indexKeys : [indexKeys];
|
|
771
|
+
if (keys.length === 1) return readOne(keys[0]);
|
|
772
|
+
const merged = (await Promise.all(keys.map(readOne))).flat();
|
|
773
|
+
const direction = order === "desc" ? -1 : 1;
|
|
774
|
+
merged.sort((x, y) => direction * (x.score - y.score || (x.id < y.id ? -1 : x.id > y.id ? 1 : 0)));
|
|
775
|
+
return merged.slice(0, want);
|
|
776
|
+
}
|
|
777
|
+
/**
|
|
778
|
+
* Atomically create a run entity with all its indexes AND its creation
|
|
779
|
+
* event. SETNX arbitrates duplicates: when the run already exists, NO
|
|
780
|
+
* index, meta, or event write happens (a replayed run_created must not
|
|
781
|
+
* reset index scores, re-insert a terminal run into the pending status
|
|
782
|
+
* index, or append a second creation event).
|
|
783
|
+
*
|
|
784
|
+
* KEYS[1] = run key
|
|
785
|
+
* KEYS[2] = runs index key
|
|
786
|
+
* KEYS[3] = runs by name key
|
|
787
|
+
* KEYS[4] = runs by status key (pending)
|
|
788
|
+
* KEYS[5] = run meta key
|
|
789
|
+
* KEYS[6] = events key (the run's log)
|
|
790
|
+
* KEYS[7] = run attributes key
|
|
791
|
+
* ARGV[1] = run JSON
|
|
792
|
+
* ARGV[2] = run ID
|
|
793
|
+
* ARGV[3] = score (timestamp)
|
|
794
|
+
* ARGV[4] = run meta value (`<status>|<specVersion>`)
|
|
795
|
+
* ARGV[5] = event JSON (event id left as the placeholder)
|
|
796
|
+
* ARGV[6..] = attribute key, value pairs to seed
|
|
797
|
+
* Returns: [1, eventId] when created, [0, runJson] on replay.
|
|
798
|
+
*/
|
|
799
|
+
const LUA_CREATE_RUN_WITH_EVENT = `${LUA_APPEND_EVENT}
|
|
800
|
+
local wasCreated = redis.call('SETNX', KEYS[1], ARGV[1])
|
|
801
|
+
if wasCreated == 0 then
|
|
802
|
+
return {0, redis.call('GET', KEYS[1])}
|
|
803
|
+
end
|
|
804
|
+
local score = tonumber(ARGV[3])
|
|
805
|
+
redis.call('ZADD', KEYS[2], score, ARGV[2])
|
|
806
|
+
redis.call('ZADD', KEYS[3], score, ARGV[2])
|
|
807
|
+
redis.call('ZADD', KEYS[4], score, ARGV[2])
|
|
808
|
+
redis.call('SET', KEYS[5], ARGV[4])
|
|
809
|
+
for i = 6, #ARGV, 2 do
|
|
810
|
+
redis.call('HSET', KEYS[7], ARGV[i], ARGV[i + 1])
|
|
811
|
+
end
|
|
812
|
+
return {1, appendEvent(KEYS[6], nil, ARGV[5])}
|
|
813
|
+
`;
|
|
814
|
+
/**
|
|
815
|
+
* Move a run to a new status, release its hooks when it finishes, and append the
|
|
816
|
+
* lifecycle event, all in one execution. They are one script because a crash
|
|
817
|
+
* between them would leave a terminal run with no terminal event (every retry is
|
|
818
|
+
* then refused as terminal) or a started run with no `run_started` (the redelivery
|
|
819
|
+
* takes the resume path and appends nothing).
|
|
820
|
+
*
|
|
821
|
+
* KEYS[1] = run key
|
|
822
|
+
* KEYS[2] = old status index key
|
|
823
|
+
* KEYS[3] = new status index key
|
|
824
|
+
* KEYS[4] = run meta key
|
|
825
|
+
* KEYS[5] = events key
|
|
826
|
+
* KEYS[6] = hooks index key
|
|
827
|
+
* KEYS[7] = hooks token-map key
|
|
828
|
+
* ARGV[1] = updated run JSON
|
|
829
|
+
* ARGV[2] = run ID
|
|
830
|
+
* ARGV[3] = score (timestamp)
|
|
831
|
+
* ARGV[4] = run meta value (`<status>|<specVersion>`)
|
|
832
|
+
* ARGV[5] = the status the caller read the run in
|
|
833
|
+
* ARGV[6] = event JSON
|
|
834
|
+
* ARGV[7] = "1" to release the run's hooks (a terminal transition)
|
|
835
|
+
* ARGV[8] = hook key prefix
|
|
836
|
+
* ARGV[9] = hooks-by-token key prefix
|
|
837
|
+
* Returns: [1, eventId] on success, [0, ''] if the run is absent, [-2, status] if
|
|
838
|
+
* the run is no longer in the status the caller read (another writer
|
|
839
|
+
* moved it first; nothing is written)
|
|
840
|
+
*/
|
|
841
|
+
const LUA_TRANSITION_RUN = `${LUA_APPEND_EVENT}
|
|
842
|
+
if redis.call('EXISTS', KEYS[1]) == 0 then
|
|
843
|
+
return {0, ''}
|
|
844
|
+
end
|
|
845
|
+
|
|
846
|
+
local current = runStatus(KEYS[4])
|
|
847
|
+
if current and current ~= ARGV[5] then
|
|
848
|
+
return {-2, current}
|
|
849
|
+
end
|
|
850
|
+
|
|
851
|
+
redis.call('SET', KEYS[1], ARGV[1])
|
|
852
|
+
redis.call('SET', KEYS[4], ARGV[4])
|
|
853
|
+
redis.call('ZREM', KEYS[2], ARGV[2])
|
|
854
|
+
redis.call('ZADD', KEYS[3], tonumber(ARGV[3]), ARGV[2])
|
|
855
|
+
|
|
856
|
+
if ARGV[7] == '1' then
|
|
857
|
+
local pairsFlat = redis.call('HGETALL', KEYS[7])
|
|
858
|
+
for i = 1, #pairsFlat, 2 do
|
|
859
|
+
local hookId = pairsFlat[i]
|
|
860
|
+
local tokenKey = ARGV[9] .. pairsFlat[i + 1]
|
|
861
|
+
redis.call('DEL', ARGV[8] .. hookId)
|
|
862
|
+
-- Only a token this hook still owns: a stale entry must not wipe a claim
|
|
863
|
+
-- another run has taken since.
|
|
864
|
+
if redis.call('GET', tokenKey) == hookId then
|
|
865
|
+
redis.call('DEL', tokenKey)
|
|
866
|
+
end
|
|
867
|
+
end
|
|
868
|
+
redis.call('DEL', KEYS[7])
|
|
869
|
+
redis.call('DEL', KEYS[6])
|
|
870
|
+
end
|
|
871
|
+
|
|
872
|
+
return {1, appendEvent(KEYS[5], nil, ARGV[6])}
|
|
873
|
+
`;
|
|
874
|
+
/**
|
|
875
|
+
* Atomically create a step entity with its index AND its creation event.
|
|
876
|
+
* SETNX arbitrates duplicates: a replayed step_created writes nothing. A step is
|
|
877
|
+
* never created on a finished run.
|
|
878
|
+
*
|
|
879
|
+
* KEYS[1] = step key
|
|
880
|
+
* KEYS[2] = steps index key
|
|
881
|
+
* KEYS[3] = events key
|
|
882
|
+
* KEYS[4] = events by correlation key
|
|
883
|
+
* KEYS[5] = step status key (see `LUA_STORE_EVENT`)
|
|
884
|
+
* KEYS[6] = run meta key
|
|
885
|
+
* ARGV[1] = step JSON
|
|
886
|
+
* ARGV[2] = step ID (correlationId)
|
|
887
|
+
* ARGV[3] = score (timestamp)
|
|
888
|
+
* ARGV[4] = event JSON
|
|
889
|
+
* ARGV[5] = step status value (`<status>|<attempt>`)
|
|
890
|
+
* Returns: [1, eventId] when created, [0, stepJson] on replay, [-6, status] when
|
|
891
|
+
* the run is terminal
|
|
892
|
+
*/
|
|
893
|
+
const LUA_CREATE_STEP_WITH_EVENT = `${LUA_APPEND_EVENT}
|
|
894
|
+
local status = runStatus(KEYS[6])
|
|
895
|
+
if status and isTerminal(status) then
|
|
896
|
+
return {-6, status}
|
|
897
|
+
end
|
|
898
|
+
local wasCreated = redis.call('SETNX', KEYS[1], ARGV[1])
|
|
899
|
+
if wasCreated == 0 then
|
|
900
|
+
return {0, redis.call('GET', KEYS[1])}
|
|
901
|
+
end
|
|
902
|
+
redis.call('ZADD', KEYS[2], tonumber(ARGV[3]), ARGV[2])
|
|
903
|
+
redis.call('SET', KEYS[5], ARGV[5])
|
|
904
|
+
return {1, appendEvent(KEYS[3], KEYS[4], ARGV[4])}
|
|
905
|
+
`;
|
|
906
|
+
/**
|
|
907
|
+
* Lazy step start: a `step_started` that carries the step's input also creates
|
|
908
|
+
* the step. The step entity (already running, attempt 1), the synthetic
|
|
909
|
+
* `step_created` and the `step_started` land together, so replay still sees the
|
|
910
|
+
* creation, and exactly one concurrent caller wins the SETNX and runs the body.
|
|
911
|
+
*
|
|
912
|
+
* KEYS[1] = step key
|
|
913
|
+
* KEYS[2] = steps index key
|
|
914
|
+
* KEYS[3] = events key
|
|
915
|
+
* KEYS[4] = events by correlation key
|
|
916
|
+
* KEYS[5] = step status key (see `LUA_STORE_EVENT`)
|
|
917
|
+
* KEYS[6] = run meta key
|
|
918
|
+
* ARGV[1] = step JSON (running)
|
|
919
|
+
* ARGV[2] = step ID (correlationId)
|
|
920
|
+
* ARGV[3] = score (timestamp)
|
|
921
|
+
* ARGV[4] = step_created event JSON
|
|
922
|
+
* ARGV[5] = step_started event JSON
|
|
923
|
+
* ARGV[6] = step status value (`running|1`)
|
|
924
|
+
* Returns: [1, createdEventId, startedEventId] when created, [0, stepJson] when
|
|
925
|
+
* the step already exists (the caller lost the claim), [-6, status]
|
|
926
|
+
* when the run is terminal
|
|
927
|
+
*/
|
|
928
|
+
const LUA_CREATE_STEP_STARTED = `${LUA_APPEND_EVENT}
|
|
929
|
+
local status = runStatus(KEYS[6])
|
|
930
|
+
if status and isTerminal(status) then
|
|
931
|
+
return {-6, status}
|
|
932
|
+
end
|
|
933
|
+
local wasCreated = redis.call('SETNX', KEYS[1], ARGV[1])
|
|
934
|
+
if wasCreated == 0 then
|
|
935
|
+
return {0, redis.call('GET', KEYS[1])}
|
|
936
|
+
end
|
|
937
|
+
redis.call('ZADD', KEYS[2], tonumber(ARGV[3]), ARGV[2])
|
|
938
|
+
redis.call('SET', KEYS[5], ARGV[6])
|
|
939
|
+
local createdId = appendEvent(KEYS[3], KEYS[4], ARGV[4])
|
|
940
|
+
local startedId = appendEvent(KEYS[3], KEYS[4], ARGV[5])
|
|
941
|
+
return {1, createdId, startedId}
|
|
942
|
+
`;
|
|
943
|
+
/**
|
|
944
|
+
* Atomically claim a hook token and create the hook entity, its indexes,
|
|
945
|
+
* token-map entry, AND its creation event. The by-token key is the claim
|
|
946
|
+
* arbiter (SETNX):
|
|
947
|
+
* - A DIFFERENT hookId already owning the token -> cross-run conflict; the
|
|
948
|
+
* rightful owner's token mapping is left untouched. If that owner's hook no
|
|
949
|
+
* longer exists (a crash during cleanup left the token behind), the token is
|
|
950
|
+
* free and is taken over here, in the same execution, so a concurrent claim
|
|
951
|
+
* can never be wiped.
|
|
952
|
+
* - The SAME hookId owning the token with the entity present -> duplicate.
|
|
953
|
+
* Entity + event are written in the same atomic script, so a crash can
|
|
954
|
+
* never leave a hook entity without its hook_created event (or vice versa). A
|
|
955
|
+
* hook is never created on a finished run, which would hold its token forever.
|
|
956
|
+
*
|
|
957
|
+
* KEYS[1] = hook key
|
|
958
|
+
* KEYS[2] = hooks by token key
|
|
959
|
+
* KEYS[3] = hooks index key
|
|
960
|
+
* KEYS[4] = hooks token-map key (hash of hookId -> token, per run)
|
|
961
|
+
* KEYS[5] = events key
|
|
962
|
+
* KEYS[6] = events by correlation key
|
|
963
|
+
* KEYS[7] = run meta key
|
|
964
|
+
* ARGV[1] = hook JSON
|
|
965
|
+
* ARGV[2] = hook ID (correlationId)
|
|
966
|
+
* ARGV[3] = score (timestamp)
|
|
967
|
+
* ARGV[4] = the token, recorded in the token map so terminal-run cleanup can
|
|
968
|
+
* drop token lookups without reading every hook body
|
|
969
|
+
* ARGV[5] = event JSON
|
|
970
|
+
* ARGV[6] = hook key prefix
|
|
971
|
+
* Returns: [2, owningHookId] on token conflict,
|
|
972
|
+
* [0, hookJson] when the hook already exists, or has existed (created
|
|
973
|
+
* and since disposed),
|
|
974
|
+
* [1, eventId] on success,
|
|
975
|
+
* [-6, status] when the run is terminal
|
|
976
|
+
*/
|
|
977
|
+
const LUA_CREATE_HOOK_WITH_EVENT = `${LUA_APPEND_EVENT}
|
|
978
|
+
local status = runStatus(KEYS[7])
|
|
979
|
+
if status and isTerminal(status) then
|
|
980
|
+
return {-6, status}
|
|
981
|
+
end
|
|
982
|
+
-- Disposal removes the hook and its token, so a stale writer that loaded the log
|
|
983
|
+
-- before the hook existed would otherwise create it again behind its disposal.
|
|
984
|
+
-- The creation event is the record that it already happened.
|
|
985
|
+
if redis.call('ZCARD', KEYS[6]) > 0 then
|
|
986
|
+
return {0, ''}
|
|
987
|
+
end
|
|
988
|
+
local claimed = redis.call('SETNX', KEYS[2], ARGV[2])
|
|
989
|
+
if claimed == 0 then
|
|
990
|
+
local owner = redis.call('GET', KEYS[2])
|
|
991
|
+
if owner ~= ARGV[2] then
|
|
992
|
+
if redis.call('EXISTS', ARGV[6] .. owner) == 1 then
|
|
993
|
+
return {2, owner}
|
|
994
|
+
end
|
|
995
|
+
redis.call('SET', KEYS[2], ARGV[2])
|
|
996
|
+
end
|
|
997
|
+
end
|
|
998
|
+
local wasCreated = redis.call('SETNX', KEYS[1], ARGV[1])
|
|
999
|
+
if wasCreated == 0 then
|
|
1000
|
+
return {0, redis.call('GET', KEYS[1])}
|
|
1001
|
+
end
|
|
1002
|
+
redis.call('ZADD', KEYS[3], tonumber(ARGV[3]), ARGV[2])
|
|
1003
|
+
redis.call('HSET', KEYS[4], ARGV[2], ARGV[4])
|
|
1004
|
+
return {1, appendEvent(KEYS[5], KEYS[6], ARGV[5])}
|
|
1005
|
+
`;
|
|
1006
|
+
/**
|
|
1007
|
+
* Atomically dispose a hook: delete the hook key, its token lookup (only while
|
|
1008
|
+
* it still points at this hook), its index entry and its token-map entry, and
|
|
1009
|
+
* append `hook_disposed`. One script, because a crash between the delete and the
|
|
1010
|
+
* append would lose the event, and a delete racing a `hook_received` must not
|
|
1011
|
+
* leave the received event behind the disposal in the log.
|
|
1012
|
+
*
|
|
1013
|
+
* KEYS[1] = hook key
|
|
1014
|
+
* KEYS[2] = hooks by token key
|
|
1015
|
+
* KEYS[3] = hooks index key
|
|
1016
|
+
* KEYS[4] = hooks token-map key
|
|
1017
|
+
* KEYS[5] = events key
|
|
1018
|
+
* KEYS[6] = events by correlation key
|
|
1019
|
+
* ARGV[1] = hook ID (correlationId)
|
|
1020
|
+
* ARGV[2] = event JSON
|
|
1021
|
+
* Returns: [1, eventId] if disposed, [0, ''] if the hook was not found
|
|
1022
|
+
*/
|
|
1023
|
+
const LUA_DISPOSE_HOOK = `${LUA_APPEND_EVENT}
|
|
1024
|
+
if redis.call('DEL', KEYS[1]) == 0 then
|
|
1025
|
+
return {0, ''}
|
|
1026
|
+
end
|
|
1027
|
+
if redis.call('GET', KEYS[2]) == ARGV[1] then
|
|
1028
|
+
redis.call('DEL', KEYS[2])
|
|
1029
|
+
end
|
|
1030
|
+
redis.call('ZREM', KEYS[3], ARGV[1])
|
|
1031
|
+
redis.call('HDEL', KEYS[4], ARGV[1])
|
|
1032
|
+
return {1, appendEvent(KEYS[5], KEYS[6], ARGV[2])}
|
|
1033
|
+
`;
|
|
1034
|
+
/**
|
|
1035
|
+
* Atomically append an event to the run's log and its correlation index,
|
|
1036
|
+
* optionally writing an entity first so the mutation and its event commit
|
|
1037
|
+
* together.
|
|
1038
|
+
*
|
|
1039
|
+
* KEYS[1] = events key
|
|
1040
|
+
* KEYS[2] = events by correlation key (or unused placeholder)
|
|
1041
|
+
* KEYS[3] = entity key (or unused placeholder)
|
|
1042
|
+
* KEYS[4] = hook resume claim key (or unused placeholder)
|
|
1043
|
+
* KEYS[5] = step status key (or unused placeholder)
|
|
1044
|
+
* KEYS[6] = run meta key (or unused placeholder)
|
|
1045
|
+
* KEYS[7] = hook key (or unused placeholder)
|
|
1046
|
+
* ARGV[1] = event JSON
|
|
1047
|
+
* ARGV[2] = has correlation ("1" or "0")
|
|
1048
|
+
* ARGV[3] = creation dedup: "1" rejects when the correlation index already holds
|
|
1049
|
+
* an event (a wait or hook conflict recorded once), "2" rejects when it
|
|
1050
|
+
* already holds two (a wait completed once)
|
|
1051
|
+
* ARGV[4] = entity JSON to write to KEYS[3] ('' to write nothing)
|
|
1052
|
+
* ARGV[5] = resume claim payload ('' when the event is not a lazy hook resume)
|
|
1053
|
+
* ARGV[6] = the step's new status ('' when the event writes no step)
|
|
1054
|
+
* ARGV[7] = the attempt the caller read ('' to skip the check)
|
|
1055
|
+
* ARGV[8] = the step's new attempt ('' to keep the current one)
|
|
1056
|
+
* ARGV[9] = "1" to refuse when the run is terminal
|
|
1057
|
+
* ARGV[10] = "1" to refuse when the hook (KEYS[7]) does not exist
|
|
1058
|
+
*
|
|
1059
|
+
* A lazy hook resume is claimed per (run, resumeId): the first writer appends and
|
|
1060
|
+
* records `<eventId>|<claim payload>`, a later one appends nothing and gets the
|
|
1061
|
+
* recorded claim back, so both writers converge on one event.
|
|
1062
|
+
*
|
|
1063
|
+
* A step write is a transition out of the state the caller read, so it is checked
|
|
1064
|
+
* where the state lives: a terminal step takes no further write, and a write that
|
|
1065
|
+
* read an older attempt than the step now has lost to another writer. Without
|
|
1066
|
+
* this two deliveries could both record an outcome, and the log would hold both.
|
|
1067
|
+
* The attempt counter is kept here, not taken from the caller's possibly stale
|
|
1068
|
+
* copy of the step, so it only moves forward.
|
|
1069
|
+
*
|
|
1070
|
+
* The run and hook guards live here because the checks the caller makes before
|
|
1071
|
+
* calling are stale by the time the event is appended: a `hook_received` must not
|
|
1072
|
+
* land behind the hook's disposal or the run's end, which no replay can consume.
|
|
1073
|
+
*
|
|
1074
|
+
* Returns: [1, eventId] on success, [-8, ''] when creation dedup rejects,
|
|
1075
|
+
* [-7, ''] when completion dedup rejects, [3, claim] when a resume claim
|
|
1076
|
+
* already exists, [-3, status] when the step is already terminal,
|
|
1077
|
+
* [-4, attempt] when the step has moved past the attempt the caller
|
|
1078
|
+
* read, [-6, status] when the run is terminal, [-5, ''] when the hook
|
|
1079
|
+
* is gone
|
|
1080
|
+
*/
|
|
1081
|
+
const LUA_STORE_EVENT = `${LUA_APPEND_EVENT}
|
|
1082
|
+
if ARGV[5] ~= '' then
|
|
1083
|
+
local claim = redis.call('GET', KEYS[4])
|
|
1084
|
+
if claim then
|
|
1085
|
+
return {3, claim}
|
|
1086
|
+
end
|
|
1087
|
+
end
|
|
1088
|
+
|
|
1089
|
+
if ARGV[10] == '1' and redis.call('EXISTS', KEYS[7]) == 0 then
|
|
1090
|
+
return {-5, ''}
|
|
1091
|
+
end
|
|
1092
|
+
|
|
1093
|
+
if ARGV[9] == '1' then
|
|
1094
|
+
local status = runStatus(KEYS[6])
|
|
1095
|
+
if status and isTerminal(status) then
|
|
1096
|
+
return {-6, status}
|
|
1097
|
+
end
|
|
1098
|
+
end
|
|
1099
|
+
|
|
1100
|
+
if ARGV[3] == '1' and redis.call('ZCARD', KEYS[2]) > 0 then
|
|
1101
|
+
return {${LUA_DUPLICATE_CREATION}, ''}
|
|
1102
|
+
end
|
|
1103
|
+
if ARGV[3] == '2' and redis.call('ZCARD', KEYS[2]) > 1 then
|
|
1104
|
+
return {-7, ''}
|
|
1105
|
+
end
|
|
1106
|
+
|
|
1107
|
+
if ARGV[6] ~= '' then
|
|
1108
|
+
local current = redis.call('GET', KEYS[5])
|
|
1109
|
+
-- No key (a step written before it existed): the caller's read is the best
|
|
1110
|
+
-- attempt there is. Assuming 0 would make every later start look stale.
|
|
1111
|
+
local attempt = '0'
|
|
1112
|
+
if ARGV[7] ~= '' then
|
|
1113
|
+
attempt = ARGV[7]
|
|
1114
|
+
end
|
|
1115
|
+
if current then
|
|
1116
|
+
local status
|
|
1117
|
+
status, attempt = string.match(current, '^([^|]*)|?(.*)$')
|
|
1118
|
+
if status == 'completed' or status == 'failed' or status == 'cancelled' then
|
|
1119
|
+
return {-3, status}
|
|
1120
|
+
end
|
|
1121
|
+
if ARGV[7] ~= '' and attempt ~= ARGV[7] then
|
|
1122
|
+
return {-4, attempt}
|
|
1123
|
+
end
|
|
1124
|
+
end
|
|
1125
|
+
if ARGV[8] ~= '' then
|
|
1126
|
+
attempt = ARGV[8]
|
|
1127
|
+
end
|
|
1128
|
+
redis.call('SET', KEYS[5], ARGV[6] .. '|' .. attempt)
|
|
1129
|
+
end
|
|
1130
|
+
|
|
1131
|
+
if ARGV[4] ~= '' then
|
|
1132
|
+
redis.call('SET', KEYS[3], ARGV[4])
|
|
1133
|
+
end
|
|
1134
|
+
|
|
1135
|
+
local correlationKey = nil
|
|
1136
|
+
if ARGV[2] == '1' then
|
|
1137
|
+
correlationKey = KEYS[2]
|
|
1138
|
+
end
|
|
1139
|
+
local eventId = appendEvent(KEYS[1], correlationKey, ARGV[1])
|
|
1140
|
+
if ARGV[5] ~= '' then
|
|
1141
|
+
redis.call('SET', KEYS[4], eventId .. '|' .. ARGV[5])
|
|
1142
|
+
end
|
|
1143
|
+
return {1, eventId}
|
|
1144
|
+
`;
|
|
1145
|
+
/**
|
|
1146
|
+
* Atomically apply a batch of attribute changes and append the `attr_set` event.
|
|
1147
|
+
* Attributes live in a hash of their own, not in the run JSON, so two writers
|
|
1148
|
+
* changing different keys never overwrite each other.
|
|
1149
|
+
*
|
|
1150
|
+
* KEYS[1] = run attributes key
|
|
1151
|
+
* KEYS[2] = events key
|
|
1152
|
+
* KEYS[3] = events by correlation key (or unused placeholder)
|
|
1153
|
+
* KEYS[4] = run meta key
|
|
1154
|
+
* ARGV[1] = event JSON
|
|
1155
|
+
* ARGV[2] = has correlation ("1" or "0")
|
|
1156
|
+
* ARGV[3] = "1" to refuse a repeat for this correlation id (a replayed write)
|
|
1157
|
+
* ARGV[4..] = triples: attribute key, value, op ("set" or "del")
|
|
1158
|
+
* Returns: [1, eventId], [-6, status] when the run is terminal, [-8, ''] when
|
|
1159
|
+
* this correlation id already set attributes
|
|
1160
|
+
*/
|
|
1161
|
+
const LUA_SET_ATTRIBUTES = `${LUA_APPEND_EVENT}
|
|
1162
|
+
local status = runStatus(KEYS[4])
|
|
1163
|
+
if status and isTerminal(status) then
|
|
1164
|
+
return {-6, status}
|
|
1165
|
+
end
|
|
1166
|
+
if ARGV[3] == '1' and ARGV[2] == '1' and redis.call('ZCARD', KEYS[3]) > 0 then
|
|
1167
|
+
return {${LUA_DUPLICATE_CREATION}, ''}
|
|
1168
|
+
end
|
|
1169
|
+
for i = 4, #ARGV, 3 do
|
|
1170
|
+
if ARGV[i + 2] == 'del' then
|
|
1171
|
+
redis.call('HDEL', KEYS[1], ARGV[i])
|
|
1172
|
+
else
|
|
1173
|
+
redis.call('HSET', KEYS[1], ARGV[i], ARGV[i + 1])
|
|
1174
|
+
end
|
|
1175
|
+
end
|
|
1176
|
+
local correlationKey = nil
|
|
1177
|
+
if ARGV[2] == '1' then
|
|
1178
|
+
correlationKey = KEYS[3]
|
|
1179
|
+
end
|
|
1180
|
+
return {1, appendEvent(KEYS[2], correlationKey, ARGV[1])}
|
|
1181
|
+
`;
|
|
1182
|
+
/** Registered via `defineCommand` so ioredis dispatches with `EVALSHA` rather
|
|
1183
|
+
* than shipping the script body every call. `LUA_STORE_EVENT` is ~1 KB and
|
|
1184
|
+
* runs once per workflow event. */
|
|
1185
|
+
const SCRIPTS = {
|
|
1186
|
+
wfCreateRunWithEvent: {
|
|
1187
|
+
numberOfKeys: 7,
|
|
1188
|
+
lua: LUA_CREATE_RUN_WITH_EVENT
|
|
1189
|
+
},
|
|
1190
|
+
wfSetAttributes: {
|
|
1191
|
+
numberOfKeys: 4,
|
|
1192
|
+
lua: LUA_SET_ATTRIBUTES
|
|
1193
|
+
},
|
|
1194
|
+
wfTransitionRun: {
|
|
1195
|
+
numberOfKeys: 7,
|
|
1196
|
+
lua: LUA_TRANSITION_RUN
|
|
1197
|
+
},
|
|
1198
|
+
wfCreateStepWithEvent: {
|
|
1199
|
+
numberOfKeys: 6,
|
|
1200
|
+
lua: LUA_CREATE_STEP_WITH_EVENT
|
|
1201
|
+
},
|
|
1202
|
+
wfCreateStepStarted: {
|
|
1203
|
+
numberOfKeys: 6,
|
|
1204
|
+
lua: LUA_CREATE_STEP_STARTED
|
|
1205
|
+
},
|
|
1206
|
+
wfCreateHookWithEvent: {
|
|
1207
|
+
numberOfKeys: 7,
|
|
1208
|
+
lua: LUA_CREATE_HOOK_WITH_EVENT
|
|
1209
|
+
},
|
|
1210
|
+
wfDisposeHook: {
|
|
1211
|
+
numberOfKeys: 6,
|
|
1212
|
+
lua: LUA_DISPOSE_HOOK
|
|
1213
|
+
},
|
|
1214
|
+
wfStoreEvent: {
|
|
1215
|
+
numberOfKeys: 7,
|
|
1216
|
+
lua: LUA_STORE_EVENT
|
|
1217
|
+
}
|
|
1218
|
+
};
|
|
1219
|
+
/** Clients that already have the scripts attached; every storage factory
|
|
1220
|
+
* calls this with the same shared connection, so it must be idempotent. */
|
|
1221
|
+
const scriptState = globalSingleton("@brickkit/redis-world//scripts", 1, () => ({ clients: /* @__PURE__ */ new WeakSet() }));
|
|
1222
|
+
function withScripts(redis) {
|
|
1223
|
+
if (!scriptState.clients.has(redis)) {
|
|
1224
|
+
for (const [name, def] of Object.entries(SCRIPTS)) redis.defineCommand(name, {
|
|
1225
|
+
numberOfKeys: def.numberOfKeys,
|
|
1226
|
+
lua: def.lua
|
|
1227
|
+
});
|
|
1228
|
+
scriptState.clients.add(redis);
|
|
1229
|
+
}
|
|
1230
|
+
return redis;
|
|
1231
|
+
}
|
|
1232
|
+
/** Compact `<status>|<specVersion>` marker beside the run blob, so validation
|
|
1233
|
+
* reads those two fields without transferring the run's input/output. */
|
|
1234
|
+
function runMetaValue(status, specVersion) {
|
|
1235
|
+
return `${status}|${specVersion ?? ""}`;
|
|
1236
|
+
}
|
|
1237
|
+
function parseRunMeta(value) {
|
|
1238
|
+
const sep = value.indexOf("|");
|
|
1239
|
+
if (sep === -1) return { status: value };
|
|
1240
|
+
const version = value.slice(sep + 1);
|
|
1241
|
+
return {
|
|
1242
|
+
status: value.slice(0, sep),
|
|
1243
|
+
specVersion: version === "" ? void 0 : Number(version)
|
|
1244
|
+
};
|
|
1245
|
+
}
|
|
1246
|
+
/**
|
|
1247
|
+
* Create storage for workflow runs using Redis hashes and sorted sets
|
|
1248
|
+
*/
|
|
1249
|
+
function createRunsStorage(config) {
|
|
1250
|
+
const { redis, keyPrefix } = config;
|
|
1251
|
+
const runKey = (id) => `${keyPrefix}run:${id}`;
|
|
1252
|
+
const runAttributesKey = (id) => `${keyPrefix}run:attrs:${id}`;
|
|
1253
|
+
const runsIndexKey = () => `${keyPrefix}runs:index`;
|
|
1254
|
+
const runsByNameKey = (name) => `${keyPrefix}runs:by_name:${name}`;
|
|
1255
|
+
const runsByStatusKey = (status) => `${keyPrefix}runs:by_status:${status}`;
|
|
1256
|
+
/** The statuses a list filter allows, or undefined for any. `status` takes one
|
|
1257
|
+
* status or an array; an empty array matches nothing, unlike omitting it. */
|
|
1258
|
+
function allowedStatuses(params) {
|
|
1259
|
+
const status = params?.status;
|
|
1260
|
+
if (status === void 0) return void 0;
|
|
1261
|
+
return Array.isArray(status) ? status : [status];
|
|
1262
|
+
}
|
|
1263
|
+
function selectIndexKeys(params) {
|
|
1264
|
+
if (params?.workflowName) return [runsByNameKey(params.workflowName)];
|
|
1265
|
+
const statuses = allowedStatuses(params);
|
|
1266
|
+
if (statuses) return statuses.map(runsByStatusKey);
|
|
1267
|
+
return [runsIndexKey()];
|
|
1268
|
+
}
|
|
1269
|
+
function parseRunsFromPipeline(results, params) {
|
|
1270
|
+
const runs = [];
|
|
1271
|
+
for (let i = 0; i + 1 < (results?.length ?? 0); i += 2) {
|
|
1272
|
+
const result = results[i];
|
|
1273
|
+
if (!result?.[1]) continue;
|
|
1274
|
+
const run = parseWithUint8Array(result[1]);
|
|
1275
|
+
run.attributes = results[i + 1]?.[1] ?? {};
|
|
1276
|
+
const statuses = allowedStatuses(params);
|
|
1277
|
+
const statusMatches = !statuses || statuses.includes(run.status);
|
|
1278
|
+
const nameMatches = !params?.workflowName || run.workflowName === params.workflowName;
|
|
1279
|
+
if (statusMatches && nameMatches) {
|
|
1280
|
+
const resolveData = params?.resolveData ?? "all";
|
|
1281
|
+
const parsed = WorkflowRunSchema.parse(compact(run));
|
|
1282
|
+
runs.push(filterRunData(parsed, resolveData));
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
return runs;
|
|
1286
|
+
}
|
|
1287
|
+
return {
|
|
1288
|
+
get: (async (id, params) => {
|
|
1289
|
+
const replies = await redis.pipeline().get(runKey(id)).hgetall(runAttributesKey(id)).exec();
|
|
1290
|
+
const data = replies?.[0]?.[1];
|
|
1291
|
+
if (!data) throw new WorkflowRunNotFoundError(id);
|
|
1292
|
+
const run = parseWithUint8Array(data);
|
|
1293
|
+
run.attributes = replies?.[1]?.[1] ?? {};
|
|
1294
|
+
return filterRunData(WorkflowRunSchema.parse(compact(run)), params?.resolveData ?? "all");
|
|
1295
|
+
}),
|
|
1296
|
+
list: (async (params) => {
|
|
1297
|
+
const limit = params?.pagination?.limit ?? 20;
|
|
1298
|
+
const fromCursor = params?.pagination?.cursor;
|
|
1299
|
+
if (allowedStatuses(params)?.length === 0) return {
|
|
1300
|
+
data: [],
|
|
1301
|
+
hasMore: false,
|
|
1302
|
+
cursor: null
|
|
1303
|
+
};
|
|
1304
|
+
const indexKeys = selectIndexKeys(params);
|
|
1305
|
+
const order = params?.pagination?.sortOrder ?? "desc";
|
|
1306
|
+
const runs = [];
|
|
1307
|
+
let scanFrom = fromCursor;
|
|
1308
|
+
while (runs.length <= limit) {
|
|
1309
|
+
const entries = await pageIndex(redis, indexKeys, scanFrom, limit + 1, order);
|
|
1310
|
+
if (entries.length === 0) break;
|
|
1311
|
+
const pipeline = redis.pipeline();
|
|
1312
|
+
for (const { id } of entries) {
|
|
1313
|
+
pipeline.get(runKey(id));
|
|
1314
|
+
pipeline.hgetall(runAttributesKey(id));
|
|
1315
|
+
}
|
|
1316
|
+
const results = await pipeline.exec();
|
|
1317
|
+
const cursorOf = new Map(entries.map((entry) => [entry.id, entry.cursor]));
|
|
1318
|
+
for (const run of parseRunsFromPipeline(results, params)) runs.push({
|
|
1319
|
+
run,
|
|
1320
|
+
cursor: cursorOf.get(run.runId)
|
|
1321
|
+
});
|
|
1322
|
+
if (entries.length < limit + 1) break;
|
|
1323
|
+
scanFrom = entries.at(-1).cursor;
|
|
1324
|
+
}
|
|
1325
|
+
const values = runs.slice(0, limit);
|
|
1326
|
+
const hasMore = runs.length > limit;
|
|
1327
|
+
return {
|
|
1328
|
+
data: values.map((entry) => entry.run),
|
|
1329
|
+
hasMore,
|
|
1330
|
+
cursor: values.at(-1)?.cursor ?? null
|
|
1331
|
+
};
|
|
1332
|
+
})
|
|
1333
|
+
};
|
|
1334
|
+
}
|
|
1335
|
+
/**
|
|
1336
|
+
* Create storage for workflow events using Redis hashes and sorted sets.
|
|
1337
|
+
*
|
|
1338
|
+
* A run's log is one hash (`eventId -> JSON`). Event ids are log positions, so
|
|
1339
|
+
* the log is dense and 1-based, and every script that appends to it picks the
|
|
1340
|
+
* next position itself, atomically with the write (see `LUA_APPEND_EVENT`).
|
|
1341
|
+
*/
|
|
1342
|
+
function createEventsStorage(config) {
|
|
1343
|
+
const { redis, keyPrefix } = config;
|
|
1344
|
+
const scripts = withScripts(redis);
|
|
1345
|
+
const ulid = monotonicFactory();
|
|
1346
|
+
const maxEventsPerRun = resolveMaxEventsPerRun(config.maxEventsPerRun);
|
|
1347
|
+
const eventsKey = (runId) => `${keyPrefix}events:data:${runId}`;
|
|
1348
|
+
const eventsByCorrelationKey = (runId, correlationId) => `${keyPrefix}events:by_correlation:${runId}:${correlationId}`;
|
|
1349
|
+
const resumeClaimKey = (runId, resumeId) => `${keyPrefix}events:resume:${runId}:${resumeId}`;
|
|
1350
|
+
const runKey = (id) => `${keyPrefix}run:${id}`;
|
|
1351
|
+
const runAttributesKey = (id) => `${keyPrefix}run:attrs:${id}`;
|
|
1352
|
+
const runMetaKey = (id) => `${keyPrefix}run:meta:${id}`;
|
|
1353
|
+
const runsIndexKey = () => `${keyPrefix}runs:index`;
|
|
1354
|
+
const runsByNameKey = (name) => `${keyPrefix}runs:by_name:${name}`;
|
|
1355
|
+
const runsByStatusKey = (status) => `${keyPrefix}runs:by_status:${status}`;
|
|
1356
|
+
const stepKey = (runId, stepId) => `${keyPrefix}step:${runId}:${stepId}`;
|
|
1357
|
+
const stepsIndexKey = (runId) => `${keyPrefix}steps:by_run:${runId}`;
|
|
1358
|
+
const stepStatusKey = (runId, stepId) => `${keyPrefix}step:status:${runId}:${stepId}`;
|
|
1359
|
+
const hookKey = (hookId) => `${keyPrefix}hook:${hookId}`;
|
|
1360
|
+
const hooksByTokenKey = (token) => `${keyPrefix}hooks:by_token:${token}`;
|
|
1361
|
+
const hooksIndexKey = (runId) => `${keyPrefix}hooks:by_run:${runId}`;
|
|
1362
|
+
const hooksTokenMapKey = (runId) => `${keyPrefix}hooks:tokens:${runId}`;
|
|
1363
|
+
/** Events per `HMGET`; keeps a long log from building one huge command. */
|
|
1364
|
+
const READ_CHUNK = 500;
|
|
1365
|
+
/** The most events an inline delta or skipped-slot report carries; the
|
|
1366
|
+
* caller continues from the cursor (or reloads) when `hasMore` is set. */
|
|
1367
|
+
const MAX_INLINE_EVENTS = 200;
|
|
1368
|
+
/** A run with its current attributes, which live apart from the run body. */
|
|
1369
|
+
async function withAttributes(run) {
|
|
1370
|
+
const attributes = await redis.hgetall(runAttributesKey(run.runId));
|
|
1371
|
+
return {
|
|
1372
|
+
...run,
|
|
1373
|
+
attributes
|
|
1374
|
+
};
|
|
1375
|
+
}
|
|
1376
|
+
/** `[key, value, ...]` for a script, from an attribute record. */
|
|
1377
|
+
const flattenAttributes = (attributes) => Object.entries(attributes ?? {}).flat();
|
|
1378
|
+
/** Serialize an event for a script: `eventId` first, holding the placeholder. */
|
|
1379
|
+
function serializeEvent(event) {
|
|
1380
|
+
const { eventId: _eventId, ...rest } = event;
|
|
1381
|
+
return stringifyWithUint8Array({
|
|
1382
|
+
eventId: EVENT_ID_PLACEHOLDER,
|
|
1383
|
+
...rest
|
|
1384
|
+
});
|
|
1385
|
+
}
|
|
1386
|
+
/** Read events by id out of one run's log, skipping any that are absent. */
|
|
1387
|
+
async function readEvents(runId, eventIds) {
|
|
1388
|
+
const events = [];
|
|
1389
|
+
for (let i = 0; i < eventIds.length; i += READ_CHUNK) {
|
|
1390
|
+
const values = await redis.hmget(eventsKey(runId), ...eventIds.slice(i, i + READ_CHUNK));
|
|
1391
|
+
for (const value of values) if (value) events.push(EventSchema.parse(compact(parseWithUint8Array(value))));
|
|
1392
|
+
}
|
|
1393
|
+
return events;
|
|
1394
|
+
}
|
|
1395
|
+
/** Events at slots `from..to` inclusive, in slot order. */
|
|
1396
|
+
async function readSlots(runId, from, to) {
|
|
1397
|
+
const ids = [];
|
|
1398
|
+
for (let slot = Math.max(from, FIRST_EVENT_SLOT); slot <= to; slot++) ids.push(slotToEventId(slot));
|
|
1399
|
+
return readEvents(runId, ids);
|
|
1400
|
+
}
|
|
1401
|
+
/**
|
|
1402
|
+
* Move a run from `oldStatus` to `newStatus`, release its hooks when it
|
|
1403
|
+
* finishes, and append the lifecycle event: one script, so a crash or a racing
|
|
1404
|
+
* writer can never leave the status and the log disagreeing. `moved` means the
|
|
1405
|
+
* run was no longer in `oldStatus` by the time the script ran, and nothing was
|
|
1406
|
+
* written.
|
|
1407
|
+
*/
|
|
1408
|
+
async function transitionRun(runId, oldStatus, newStatus, updatedRun, at, event, releaseHooks) {
|
|
1409
|
+
const result = await scripts.wfTransitionRun(runKey(runId), runsByStatusKey(oldStatus), runsByStatusKey(newStatus), runMetaKey(runId), eventsKey(runId), hooksIndexKey(runId), hooksTokenMapKey(runId), stringifyWithUint8Array(updatedRun), runId, at.getTime().toString(), runMetaValue(newStatus, updatedRun.specVersion), oldStatus, serializeEvent(event), releaseHooks ? "1" : "0", `${keyPrefix}hook:`, `${keyPrefix}hooks:by_token:`);
|
|
1410
|
+
if (result[0] === 1) return {
|
|
1411
|
+
kind: "ok",
|
|
1412
|
+
eventId: result[1]
|
|
1413
|
+
};
|
|
1414
|
+
if (result[0] === -2) return {
|
|
1415
|
+
kind: "moved",
|
|
1416
|
+
status: result[1]
|
|
1417
|
+
};
|
|
1418
|
+
return { kind: "absent" };
|
|
1419
|
+
}
|
|
1420
|
+
/** Append an event whose id the script picks. Returns the stored event. */
|
|
1421
|
+
async function appendEvent(runId, event, options = {}) {
|
|
1422
|
+
const correlationId = event.correlationId;
|
|
1423
|
+
const result = await scripts.wfStoreEvent(eventsKey(runId), correlationId ? eventsByCorrelationKey(runId, correlationId) : "__unused__", options.entityKey ?? "__unused__", options.resume ? resumeClaimKey(runId, options.resume.id) : "__unused__", options.step?.key ?? "__unused__", options.runMustBeLive ? runMetaKey(runId) : "__unused__", options.hookMustExist && correlationId ? hookKey(correlationId) : "__unused__", serializeEvent(event), correlationId ? "1" : "0", options.dedup === "creation" ? "1" : options.dedup === "completion" ? "2" : "0", options.entityJson ?? "", options.resume?.claim ?? "", options.step?.status ?? "", options.step?.expectedAttempt === void 0 ? "" : String(options.step.expectedAttempt), options.step?.newAttempt === void 0 ? "" : String(options.step.newAttempt), options.runMustBeLive ? "1" : "0", options.hookMustExist ? "1" : "0");
|
|
1424
|
+
switch (result[0]) {
|
|
1425
|
+
case LUA_DUPLICATE_CREATION: return { kind: "duplicate" };
|
|
1426
|
+
case -7: return { kind: "alreadyCompleted" };
|
|
1427
|
+
case -3: return {
|
|
1428
|
+
kind: "stepTerminal",
|
|
1429
|
+
status: result[1]
|
|
1430
|
+
};
|
|
1431
|
+
case -4: return {
|
|
1432
|
+
kind: "stepMoved",
|
|
1433
|
+
attempt: result[1]
|
|
1434
|
+
};
|
|
1435
|
+
case -5: return { kind: "hookGone" };
|
|
1436
|
+
case -6: return {
|
|
1437
|
+
kind: "runTerminal",
|
|
1438
|
+
status: result[1]
|
|
1439
|
+
};
|
|
1440
|
+
case 3: return {
|
|
1441
|
+
kind: "claimed",
|
|
1442
|
+
claim: result[1]
|
|
1443
|
+
};
|
|
1444
|
+
default: return {
|
|
1445
|
+
kind: "stored",
|
|
1446
|
+
event: EventSchema.parse({
|
|
1447
|
+
...event,
|
|
1448
|
+
eventId: result[1]
|
|
1449
|
+
})
|
|
1450
|
+
};
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
/** The events a bumped write skipped over: those between the slot the writer
|
|
1454
|
+
* asked for (`eventCount + 1`) and the one it landed on. */
|
|
1455
|
+
async function reportSkippedSlots(result, askedFor, resolveData) {
|
|
1456
|
+
if (!result.event) return result;
|
|
1457
|
+
const committedSlot = eventIdToSlot(result.event.eventId);
|
|
1458
|
+
if (committedSlot === null || askedFor < 0 || committedSlot <= askedFor + 1) return result;
|
|
1459
|
+
const span = committedSlot - askedFor - 1;
|
|
1460
|
+
const upto = Math.min(committedSlot - 1, askedFor + MAX_INLINE_EVENTS);
|
|
1461
|
+
const skipped = await readSlots(result.event.runId, askedFor + 1, upto);
|
|
1462
|
+
return {
|
|
1463
|
+
...result,
|
|
1464
|
+
events: skipped.map((event) => stripEventDataRefs(event, resolveData)),
|
|
1465
|
+
cursor: null,
|
|
1466
|
+
hasMore: skipped.length < span
|
|
1467
|
+
};
|
|
1468
|
+
}
|
|
1469
|
+
/** The inline delta: everything after `sinceCursor`, as `events.list` would
|
|
1470
|
+
* return it right after the write. */
|
|
1471
|
+
async function deltaSince(result, sinceCursor, resolveData) {
|
|
1472
|
+
if (!result.event) return result;
|
|
1473
|
+
const sinceSlot = eventIdToSlot(sinceCursor);
|
|
1474
|
+
if (sinceSlot === null) return result;
|
|
1475
|
+
const total = await redis.hlen(eventsKey(result.event.runId));
|
|
1476
|
+
const upto = Math.min(total, sinceSlot + MAX_INLINE_EVENTS);
|
|
1477
|
+
const page = await readSlots(result.event.runId, sinceSlot + 1, upto);
|
|
1478
|
+
return {
|
|
1479
|
+
...result,
|
|
1480
|
+
events: page.map((event) => stripEventDataRefs(event, resolveData)),
|
|
1481
|
+
cursor: page.at(-1)?.eventId ?? sinceCursor,
|
|
1482
|
+
hasMore: upto < total
|
|
1483
|
+
};
|
|
1484
|
+
}
|
|
1485
|
+
const isTerminal = (status) => [
|
|
1486
|
+
"completed",
|
|
1487
|
+
"failed",
|
|
1488
|
+
"cancelled"
|
|
1489
|
+
].includes(status);
|
|
1490
|
+
/** How many times a step write recomputes after losing a race for the step. */
|
|
1491
|
+
const MAX_STALE_RETRIES = 5;
|
|
1492
|
+
async function createEvent(runId, data, params, staleRetries = 0) {
|
|
1493
|
+
const now = /* @__PURE__ */ new Date();
|
|
1494
|
+
const resolveData = params?.resolveData ?? "all";
|
|
1495
|
+
const entityData = entityResolveData(resolveData);
|
|
1496
|
+
let effectiveRunId;
|
|
1497
|
+
if (data.eventType === "run_created" && (!runId || runId === "")) effectiveRunId = `wrun_${ulid()}`;
|
|
1498
|
+
else if (!runId) throw new Error("runId is required for non-run_created events");
|
|
1499
|
+
else effectiveRunId = runId;
|
|
1500
|
+
const effectiveSpecVersion = data.specVersion ?? mintedSpecVersion();
|
|
1501
|
+
const buildEvent = (overrides = {}) => ({
|
|
1502
|
+
...data,
|
|
1503
|
+
runId: effectiveRunId,
|
|
1504
|
+
createdAt: /* @__PURE__ */ new Date(),
|
|
1505
|
+
specVersion: effectiveSpecVersion,
|
|
1506
|
+
...overrides
|
|
1507
|
+
});
|
|
1508
|
+
let run;
|
|
1509
|
+
let step;
|
|
1510
|
+
let hook;
|
|
1511
|
+
let lifecycleEvent;
|
|
1512
|
+
let pendingEntityKey;
|
|
1513
|
+
let pendingEntityJson;
|
|
1514
|
+
let pendingStep;
|
|
1515
|
+
let currentRun = null;
|
|
1516
|
+
let existingRun = null;
|
|
1517
|
+
async function loadRunBody() {
|
|
1518
|
+
const runData = await redis.get(runKey(effectiveRunId));
|
|
1519
|
+
if (runData) {
|
|
1520
|
+
existingRun = parseWithUint8Array(runData);
|
|
1521
|
+
currentRun = {
|
|
1522
|
+
status: existingRun.status,
|
|
1523
|
+
specVersion: existingRun.specVersion
|
|
1524
|
+
};
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
if (data.eventType !== "run_created" && !["step_completed", "step_retrying"].includes(data.eventType)) {
|
|
1528
|
+
if (RUN_LIFECYCLE_EVENT_TYPES.has(data.eventType)) await loadRunBody();
|
|
1529
|
+
else {
|
|
1530
|
+
const meta = await redis.get(runMetaKey(effectiveRunId));
|
|
1531
|
+
if (meta) currentRun = parseRunMeta(meta);
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
if (data.eventType === "run_started" && !currentRun && "eventData" in data && data.eventData) {
|
|
1535
|
+
const runInputData = data.eventData;
|
|
1536
|
+
if (runInputData.deploymentId && runInputData.workflowName && runInputData.input !== void 0) {
|
|
1537
|
+
const seeds = runInputData.attributes ?? {};
|
|
1538
|
+
validateAttributeChanges(Object.entries(seeds).map(([key, value]) => ({
|
|
1539
|
+
key,
|
|
1540
|
+
value
|
|
1541
|
+
})), { allowReservedAttributes: runInputData.allowReservedAttributes === true });
|
|
1542
|
+
const newRun = {
|
|
1543
|
+
runId: effectiveRunId,
|
|
1544
|
+
deploymentId: runInputData.deploymentId,
|
|
1545
|
+
workflowName: runInputData.workflowName,
|
|
1546
|
+
specVersion: effectiveSpecVersion,
|
|
1547
|
+
input: runInputData.input,
|
|
1548
|
+
executionContext: runInputData.executionContext,
|
|
1549
|
+
...runInputData.encryptionPublicKey && { encryptionPublicKey: runInputData.encryptionPublicKey },
|
|
1550
|
+
...runInputData.dynamicWorkflowCode !== void 0 && { dynamicWorkflowCode: runInputData.dynamicWorkflowCode },
|
|
1551
|
+
status: "pending",
|
|
1552
|
+
output: void 0,
|
|
1553
|
+
error: void 0,
|
|
1554
|
+
completedAt: void 0,
|
|
1555
|
+
startedAt: void 0,
|
|
1556
|
+
createdAt: now,
|
|
1557
|
+
updatedAt: now
|
|
1558
|
+
};
|
|
1559
|
+
const result = await scripts.wfCreateRunWithEvent(runKey(effectiveRunId), runsIndexKey(), runsByNameKey(runInputData.workflowName), runsByStatusKey("pending"), runMetaKey(effectiveRunId), eventsKey(effectiveRunId), runAttributesKey(effectiveRunId), stringifyWithUint8Array(newRun), effectiveRunId, now.getTime().toString(), runMetaValue("pending", effectiveSpecVersion), serializeEvent({
|
|
1560
|
+
eventType: "run_created",
|
|
1561
|
+
eventData: {
|
|
1562
|
+
deploymentId: runInputData.deploymentId,
|
|
1563
|
+
workflowName: runInputData.workflowName,
|
|
1564
|
+
input: runInputData.input,
|
|
1565
|
+
executionContext: runInputData.executionContext,
|
|
1566
|
+
...runInputData.attributes && { attributes: runInputData.attributes },
|
|
1567
|
+
...runInputData.allowReservedAttributes && { allowReservedAttributes: true },
|
|
1568
|
+
...runInputData.encryptionPublicKey && { encryptionPublicKey: runInputData.encryptionPublicKey }
|
|
1569
|
+
},
|
|
1570
|
+
runId: effectiveRunId,
|
|
1571
|
+
createdAt: now,
|
|
1572
|
+
specVersion: effectiveSpecVersion
|
|
1573
|
+
}), ...flattenAttributes(seeds));
|
|
1574
|
+
if (result[0] === 1) {
|
|
1575
|
+
currentRun = {
|
|
1576
|
+
status: "pending",
|
|
1577
|
+
specVersion: effectiveSpecVersion
|
|
1578
|
+
};
|
|
1579
|
+
existingRun = newRun;
|
|
1580
|
+
} else {
|
|
1581
|
+
const parsed = parseWithUint8Array(result[1]);
|
|
1582
|
+
currentRun = {
|
|
1583
|
+
status: parsed.status,
|
|
1584
|
+
specVersion: parsed.specVersion
|
|
1585
|
+
};
|
|
1586
|
+
existingRun = parsed;
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
if (currentRun) {
|
|
1591
|
+
if (requiresNewerWorld(currentRun.specVersion)) throw new RunNotSupportedError(currentRun.specVersion, mintedSpecVersion());
|
|
1592
|
+
if (isLegacySpecVersion(currentRun.specVersion)) throw new RunNotSupportedError(currentRun.specVersion, mintedSpecVersion());
|
|
1593
|
+
}
|
|
1594
|
+
if (currentRun && isTerminal(currentRun.status)) {
|
|
1595
|
+
const terminalStatus = currentRun.status;
|
|
1596
|
+
const runTerminalEvents = [
|
|
1597
|
+
"run_started",
|
|
1598
|
+
"run_completed",
|
|
1599
|
+
"run_failed"
|
|
1600
|
+
];
|
|
1601
|
+
if (data.eventType === "run_cancelled" && terminalStatus === "cancelled") {
|
|
1602
|
+
const stored = await appendEvent(effectiveRunId, buildEvent());
|
|
1603
|
+
if (stored.kind !== "stored") throw new EntityConflictError("Duplicate run_cancelled");
|
|
1604
|
+
return {
|
|
1605
|
+
event: stripEventDataRefs(stored.event, entityData),
|
|
1606
|
+
run: existingRun ? await withAttributes(WorkflowRunSchema.parse(compact(existingRun))) : void 0
|
|
1607
|
+
};
|
|
1608
|
+
}
|
|
1609
|
+
if (data.eventType === "run_started") throw new RunExpiredError(`Workflow run "${effectiveRunId}" is already in terminal state "${terminalStatus}"`);
|
|
1610
|
+
if (data.eventType === "step_started") throw new RunExpiredError(`Cannot start step on run in terminal state "${terminalStatus}"`);
|
|
1611
|
+
if (runTerminalEvents.includes(data.eventType) || data.eventType === "run_cancelled") throw new EntityConflictError(`Cannot transition run from terminal state "${terminalStatus}"`);
|
|
1612
|
+
if (data.eventType === "step_created" || data.eventType === "hook_created" || data.eventType === "wait_created") throw new EntityConflictError(`Cannot create new entities on run in terminal state "${terminalStatus}"`);
|
|
1613
|
+
}
|
|
1614
|
+
let validatedStep = null;
|
|
1615
|
+
const lazyStepStart = data.eventType === "step_started" && typeof data.eventData?.stepName === "string" && data.eventData?.input !== void 0;
|
|
1616
|
+
if (["step_started", "step_retrying"].includes(data.eventType) && data.correlationId) {
|
|
1617
|
+
const stepData = await redis.get(stepKey(effectiveRunId, data.correlationId));
|
|
1618
|
+
if (stepData) validatedStep = parseWithUint8Array(stepData);
|
|
1619
|
+
if (!validatedStep && !lazyStepStart) throw new WorkflowWorldError(`Step "${data.correlationId}" not found`, { status: 404 });
|
|
1620
|
+
if (lazyStepStart && validatedStep) throw new EntityConflictError(`Step "${data.correlationId}" already created`);
|
|
1621
|
+
if (validatedStep) {
|
|
1622
|
+
if (isTerminal(validatedStep.status)) throw new EntityConflictError(`Cannot modify step in terminal state "${validatedStep.status}"`);
|
|
1623
|
+
if (currentRun && isTerminal(currentRun.status)) {
|
|
1624
|
+
if (validatedStep.status !== "running") throw new RunExpiredError(`Cannot modify non-running step on run in terminal state "${currentRun.status}"`);
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
let validatedHook = null;
|
|
1629
|
+
const hookEventsRequiringExistence = ["hook_disposed", "hook_received"];
|
|
1630
|
+
const isResumeRetryCandidate = data.eventType === "hook_received" && params?.resumeId;
|
|
1631
|
+
if (hookEventsRequiringExistence.includes(data.eventType) && data.correlationId && !isResumeRetryCandidate) {
|
|
1632
|
+
const existingHook = await redis.get(hookKey(data.correlationId));
|
|
1633
|
+
if (!existingHook) throw new HookNotFoundError(data.correlationId);
|
|
1634
|
+
validatedHook = parseWithUint8Array(existingHook);
|
|
1635
|
+
}
|
|
1636
|
+
if (data.eventType === "run_created") {
|
|
1637
|
+
const eventData = data.eventData;
|
|
1638
|
+
if (eventData.dynamicWorkflowCodeRef !== void 0) throw new WorkflowWorldError("dynamicWorkflowCodeRef is not supported by this world", { status: 501 });
|
|
1639
|
+
const seeds = eventData.attributes ?? {};
|
|
1640
|
+
validateAttributeChanges(Object.entries(seeds).map(([key, value]) => ({
|
|
1641
|
+
key,
|
|
1642
|
+
value
|
|
1643
|
+
})), { allowReservedAttributes: eventData.allowReservedAttributes === true });
|
|
1644
|
+
const newRun = {
|
|
1645
|
+
runId: effectiveRunId,
|
|
1646
|
+
deploymentId: eventData.deploymentId,
|
|
1647
|
+
workflowName: eventData.workflowName,
|
|
1648
|
+
specVersion: effectiveSpecVersion,
|
|
1649
|
+
input: eventData.input,
|
|
1650
|
+
executionContext: eventData.executionContext,
|
|
1651
|
+
...eventData.encryptionPublicKey && { encryptionPublicKey: eventData.encryptionPublicKey },
|
|
1652
|
+
...eventData.dynamicWorkflowCode !== void 0 && { dynamicWorkflowCode: eventData.dynamicWorkflowCode },
|
|
1653
|
+
status: "pending",
|
|
1654
|
+
output: void 0,
|
|
1655
|
+
error: void 0,
|
|
1656
|
+
completedAt: void 0,
|
|
1657
|
+
startedAt: void 0,
|
|
1658
|
+
createdAt: now,
|
|
1659
|
+
updatedAt: now
|
|
1660
|
+
};
|
|
1661
|
+
const event = buildEvent({ createdAt: now });
|
|
1662
|
+
const result = await scripts.wfCreateRunWithEvent(runKey(effectiveRunId), runsIndexKey(), runsByNameKey(eventData.workflowName), runsByStatusKey("pending"), runMetaKey(effectiveRunId), eventsKey(effectiveRunId), runAttributesKey(effectiveRunId), stringifyWithUint8Array(newRun), effectiveRunId, now.getTime().toString(), runMetaValue("pending", effectiveSpecVersion), serializeEvent(event), ...flattenAttributes(seeds));
|
|
1663
|
+
if (result[0] === 0) throw new EntityConflictError(`Workflow run "${effectiveRunId}" already exists`);
|
|
1664
|
+
run = WorkflowRunSchema.parse({
|
|
1665
|
+
...compact(newRun),
|
|
1666
|
+
attributes: seeds
|
|
1667
|
+
});
|
|
1668
|
+
const parsed = EventSchema.parse({
|
|
1669
|
+
...event,
|
|
1670
|
+
eventId: result[1]
|
|
1671
|
+
});
|
|
1672
|
+
return {
|
|
1673
|
+
event: stripEventDataRefs(parsed, entityData),
|
|
1674
|
+
run,
|
|
1675
|
+
maxEvents: maxEventsPerRun
|
|
1676
|
+
};
|
|
1677
|
+
}
|
|
1678
|
+
if (data.eventType === "run_started") {
|
|
1679
|
+
const resumeResponse = async (current) => {
|
|
1680
|
+
const resumed = current ? await withAttributes(WorkflowRunSchema.parse(compact(current))) : void 0;
|
|
1681
|
+
return {
|
|
1682
|
+
run: resumed ? filterRunData(resumed, entityData) : void 0,
|
|
1683
|
+
...resumed ? { maxEvents: maxEventsPerRun } : {}
|
|
1684
|
+
};
|
|
1685
|
+
};
|
|
1686
|
+
if (currentRun?.status === "running") return resumeResponse(existingRun);
|
|
1687
|
+
if (!existingRun) throw new WorkflowRunNotFoundError(effectiveRunId);
|
|
1688
|
+
const existing = existingRun;
|
|
1689
|
+
const updatedRun = {
|
|
1690
|
+
...existing,
|
|
1691
|
+
status: "running",
|
|
1692
|
+
startedAt: now,
|
|
1693
|
+
updatedAt: now
|
|
1694
|
+
};
|
|
1695
|
+
const { eventData: _startData, ...startEvent } = buildEvent({ createdAt: now });
|
|
1696
|
+
const moved = await transitionRun(effectiveRunId, existing.status, "running", updatedRun, now, startEvent, false);
|
|
1697
|
+
if (moved.kind === "absent") throw new WorkflowRunNotFoundError(effectiveRunId);
|
|
1698
|
+
if (moved.kind === "moved") {
|
|
1699
|
+
if (isTerminal(moved.status)) throw new RunExpiredError(`Workflow run "${effectiveRunId}" is already in terminal state "${moved.status}"`);
|
|
1700
|
+
await loadRunBody();
|
|
1701
|
+
return resumeResponse(existingRun);
|
|
1702
|
+
}
|
|
1703
|
+
run = WorkflowRunSchema.parse(compact(updatedRun));
|
|
1704
|
+
lifecycleEvent = EventSchema.parse({
|
|
1705
|
+
...startEvent,
|
|
1706
|
+
eventId: moved.eventId
|
|
1707
|
+
});
|
|
1708
|
+
}
|
|
1709
|
+
if (data.eventType === "run_completed" || data.eventType === "run_failed" || data.eventType === "run_cancelled") {
|
|
1710
|
+
if (!existingRun) throw new WorkflowRunNotFoundError(effectiveRunId);
|
|
1711
|
+
const existing = existingRun;
|
|
1712
|
+
let updatedRun;
|
|
1713
|
+
let newStatus;
|
|
1714
|
+
if (data.eventType === "run_completed") {
|
|
1715
|
+
const eventData = data.eventData;
|
|
1716
|
+
newStatus = "completed";
|
|
1717
|
+
updatedRun = {
|
|
1718
|
+
...existing,
|
|
1719
|
+
status: "completed",
|
|
1720
|
+
output: eventData.output,
|
|
1721
|
+
completedAt: now,
|
|
1722
|
+
updatedAt: now
|
|
1723
|
+
};
|
|
1724
|
+
} else if (data.eventType === "run_failed") {
|
|
1725
|
+
const eventData = data.eventData;
|
|
1726
|
+
newStatus = "failed";
|
|
1727
|
+
updatedRun = {
|
|
1728
|
+
...existing,
|
|
1729
|
+
status: "failed",
|
|
1730
|
+
error: eventData.error,
|
|
1731
|
+
errorCode: eventData.errorCode,
|
|
1732
|
+
completedAt: now,
|
|
1733
|
+
updatedAt: now
|
|
1734
|
+
};
|
|
1735
|
+
} else {
|
|
1736
|
+
newStatus = "cancelled";
|
|
1737
|
+
updatedRun = {
|
|
1738
|
+
...existing,
|
|
1739
|
+
status: "cancelled",
|
|
1740
|
+
completedAt: now,
|
|
1741
|
+
updatedAt: now
|
|
1742
|
+
};
|
|
1743
|
+
}
|
|
1744
|
+
const terminalEvent = buildEvent({ createdAt: now });
|
|
1745
|
+
const moved = await transitionRun(effectiveRunId, existing.status, newStatus, updatedRun, now, terminalEvent, true);
|
|
1746
|
+
if (moved.kind === "absent") throw new WorkflowRunNotFoundError(effectiveRunId);
|
|
1747
|
+
if (moved.kind === "moved") throw new EntityConflictError(`Cannot transition run "${effectiveRunId}" to "${newStatus}": it already left "${existing.status}" for "${moved.status}"`);
|
|
1748
|
+
run = WorkflowRunSchema.parse(compact(updatedRun));
|
|
1749
|
+
lifecycleEvent = EventSchema.parse({
|
|
1750
|
+
...terminalEvent,
|
|
1751
|
+
eventId: moved.eventId
|
|
1752
|
+
});
|
|
1753
|
+
}
|
|
1754
|
+
if (data.eventType === "step_created") {
|
|
1755
|
+
const eventData = data.eventData;
|
|
1756
|
+
const newStep = {
|
|
1757
|
+
runId: effectiveRunId,
|
|
1758
|
+
stepId: data.correlationId,
|
|
1759
|
+
stepName: eventData.stepName,
|
|
1760
|
+
input: eventData.input,
|
|
1761
|
+
status: "pending",
|
|
1762
|
+
attempt: 0,
|
|
1763
|
+
specVersion: effectiveSpecVersion,
|
|
1764
|
+
createdAt: now,
|
|
1765
|
+
updatedAt: now
|
|
1766
|
+
};
|
|
1767
|
+
const event = buildEvent({ createdAt: now });
|
|
1768
|
+
const result = await scripts.wfCreateStepWithEvent(stepKey(effectiveRunId, data.correlationId), stepsIndexKey(effectiveRunId), eventsKey(effectiveRunId), eventsByCorrelationKey(effectiveRunId, data.correlationId), stepStatusKey(effectiveRunId, data.correlationId), runMetaKey(effectiveRunId), stringifyWithUint8Array(newStep), data.correlationId, now.getTime().toString(), serializeEvent(event), "pending|0");
|
|
1769
|
+
if (result[0] === -6) throw new EntityConflictError(`Cannot create new entities on run in terminal state "${result[1]}"`);
|
|
1770
|
+
if (result[0] === 0) throw new EntityConflictError(`Step "${data.correlationId}" already exists`);
|
|
1771
|
+
step = StepSchema.parse(compact(newStep));
|
|
1772
|
+
const parsed = EventSchema.parse({
|
|
1773
|
+
...event,
|
|
1774
|
+
eventId: result[1]
|
|
1775
|
+
});
|
|
1776
|
+
return {
|
|
1777
|
+
event: stripEventDataRefs(parsed, entityData),
|
|
1778
|
+
step
|
|
1779
|
+
};
|
|
1780
|
+
}
|
|
1781
|
+
const dedup = data.eventType === "wait_created" ? "creation" : data.eventType === "wait_completed" ? "completion" : void 0;
|
|
1782
|
+
if (data.eventType === "step_started") {
|
|
1783
|
+
if (validatedStep?.retryAfter && validatedStep.retryAfter.getTime() > Date.now()) throw new TooEarlyError(`Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, { retryAfter: Math.ceil((validatedStep.retryAfter.getTime() - Date.now()) / 1e3) });
|
|
1784
|
+
if (lazyStepStart) {
|
|
1785
|
+
const lazyData = data.eventData;
|
|
1786
|
+
const startedStep = {
|
|
1787
|
+
runId: effectiveRunId,
|
|
1788
|
+
stepId: data.correlationId,
|
|
1789
|
+
stepName: lazyData.stepName,
|
|
1790
|
+
input: lazyData.input,
|
|
1791
|
+
status: "running",
|
|
1792
|
+
attempt: 1,
|
|
1793
|
+
startedAt: now,
|
|
1794
|
+
specVersion: effectiveSpecVersion,
|
|
1795
|
+
createdAt: now,
|
|
1796
|
+
updatedAt: now
|
|
1797
|
+
};
|
|
1798
|
+
const { input: _input, ...startedEventData } = lazyData;
|
|
1799
|
+
const createdEvent = {
|
|
1800
|
+
eventType: "step_created",
|
|
1801
|
+
correlationId: data.correlationId,
|
|
1802
|
+
runId: effectiveRunId,
|
|
1803
|
+
createdAt: now,
|
|
1804
|
+
specVersion: effectiveSpecVersion,
|
|
1805
|
+
eventData: {
|
|
1806
|
+
stepName: lazyData.stepName,
|
|
1807
|
+
...lazyData.workflowName !== void 0 && { workflowName: lazyData.workflowName },
|
|
1808
|
+
input: lazyData.input
|
|
1809
|
+
}
|
|
1810
|
+
};
|
|
1811
|
+
const startedEvent = buildEvent({
|
|
1812
|
+
createdAt: now,
|
|
1813
|
+
eventData: startedEventData
|
|
1814
|
+
});
|
|
1815
|
+
const result = await scripts.wfCreateStepStarted(stepKey(effectiveRunId, data.correlationId), stepsIndexKey(effectiveRunId), eventsKey(effectiveRunId), eventsByCorrelationKey(effectiveRunId, data.correlationId), stepStatusKey(effectiveRunId, data.correlationId), runMetaKey(effectiveRunId), stringifyWithUint8Array(startedStep), data.correlationId, now.getTime().toString(), serializeEvent(createdEvent), serializeEvent(startedEvent), "running|1");
|
|
1816
|
+
if (result[0] === -6) throw new RunExpiredError(`Cannot start step on run in terminal state "${result[1]}"`);
|
|
1817
|
+
if (result[0] === 0) throw new EntityConflictError(`Step "${data.correlationId}" already created`);
|
|
1818
|
+
step = StepSchema.parse(compact(startedStep));
|
|
1819
|
+
const parsed = EventSchema.parse({
|
|
1820
|
+
...startedEvent,
|
|
1821
|
+
eventId: result[2]
|
|
1822
|
+
});
|
|
1823
|
+
return {
|
|
1824
|
+
event: stripEventDataRefs(parsed, entityData),
|
|
1825
|
+
step,
|
|
1826
|
+
stepCreated: true
|
|
1827
|
+
};
|
|
1828
|
+
}
|
|
1829
|
+
const isFirstStart = !validatedStep?.startedAt;
|
|
1830
|
+
const existing = validatedStep;
|
|
1831
|
+
if (existing) {
|
|
1832
|
+
const updatedStep = {
|
|
1833
|
+
...existing,
|
|
1834
|
+
status: "running",
|
|
1835
|
+
attempt: existing.attempt + 1,
|
|
1836
|
+
...isFirstStart ? { startedAt: now } : {},
|
|
1837
|
+
retryAfter: void 0,
|
|
1838
|
+
updatedAt: now
|
|
1839
|
+
};
|
|
1840
|
+
pendingEntityKey = stepKey(effectiveRunId, data.correlationId);
|
|
1841
|
+
pendingEntityJson = stringifyWithUint8Array(updatedStep);
|
|
1842
|
+
pendingStep = {
|
|
1843
|
+
key: stepStatusKey(effectiveRunId, data.correlationId),
|
|
1844
|
+
status: "running",
|
|
1845
|
+
expectedAttempt: existing.attempt,
|
|
1846
|
+
newAttempt: updatedStep.attempt
|
|
1847
|
+
};
|
|
1848
|
+
step = StepSchema.parse(compact(updatedStep));
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
if (data.eventType === "step_completed") {
|
|
1852
|
+
const eventData = data.eventData;
|
|
1853
|
+
const existingData = await redis.get(stepKey(effectiveRunId, data.correlationId));
|
|
1854
|
+
if (existingData) {
|
|
1855
|
+
const existing = parseWithUint8Array(existingData);
|
|
1856
|
+
if (isTerminal(existing.status)) throw new EntityConflictError(`Cannot modify step in terminal state "${existing.status}"`);
|
|
1857
|
+
const updatedStep = {
|
|
1858
|
+
...existing,
|
|
1859
|
+
status: "completed",
|
|
1860
|
+
output: eventData.result,
|
|
1861
|
+
completedAt: now,
|
|
1862
|
+
updatedAt: now
|
|
1863
|
+
};
|
|
1864
|
+
pendingEntityKey = stepKey(effectiveRunId, data.correlationId);
|
|
1865
|
+
pendingEntityJson = stringifyWithUint8Array(updatedStep);
|
|
1866
|
+
pendingStep = {
|
|
1867
|
+
key: stepStatusKey(effectiveRunId, data.correlationId),
|
|
1868
|
+
status: "completed",
|
|
1869
|
+
expectedAttempt: existing.attempt
|
|
1870
|
+
};
|
|
1871
|
+
step = StepSchema.parse(compact(updatedStep));
|
|
1872
|
+
} else throw new WorkflowWorldError(`Step "${data.correlationId}" not found`, { status: 404 });
|
|
1873
|
+
}
|
|
1874
|
+
if (data.eventType === "step_failed") {
|
|
1875
|
+
const eventData = data.eventData;
|
|
1876
|
+
const existingData = await redis.get(stepKey(effectiveRunId, data.correlationId));
|
|
1877
|
+
if (existingData) {
|
|
1878
|
+
const existing = parseWithUint8Array(existingData);
|
|
1879
|
+
if (isTerminal(existing.status)) throw new EntityConflictError(`Cannot modify step in terminal state "${existing.status}"`);
|
|
1880
|
+
const updatedStep = {
|
|
1881
|
+
...existing,
|
|
1882
|
+
status: "failed",
|
|
1883
|
+
error: eventData.error,
|
|
1884
|
+
completedAt: now,
|
|
1885
|
+
updatedAt: now
|
|
1886
|
+
};
|
|
1887
|
+
pendingEntityKey = stepKey(effectiveRunId, data.correlationId);
|
|
1888
|
+
pendingEntityJson = stringifyWithUint8Array(updatedStep);
|
|
1889
|
+
pendingStep = {
|
|
1890
|
+
key: stepStatusKey(effectiveRunId, data.correlationId),
|
|
1891
|
+
status: "failed",
|
|
1892
|
+
expectedAttempt: existing.attempt
|
|
1893
|
+
};
|
|
1894
|
+
step = StepSchema.parse(compact(updatedStep));
|
|
1895
|
+
} else throw new WorkflowWorldError(`Step "${data.correlationId}" not found`, { status: 404 });
|
|
1896
|
+
}
|
|
1897
|
+
if (data.eventType === "step_retrying") {
|
|
1898
|
+
const eventData = data.eventData;
|
|
1899
|
+
const existing = validatedStep;
|
|
1900
|
+
if (existing) {
|
|
1901
|
+
const updatedStep = {
|
|
1902
|
+
...existing,
|
|
1903
|
+
status: "pending",
|
|
1904
|
+
error: eventData.error,
|
|
1905
|
+
retryAfter: eventData.retryAfter,
|
|
1906
|
+
updatedAt: now
|
|
1907
|
+
};
|
|
1908
|
+
pendingEntityKey = stepKey(effectiveRunId, data.correlationId);
|
|
1909
|
+
pendingEntityJson = stringifyWithUint8Array(updatedStep);
|
|
1910
|
+
pendingStep = {
|
|
1911
|
+
key: stepStatusKey(effectiveRunId, data.correlationId),
|
|
1912
|
+
status: "pending",
|
|
1913
|
+
expectedAttempt: existing.attempt
|
|
1914
|
+
};
|
|
1915
|
+
step = StepSchema.parse(compact(updatedStep));
|
|
1916
|
+
}
|
|
1917
|
+
}
|
|
1918
|
+
if (data.eventType === "hook_created") {
|
|
1919
|
+
const eventData = data.eventData;
|
|
1920
|
+
const newHook = {
|
|
1921
|
+
runId: effectiveRunId,
|
|
1922
|
+
hookId: data.correlationId,
|
|
1923
|
+
token: eventData.token,
|
|
1924
|
+
ownerId: "",
|
|
1925
|
+
projectId: "",
|
|
1926
|
+
environment: "",
|
|
1927
|
+
metadata: eventData.metadata,
|
|
1928
|
+
specVersion: effectiveSpecVersion,
|
|
1929
|
+
createdAt: now,
|
|
1930
|
+
...eventData.isWebhook !== void 0 && { isWebhook: eventData.isWebhook },
|
|
1931
|
+
...eventData.isSystem !== void 0 && { isSystem: eventData.isSystem }
|
|
1932
|
+
};
|
|
1933
|
+
const event = buildEvent({ createdAt: now });
|
|
1934
|
+
const result = await scripts.wfCreateHookWithEvent(hookKey(data.correlationId), hooksByTokenKey(eventData.token), hooksIndexKey(effectiveRunId), hooksTokenMapKey(effectiveRunId), eventsKey(effectiveRunId), eventsByCorrelationKey(effectiveRunId, data.correlationId), runMetaKey(effectiveRunId), stringifyWithUint8Array(newHook), data.correlationId, now.getTime().toString(), eventData.token, serializeEvent(event), `${keyPrefix}hook:`);
|
|
1935
|
+
if (result[0] === -6) throw new EntityConflictError(`Cannot create new entities on run in terminal state "${result[1]}"`);
|
|
1936
|
+
if (result[0] === 2) {
|
|
1937
|
+
const owningHookId = result[1];
|
|
1938
|
+
let conflictingRunId;
|
|
1939
|
+
const owningHookData = await redis.get(hookKey(owningHookId));
|
|
1940
|
+
if (owningHookData) conflictingRunId = parseWithUint8Array(owningHookData).runId;
|
|
1941
|
+
const stored = await appendEvent(effectiveRunId, {
|
|
1942
|
+
eventType: "hook_conflict",
|
|
1943
|
+
correlationId: data.correlationId,
|
|
1944
|
+
eventData: {
|
|
1945
|
+
token: eventData.token,
|
|
1946
|
+
...conflictingRunId ? { conflictingRunId } : {}
|
|
1947
|
+
},
|
|
1948
|
+
runId: effectiveRunId,
|
|
1949
|
+
createdAt: now,
|
|
1950
|
+
specVersion: effectiveSpecVersion
|
|
1951
|
+
}, {
|
|
1952
|
+
dedup: "creation",
|
|
1953
|
+
runMustBeLive: true
|
|
1954
|
+
});
|
|
1955
|
+
if (stored.kind === "runTerminal") throw new EntityConflictError(`Cannot create new entities on run in terminal state "${stored.status}"`);
|
|
1956
|
+
if (stored.kind !== "stored") throw new EntityConflictError(`Hook "${data.correlationId}" conflict already recorded`);
|
|
1957
|
+
return {
|
|
1958
|
+
event: stripEventDataRefs(stored.event, entityData),
|
|
1959
|
+
run,
|
|
1960
|
+
step,
|
|
1961
|
+
hook: void 0
|
|
1962
|
+
};
|
|
1963
|
+
}
|
|
1964
|
+
if (result[0] === 0) throw new EntityConflictError(`Hook "${data.correlationId}" already created`);
|
|
1965
|
+
hook = HookSchema.parse(compact(newHook));
|
|
1966
|
+
const parsed = EventSchema.parse({
|
|
1967
|
+
...event,
|
|
1968
|
+
eventId: result[1]
|
|
1969
|
+
});
|
|
1970
|
+
return {
|
|
1971
|
+
event: stripEventDataRefs(parsed, entityData),
|
|
1972
|
+
hook
|
|
1973
|
+
};
|
|
1974
|
+
}
|
|
1975
|
+
if (data.eventType === "hook_disposed" && data.correlationId) {
|
|
1976
|
+
if (!validatedHook) throw new EntityConflictError(`Hook "${data.correlationId}" already disposed`);
|
|
1977
|
+
const event = buildEvent();
|
|
1978
|
+
const result = await scripts.wfDisposeHook(hookKey(data.correlationId), hooksByTokenKey(validatedHook.token), hooksIndexKey(effectiveRunId), hooksTokenMapKey(effectiveRunId), eventsKey(effectiveRunId), eventsByCorrelationKey(effectiveRunId, data.correlationId), data.correlationId, serializeEvent(event));
|
|
1979
|
+
if (result[0] === 0) throw new EntityConflictError(`Hook "${data.correlationId}" already disposed`);
|
|
1980
|
+
const parsed = EventSchema.parse({
|
|
1981
|
+
...event,
|
|
1982
|
+
eventId: result[1]
|
|
1983
|
+
});
|
|
1984
|
+
return { event: stripEventDataRefs(parsed, entityData) };
|
|
1985
|
+
}
|
|
1986
|
+
if (data.eventType === "attr_set") {
|
|
1987
|
+
if (!currentRun) throw new WorkflowRunNotFoundError(effectiveRunId);
|
|
1988
|
+
const eventData = data.eventData;
|
|
1989
|
+
validateAttributeChanges(eventData.changes, {
|
|
1990
|
+
existingKeys: await redis.hkeys(runAttributesKey(effectiveRunId)),
|
|
1991
|
+
allowReservedAttributes: eventData.allowReservedAttributes === true
|
|
1992
|
+
});
|
|
1993
|
+
const event = buildEvent();
|
|
1994
|
+
const correlationId = data.correlationId;
|
|
1995
|
+
const result = await scripts.wfSetAttributes(runAttributesKey(effectiveRunId), eventsKey(effectiveRunId), correlationId ? eventsByCorrelationKey(effectiveRunId, correlationId) : "__unused__", runMetaKey(effectiveRunId), serializeEvent(event), correlationId ? "1" : "0", data.eventData?.writer?.type === "workflow" ? "1" : "0", ...eventData.changes.flatMap((change) => [
|
|
1996
|
+
change.key,
|
|
1997
|
+
change.value ?? "",
|
|
1998
|
+
change.value === null ? "del" : "set"
|
|
1999
|
+
]));
|
|
2000
|
+
if (result[0] === -6) throw new EntityConflictError(`Cannot set attributes on run in terminal state "${result[1]}"`);
|
|
2001
|
+
if (result[0] === LUA_DUPLICATE_CREATION) throw new EntityConflictError(`attr_set for correlationId "${correlationId}" already exists in run "${effectiveRunId}"`);
|
|
2002
|
+
await loadRunBody();
|
|
2003
|
+
return {
|
|
2004
|
+
event: stripEventDataRefs(EventSchema.parse({
|
|
2005
|
+
...event,
|
|
2006
|
+
eventId: result[1]
|
|
2007
|
+
}), entityData),
|
|
2008
|
+
run: existingRun ? await withAttributes(WorkflowRunSchema.parse(compact(existingRun))) : void 0
|
|
2009
|
+
};
|
|
2010
|
+
}
|
|
2011
|
+
const resume = data.eventType === "hook_received" && params?.resumeId ? {
|
|
2012
|
+
id: params.resumeId,
|
|
2013
|
+
claim: `${params.resumePayloadDigest ?? ""}|${data.correlationId}`
|
|
2014
|
+
} : void 0;
|
|
2015
|
+
const stored = lifecycleEvent ? {
|
|
2016
|
+
kind: "stored",
|
|
2017
|
+
event: lifecycleEvent
|
|
2018
|
+
} : await appendEvent(effectiveRunId, buildEvent(), {
|
|
2019
|
+
entityKey: pendingEntityKey,
|
|
2020
|
+
entityJson: pendingEntityJson,
|
|
2021
|
+
dedup,
|
|
2022
|
+
resume,
|
|
2023
|
+
step: pendingStep,
|
|
2024
|
+
runMustBeLive: data.eventType === "hook_received" || data.eventType === "wait_created" || data.eventType === "step_started",
|
|
2025
|
+
hookMustExist: data.eventType === "hook_received"
|
|
2026
|
+
});
|
|
2027
|
+
if (stored.kind === "duplicate") throw new EntityConflictError(`"${data.correlationId}" already exists`);
|
|
2028
|
+
if (stored.kind === "alreadyCompleted") throw new EntityConflictError(`Wait "${data.correlationId}" already completed`);
|
|
2029
|
+
if (stored.kind === "stepTerminal") throw new EntityConflictError(`Cannot modify step in terminal state "${stored.status}"`);
|
|
2030
|
+
if (stored.kind === "stepMoved") {
|
|
2031
|
+
if (data.eventType !== "step_started" && staleRetries < MAX_STALE_RETRIES) return createEvent(runId, data, params, staleRetries + 1);
|
|
2032
|
+
throw new EntityConflictError(`Step "${data.correlationId}" was moved by another delivery (attempt ${stored.attempt})`);
|
|
2033
|
+
}
|
|
2034
|
+
if (stored.kind === "runTerminal") {
|
|
2035
|
+
if (data.eventType === "step_started") throw new RunExpiredError(`Cannot start step on run in terminal state "${stored.status}"`);
|
|
2036
|
+
throw new EntityConflictError(`Cannot add ${data.eventType} to run in terminal state "${stored.status}"`);
|
|
2037
|
+
}
|
|
2038
|
+
if (stored.kind === "hookGone") throw new HookNotFoundError(data.correlationId);
|
|
2039
|
+
if (stored.kind === "claimed") {
|
|
2040
|
+
const [claimedEventId, claimedDigest, claimedHookId] = stored.claim.split("|");
|
|
2041
|
+
if (claimedHookId !== data.correlationId) throw new EntityConflictError(`hook_received resumeId "${params.resumeId}" already recorded for a different hook`);
|
|
2042
|
+
if (params?.resumePayloadDigest && claimedDigest && claimedDigest !== params.resumePayloadDigest) throw new EntityConflictError(`hook_received resumeId "${params.resumeId}" already recorded with a different payload`);
|
|
2043
|
+
const [existingEvent] = await readEvents(effectiveRunId, [claimedEventId]);
|
|
2044
|
+
if (!existingEvent) throw new WorkflowWorldError(`Event not found: ${claimedEventId}`, { status: 404 });
|
|
2045
|
+
return { event: stripEventDataRefs(existingEvent, entityData) };
|
|
2046
|
+
}
|
|
2047
|
+
let preloaded;
|
|
2048
|
+
if (data.eventType === "run_started" && run && params?.skipPreload !== true) {
|
|
2049
|
+
const total = await redis.hlen(eventsKey(effectiveRunId));
|
|
2050
|
+
const all = await readSlots(effectiveRunId, FIRST_EVENT_SLOT, total);
|
|
2051
|
+
preloaded = {
|
|
2052
|
+
events: all.map((e) => stripEventDataRefs(e, resolveData)),
|
|
2053
|
+
cursor: all.at(-1)?.eventId ?? null,
|
|
2054
|
+
hasMore: false
|
|
2055
|
+
};
|
|
2056
|
+
}
|
|
2057
|
+
return {
|
|
2058
|
+
event: stripEventDataRefs(stored.event, entityData),
|
|
2059
|
+
run: run ? await withAttributes(run) : void 0,
|
|
2060
|
+
step,
|
|
2061
|
+
hook,
|
|
2062
|
+
...preloaded,
|
|
2063
|
+
...run ? { maxEvents: maxEventsPerRun } : {}
|
|
2064
|
+
};
|
|
2065
|
+
}
|
|
2066
|
+
return {
|
|
2067
|
+
create: (async (runId, data, params) => {
|
|
2068
|
+
const result = await createEvent(runId, data, params);
|
|
2069
|
+
if (!result.event || result.events !== void 0) return result;
|
|
2070
|
+
const resolveData = params?.resolveData ?? "all";
|
|
2071
|
+
if (typeof params?.sinceCursor === "string") return deltaSince(result, params.sinceCursor, resolveData);
|
|
2072
|
+
if (params?.eventCount !== void 0) return reportSkippedSlots(result, params.eventCount, resolveData);
|
|
2073
|
+
return result;
|
|
2074
|
+
}),
|
|
2075
|
+
async get(runId, eventId, params) {
|
|
2076
|
+
const data = await redis.hget(eventsKey(runId), eventId);
|
|
2077
|
+
if (!data) throw new WorkflowWorldError(`Event not found: ${eventId}`, { status: 404 });
|
|
2078
|
+
const event = EventSchema.parse(compact(parseWithUint8Array(data)));
|
|
2079
|
+
return stripEventDataRefs(event, params?.resolveData ?? "all");
|
|
2080
|
+
},
|
|
2081
|
+
async list(params) {
|
|
2082
|
+
const sortOrder = params.pagination?.sortOrder || "asc";
|
|
2083
|
+
const fromCursor = params.pagination?.cursor;
|
|
2084
|
+
const total = await redis.hlen(eventsKey(params.runId));
|
|
2085
|
+
const limit = params.pagination?.limit ?? Math.max(total, 1);
|
|
2086
|
+
let cursorSlot = null;
|
|
2087
|
+
if (fromCursor) {
|
|
2088
|
+
cursorSlot = eventIdToSlot(fromCursor);
|
|
2089
|
+
if (cursorSlot === null) throw new WorkflowWorldError(`Invalid pagination cursor "${fromCursor}"`, { status: 400 });
|
|
2090
|
+
}
|
|
2091
|
+
let events;
|
|
2092
|
+
let hasMore;
|
|
2093
|
+
if (sortOrder === "desc") {
|
|
2094
|
+
const last = cursorSlot === null ? total : Math.min(cursorSlot - 1, total);
|
|
2095
|
+
const first = Math.max(FIRST_EVENT_SLOT, last - limit + 1);
|
|
2096
|
+
events = (await readSlots(params.runId, first, last)).reverse();
|
|
2097
|
+
hasMore = first > FIRST_EVENT_SLOT;
|
|
2098
|
+
} else {
|
|
2099
|
+
const first = (cursorSlot ?? 0) + 1;
|
|
2100
|
+
const last = Math.min(total, first + limit - 1);
|
|
2101
|
+
events = await readSlots(params.runId, first, last);
|
|
2102
|
+
hasMore = last < total;
|
|
2103
|
+
}
|
|
2104
|
+
const resolveData = params.resolveData ?? "all";
|
|
2105
|
+
return {
|
|
2106
|
+
data: events.map((event) => stripEventDataRefs(event, resolveData)),
|
|
2107
|
+
cursor: events.at(-1)?.eventId ?? fromCursor ?? null,
|
|
2108
|
+
hasMore
|
|
2109
|
+
};
|
|
2110
|
+
},
|
|
2111
|
+
async listByCorrelationId(params) {
|
|
2112
|
+
const limit = params.pagination?.limit ?? 100;
|
|
2113
|
+
const sortOrder = params.pagination?.sortOrder || "asc";
|
|
2114
|
+
const fromCursor = params.pagination?.cursor;
|
|
2115
|
+
const indexKey = eventsByCorrelationKey(params.runId, params.correlationId);
|
|
2116
|
+
let ids;
|
|
2117
|
+
if (sortOrder === "desc") {
|
|
2118
|
+
const cursorSlot = fromCursor ? eventIdToSlot(fromCursor) : null;
|
|
2119
|
+
const max = cursorSlot === null ? "+inf" : `(${cursorSlot}`;
|
|
2120
|
+
ids = await redis.zrevrangebyscore(indexKey, max, "-inf", "LIMIT", 0, limit + 1);
|
|
2121
|
+
} else {
|
|
2122
|
+
const cursorSlot = fromCursor ? eventIdToSlot(fromCursor) : null;
|
|
2123
|
+
const min = cursorSlot === null ? "-inf" : `(${cursorSlot}`;
|
|
2124
|
+
ids = await redis.zrangebyscore(indexKey, min, "+inf", "LIMIT", 0, limit + 1);
|
|
2125
|
+
}
|
|
2126
|
+
const page = ids.slice(0, limit);
|
|
2127
|
+
const events = await readEvents(params.runId, page);
|
|
2128
|
+
const resolveData = params.resolveData ?? "all";
|
|
2129
|
+
return {
|
|
2130
|
+
data: events.map((event) => stripEventDataRefs(event, resolveData)),
|
|
2131
|
+
cursor: events.at(-1)?.eventId ?? fromCursor ?? null,
|
|
2132
|
+
hasMore: ids.length > limit
|
|
2133
|
+
};
|
|
2134
|
+
}
|
|
2135
|
+
};
|
|
2136
|
+
}
|
|
2137
|
+
/**
|
|
2138
|
+
* Create storage for workflow steps using Redis hashes and sorted sets
|
|
2139
|
+
*/
|
|
2140
|
+
function createStepsStorage(config) {
|
|
2141
|
+
const { redis, keyPrefix } = config;
|
|
2142
|
+
const stepKey = (runId, stepId) => `${keyPrefix}step:${runId}:${stepId}`;
|
|
2143
|
+
const stepsIndexKey = (runId) => `${keyPrefix}steps:by_run:${runId}`;
|
|
2144
|
+
async function getStepData(key, stepId, params) {
|
|
2145
|
+
const data = await redis.get(key);
|
|
2146
|
+
if (!data) throw new WorkflowWorldError(`Step not found: ${stepId}`, { status: 404 });
|
|
2147
|
+
const step = parseWithUint8Array(data);
|
|
2148
|
+
return filterStepData(StepSchema.parse(compact(step)), params?.resolveData ?? "all");
|
|
2149
|
+
}
|
|
2150
|
+
return {
|
|
2151
|
+
get: (async (runId, stepId, params) => getStepData(stepKey(runId, stepId), stepId, params)),
|
|
2152
|
+
list: (async (params) => {
|
|
2153
|
+
const limit = params?.pagination?.limit ?? 20;
|
|
2154
|
+
const fromCursor = params?.pagination?.cursor;
|
|
2155
|
+
const entries = await pageIndex(redis, stepsIndexKey(params.runId), fromCursor, limit + 1, params?.pagination?.sortOrder ?? "desc");
|
|
2156
|
+
const pipeline = redis.pipeline();
|
|
2157
|
+
for (const { id } of entries) pipeline.get(stepKey(params.runId, id));
|
|
2158
|
+
const results = await pipeline.exec();
|
|
2159
|
+
const resolveData = params?.resolveData ?? "all";
|
|
2160
|
+
const steps = [];
|
|
2161
|
+
for (const [i, result] of (results ?? []).entries()) if (result?.[1]) {
|
|
2162
|
+
const step = parseWithUint8Array(result[1]);
|
|
2163
|
+
const parsed = StepSchema.parse(compact(step));
|
|
2164
|
+
steps.push({
|
|
2165
|
+
step: filterStepData(parsed, resolveData),
|
|
2166
|
+
cursor: entries[i].cursor
|
|
2167
|
+
});
|
|
2168
|
+
}
|
|
2169
|
+
const values = steps.slice(0, limit);
|
|
2170
|
+
const hasMore = steps.length > limit;
|
|
2171
|
+
return {
|
|
2172
|
+
data: values.map((entry) => entry.step),
|
|
2173
|
+
hasMore,
|
|
2174
|
+
cursor: values.at(-1)?.cursor ?? null
|
|
2175
|
+
};
|
|
2176
|
+
})
|
|
2177
|
+
};
|
|
2178
|
+
}
|
|
2179
|
+
/**
|
|
2180
|
+
* Create storage for hooks using Redis hashes and sorted sets
|
|
2181
|
+
*/
|
|
2182
|
+
function createHooksStorage(config) {
|
|
2183
|
+
const { redis, keyPrefix } = config;
|
|
2184
|
+
const hookKeyFn = (hookId) => `${keyPrefix}hook:${hookId}`;
|
|
2185
|
+
const hooksByTokenKey = (token) => `${keyPrefix}hooks:by_token:${token}`;
|
|
2186
|
+
const hooksIndexKey = (runId) => `${keyPrefix}hooks:by_run:${runId}`;
|
|
2187
|
+
async function liveHooksIndexKeys() {
|
|
2188
|
+
return (await Promise.all(["pending", "running"].map((status) => redis.zrange(`${keyPrefix}runs:by_status:${status}`, "0", "-1")))).flat().map(hooksIndexKey);
|
|
2189
|
+
}
|
|
2190
|
+
return {
|
|
2191
|
+
async get(hookId, params) {
|
|
2192
|
+
const data = await redis.get(hookKeyFn(hookId));
|
|
2193
|
+
if (!data) throw new HookNotFoundError(hookId);
|
|
2194
|
+
const hook = parseWithUint8Array(data);
|
|
2195
|
+
return filterHookData(HookSchema.parse(compact(hook)), params?.resolveData ?? "all");
|
|
2196
|
+
},
|
|
2197
|
+
async getByToken(token, params) {
|
|
2198
|
+
const hookId = await redis.get(hooksByTokenKey(token));
|
|
2199
|
+
if (!hookId) throw new HookNotFoundError(token);
|
|
2200
|
+
return this.get(hookId, params);
|
|
2201
|
+
},
|
|
2202
|
+
async list(params) {
|
|
2203
|
+
const limit = params?.pagination?.limit ?? 100;
|
|
2204
|
+
const fromCursor = params?.pagination?.cursor;
|
|
2205
|
+
const entries = await pageIndex(redis, params.runId ? hooksIndexKey(params.runId) : await liveHooksIndexKeys(), fromCursor, limit + 1, params?.pagination?.sortOrder ?? "asc");
|
|
2206
|
+
const pipeline = redis.pipeline();
|
|
2207
|
+
for (const { id } of entries) pipeline.get(hookKeyFn(id));
|
|
2208
|
+
const results = await pipeline.exec();
|
|
2209
|
+
const hooks = [];
|
|
2210
|
+
for (const [i, result] of (results ?? []).entries()) if (result?.[1]) {
|
|
2211
|
+
const hook = parseWithUint8Array(result[1]);
|
|
2212
|
+
const parsed = HookSchema.parse(compact(hook));
|
|
2213
|
+
hooks.push({
|
|
2214
|
+
hook: filterHookData(parsed, params?.resolveData ?? "all"),
|
|
2215
|
+
cursor: entries[i].cursor
|
|
2216
|
+
});
|
|
2217
|
+
}
|
|
2218
|
+
const values = hooks.slice(0, limit);
|
|
2219
|
+
const hasMore = hooks.length > limit;
|
|
2220
|
+
return {
|
|
2221
|
+
data: values.map((entry) => entry.hook),
|
|
2222
|
+
cursor: values.at(-1)?.cursor ?? null,
|
|
2223
|
+
hasMore
|
|
2224
|
+
};
|
|
2225
|
+
}
|
|
2226
|
+
};
|
|
2227
|
+
}
|
|
2228
|
+
//#endregion
|
|
2229
|
+
//#region src/reaper.ts
|
|
2230
|
+
const TERMINAL = [
|
|
2231
|
+
"completed",
|
|
2232
|
+
"failed",
|
|
2233
|
+
"cancelled"
|
|
2234
|
+
];
|
|
2235
|
+
const DEFAULT_SWEEP_INTERVAL_MS = 6e4;
|
|
2236
|
+
const DEFAULT_BATCH_SIZE = 200;
|
|
2237
|
+
const UNLINK_CHUNK = 500;
|
|
2238
|
+
function validateRetention(retention) {
|
|
2239
|
+
if (!retention) return;
|
|
2240
|
+
for (const status of TERMINAL) {
|
|
2241
|
+
const keep = retention[status];
|
|
2242
|
+
if (!keep) continue;
|
|
2243
|
+
for (const field of ["age", "count"]) {
|
|
2244
|
+
const value = keep[field];
|
|
2245
|
+
if (value !== void 0 && (!Number.isInteger(value) || value < 0)) throw new TypeError(`retention.${status}.${field} must be a non-negative integer, received ${String(value)}`);
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
2248
|
+
for (const field of ["sweepIntervalMs", "batchSize"]) {
|
|
2249
|
+
const value = retention[field];
|
|
2250
|
+
if (value !== void 0 && (!Number.isInteger(value) || value < 1)) throw new TypeError(`retention.${field} must be a positive integer, received ${String(value)}`);
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2253
|
+
function hasRetention(retention) {
|
|
2254
|
+
return !!retention && TERMINAL.some((status) => retention[status] !== void 0);
|
|
2255
|
+
}
|
|
2256
|
+
/**
|
|
2257
|
+
* Deletes finished runs once they pass their retention. Safe to run on every
|
|
2258
|
+
* worker: one lock per interval picks a single sweeper, and deleting a run is
|
|
2259
|
+
* idempotent anyway. A run's own key and index entries go last, in one
|
|
2260
|
+
* transaction, so a crash part-way leaves the run listed and the next sweep
|
|
2261
|
+
* finishes the job.
|
|
2262
|
+
*/
|
|
2263
|
+
function createReaper(options) {
|
|
2264
|
+
const { redis, keyPrefix, retention } = options;
|
|
2265
|
+
const intervalMs = retention.sweepIntervalMs ?? DEFAULT_SWEEP_INTERVAL_MS;
|
|
2266
|
+
const batchSize = retention.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
2267
|
+
const lockKey = `${keyPrefix}reaper:lock`;
|
|
2268
|
+
const byStatusKey = (status) => `${keyPrefix}runs:by_status:${status}`;
|
|
2269
|
+
let timer;
|
|
2270
|
+
let running = Promise.resolve();
|
|
2271
|
+
/** Runs past retention, oldest first: too old, or beyond the newest `count`. */
|
|
2272
|
+
async function expired(status) {
|
|
2273
|
+
const keep = retention[status];
|
|
2274
|
+
if (!keep) return [];
|
|
2275
|
+
const ids = /* @__PURE__ */ new Set();
|
|
2276
|
+
if (keep.age !== void 0) {
|
|
2277
|
+
const cutoff = Date.now() - keep.age * 1e3;
|
|
2278
|
+
for (const id of await redis.zrangebyscore(byStatusKey(status), "-inf", cutoff, "LIMIT", 0, batchSize)) ids.add(id);
|
|
2279
|
+
}
|
|
2280
|
+
if (keep.count !== void 0) {
|
|
2281
|
+
const stop = -(keep.count + 1);
|
|
2282
|
+
for (const id of await redis.zrange(byStatusKey(status), "0", String(stop))) {
|
|
2283
|
+
if (ids.size >= batchSize) break;
|
|
2284
|
+
ids.add(id);
|
|
2285
|
+
}
|
|
2286
|
+
}
|
|
2287
|
+
return [...ids].slice(0, batchSize);
|
|
2288
|
+
}
|
|
2289
|
+
async function unlink(keys) {
|
|
2290
|
+
for (let i = 0; i < keys.length; i += UNLINK_CHUNK) await redis.unlink(...keys.slice(i, i + UNLINK_CHUNK));
|
|
2291
|
+
}
|
|
2292
|
+
async function scan(pattern) {
|
|
2293
|
+
const found = [];
|
|
2294
|
+
let cursor = "0";
|
|
2295
|
+
do {
|
|
2296
|
+
const [next, batch] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 1e3);
|
|
2297
|
+
cursor = next;
|
|
2298
|
+
found.push(...batch);
|
|
2299
|
+
} while (cursor !== "0");
|
|
2300
|
+
return found;
|
|
2301
|
+
}
|
|
2302
|
+
/** Delete one run if it is still finished. Returns whether it was deleted. */
|
|
2303
|
+
async function deleteRun(runId, listedAs) {
|
|
2304
|
+
const metaKey = `${keyPrefix}run:meta:${runId}`;
|
|
2305
|
+
const meta = await redis.get(metaKey);
|
|
2306
|
+
const status = meta?.split("|")[0];
|
|
2307
|
+
if (!status || !TERMINAL.includes(status)) {
|
|
2308
|
+
if (!meta) await redis.zrem(byStatusKey(listedAs), runId);
|
|
2309
|
+
return false;
|
|
2310
|
+
}
|
|
2311
|
+
const runKey = `${keyPrefix}run:${runId}`;
|
|
2312
|
+
const runJson = await redis.get(runKey);
|
|
2313
|
+
const workflowName = runJson ? JSON.parse(runJson).workflowName : void 0;
|
|
2314
|
+
const eventsKey = `${keyPrefix}events:data:${runId}`;
|
|
2315
|
+
const correlationIds = /* @__PURE__ */ new Set();
|
|
2316
|
+
let hasResume = false;
|
|
2317
|
+
for (const raw of await redis.hvals(eventsKey)) try {
|
|
2318
|
+
const event = JSON.parse(raw);
|
|
2319
|
+
if (event.correlationId) correlationIds.add(event.correlationId);
|
|
2320
|
+
if (event.eventType === "hook_received") hasResume = true;
|
|
2321
|
+
} catch {}
|
|
2322
|
+
const stepIds = await redis.zrange(`${keyPrefix}steps:by_run:${runId}`, "0", "-1");
|
|
2323
|
+
const hookIds = await redis.zrange(`${keyPrefix}hooks:by_run:${runId}`, "0", "-1");
|
|
2324
|
+
const streamNames = await redis.smembers(`${keyPrefix}streams:by_run:${runId}`);
|
|
2325
|
+
const keys = [
|
|
2326
|
+
eventsKey,
|
|
2327
|
+
`${keyPrefix}run:attrs:${runId}`,
|
|
2328
|
+
`${keyPrefix}steps:by_run:${runId}`,
|
|
2329
|
+
`${keyPrefix}hooks:by_run:${runId}`,
|
|
2330
|
+
`${keyPrefix}hooks:tokens:${runId}`,
|
|
2331
|
+
`${keyPrefix}streams:by_run:${runId}`,
|
|
2332
|
+
...[...correlationIds].map((id) => `${keyPrefix}events:by_correlation:${runId}:${id}`),
|
|
2333
|
+
...stepIds.flatMap((id) => [`${keyPrefix}step:${runId}:${id}`, `${keyPrefix}step:status:${runId}:${id}`]),
|
|
2334
|
+
...hookIds.map((id) => `${keyPrefix}hook:${id}`),
|
|
2335
|
+
...streamNames.map((name) => `${keyPrefix}stream:${runId}:${name}`)
|
|
2336
|
+
];
|
|
2337
|
+
if (hasResume) keys.push(...await scan(`${keyPrefix}events:resume:${runId}:*`));
|
|
2338
|
+
await unlink(keys);
|
|
2339
|
+
const tx = redis.multi().zrem(`${keyPrefix}runs:index`, runId).zrem(byStatusKey(status), runId).unlink(metaKey, runKey);
|
|
2340
|
+
if (workflowName) tx.zrem(`${keyPrefix}runs:by_name:${workflowName}`, runId);
|
|
2341
|
+
await tx.exec();
|
|
2342
|
+
return true;
|
|
2343
|
+
}
|
|
2344
|
+
async function sweep() {
|
|
2345
|
+
if (await redis.set(lockKey, "1", "PX", Math.max(1e3, Math.floor(intervalMs / 2)), "NX") !== "OK") return 0;
|
|
2346
|
+
let deleted = 0;
|
|
2347
|
+
for (const status of TERMINAL) for (const runId of await expired(status)) try {
|
|
2348
|
+
if (await deleteRun(runId, status)) deleted++;
|
|
2349
|
+
} catch (err) {
|
|
2350
|
+
debug("reaper failed to delete run", runId, err);
|
|
2351
|
+
}
|
|
2352
|
+
return deleted;
|
|
2353
|
+
}
|
|
2354
|
+
return {
|
|
2355
|
+
sweep,
|
|
2356
|
+
start() {
|
|
2357
|
+
if (timer) return;
|
|
2358
|
+
const tick = () => {
|
|
2359
|
+
running = sweep().catch((err) => debug("reaper sweep failed", err));
|
|
2360
|
+
};
|
|
2361
|
+
timer = setInterval(tick, intervalMs);
|
|
2362
|
+
timer.unref();
|
|
2363
|
+
},
|
|
2364
|
+
async stop() {
|
|
2365
|
+
if (timer) clearInterval(timer);
|
|
2366
|
+
timer = void 0;
|
|
2367
|
+
await running;
|
|
2368
|
+
}
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
//#endregion
|
|
2372
|
+
//#region src/streamer.ts
|
|
2373
|
+
function parseStreamPublishMessage(raw) {
|
|
2374
|
+
let parsed;
|
|
2375
|
+
try {
|
|
2376
|
+
parsed = JSON.parse(raw);
|
|
2377
|
+
} catch {
|
|
2378
|
+
return null;
|
|
2379
|
+
}
|
|
2380
|
+
if (!parsed || typeof parsed !== "object") return null;
|
|
2381
|
+
const { runId, streamId, entryId, data, eof, prevId } = parsed;
|
|
2382
|
+
if (typeof runId !== "string" || typeof streamId !== "string") return null;
|
|
2383
|
+
if (typeof entryId !== "string" || typeof prevId !== "string") return null;
|
|
2384
|
+
if (typeof data !== "string" || typeof eof !== "boolean") return null;
|
|
2385
|
+
return {
|
|
2386
|
+
runId,
|
|
2387
|
+
streamId,
|
|
2388
|
+
entryId,
|
|
2389
|
+
data,
|
|
2390
|
+
eof,
|
|
2391
|
+
prevId
|
|
2392
|
+
};
|
|
2393
|
+
}
|
|
2394
|
+
/** A reader also re-reads the stream this often, in case a notification and the
|
|
2395
|
+
* reconnect hook both missed something. Notifications are the fast path. */
|
|
2396
|
+
const CATCH_UP_INTERVAL_MS = 2e3;
|
|
2397
|
+
/**
|
|
2398
|
+
* Compare two Redis stream entry IDs (`<ms>-<seq>`) numerically.
|
|
2399
|
+
*
|
|
2400
|
+
* String comparison is NOT safe here: within one millisecond `...-10` compares
|
|
2401
|
+
* lexicographically smaller than `...-9`, which would drop chunks (and the eof
|
|
2402
|
+
* marker) once the sequence number crosses a digit boundary.
|
|
2403
|
+
*/
|
|
2404
|
+
function compareStreamEntryIds(a, b) {
|
|
2405
|
+
const [aMs, aSeq] = a.split("-").map(Number);
|
|
2406
|
+
const [bMs, bSeq] = b.split("-").map(Number);
|
|
2407
|
+
return aMs - bMs || aSeq - bSeq;
|
|
2408
|
+
}
|
|
2409
|
+
/**
|
|
2410
|
+
* Append one or more chunks, index the stream under its run, and notify
|
|
2411
|
+
* subscribers in one round trip. The publish could not be pipelined with the
|
|
2412
|
+
* append because it needs the entry ID `XADD` generates; Lua makes that
|
|
2413
|
+
* dependency server-side. Each chunk rides along in its message so subscribers
|
|
2414
|
+
* do not `XRANGE` per notification.
|
|
2415
|
+
*
|
|
2416
|
+
* A stream that already holds its eof marker is closed: the append is dropped
|
|
2417
|
+
* and nothing is published. Readers stop at the first eof anyway, so this only
|
|
2418
|
+
* keeps the invariant that the eof marker, when present, is the final entry.
|
|
2419
|
+
* `getInfo` relies on it to answer from `XLEN` and the tail entry alone, and a
|
|
2420
|
+
* producer retrying a lost-ACK `close()` would otherwise break it.
|
|
2421
|
+
*
|
|
2422
|
+
* KEYS[1] = stream key
|
|
2423
|
+
* KEYS[2] = streams-by-run key
|
|
2424
|
+
* ARGV[1] = notification channel
|
|
2425
|
+
* ARGV[2] = run ID
|
|
2426
|
+
* ARGV[3] = stream name
|
|
2427
|
+
* ARGV[4] = 'true' | 'false' (eof marker, applied to every chunk below)
|
|
2428
|
+
* ARGV[5..] = base64 chunk data, in write order
|
|
2429
|
+
* Returns: 1 if appended, 0 if the stream was already closed
|
|
2430
|
+
*/
|
|
2431
|
+
const LUA_APPEND_AND_PUBLISH = `
|
|
2432
|
+
local streamKey = KEYS[1]
|
|
2433
|
+
local runIndexKey = KEYS[2]
|
|
2434
|
+
local channel = ARGV[1]
|
|
2435
|
+
local runId = ARGV[2]
|
|
2436
|
+
local streamId = ARGV[3]
|
|
2437
|
+
local eof = ARGV[4]
|
|
2438
|
+
|
|
2439
|
+
local tail = redis.call('XREVRANGE', streamKey, '+', '-', 'COUNT', 1)
|
|
2440
|
+
if #tail > 0 and tail[1][2][4] == 'true' then
|
|
2441
|
+
return 0
|
|
2442
|
+
end
|
|
2443
|
+
|
|
2444
|
+
local prevId = ''
|
|
2445
|
+
if #tail > 0 then
|
|
2446
|
+
prevId = tail[1][1]
|
|
2447
|
+
end
|
|
2448
|
+
|
|
2449
|
+
for i = 5, #ARGV do
|
|
2450
|
+
local data = ARGV[i]
|
|
2451
|
+
local entryId = redis.call('XADD', streamKey, '*', 'data', data, 'eof', eof)
|
|
2452
|
+
redis.call('PUBLISH', channel, cjson.encode({
|
|
2453
|
+
runId = runId,
|
|
2454
|
+
streamId = streamId,
|
|
2455
|
+
entryId = entryId,
|
|
2456
|
+
data = data,
|
|
2457
|
+
eof = eof == 'true',
|
|
2458
|
+
prevId = prevId
|
|
2459
|
+
}))
|
|
2460
|
+
prevId = entryId
|
|
2461
|
+
end
|
|
2462
|
+
redis.call('SADD', runIndexKey, streamId)
|
|
2463
|
+
return 1
|
|
2464
|
+
`;
|
|
2465
|
+
/**
|
|
2466
|
+
* Create a streamer implementation using Redis Streams.
|
|
2467
|
+
*
|
|
2468
|
+
* Uses Redis auto-generated stream entry IDs (timestamp-sequence format)
|
|
2469
|
+
* instead of custom IDs, since Redis Streams require IDs in
|
|
2470
|
+
* `<millisecondsTime>-<sequenceNumber>` format.
|
|
2471
|
+
*
|
|
2472
|
+
* Key layout (every key is scoped to the run that owns the stream, so two runs
|
|
2473
|
+
* can use the same stream name without sharing chunks):
|
|
2474
|
+
* - `<prefix>stream:<runId>:<name>` Redis Stream of the chunks plus the eof marker
|
|
2475
|
+
* - `<prefix>streams:by_run:<runId>` Set of the run's stream names
|
|
2476
|
+
* - `<prefix>stream_notify` pub/sub channel carrying live chunks
|
|
2477
|
+
*/
|
|
2478
|
+
function createStreamer(config) {
|
|
2479
|
+
const { redis, keyPrefix } = config;
|
|
2480
|
+
const events = new EventEmitter();
|
|
2481
|
+
const mutexes = /* @__PURE__ */ new Map();
|
|
2482
|
+
const withStreamMutex = async (key, fn) => {
|
|
2483
|
+
let entry = mutexes.get(key);
|
|
2484
|
+
if (!entry) {
|
|
2485
|
+
entry = {
|
|
2486
|
+
mutex: new Mutex(),
|
|
2487
|
+
rc: new Rc(() => {
|
|
2488
|
+
mutexes.delete(key);
|
|
2489
|
+
})
|
|
2490
|
+
};
|
|
2491
|
+
mutexes.set(key, entry);
|
|
2492
|
+
}
|
|
2493
|
+
entry.rc.inc();
|
|
2494
|
+
try {
|
|
2495
|
+
return await entry.mutex.andThen(fn);
|
|
2496
|
+
} finally {
|
|
2497
|
+
await entry.rc.dec();
|
|
2498
|
+
}
|
|
2499
|
+
};
|
|
2500
|
+
redis.defineCommand("wfStreamAppend", {
|
|
2501
|
+
numberOfKeys: 2,
|
|
2502
|
+
lua: LUA_APPEND_AND_PUBLISH
|
|
2503
|
+
});
|
|
2504
|
+
const scripted = redis;
|
|
2505
|
+
const streamKey = (runId, name) => `${keyPrefix}stream:${runId}:${name}`;
|
|
2506
|
+
const streamsByRunKey = (runId) => `${keyPrefix}streams:by_run:${runId}`;
|
|
2507
|
+
const STREAM_CHANNEL = `${keyPrefix}stream_notify`;
|
|
2508
|
+
const emitterKey = (runId, name) => `strm:${runId}:${name}`;
|
|
2509
|
+
const subscriber = redis.duplicate();
|
|
2510
|
+
const subscriptionReady = subscriber.subscribe(STREAM_CHANNEL).then(() => {});
|
|
2511
|
+
const resync = new EventEmitter();
|
|
2512
|
+
resync.setMaxListeners(0);
|
|
2513
|
+
let subscriberConnected = false;
|
|
2514
|
+
subscriber.on("ready", () => {
|
|
2515
|
+
if (subscriberConnected) resync.emit("resync");
|
|
2516
|
+
subscriberConnected = true;
|
|
2517
|
+
});
|
|
2518
|
+
subscriber.on("message", async (channel, message) => {
|
|
2519
|
+
if (channel !== STREAM_CHANNEL) return;
|
|
2520
|
+
try {
|
|
2521
|
+
const parsed = parseStreamPublishMessage(message);
|
|
2522
|
+
if (!parsed) return;
|
|
2523
|
+
const key = emitterKey(parsed.runId, parsed.streamId);
|
|
2524
|
+
if (!events.listenerCount(key)) return;
|
|
2525
|
+
await withStreamMutex(key, async () => {
|
|
2526
|
+
events.emit(key, {
|
|
2527
|
+
id: parsed.entryId,
|
|
2528
|
+
data: Buffer.from(parsed.data, "base64"),
|
|
2529
|
+
eof: parsed.eof,
|
|
2530
|
+
prevId: parsed.prevId
|
|
2531
|
+
});
|
|
2532
|
+
});
|
|
2533
|
+
} catch (error) {
|
|
2534
|
+
console.error("Error processing stream notification:", error);
|
|
2535
|
+
}
|
|
2536
|
+
});
|
|
2537
|
+
const toBase64 = (chunk) => (Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)).toString("base64");
|
|
2538
|
+
const append = async (runId, name, eof, encodedChunks) => {
|
|
2539
|
+
await subscriptionReady;
|
|
2540
|
+
await scripted.wfStreamAppend(streamKey(runId, name), streamsByRunKey(runId), STREAM_CHANNEL, runId, name, eof ? "true" : "false", ...encodedChunks);
|
|
2541
|
+
};
|
|
2542
|
+
return {
|
|
2543
|
+
streams: {
|
|
2544
|
+
async write(_runId, name, chunk) {
|
|
2545
|
+
const runId = await _runId;
|
|
2546
|
+
await append(runId, name, false, [toBase64(chunk)]);
|
|
2547
|
+
},
|
|
2548
|
+
async writeMulti(_runId, name, chunks) {
|
|
2549
|
+
if (chunks.length === 0) return;
|
|
2550
|
+
const runId = await _runId;
|
|
2551
|
+
await append(runId, name, false, chunks.map(toBase64));
|
|
2552
|
+
},
|
|
2553
|
+
async close(_runId, name) {
|
|
2554
|
+
const runId = await _runId;
|
|
2555
|
+
await append(runId, name, true, [""]);
|
|
2556
|
+
},
|
|
2557
|
+
async get(runId, name, startIndex) {
|
|
2558
|
+
const key = emitterKey(runId, name);
|
|
2559
|
+
const cleanups = [];
|
|
2560
|
+
let cleanedUp = false;
|
|
2561
|
+
const cleanup = () => {
|
|
2562
|
+
if (cleanedUp) return;
|
|
2563
|
+
cleanedUp = true;
|
|
2564
|
+
for (const fn of cleanups) fn();
|
|
2565
|
+
};
|
|
2566
|
+
return new ReadableStream({
|
|
2567
|
+
async start(controller) {
|
|
2568
|
+
let lastEntryId = "";
|
|
2569
|
+
let offset = startIndex ?? 0;
|
|
2570
|
+
let buffer = [];
|
|
2571
|
+
function enqueue(msg) {
|
|
2572
|
+
if (cleanedUp) return;
|
|
2573
|
+
if (lastEntryId !== "" && compareStreamEntryIds(lastEntryId, msg.id) >= 0) return;
|
|
2574
|
+
lastEntryId = msg.id;
|
|
2575
|
+
if (offset > 0 && !msg.eof) {
|
|
2576
|
+
offset--;
|
|
2577
|
+
return;
|
|
2578
|
+
}
|
|
2579
|
+
if (msg.data.byteLength) controller.enqueue(new Uint8Array(msg.data));
|
|
2580
|
+
if (msg.eof) {
|
|
2581
|
+
cleanup();
|
|
2582
|
+
controller.close();
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
let catchingUp = false;
|
|
2586
|
+
let again = false;
|
|
2587
|
+
async function catchUp() {
|
|
2588
|
+
if (cleanedUp || buffer) return;
|
|
2589
|
+
if (catchingUp) {
|
|
2590
|
+
again = true;
|
|
2591
|
+
return;
|
|
2592
|
+
}
|
|
2593
|
+
catchingUp = true;
|
|
2594
|
+
try {
|
|
2595
|
+
do {
|
|
2596
|
+
again = false;
|
|
2597
|
+
const entries = await redis.xrange(streamKey(runId, name), lastEntryId === "" ? "-" : `(${lastEntryId}`, "+");
|
|
2598
|
+
for (const [id, fields] of entries) enqueue({
|
|
2599
|
+
id,
|
|
2600
|
+
data: Buffer.from(fields[1], "base64"),
|
|
2601
|
+
eof: fields[3] === "true"
|
|
2602
|
+
});
|
|
2603
|
+
} while (again && !cleanedUp);
|
|
2604
|
+
} catch {} finally {
|
|
2605
|
+
catchingUp = false;
|
|
2606
|
+
}
|
|
2607
|
+
}
|
|
2608
|
+
function deliverLive(data) {
|
|
2609
|
+
if (data.prevId !== void 0 && data.prevId !== lastEntryId) {
|
|
2610
|
+
catchUp();
|
|
2611
|
+
return;
|
|
2612
|
+
}
|
|
2613
|
+
enqueue(data);
|
|
2614
|
+
}
|
|
2615
|
+
function onData(data) {
|
|
2616
|
+
if (buffer) {
|
|
2617
|
+
buffer.push(data);
|
|
2618
|
+
return;
|
|
2619
|
+
}
|
|
2620
|
+
deliverLive(data);
|
|
2621
|
+
}
|
|
2622
|
+
events.on(key, onData);
|
|
2623
|
+
cleanups.push(() => {
|
|
2624
|
+
events.off(key, onData);
|
|
2625
|
+
});
|
|
2626
|
+
const chunks = await redis.xrange(streamKey(runId, name), "-", "+").catch((err) => {
|
|
2627
|
+
cleanup();
|
|
2628
|
+
throw err;
|
|
2629
|
+
});
|
|
2630
|
+
const parsedChunks = [];
|
|
2631
|
+
for (const [id, fields] of chunks) {
|
|
2632
|
+
const data = Buffer.from(fields[1], "base64");
|
|
2633
|
+
const eof = fields[3] === "true";
|
|
2634
|
+
parsedChunks.push({
|
|
2635
|
+
id,
|
|
2636
|
+
data,
|
|
2637
|
+
eof
|
|
2638
|
+
});
|
|
2639
|
+
}
|
|
2640
|
+
if (offset < 0) {
|
|
2641
|
+
const firstEof = parsedChunks.findIndex((chunk) => chunk.eof);
|
|
2642
|
+
const dataCount = firstEof === -1 ? parsedChunks.length : firstEof;
|
|
2643
|
+
offset = Math.max(0, dataCount + offset);
|
|
2644
|
+
}
|
|
2645
|
+
const buffered = buffer ?? [];
|
|
2646
|
+
buffer = null;
|
|
2647
|
+
for (const chunk of parsedChunks) enqueue(chunk);
|
|
2648
|
+
for (const chunk of buffered) deliverLive(chunk);
|
|
2649
|
+
resync.on("resync", catchUp);
|
|
2650
|
+
const timer = setInterval(() => void catchUp(), CATCH_UP_INTERVAL_MS);
|
|
2651
|
+
timer.unref();
|
|
2652
|
+
cleanups.push(() => {
|
|
2653
|
+
resync.off("resync", catchUp);
|
|
2654
|
+
clearInterval(timer);
|
|
2655
|
+
});
|
|
2656
|
+
await catchUp();
|
|
2657
|
+
},
|
|
2658
|
+
cancel() {
|
|
2659
|
+
cleanup();
|
|
2660
|
+
}
|
|
2661
|
+
});
|
|
2662
|
+
},
|
|
2663
|
+
async list(runId) {
|
|
2664
|
+
return await redis.smembers(streamsByRunKey(runId)) ?? [];
|
|
2665
|
+
},
|
|
2666
|
+
async getChunks(runId, name, options) {
|
|
2667
|
+
const limit = Math.min(options?.limit ?? 100, 1e3);
|
|
2668
|
+
let dataIndex = 0;
|
|
2669
|
+
let start = "-";
|
|
2670
|
+
if (options?.cursor) try {
|
|
2671
|
+
const decoded = JSON.parse(Buffer.from(options.cursor, "base64").toString("utf-8"));
|
|
2672
|
+
if (typeof decoded.i === "number") dataIndex = decoded.i;
|
|
2673
|
+
if (typeof decoded.id === "string") start = `(${decoded.id}`;
|
|
2674
|
+
} catch {
|
|
2675
|
+
dataIndex = 0;
|
|
2676
|
+
start = "-";
|
|
2677
|
+
}
|
|
2678
|
+
const entries = await redis.xrange(streamKey(runId, name), start, "+", "COUNT", limit + 1);
|
|
2679
|
+
const data = [];
|
|
2680
|
+
let done = false;
|
|
2681
|
+
let hasMore = false;
|
|
2682
|
+
let lastId = null;
|
|
2683
|
+
for (const [id, fields] of entries) {
|
|
2684
|
+
if (fields[3] === "true") {
|
|
2685
|
+
done = true;
|
|
2686
|
+
break;
|
|
2687
|
+
}
|
|
2688
|
+
if (data.length >= limit) {
|
|
2689
|
+
hasMore = true;
|
|
2690
|
+
break;
|
|
2691
|
+
}
|
|
2692
|
+
data.push({
|
|
2693
|
+
index: dataIndex,
|
|
2694
|
+
data: new Uint8Array(Buffer.from(fields[1], "base64"))
|
|
2695
|
+
});
|
|
2696
|
+
dataIndex++;
|
|
2697
|
+
lastId = id;
|
|
2698
|
+
}
|
|
2699
|
+
return {
|
|
2700
|
+
data,
|
|
2701
|
+
cursor: hasMore && lastId ? Buffer.from(JSON.stringify({
|
|
2702
|
+
i: dataIndex,
|
|
2703
|
+
id: lastId
|
|
2704
|
+
})).toString("base64") : null,
|
|
2705
|
+
hasMore,
|
|
2706
|
+
done
|
|
2707
|
+
};
|
|
2708
|
+
},
|
|
2709
|
+
async getInfo(runId, name) {
|
|
2710
|
+
const key = streamKey(runId, name);
|
|
2711
|
+
const [length, tail] = await Promise.all([redis.xlen(key), redis.xrevrange(key, "+", "-", "COUNT", 1)]);
|
|
2712
|
+
const done = tail.length > 0 && tail[0][1][3] === "true";
|
|
2713
|
+
return {
|
|
2714
|
+
tailIndex: length - (done ? 1 : 0) - 1,
|
|
2715
|
+
done
|
|
2716
|
+
};
|
|
2717
|
+
}
|
|
2718
|
+
},
|
|
2719
|
+
async close() {
|
|
2720
|
+
await subscriber.quit();
|
|
2721
|
+
}
|
|
2722
|
+
};
|
|
2723
|
+
}
|
|
2724
|
+
//#endregion
|
|
2725
|
+
//#region src/index.ts
|
|
2726
|
+
function createStorage(redis, keyPrefix, maxEventsPerRun) {
|
|
2727
|
+
const config = {
|
|
2728
|
+
redis,
|
|
2729
|
+
keyPrefix,
|
|
2730
|
+
maxEventsPerRun
|
|
2731
|
+
};
|
|
2732
|
+
return {
|
|
2733
|
+
runs: createRunsStorage(config),
|
|
2734
|
+
events: createEventsStorage(config),
|
|
2735
|
+
hooks: createHooksStorage(config),
|
|
2736
|
+
steps: createStepsStorage(config)
|
|
2737
|
+
};
|
|
2738
|
+
}
|
|
2739
|
+
function createWorld(config = {
|
|
2740
|
+
redis: process.env.WORKFLOW_REDIS_URL || process.env.REDIS_URL || "redis://localhost:6379",
|
|
2741
|
+
queueConcurrency: Number.parseInt(process.env.WORKFLOW_REDIS_WORKER_CONCURRENCY || "10", 10) || 10,
|
|
2742
|
+
keyPrefix: process.env.WORKFLOW_REDIS_KEY_PREFIX || "workflow:"
|
|
2743
|
+
}) {
|
|
2744
|
+
const autoPipelining = config.enableAutoPipelining ?? true;
|
|
2745
|
+
const redis = typeof config.redis === "string" ? new Redis(config.redis, { enableAutoPipelining: autoPipelining }) : new Redis({
|
|
2746
|
+
enableAutoPipelining: autoPipelining,
|
|
2747
|
+
...config.redis
|
|
2748
|
+
});
|
|
2749
|
+
const keyPrefix = config.keyPrefix || "workflow:";
|
|
2750
|
+
validateRetention(config.retention);
|
|
2751
|
+
const reaper = hasRetention(config.retention) ? createReaper({
|
|
2752
|
+
redis,
|
|
2753
|
+
keyPrefix,
|
|
2754
|
+
retention: config.retention
|
|
2755
|
+
}) : void 0;
|
|
2756
|
+
const storage = createStorage(redis, keyPrefix, config.maxEventsPerRun);
|
|
2757
|
+
const queue = createQueue(redis, config, { runHasStarted: async (runId) => {
|
|
2758
|
+
try {
|
|
2759
|
+
return (await storage.runs.get(runId, { resolveData: "none" })).status !== "pending";
|
|
2760
|
+
} catch (error) {
|
|
2761
|
+
if (WorkflowRunNotFoundError.is(error)) return false;
|
|
2762
|
+
throw error;
|
|
2763
|
+
}
|
|
2764
|
+
} });
|
|
2765
|
+
const streamer = createStreamer({
|
|
2766
|
+
redis,
|
|
2767
|
+
keyPrefix
|
|
2768
|
+
});
|
|
2769
|
+
return {
|
|
2770
|
+
...storage,
|
|
2771
|
+
...streamer,
|
|
2772
|
+
...queue,
|
|
2773
|
+
specVersion: mintedSpecVersion(),
|
|
2774
|
+
createRunId,
|
|
2775
|
+
async start() {
|
|
2776
|
+
await queue.start();
|
|
2777
|
+
reaper?.start();
|
|
2778
|
+
await reenqueueActiveRuns(storage.runs, (queueName, message, opts) => queue.queue(queueName, message, {
|
|
2779
|
+
...opts,
|
|
2780
|
+
idempotencyKey: `recover:${message.runId}`
|
|
2781
|
+
}), "world-redis");
|
|
2782
|
+
},
|
|
2783
|
+
async sweepRetention() {
|
|
2784
|
+
return reaper ? reaper.sweep() : 0;
|
|
2785
|
+
},
|
|
2786
|
+
async close() {
|
|
2787
|
+
await reaper?.stop();
|
|
2788
|
+
await queue.close();
|
|
2789
|
+
await streamer.close();
|
|
2790
|
+
await redis.quit();
|
|
2791
|
+
}
|
|
2792
|
+
};
|
|
2793
|
+
}
|
|
2794
|
+
//#endregion
|
|
2795
|
+
export { createWorld };
|
|
2796
|
+
|
|
2797
|
+
//# sourceMappingURL=index.js.map
|