@ewhauser/world-celld 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/LICENSE +201 -0
- package/NOTICE +24 -0
- package/README.md +292 -0
- package/celld-worker/README.md +34 -0
- package/celld-worker/worker.ts +5 -0
- package/celld-worker/wrangler.jsonc +32 -0
- package/dist/index.d.ts +250 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +949 -0
- package/dist/index.js.map +1 -0
- package/dist/testing.d.ts +134 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +1808 -0
- package/dist/testing.js.map +1 -0
- package/dist/util-BvWlkqgV.js +122 -0
- package/dist/util-BvWlkqgV.js.map +1 -0
- package/dist/worker.d.ts +406 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +1582 -0
- package/dist/worker.js.map +1 -0
- package/package.json +90 -0
package/dist/worker.js
ADDED
|
@@ -0,0 +1,1582 @@
|
|
|
1
|
+
import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WorkflowRunSchema, isLegacySpecVersion, isTerminalStepStatus, isTerminalWorkflowRunStatus, requiresNewerWorld } from "@workflow/world";
|
|
2
|
+
import { DurableObject } from "cloudflare:workers";
|
|
3
|
+
//#region src/vendor/shared/serialization.ts
|
|
4
|
+
/** Standard Date fields used across all worlds */
|
|
5
|
+
const DATE_FIELDS = /* @__PURE__ */ new Set([
|
|
6
|
+
"createdAt",
|
|
7
|
+
"updatedAt",
|
|
8
|
+
"startedAt",
|
|
9
|
+
"completedAt",
|
|
10
|
+
"retryAfter"
|
|
11
|
+
]);
|
|
12
|
+
/**
|
|
13
|
+
* JSON reviver that converts ISO date strings to Date objects.
|
|
14
|
+
* Use with JSON.parse(json, dateReviver).
|
|
15
|
+
*/
|
|
16
|
+
function dateReviver(key, value) {
|
|
17
|
+
if (DATE_FIELDS.has(key) && typeof value === "string") {
|
|
18
|
+
const date = new Date(value);
|
|
19
|
+
return Number.isNaN(date.getTime()) ? value : date;
|
|
20
|
+
}
|
|
21
|
+
return value;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Base64 helpers. Modified vs upstream: implemented over atob/btoa instead of
|
|
25
|
+
* Buffer so this module is safe inside the celld worker bundle (workerd has
|
|
26
|
+
* no Buffer global); atob/btoa are global in both workerd and Node >= 16.
|
|
27
|
+
*/
|
|
28
|
+
function b64encode(bytes) {
|
|
29
|
+
let bin = "";
|
|
30
|
+
const CHUNK = 32768;
|
|
31
|
+
for (let i = 0; i < bytes.length; i += CHUNK) bin += String.fromCharCode(...bytes.subarray(i, i + CHUNK));
|
|
32
|
+
return btoa(bin);
|
|
33
|
+
}
|
|
34
|
+
function b64decode(text) {
|
|
35
|
+
const bin = atob(text);
|
|
36
|
+
const out = new Uint8Array(bin.length);
|
|
37
|
+
for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i);
|
|
38
|
+
return out;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* JSON reviver that decodes tagged Uint8Array objects and ISO date strings.
|
|
42
|
+
* Supports both:
|
|
43
|
+
* - New format: { __type: 'Uint8Array', data: '<base64>' }
|
|
44
|
+
* - Legacy format: { __uint8array: true, data: [1, 2, 3] }
|
|
45
|
+
*/
|
|
46
|
+
function uint8ArrayReviver(key, value) {
|
|
47
|
+
if (value && typeof value === "object") {
|
|
48
|
+
const obj = value;
|
|
49
|
+
if (obj.__type === "Uint8Array" && typeof obj.data === "string") return b64decode(obj.data);
|
|
50
|
+
if (obj.__uint8array === true && Array.isArray(obj.data)) return new Uint8Array(obj.data);
|
|
51
|
+
}
|
|
52
|
+
return dateReviver(key, value);
|
|
53
|
+
}
|
|
54
|
+
/** Parse with Uint8Array and Date support. */
|
|
55
|
+
function parse(text) {
|
|
56
|
+
return JSON.parse(text, uint8ArrayReviver);
|
|
57
|
+
}
|
|
58
|
+
//#endregion
|
|
59
|
+
//#region src/codec.ts
|
|
60
|
+
/**
|
|
61
|
+
* RPC wire codec for the Node client <-> worker router HTTP boundary.
|
|
62
|
+
*
|
|
63
|
+
* Extends the shared tagged-JSON format (`{__type:'Uint8Array', data}` +
|
|
64
|
+
* DATE_FIELDS revival) with explicitly tagged Dates: on Cloudflare the DO RPC
|
|
65
|
+
* boundary is structured clone, which preserves EVERY Date — including ones
|
|
66
|
+
* outside the well-known DATE_FIELDS (e.g. inside applyEvent outcomes and
|
|
67
|
+
* user payloads). The HTTP hop must preserve them too, so Dates travel as
|
|
68
|
+
* `{__type:'Date', iso}`.
|
|
69
|
+
*
|
|
70
|
+
* Runs on both sides of the wire, including inside the celld worker — so no
|
|
71
|
+
* Node built-ins (Buffer): base64 goes through atob/btoa.
|
|
72
|
+
*/
|
|
73
|
+
function rpcReplacer(key, value) {
|
|
74
|
+
const original = this && typeof this === "object" ? this[key] : void 0;
|
|
75
|
+
if (original instanceof Date) return {
|
|
76
|
+
__type: "Date",
|
|
77
|
+
iso: original.toISOString()
|
|
78
|
+
};
|
|
79
|
+
if (value instanceof Uint8Array) return {
|
|
80
|
+
__type: "Uint8Array",
|
|
81
|
+
data: b64encode(value)
|
|
82
|
+
};
|
|
83
|
+
return value;
|
|
84
|
+
}
|
|
85
|
+
function rpcReviver(key, value) {
|
|
86
|
+
if (value && typeof value === "object") {
|
|
87
|
+
const obj = value;
|
|
88
|
+
if (obj.__type === "Date" && typeof obj.iso === "string") return new Date(obj.iso);
|
|
89
|
+
if (obj.__type === "Uint8Array" && typeof obj.data === "string") return b64decode(obj.data);
|
|
90
|
+
}
|
|
91
|
+
return dateReviver(key, value);
|
|
92
|
+
}
|
|
93
|
+
function rpcStringify(value) {
|
|
94
|
+
return JSON.stringify(value, rpcReplacer);
|
|
95
|
+
}
|
|
96
|
+
function rpcParse(text) {
|
|
97
|
+
return JSON.parse(text, rpcReviver);
|
|
98
|
+
}
|
|
99
|
+
//#endregion
|
|
100
|
+
//#region src/worker/auth.ts
|
|
101
|
+
/**
|
|
102
|
+
* Bearer-token auth for the worker router.
|
|
103
|
+
*
|
|
104
|
+
* Fails closed: when the WORLD_SECRET var is not configured, every protected
|
|
105
|
+
* route returns 503 instead of forwarding unauthenticated traffic (the
|
|
106
|
+
* eve-ambient celld-worker pattern).
|
|
107
|
+
*/
|
|
108
|
+
/**
|
|
109
|
+
* Constant-time string comparison via SHA-256 digests (digest-then-compare
|
|
110
|
+
* is length-independent). crypto.subtle is global in both workerd and Node.
|
|
111
|
+
*/
|
|
112
|
+
async function timingSafeEqual(a, b) {
|
|
113
|
+
const enc = new TextEncoder();
|
|
114
|
+
const [da, db] = await Promise.all([crypto.subtle.digest("SHA-256", enc.encode(a)), crypto.subtle.digest("SHA-256", enc.encode(b))]);
|
|
115
|
+
const va = new Uint8Array(da);
|
|
116
|
+
const vb = new Uint8Array(db);
|
|
117
|
+
let diff = 0;
|
|
118
|
+
for (let i = 0; i < va.length; i++) diff |= va[i] ^ vb[i];
|
|
119
|
+
return diff === 0;
|
|
120
|
+
}
|
|
121
|
+
async function authenticate(request, secret) {
|
|
122
|
+
if (!secret) return {
|
|
123
|
+
ok: false,
|
|
124
|
+
response: Response.json({ error: {
|
|
125
|
+
name: "WorldNotConfigured",
|
|
126
|
+
message: "WORLD_SECRET is not configured"
|
|
127
|
+
} }, { status: 503 })
|
|
128
|
+
};
|
|
129
|
+
const header = request.headers.get("authorization") ?? "";
|
|
130
|
+
const token = header.startsWith("Bearer ") ? header.slice(7) : "";
|
|
131
|
+
if (!token || !await timingSafeEqual(token, secret)) return {
|
|
132
|
+
ok: false,
|
|
133
|
+
response: Response.json({ error: {
|
|
134
|
+
name: "Unauthorized",
|
|
135
|
+
message: "invalid or missing bearer token"
|
|
136
|
+
} }, { status: 401 })
|
|
137
|
+
};
|
|
138
|
+
return { ok: true };
|
|
139
|
+
}
|
|
140
|
+
//#endregion
|
|
141
|
+
//#region src/worker/router.ts
|
|
142
|
+
/**
|
|
143
|
+
* Worker-level HTTP router: the fleet-facing RPC surface of the celld world.
|
|
144
|
+
*
|
|
145
|
+
* Routes:
|
|
146
|
+
* GET /v1/health — liveness + spec version (unauthenticated)
|
|
147
|
+
* POST /v1/rpc/{binding}/{name}/{method} — DO RPC (bearer auth)
|
|
148
|
+
*
|
|
149
|
+
* The RPC body is an rpc-codec-encoded JSON array of arguments; the response
|
|
150
|
+
* body is the rpc-codec-encoded return value. Only whitelisted methods
|
|
151
|
+
* dispatch — everything else (including fetch/alarm) is unreachable.
|
|
152
|
+
*/
|
|
153
|
+
const WORLD_NAME = "world-celld";
|
|
154
|
+
const WORLD_VERSION = "0.1.0";
|
|
155
|
+
const BINDINGS = {
|
|
156
|
+
runs: {
|
|
157
|
+
env: "WORKFLOW_DB",
|
|
158
|
+
methods: /* @__PURE__ */ new Set([
|
|
159
|
+
"applyEvent",
|
|
160
|
+
"getRun",
|
|
161
|
+
"getStep",
|
|
162
|
+
"getEvent",
|
|
163
|
+
"listEvents",
|
|
164
|
+
"listSteps",
|
|
165
|
+
"listHooks"
|
|
166
|
+
])
|
|
167
|
+
},
|
|
168
|
+
streams: {
|
|
169
|
+
env: "WORKFLOW_STREAMS",
|
|
170
|
+
methods: /* @__PURE__ */ new Set([
|
|
171
|
+
"writeChunk",
|
|
172
|
+
"closeStream",
|
|
173
|
+
"getChunks",
|
|
174
|
+
"getInfo",
|
|
175
|
+
"registerStream",
|
|
176
|
+
"listStreams"
|
|
177
|
+
])
|
|
178
|
+
},
|
|
179
|
+
index: {
|
|
180
|
+
env: "WORKFLOW_INDEX",
|
|
181
|
+
methods: /* @__PURE__ */ new Set([
|
|
182
|
+
"get",
|
|
183
|
+
"put",
|
|
184
|
+
"delete",
|
|
185
|
+
"list",
|
|
186
|
+
"reserveHookToken",
|
|
187
|
+
"finalizeHookIndexes",
|
|
188
|
+
"releaseHookToken",
|
|
189
|
+
"deleteHookIndexes"
|
|
190
|
+
])
|
|
191
|
+
},
|
|
192
|
+
queue: {
|
|
193
|
+
env: "WORKFLOW_QUEUE",
|
|
194
|
+
methods: /* @__PURE__ */ new Set([
|
|
195
|
+
"enqueue",
|
|
196
|
+
"stats",
|
|
197
|
+
"listDeadLetters",
|
|
198
|
+
"redriveDeadLetter",
|
|
199
|
+
"purgeDeadLetters",
|
|
200
|
+
"rearmAlarm"
|
|
201
|
+
])
|
|
202
|
+
}
|
|
203
|
+
};
|
|
204
|
+
/** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
|
|
205
|
+
const MAX_BODY_BYTES = 33554432;
|
|
206
|
+
function errorResponse(status, name, message) {
|
|
207
|
+
return Response.json({ error: {
|
|
208
|
+
name,
|
|
209
|
+
message
|
|
210
|
+
} }, { status });
|
|
211
|
+
}
|
|
212
|
+
async function readBoundedBody(request) {
|
|
213
|
+
if (!request.body) return "";
|
|
214
|
+
const reader = request.body.getReader();
|
|
215
|
+
const decoder = new TextDecoder();
|
|
216
|
+
let total = 0;
|
|
217
|
+
let text = "";
|
|
218
|
+
try {
|
|
219
|
+
while (true) {
|
|
220
|
+
const { done, value } = await reader.read();
|
|
221
|
+
if (done) break;
|
|
222
|
+
total += value.byteLength;
|
|
223
|
+
if (total > MAX_BODY_BYTES) {
|
|
224
|
+
await reader.cancel("RPC body exceeds configured limit").catch(() => void 0);
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
text += decoder.decode(value, { stream: true });
|
|
228
|
+
}
|
|
229
|
+
return text + decoder.decode();
|
|
230
|
+
} finally {
|
|
231
|
+
reader.releaseLock();
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
function createRouter(env) {
|
|
235
|
+
return async (request) => {
|
|
236
|
+
const url = new URL(request.url);
|
|
237
|
+
const parts = url.pathname.split("/").filter(Boolean);
|
|
238
|
+
if (parts[0] !== "v1") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
|
|
239
|
+
if (parts[1] === "health" && parts.length === 2) return Response.json({
|
|
240
|
+
ok: true,
|
|
241
|
+
name: WORLD_NAME,
|
|
242
|
+
version: WORLD_VERSION,
|
|
243
|
+
specVersion: SPEC_VERSION_CURRENT
|
|
244
|
+
});
|
|
245
|
+
if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
|
|
246
|
+
const auth = await authenticate(request, env.WORLD_SECRET);
|
|
247
|
+
if (!auth.ok) return auth.response;
|
|
248
|
+
if (request.method !== "POST" || parts.length !== 5) return errorResponse(404, "NotFound", "expected POST /v1/rpc/{binding}/{name}/{method}");
|
|
249
|
+
const [, , bindingKey, encodedName, method] = parts;
|
|
250
|
+
const binding = BINDINGS[bindingKey];
|
|
251
|
+
if (!binding) return errorResponse(404, "NotFound", `unknown binding: ${bindingKey}`);
|
|
252
|
+
if (!binding.methods.has(method)) return errorResponse(404, "NotFound", `unknown method: ${bindingKey}.${method}`);
|
|
253
|
+
if (Number(request.headers.get("content-length") ?? "0") > MAX_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
254
|
+
let args;
|
|
255
|
+
try {
|
|
256
|
+
const text = await readBoundedBody(request);
|
|
257
|
+
if (text === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
258
|
+
const parsed = rpcParse(text);
|
|
259
|
+
if (!Array.isArray(parsed)) return errorResponse(400, "BadRequest", "body must be an argument array");
|
|
260
|
+
args = parsed;
|
|
261
|
+
} catch {
|
|
262
|
+
return errorResponse(400, "BadRequest", "malformed rpc body");
|
|
263
|
+
}
|
|
264
|
+
const name = decodeURIComponent(encodedName);
|
|
265
|
+
const namespace = env[binding.env];
|
|
266
|
+
if (!namespace) return errorResponse(500, "WorldMisconfigured", `missing binding: ${binding.env}`);
|
|
267
|
+
try {
|
|
268
|
+
const result = await namespace.get(namespace.idFromName(name))[method](...args);
|
|
269
|
+
return new Response(rpcStringify(result ?? null), {
|
|
270
|
+
status: 200,
|
|
271
|
+
headers: { "content-type": "application/json" }
|
|
272
|
+
});
|
|
273
|
+
} catch (error) {
|
|
274
|
+
const err = error;
|
|
275
|
+
return Response.json({ error: {
|
|
276
|
+
name: err?.name ?? "Error",
|
|
277
|
+
message: err?.message ?? String(error),
|
|
278
|
+
status: typeof err?.status === "number" ? err.status : void 0
|
|
279
|
+
} }, { status: 500 });
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
//#endregion
|
|
284
|
+
//#region src/vendor/shared/util.ts
|
|
285
|
+
function compact(obj) {
|
|
286
|
+
const value = {};
|
|
287
|
+
for (const key in obj) if (obj[key] !== null) value[key] = obj[key];
|
|
288
|
+
else value[key] = void 0;
|
|
289
|
+
return value;
|
|
290
|
+
}
|
|
291
|
+
//#endregion
|
|
292
|
+
//#region src/worker/durable-objects/IndexDO.ts
|
|
293
|
+
function sameOwner(left, right) {
|
|
294
|
+
return left.runId === right.runId && left.hookId === right.hookId;
|
|
295
|
+
}
|
|
296
|
+
function ownerFromHook(raw) {
|
|
297
|
+
const hook = parse(raw);
|
|
298
|
+
return {
|
|
299
|
+
runId: hook.runId,
|
|
300
|
+
hookId: hook.hookId
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* Global index cell, replacing the Workers KV namespace world-cloudflare
|
|
305
|
+
* uses for run listing and hook token/id lookup.
|
|
306
|
+
*
|
|
307
|
+
* A single cell (named `index`) serializes every read and write. Hook-token
|
|
308
|
+
* reservation, publication, and release use storage transactions so
|
|
309
|
+
* concurrent runs cannot both acquire the same token.
|
|
310
|
+
*
|
|
311
|
+
* The RPC surface matches the storage-layer KVNamespace interface bit-for-bit
|
|
312
|
+
* (including MockKVNamespace's pagination contract: cursor = last returned
|
|
313
|
+
* key, `list_complete` reflects whether further keys exist), so the vendored
|
|
314
|
+
* storage.ts works against it unchanged.
|
|
315
|
+
*/
|
|
316
|
+
var IndexDO = class extends DurableObject {
|
|
317
|
+
async get(key) {
|
|
318
|
+
return await this.ctx.storage.get(key) ?? null;
|
|
319
|
+
}
|
|
320
|
+
async put(key, value) {
|
|
321
|
+
await this.ctx.storage.put(key, value);
|
|
322
|
+
}
|
|
323
|
+
async delete(key) {
|
|
324
|
+
await this.ctx.storage.delete(key);
|
|
325
|
+
}
|
|
326
|
+
async list(options) {
|
|
327
|
+
const prefix = options?.prefix ?? "";
|
|
328
|
+
const limit = options?.limit ?? 1e3;
|
|
329
|
+
const entries = await this.ctx.storage.list({
|
|
330
|
+
prefix,
|
|
331
|
+
...options?.reverse ? {
|
|
332
|
+
reverse: true,
|
|
333
|
+
end: options.cursor
|
|
334
|
+
} : { startAfter: options?.cursor },
|
|
335
|
+
limit: limit + 1
|
|
336
|
+
});
|
|
337
|
+
const names = Array.from(entries.keys());
|
|
338
|
+
const page = names.slice(0, limit);
|
|
339
|
+
const listComplete = names.length <= limit;
|
|
340
|
+
return {
|
|
341
|
+
keys: page.map((name) => ({ name })),
|
|
342
|
+
list_complete: listComplete,
|
|
343
|
+
cursor: listComplete ? void 0 : page.at(-1)
|
|
344
|
+
};
|
|
345
|
+
}
|
|
346
|
+
async reserveHookToken(token, owner) {
|
|
347
|
+
return this.ctx.storage.transaction(async (txn) => {
|
|
348
|
+
const indexedHook = await txn.get(`hook:${token}`);
|
|
349
|
+
if (indexedHook !== void 0) return {
|
|
350
|
+
claimed: false,
|
|
351
|
+
holder: ownerFromHook(indexedHook)
|
|
352
|
+
};
|
|
353
|
+
const claimKey = `hookclaim:${token}`;
|
|
354
|
+
const claim = await txn.get(claimKey);
|
|
355
|
+
if (claim !== void 0 && !sameOwner(claim.owner, owner)) return {
|
|
356
|
+
claimed: false,
|
|
357
|
+
holder: claim.owner
|
|
358
|
+
};
|
|
359
|
+
await txn.put(claimKey, {
|
|
360
|
+
owner,
|
|
361
|
+
reservedAt: Date.now()
|
|
362
|
+
});
|
|
363
|
+
return { claimed: true };
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
async finalizeHookIndexes(token, hookId, serializedHook, owner) {
|
|
367
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
368
|
+
const indexedHook = await txn.get(`hook:${token}`);
|
|
369
|
+
if (indexedHook !== void 0 && !sameOwner(ownerFromHook(indexedHook), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
|
|
370
|
+
const claim = await txn.get(`hookclaim:${token}`);
|
|
371
|
+
if (claim !== void 0 && !sameOwner(claim.owner, owner)) throw new Error(`Hook token ${token} is reserved by another hook`);
|
|
372
|
+
await txn.put(`hook:${token}`, serializedHook);
|
|
373
|
+
await txn.put(`hookid:${hookId}`, serializedHook);
|
|
374
|
+
if (claim !== void 0) await txn.delete(`hookclaim:${token}`);
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
async releaseHookToken(token, owner) {
|
|
378
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
379
|
+
const key = `hookclaim:${token}`;
|
|
380
|
+
const claim = await txn.get(key);
|
|
381
|
+
if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(key);
|
|
382
|
+
});
|
|
383
|
+
}
|
|
384
|
+
async deleteHookIndexes(token, hookId, owner) {
|
|
385
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
386
|
+
const tokenKey = `hook:${token}`;
|
|
387
|
+
const idKey = `hookid:${hookId}`;
|
|
388
|
+
const claimKey = `hookclaim:${token}`;
|
|
389
|
+
const [byToken, byId, claim] = await Promise.all([
|
|
390
|
+
txn.get(tokenKey),
|
|
391
|
+
txn.get(idKey),
|
|
392
|
+
txn.get(claimKey)
|
|
393
|
+
]);
|
|
394
|
+
if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) await txn.delete(tokenKey);
|
|
395
|
+
if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) await txn.delete(idKey);
|
|
396
|
+
if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(claimKey);
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
};
|
|
400
|
+
//#endregion
|
|
401
|
+
//#region src/worker/durable-objects/QueueDO.ts
|
|
402
|
+
const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
|
|
403
|
+
404,
|
|
404
|
+
409,
|
|
405
|
+
410,
|
|
406
|
+
422
|
|
407
|
+
]);
|
|
408
|
+
const DEFAULT_MAX_ATTEMPTS = 5;
|
|
409
|
+
const DEFAULT_MAX_INFLIGHT = 5;
|
|
410
|
+
const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
|
|
411
|
+
/** Grace period past the delivery timeout before a claim counts as lost. */
|
|
412
|
+
const INFLIGHT_GRACE_MS = 3e4;
|
|
413
|
+
/** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
|
|
414
|
+
const MIN_ALARM_DELAY_MS = 1;
|
|
415
|
+
function pad(ms) {
|
|
416
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
417
|
+
}
|
|
418
|
+
function dueKey(atMs, messageId) {
|
|
419
|
+
return `due:${pad(atMs)}:${messageId}`;
|
|
420
|
+
}
|
|
421
|
+
function backoffSeconds(attempt) {
|
|
422
|
+
return Math.min(60, 2 ** attempt);
|
|
423
|
+
}
|
|
424
|
+
var QueueDO = class extends DurableObject {
|
|
425
|
+
#now() {
|
|
426
|
+
const clock = this.env?.clock;
|
|
427
|
+
return typeof clock === "function" ? clock() : Date.now();
|
|
428
|
+
}
|
|
429
|
+
#fetch() {
|
|
430
|
+
const impl = this.env?.fetch;
|
|
431
|
+
return typeof impl === "function" ? impl : fetch;
|
|
432
|
+
}
|
|
433
|
+
#intVar(name, fallback) {
|
|
434
|
+
const raw = this.env?.[name];
|
|
435
|
+
const parsed = raw ? Number.parseInt(raw, 10) : NaN;
|
|
436
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
437
|
+
}
|
|
438
|
+
async enqueue(request) {
|
|
439
|
+
const storage = this.ctx.storage;
|
|
440
|
+
const now = this.#now();
|
|
441
|
+
return await storage.transaction(async (txn) => {
|
|
442
|
+
const pinned = await txn.get("cfg");
|
|
443
|
+
if (!pinned) await txn.put("cfg", {
|
|
444
|
+
queueShards: request.config.queueShards,
|
|
445
|
+
pinnedAt: now
|
|
446
|
+
});
|
|
447
|
+
else if (pinned.queueShards !== request.config.queueShards) return {
|
|
448
|
+
ok: false,
|
|
449
|
+
code: "CONFIG_MISMATCH",
|
|
450
|
+
message: `queue cell shard-count mismatch: pinned ${pinned.queueShards}, got ${request.config.queueShards} — changing queueShards requires draining the fleet`
|
|
451
|
+
};
|
|
452
|
+
if (request.idempotencyKey) {
|
|
453
|
+
const existing = await txn.get(`key:${request.idempotencyKey}`);
|
|
454
|
+
if (existing) {
|
|
455
|
+
if (await txn.get(`msg:${existing}`)) {
|
|
456
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
457
|
+
return {
|
|
458
|
+
ok: true,
|
|
459
|
+
messageId: existing,
|
|
460
|
+
deduped: true
|
|
461
|
+
};
|
|
462
|
+
}
|
|
463
|
+
await txn.delete(`key:${request.idempotencyKey}`);
|
|
464
|
+
}
|
|
465
|
+
await txn.put(`key:${request.idempotencyKey}`, request.messageId);
|
|
466
|
+
}
|
|
467
|
+
const row = {
|
|
468
|
+
messageId: request.messageId,
|
|
469
|
+
queueName: request.queueName,
|
|
470
|
+
pathname: request.pathname,
|
|
471
|
+
body: request.body,
|
|
472
|
+
targetBaseUrl: request.config.targetBaseUrl,
|
|
473
|
+
idempotencyKey: request.idempotencyKey,
|
|
474
|
+
attempt: 0,
|
|
475
|
+
enqueuedAt: now
|
|
476
|
+
};
|
|
477
|
+
const dueAt = now + Math.max(0, request.delaySeconds ?? 0) * 1e3;
|
|
478
|
+
await txn.put(`msg:${row.messageId}`, row);
|
|
479
|
+
await txn.put(dueKey(dueAt, row.messageId), row.messageId);
|
|
480
|
+
await this.#armAlarmAtMost(txn, dueAt, now);
|
|
481
|
+
return {
|
|
482
|
+
ok: true,
|
|
483
|
+
messageId: row.messageId,
|
|
484
|
+
deduped: false
|
|
485
|
+
};
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
async alarm() {
|
|
489
|
+
let claimed = [];
|
|
490
|
+
let failure = null;
|
|
491
|
+
const attempt = async () => {
|
|
492
|
+
try {
|
|
493
|
+
claimed = await this.#claimPhase();
|
|
494
|
+
} catch (error) {
|
|
495
|
+
failure = error;
|
|
496
|
+
}
|
|
497
|
+
};
|
|
498
|
+
if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
|
|
499
|
+
else await attempt();
|
|
500
|
+
if (failure !== null) throw failure;
|
|
501
|
+
for (const row of claimed) this.ctx.waitUntil(this.#deliver(row));
|
|
502
|
+
}
|
|
503
|
+
/**
|
|
504
|
+
* Storage-only: recover lost inflight claims, claim due messages up to the
|
|
505
|
+
* inflight cap, and re-arm the alarm. Never throws for data conditions.
|
|
506
|
+
*/
|
|
507
|
+
async #claimPhase() {
|
|
508
|
+
const storage = this.ctx.storage;
|
|
509
|
+
const now = this.#now();
|
|
510
|
+
const maxInflight = this.#intVar("QUEUE_MAX_INFLIGHT", DEFAULT_MAX_INFLIGHT);
|
|
511
|
+
return await storage.transaction(async (txn) => {
|
|
512
|
+
const inflight = await txn.list({ prefix: "inflight:" });
|
|
513
|
+
let inflightCount = 0;
|
|
514
|
+
for (const [key, deadline] of inflight) if (deadline <= now) {
|
|
515
|
+
const messageId = key.slice(9);
|
|
516
|
+
await txn.delete(key);
|
|
517
|
+
if (await txn.get(`msg:${messageId}`)) await txn.put(dueKey(now, messageId), messageId);
|
|
518
|
+
} else inflightCount++;
|
|
519
|
+
const rows = [];
|
|
520
|
+
if (inflightCount < maxInflight) {
|
|
521
|
+
const due = await txn.list({
|
|
522
|
+
prefix: "due:",
|
|
523
|
+
end: `due:${pad(now + 1)}`,
|
|
524
|
+
limit: maxInflight - inflightCount
|
|
525
|
+
});
|
|
526
|
+
const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
527
|
+
for (const [key, messageId] of due) {
|
|
528
|
+
await txn.delete(key);
|
|
529
|
+
const row = await txn.get(`msg:${messageId}`);
|
|
530
|
+
if (!row) continue;
|
|
531
|
+
await txn.put(`inflight:${messageId}`, now + timeoutMs + INFLIGHT_GRACE_MS);
|
|
532
|
+
rows.push(row);
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
536
|
+
return rows;
|
|
537
|
+
});
|
|
538
|
+
}
|
|
539
|
+
async #deliver(row) {
|
|
540
|
+
const storage = this.ctx.storage;
|
|
541
|
+
const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
542
|
+
const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
|
|
543
|
+
const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
|
|
544
|
+
let response = null;
|
|
545
|
+
let transportError = null;
|
|
546
|
+
try {
|
|
547
|
+
response = await this.#fetch()(url, {
|
|
548
|
+
method: "POST",
|
|
549
|
+
headers: {
|
|
550
|
+
"content-type": "application/json",
|
|
551
|
+
"x-vqs-queue-name": row.queueName,
|
|
552
|
+
"x-vqs-message-id": row.messageId,
|
|
553
|
+
"x-vqs-message-attempt": String(row.attempt + 1),
|
|
554
|
+
...callbackSecret ? { "x-workflow-callback-secret": callbackSecret } : {}
|
|
555
|
+
},
|
|
556
|
+
body: row.body,
|
|
557
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
558
|
+
});
|
|
559
|
+
} catch (error) {
|
|
560
|
+
transportError = error;
|
|
561
|
+
}
|
|
562
|
+
const now = this.#now();
|
|
563
|
+
if (response?.ok) await this.#ack(row);
|
|
564
|
+
else if (response && response.status === 503) {
|
|
565
|
+
let timeoutSeconds;
|
|
566
|
+
try {
|
|
567
|
+
timeoutSeconds = (await response.json())?.timeoutSeconds;
|
|
568
|
+
} catch {
|
|
569
|
+
timeoutSeconds = void 0;
|
|
570
|
+
}
|
|
571
|
+
if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
|
|
572
|
+
await txn.delete(`inflight:${row.messageId}`);
|
|
573
|
+
await txn.put(dueKey(now + timeoutSeconds * 1e3, row.messageId), row.messageId);
|
|
574
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
575
|
+
});
|
|
576
|
+
else await this.#retry(row, "HTTP 503");
|
|
577
|
+
} else if (response && PERMANENT_STATUSES.has(response.status)) await this.#ack(row);
|
|
578
|
+
else {
|
|
579
|
+
const reason = response ? `HTTP ${response.status}` : `transport error: ${String(transportError)}`;
|
|
580
|
+
await this.#retry(row, reason);
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
async #ack(row) {
|
|
584
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
585
|
+
await txn.delete(`msg:${row.messageId}`);
|
|
586
|
+
await txn.delete(`inflight:${row.messageId}`);
|
|
587
|
+
if (row.idempotencyKey) {
|
|
588
|
+
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
589
|
+
}
|
|
590
|
+
await this.#scheduleNextAlarm(txn, this.#now());
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
async #retry(row, reason) {
|
|
594
|
+
const storage = this.ctx.storage;
|
|
595
|
+
const now = this.#now();
|
|
596
|
+
const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
597
|
+
const attempt = row.attempt + 1;
|
|
598
|
+
await storage.transaction(async (txn) => {
|
|
599
|
+
await txn.delete(`inflight:${row.messageId}`);
|
|
600
|
+
if (attempt >= maxAttempts) {
|
|
601
|
+
const dead = {
|
|
602
|
+
...row,
|
|
603
|
+
attempt,
|
|
604
|
+
lastError: reason,
|
|
605
|
+
failedAt: now
|
|
606
|
+
};
|
|
607
|
+
await txn.put(`dlq:${pad(now)}:${row.messageId}`, dead);
|
|
608
|
+
await txn.delete(`msg:${row.messageId}`);
|
|
609
|
+
if (row.idempotencyKey) {
|
|
610
|
+
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
611
|
+
}
|
|
612
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
const updated = {
|
|
616
|
+
...row,
|
|
617
|
+
attempt,
|
|
618
|
+
lastError: reason
|
|
619
|
+
};
|
|
620
|
+
await txn.put(`msg:${row.messageId}`, updated);
|
|
621
|
+
await txn.put(dueKey(now + backoffSeconds(attempt) * 1e3, row.messageId), row.messageId);
|
|
622
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
623
|
+
});
|
|
624
|
+
}
|
|
625
|
+
/** Arm the alarm no later than `atMs` (durable backoff never throws). */
|
|
626
|
+
async #armAlarmAtMost(storage, atMs, now) {
|
|
627
|
+
const current = await storage.getAlarm();
|
|
628
|
+
if (current !== null && current <= now) {
|
|
629
|
+
await storage.setAlarm(now + MIN_ALARM_DELAY_MS);
|
|
630
|
+
return;
|
|
631
|
+
}
|
|
632
|
+
const target = atMs <= now ? now + MIN_ALARM_DELAY_MS : atMs;
|
|
633
|
+
if (current === null || target < current) await storage.setAlarm(target);
|
|
634
|
+
}
|
|
635
|
+
/** Recompute the alarm from the earliest due entry / inflight deadline. */
|
|
636
|
+
async #rearm() {
|
|
637
|
+
await this.#scheduleNextAlarm(this.ctx.storage, this.#now());
|
|
638
|
+
}
|
|
639
|
+
/** Keep the alarm index consistent with message state in the same transaction. */
|
|
640
|
+
async #scheduleNextAlarm(storage, now) {
|
|
641
|
+
let next = null;
|
|
642
|
+
const due = await storage.list({
|
|
643
|
+
prefix: "due:",
|
|
644
|
+
limit: 1
|
|
645
|
+
});
|
|
646
|
+
for (const key of due.keys()) next = Number.parseInt(key.slice(4, 17), 10);
|
|
647
|
+
const inflight = await storage.list({ prefix: "inflight:" });
|
|
648
|
+
for (const deadline of inflight.values()) if (next === null || deadline < next) next = deadline;
|
|
649
|
+
if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
|
|
650
|
+
else await storage.deleteAlarm();
|
|
651
|
+
}
|
|
652
|
+
async stats() {
|
|
653
|
+
const storage = this.ctx.storage;
|
|
654
|
+
const [due, inflight, dlq, alarmAt] = await Promise.all([
|
|
655
|
+
storage.list({ prefix: "due:" }),
|
|
656
|
+
storage.list({ prefix: "inflight:" }),
|
|
657
|
+
storage.list({ prefix: "dlq:" }),
|
|
658
|
+
storage.getAlarm()
|
|
659
|
+
]);
|
|
660
|
+
return {
|
|
661
|
+
pending: due.size,
|
|
662
|
+
inflight: inflight.size,
|
|
663
|
+
deadLetters: dlq.size,
|
|
664
|
+
alarmAt
|
|
665
|
+
};
|
|
666
|
+
}
|
|
667
|
+
async listDeadLetters(params) {
|
|
668
|
+
const limit = params?.limit ?? 100;
|
|
669
|
+
const entries = await this.ctx.storage.list({
|
|
670
|
+
prefix: "dlq:",
|
|
671
|
+
startAfter: params?.cursor,
|
|
672
|
+
limit: limit + 1
|
|
673
|
+
});
|
|
674
|
+
const keys = Array.from(entries.keys());
|
|
675
|
+
const page = keys.slice(0, limit);
|
|
676
|
+
const hasMore = keys.length > limit;
|
|
677
|
+
return {
|
|
678
|
+
data: page.map((k) => entries.get(k)),
|
|
679
|
+
cursor: hasMore ? page.at(-1) : null,
|
|
680
|
+
hasMore
|
|
681
|
+
};
|
|
682
|
+
}
|
|
683
|
+
/** Move a dead letter back to the live queue (attempt count reset). */
|
|
684
|
+
async redriveDeadLetter(messageId) {
|
|
685
|
+
const storage = this.ctx.storage;
|
|
686
|
+
const now = this.#now();
|
|
687
|
+
return await storage.transaction(async (txn) => {
|
|
688
|
+
const dlq = await txn.list({ prefix: "dlq:" });
|
|
689
|
+
for (const [key, dead] of dlq) {
|
|
690
|
+
if (dead.messageId !== messageId) continue;
|
|
691
|
+
const { failedAt: _failedAt, ...rest } = dead;
|
|
692
|
+
const row = {
|
|
693
|
+
...rest,
|
|
694
|
+
attempt: 0,
|
|
695
|
+
lastError: void 0
|
|
696
|
+
};
|
|
697
|
+
if (row.idempotencyKey) {
|
|
698
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
699
|
+
const holder = await txn.get(dedupKey);
|
|
700
|
+
if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
|
|
701
|
+
if (holder && holder !== row.messageId) await txn.delete(dedupKey);
|
|
702
|
+
await txn.put(dedupKey, row.messageId);
|
|
703
|
+
}
|
|
704
|
+
await txn.put(`msg:${row.messageId}`, row);
|
|
705
|
+
await txn.put(dueKey(now, row.messageId), row.messageId);
|
|
706
|
+
await txn.delete(key);
|
|
707
|
+
await this.#armAlarmAtMost(txn, now, now);
|
|
708
|
+
return { ok: true };
|
|
709
|
+
}
|
|
710
|
+
return { ok: false };
|
|
711
|
+
});
|
|
712
|
+
}
|
|
713
|
+
async purgeDeadLetters() {
|
|
714
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
715
|
+
const dlq = await txn.list({ prefix: "dlq:" });
|
|
716
|
+
for (const key of dlq.keys()) await txn.delete(key);
|
|
717
|
+
return { purged: dlq.size };
|
|
718
|
+
});
|
|
719
|
+
}
|
|
720
|
+
/**
|
|
721
|
+
* Recovery hatch for celld's six-failure alarm abandonment: derive the due
|
|
722
|
+
* time again and arm it.
|
|
723
|
+
*/
|
|
724
|
+
async rearmAlarm() {
|
|
725
|
+
await this.#rearm();
|
|
726
|
+
return { alarmAt: await this.ctx.storage.getAlarm() };
|
|
727
|
+
}
|
|
728
|
+
};
|
|
729
|
+
//#endregion
|
|
730
|
+
//#region src/worker/durable-objects/StreamDO.ts
|
|
731
|
+
const META_KEY = "meta";
|
|
732
|
+
const CHUNK_KEY_PREFIX = "chunk:";
|
|
733
|
+
/** Registry keys used when this DO instance acts as a per-run stream index. */
|
|
734
|
+
const STREAM_REGISTRY_PREFIX = "stream:";
|
|
735
|
+
/**
|
|
736
|
+
* Zero-pad chunk indexes so `storage.list({ prefix })` returns chunks in
|
|
737
|
+
* write order. 12 digits comfortably exceeds any realistic chunk count.
|
|
738
|
+
*/
|
|
739
|
+
function chunkKey(index) {
|
|
740
|
+
return `${CHUNK_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
|
|
741
|
+
}
|
|
742
|
+
/**
|
|
743
|
+
* Durable Object backing workflow streams.
|
|
744
|
+
*
|
|
745
|
+
* Two roles, selected by the DO name:
|
|
746
|
+
* - `stream:<name>`: chunk storage for a single stream. A monotonic
|
|
747
|
+
* per-stream counter (maintained transactionally inside the DO) assigns
|
|
748
|
+
* each chunk its index; one chunk per storage key.
|
|
749
|
+
* - `run-streams:<runId>`: registry of stream names owned by a run,
|
|
750
|
+
* powering `listStreamsByRunId`.
|
|
751
|
+
*/
|
|
752
|
+
var StreamDO = class extends DurableObject {
|
|
753
|
+
async getMeta() {
|
|
754
|
+
return await this.ctx.storage.get(META_KEY) ?? {
|
|
755
|
+
count: 0,
|
|
756
|
+
closed: false
|
|
757
|
+
};
|
|
758
|
+
}
|
|
759
|
+
/**
|
|
760
|
+
* Append a chunk. The index is allocated inside the transaction, so
|
|
761
|
+
* concurrent writers can never collide or skip.
|
|
762
|
+
*/
|
|
763
|
+
async writeChunk(data) {
|
|
764
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
765
|
+
const meta = await txn.get(META_KEY) ?? {
|
|
766
|
+
count: 0,
|
|
767
|
+
closed: false
|
|
768
|
+
};
|
|
769
|
+
if (meta.closed) throw new Error("Cannot write to a closed stream");
|
|
770
|
+
const index = meta.count;
|
|
771
|
+
await txn.put(chunkKey(index), data);
|
|
772
|
+
await txn.put(META_KEY, {
|
|
773
|
+
count: index + 1,
|
|
774
|
+
closed: false
|
|
775
|
+
});
|
|
776
|
+
return index;
|
|
777
|
+
});
|
|
778
|
+
}
|
|
779
|
+
/**
|
|
780
|
+
* Close the stream (idempotent). Readers observe `done: true` once all
|
|
781
|
+
* previously written chunks have been consumed.
|
|
782
|
+
*/
|
|
783
|
+
async closeStream() {
|
|
784
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
785
|
+
const meta = await txn.get(META_KEY) ?? {
|
|
786
|
+
count: 0,
|
|
787
|
+
closed: false
|
|
788
|
+
};
|
|
789
|
+
await txn.put(META_KEY, {
|
|
790
|
+
...meta,
|
|
791
|
+
closed: true
|
|
792
|
+
});
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
/**
|
|
796
|
+
* Read up to `limit` chunks starting at `startIndex` (0-based, inclusive).
|
|
797
|
+
*/
|
|
798
|
+
async getChunks(params) {
|
|
799
|
+
const meta = await this.getMeta();
|
|
800
|
+
const start = Math.max(0, params.startIndex);
|
|
801
|
+
const entries = await this.ctx.storage.list({
|
|
802
|
+
prefix: CHUNK_KEY_PREFIX,
|
|
803
|
+
start: chunkKey(start),
|
|
804
|
+
limit: params.limit
|
|
805
|
+
});
|
|
806
|
+
return {
|
|
807
|
+
chunks: Array.from(entries.values()),
|
|
808
|
+
done: meta.closed,
|
|
809
|
+
tailIndex: meta.count - 1
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
/**
|
|
813
|
+
* Lightweight stream metadata: last chunk index and completion flag.
|
|
814
|
+
*/
|
|
815
|
+
async getInfo() {
|
|
816
|
+
const meta = await this.getMeta();
|
|
817
|
+
return {
|
|
818
|
+
tailIndex: meta.count - 1,
|
|
819
|
+
done: meta.closed
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
/**
|
|
823
|
+
* Register a stream name against this per-run registry instance.
|
|
824
|
+
*/
|
|
825
|
+
async registerStream(name) {
|
|
826
|
+
await this.ctx.storage.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
|
|
827
|
+
}
|
|
828
|
+
/**
|
|
829
|
+
* List stream names registered against this per-run registry instance.
|
|
830
|
+
*/
|
|
831
|
+
async listStreams() {
|
|
832
|
+
const entries = await this.ctx.storage.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
833
|
+
return Array.from(entries.keys()).map((key) => key.slice(7));
|
|
834
|
+
}
|
|
835
|
+
};
|
|
836
|
+
//#endregion
|
|
837
|
+
//#region src/apply-event.ts
|
|
838
|
+
/** Storage key for the run entity. */
|
|
839
|
+
const RUN_KEY = "run";
|
|
840
|
+
/** Key prefixes for per-entity storage. */
|
|
841
|
+
const EVENT_KEY_PREFIX = "event:";
|
|
842
|
+
const STEP_KEY_PREFIX = "step:";
|
|
843
|
+
const HOOK_KEY_PREFIX = "hook:";
|
|
844
|
+
const STEP_CREATED_KEY_PREFIX = "stepcreated:";
|
|
845
|
+
const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
|
|
846
|
+
/**
|
|
847
|
+
* Marker recording that a hook_created event was committed for a hookId.
|
|
848
|
+
* Mirrors postgres' `workflow_events_entity_creation_unique` partial index:
|
|
849
|
+
* it outlives hook disposal so a replayed hook_created after dispose is
|
|
850
|
+
* rejected as a duplicate instead of resurrecting the hook.
|
|
851
|
+
*/
|
|
852
|
+
const HOOK_EVENT_MARKER_PREFIX = "hookevent:";
|
|
853
|
+
function failure(code, message, extra) {
|
|
854
|
+
return {
|
|
855
|
+
ok: false,
|
|
856
|
+
code,
|
|
857
|
+
message,
|
|
858
|
+
...extra
|
|
859
|
+
};
|
|
860
|
+
}
|
|
861
|
+
function readStringProp(value, key) {
|
|
862
|
+
if (value !== null && typeof value === "object") {
|
|
863
|
+
const prop = value[key];
|
|
864
|
+
if (typeof prop === "string") return prop;
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
/**
|
|
868
|
+
* Map a failure eventData.error (which may be a string, an Error-shaped
|
|
869
|
+
* object, or anything else) to the structured error stored on entities.
|
|
870
|
+
* Matches upstream: string errors keep their text, errorCode is preserved.
|
|
871
|
+
*/
|
|
872
|
+
function toStructuredError(error, opts) {
|
|
873
|
+
return {
|
|
874
|
+
message: typeof error === "string" ? error : readStringProp(error, "message") ?? "Unknown error",
|
|
875
|
+
stack: opts?.stack ?? readStringProp(error, "stack"),
|
|
876
|
+
code: opts?.code ?? readStringProp(error, "code")
|
|
877
|
+
};
|
|
878
|
+
}
|
|
879
|
+
/**
|
|
880
|
+
* Shared cursor pagination over a key prefix. Fetches `limit + 1` entries to
|
|
881
|
+
* detect `hasMore` without ever skipping the boundary entry (the cursor is
|
|
882
|
+
* derived from the last *returned* item, not the peeked one).
|
|
883
|
+
*/
|
|
884
|
+
async function listByPrefix(store, prefix, params, getId) {
|
|
885
|
+
const { limit, cursor, sortOrder = "asc" } = params;
|
|
886
|
+
const options = sortOrder === "desc" ? {
|
|
887
|
+
prefix,
|
|
888
|
+
reverse: true,
|
|
889
|
+
limit: limit + 1,
|
|
890
|
+
end: cursor ? `${prefix}${cursor}` : void 0
|
|
891
|
+
} : {
|
|
892
|
+
prefix,
|
|
893
|
+
limit: limit + 1,
|
|
894
|
+
startAfter: cursor ? `${prefix}${cursor}` : void 0
|
|
895
|
+
};
|
|
896
|
+
const entries = await store.list(options);
|
|
897
|
+
const values = Array.from(entries.values());
|
|
898
|
+
const hasMore = values.length > limit;
|
|
899
|
+
const data = values.slice(0, limit);
|
|
900
|
+
const last = data.at(-1);
|
|
901
|
+
return {
|
|
902
|
+
data,
|
|
903
|
+
cursor: hasMore && last !== void 0 ? getId(last) : null,
|
|
904
|
+
hasMore
|
|
905
|
+
};
|
|
906
|
+
}
|
|
907
|
+
function creationIndexKey(prefix, createdAt, id) {
|
|
908
|
+
return `${prefix}${createdAt.toISOString()}:${id}`;
|
|
909
|
+
}
|
|
910
|
+
/**
|
|
911
|
+
* Cursor pagination over an explicit creation-time index. Entity ids are
|
|
912
|
+
* values rather than key suffixes, so caller-supplied ids cannot disturb the
|
|
913
|
+
* requested ordering.
|
|
914
|
+
*/
|
|
915
|
+
async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
916
|
+
const { limit, cursor, sortOrder = "asc" } = params;
|
|
917
|
+
const entries = await store.list(sortOrder === "desc" ? {
|
|
918
|
+
prefix: indexPrefix,
|
|
919
|
+
reverse: true,
|
|
920
|
+
limit: limit + 1,
|
|
921
|
+
end: cursor ? `${indexPrefix}${cursor}` : void 0
|
|
922
|
+
} : {
|
|
923
|
+
prefix: indexPrefix,
|
|
924
|
+
limit: limit + 1,
|
|
925
|
+
startAfter: cursor ? `${indexPrefix}${cursor}` : void 0
|
|
926
|
+
});
|
|
927
|
+
const page = Array.from(entries.entries()).slice(0, limit);
|
|
928
|
+
const data = [];
|
|
929
|
+
for (const [, id] of page) {
|
|
930
|
+
const item = await store.get(`${entityPrefix}${id}`);
|
|
931
|
+
if (item !== void 0) data.push(item);
|
|
932
|
+
}
|
|
933
|
+
const hasMore = entries.size > limit;
|
|
934
|
+
const lastKey = page.at(-1)?.[0];
|
|
935
|
+
return {
|
|
936
|
+
data,
|
|
937
|
+
cursor: hasMore && lastKey ? lastKey.slice(indexPrefix.length) : null,
|
|
938
|
+
hasMore
|
|
939
|
+
};
|
|
940
|
+
}
|
|
941
|
+
/** Delete all hook entities for the run, returning the released tokens. */
|
|
942
|
+
async function releaseAllHooks(store) {
|
|
943
|
+
const hooks = await store.list({ prefix: HOOK_KEY_PREFIX });
|
|
944
|
+
const released = [];
|
|
945
|
+
for (const hook of hooks.values()) {
|
|
946
|
+
await store.delete(`${HOOK_KEY_PREFIX}${hook.hookId}`);
|
|
947
|
+
await store.delete(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
|
|
948
|
+
released.push({
|
|
949
|
+
hookId: hook.hookId,
|
|
950
|
+
token: hook.token
|
|
951
|
+
});
|
|
952
|
+
}
|
|
953
|
+
return released;
|
|
954
|
+
}
|
|
955
|
+
async function applyEvent(store, ctx) {
|
|
956
|
+
const { runId, data, now } = ctx;
|
|
957
|
+
const specVersion = data.specVersion ?? SPEC_VERSION_CURRENT;
|
|
958
|
+
/** Build and validate the stored event BEFORE anything is persisted. */
|
|
959
|
+
const buildEvent = (record) => {
|
|
960
|
+
const withMeta = {
|
|
961
|
+
...record,
|
|
962
|
+
runId,
|
|
963
|
+
eventId: ctx.nextEventId(),
|
|
964
|
+
createdAt: now,
|
|
965
|
+
specVersion
|
|
966
|
+
};
|
|
967
|
+
if (record.eventType === "run_started") delete withMeta.eventData;
|
|
968
|
+
return EventSchema.parse(compact(withMeta));
|
|
969
|
+
};
|
|
970
|
+
const putEvent = async (event) => {
|
|
971
|
+
await store.put(`${EVENT_KEY_PREFIX}${event.eventId}`, event);
|
|
972
|
+
};
|
|
973
|
+
let currentRun;
|
|
974
|
+
const skipRunValidation = data.eventType === "run_created" || data.eventType === "step_completed" || data.eventType === "step_retrying";
|
|
975
|
+
if (!skipRunValidation) currentRun = await store.get(RUN_KEY);
|
|
976
|
+
let bootstrapped;
|
|
977
|
+
if (data.eventType === "run_started" && !currentRun && data.eventData) {
|
|
978
|
+
const { deploymentId, workflowName, input, executionContext } = data.eventData;
|
|
979
|
+
if (deploymentId && workflowName && input !== void 0) {
|
|
980
|
+
const createdRun = WorkflowRunSchema.parse(compact({
|
|
981
|
+
runId,
|
|
982
|
+
deploymentId,
|
|
983
|
+
workflowName,
|
|
984
|
+
specVersion,
|
|
985
|
+
executionContext,
|
|
986
|
+
input,
|
|
987
|
+
status: "pending",
|
|
988
|
+
output: void 0,
|
|
989
|
+
error: void 0,
|
|
990
|
+
startedAt: void 0,
|
|
991
|
+
completedAt: void 0,
|
|
992
|
+
createdAt: now,
|
|
993
|
+
updatedAt: now
|
|
994
|
+
}));
|
|
995
|
+
const runCreatedEvent = buildEvent({
|
|
996
|
+
eventType: "run_created",
|
|
997
|
+
eventData: {
|
|
998
|
+
deploymentId,
|
|
999
|
+
workflowName,
|
|
1000
|
+
input,
|
|
1001
|
+
executionContext
|
|
1002
|
+
}
|
|
1003
|
+
});
|
|
1004
|
+
await store.put(RUN_KEY, createdRun);
|
|
1005
|
+
await putEvent(runCreatedEvent);
|
|
1006
|
+
currentRun = createdRun;
|
|
1007
|
+
bootstrapped = {
|
|
1008
|
+
workflowName,
|
|
1009
|
+
createdAt: now
|
|
1010
|
+
};
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
if (currentRun) {
|
|
1014
|
+
if (requiresNewerWorld(currentRun.specVersion)) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" requires a newer world (run specVersion ${currentRun.specVersion}, world ${SPEC_VERSION_CURRENT})`, { runSpecVersion: currentRun.specVersion });
|
|
1015
|
+
if (isLegacySpecVersion(currentRun.specVersion)) return failure("LEGACY_RUN_NOT_SUPPORTED", `Run "${runId}" was created with a legacy spec version (${currentRun.specVersion ?? "undefined"}) not supported by this world`);
|
|
1016
|
+
}
|
|
1017
|
+
if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) {
|
|
1018
|
+
if (data.eventType === "run_cancelled" && currentRun.status === "cancelled") {
|
|
1019
|
+
const event = buildEvent({ ...data });
|
|
1020
|
+
await putEvent(event);
|
|
1021
|
+
return {
|
|
1022
|
+
ok: true,
|
|
1023
|
+
event,
|
|
1024
|
+
run: currentRun,
|
|
1025
|
+
releasedHooks: []
|
|
1026
|
+
};
|
|
1027
|
+
}
|
|
1028
|
+
if (data.eventType === "run_started") return failure("RUN_EXPIRED", `Workflow run "${runId}" is already in terminal state "${currentRun.status}"`);
|
|
1029
|
+
if (data.eventType === "run_completed" || data.eventType === "run_failed" || data.eventType === "run_cancelled") return failure("ENTITY_CONFLICT", `Cannot transition run from terminal state "${currentRun.status}"`);
|
|
1030
|
+
if (data.eventType === "step_created" || data.eventType === "hook_created" || data.eventType === "wait_created") return failure("ENTITY_CONFLICT", `Cannot create new entities on run in terminal state "${currentRun.status}"`);
|
|
1031
|
+
}
|
|
1032
|
+
let validatedStep;
|
|
1033
|
+
if (data.eventType === "step_started" || data.eventType === "step_completed" || data.eventType === "step_failed" || data.eventType === "step_retrying") {
|
|
1034
|
+
validatedStep = await store.get(`${STEP_KEY_PREFIX}${data.correlationId}`);
|
|
1035
|
+
if (!validatedStep) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
1036
|
+
if (isTerminalStepStatus(validatedStep.status)) return failure("ENTITY_CONFLICT", `Cannot modify step in terminal state "${validatedStep.status}"`);
|
|
1037
|
+
if (currentRun && isTerminalWorkflowRunStatus(currentRun.status) && validatedStep.status !== "running") return failure("RUN_EXPIRED", `Cannot modify non-running step on run in terminal state "${currentRun.status}"`);
|
|
1038
|
+
}
|
|
1039
|
+
if ((data.eventType === "hook_disposed" || data.eventType === "hook_received") && data.correlationId) {
|
|
1040
|
+
if (!await store.get(`hook:${data.correlationId}`)) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
1041
|
+
}
|
|
1042
|
+
if (!skipRunValidation && data.eventType !== "run_started" && !currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1043
|
+
switch (data.eventType) {
|
|
1044
|
+
case "run_created": {
|
|
1045
|
+
const existing = await store.get(RUN_KEY);
|
|
1046
|
+
if (existing) return {
|
|
1047
|
+
ok: true,
|
|
1048
|
+
run: existing,
|
|
1049
|
+
releasedHooks: []
|
|
1050
|
+
};
|
|
1051
|
+
const run = WorkflowRunSchema.parse(compact({
|
|
1052
|
+
runId,
|
|
1053
|
+
deploymentId: data.eventData.deploymentId,
|
|
1054
|
+
workflowName: data.eventData.workflowName,
|
|
1055
|
+
specVersion,
|
|
1056
|
+
executionContext: data.eventData.executionContext,
|
|
1057
|
+
input: data.eventData.input,
|
|
1058
|
+
status: "pending",
|
|
1059
|
+
output: void 0,
|
|
1060
|
+
error: void 0,
|
|
1061
|
+
startedAt: void 0,
|
|
1062
|
+
completedAt: void 0,
|
|
1063
|
+
createdAt: now,
|
|
1064
|
+
updatedAt: now
|
|
1065
|
+
}));
|
|
1066
|
+
const event = buildEvent({ ...data });
|
|
1067
|
+
await store.put(RUN_KEY, run);
|
|
1068
|
+
await putEvent(event);
|
|
1069
|
+
return {
|
|
1070
|
+
ok: true,
|
|
1071
|
+
event,
|
|
1072
|
+
run,
|
|
1073
|
+
releasedHooks: [],
|
|
1074
|
+
runCreated: {
|
|
1075
|
+
workflowName: data.eventData.workflowName,
|
|
1076
|
+
createdAt: now
|
|
1077
|
+
}
|
|
1078
|
+
};
|
|
1079
|
+
}
|
|
1080
|
+
case "run_started": {
|
|
1081
|
+
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1082
|
+
if (currentRun.status === "running") return {
|
|
1083
|
+
ok: true,
|
|
1084
|
+
run: currentRun,
|
|
1085
|
+
releasedHooks: []
|
|
1086
|
+
};
|
|
1087
|
+
const run = WorkflowRunSchema.parse(compact({
|
|
1088
|
+
...currentRun,
|
|
1089
|
+
status: "running",
|
|
1090
|
+
output: void 0,
|
|
1091
|
+
error: void 0,
|
|
1092
|
+
completedAt: void 0,
|
|
1093
|
+
startedAt: currentRun.startedAt ?? now,
|
|
1094
|
+
updatedAt: now
|
|
1095
|
+
}));
|
|
1096
|
+
const event = buildEvent({ ...data });
|
|
1097
|
+
await store.put(RUN_KEY, run);
|
|
1098
|
+
await putEvent(event);
|
|
1099
|
+
const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
1100
|
+
limit: 1e3,
|
|
1101
|
+
sortOrder: "asc"
|
|
1102
|
+
}, (e) => e.eventId);
|
|
1103
|
+
return {
|
|
1104
|
+
ok: true,
|
|
1105
|
+
event,
|
|
1106
|
+
run,
|
|
1107
|
+
releasedHooks: [],
|
|
1108
|
+
runCreated: bootstrapped,
|
|
1109
|
+
events: allEvents.data
|
|
1110
|
+
};
|
|
1111
|
+
}
|
|
1112
|
+
case "run_completed": {
|
|
1113
|
+
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1114
|
+
const run = WorkflowRunSchema.parse(compact({
|
|
1115
|
+
...currentRun,
|
|
1116
|
+
status: "completed",
|
|
1117
|
+
output: data.eventData.output,
|
|
1118
|
+
error: void 0,
|
|
1119
|
+
completedAt: now,
|
|
1120
|
+
updatedAt: now
|
|
1121
|
+
}));
|
|
1122
|
+
const event = buildEvent({ ...data });
|
|
1123
|
+
await store.put(RUN_KEY, run);
|
|
1124
|
+
await putEvent(event);
|
|
1125
|
+
return {
|
|
1126
|
+
ok: true,
|
|
1127
|
+
event,
|
|
1128
|
+
run,
|
|
1129
|
+
releasedHooks: await releaseAllHooks(store)
|
|
1130
|
+
};
|
|
1131
|
+
}
|
|
1132
|
+
case "run_failed": {
|
|
1133
|
+
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1134
|
+
const run = WorkflowRunSchema.parse(compact({
|
|
1135
|
+
...currentRun,
|
|
1136
|
+
status: "failed",
|
|
1137
|
+
output: void 0,
|
|
1138
|
+
error: toStructuredError(data.eventData.error, { code: data.eventData.errorCode ?? readStringProp(data.eventData.error, "code") }),
|
|
1139
|
+
completedAt: now,
|
|
1140
|
+
updatedAt: now
|
|
1141
|
+
}));
|
|
1142
|
+
const event = buildEvent({ ...data });
|
|
1143
|
+
await store.put(RUN_KEY, run);
|
|
1144
|
+
await putEvent(event);
|
|
1145
|
+
return {
|
|
1146
|
+
ok: true,
|
|
1147
|
+
event,
|
|
1148
|
+
run,
|
|
1149
|
+
releasedHooks: await releaseAllHooks(store)
|
|
1150
|
+
};
|
|
1151
|
+
}
|
|
1152
|
+
case "run_cancelled": {
|
|
1153
|
+
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1154
|
+
const run = WorkflowRunSchema.parse(compact({
|
|
1155
|
+
...currentRun,
|
|
1156
|
+
status: "cancelled",
|
|
1157
|
+
output: void 0,
|
|
1158
|
+
error: void 0,
|
|
1159
|
+
completedAt: now,
|
|
1160
|
+
updatedAt: now
|
|
1161
|
+
}));
|
|
1162
|
+
const event = buildEvent({ ...data });
|
|
1163
|
+
await store.put(RUN_KEY, run);
|
|
1164
|
+
await putEvent(event);
|
|
1165
|
+
return {
|
|
1166
|
+
ok: true,
|
|
1167
|
+
event,
|
|
1168
|
+
run,
|
|
1169
|
+
releasedHooks: await releaseAllHooks(store)
|
|
1170
|
+
};
|
|
1171
|
+
}
|
|
1172
|
+
case "step_created": {
|
|
1173
|
+
const stepKey = `${STEP_KEY_PREFIX}${data.correlationId}`;
|
|
1174
|
+
if (await store.get(stepKey)) return failure("ENTITY_CONFLICT", `Step "${data.correlationId}" already exists`);
|
|
1175
|
+
const step = StepSchema.parse(compact({
|
|
1176
|
+
runId,
|
|
1177
|
+
stepId: data.correlationId,
|
|
1178
|
+
stepName: data.eventData.stepName,
|
|
1179
|
+
status: "pending",
|
|
1180
|
+
input: data.eventData.input,
|
|
1181
|
+
output: void 0,
|
|
1182
|
+
error: void 0,
|
|
1183
|
+
attempt: 0,
|
|
1184
|
+
startedAt: void 0,
|
|
1185
|
+
completedAt: void 0,
|
|
1186
|
+
retryAfter: void 0,
|
|
1187
|
+
createdAt: now,
|
|
1188
|
+
updatedAt: now,
|
|
1189
|
+
specVersion
|
|
1190
|
+
}));
|
|
1191
|
+
const event = buildEvent({ ...data });
|
|
1192
|
+
await store.put(stepKey, step);
|
|
1193
|
+
await store.put(creationIndexKey(STEP_CREATED_KEY_PREFIX, step.createdAt, step.stepId), step.stepId);
|
|
1194
|
+
await putEvent(event);
|
|
1195
|
+
return {
|
|
1196
|
+
ok: true,
|
|
1197
|
+
event,
|
|
1198
|
+
step,
|
|
1199
|
+
releasedHooks: []
|
|
1200
|
+
};
|
|
1201
|
+
}
|
|
1202
|
+
case "step_started": {
|
|
1203
|
+
const step = validatedStep;
|
|
1204
|
+
if (!step) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
1205
|
+
if (step.retryAfter && step.retryAfter.getTime() > now.getTime()) return failure("TOO_EARLY", `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, { retryAfterSeconds: Math.ceil((step.retryAfter.getTime() - now.getTime()) / 1e3) });
|
|
1206
|
+
const updated = StepSchema.parse(compact({
|
|
1207
|
+
...step,
|
|
1208
|
+
status: "running",
|
|
1209
|
+
startedAt: step.startedAt ?? now,
|
|
1210
|
+
attempt: step.attempt + 1,
|
|
1211
|
+
retryAfter: void 0,
|
|
1212
|
+
updatedAt: now
|
|
1213
|
+
}));
|
|
1214
|
+
const event = buildEvent({ ...data });
|
|
1215
|
+
await store.put(`${STEP_KEY_PREFIX}${data.correlationId}`, updated);
|
|
1216
|
+
await putEvent(event);
|
|
1217
|
+
return {
|
|
1218
|
+
ok: true,
|
|
1219
|
+
event,
|
|
1220
|
+
step: updated,
|
|
1221
|
+
releasedHooks: []
|
|
1222
|
+
};
|
|
1223
|
+
}
|
|
1224
|
+
case "step_completed": {
|
|
1225
|
+
const step = validatedStep;
|
|
1226
|
+
if (!step) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
1227
|
+
const updated = StepSchema.parse(compact({
|
|
1228
|
+
...step,
|
|
1229
|
+
status: "completed",
|
|
1230
|
+
output: data.eventData.result,
|
|
1231
|
+
completedAt: now,
|
|
1232
|
+
updatedAt: now
|
|
1233
|
+
}));
|
|
1234
|
+
const event = buildEvent({ ...data });
|
|
1235
|
+
await store.put(`${STEP_KEY_PREFIX}${data.correlationId}`, updated);
|
|
1236
|
+
await putEvent(event);
|
|
1237
|
+
return {
|
|
1238
|
+
ok: true,
|
|
1239
|
+
event,
|
|
1240
|
+
step: updated,
|
|
1241
|
+
releasedHooks: []
|
|
1242
|
+
};
|
|
1243
|
+
}
|
|
1244
|
+
case "step_failed": {
|
|
1245
|
+
const step = validatedStep;
|
|
1246
|
+
if (!step) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
1247
|
+
const updated = StepSchema.parse(compact({
|
|
1248
|
+
...step,
|
|
1249
|
+
status: "failed",
|
|
1250
|
+
error: toStructuredError(data.eventData.error, { stack: data.eventData.stack }),
|
|
1251
|
+
completedAt: now,
|
|
1252
|
+
updatedAt: now
|
|
1253
|
+
}));
|
|
1254
|
+
const event = buildEvent({ ...data });
|
|
1255
|
+
await store.put(`${STEP_KEY_PREFIX}${data.correlationId}`, updated);
|
|
1256
|
+
await putEvent(event);
|
|
1257
|
+
return {
|
|
1258
|
+
ok: true,
|
|
1259
|
+
event,
|
|
1260
|
+
step: updated,
|
|
1261
|
+
releasedHooks: []
|
|
1262
|
+
};
|
|
1263
|
+
}
|
|
1264
|
+
case "step_retrying": {
|
|
1265
|
+
const step = validatedStep;
|
|
1266
|
+
if (!step) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
1267
|
+
const updated = StepSchema.parse(compact({
|
|
1268
|
+
...step,
|
|
1269
|
+
status: "pending",
|
|
1270
|
+
error: toStructuredError(data.eventData.error, { stack: data.eventData.stack }),
|
|
1271
|
+
retryAfter: data.eventData.retryAfter ? new Date(data.eventData.retryAfter) : void 0,
|
|
1272
|
+
updatedAt: now
|
|
1273
|
+
}));
|
|
1274
|
+
const event = buildEvent({ ...data });
|
|
1275
|
+
await store.put(`${STEP_KEY_PREFIX}${data.correlationId}`, updated);
|
|
1276
|
+
await putEvent(event);
|
|
1277
|
+
return {
|
|
1278
|
+
ok: true,
|
|
1279
|
+
event,
|
|
1280
|
+
step: updated,
|
|
1281
|
+
releasedHooks: []
|
|
1282
|
+
};
|
|
1283
|
+
}
|
|
1284
|
+
case "hook_created": {
|
|
1285
|
+
const hookId = data.correlationId;
|
|
1286
|
+
const token = data.eventData.token;
|
|
1287
|
+
const holder = ctx.tokenHolder ?? null;
|
|
1288
|
+
const markerKey = `${HOOK_EVENT_MARKER_PREFIX}${hookId}`;
|
|
1289
|
+
const hookKey = `${HOOK_KEY_PREFIX}${hookId}`;
|
|
1290
|
+
if (holder && (holder.runId !== runId || holder.hookId !== hookId)) {
|
|
1291
|
+
const event = buildEvent({
|
|
1292
|
+
eventType: "hook_conflict",
|
|
1293
|
+
correlationId: hookId,
|
|
1294
|
+
eventData: {
|
|
1295
|
+
token,
|
|
1296
|
+
conflictingRunId: holder.runId
|
|
1297
|
+
}
|
|
1298
|
+
});
|
|
1299
|
+
await putEvent(event);
|
|
1300
|
+
return {
|
|
1301
|
+
ok: true,
|
|
1302
|
+
event,
|
|
1303
|
+
releasedHooks: []
|
|
1304
|
+
};
|
|
1305
|
+
}
|
|
1306
|
+
if (await store.get(markerKey) !== void 0) {
|
|
1307
|
+
const existingHook = await store.get(hookKey);
|
|
1308
|
+
if (existingHook && !holder) return {
|
|
1309
|
+
ok: true,
|
|
1310
|
+
hook: existingHook,
|
|
1311
|
+
hookToIndex: existingHook,
|
|
1312
|
+
releasedHooks: []
|
|
1313
|
+
};
|
|
1314
|
+
return failure("ENTITY_CONFLICT", `Hook "${hookId}" already created`);
|
|
1315
|
+
}
|
|
1316
|
+
const hook = HookSchema.parse(compact({
|
|
1317
|
+
runId,
|
|
1318
|
+
hookId,
|
|
1319
|
+
token,
|
|
1320
|
+
metadata: data.eventData.metadata,
|
|
1321
|
+
ownerId: "",
|
|
1322
|
+
projectId: "",
|
|
1323
|
+
environment: "",
|
|
1324
|
+
createdAt: now,
|
|
1325
|
+
specVersion,
|
|
1326
|
+
isWebhook: data.eventData.isWebhook ?? false
|
|
1327
|
+
}));
|
|
1328
|
+
const event = buildEvent({ ...data });
|
|
1329
|
+
await store.put(hookKey, hook);
|
|
1330
|
+
await store.put(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId), hook.hookId);
|
|
1331
|
+
await store.put(markerKey, event.eventId);
|
|
1332
|
+
await putEvent(event);
|
|
1333
|
+
return {
|
|
1334
|
+
ok: true,
|
|
1335
|
+
event,
|
|
1336
|
+
hook,
|
|
1337
|
+
hookToIndex: hook,
|
|
1338
|
+
releasedHooks: []
|
|
1339
|
+
};
|
|
1340
|
+
}
|
|
1341
|
+
case "hook_disposed": {
|
|
1342
|
+
const hookKey = `${HOOK_KEY_PREFIX}${data.correlationId}`;
|
|
1343
|
+
const hook = await store.get(hookKey);
|
|
1344
|
+
if (!hook) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
1345
|
+
await store.delete(hookKey);
|
|
1346
|
+
await store.delete(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
|
|
1347
|
+
const event = buildEvent({ ...data });
|
|
1348
|
+
await putEvent(event);
|
|
1349
|
+
return {
|
|
1350
|
+
ok: true,
|
|
1351
|
+
event,
|
|
1352
|
+
releasedHooks: [{
|
|
1353
|
+
hookId: hook.hookId,
|
|
1354
|
+
token: hook.token
|
|
1355
|
+
}]
|
|
1356
|
+
};
|
|
1357
|
+
}
|
|
1358
|
+
default: {
|
|
1359
|
+
const event = buildEvent({ ...data });
|
|
1360
|
+
await putEvent(event);
|
|
1361
|
+
return {
|
|
1362
|
+
ok: true,
|
|
1363
|
+
event,
|
|
1364
|
+
releasedHooks: []
|
|
1365
|
+
};
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
//#endregion
|
|
1370
|
+
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
1371
|
+
/**
|
|
1372
|
+
* Current schema version for DO storage.
|
|
1373
|
+
* Increment when making breaking changes to the storage layout.
|
|
1374
|
+
* Migrations run lazily on first request after deploy.
|
|
1375
|
+
*
|
|
1376
|
+
* v2: one storage key per event ('event:<eventId>'), step ('step:<stepId>')
|
|
1377
|
+
* and hook ('hook:<hookId>') instead of single aggregate values. This package
|
|
1378
|
+
* is pre-production, so the migration simply drops the legacy aggregates.
|
|
1379
|
+
*/
|
|
1380
|
+
const CURRENT_SCHEMA_VERSION = 3;
|
|
1381
|
+
/**
|
|
1382
|
+
* Idempotent migration functions keyed by target version. The schema version
|
|
1383
|
+
* is written only after the migration completes, so eviction safely retries
|
|
1384
|
+
* an interrupted migration.
|
|
1385
|
+
*/
|
|
1386
|
+
const MIGRATIONS = {
|
|
1387
|
+
2: async (storage) => {
|
|
1388
|
+
await storage.transaction(async (txn) => {
|
|
1389
|
+
await txn.delete("events");
|
|
1390
|
+
await txn.delete("steps");
|
|
1391
|
+
await txn.delete("hooks");
|
|
1392
|
+
});
|
|
1393
|
+
},
|
|
1394
|
+
3: async (storage) => {
|
|
1395
|
+
await backfillCreationIndex(storage, STEP_KEY_PREFIX, STEP_CREATED_KEY_PREFIX, (step) => step.stepId);
|
|
1396
|
+
await backfillCreationIndex(storage, HOOK_KEY_PREFIX, HOOK_CREATED_KEY_PREFIX, (hook) => hook.hookId);
|
|
1397
|
+
}
|
|
1398
|
+
};
|
|
1399
|
+
async function backfillCreationIndex(storage, entityPrefix, indexPrefix, getId) {
|
|
1400
|
+
let startAfter;
|
|
1401
|
+
while (true) {
|
|
1402
|
+
const entities = await storage.list({
|
|
1403
|
+
prefix: entityPrefix,
|
|
1404
|
+
startAfter,
|
|
1405
|
+
limit: 100
|
|
1406
|
+
});
|
|
1407
|
+
if (entities.size === 0) return;
|
|
1408
|
+
for (const [key, entity] of entities) {
|
|
1409
|
+
const id = getId(entity);
|
|
1410
|
+
await storage.put(`${indexPrefix}${entity.createdAt.toISOString()}:${id}`, id);
|
|
1411
|
+
startAfter = key;
|
|
1412
|
+
}
|
|
1413
|
+
if (entities.size < 100) return;
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
/**
|
|
1417
|
+
* Ensure the DO storage schema is up to date.
|
|
1418
|
+
* Runs any pending migrations sequentially inside transactions,
|
|
1419
|
+
* persisting the version after each successful migration.
|
|
1420
|
+
* Safe to call on every request -- it's a no-op when already current.
|
|
1421
|
+
*/
|
|
1422
|
+
async function ensureSchemaUpToDate(storage) {
|
|
1423
|
+
const stored = await storage.get("schema_version") ?? 1;
|
|
1424
|
+
for (let v = stored + 1; v <= CURRENT_SCHEMA_VERSION; v++) {
|
|
1425
|
+
const migration = MIGRATIONS[v];
|
|
1426
|
+
if (migration) await migration(storage);
|
|
1427
|
+
await storage.put("schema_version", v);
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
/**
|
|
1431
|
+
* Adapt DO storage (or a transaction) to the {@link EventStore} interface
|
|
1432
|
+
* shared with the test mocks. No casts: the methods line up structurally,
|
|
1433
|
+
* this just pins the subset we rely on.
|
|
1434
|
+
*/
|
|
1435
|
+
function storeFrom(storage) {
|
|
1436
|
+
return {
|
|
1437
|
+
get: (key) => storage.get(key),
|
|
1438
|
+
put: (key, value) => storage.put(key, value),
|
|
1439
|
+
delete: (key) => storage.delete(key),
|
|
1440
|
+
list: (options) => storage.list(options)
|
|
1441
|
+
};
|
|
1442
|
+
}
|
|
1443
|
+
/**
|
|
1444
|
+
* Durable Object for managing a single workflow run's state.
|
|
1445
|
+
*
|
|
1446
|
+
* All event-sourced writes go through {@link applyEvent} inside a single
|
|
1447
|
+
* storage transaction: guard checks, the event append, and the entity
|
|
1448
|
+
* mutation are atomic even if the DO is evicted mid-operation.
|
|
1449
|
+
*
|
|
1450
|
+
* Instances named `claim:<queueName>:<idempotencyKey>` are used purely as
|
|
1451
|
+
* queue-message dedup claims (see claimInflight/releaseInflight).
|
|
1452
|
+
*/
|
|
1453
|
+
var WorkflowRunDO = class extends DurableObject {
|
|
1454
|
+
schemaReady = null;
|
|
1455
|
+
/**
|
|
1456
|
+
* Lazily ensure schema migrations have run.
|
|
1457
|
+
* Memoized so it only runs once per DO instantiation.
|
|
1458
|
+
*/
|
|
1459
|
+
ensureSchema() {
|
|
1460
|
+
if (!this.schemaReady) this.schemaReady = ensureSchemaUpToDate(this.ctx.storage);
|
|
1461
|
+
return this.schemaReady;
|
|
1462
|
+
}
|
|
1463
|
+
/**
|
|
1464
|
+
* Apply an event: validate, guard, append the event and mutate the entity
|
|
1465
|
+
* in ONE storage transaction. Guard failures are returned as structured
|
|
1466
|
+
* outcomes (not thrown) because typed errors do not survive the RPC
|
|
1467
|
+
* boundary with their `name` intact.
|
|
1468
|
+
*/
|
|
1469
|
+
async applyEvent(request) {
|
|
1470
|
+
await this.ensureSchema();
|
|
1471
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1472
|
+
let eventSequence = await txn.get("event_sequence") ?? 0;
|
|
1473
|
+
const outcome = await applyEvent(storeFrom(txn), {
|
|
1474
|
+
...request,
|
|
1475
|
+
nextEventId: () => `wevt_z${String(++eventSequence).padStart(20, "0")}`,
|
|
1476
|
+
now: /* @__PURE__ */ new Date()
|
|
1477
|
+
});
|
|
1478
|
+
if (outcome.ok) await txn.put("event_sequence", eventSequence);
|
|
1479
|
+
return outcome;
|
|
1480
|
+
});
|
|
1481
|
+
}
|
|
1482
|
+
/**
|
|
1483
|
+
* Get the workflow run
|
|
1484
|
+
*/
|
|
1485
|
+
async getRun() {
|
|
1486
|
+
await this.ensureSchema();
|
|
1487
|
+
return await this.ctx.storage.get("run") ?? null;
|
|
1488
|
+
}
|
|
1489
|
+
/**
|
|
1490
|
+
* Get a specific step
|
|
1491
|
+
*/
|
|
1492
|
+
async getStep(stepId) {
|
|
1493
|
+
await this.ensureSchema();
|
|
1494
|
+
return await this.ctx.storage.get(`step:${stepId}`) ?? null;
|
|
1495
|
+
}
|
|
1496
|
+
/**
|
|
1497
|
+
* List steps with real limit/cursor pagination (cursor = last stepId).
|
|
1498
|
+
*/
|
|
1499
|
+
async listSteps(params) {
|
|
1500
|
+
await this.ensureSchema();
|
|
1501
|
+
return listByCreationTime(storeFrom(this.ctx.storage), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
|
|
1502
|
+
limit: params?.limit ?? 20,
|
|
1503
|
+
cursor: params?.cursor,
|
|
1504
|
+
sortOrder: params?.sortOrder ?? "asc"
|
|
1505
|
+
});
|
|
1506
|
+
}
|
|
1507
|
+
/**
|
|
1508
|
+
* Get a specific event
|
|
1509
|
+
*/
|
|
1510
|
+
async getEvent(eventId) {
|
|
1511
|
+
await this.ensureSchema();
|
|
1512
|
+
return await this.ctx.storage.get(`event:${eventId}`) ?? null;
|
|
1513
|
+
}
|
|
1514
|
+
/**
|
|
1515
|
+
* List events with real limit/cursor pagination. Events are keyed by their
|
|
1516
|
+
* monotonic ULID eventId, so storage order == event order.
|
|
1517
|
+
*/
|
|
1518
|
+
async listEvents(params) {
|
|
1519
|
+
await this.ensureSchema();
|
|
1520
|
+
return listByPrefix(storeFrom(this.ctx.storage), EVENT_KEY_PREFIX, {
|
|
1521
|
+
limit: params?.limit ?? 100,
|
|
1522
|
+
cursor: params?.cursor,
|
|
1523
|
+
sortOrder: params?.sortOrder ?? "asc"
|
|
1524
|
+
}, (event) => event.eventId);
|
|
1525
|
+
}
|
|
1526
|
+
/**
|
|
1527
|
+
* List hooks with real limit/cursor pagination (cursor = last hookId).
|
|
1528
|
+
*/
|
|
1529
|
+
async listHooks(params) {
|
|
1530
|
+
await this.ensureSchema();
|
|
1531
|
+
return listByCreationTime(storeFrom(this.ctx.storage), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
|
|
1532
|
+
limit: params?.limit ?? 100,
|
|
1533
|
+
cursor: params?.cursor,
|
|
1534
|
+
sortOrder: params?.sortOrder ?? "asc"
|
|
1535
|
+
});
|
|
1536
|
+
}
|
|
1537
|
+
/**
|
|
1538
|
+
* Claim a queue-message dedup slot. Used on DO instances dedicated to a
|
|
1539
|
+
* single idempotency key (`claim:<queueName>:<idempotencyKey>`).
|
|
1540
|
+
*
|
|
1541
|
+
* - Unclaimed -> claim and proceed.
|
|
1542
|
+
* - Claimed by the SAME messageId -> redelivery of the same message
|
|
1543
|
+
* (previous attempt failed transiently) -> proceed.
|
|
1544
|
+
* - Claimed by a DIFFERENT messageId -> concurrent duplicate -> reject,
|
|
1545
|
+
* unless the claim is stale (holder crashed / went to the DLQ without
|
|
1546
|
+
* releasing), in which case the claim is stolen.
|
|
1547
|
+
*/
|
|
1548
|
+
async claimInflight(params) {
|
|
1549
|
+
await this.ensureSchema();
|
|
1550
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1551
|
+
const existing = await txn.get("claim");
|
|
1552
|
+
const now = Date.now();
|
|
1553
|
+
if (existing && existing.messageId !== params.messageId && now - existing.claimedAt < params.staleMs) return { claimed: false };
|
|
1554
|
+
await txn.put("claim", {
|
|
1555
|
+
messageId: params.messageId,
|
|
1556
|
+
claimedAt: now
|
|
1557
|
+
});
|
|
1558
|
+
return { claimed: true };
|
|
1559
|
+
});
|
|
1560
|
+
}
|
|
1561
|
+
/**
|
|
1562
|
+
* Release a queue-message dedup claim after the message was fully handled.
|
|
1563
|
+
*/
|
|
1564
|
+
async releaseInflight() {
|
|
1565
|
+
await this.ensureSchema();
|
|
1566
|
+
await this.ctx.storage.delete("claim");
|
|
1567
|
+
}
|
|
1568
|
+
};
|
|
1569
|
+
//#endregion
|
|
1570
|
+
//#region src/worker/worker.ts
|
|
1571
|
+
/**
|
|
1572
|
+
* celld worker entry: exports the Durable Object (cell) classes and the
|
|
1573
|
+
* default fetch router. This module (and everything it imports) must stay
|
|
1574
|
+
* free of Node built-ins — it runs inside celld's workerd runtime.
|
|
1575
|
+
*/
|
|
1576
|
+
var worker_default = { async fetch(request, env) {
|
|
1577
|
+
return createRouter(env)(request);
|
|
1578
|
+
} };
|
|
1579
|
+
//#endregion
|
|
1580
|
+
export { IndexDO, QueueDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
|
|
1581
|
+
|
|
1582
|
+
//# sourceMappingURL=worker.js.map
|