@ewhauser/world-celld 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +47 -19
- package/celld-worker/README.md +5 -3
- package/celld-worker/worker.ts +10 -1
- package/celld-worker/wrangler.jsonc +15 -4
- package/dist/index.d.ts +163 -60
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +524 -197
- package/dist/index.js.map +1 -1
- package/dist/testing.d.ts +47 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +1874 -582
- package/dist/testing.js.map +1 -1
- package/dist/util-DmO0XkFw.js +545 -0
- package/dist/util-DmO0XkFw.js.map +1 -0
- package/dist/worker.d.ts +223 -111
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +1960 -551
- package/dist/worker.js.map +1 -1
- package/package.json +7 -5
- package/dist/retention-bLY1vcOu.js +0 -164
- package/dist/retention-bLY1vcOu.js.map +0 -1
package/dist/worker.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WorkflowRunSchema,
|
|
1
|
+
import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
|
|
2
2
|
import { DurableObject } from "cloudflare:workers";
|
|
3
3
|
//#region src/vendor/shared/serialization.ts
|
|
4
4
|
/** Standard Date fields used across all worlds */
|
|
@@ -96,6 +96,342 @@ function rpcStringify(value) {
|
|
|
96
96
|
function rpcParse(text) {
|
|
97
97
|
return JSON.parse(text, rpcReviver);
|
|
98
98
|
}
|
|
99
|
+
const INDEX_PROTOCOL_VERSION = "v1";
|
|
100
|
+
const HOOK_RELEASE_BATCH = 64;
|
|
101
|
+
/** Stable FNV-1a routing hash. Collisions share a shard but never a storage key. */
|
|
102
|
+
function stableIndexHash(value) {
|
|
103
|
+
let hash = 2166136261;
|
|
104
|
+
for (const byte of new TextEncoder().encode(value)) {
|
|
105
|
+
hash ^= byte;
|
|
106
|
+
hash = Math.imul(hash, 16777619);
|
|
107
|
+
}
|
|
108
|
+
return hash >>> 0;
|
|
109
|
+
}
|
|
110
|
+
function shardName(domain, value, cardinality) {
|
|
111
|
+
const width = Math.max(2, (cardinality - 1).toString(16).length);
|
|
112
|
+
const shard = stableIndexHash(value) % cardinality;
|
|
113
|
+
return `${domain}:${INDEX_PROTOCOL_VERSION}:${shard.toString(16).padStart(width, "0")}`;
|
|
114
|
+
}
|
|
115
|
+
function runCatalogShardName(runId) {
|
|
116
|
+
return shardName("run-catalog", runId, 16);
|
|
117
|
+
}
|
|
118
|
+
function hookTokenShardName(token) {
|
|
119
|
+
return shardName("hook-token", token, 32);
|
|
120
|
+
}
|
|
121
|
+
function hookIdShardName(hookId) {
|
|
122
|
+
return shardName("hook-id", hookId, 32);
|
|
123
|
+
}
|
|
124
|
+
function runFenceCellName(runId) {
|
|
125
|
+
return `run-fence:${INDEX_PROTOCOL_VERSION}:${runId}`;
|
|
126
|
+
}
|
|
127
|
+
function allRunCatalogShardNames() {
|
|
128
|
+
return Array.from({ length: 16 }, (_, shard) => `run-catalog:${INDEX_PROTOCOL_VERSION}:${shard.toString(16).padStart(2, "0")}`);
|
|
129
|
+
}
|
|
130
|
+
function stub(namespace, name) {
|
|
131
|
+
return namespace.get(namespace.idFromName(name));
|
|
132
|
+
}
|
|
133
|
+
function runKeys(run) {
|
|
134
|
+
const timestamp = run.createdAt.getTime().toString().padStart(13, "0");
|
|
135
|
+
return [`run:${run.workflowName}:${timestamp}:${run.runId}`, `runall:${timestamp}:${run.runId}`];
|
|
136
|
+
}
|
|
137
|
+
function groupByShard(values, route) {
|
|
138
|
+
const groups = /* @__PURE__ */ new Map();
|
|
139
|
+
for (const value of values) {
|
|
140
|
+
const name = route(value);
|
|
141
|
+
const group = groups.get(name);
|
|
142
|
+
if (group) group.push(value);
|
|
143
|
+
else groups.set(name, [value]);
|
|
144
|
+
}
|
|
145
|
+
return groups;
|
|
146
|
+
}
|
|
147
|
+
function sameOwner$1(left, right) {
|
|
148
|
+
return left.runId === right.runId && left.hookId === right.hookId;
|
|
149
|
+
}
|
|
150
|
+
function compareBytes(left, right) {
|
|
151
|
+
const length = Math.min(left.byteLength, right.byteLength);
|
|
152
|
+
for (let index = 0; index < length; index++) {
|
|
153
|
+
const difference = left[index] - right[index];
|
|
154
|
+
if (difference !== 0) return difference;
|
|
155
|
+
}
|
|
156
|
+
return left.byteLength - right.byteLength;
|
|
157
|
+
}
|
|
158
|
+
async function inBatches(values, operation) {
|
|
159
|
+
let deleted = 0;
|
|
160
|
+
for (let offset = 0; offset < values.length; offset += HOOK_RELEASE_BATCH) deleted += (await operation(values.slice(offset, offset + HOOK_RELEASE_BATCH))).deleted;
|
|
161
|
+
return deleted;
|
|
162
|
+
}
|
|
163
|
+
function createWorkflowIndex(bindings) {
|
|
164
|
+
const fence = (runId) => stub(bindings.runFences, runFenceCellName(runId));
|
|
165
|
+
const releaseHooks = async (request, fencedAt) => {
|
|
166
|
+
const tokenGroups = groupByShard(request.hooks, (hook) => hookTokenShardName(hook.token));
|
|
167
|
+
const idGroups = groupByShard(request.hooks, (hook) => hookIdShardName(hook.hookId));
|
|
168
|
+
const releases = [];
|
|
169
|
+
for (const [name, hooks] of tokenGroups) {
|
|
170
|
+
const target = stub(bindings.hookTokens, name);
|
|
171
|
+
releases.push(inBatches(hooks.map((hook) => ({
|
|
172
|
+
token: hook.token,
|
|
173
|
+
owner: {
|
|
174
|
+
runId: request.runId,
|
|
175
|
+
hookId: hook.hookId
|
|
176
|
+
}
|
|
177
|
+
})), (batch) => target.releaseBatch(batch, fencedAt, request.terminal)));
|
|
178
|
+
}
|
|
179
|
+
for (const [name, hooks] of idGroups) {
|
|
180
|
+
const target = stub(bindings.hookIds, name);
|
|
181
|
+
releases.push(inBatches(hooks.map((hook) => ({
|
|
182
|
+
hookId: hook.hookId,
|
|
183
|
+
owner: {
|
|
184
|
+
runId: request.runId,
|
|
185
|
+
hookId: hook.hookId
|
|
186
|
+
}
|
|
187
|
+
})), (batch) => target.releaseBatch(batch, fencedAt, request.terminal)));
|
|
188
|
+
}
|
|
189
|
+
return (await Promise.all(releases)).reduce((total, count) => total + count, 0);
|
|
190
|
+
};
|
|
191
|
+
return {
|
|
192
|
+
async commitRun(run, serializedMetadata) {
|
|
193
|
+
if (isTerminalWorkflowRunStatus(run.status)) await fence(run.runId).fenceTerminal(Date.now());
|
|
194
|
+
return await stub(bindings.runCatalog, runCatalogShardName(run.runId)).upsertRun(run.runId, runKeys(run), serializedMetadata);
|
|
195
|
+
},
|
|
196
|
+
async listRuns(options) {
|
|
197
|
+
const suppliedLimit = options?.limit;
|
|
198
|
+
const limit = Math.min(1e3, Math.max(1, typeof suppliedLimit === "number" && Number.isFinite(suppliedLimit) ? Math.floor(suppliedLimit) : 1e3));
|
|
199
|
+
const pages = await Promise.all(allRunCatalogShardNames().map((name) => stub(bindings.runCatalog, name).list({
|
|
200
|
+
...options,
|
|
201
|
+
limit: limit + 1
|
|
202
|
+
})));
|
|
203
|
+
const encoder = new TextEncoder();
|
|
204
|
+
const candidates = pages.flatMap((shardPage) => shardPage.keys.map((entry) => ({
|
|
205
|
+
entry,
|
|
206
|
+
encodedName: encoder.encode(entry.name)
|
|
207
|
+
}))).toSorted((left, right) => {
|
|
208
|
+
const compared = compareBytes(left.encodedName, right.encodedName);
|
|
209
|
+
return options?.reverse ? -compared : compared;
|
|
210
|
+
});
|
|
211
|
+
const page = candidates.slice(0, limit).map(({ entry }) => entry);
|
|
212
|
+
const listComplete = candidates.length <= limit && pages.every((entry) => entry.list_complete);
|
|
213
|
+
return {
|
|
214
|
+
keys: page,
|
|
215
|
+
list_complete: listComplete,
|
|
216
|
+
cursor: listComplete ? void 0 : page.at(-1)?.name
|
|
217
|
+
};
|
|
218
|
+
},
|
|
219
|
+
getHookByToken(token) {
|
|
220
|
+
return stub(bindings.hookTokens, hookTokenShardName(token)).get(token);
|
|
221
|
+
},
|
|
222
|
+
getHookById(hookId) {
|
|
223
|
+
return stub(bindings.hookIds, hookIdShardName(hookId)).get(hookId);
|
|
224
|
+
},
|
|
225
|
+
async reserveHook(token, owner) {
|
|
226
|
+
const proposedClaimId = crypto.randomUUID();
|
|
227
|
+
const tokenTarget = stub(bindings.hookTokens, hookTokenShardName(token));
|
|
228
|
+
const tokenClaim = await tokenTarget.reserve(token, owner, proposedClaimId);
|
|
229
|
+
if (!tokenClaim.claimed && !sameOwner$1(tokenClaim.holder, owner)) return {
|
|
230
|
+
admitted: false,
|
|
231
|
+
holder: tokenClaim.holder
|
|
232
|
+
};
|
|
233
|
+
const tokenClaimId = tokenClaim.claimed ? tokenClaim.claimId : void 0;
|
|
234
|
+
const hookIdTarget = stub(bindings.hookIds, hookIdShardName(owner.hookId));
|
|
235
|
+
const sharedClaimId = tokenClaimId ?? proposedClaimId;
|
|
236
|
+
let hookIdClaim;
|
|
237
|
+
try {
|
|
238
|
+
hookIdClaim = await hookIdTarget.reserve(owner.hookId, owner, sharedClaimId);
|
|
239
|
+
} catch (error) {
|
|
240
|
+
await Promise.allSettled([...tokenClaimId ? [tokenTarget.releaseClaim(token, owner, tokenClaimId)] : [], hookIdTarget.releaseClaim(owner.hookId, owner, sharedClaimId)]);
|
|
241
|
+
throw error;
|
|
242
|
+
}
|
|
243
|
+
if (!hookIdClaim.claimed) {
|
|
244
|
+
if (tokenClaimId) await tokenTarget.releaseClaim(token, owner, tokenClaimId);
|
|
245
|
+
return {
|
|
246
|
+
admitted: false,
|
|
247
|
+
holder: hookIdClaim.holder
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
return {
|
|
251
|
+
admitted: true,
|
|
252
|
+
reservation: {
|
|
253
|
+
claimId: `${tokenClaimId ?? "-"}:${hookIdClaim.claimId}`,
|
|
254
|
+
tokenClaimId,
|
|
255
|
+
hookIdClaimId: hookIdClaim.claimId
|
|
256
|
+
}
|
|
257
|
+
};
|
|
258
|
+
},
|
|
259
|
+
async finalizeHookIndexes(token, hookId, serializedHook, owner, reservation) {
|
|
260
|
+
if (!(await stub(bindings.hookTokens, hookTokenShardName(token)).finalize(token, serializedHook, owner, reservation?.tokenClaimId)).stored) return;
|
|
261
|
+
await stub(bindings.hookIds, hookIdShardName(hookId)).publish(hookId, serializedHook, owner, reservation?.hookIdClaimId);
|
|
262
|
+
},
|
|
263
|
+
async releaseHookReservation(token, owner, reservation) {
|
|
264
|
+
const releases = [];
|
|
265
|
+
if (reservation.tokenClaimId) releases.push(stub(bindings.hookTokens, hookTokenShardName(token)).releaseClaim(token, owner, reservation.tokenClaimId));
|
|
266
|
+
if (reservation.hookIdClaimId) releases.push(stub(bindings.hookIds, hookIdShardName(owner.hookId)).releaseClaim(owner.hookId, owner, reservation.hookIdClaimId));
|
|
267
|
+
await Promise.all(releases);
|
|
268
|
+
},
|
|
269
|
+
async releaseHookIndexes(request) {
|
|
270
|
+
if (request.terminal) await fence(request.runId).fenceTerminal(Date.now());
|
|
271
|
+
return { deleted: await releaseHooks(request, Date.now()) };
|
|
272
|
+
},
|
|
273
|
+
async expireRun(request) {
|
|
274
|
+
await fence(request.runId).fenceExpired(request.expiredAt);
|
|
275
|
+
const [catalog, hookDeletes] = await Promise.all([stub(bindings.runCatalog, runCatalogShardName(request.runId)).expireRun(request.runId, request.keys, request.expiredAt), releaseHooks({
|
|
276
|
+
runId: request.runId,
|
|
277
|
+
hooks: request.hooks,
|
|
278
|
+
terminal: true
|
|
279
|
+
}, request.expiredAt)]);
|
|
280
|
+
return { deleted: catalog.deleted + hookDeletes };
|
|
281
|
+
}
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
//#endregion
|
|
285
|
+
//#region src/stream-protocol.ts
|
|
286
|
+
/**
|
|
287
|
+
* Hard-cutover stream protocol shared by the client, Worker router, and
|
|
288
|
+
* StreamDO. Chunk payloads use a compact binary envelope on the public HTTP
|
|
289
|
+
* boundary and Uint8Array values on the DO/storage boundary.
|
|
290
|
+
*/
|
|
291
|
+
const STREAM_BATCH_CONTENT_TYPE = "application/vnd.world-celld.stream-batch.v1";
|
|
292
|
+
/** A chunk remains below the 2 MiB SQLite-backed DO key/value limit. */
|
|
293
|
+
const MAX_STREAM_CHUNK_BYTES = 1048576;
|
|
294
|
+
/** Keeps serialized RPC bodies well below the 32 MiB Workers RPC limit. */
|
|
295
|
+
const MAX_STREAM_BATCH_BYTES = 8388608;
|
|
296
|
+
const MAX_STREAM_READ_BYTES = 8388608;
|
|
297
|
+
const MAX_STREAM_ERROR_BYTES = 16384;
|
|
298
|
+
/** Must remain below the default 30 second fleet request deadline. */
|
|
299
|
+
const MAX_STREAM_LONG_POLL_MS = 2e4;
|
|
300
|
+
const WRITE_MAGIC = 1464030039;
|
|
301
|
+
const WRITE_ACK_MAGIC = 1464030017;
|
|
302
|
+
const READ_MAGIC = 1464030034;
|
|
303
|
+
const PROTOCOL_VERSION = 1;
|
|
304
|
+
const WRITE_HEADER_BYTES = 8;
|
|
305
|
+
const WRITE_ACK_BYTES = 16;
|
|
306
|
+
const READ_HEADER_BYTES = 28;
|
|
307
|
+
const STATE_TO_WIRE = {
|
|
308
|
+
open: 0,
|
|
309
|
+
closed: 1,
|
|
310
|
+
errored: 2,
|
|
311
|
+
expired: 3
|
|
312
|
+
};
|
|
313
|
+
function protocolError(message) {
|
|
314
|
+
const error = /* @__PURE__ */ new Error(`world-celld: invalid stream protocol: ${message}`);
|
|
315
|
+
error.name = "StreamProtocolError";
|
|
316
|
+
return error;
|
|
317
|
+
}
|
|
318
|
+
function requireSafeIndex(value, field) {
|
|
319
|
+
if (!Number.isSafeInteger(value) || value < -1 || value > 2147483647) throw protocolError(`${field} is out of range`);
|
|
320
|
+
}
|
|
321
|
+
function validateStreamWriteChunks(chunks) {
|
|
322
|
+
if (chunks.length === 0) throw protocolError("batch must contain at least one chunk");
|
|
323
|
+
if (chunks.length > 32) throw protocolError(`batch exceeds 32 chunks`);
|
|
324
|
+
let totalBytes = 0;
|
|
325
|
+
for (const chunk of chunks) {
|
|
326
|
+
if (!(chunk instanceof Uint8Array)) throw protocolError("chunk is not binary data");
|
|
327
|
+
if (chunk.byteLength > 1048576) throw protocolError(`chunk exceeds ${MAX_STREAM_CHUNK_BYTES} bytes`);
|
|
328
|
+
totalBytes += chunk.byteLength;
|
|
329
|
+
if (totalBytes > 8388608) throw protocolError(`batch exceeds ${MAX_STREAM_BATCH_BYTES} bytes`);
|
|
330
|
+
}
|
|
331
|
+
return totalBytes;
|
|
332
|
+
}
|
|
333
|
+
function validateStreamReadRequest(request) {
|
|
334
|
+
if (!request.runId) throw protocolError("runId is required");
|
|
335
|
+
if (!Number.isSafeInteger(request.startIndex) || request.startIndex < 0) throw protocolError("startIndex must be a non-negative safe integer");
|
|
336
|
+
if (!Number.isSafeInteger(request.maxChunks) || request.maxChunks < 0 || request.maxChunks > 32) throw protocolError(`maxChunks must be between 0 and 32`);
|
|
337
|
+
if (!Number.isSafeInteger(request.maxBytes) || request.maxBytes < 1048576 || request.maxBytes > 8388608) throw protocolError(`maxBytes must be between ${MAX_STREAM_CHUNK_BYTES} and ${MAX_STREAM_READ_BYTES}`);
|
|
338
|
+
if (!Number.isSafeInteger(request.waitMs) || request.waitMs < 0 || request.waitMs > 2e4) throw protocolError(`waitMs must be between 0 and ${MAX_STREAM_LONG_POLL_MS}`);
|
|
339
|
+
}
|
|
340
|
+
function decodeStreamWriteBatch(encoded) {
|
|
341
|
+
if (encoded.byteLength < WRITE_HEADER_BYTES) throw protocolError("truncated write header");
|
|
342
|
+
const view = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength);
|
|
343
|
+
if (view.getUint32(0) !== WRITE_MAGIC) throw protocolError("bad write magic");
|
|
344
|
+
if (view.getUint16(4) !== PROTOCOL_VERSION) throw protocolError("unsupported write version");
|
|
345
|
+
const count = view.getUint16(6);
|
|
346
|
+
const chunks = [];
|
|
347
|
+
let offset = WRITE_HEADER_BYTES;
|
|
348
|
+
for (let index = 0; index < count; index++) {
|
|
349
|
+
if (offset + 4 > encoded.byteLength) throw protocolError("truncated chunk length");
|
|
350
|
+
const length = view.getUint32(offset);
|
|
351
|
+
offset += 4;
|
|
352
|
+
if (offset + length > encoded.byteLength) throw protocolError("truncated chunk data");
|
|
353
|
+
chunks.push(new Uint8Array(encoded.subarray(offset, offset + length)));
|
|
354
|
+
offset += length;
|
|
355
|
+
}
|
|
356
|
+
if (offset !== encoded.byteLength) throw protocolError("trailing write data");
|
|
357
|
+
validateStreamWriteChunks(chunks);
|
|
358
|
+
return chunks;
|
|
359
|
+
}
|
|
360
|
+
function encodeStreamWriteResult(result) {
|
|
361
|
+
requireSafeIndex(result.startIndex, "startIndex");
|
|
362
|
+
requireSafeIndex(result.tailIndex, "tailIndex");
|
|
363
|
+
if (!Number.isSafeInteger(result.count) || result.count < 1 || result.count > 32) throw protocolError("write count is out of range");
|
|
364
|
+
const encoded = new Uint8Array(WRITE_ACK_BYTES);
|
|
365
|
+
const view = new DataView(encoded.buffer);
|
|
366
|
+
view.setUint32(0, WRITE_ACK_MAGIC);
|
|
367
|
+
view.setUint16(4, PROTOCOL_VERSION);
|
|
368
|
+
view.setUint16(6, result.count);
|
|
369
|
+
view.setInt32(8, result.startIndex);
|
|
370
|
+
view.setInt32(12, result.tailIndex);
|
|
371
|
+
return encoded;
|
|
372
|
+
}
|
|
373
|
+
function encodeStreamReadResult(result) {
|
|
374
|
+
requireSafeIndex(result.startIndex, "startIndex");
|
|
375
|
+
requireSafeIndex(result.tailIndex, "tailIndex");
|
|
376
|
+
let chunkBytes = 0;
|
|
377
|
+
for (const chunk of result.chunks) chunkBytes += 4 + chunk.byteLength;
|
|
378
|
+
if (result.chunks.length > 32 || chunkBytes > 8388736) throw protocolError("read result exceeds configured bounds");
|
|
379
|
+
const encoder = new TextEncoder();
|
|
380
|
+
const errorName = result.error ? encoder.encode(result.error.name) : /* @__PURE__ */ new Uint8Array();
|
|
381
|
+
const errorMessage = result.error ? encoder.encode(result.error.message) : /* @__PURE__ */ new Uint8Array();
|
|
382
|
+
if (errorName.byteLength + errorMessage.byteLength > 16384) throw protocolError(`stream error exceeds ${MAX_STREAM_ERROR_BYTES} bytes`);
|
|
383
|
+
const encoded = new Uint8Array(READ_HEADER_BYTES + chunkBytes + errorName.byteLength + errorMessage.byteLength);
|
|
384
|
+
const view = new DataView(encoded.buffer);
|
|
385
|
+
view.setUint32(0, READ_MAGIC);
|
|
386
|
+
view.setUint8(4, PROTOCOL_VERSION);
|
|
387
|
+
const state = STATE_TO_WIRE[result.state];
|
|
388
|
+
if (state === void 0) throw protocolError("unknown stream state");
|
|
389
|
+
view.setUint8(5, state);
|
|
390
|
+
view.setUint8(6, result.timedOut ? 1 : 0);
|
|
391
|
+
view.setUint8(7, 0);
|
|
392
|
+
view.setInt32(8, result.startIndex);
|
|
393
|
+
view.setInt32(12, result.tailIndex);
|
|
394
|
+
view.setUint32(16, result.chunks.length);
|
|
395
|
+
view.setUint32(20, errorName.byteLength);
|
|
396
|
+
view.setUint32(24, errorMessage.byteLength);
|
|
397
|
+
let offset = READ_HEADER_BYTES;
|
|
398
|
+
for (const chunk of result.chunks) {
|
|
399
|
+
view.setUint32(offset, chunk.byteLength);
|
|
400
|
+
offset += 4;
|
|
401
|
+
encoded.set(chunk, offset);
|
|
402
|
+
offset += chunk.byteLength;
|
|
403
|
+
}
|
|
404
|
+
encoded.set(errorName, offset);
|
|
405
|
+
offset += errorName.byteLength;
|
|
406
|
+
encoded.set(errorMessage, offset);
|
|
407
|
+
return encoded;
|
|
408
|
+
}
|
|
409
|
+
function normalizeStreamError(error) {
|
|
410
|
+
let normalized;
|
|
411
|
+
if (error instanceof Error) normalized = {
|
|
412
|
+
name: error.name || "Error",
|
|
413
|
+
message: error.message
|
|
414
|
+
};
|
|
415
|
+
else if (typeof error === "string") normalized = {
|
|
416
|
+
name: "Error",
|
|
417
|
+
message: error
|
|
418
|
+
};
|
|
419
|
+
else if (error && typeof error === "object" && typeof error.name === "string" && typeof error.message === "string") normalized = {
|
|
420
|
+
name: error.name,
|
|
421
|
+
message: error.message
|
|
422
|
+
};
|
|
423
|
+
else normalized = {
|
|
424
|
+
name: "Error",
|
|
425
|
+
message: String(error)
|
|
426
|
+
};
|
|
427
|
+
normalized.name = normalized.name.slice(0, 1024);
|
|
428
|
+
normalized.message = normalized.message.slice(0, 4096);
|
|
429
|
+
while (new TextEncoder().encode(normalized.name).byteLength + new TextEncoder().encode(normalized.message).byteLength > MAX_STREAM_ERROR_BYTES) {
|
|
430
|
+
normalized.message = normalized.message.slice(0, Math.max(0, normalized.message.length - 256));
|
|
431
|
+
if (normalized.message.length === 0) normalized.name = normalized.name.slice(0, Math.max(0, normalized.name.length - 256));
|
|
432
|
+
}
|
|
433
|
+
return normalized;
|
|
434
|
+
}
|
|
99
435
|
//#endregion
|
|
100
436
|
//#region src/worker/auth.ts
|
|
101
437
|
/**
|
|
@@ -145,13 +481,23 @@ async function authenticate(request, secret) {
|
|
|
145
481
|
* Routes:
|
|
146
482
|
* GET /v1/health — liveness + spec version (unauthenticated)
|
|
147
483
|
* POST /v1/rpc/{binding}/{name}/{method} — DO RPC (bearer auth)
|
|
484
|
+
* POST /v1/index/{domain}/{method} — cohesive index operation
|
|
485
|
+
* GET /v1/streams/{name}/chunks — bounded binary long-poll read
|
|
486
|
+
* POST /v1/streams/{name}/chunks — bounded binary batch append
|
|
148
487
|
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
* dispatch — everything else (including fetch/alarm) is unreachable.
|
|
488
|
+
* Generic RPC bodies use the tagged JSON codec. Stream chunk bodies use the
|
|
489
|
+
* compact binary stream protocol. Only whitelisted routes/methods dispatch.
|
|
152
490
|
*/
|
|
153
491
|
const WORLD_NAME = "world-celld";
|
|
154
492
|
const WORLD_VERSION = "0.1.0";
|
|
493
|
+
function workflowIndex(env) {
|
|
494
|
+
return createWorkflowIndex({
|
|
495
|
+
runCatalog: env.WORKFLOW_RUN_CATALOG,
|
|
496
|
+
runFences: env.WORKFLOW_RUN_FENCES,
|
|
497
|
+
hookTokens: env.WORKFLOW_HOOK_TOKENS,
|
|
498
|
+
hookIds: env.WORKFLOW_HOOK_IDS
|
|
499
|
+
});
|
|
500
|
+
}
|
|
155
501
|
const BINDINGS = {
|
|
156
502
|
runs: {
|
|
157
503
|
env: "WORKFLOW_DB",
|
|
@@ -166,16 +512,15 @@ const BINDINGS = {
|
|
|
166
512
|
"getCleanupStatus",
|
|
167
513
|
"scheduleCleanup",
|
|
168
514
|
"cleanupNow",
|
|
169
|
-
"rearmCleanup"
|
|
515
|
+
"rearmCleanup",
|
|
516
|
+
"resolveHookTokenClaim"
|
|
170
517
|
])
|
|
171
518
|
},
|
|
172
519
|
streams: {
|
|
173
520
|
env: "WORKFLOW_STREAMS",
|
|
174
521
|
methods: /* @__PURE__ */ new Set([
|
|
175
|
-
"writeChunk",
|
|
176
522
|
"closeStream",
|
|
177
|
-
"
|
|
178
|
-
"getInfo",
|
|
523
|
+
"failStream",
|
|
179
524
|
"registerStream",
|
|
180
525
|
"listStreams",
|
|
181
526
|
"expireRegistry",
|
|
@@ -183,20 +528,13 @@ const BINDINGS = {
|
|
|
183
528
|
"expireStream"
|
|
184
529
|
])
|
|
185
530
|
},
|
|
186
|
-
|
|
187
|
-
env: "
|
|
188
|
-
methods: /* @__PURE__ */ new Set([
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
"expireRun",
|
|
194
|
-
"list",
|
|
195
|
-
"reserveHookToken",
|
|
196
|
-
"finalizeHookIndexes",
|
|
197
|
-
"releaseHookToken",
|
|
198
|
-
"deleteHookIndexes"
|
|
199
|
-
])
|
|
531
|
+
"hook-tokens": {
|
|
532
|
+
env: "WORKFLOW_HOOK_TOKENS",
|
|
533
|
+
methods: /* @__PURE__ */ new Set(["get"])
|
|
534
|
+
},
|
|
535
|
+
"hook-ids": {
|
|
536
|
+
env: "WORKFLOW_HOOK_IDS",
|
|
537
|
+
methods: /* @__PURE__ */ new Set(["get"])
|
|
200
538
|
},
|
|
201
539
|
queue: {
|
|
202
540
|
env: "WORKFLOW_QUEUE",
|
|
@@ -213,6 +551,7 @@ const BINDINGS = {
|
|
|
213
551
|
};
|
|
214
552
|
/** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
|
|
215
553
|
const MAX_BODY_BYTES = 33554432;
|
|
554
|
+
const MAX_STREAM_WRITE_BODY_BYTES = 8389632;
|
|
216
555
|
function errorResponse(status, name, message) {
|
|
217
556
|
return Response.json({ error: {
|
|
218
557
|
name,
|
|
@@ -241,6 +580,49 @@ async function readBoundedBody(request) {
|
|
|
241
580
|
reader.releaseLock();
|
|
242
581
|
}
|
|
243
582
|
}
|
|
583
|
+
async function readBoundedBytes(request, maxBytes) {
|
|
584
|
+
if (!request.body) return /* @__PURE__ */ new Uint8Array();
|
|
585
|
+
const reader = request.body.getReader();
|
|
586
|
+
const chunks = [];
|
|
587
|
+
let total = 0;
|
|
588
|
+
try {
|
|
589
|
+
for (;;) {
|
|
590
|
+
const { done, value } = await reader.read();
|
|
591
|
+
if (done) break;
|
|
592
|
+
total += value.byteLength;
|
|
593
|
+
if (total > maxBytes) {
|
|
594
|
+
await reader.cancel("stream body exceeds configured limit").catch(() => void 0);
|
|
595
|
+
return null;
|
|
596
|
+
}
|
|
597
|
+
chunks.push(value);
|
|
598
|
+
}
|
|
599
|
+
} finally {
|
|
600
|
+
reader.releaseLock();
|
|
601
|
+
}
|
|
602
|
+
const body = new Uint8Array(total);
|
|
603
|
+
let offset = 0;
|
|
604
|
+
for (const chunk of chunks) {
|
|
605
|
+
body.set(chunk, offset);
|
|
606
|
+
offset += chunk.byteLength;
|
|
607
|
+
}
|
|
608
|
+
return body;
|
|
609
|
+
}
|
|
610
|
+
function parseBoundedInteger(url, name, minimum, maximum) {
|
|
611
|
+
const raw = url.searchParams.get(name);
|
|
612
|
+
const value = raw === null ? NaN : Number(raw);
|
|
613
|
+
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
|
614
|
+
const error = /* @__PURE__ */ new Error(`${name} must be an integer between ${minimum} and ${maximum}`);
|
|
615
|
+
error.name = "StreamProtocolError";
|
|
616
|
+
throw error;
|
|
617
|
+
}
|
|
618
|
+
return value;
|
|
619
|
+
}
|
|
620
|
+
function streamResponse(body) {
|
|
621
|
+
return new Response(body, {
|
|
622
|
+
status: 200,
|
|
623
|
+
headers: { "content-type": STREAM_BATCH_CONTENT_TYPE }
|
|
624
|
+
});
|
|
625
|
+
}
|
|
244
626
|
function createRouter(env) {
|
|
245
627
|
return async (request) => {
|
|
246
628
|
const url = new URL(request.url);
|
|
@@ -252,9 +634,79 @@ function createRouter(env) {
|
|
|
252
634
|
version: WORLD_VERSION,
|
|
253
635
|
specVersion: SPEC_VERSION_CURRENT
|
|
254
636
|
});
|
|
255
|
-
if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
|
|
256
637
|
const auth = await authenticate(request, env.WORLD_SECRET);
|
|
257
638
|
if (!auth.ok) return auth.response;
|
|
639
|
+
if (parts[1] === "index" && parts.length === 4) {
|
|
640
|
+
if (request.method !== "POST") return errorResponse(405, "MethodNotAllowed", "expected POST");
|
|
641
|
+
if (Number(request.headers.get("content-length") ?? "0") > MAX_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
642
|
+
let args;
|
|
643
|
+
try {
|
|
644
|
+
const text = await readBoundedBody(request);
|
|
645
|
+
if (text === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
646
|
+
const parsed = rpcParse(text);
|
|
647
|
+
if (!Array.isArray(parsed)) return errorResponse(400, "BadRequest", "body must be an argument array");
|
|
648
|
+
args = parsed;
|
|
649
|
+
} catch {
|
|
650
|
+
return errorResponse(400, "BadRequest", "malformed rpc body");
|
|
651
|
+
}
|
|
652
|
+
try {
|
|
653
|
+
const index = workflowIndex(env);
|
|
654
|
+
let result;
|
|
655
|
+
const operation = `${parts[2]}.${parts[3]}`;
|
|
656
|
+
if (operation === "runs.list") result = await index.listRuns(args[0]);
|
|
657
|
+
else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1]);
|
|
658
|
+
else if (operation === "runs.expire") result = await index.expireRun(args[0]);
|
|
659
|
+
else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
|
|
660
|
+
else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
|
|
661
|
+
else if (operation === "hooks.release-reservation") result = await index.releaseHookReservation(args[0], args[1], args[2]);
|
|
662
|
+
else if (operation === "hooks.release") result = await index.releaseHookIndexes(args[0]);
|
|
663
|
+
else return errorResponse(404, "NotFound", `unknown index operation: ${operation}`);
|
|
664
|
+
return new Response(rpcStringify(result ?? null), {
|
|
665
|
+
status: 200,
|
|
666
|
+
headers: { "content-type": "application/json" }
|
|
667
|
+
});
|
|
668
|
+
} catch (error) {
|
|
669
|
+
const err = error;
|
|
670
|
+
return Response.json({ error: {
|
|
671
|
+
name: err?.name ?? "Error",
|
|
672
|
+
message: err?.message ?? String(error),
|
|
673
|
+
status: typeof err?.status === "number" ? err.status : void 0
|
|
674
|
+
} }, { status: 500 });
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
if (parts[1] === "streams" && parts.length === 4 && parts[3] === "chunks") {
|
|
678
|
+
const namespace = env.WORKFLOW_STREAMS;
|
|
679
|
+
if (!namespace) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_STREAMS");
|
|
680
|
+
const name = decodeURIComponent(parts[2]);
|
|
681
|
+
const runId = url.searchParams.get("runId");
|
|
682
|
+
if (!runId) return errorResponse(400, "BadRequest", "runId is required");
|
|
683
|
+
const stub = namespace.get(namespace.idFromName(name));
|
|
684
|
+
try {
|
|
685
|
+
if (request.method === "POST") {
|
|
686
|
+
if (!request.headers.get("content-type")?.startsWith("application/vnd.world-celld.stream-batch.v1")) return errorResponse(415, "UnsupportedMediaType", STREAM_BATCH_CONTENT_TYPE);
|
|
687
|
+
if (Number(request.headers.get("content-length") ?? "0") > MAX_STREAM_WRITE_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
|
|
688
|
+
const body = await readBoundedBytes(request, MAX_STREAM_WRITE_BODY_BYTES);
|
|
689
|
+
if (body === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
|
|
690
|
+
return streamResponse(encodeStreamWriteResult(await stub.writeChunks(runId, decodeStreamWriteBatch(body))));
|
|
691
|
+
}
|
|
692
|
+
if (request.method === "GET") return streamResponse(encodeStreamReadResult(await stub.readChunks({
|
|
693
|
+
runId,
|
|
694
|
+
startIndex: parseBoundedInteger(url, "startIndex", 0, 2147483647),
|
|
695
|
+
maxChunks: parseBoundedInteger(url, "maxChunks", 0, 32),
|
|
696
|
+
maxBytes: parseBoundedInteger(url, "maxBytes", MAX_STREAM_CHUNK_BYTES, MAX_STREAM_READ_BYTES),
|
|
697
|
+
waitMs: parseBoundedInteger(url, "waitMs", 0, MAX_STREAM_LONG_POLL_MS)
|
|
698
|
+
}, request.signal)));
|
|
699
|
+
return errorResponse(405, "MethodNotAllowed", "expected GET or POST");
|
|
700
|
+
} catch (error) {
|
|
701
|
+
const err = error;
|
|
702
|
+
return Response.json({ error: {
|
|
703
|
+
name: err?.name ?? "Error",
|
|
704
|
+
message: err?.message ?? String(error),
|
|
705
|
+
status: typeof err?.status === "number" ? err.status : void 0
|
|
706
|
+
} }, { status: err?.name === "StreamProtocolError" ? 400 : 500 });
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
|
|
258
710
|
if (request.method !== "POST" || parts.length !== 5) return errorResponse(404, "NotFound", "expected POST /v1/rpc/{binding}/{name}/{method}");
|
|
259
711
|
const [, , bindingKey, encodedName, method] = parts;
|
|
260
712
|
const binding = BINDINGS[bindingKey];
|
|
@@ -299,160 +751,242 @@ function compact(obj) {
|
|
|
299
751
|
return value;
|
|
300
752
|
}
|
|
301
753
|
//#endregion
|
|
302
|
-
//#region src/worker/durable-objects/
|
|
754
|
+
//#region src/worker/durable-objects/hook-index-shared.ts
|
|
303
755
|
function sameOwner(left, right) {
|
|
304
756
|
return left.runId === right.runId && left.hookId === right.hookId;
|
|
305
757
|
}
|
|
306
|
-
function
|
|
758
|
+
function ownerFromSerializedHook(raw) {
|
|
307
759
|
const hook = parse(raw);
|
|
308
760
|
return {
|
|
309
761
|
runId: hook.runId,
|
|
310
762
|
hookId: hook.hookId
|
|
311
763
|
};
|
|
312
764
|
}
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
765
|
+
function indexKey(prefix, value) {
|
|
766
|
+
return `${prefix}:${encodeURIComponent(value)}`;
|
|
767
|
+
}
|
|
768
|
+
/** Compact shard-local fence for every hook owned by one terminal or expired run. */
|
|
769
|
+
function shardRunFenceKey(runId) {
|
|
770
|
+
return `runfence:${encodeURIComponent(runId)}`;
|
|
771
|
+
}
|
|
772
|
+
async function runIsActive(env, runId) {
|
|
773
|
+
const namespace = env.WORKFLOW_RUN_FENCES;
|
|
774
|
+
if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_RUN_FENCES binding");
|
|
775
|
+
return await namespace.get(namespace.idFromName(runFenceCellName(runId))).getStatus() === "active";
|
|
776
|
+
}
|
|
777
|
+
//#endregion
|
|
778
|
+
//#region src/worker/durable-objects/HookIdDO.ts
|
|
779
|
+
const MAX_RELEASE_BATCH$1 = 64;
|
|
780
|
+
function recordKey$1(hookId) {
|
|
781
|
+
return indexKey("hookid", hookId);
|
|
782
|
+
}
|
|
783
|
+
function claimKey$1(hookId) {
|
|
784
|
+
return indexKey("claim", hookId);
|
|
785
|
+
}
|
|
786
|
+
/** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
|
|
787
|
+
var HookIdDO = class extends DurableObject {
|
|
788
|
+
async get(hookId) {
|
|
789
|
+
const raw = await this.ctx.storage.get(recordKey$1(hookId));
|
|
790
|
+
if (!raw) return null;
|
|
791
|
+
const owner = ownerFromSerializedHook(raw);
|
|
792
|
+
return await runIsActive(this.env, owner.runId) ? raw : null;
|
|
793
|
+
}
|
|
794
|
+
async reserve(hookId, owner, proposedClaimId) {
|
|
795
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
796
|
+
const record = recordKey$1(hookId);
|
|
797
|
+
const claim = claimKey$1(hookId);
|
|
798
|
+
const values = await txn.get([record, claim]);
|
|
799
|
+
const indexed = values.get(record);
|
|
800
|
+
if (indexed !== void 0) return {
|
|
801
|
+
claimed: false,
|
|
802
|
+
holder: ownerFromSerializedHook(indexed)
|
|
803
|
+
};
|
|
804
|
+
const existing = values.get(claim);
|
|
805
|
+
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
806
|
+
claimed: false,
|
|
807
|
+
holder: existing.owner
|
|
808
|
+
};
|
|
809
|
+
if (existing === void 0 || existing.claimId !== proposedClaimId) {
|
|
810
|
+
await txn.put(claim, {
|
|
811
|
+
owner,
|
|
812
|
+
claimId: proposedClaimId,
|
|
813
|
+
reservedAt: Date.now()
|
|
814
|
+
});
|
|
815
|
+
return {
|
|
816
|
+
claimed: true,
|
|
817
|
+
claimId: proposedClaimId
|
|
818
|
+
};
|
|
819
|
+
}
|
|
820
|
+
return {
|
|
821
|
+
claimed: true,
|
|
822
|
+
claimId: existing.claimId
|
|
823
|
+
};
|
|
824
|
+
});
|
|
332
825
|
}
|
|
333
|
-
|
|
334
|
-
|
|
826
|
+
async publish(hookId, serializedHook, owner, claimId) {
|
|
827
|
+
if (!await runIsActive(this.env, owner.runId)) {
|
|
828
|
+
await this.releaseBatch([{
|
|
829
|
+
hookId,
|
|
830
|
+
owner
|
|
831
|
+
}], Date.now(), true);
|
|
832
|
+
return { stored: false };
|
|
833
|
+
}
|
|
335
834
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
336
|
-
|
|
337
|
-
|
|
835
|
+
const record = recordKey$1(hookId);
|
|
836
|
+
const claim = claimKey$1(hookId);
|
|
837
|
+
const runFence = shardRunFenceKey(owner.runId);
|
|
838
|
+
const values = await txn.get([
|
|
839
|
+
record,
|
|
840
|
+
claim,
|
|
841
|
+
runFence
|
|
842
|
+
]);
|
|
843
|
+
if (values.has(runFence)) {
|
|
844
|
+
const existingClaim = values.get(claim);
|
|
845
|
+
if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
|
|
846
|
+
return { stored: false };
|
|
847
|
+
}
|
|
848
|
+
const indexed = values.get(record);
|
|
849
|
+
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
|
|
850
|
+
const existingClaim = values.get(claim);
|
|
851
|
+
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
|
|
852
|
+
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
|
|
853
|
+
await txn.put(record, serializedHook);
|
|
854
|
+
if (existingClaim !== void 0) await txn.delete(claim);
|
|
338
855
|
return { stored: true };
|
|
339
856
|
});
|
|
340
857
|
}
|
|
341
|
-
async
|
|
342
|
-
await this.ctx.storage.
|
|
858
|
+
async releaseClaim(hookId, owner, claimId) {
|
|
859
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
860
|
+
const key = claimKey$1(hookId);
|
|
861
|
+
const claim = await txn.get(key);
|
|
862
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
|
|
863
|
+
});
|
|
343
864
|
}
|
|
344
|
-
async
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
865
|
+
async releaseBatch(hooks, fencedAt, runFenced) {
|
|
866
|
+
if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
|
|
867
|
+
if (hooks.length === 0) return { deleted: 0 };
|
|
868
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
869
|
+
const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
|
|
870
|
+
const values = await txn.get(keys);
|
|
871
|
+
const deletes = [];
|
|
872
|
+
for (const { hookId, owner } of hooks) {
|
|
873
|
+
const record = recordKey$1(hookId);
|
|
874
|
+
const claim = claimKey$1(hookId);
|
|
875
|
+
const indexed = values.get(record);
|
|
876
|
+
const existingClaim = values.get(claim);
|
|
877
|
+
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
878
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
|
|
879
|
+
}
|
|
880
|
+
if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
|
|
881
|
+
return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
|
|
354
882
|
});
|
|
355
|
-
const names = Array.from(entries.keys());
|
|
356
|
-
const page = names.slice(0, limit);
|
|
357
|
-
const listComplete = names.length <= limit;
|
|
358
|
-
return {
|
|
359
|
-
keys: page.map((name) => ({ name })),
|
|
360
|
-
list_complete: listComplete,
|
|
361
|
-
cursor: listComplete ? void 0 : page.at(-1)
|
|
362
|
-
};
|
|
363
883
|
}
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
884
|
+
};
|
|
885
|
+
//#endregion
|
|
886
|
+
//#region src/worker/durable-objects/HookTokenDO.ts
|
|
887
|
+
const MAX_RELEASE_BATCH = 64;
|
|
888
|
+
function recordKey(token) {
|
|
889
|
+
return indexKey("hook", token);
|
|
890
|
+
}
|
|
891
|
+
function claimKey(token) {
|
|
892
|
+
return indexKey("claim", token);
|
|
893
|
+
}
|
|
894
|
+
/** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
|
|
895
|
+
var HookTokenDO = class extends DurableObject {
|
|
896
|
+
async get(token) {
|
|
897
|
+
const raw = await this.ctx.storage.get(recordKey(token));
|
|
898
|
+
if (!raw) return null;
|
|
899
|
+
const owner = ownerFromSerializedHook(raw);
|
|
900
|
+
return await runIsActive(this.env, owner.runId) ? raw : null;
|
|
901
|
+
}
|
|
902
|
+
async reserve(token, owner, proposedClaimId) {
|
|
903
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
904
|
+
const record = recordKey(token);
|
|
905
|
+
const claim = claimKey(token);
|
|
906
|
+
const values = await txn.get([record, claim]);
|
|
907
|
+
const indexed = values.get(record);
|
|
908
|
+
if (indexed !== void 0) return {
|
|
368
909
|
claimed: false,
|
|
369
|
-
holder:
|
|
910
|
+
holder: ownerFromSerializedHook(indexed)
|
|
370
911
|
};
|
|
371
|
-
const
|
|
372
|
-
|
|
373
|
-
if (claim !== void 0 && !sameOwner(claim.owner, owner)) return {
|
|
912
|
+
const existing = values.get(claim);
|
|
913
|
+
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
374
914
|
claimed: false,
|
|
375
|
-
holder:
|
|
915
|
+
holder: existing.owner
|
|
916
|
+
};
|
|
917
|
+
if (existing === void 0) {
|
|
918
|
+
await txn.put(claim, {
|
|
919
|
+
owner,
|
|
920
|
+
claimId: proposedClaimId,
|
|
921
|
+
reservedAt: Date.now()
|
|
922
|
+
});
|
|
923
|
+
return {
|
|
924
|
+
claimed: true,
|
|
925
|
+
claimId: proposedClaimId
|
|
926
|
+
};
|
|
927
|
+
}
|
|
928
|
+
return {
|
|
929
|
+
claimed: true,
|
|
930
|
+
claimId: existing.claimId
|
|
376
931
|
};
|
|
377
|
-
await txn.put(claimKey, {
|
|
378
|
-
owner,
|
|
379
|
-
reservedAt: Date.now()
|
|
380
|
-
});
|
|
381
|
-
return { claimed: true };
|
|
382
932
|
});
|
|
383
933
|
}
|
|
384
|
-
async
|
|
385
|
-
await this.
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
934
|
+
async finalize(token, serializedHook, owner, claimId) {
|
|
935
|
+
if (!await runIsActive(this.env, owner.runId)) {
|
|
936
|
+
await this.releaseBatch([{
|
|
937
|
+
token,
|
|
938
|
+
owner
|
|
939
|
+
}], Date.now(), true);
|
|
940
|
+
return { stored: false };
|
|
941
|
+
}
|
|
942
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
943
|
+
const record = recordKey(token);
|
|
944
|
+
const claim = claimKey(token);
|
|
945
|
+
const runFence = shardRunFenceKey(owner.runId);
|
|
946
|
+
const values = await txn.get([
|
|
947
|
+
record,
|
|
948
|
+
claim,
|
|
949
|
+
runFence
|
|
950
|
+
]);
|
|
951
|
+
if (values.has(runFence)) {
|
|
952
|
+
const existingClaim = values.get(claim);
|
|
953
|
+
if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
|
|
954
|
+
return { stored: false };
|
|
390
955
|
}
|
|
391
|
-
const
|
|
392
|
-
if (
|
|
393
|
-
const
|
|
394
|
-
if (
|
|
395
|
-
|
|
396
|
-
await txn.put(
|
|
397
|
-
if (
|
|
956
|
+
const indexed = values.get(record);
|
|
957
|
+
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
|
|
958
|
+
const existingClaim = values.get(claim);
|
|
959
|
+
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
|
|
960
|
+
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
|
|
961
|
+
await txn.put(record, serializedHook);
|
|
962
|
+
if (existingClaim !== void 0) await txn.delete(claim);
|
|
963
|
+
return { stored: true };
|
|
398
964
|
});
|
|
399
965
|
}
|
|
400
|
-
async
|
|
966
|
+
async releaseClaim(token, owner, claimId) {
|
|
401
967
|
await this.ctx.storage.transaction(async (txn) => {
|
|
402
|
-
const key =
|
|
968
|
+
const key = claimKey(token);
|
|
403
969
|
const claim = await txn.get(key);
|
|
404
|
-
if (claim
|
|
405
|
-
});
|
|
406
|
-
}
|
|
407
|
-
async deleteHookIndexes(token, hookId, owner) {
|
|
408
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
409
|
-
const tokenKey = `hook:${token}`;
|
|
410
|
-
const idKey = `hookid:${hookId}`;
|
|
411
|
-
const claimKey = `hookclaim:${token}`;
|
|
412
|
-
const [byToken, byId, claim] = await Promise.all([
|
|
413
|
-
txn.get(tokenKey),
|
|
414
|
-
txn.get(idKey),
|
|
415
|
-
txn.get(claimKey)
|
|
416
|
-
]);
|
|
417
|
-
if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) await txn.delete(tokenKey);
|
|
418
|
-
if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) await txn.delete(idKey);
|
|
419
|
-
if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(claimKey);
|
|
970
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
|
|
420
971
|
});
|
|
421
972
|
}
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
* entries from a delayed request.
|
|
426
|
-
*/
|
|
427
|
-
async expireRun(request) {
|
|
973
|
+
async releaseBatch(hooks, fencedAt, runFenced) {
|
|
974
|
+
if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
|
|
975
|
+
if (hooks.length === 0) return { deleted: 0 };
|
|
428
976
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
for (const
|
|
433
|
-
const
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
const claimKey = `hookclaim:${hook.token}`;
|
|
440
|
-
const [byToken, byId, claim] = await Promise.all([
|
|
441
|
-
txn.get(tokenKey),
|
|
442
|
-
txn.get(idKey),
|
|
443
|
-
txn.get(claimKey)
|
|
444
|
-
]);
|
|
445
|
-
if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) {
|
|
446
|
-
if (await txn.delete(tokenKey)) deleted++;
|
|
447
|
-
}
|
|
448
|
-
if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) {
|
|
449
|
-
if (await txn.delete(idKey)) deleted++;
|
|
450
|
-
}
|
|
451
|
-
if (claim !== void 0 && sameOwner(claim.owner, owner)) {
|
|
452
|
-
if (await txn.delete(claimKey)) deleted++;
|
|
453
|
-
}
|
|
977
|
+
const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
|
|
978
|
+
const values = await txn.get(keys);
|
|
979
|
+
const deletes = [];
|
|
980
|
+
for (const { token, owner } of hooks) {
|
|
981
|
+
const record = recordKey(token);
|
|
982
|
+
const claim = claimKey(token);
|
|
983
|
+
const indexed = values.get(record);
|
|
984
|
+
const existingClaim = values.get(claim);
|
|
985
|
+
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
986
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
|
|
454
987
|
}
|
|
455
|
-
|
|
988
|
+
if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
|
|
989
|
+
return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
|
|
456
990
|
});
|
|
457
991
|
}
|
|
458
992
|
};
|
|
@@ -471,21 +1005,61 @@ const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
|
|
|
471
1005
|
const INFLIGHT_GRACE_MS = 3e4;
|
|
472
1006
|
/** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
|
|
473
1007
|
const MIN_ALARM_DELAY_MS = 1;
|
|
1008
|
+
/** Bound expiry recovery so one alarm cannot monopolize the cell. */
|
|
1009
|
+
const EXPIRED_INFLIGHT_BATCH = 128;
|
|
1010
|
+
const STORAGE_BATCH_SIZE$2 = 128;
|
|
1011
|
+
const EXPIRE_RUN_REFERENCE_BATCH = 64;
|
|
1012
|
+
const PURGE_DEAD_LETTER_BATCH = 128;
|
|
1013
|
+
const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
|
|
474
1014
|
function pad(ms) {
|
|
475
1015
|
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
476
1016
|
}
|
|
477
1017
|
function dueKey(atMs, messageId) {
|
|
478
1018
|
return `due:${pad(atMs)}:${messageId}`;
|
|
479
1019
|
}
|
|
1020
|
+
function inflightKey(deadlineMs, messageId) {
|
|
1021
|
+
return `${INFLIGHT_DEADLINE_PREFIX}${pad(deadlineMs)}:${messageId}`;
|
|
1022
|
+
}
|
|
1023
|
+
function deadlineFromInflightKey(key) {
|
|
1024
|
+
return Number.parseInt(key.slice(18, 31), 10);
|
|
1025
|
+
}
|
|
480
1026
|
function runReferenceKey(runId, messageId) {
|
|
481
1027
|
return `run:${runId}:${messageId}`;
|
|
482
1028
|
}
|
|
483
1029
|
function expiredRunKey(runId) {
|
|
484
1030
|
return `expired-run:${runId}`;
|
|
485
1031
|
}
|
|
1032
|
+
function boundedLimit$1(value, fallback, maximum) {
|
|
1033
|
+
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
1034
|
+
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
1035
|
+
}
|
|
1036
|
+
async function getMany$1(storage, keys) {
|
|
1037
|
+
const result = /* @__PURE__ */ new Map();
|
|
1038
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$2) {
|
|
1039
|
+
const page = await storage.get(keys.slice(offset, offset + STORAGE_BATCH_SIZE$2));
|
|
1040
|
+
for (const [key, value] of page) result.set(key, value);
|
|
1041
|
+
}
|
|
1042
|
+
return result;
|
|
1043
|
+
}
|
|
1044
|
+
async function putMany(storage, entries) {
|
|
1045
|
+
const all = Array.from(entries);
|
|
1046
|
+
for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE$2) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE$2)));
|
|
1047
|
+
}
|
|
1048
|
+
async function deleteMany$1(storage, keys) {
|
|
1049
|
+
let deleted = 0;
|
|
1050
|
+
const unique = Array.from(new Set(keys));
|
|
1051
|
+
for (let offset = 0; offset < unique.length; offset += STORAGE_BATCH_SIZE$2) deleted += await storage.delete(unique.slice(offset, offset + STORAGE_BATCH_SIZE$2));
|
|
1052
|
+
return deleted;
|
|
1053
|
+
}
|
|
486
1054
|
function backoffSeconds(attempt) {
|
|
487
1055
|
return Math.min(60, 2 ** attempt);
|
|
488
1056
|
}
|
|
1057
|
+
async function cancelUnusedResponseBody(response) {
|
|
1058
|
+
try {
|
|
1059
|
+
await response.body?.cancel();
|
|
1060
|
+
} catch {}
|
|
1061
|
+
}
|
|
1062
|
+
const MAX_QUEUE_INFLIGHT = 128;
|
|
489
1063
|
var QueueDO = class extends DurableObject {
|
|
490
1064
|
#now() {
|
|
491
1065
|
const clock = this.env?.clock;
|
|
@@ -500,7 +1074,13 @@ var QueueDO = class extends DurableObject {
|
|
|
500
1074
|
const parsed = raw ? Number.parseInt(raw, 10) : NaN;
|
|
501
1075
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
502
1076
|
}
|
|
1077
|
+
#maxInflight() {
|
|
1078
|
+
const configured = this.#intVar("QUEUE_MAX_INFLIGHT", DEFAULT_MAX_INFLIGHT);
|
|
1079
|
+
if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
|
|
1080
|
+
return configured;
|
|
1081
|
+
}
|
|
503
1082
|
async enqueue(request) {
|
|
1083
|
+
this.#maxInflight();
|
|
504
1084
|
const storage = this.ctx.storage;
|
|
505
1085
|
const now = this.#now();
|
|
506
1086
|
return await storage.transaction(async (txn) => {
|
|
@@ -575,7 +1155,7 @@ var QueueDO = class extends DurableObject {
|
|
|
575
1155
|
if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
|
|
576
1156
|
else await attempt();
|
|
577
1157
|
if (failure !== null) throw failure;
|
|
578
|
-
for (const
|
|
1158
|
+
for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
|
|
579
1159
|
}
|
|
580
1160
|
/**
|
|
581
1161
|
* Storage-only: recover lost inflight claims, claim due messages up to the
|
|
@@ -584,24 +1164,38 @@ var QueueDO = class extends DurableObject {
|
|
|
584
1164
|
async #claimPhase() {
|
|
585
1165
|
const storage = this.ctx.storage;
|
|
586
1166
|
const now = this.#now();
|
|
587
|
-
const maxInflight = this.#
|
|
1167
|
+
const maxInflight = this.#maxInflight();
|
|
588
1168
|
return await storage.transaction(async (txn) => {
|
|
589
|
-
const
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
1169
|
+
const expiredPage = await txn.list({
|
|
1170
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1171
|
+
end: `${INFLIGHT_DEADLINE_PREFIX}${pad(now + 1)}`,
|
|
1172
|
+
limit: 129
|
|
1173
|
+
});
|
|
1174
|
+
const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
|
|
1175
|
+
const expiredRows = await getMany$1(txn, expired.map(([, messageId]) => `msg:${messageId}`));
|
|
1176
|
+
await deleteMany$1(txn, expired.map(([key]) => key));
|
|
1177
|
+
const recoveryWrites = [];
|
|
1178
|
+
for (const [, messageId] of expired) {
|
|
1179
|
+
const row = expiredRows.get(`msg:${messageId}`);
|
|
595
1180
|
if (row) {
|
|
596
1181
|
const scheduleKey = dueKey(now, messageId);
|
|
597
|
-
|
|
598
|
-
if (row.runId)
|
|
1182
|
+
recoveryWrites.push([scheduleKey, messageId]);
|
|
1183
|
+
if (row.runId) recoveryWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
599
1184
|
messageId: row.messageId,
|
|
600
1185
|
dueKey: scheduleKey,
|
|
601
1186
|
idempotencyKey: row.idempotencyKey
|
|
602
|
-
});
|
|
1187
|
+
}]);
|
|
603
1188
|
}
|
|
604
|
-
}
|
|
1189
|
+
}
|
|
1190
|
+
await putMany(txn, recoveryWrites);
|
|
1191
|
+
if (expiredPage.size > EXPIRED_INFLIGHT_BATCH) {
|
|
1192
|
+
await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
|
|
1193
|
+
return [];
|
|
1194
|
+
}
|
|
1195
|
+
const inflightCount = (await txn.list({
|
|
1196
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1197
|
+
limit: maxInflight
|
|
1198
|
+
})).size;
|
|
605
1199
|
const rows = [];
|
|
606
1200
|
if (inflightCount < maxInflight) {
|
|
607
1201
|
const due = await txn.list({
|
|
@@ -610,19 +1204,31 @@ var QueueDO = class extends DurableObject {
|
|
|
610
1204
|
limit: maxInflight - inflightCount
|
|
611
1205
|
});
|
|
612
1206
|
const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
1207
|
+
const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
|
|
1208
|
+
await deleteMany$1(txn, Array.from(due.keys()));
|
|
1209
|
+
const claimWrites = [];
|
|
1210
|
+
for (const [, messageId] of due) {
|
|
1211
|
+
const row = dueRows.get(`msg:${messageId}`);
|
|
616
1212
|
if (!row) continue;
|
|
617
|
-
|
|
618
|
-
|
|
1213
|
+
const claimKey = inflightKey(now + timeoutMs + INFLIGHT_GRACE_MS, messageId);
|
|
1214
|
+
claimWrites.push([claimKey, messageId]);
|
|
1215
|
+
if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
1216
|
+
messageId: row.messageId,
|
|
1217
|
+
inflightKey: claimKey,
|
|
1218
|
+
idempotencyKey: row.idempotencyKey
|
|
1219
|
+
}]);
|
|
1220
|
+
rows.push({
|
|
1221
|
+
row,
|
|
1222
|
+
inflightKey: claimKey
|
|
1223
|
+
});
|
|
619
1224
|
}
|
|
1225
|
+
await putMany(txn, claimWrites);
|
|
620
1226
|
}
|
|
621
1227
|
await this.#scheduleNextAlarm(txn, now);
|
|
622
1228
|
return rows;
|
|
623
1229
|
});
|
|
624
1230
|
}
|
|
625
|
-
async #deliver(row) {
|
|
1231
|
+
async #deliver({ row, inflightKey: claimKey }) {
|
|
626
1232
|
const storage = this.ctx.storage;
|
|
627
1233
|
const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
628
1234
|
const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
|
|
@@ -646,8 +1252,10 @@ var QueueDO = class extends DurableObject {
|
|
|
646
1252
|
transportError = error;
|
|
647
1253
|
}
|
|
648
1254
|
const now = this.#now();
|
|
649
|
-
if (response?.ok)
|
|
650
|
-
|
|
1255
|
+
if (response?.ok) {
|
|
1256
|
+
await cancelUnusedResponseBody(response);
|
|
1257
|
+
await this.#ack(row, claimKey);
|
|
1258
|
+
} else if (response && response.status === 503) {
|
|
651
1259
|
let timeoutSeconds;
|
|
652
1260
|
try {
|
|
653
1261
|
timeoutSeconds = (await response.json())?.timeoutSeconds;
|
|
@@ -655,9 +1263,9 @@ var QueueDO = class extends DurableObject {
|
|
|
655
1263
|
timeoutSeconds = void 0;
|
|
656
1264
|
}
|
|
657
1265
|
if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
|
|
1266
|
+
if (!await txn.delete(claimKey)) return;
|
|
658
1267
|
if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
|
|
659
1268
|
await txn.delete(`msg:${row.messageId}`);
|
|
660
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
661
1269
|
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
662
1270
|
if (row.idempotencyKey) {
|
|
663
1271
|
const dedupKey = `key:${row.idempotencyKey}`;
|
|
@@ -666,7 +1274,6 @@ var QueueDO = class extends DurableObject {
|
|
|
666
1274
|
await this.#scheduleNextAlarm(txn, now);
|
|
667
1275
|
return;
|
|
668
1276
|
}
|
|
669
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
670
1277
|
const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
|
|
671
1278
|
await txn.put(scheduleKey, row.messageId);
|
|
672
1279
|
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
@@ -676,17 +1283,20 @@ var QueueDO = class extends DurableObject {
|
|
|
676
1283
|
});
|
|
677
1284
|
await this.#scheduleNextAlarm(txn, now);
|
|
678
1285
|
});
|
|
679
|
-
else await this.#retry(row, "HTTP 503");
|
|
680
|
-
} else if (response && PERMANENT_STATUSES.has(response.status))
|
|
681
|
-
|
|
1286
|
+
else await this.#retry(row, claimKey, "HTTP 503");
|
|
1287
|
+
} else if (response && PERMANENT_STATUSES.has(response.status)) {
|
|
1288
|
+
await cancelUnusedResponseBody(response);
|
|
1289
|
+
await this.#ack(row, claimKey);
|
|
1290
|
+
} else {
|
|
682
1291
|
const reason = response ? `HTTP ${response.status}` : `transport error: ${String(transportError)}`;
|
|
683
|
-
await
|
|
1292
|
+
if (response) await cancelUnusedResponseBody(response);
|
|
1293
|
+
await this.#retry(row, claimKey, reason);
|
|
684
1294
|
}
|
|
685
1295
|
}
|
|
686
|
-
async #ack(row) {
|
|
1296
|
+
async #ack(row, claimKey) {
|
|
687
1297
|
await this.ctx.storage.transaction(async (txn) => {
|
|
1298
|
+
if (!await txn.delete(claimKey)) return;
|
|
688
1299
|
await txn.delete(`msg:${row.messageId}`);
|
|
689
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
690
1300
|
if (row.idempotencyKey) {
|
|
691
1301
|
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
692
1302
|
}
|
|
@@ -694,15 +1304,15 @@ var QueueDO = class extends DurableObject {
|
|
|
694
1304
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
695
1305
|
});
|
|
696
1306
|
}
|
|
697
|
-
async #retry(row, reason) {
|
|
1307
|
+
async #retry(row, claimKey, reason) {
|
|
698
1308
|
const storage = this.ctx.storage;
|
|
699
1309
|
const now = this.#now();
|
|
700
1310
|
const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
701
1311
|
const attempt = row.attempt + 1;
|
|
702
1312
|
await storage.transaction(async (txn) => {
|
|
1313
|
+
if (!await txn.delete(claimKey)) return;
|
|
703
1314
|
if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
|
|
704
1315
|
await txn.delete(`msg:${row.messageId}`);
|
|
705
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
706
1316
|
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
707
1317
|
if (row.idempotencyKey) {
|
|
708
1318
|
const dedupKey = `key:${row.idempotencyKey}`;
|
|
@@ -711,7 +1321,6 @@ var QueueDO = class extends DurableObject {
|
|
|
711
1321
|
await this.#scheduleNextAlarm(txn, now);
|
|
712
1322
|
return;
|
|
713
1323
|
}
|
|
714
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
715
1324
|
if (attempt >= maxAttempts) {
|
|
716
1325
|
const dead = {
|
|
717
1326
|
...row,
|
|
@@ -771,8 +1380,14 @@ var QueueDO = class extends DurableObject {
|
|
|
771
1380
|
limit: 1
|
|
772
1381
|
});
|
|
773
1382
|
for (const key of due.keys()) next = Number.parseInt(key.slice(4, 17), 10);
|
|
774
|
-
const inflight = await storage.list({
|
|
775
|
-
|
|
1383
|
+
const inflight = await storage.list({
|
|
1384
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1385
|
+
limit: 1
|
|
1386
|
+
});
|
|
1387
|
+
for (const key of inflight.keys()) {
|
|
1388
|
+
const deadline = deadlineFromInflightKey(key);
|
|
1389
|
+
if (next === null || deadline < next) next = deadline;
|
|
1390
|
+
}
|
|
776
1391
|
if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
|
|
777
1392
|
else await storage.deleteAlarm();
|
|
778
1393
|
}
|
|
@@ -780,7 +1395,7 @@ var QueueDO = class extends DurableObject {
|
|
|
780
1395
|
const storage = this.ctx.storage;
|
|
781
1396
|
const [due, inflight, dlq, alarmAt] = await Promise.all([
|
|
782
1397
|
storage.list({ prefix: "due:" }),
|
|
783
|
-
storage.list({ prefix:
|
|
1398
|
+
storage.list({ prefix: INFLIGHT_DEADLINE_PREFIX }),
|
|
784
1399
|
storage.list({ prefix: "dlq:" }),
|
|
785
1400
|
storage.getAlarm()
|
|
786
1401
|
]);
|
|
@@ -845,13 +1460,16 @@ var QueueDO = class extends DurableObject {
|
|
|
845
1460
|
}
|
|
846
1461
|
async purgeDeadLetters() {
|
|
847
1462
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
848
|
-
const
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
return {
|
|
1463
|
+
const entries = await txn.list({
|
|
1464
|
+
prefix: "dlq:",
|
|
1465
|
+
limit: 129
|
|
1466
|
+
});
|
|
1467
|
+
const page = Array.from(entries).slice(0, PURGE_DEAD_LETTER_BATCH);
|
|
1468
|
+
await deleteMany$1(txn, page.flatMap(([key, dead]) => [key, ...dead.runId ? [runReferenceKey(dead.runId, dead.messageId)] : []]));
|
|
1469
|
+
return {
|
|
1470
|
+
purged: page.length,
|
|
1471
|
+
hasMore: entries.size > PURGE_DEAD_LETTER_BATCH
|
|
1472
|
+
};
|
|
855
1473
|
});
|
|
856
1474
|
}
|
|
857
1475
|
/**
|
|
@@ -863,140 +1481,384 @@ var QueueDO = class extends DurableObject {
|
|
|
863
1481
|
return { alarmAt: await this.ctx.storage.getAlarm() };
|
|
864
1482
|
}
|
|
865
1483
|
/** Fence a run and purge every live, inflight, and dead-letter message it owns. */
|
|
866
|
-
async expireRun(runId, expiredAt) {
|
|
867
|
-
|
|
1484
|
+
async expireRun(runId, expiredAt, options) {
|
|
1485
|
+
const storage = this.ctx.storage;
|
|
1486
|
+
const limit = boundedLimit$1(options?.limit, EXPIRE_RUN_REFERENCE_BATCH, EXPIRE_RUN_REFERENCE_BATCH);
|
|
1487
|
+
return await storage.transaction(async (txn) => {
|
|
868
1488
|
const existing = await txn.get(expiredRunKey(runId));
|
|
869
|
-
const references = await txn.list({
|
|
1489
|
+
const references = await txn.list({
|
|
1490
|
+
prefix: `run:${runId}:`,
|
|
1491
|
+
limit: limit + 1
|
|
1492
|
+
});
|
|
1493
|
+
const page = Array.from(references).slice(0, limit);
|
|
1494
|
+
const dedupHolders = await getMany$1(txn, page.flatMap(([, reference]) => reference.idempotencyKey ? [`key:${reference.idempotencyKey}`] : []));
|
|
1495
|
+
const keys = [];
|
|
870
1496
|
let count = existing?.deleted ?? 0;
|
|
871
|
-
for (const [key, reference] of
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
if (reference.dueKey)
|
|
875
|
-
if (reference.dlqKey)
|
|
1497
|
+
for (const [key, reference] of page) {
|
|
1498
|
+
keys.push(`msg:${reference.messageId}`, key);
|
|
1499
|
+
if (reference.inflightKey) keys.push(reference.inflightKey);
|
|
1500
|
+
if (reference.dueKey) keys.push(reference.dueKey);
|
|
1501
|
+
if (reference.dlqKey) keys.push(reference.dlqKey);
|
|
876
1502
|
if (reference.idempotencyKey) {
|
|
877
1503
|
const dedupKey = `key:${reference.idempotencyKey}`;
|
|
878
|
-
if (
|
|
1504
|
+
if (dedupHolders.get(dedupKey) === reference.messageId) keys.push(dedupKey);
|
|
879
1505
|
}
|
|
880
|
-
await txn.delete(key);
|
|
881
1506
|
count++;
|
|
882
1507
|
}
|
|
1508
|
+
await deleteMany$1(txn, keys);
|
|
883
1509
|
await txn.put(expiredRunKey(runId), {
|
|
884
1510
|
expiredAt,
|
|
885
1511
|
deleted: count
|
|
886
1512
|
});
|
|
887
1513
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
888
|
-
return
|
|
889
|
-
|
|
1514
|
+
return {
|
|
1515
|
+
deleted: count,
|
|
1516
|
+
done: references.size <= limit
|
|
1517
|
+
};
|
|
1518
|
+
});
|
|
1519
|
+
}
|
|
1520
|
+
};
|
|
1521
|
+
//#endregion
|
|
1522
|
+
//#region src/worker/durable-objects/RunCatalogDO.ts
|
|
1523
|
+
const MAX_INDEX_LIST_SIZE = 1001;
|
|
1524
|
+
/** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
|
|
1525
|
+
var RunCatalogDO = class extends DurableObject {
|
|
1526
|
+
async upsertRun(runId, keys, serializedMetadata) {
|
|
1527
|
+
if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
|
|
1528
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1529
|
+
if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
|
|
1530
|
+
await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
|
|
1531
|
+
return { stored: true };
|
|
1532
|
+
});
|
|
1533
|
+
}
|
|
1534
|
+
async list(options = {}) {
|
|
1535
|
+
const suppliedLimit = options.limit;
|
|
1536
|
+
const limit = Math.min(MAX_INDEX_LIST_SIZE, Math.max(1, typeof suppliedLimit === "number" && Number.isFinite(suppliedLimit) ? Math.floor(suppliedLimit) : MAX_INDEX_LIST_SIZE));
|
|
1537
|
+
const entries = await this.ctx.storage.list({
|
|
1538
|
+
prefix: options.prefix ?? "",
|
|
1539
|
+
...options.reverse ? {
|
|
1540
|
+
reverse: true,
|
|
1541
|
+
end: options.cursor
|
|
1542
|
+
} : { startAfter: options.cursor },
|
|
1543
|
+
limit: limit + 1
|
|
1544
|
+
});
|
|
1545
|
+
const page = Array.from(entries).slice(0, limit);
|
|
1546
|
+
const listComplete = entries.size <= limit;
|
|
1547
|
+
return {
|
|
1548
|
+
keys: page.map(([name, value]) => ({
|
|
1549
|
+
name,
|
|
1550
|
+
value
|
|
1551
|
+
})),
|
|
1552
|
+
list_complete: listComplete,
|
|
1553
|
+
cursor: listComplete ? void 0 : page.at(-1)?.[0]
|
|
1554
|
+
};
|
|
1555
|
+
}
|
|
1556
|
+
async expireRun(runId, keys, expiredAt) {
|
|
1557
|
+
if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
|
|
1558
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1559
|
+
await txn.put(`expired:${runId}`, expiredAt);
|
|
1560
|
+
return { deleted: await txn.delete(keys) };
|
|
1561
|
+
});
|
|
1562
|
+
}
|
|
1563
|
+
};
|
|
1564
|
+
//#endregion
|
|
1565
|
+
//#region src/worker/durable-objects/RunFenceDO.ts
|
|
1566
|
+
/** Per-run lifecycle fence used by hook shards without a global lookup cell. */
|
|
1567
|
+
var RunFenceDO = class extends DurableObject {
|
|
1568
|
+
async getStatus() {
|
|
1569
|
+
return (await this.ctx.storage.get("fence"))?.status ?? "active";
|
|
1570
|
+
}
|
|
1571
|
+
async fenceTerminal(at) {
|
|
1572
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1573
|
+
const current = await txn.get("fence");
|
|
1574
|
+
if (current) return current.status;
|
|
1575
|
+
await txn.put("fence", {
|
|
1576
|
+
status: "terminal",
|
|
1577
|
+
at
|
|
1578
|
+
});
|
|
1579
|
+
return "terminal";
|
|
1580
|
+
});
|
|
1581
|
+
}
|
|
1582
|
+
async fenceExpired(at) {
|
|
1583
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1584
|
+
const current = await txn.get("fence");
|
|
1585
|
+
if (current?.status === "expired") return current.status;
|
|
1586
|
+
await txn.put("fence", {
|
|
1587
|
+
status: "expired",
|
|
1588
|
+
at
|
|
1589
|
+
});
|
|
1590
|
+
return "expired";
|
|
1591
|
+
});
|
|
890
1592
|
}
|
|
891
1593
|
};
|
|
892
1594
|
//#endregion
|
|
893
1595
|
//#region src/worker/durable-objects/StreamDO.ts
|
|
894
1596
|
const META_KEY = "meta";
|
|
895
1597
|
const CHUNK_KEY_PREFIX = "chunk:";
|
|
1598
|
+
const CHUNK_SIZE_KEY_PREFIX = "chunk-size:";
|
|
896
1599
|
/** Registry keys used when this DO instance acts as a per-run stream index. */
|
|
897
1600
|
const STREAM_REGISTRY_PREFIX = "stream:";
|
|
898
1601
|
const REGISTRY_OWNER_KEY = "registry:owner";
|
|
899
1602
|
const REGISTRY_EXPIRED_KEY = "registry:expired";
|
|
900
|
-
/**
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
*/
|
|
1603
|
+
/** Each chunk occupies a payload key and a size key; one delete accepts at most 128 keys. */
|
|
1604
|
+
const DEFAULT_EXPIRE_CHUNK_LIMIT = 64;
|
|
1605
|
+
const MAX_EXPIRE_CHUNK_LIMIT = 64;
|
|
1606
|
+
/** Bound storage deletion work as well as item count; one oversized chunk still makes progress. */
|
|
1607
|
+
const DEFAULT_EXPIRE_BYTE_LIMIT = 16777216;
|
|
1608
|
+
const MAX_EXPIRE_BYTE_LIMIT = DEFAULT_EXPIRE_BYTE_LIMIT;
|
|
1609
|
+
function boundedLimit(value, fallback, maximum) {
|
|
1610
|
+
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
1611
|
+
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
1612
|
+
}
|
|
1613
|
+
function emptyMeta() {
|
|
1614
|
+
return {
|
|
1615
|
+
count: 0,
|
|
1616
|
+
state: "open"
|
|
1617
|
+
};
|
|
1618
|
+
}
|
|
1619
|
+
function validateMeta(meta) {
|
|
1620
|
+
if (!Number.isSafeInteger(meta.count) || meta.count < 0 || meta.count > 2147483647 || ![
|
|
1621
|
+
"open",
|
|
1622
|
+
"closed",
|
|
1623
|
+
"errored",
|
|
1624
|
+
"expired"
|
|
1625
|
+
].includes(meta.state)) throw new Error("Invalid persisted stream metadata");
|
|
1626
|
+
if (meta.state === "errored" && !meta.error) throw new Error("Invalid persisted stream error metadata");
|
|
1627
|
+
return meta;
|
|
1628
|
+
}
|
|
1629
|
+
/** Zero-padding keeps storage.list() results in stream offset order. */
|
|
904
1630
|
function chunkKey(index) {
|
|
905
1631
|
return `${CHUNK_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
|
|
906
1632
|
}
|
|
1633
|
+
function chunkSizeKey(index) {
|
|
1634
|
+
return `${CHUNK_SIZE_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
|
|
1635
|
+
}
|
|
1636
|
+
function indexFromChunkSizeKey(key) {
|
|
1637
|
+
const suffix = key.slice(11);
|
|
1638
|
+
const index = Number(suffix);
|
|
1639
|
+
if (!/^\d{12}$/.test(suffix) || !Number.isSafeInteger(index) || index < 0) throw new Error(`Invalid persisted stream chunk size key "${key}"`);
|
|
1640
|
+
return index;
|
|
1641
|
+
}
|
|
1642
|
+
function validateChunkSize(size, index) {
|
|
1643
|
+
if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0 || size > 1048576) throw new Error(`Invalid persisted stream chunk size at index ${index}`);
|
|
1644
|
+
return size;
|
|
1645
|
+
}
|
|
1646
|
+
function compactChunk(chunk) {
|
|
1647
|
+
return chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength ? chunk : new Uint8Array(chunk);
|
|
1648
|
+
}
|
|
1649
|
+
function abortReason(signal) {
|
|
1650
|
+
return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
|
1651
|
+
}
|
|
907
1652
|
/**
|
|
908
1653
|
* Durable Object backing workflow streams.
|
|
909
1654
|
*
|
|
910
|
-
*
|
|
911
|
-
*
|
|
912
|
-
*
|
|
913
|
-
* each chunk its index; one chunk per storage key.
|
|
914
|
-
* - `run-streams:<runId>`: registry of stream names owned by a run,
|
|
915
|
-
* powering `listStreamsByRunId`.
|
|
1655
|
+
* Stream cells persist binary chunks under monotonic offset keys. Public
|
|
1656
|
+
* reads are bounded long polls, and writes append a bounded ordered batch in
|
|
1657
|
+
* one storage transaction. Per-run registry cells retain their existing role.
|
|
916
1658
|
*/
|
|
917
1659
|
var StreamDO = class extends DurableObject {
|
|
1660
|
+
meta;
|
|
1661
|
+
metaLoad;
|
|
1662
|
+
mutationTail = Promise.resolve();
|
|
1663
|
+
changeVersion = 0;
|
|
1664
|
+
waiters = /* @__PURE__ */ new Set();
|
|
918
1665
|
async getMeta() {
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1666
|
+
if (this.meta) return this.meta;
|
|
1667
|
+
this.metaLoad ??= this.ctx.storage.get(META_KEY).then((meta) => {
|
|
1668
|
+
const loaded = meta ? validateMeta(meta) : emptyMeta();
|
|
1669
|
+
this.meta = loaded;
|
|
1670
|
+
return loaded;
|
|
1671
|
+
});
|
|
1672
|
+
try {
|
|
1673
|
+
return await this.metaLoad;
|
|
1674
|
+
} catch (error) {
|
|
1675
|
+
this.metaLoad = void 0;
|
|
1676
|
+
throw error;
|
|
1677
|
+
}
|
|
923
1678
|
}
|
|
924
1679
|
assertOwner(meta, runId) {
|
|
925
|
-
if (
|
|
1680
|
+
if (!runId) throw new Error("Stream runId is required");
|
|
926
1681
|
if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
|
|
927
1682
|
}
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
1683
|
+
assertWritable(meta, runId) {
|
|
1684
|
+
this.assertOwner(meta, runId);
|
|
1685
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
1686
|
+
if (meta.state === "closed") throw new Error("Cannot write to a closed stream");
|
|
1687
|
+
if (meta.state === "errored") throw new Error("Cannot write to an errored stream");
|
|
1688
|
+
}
|
|
1689
|
+
runMutation(operation) {
|
|
1690
|
+
const result = this.mutationTail.then(operation, operation);
|
|
1691
|
+
this.mutationTail = result.then(() => void 0, () => void 0);
|
|
1692
|
+
return result;
|
|
1693
|
+
}
|
|
1694
|
+
wakeReaders() {
|
|
1695
|
+
this.changeVersion += 1;
|
|
1696
|
+
for (const waiter of Array.from(this.waiters)) waiter.resolve("change");
|
|
1697
|
+
}
|
|
1698
|
+
waitForChange(observedVersion, waitMs, signal) {
|
|
1699
|
+
if (signal?.aborted) return Promise.reject(abortReason(signal));
|
|
1700
|
+
return new Promise((resolve, reject) => {
|
|
1701
|
+
let settled = false;
|
|
1702
|
+
const waiter = {
|
|
1703
|
+
timer: setTimeout(() => finish("timeout"), waitMs),
|
|
1704
|
+
signal,
|
|
1705
|
+
resolve: (reason) => finish(reason)
|
|
937
1706
|
};
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
await txn.put(META_KEY, {
|
|
943
|
-
count: index + 1,
|
|
944
|
-
closed: false,
|
|
945
|
-
ownerRunId: runId
|
|
946
|
-
});
|
|
947
|
-
return index;
|
|
948
|
-
});
|
|
949
|
-
}
|
|
950
|
-
/**
|
|
951
|
-
* Close the stream (idempotent). Readers observe `done: true` once all
|
|
952
|
-
* previously written chunks have been consumed.
|
|
953
|
-
*/
|
|
954
|
-
async closeStream(runId) {
|
|
955
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
956
|
-
const meta = await txn.get(META_KEY) ?? {
|
|
957
|
-
count: 0,
|
|
958
|
-
closed: false
|
|
1707
|
+
const cleanup = () => {
|
|
1708
|
+
clearTimeout(waiter.timer);
|
|
1709
|
+
this.waiters.delete(waiter);
|
|
1710
|
+
if (waiter.signal && waiter.onAbort) waiter.signal.removeEventListener("abort", waiter.onAbort);
|
|
959
1711
|
};
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
}
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
const entries = await this.ctx.storage.list({
|
|
976
|
-
prefix: CHUNK_KEY_PREFIX,
|
|
977
|
-
start: chunkKey(start),
|
|
978
|
-
limit: params.limit
|
|
1712
|
+
const finish = (reason) => {
|
|
1713
|
+
if (settled) return;
|
|
1714
|
+
settled = true;
|
|
1715
|
+
cleanup();
|
|
1716
|
+
resolve(reason);
|
|
1717
|
+
};
|
|
1718
|
+
waiter.onAbort = () => {
|
|
1719
|
+
if (settled) return;
|
|
1720
|
+
settled = true;
|
|
1721
|
+
cleanup();
|
|
1722
|
+
reject(abortReason(signal));
|
|
1723
|
+
};
|
|
1724
|
+
this.waiters.add(waiter);
|
|
1725
|
+
signal?.addEventListener("abort", waiter.onAbort, { once: true });
|
|
1726
|
+
if (this.changeVersion !== observedVersion) finish("change");
|
|
979
1727
|
});
|
|
980
|
-
return {
|
|
981
|
-
chunks: Array.from(entries.values()),
|
|
982
|
-
done: meta.closed,
|
|
983
|
-
tailIndex: meta.count - 1
|
|
984
|
-
};
|
|
985
1728
|
}
|
|
986
|
-
|
|
987
|
-
* Lightweight stream metadata: last chunk index and completion flag.
|
|
988
|
-
*/
|
|
989
|
-
async getInfo() {
|
|
1729
|
+
async readSnapshot(request, timedOut) {
|
|
990
1730
|
const meta = await this.getMeta();
|
|
991
|
-
|
|
1731
|
+
this.assertOwner(meta, request.runId);
|
|
1732
|
+
const available = meta.state === "expired" ? 0 : Math.max(0, Math.min(request.maxChunks, meta.count - request.startIndex));
|
|
1733
|
+
const chunks = [];
|
|
1734
|
+
if (available > 0) {
|
|
1735
|
+
const indexes = Array.from({ length: available }, (_, offset) => request.startIndex + offset);
|
|
1736
|
+
const sizeKeys = indexes.map(chunkSizeKey);
|
|
1737
|
+
const storedSizes = await this.ctx.storage.get(sizeKeys);
|
|
1738
|
+
const selectedIndexes = [];
|
|
1739
|
+
const selectedSizes = [];
|
|
1740
|
+
let bytes = 0;
|
|
1741
|
+
for (const index of indexes) {
|
|
1742
|
+
const size = validateChunkSize(storedSizes.get(chunkSizeKey(index)), index);
|
|
1743
|
+
if (bytes + size > request.maxBytes) break;
|
|
1744
|
+
selectedIndexes.push(index);
|
|
1745
|
+
selectedSizes.push(size);
|
|
1746
|
+
bytes += size;
|
|
1747
|
+
}
|
|
1748
|
+
const payloadKeys = selectedIndexes.map(chunkKey);
|
|
1749
|
+
const storedChunks = await this.ctx.storage.get(payloadKeys);
|
|
1750
|
+
for (let offset = 0; offset < payloadKeys.length; offset++) {
|
|
1751
|
+
const chunk = storedChunks.get(payloadKeys[offset]);
|
|
1752
|
+
if (!(chunk instanceof Uint8Array) || chunk.byteLength !== selectedSizes[offset]) throw new Error(`Invalid persisted stream chunk at index ${selectedIndexes[offset]}`);
|
|
1753
|
+
chunks.push(chunk);
|
|
1754
|
+
}
|
|
1755
|
+
}
|
|
992
1756
|
return {
|
|
1757
|
+
startIndex: request.startIndex,
|
|
993
1758
|
tailIndex: meta.count - 1,
|
|
994
|
-
|
|
1759
|
+
chunks,
|
|
1760
|
+
state: meta.state,
|
|
1761
|
+
timedOut,
|
|
1762
|
+
error: meta.error
|
|
995
1763
|
};
|
|
996
1764
|
}
|
|
1765
|
+
/** Append one bounded binary batch with contiguous indexes. */
|
|
1766
|
+
async writeChunks(runId, chunks) {
|
|
1767
|
+
validateStreamWriteChunks(chunks);
|
|
1768
|
+
const storedChunks = chunks.map(compactChunk);
|
|
1769
|
+
return await this.runMutation(async () => {
|
|
1770
|
+
const committed = await this.ctx.storage.transaction(async (txn) => {
|
|
1771
|
+
const stored = await txn.get(META_KEY);
|
|
1772
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
1773
|
+
this.assertWritable(meta, runId);
|
|
1774
|
+
const startIndex = meta.count;
|
|
1775
|
+
if (startIndex + chunks.length > 2147483647) throw new Error("Stream offset limit exceeded");
|
|
1776
|
+
const nextMeta = {
|
|
1777
|
+
count: startIndex + chunks.length,
|
|
1778
|
+
state: "open",
|
|
1779
|
+
ownerRunId: runId
|
|
1780
|
+
};
|
|
1781
|
+
const entries = { [META_KEY]: nextMeta };
|
|
1782
|
+
for (let offset = 0; offset < storedChunks.length; offset++) {
|
|
1783
|
+
const index = startIndex + offset;
|
|
1784
|
+
entries[chunkKey(index)] = storedChunks[offset];
|
|
1785
|
+
entries[chunkSizeKey(index)] = storedChunks[offset].byteLength;
|
|
1786
|
+
}
|
|
1787
|
+
await txn.put(entries);
|
|
1788
|
+
return {
|
|
1789
|
+
meta: nextMeta,
|
|
1790
|
+
result: {
|
|
1791
|
+
startIndex,
|
|
1792
|
+
count: chunks.length,
|
|
1793
|
+
tailIndex: nextMeta.count - 1
|
|
1794
|
+
}
|
|
1795
|
+
};
|
|
1796
|
+
});
|
|
1797
|
+
this.meta = committed.meta;
|
|
1798
|
+
this.metaLoad = Promise.resolve(committed.meta);
|
|
1799
|
+
this.wakeReaders();
|
|
1800
|
+
return committed.result;
|
|
1801
|
+
});
|
|
1802
|
+
}
|
|
997
1803
|
/**
|
|
998
|
-
*
|
|
1804
|
+
* Return available binary chunks and terminal metadata in one bounded
|
|
1805
|
+
* operation, waiting at most waitMs when the requested offset is idle.
|
|
999
1806
|
*/
|
|
1807
|
+
async readChunks(request, signal) {
|
|
1808
|
+
validateStreamReadRequest(request);
|
|
1809
|
+
const observedVersion = this.changeVersion;
|
|
1810
|
+
const immediate = await this.readSnapshot(request, false);
|
|
1811
|
+
if (immediate.chunks.length > 0 || immediate.state !== "open" || request.waitMs === 0 || request.maxChunks === 0) return immediate;
|
|
1812
|
+
const reason = await this.waitForChange(observedVersion, request.waitMs, signal);
|
|
1813
|
+
return await this.readSnapshot(request, reason === "timeout");
|
|
1814
|
+
}
|
|
1815
|
+
/** Close idempotently; the first terminal state wins. */
|
|
1816
|
+
async closeStream(runId) {
|
|
1817
|
+
await this.runMutation(async () => {
|
|
1818
|
+
const nextMeta = await this.ctx.storage.transaction(async (txn) => {
|
|
1819
|
+
const stored = await txn.get(META_KEY);
|
|
1820
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
1821
|
+
this.assertOwner(meta, runId);
|
|
1822
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
1823
|
+
if (meta.state !== "open") return meta;
|
|
1824
|
+
const closed = {
|
|
1825
|
+
...meta,
|
|
1826
|
+
ownerRunId: runId,
|
|
1827
|
+
state: "closed"
|
|
1828
|
+
};
|
|
1829
|
+
await txn.put(META_KEY, closed);
|
|
1830
|
+
return closed;
|
|
1831
|
+
});
|
|
1832
|
+
this.meta = nextMeta;
|
|
1833
|
+
this.metaLoad = Promise.resolve(nextMeta);
|
|
1834
|
+
this.wakeReaders();
|
|
1835
|
+
});
|
|
1836
|
+
}
|
|
1837
|
+
/** Persist a terminal stream failure and wake every pending reader. */
|
|
1838
|
+
async failStream(runId, error) {
|
|
1839
|
+
const normalized = normalizeStreamError(error);
|
|
1840
|
+
await this.runMutation(async () => {
|
|
1841
|
+
const nextMeta = await this.ctx.storage.transaction(async (txn) => {
|
|
1842
|
+
const stored = await txn.get(META_KEY);
|
|
1843
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
1844
|
+
this.assertOwner(meta, runId);
|
|
1845
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
1846
|
+
if (meta.state !== "open") return meta;
|
|
1847
|
+
const failed = {
|
|
1848
|
+
...meta,
|
|
1849
|
+
ownerRunId: runId,
|
|
1850
|
+
state: "errored",
|
|
1851
|
+
error: normalized
|
|
1852
|
+
};
|
|
1853
|
+
await txn.put(META_KEY, failed);
|
|
1854
|
+
return failed;
|
|
1855
|
+
});
|
|
1856
|
+
this.meta = nextMeta;
|
|
1857
|
+
this.metaLoad = Promise.resolve(nextMeta);
|
|
1858
|
+
this.wakeReaders();
|
|
1859
|
+
});
|
|
1860
|
+
}
|
|
1861
|
+
/** Register a stream name against this per-run registry instance. */
|
|
1000
1862
|
async registerStream(runId, name) {
|
|
1001
1863
|
await this.ctx.storage.transaction(async (txn) => {
|
|
1002
1864
|
if (await txn.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
|
|
@@ -1006,57 +1868,124 @@ var StreamDO = class extends DurableObject {
|
|
|
1006
1868
|
await txn.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
|
|
1007
1869
|
});
|
|
1008
1870
|
}
|
|
1009
|
-
/**
|
|
1010
|
-
* List stream names registered against this per-run registry instance.
|
|
1011
|
-
*/
|
|
1012
1871
|
async listStreams() {
|
|
1013
1872
|
if (await this.ctx.storage.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error("Workflow run streams have expired");
|
|
1014
1873
|
const entries = await this.ctx.storage.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1015
1874
|
return Array.from(entries.keys()).map((key) => key.slice(7));
|
|
1016
1875
|
}
|
|
1017
1876
|
/** Fence a run's registry before any stream cells are removed. */
|
|
1018
|
-
async expireRegistry(runId, expiredAt) {
|
|
1877
|
+
async expireRegistry(runId, expiredAt, options) {
|
|
1878
|
+
const limit = boundedLimit(options?.limit, 16, 128);
|
|
1019
1879
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1020
|
-
const
|
|
1880
|
+
const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
|
|
1881
|
+
const owner = state.get(REGISTRY_OWNER_KEY);
|
|
1021
1882
|
if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
|
|
1022
|
-
|
|
1023
|
-
await txn.put(
|
|
1024
|
-
|
|
1883
|
+
const expiry = state.get(REGISTRY_EXPIRED_KEY);
|
|
1884
|
+
await txn.put({
|
|
1885
|
+
[REGISTRY_OWNER_KEY]: runId,
|
|
1886
|
+
[REGISTRY_EXPIRED_KEY]: expiry ?? {
|
|
1887
|
+
expiredAt,
|
|
1888
|
+
deleted: 0
|
|
1889
|
+
}
|
|
1890
|
+
});
|
|
1891
|
+
const entries = await txn.list({
|
|
1892
|
+
prefix: STREAM_REGISTRY_PREFIX,
|
|
1893
|
+
limit
|
|
1894
|
+
});
|
|
1025
1895
|
return { streams: Array.from(entries.keys()).map((key) => key.slice(7)) };
|
|
1026
1896
|
});
|
|
1027
1897
|
}
|
|
1028
|
-
/** Remove
|
|
1029
|
-
async finalizeRegistry(runId) {
|
|
1030
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
1031
|
-
const owner = await txn.get(REGISTRY_OWNER_KEY);
|
|
1032
|
-
const expiredAt = await txn.get(REGISTRY_EXPIRED_KEY);
|
|
1033
|
-
if (owner !== runId || expiredAt === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
|
|
1034
|
-
const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1035
|
-
for (const key of entries.keys()) await txn.delete(key);
|
|
1036
|
-
});
|
|
1037
|
-
}
|
|
1038
|
-
/** Delete stream payload while leaving a small ownership tombstone. */
|
|
1039
|
-
async expireStream(runId, expiredAt) {
|
|
1898
|
+
/** Remove completed registry entries in a bounded, retry-safe batch. */
|
|
1899
|
+
async finalizeRegistry(runId, streams) {
|
|
1040
1900
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1041
|
-
const
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
};
|
|
1045
|
-
|
|
1046
|
-
const
|
|
1047
|
-
|
|
1048
|
-
await txn.put(
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
ownerRunId: runId,
|
|
1052
|
-
expiredAt
|
|
1901
|
+
const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
|
|
1902
|
+
const owner = state.get(REGISTRY_OWNER_KEY);
|
|
1903
|
+
const expiry = state.get(REGISTRY_EXPIRED_KEY);
|
|
1904
|
+
if (owner !== runId || expiry === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
|
|
1905
|
+
const keys = Array.from(new Set(streams.map((name) => `${STREAM_REGISTRY_PREFIX}${name}`)));
|
|
1906
|
+
const removed = keys.length === 0 ? 0 : await txn.delete(keys);
|
|
1907
|
+
const deleted = expiry.deleted + removed;
|
|
1908
|
+
await txn.put(REGISTRY_EXPIRED_KEY, {
|
|
1909
|
+
...expiry,
|
|
1910
|
+
deleted
|
|
1053
1911
|
});
|
|
1054
1912
|
return {
|
|
1055
|
-
deleted
|
|
1056
|
-
|
|
1913
|
+
deleted,
|
|
1914
|
+
done: (await txn.list({
|
|
1915
|
+
prefix: STREAM_REGISTRY_PREFIX,
|
|
1916
|
+
limit: 1
|
|
1917
|
+
})).size === 0
|
|
1057
1918
|
};
|
|
1058
1919
|
});
|
|
1059
1920
|
}
|
|
1921
|
+
/**
|
|
1922
|
+
* Fence the stream and delete one bounded page. Listing size keys avoids
|
|
1923
|
+
* loading payloads, and the transaction makes a crash retry resume safely.
|
|
1924
|
+
*/
|
|
1925
|
+
async expireStream(runId, expiredAt, options) {
|
|
1926
|
+
const limit = boundedLimit(options?.limit, DEFAULT_EXPIRE_CHUNK_LIMIT, MAX_EXPIRE_CHUNK_LIMIT);
|
|
1927
|
+
const byteLimit = boundedLimit(options?.byteLimit, DEFAULT_EXPIRE_BYTE_LIMIT, MAX_EXPIRE_BYTE_LIMIT);
|
|
1928
|
+
return await this.runMutation(async () => {
|
|
1929
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
1930
|
+
const stored = await txn.get(META_KEY);
|
|
1931
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
1932
|
+
this.assertOwner(meta, runId);
|
|
1933
|
+
const firstFence = meta.state !== "expired";
|
|
1934
|
+
if (meta.state === "expired" && meta.payloadDeleted) return {
|
|
1935
|
+
meta,
|
|
1936
|
+
result: {
|
|
1937
|
+
deleted: false,
|
|
1938
|
+
chunks: 0,
|
|
1939
|
+
bytes: 0,
|
|
1940
|
+
done: true
|
|
1941
|
+
}
|
|
1942
|
+
};
|
|
1943
|
+
const candidates = await txn.list({
|
|
1944
|
+
prefix: CHUNK_SIZE_KEY_PREFIX,
|
|
1945
|
+
limit: limit + 1
|
|
1946
|
+
});
|
|
1947
|
+
const page = [];
|
|
1948
|
+
let bytes = 0;
|
|
1949
|
+
for (const [key, storedSize] of Array.from(candidates).slice(0, limit)) {
|
|
1950
|
+
const index = indexFromChunkSizeKey(key);
|
|
1951
|
+
const size = validateChunkSize(storedSize, index);
|
|
1952
|
+
if (page.length > 0 && bytes + size > byteLimit) break;
|
|
1953
|
+
page.push({
|
|
1954
|
+
index,
|
|
1955
|
+
size
|
|
1956
|
+
});
|
|
1957
|
+
bytes += size;
|
|
1958
|
+
}
|
|
1959
|
+
const keys = page.flatMap(({ index }) => [chunkKey(index), chunkSizeKey(index)]);
|
|
1960
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
1961
|
+
const done = candidates.size === page.length;
|
|
1962
|
+
const nextMeta = {
|
|
1963
|
+
...meta,
|
|
1964
|
+
count: done ? 0 : meta.count,
|
|
1965
|
+
ownerRunId: runId,
|
|
1966
|
+
state: "expired",
|
|
1967
|
+
error: void 0,
|
|
1968
|
+
expiredAt: meta.expiredAt ?? expiredAt,
|
|
1969
|
+
expiredChunkCount: meta.expiredChunkCount ?? meta.count,
|
|
1970
|
+
payloadDeleted: done
|
|
1971
|
+
};
|
|
1972
|
+
await txn.put(META_KEY, nextMeta);
|
|
1973
|
+
return {
|
|
1974
|
+
meta: nextMeta,
|
|
1975
|
+
result: {
|
|
1976
|
+
deleted: firstFence,
|
|
1977
|
+
chunks: page.length,
|
|
1978
|
+
bytes,
|
|
1979
|
+
done
|
|
1980
|
+
}
|
|
1981
|
+
};
|
|
1982
|
+
});
|
|
1983
|
+
this.meta = result.meta;
|
|
1984
|
+
this.metaLoad = Promise.resolve(result.meta);
|
|
1985
|
+
this.wakeReaders();
|
|
1986
|
+
return result.result;
|
|
1987
|
+
});
|
|
1988
|
+
}
|
|
1060
1989
|
};
|
|
1061
1990
|
//#endregion
|
|
1062
1991
|
//#region src/apply-event.ts
|
|
@@ -1066,6 +1995,7 @@ const RUN_KEY = "run";
|
|
|
1066
1995
|
const EVENT_KEY_PREFIX = "event:";
|
|
1067
1996
|
const STEP_KEY_PREFIX = "step:";
|
|
1068
1997
|
const HOOK_KEY_PREFIX = "hook:";
|
|
1998
|
+
const WAIT_KEY_PREFIX = "wait:";
|
|
1069
1999
|
const STEP_CREATED_KEY_PREFIX = "stepcreated:";
|
|
1070
2000
|
const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
|
|
1071
2001
|
/**
|
|
@@ -1075,6 +2005,36 @@ const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
|
|
|
1075
2005
|
* rejected as a duplicate instead of resurrecting the hook.
|
|
1076
2006
|
*/
|
|
1077
2007
|
const HOOK_EVENT_MARKER_PREFIX = "hookevent:";
|
|
2008
|
+
/** Durable repair record for idempotent post-commit hook-index release. */
|
|
2009
|
+
const HOOK_RELEASE_MARKER_PREFIX = "hookreleased:";
|
|
2010
|
+
/** Exact ambiguous reservations canceled by a serialized RunDO resolution. */
|
|
2011
|
+
const HOOK_CLAIM_CANCELLATION_PREFIX = "hookclaimcancelled:";
|
|
2012
|
+
/** Workflow-authored attr_set correlation claims. */
|
|
2013
|
+
const ATTRIBUTE_EVENT_MARKER_PREFIX = "attrevent:";
|
|
2014
|
+
/** Durable lazy-hook resume claims, scoped by the containing run DO. */
|
|
2015
|
+
const HOOK_RESUME_CLAIM_PREFIX = "hookresume:";
|
|
2016
|
+
const STORAGE_BATCH_SIZE$1 = 128;
|
|
2017
|
+
const MAX_ENTITY_PAGE_SIZE = 1e3;
|
|
2018
|
+
async function getMany(store, keys) {
|
|
2019
|
+
if (store.getMany) {
|
|
2020
|
+
const result = /* @__PURE__ */ new Map();
|
|
2021
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) {
|
|
2022
|
+
const page = await store.getMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
2023
|
+
for (const entry of page) result.set(...entry);
|
|
2024
|
+
}
|
|
2025
|
+
return result;
|
|
2026
|
+
}
|
|
2027
|
+
const values = await Promise.all(keys.map((key) => store.get(key)));
|
|
2028
|
+
return new Map(keys.flatMap((key, index) => values[index] === void 0 ? [] : [[key, values[index]]]));
|
|
2029
|
+
}
|
|
2030
|
+
async function deleteMany(store, keys) {
|
|
2031
|
+
if (store.deleteMany) {
|
|
2032
|
+
let deleted = 0;
|
|
2033
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) deleted += await store.deleteMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
2034
|
+
return deleted;
|
|
2035
|
+
}
|
|
2036
|
+
return (await Promise.all(keys.map((key) => store.delete(key)))).filter(Boolean).length;
|
|
2037
|
+
}
|
|
1078
2038
|
function failure(code, message, extra) {
|
|
1079
2039
|
return {
|
|
1080
2040
|
ok: false,
|
|
@@ -1083,6 +2043,9 @@ function failure(code, message, extra) {
|
|
|
1083
2043
|
...extra
|
|
1084
2044
|
};
|
|
1085
2045
|
}
|
|
2046
|
+
function hookClaimCancellationKey(claimId) {
|
|
2047
|
+
return `${HOOK_CLAIM_CANCELLATION_PREFIX}${encodeURIComponent(claimId)}`;
|
|
2048
|
+
}
|
|
1086
2049
|
function readStringProp(value, key) {
|
|
1087
2050
|
if (value !== null && typeof value === "object") {
|
|
1088
2051
|
const prop = value[key];
|
|
@@ -1138,7 +2101,9 @@ function creationIndexKey(prefix, createdAt, id) {
|
|
|
1138
2101
|
* requested ordering.
|
|
1139
2102
|
*/
|
|
1140
2103
|
async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
1141
|
-
const
|
|
2104
|
+
const requestedLimit = Number.isFinite(params.limit) ? Math.floor(params.limit) : 1;
|
|
2105
|
+
const limit = Math.min(MAX_ENTITY_PAGE_SIZE, Math.max(1, requestedLimit));
|
|
2106
|
+
const { cursor, sortOrder = "asc" } = params;
|
|
1142
2107
|
const entries = await store.list(sortOrder === "desc" ? {
|
|
1143
2108
|
prefix: indexPrefix,
|
|
1144
2109
|
reverse: true,
|
|
@@ -1150,11 +2115,12 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
|
1150
2115
|
startAfter: cursor ? `${indexPrefix}${cursor}` : void 0
|
|
1151
2116
|
});
|
|
1152
2117
|
const page = Array.from(entries.entries()).slice(0, limit);
|
|
1153
|
-
const
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
2118
|
+
const entityKeys = page.map(([, id]) => `${entityPrefix}${id}`);
|
|
2119
|
+
const entities = await getMany(store, entityKeys);
|
|
2120
|
+
const data = entityKeys.flatMap((key) => {
|
|
2121
|
+
const item = entities.get(key);
|
|
2122
|
+
return item === void 0 ? [] : [item];
|
|
2123
|
+
});
|
|
1158
2124
|
const hasMore = entries.size > limit;
|
|
1159
2125
|
const lastKey = page.at(-1)?.[0];
|
|
1160
2126
|
return {
|
|
@@ -1165,21 +2131,31 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
|
1165
2131
|
}
|
|
1166
2132
|
/** Delete all hook entities for the run, returning the released tokens. */
|
|
1167
2133
|
async function releaseAllHooks(store) {
|
|
2134
|
+
if (store.deferTerminalCleanup) return [];
|
|
1168
2135
|
const hooks = await store.list({ prefix: HOOK_KEY_PREFIX });
|
|
1169
2136
|
const released = [];
|
|
2137
|
+
const keys = [];
|
|
1170
2138
|
for (const hook of hooks.values()) {
|
|
1171
|
-
|
|
1172
|
-
|
|
2139
|
+
keys.push(`${HOOK_KEY_PREFIX}${hook.hookId}`);
|
|
2140
|
+
keys.push(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
|
|
1173
2141
|
released.push({
|
|
1174
2142
|
hookId: hook.hookId,
|
|
1175
2143
|
token: hook.token
|
|
1176
2144
|
});
|
|
1177
2145
|
}
|
|
2146
|
+
await deleteMany(store, keys);
|
|
1178
2147
|
return released;
|
|
1179
2148
|
}
|
|
2149
|
+
/** Delete all persisted waits once their owning run becomes terminal. */
|
|
2150
|
+
async function releaseAllWaits(store) {
|
|
2151
|
+
if (store.deferTerminalCleanup) return;
|
|
2152
|
+
const waits = await store.list({ prefix: WAIT_KEY_PREFIX });
|
|
2153
|
+
await deleteMany(store, Array.from(waits.keys()));
|
|
2154
|
+
}
|
|
1180
2155
|
async function applyEvent(store, ctx) {
|
|
1181
2156
|
const { runId, data, now } = ctx;
|
|
1182
2157
|
const specVersion = data.specVersion ?? SPEC_VERSION_CURRENT;
|
|
2158
|
+
if (specVersion !== SPEC_VERSION_CURRENT) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" uses unsupported specVersion ${specVersion}; this world requires ${SPEC_VERSION_CURRENT}`, { runSpecVersion: specVersion });
|
|
1183
2159
|
/** Build and validate the stored event BEFORE anything is persisted. */
|
|
1184
2160
|
const buildEvent = (record) => {
|
|
1185
2161
|
const withMeta = {
|
|
@@ -1187,9 +2163,16 @@ async function applyEvent(store, ctx) {
|
|
|
1187
2163
|
runId,
|
|
1188
2164
|
eventId: ctx.nextEventId(),
|
|
1189
2165
|
createdAt: now,
|
|
2166
|
+
occurredAt: ctx.params?.occurredAt,
|
|
2167
|
+
resumeId: ctx.params?.resumeId,
|
|
1190
2168
|
specVersion
|
|
1191
2169
|
};
|
|
2170
|
+
if (record.eventType !== "hook_received") delete withMeta.resumeId;
|
|
1192
2171
|
if (record.eventType === "run_started") delete withMeta.eventData;
|
|
2172
|
+
if (record.eventType === "step_started" && withMeta.eventData !== null && typeof withMeta.eventData === "object") {
|
|
2173
|
+
const { input: _input, ...eventData } = withMeta.eventData;
|
|
2174
|
+
withMeta.eventData = eventData;
|
|
2175
|
+
}
|
|
1193
2176
|
return EventSchema.parse(compact(withMeta));
|
|
1194
2177
|
};
|
|
1195
2178
|
const putEvent = async (event) => {
|
|
@@ -1200,14 +2183,21 @@ async function applyEvent(store, ctx) {
|
|
|
1200
2183
|
if (!skipRunValidation) currentRun = await store.get(RUN_KEY);
|
|
1201
2184
|
let bootstrapped;
|
|
1202
2185
|
if (data.eventType === "run_started" && !currentRun && data.eventData) {
|
|
1203
|
-
const { deploymentId, workflowName, input, executionContext } = data.eventData;
|
|
2186
|
+
const { deploymentId, workflowName, input, executionContext, attributes, allowReservedAttributes, encryptionPublicKey } = data.eventData;
|
|
1204
2187
|
if (deploymentId && workflowName && input !== void 0) {
|
|
2188
|
+
const attributeChanges = Object.entries(attributes ?? {}).map(([key, value]) => ({
|
|
2189
|
+
key,
|
|
2190
|
+
value
|
|
2191
|
+
}));
|
|
2192
|
+
validateAttributeChanges(attributeChanges, { allowReservedAttributes });
|
|
1205
2193
|
const createdRun = WorkflowRunSchema.parse(compact({
|
|
1206
2194
|
runId,
|
|
1207
2195
|
deploymentId,
|
|
1208
2196
|
workflowName,
|
|
1209
2197
|
specVersion,
|
|
1210
2198
|
executionContext,
|
|
2199
|
+
attributes,
|
|
2200
|
+
encryptionPublicKey,
|
|
1211
2201
|
input,
|
|
1212
2202
|
status: "pending",
|
|
1213
2203
|
output: void 0,
|
|
@@ -1223,7 +2213,10 @@ async function applyEvent(store, ctx) {
|
|
|
1223
2213
|
deploymentId,
|
|
1224
2214
|
workflowName,
|
|
1225
2215
|
input,
|
|
1226
|
-
executionContext
|
|
2216
|
+
executionContext,
|
|
2217
|
+
attributes,
|
|
2218
|
+
allowReservedAttributes,
|
|
2219
|
+
encryptionPublicKey
|
|
1227
2220
|
}
|
|
1228
2221
|
});
|
|
1229
2222
|
await store.put(RUN_KEY, createdRun);
|
|
@@ -1235,10 +2228,7 @@ async function applyEvent(store, ctx) {
|
|
|
1235
2228
|
};
|
|
1236
2229
|
}
|
|
1237
2230
|
}
|
|
1238
|
-
if (currentRun) {
|
|
1239
|
-
if (requiresNewerWorld(currentRun.specVersion)) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" requires a newer world (run specVersion ${currentRun.specVersion}, world ${SPEC_VERSION_CURRENT})`, { runSpecVersion: currentRun.specVersion });
|
|
1240
|
-
if (isLegacySpecVersion(currentRun.specVersion)) return failure("LEGACY_RUN_NOT_SUPPORTED", `Run "${runId}" was created with a legacy spec version (${currentRun.specVersion ?? "undefined"}) not supported by this world`);
|
|
1241
|
-
}
|
|
2231
|
+
if (currentRun && currentRun.specVersion !== SPEC_VERSION_CURRENT) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" uses unsupported specVersion ${currentRun.specVersion}; this world requires ${SPEC_VERSION_CURRENT}`, { runSpecVersion: currentRun.specVersion });
|
|
1242
2232
|
if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) {
|
|
1243
2233
|
if (data.eventType === "run_cancelled" && currentRun.status === "cancelled") {
|
|
1244
2234
|
const event = buildEvent({ ...data });
|
|
@@ -1252,17 +2242,44 @@ async function applyEvent(store, ctx) {
|
|
|
1252
2242
|
}
|
|
1253
2243
|
if (data.eventType === "run_started") return failure("RUN_EXPIRED", `Workflow run "${runId}" is already in terminal state "${currentRun.status}"`);
|
|
1254
2244
|
if (data.eventType === "run_completed" || data.eventType === "run_failed" || data.eventType === "run_cancelled") return failure("ENTITY_CONFLICT", `Cannot transition run from terminal state "${currentRun.status}"`);
|
|
1255
|
-
if (data
|
|
2245
|
+
if (isChildEntityCreationEvent(data)) return failure("ENTITY_CONFLICT", `Cannot create new entities on run in terminal state "${currentRun.status}"`);
|
|
2246
|
+
if (data.eventType === "attr_set") return failure("ENTITY_CONFLICT", `Cannot set attributes on run in terminal state "${currentRun.status}"`);
|
|
1256
2247
|
}
|
|
1257
2248
|
let validatedStep;
|
|
1258
2249
|
if (data.eventType === "step_started" || data.eventType === "step_completed" || data.eventType === "step_failed" || data.eventType === "step_retrying") {
|
|
2250
|
+
const lazyStepStart = data.eventType === "step_started" && isChildEntityCreationEvent(data);
|
|
1259
2251
|
validatedStep = await store.get(`${STEP_KEY_PREFIX}${data.correlationId}`);
|
|
1260
|
-
if (!validatedStep)
|
|
1261
|
-
|
|
1262
|
-
|
|
2252
|
+
if (!validatedStep) {
|
|
2253
|
+
if (!lazyStepStart) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
2254
|
+
} else if (lazyStepStart) return failure("ENTITY_CONFLICT", `Step "${data.correlationId}" already exists`);
|
|
2255
|
+
if (validatedStep && isTerminalStepStatus(validatedStep.status)) return failure("ENTITY_CONFLICT", `Cannot modify step in terminal state "${validatedStep.status}"`);
|
|
2256
|
+
if (currentRun && isTerminalWorkflowRunStatus(currentRun.status) && validatedStep && validatedStep.status !== "running") return failure("RUN_EXPIRED", `Cannot modify non-running step on run in terminal state "${currentRun.status}"`);
|
|
2257
|
+
}
|
|
2258
|
+
if (data.eventType === "hook_received" && ctx.params?.resumeId) {
|
|
2259
|
+
const claim = await store.get(`${HOOK_RESUME_CLAIM_PREFIX}${ctx.params.resumeId}`);
|
|
2260
|
+
if (claim) {
|
|
2261
|
+
if (claim.hookId !== data.correlationId) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" already recorded for a different hook`);
|
|
2262
|
+
if (claim.payloadDigest && ctx.params.resumePayloadDigest && claim.payloadDigest !== ctx.params.resumePayloadDigest) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" already recorded with a different payload`);
|
|
2263
|
+
const committed = await store.get(`${EVENT_KEY_PREFIX}${claim.eventId}`);
|
|
2264
|
+
if (committed?.eventType === "hook_received" && committed.correlationId === claim.hookId && committed.resumeId === ctx.params.resumeId) return {
|
|
2265
|
+
ok: true,
|
|
2266
|
+
event: committed,
|
|
2267
|
+
releasedHooks: []
|
|
2268
|
+
};
|
|
2269
|
+
if (committed) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" points to a different event`);
|
|
2270
|
+
}
|
|
1263
2271
|
}
|
|
1264
2272
|
if ((data.eventType === "hook_disposed" || data.eventType === "hook_received") && data.correlationId) {
|
|
1265
|
-
if (!await store.get(`hook:${data.correlationId}`))
|
|
2273
|
+
if (!await store.get(`hook:${data.correlationId}`)) {
|
|
2274
|
+
if (data.eventType === "hook_disposed") {
|
|
2275
|
+
const released = await store.get(`${HOOK_RELEASE_MARKER_PREFIX}${data.correlationId}`);
|
|
2276
|
+
if (released) return {
|
|
2277
|
+
ok: true,
|
|
2278
|
+
releasedHooks: [released]
|
|
2279
|
+
};
|
|
2280
|
+
}
|
|
2281
|
+
return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
2282
|
+
}
|
|
1266
2283
|
}
|
|
1267
2284
|
if (!skipRunValidation && data.eventType !== "run_started" && !currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1268
2285
|
switch (data.eventType) {
|
|
@@ -1273,12 +2290,19 @@ async function applyEvent(store, ctx) {
|
|
|
1273
2290
|
run: existing,
|
|
1274
2291
|
releasedHooks: []
|
|
1275
2292
|
};
|
|
2293
|
+
const attributeChanges = Object.entries(data.eventData.attributes ?? {}).map(([key, value]) => ({
|
|
2294
|
+
key,
|
|
2295
|
+
value
|
|
2296
|
+
}));
|
|
2297
|
+
validateAttributeChanges(attributeChanges, { allowReservedAttributes: data.eventData.allowReservedAttributes });
|
|
1276
2298
|
const run = WorkflowRunSchema.parse(compact({
|
|
1277
2299
|
runId,
|
|
1278
2300
|
deploymentId: data.eventData.deploymentId,
|
|
1279
2301
|
workflowName: data.eventData.workflowName,
|
|
1280
2302
|
specVersion,
|
|
1281
2303
|
executionContext: data.eventData.executionContext,
|
|
2304
|
+
attributes: data.eventData.attributes,
|
|
2305
|
+
encryptionPublicKey: data.eventData.encryptionPublicKey,
|
|
1282
2306
|
input: data.eventData.input,
|
|
1283
2307
|
status: "pending",
|
|
1284
2308
|
output: void 0,
|
|
@@ -1304,11 +2328,25 @@ async function applyEvent(store, ctx) {
|
|
|
1304
2328
|
}
|
|
1305
2329
|
case "run_started": {
|
|
1306
2330
|
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1307
|
-
if (currentRun.status === "running")
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
2331
|
+
if (currentRun.status === "running") {
|
|
2332
|
+
const result = {
|
|
2333
|
+
ok: true,
|
|
2334
|
+
run: currentRun,
|
|
2335
|
+
releasedHooks: [],
|
|
2336
|
+
maxEvents: getMaxEventsPerRun()
|
|
2337
|
+
};
|
|
2338
|
+
if (ctx.params?.skipPreload) return result;
|
|
2339
|
+
const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
2340
|
+
limit: getMaxEventsPerRun(),
|
|
2341
|
+
sortOrder: "asc"
|
|
2342
|
+
}, (event) => event.eventId);
|
|
2343
|
+
return {
|
|
2344
|
+
...result,
|
|
2345
|
+
events: allEvents.data,
|
|
2346
|
+
cursor: allEvents.cursor,
|
|
2347
|
+
hasMore: allEvents.hasMore
|
|
2348
|
+
};
|
|
2349
|
+
}
|
|
1312
2350
|
const run = WorkflowRunSchema.parse(compact({
|
|
1313
2351
|
...currentRun,
|
|
1314
2352
|
status: "running",
|
|
@@ -1321,17 +2359,24 @@ async function applyEvent(store, ctx) {
|
|
|
1321
2359
|
const event = buildEvent({ ...data });
|
|
1322
2360
|
await store.put(RUN_KEY, run);
|
|
1323
2361
|
await putEvent(event);
|
|
1324
|
-
const
|
|
1325
|
-
limit: 1e3,
|
|
1326
|
-
sortOrder: "asc"
|
|
1327
|
-
}, (e) => e.eventId);
|
|
1328
|
-
return {
|
|
2362
|
+
const result = {
|
|
1329
2363
|
ok: true,
|
|
1330
2364
|
event,
|
|
1331
2365
|
run,
|
|
1332
2366
|
releasedHooks: [],
|
|
1333
2367
|
runCreated: bootstrapped,
|
|
1334
|
-
|
|
2368
|
+
maxEvents: getMaxEventsPerRun()
|
|
2369
|
+
};
|
|
2370
|
+
if (ctx.params?.skipPreload) return result;
|
|
2371
|
+
const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
2372
|
+
limit: getMaxEventsPerRun(),
|
|
2373
|
+
sortOrder: "asc"
|
|
2374
|
+
}, (e) => e.eventId);
|
|
2375
|
+
return {
|
|
2376
|
+
...result,
|
|
2377
|
+
events: allEvents.data,
|
|
2378
|
+
cursor: allEvents.cursor,
|
|
2379
|
+
hasMore: allEvents.hasMore
|
|
1335
2380
|
};
|
|
1336
2381
|
}
|
|
1337
2382
|
case "run_completed": {
|
|
@@ -1347,11 +2392,13 @@ async function applyEvent(store, ctx) {
|
|
|
1347
2392
|
const event = buildEvent({ ...data });
|
|
1348
2393
|
await store.put(RUN_KEY, run);
|
|
1349
2394
|
await putEvent(event);
|
|
2395
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
2396
|
+
await releaseAllWaits(store);
|
|
1350
2397
|
return {
|
|
1351
2398
|
ok: true,
|
|
1352
2399
|
event,
|
|
1353
2400
|
run,
|
|
1354
|
-
releasedHooks
|
|
2401
|
+
releasedHooks
|
|
1355
2402
|
};
|
|
1356
2403
|
}
|
|
1357
2404
|
case "run_failed": {
|
|
@@ -1367,11 +2414,13 @@ async function applyEvent(store, ctx) {
|
|
|
1367
2414
|
const event = buildEvent({ ...data });
|
|
1368
2415
|
await store.put(RUN_KEY, run);
|
|
1369
2416
|
await putEvent(event);
|
|
2417
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
2418
|
+
await releaseAllWaits(store);
|
|
1370
2419
|
return {
|
|
1371
2420
|
ok: true,
|
|
1372
2421
|
event,
|
|
1373
2422
|
run,
|
|
1374
|
-
releasedHooks
|
|
2423
|
+
releasedHooks
|
|
1375
2424
|
};
|
|
1376
2425
|
}
|
|
1377
2426
|
case "run_cancelled": {
|
|
@@ -1387,11 +2436,37 @@ async function applyEvent(store, ctx) {
|
|
|
1387
2436
|
const event = buildEvent({ ...data });
|
|
1388
2437
|
await store.put(RUN_KEY, run);
|
|
1389
2438
|
await putEvent(event);
|
|
2439
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
2440
|
+
await releaseAllWaits(store);
|
|
2441
|
+
return {
|
|
2442
|
+
ok: true,
|
|
2443
|
+
event,
|
|
2444
|
+
run,
|
|
2445
|
+
releasedHooks
|
|
2446
|
+
};
|
|
2447
|
+
}
|
|
2448
|
+
case "attr_set": {
|
|
2449
|
+
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
2450
|
+
validateAttributeChanges(data.eventData.changes, {
|
|
2451
|
+
existingKeys: Object.keys(currentRun.attributes),
|
|
2452
|
+
allowReservedAttributes: data.eventData.allowReservedAttributes
|
|
2453
|
+
});
|
|
2454
|
+
const markerKey = data.correlationId && data.eventData.writer.type === "workflow" ? `${ATTRIBUTE_EVENT_MARKER_PREFIX}${data.correlationId}` : void 0;
|
|
2455
|
+
if (markerKey && await store.get(markerKey) !== void 0) return failure("ENTITY_CONFLICT", `Attribute event "${data.correlationId}" already exists`);
|
|
2456
|
+
const run = WorkflowRunSchema.parse(compact({
|
|
2457
|
+
...currentRun,
|
|
2458
|
+
attributes: applyAttributeChanges(currentRun.attributes, data.eventData.changes),
|
|
2459
|
+
updatedAt: now
|
|
2460
|
+
}));
|
|
2461
|
+
const event = buildEvent({ ...data });
|
|
2462
|
+
await store.put(RUN_KEY, run);
|
|
2463
|
+
if (markerKey) await store.put(markerKey, event.eventId);
|
|
2464
|
+
await putEvent(event);
|
|
1390
2465
|
return {
|
|
1391
2466
|
ok: true,
|
|
1392
2467
|
event,
|
|
1393
2468
|
run,
|
|
1394
|
-
releasedHooks:
|
|
2469
|
+
releasedHooks: []
|
|
1395
2470
|
};
|
|
1396
2471
|
}
|
|
1397
2472
|
case "step_created": {
|
|
@@ -1425,7 +2500,34 @@ async function applyEvent(store, ctx) {
|
|
|
1425
2500
|
};
|
|
1426
2501
|
}
|
|
1427
2502
|
case "step_started": {
|
|
1428
|
-
|
|
2503
|
+
let step = validatedStep;
|
|
2504
|
+
let stepCreated = false;
|
|
2505
|
+
if (!step && data.eventData?.input !== void 0 && data.eventData.stepName) {
|
|
2506
|
+
step = StepSchema.parse(compact({
|
|
2507
|
+
runId,
|
|
2508
|
+
stepId: data.correlationId,
|
|
2509
|
+
stepName: data.eventData.stepName,
|
|
2510
|
+
status: "pending",
|
|
2511
|
+
input: data.eventData.input,
|
|
2512
|
+
attempt: 0,
|
|
2513
|
+
createdAt: now,
|
|
2514
|
+
updatedAt: now,
|
|
2515
|
+
specVersion
|
|
2516
|
+
}));
|
|
2517
|
+
const createdEvent = buildEvent({
|
|
2518
|
+
eventType: "step_created",
|
|
2519
|
+
correlationId: data.correlationId,
|
|
2520
|
+
eventData: {
|
|
2521
|
+
stepName: data.eventData.stepName,
|
|
2522
|
+
workflowName: data.eventData.workflowName,
|
|
2523
|
+
input: data.eventData.input
|
|
2524
|
+
}
|
|
2525
|
+
});
|
|
2526
|
+
await store.put(`${STEP_KEY_PREFIX}${data.correlationId}`, step);
|
|
2527
|
+
await store.put(creationIndexKey(STEP_CREATED_KEY_PREFIX, step.createdAt, step.stepId), step.stepId);
|
|
2528
|
+
await putEvent(createdEvent);
|
|
2529
|
+
stepCreated = true;
|
|
2530
|
+
}
|
|
1429
2531
|
if (!step) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
1430
2532
|
if (step.retryAfter && step.retryAfter.getTime() > now.getTime()) return failure("TOO_EARLY", `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, { retryAfterSeconds: Math.ceil((step.retryAfter.getTime() - now.getTime()) / 1e3) });
|
|
1431
2533
|
const updated = StepSchema.parse(compact({
|
|
@@ -1443,7 +2545,8 @@ async function applyEvent(store, ctx) {
|
|
|
1443
2545
|
ok: true,
|
|
1444
2546
|
event,
|
|
1445
2547
|
step: updated,
|
|
1446
|
-
releasedHooks: []
|
|
2548
|
+
releasedHooks: [],
|
|
2549
|
+
...stepCreated ? { stepCreated: true } : {}
|
|
1447
2550
|
};
|
|
1448
2551
|
}
|
|
1449
2552
|
case "step_completed": {
|
|
@@ -1472,7 +2575,7 @@ async function applyEvent(store, ctx) {
|
|
|
1472
2575
|
const updated = StepSchema.parse(compact({
|
|
1473
2576
|
...step,
|
|
1474
2577
|
status: "failed",
|
|
1475
|
-
error: toStructuredError(data.eventData.error
|
|
2578
|
+
error: toStructuredError(data.eventData.error),
|
|
1476
2579
|
completedAt: now,
|
|
1477
2580
|
updatedAt: now
|
|
1478
2581
|
}));
|
|
@@ -1492,7 +2595,7 @@ async function applyEvent(store, ctx) {
|
|
|
1492
2595
|
const updated = StepSchema.parse(compact({
|
|
1493
2596
|
...step,
|
|
1494
2597
|
status: "pending",
|
|
1495
|
-
error: toStructuredError(data.eventData.error
|
|
2598
|
+
error: toStructuredError(data.eventData.error),
|
|
1496
2599
|
retryAfter: data.eventData.retryAfter ? new Date(data.eventData.retryAfter) : void 0,
|
|
1497
2600
|
updatedAt: now
|
|
1498
2601
|
}));
|
|
@@ -1512,6 +2615,7 @@ async function applyEvent(store, ctx) {
|
|
|
1512
2615
|
const holder = ctx.tokenHolder ?? null;
|
|
1513
2616
|
const markerKey = `${HOOK_EVENT_MARKER_PREFIX}${hookId}`;
|
|
1514
2617
|
const hookKey = `${HOOK_KEY_PREFIX}${hookId}`;
|
|
2618
|
+
if (ctx.hookClaimId && await store.get(hookClaimCancellationKey(ctx.hookClaimId)) !== void 0) return failure("HOOK_CLAIM_CANCELLED", `Hook reservation for "${hookId}" was canceled before commit`);
|
|
1515
2619
|
if (holder && (holder.runId !== runId || holder.hookId !== hookId)) {
|
|
1516
2620
|
const event = buildEvent({
|
|
1517
2621
|
eventType: "hook_conflict",
|
|
@@ -1548,7 +2652,9 @@ async function applyEvent(store, ctx) {
|
|
|
1548
2652
|
environment: "",
|
|
1549
2653
|
createdAt: now,
|
|
1550
2654
|
specVersion,
|
|
1551
|
-
isWebhook: data.eventData.isWebhook ?? false
|
|
2655
|
+
isWebhook: data.eventData.isWebhook ?? false,
|
|
2656
|
+
isSystem: data.eventData.isSystem,
|
|
2657
|
+
tokenRetentionUntil: data.eventData.tokenRetentionUntil
|
|
1552
2658
|
}));
|
|
1553
2659
|
const event = buildEvent({ ...data });
|
|
1554
2660
|
await store.put(hookKey, hook);
|
|
@@ -1569,6 +2675,10 @@ async function applyEvent(store, ctx) {
|
|
|
1569
2675
|
if (!hook) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
1570
2676
|
await store.delete(hookKey);
|
|
1571
2677
|
await store.delete(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
|
|
2678
|
+
await store.put(`${HOOK_RELEASE_MARKER_PREFIX}${hook.hookId}`, {
|
|
2679
|
+
hookId: hook.hookId,
|
|
2680
|
+
token: hook.token
|
|
2681
|
+
});
|
|
1572
2682
|
const event = buildEvent({ ...data });
|
|
1573
2683
|
await putEvent(event);
|
|
1574
2684
|
return {
|
|
@@ -1580,9 +2690,58 @@ async function applyEvent(store, ctx) {
|
|
|
1580
2690
|
}]
|
|
1581
2691
|
};
|
|
1582
2692
|
}
|
|
2693
|
+
case "wait_created": {
|
|
2694
|
+
const waitKey = `${WAIT_KEY_PREFIX}${data.correlationId}`;
|
|
2695
|
+
if (await store.get(waitKey) !== void 0) return failure("ENTITY_CONFLICT", `Wait "${data.correlationId}" already exists`);
|
|
2696
|
+
const wait = WaitSchema.parse({
|
|
2697
|
+
waitId: `${runId}-${data.correlationId}`,
|
|
2698
|
+
runId,
|
|
2699
|
+
status: "waiting",
|
|
2700
|
+
resumeAt: data.eventData.resumeAt,
|
|
2701
|
+
createdAt: now,
|
|
2702
|
+
updatedAt: now,
|
|
2703
|
+
specVersion
|
|
2704
|
+
});
|
|
2705
|
+
const event = buildEvent({ ...data });
|
|
2706
|
+
await store.put(waitKey, wait);
|
|
2707
|
+
await putEvent(event);
|
|
2708
|
+
return {
|
|
2709
|
+
ok: true,
|
|
2710
|
+
event,
|
|
2711
|
+
wait,
|
|
2712
|
+
releasedHooks: []
|
|
2713
|
+
};
|
|
2714
|
+
}
|
|
2715
|
+
case "wait_completed": {
|
|
2716
|
+
const waitKey = `${WAIT_KEY_PREFIX}${data.correlationId}`;
|
|
2717
|
+
const existing = await store.get(waitKey);
|
|
2718
|
+
if (!existing) return failure("WAIT_NOT_FOUND", `Wait "${data.correlationId}" not found`);
|
|
2719
|
+
if (existing.status === "completed") return failure("ENTITY_CONFLICT", `Wait "${data.correlationId}" already completed`);
|
|
2720
|
+
const event = buildEvent({ ...data });
|
|
2721
|
+
const wait = WaitSchema.parse({
|
|
2722
|
+
...existing,
|
|
2723
|
+
status: "completed",
|
|
2724
|
+
resumeAt: data.eventData?.resumeAt ?? existing.resumeAt,
|
|
2725
|
+
completedAt: now,
|
|
2726
|
+
updatedAt: now
|
|
2727
|
+
});
|
|
2728
|
+
await store.put(waitKey, wait);
|
|
2729
|
+
await putEvent(event);
|
|
2730
|
+
return {
|
|
2731
|
+
ok: true,
|
|
2732
|
+
event,
|
|
2733
|
+
wait,
|
|
2734
|
+
releasedHooks: []
|
|
2735
|
+
};
|
|
2736
|
+
}
|
|
1583
2737
|
default: {
|
|
1584
2738
|
const event = buildEvent({ ...data });
|
|
1585
2739
|
await putEvent(event);
|
|
2740
|
+
if (data.eventType === "hook_received" && ctx.params?.resumeId) await store.put(`${HOOK_RESUME_CLAIM_PREFIX}${ctx.params.resumeId}`, {
|
|
2741
|
+
hookId: data.correlationId,
|
|
2742
|
+
eventId: event.eventId,
|
|
2743
|
+
...ctx.params.resumePayloadDigest ? { payloadDigest: ctx.params.resumePayloadDigest } : {}
|
|
2744
|
+
});
|
|
1586
2745
|
return {
|
|
1587
2746
|
ok: true,
|
|
1588
2747
|
event,
|
|
@@ -1591,12 +2750,54 @@ async function applyEvent(store, ctx) {
|
|
|
1591
2750
|
}
|
|
1592
2751
|
}
|
|
1593
2752
|
}
|
|
2753
|
+
/**
|
|
2754
|
+
* Complete Workflow 5's event-page response after an atomic apply. A celld
|
|
2755
|
+
* run is a single serialized writer, so the committed sequence is dense; a
|
|
2756
|
+
* stale eventCount therefore needs only the slots between the requested
|
|
2757
|
+
* position and the event this call committed.
|
|
2758
|
+
*/
|
|
2759
|
+
async function finalizeEventPage(store, outcome, params) {
|
|
2760
|
+
if (!outcome.ok) return outcome;
|
|
2761
|
+
const result = outcome.run && outcome.maxEvents === void 0 ? {
|
|
2762
|
+
...outcome,
|
|
2763
|
+
maxEvents: getMaxEventsPerRun()
|
|
2764
|
+
} : outcome;
|
|
2765
|
+
if (!result.event) return result;
|
|
2766
|
+
if (typeof params?.sinceCursor === "string") {
|
|
2767
|
+
const page = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
2768
|
+
limit: 100,
|
|
2769
|
+
cursor: params.sinceCursor,
|
|
2770
|
+
sortOrder: "asc"
|
|
2771
|
+
}, (event) => event.eventId);
|
|
2772
|
+
return {
|
|
2773
|
+
...result,
|
|
2774
|
+
events: page.data,
|
|
2775
|
+
cursor: page.cursor,
|
|
2776
|
+
hasMore: page.hasMore
|
|
2777
|
+
};
|
|
2778
|
+
}
|
|
2779
|
+
if (params?.eventCount === void 0) return result;
|
|
2780
|
+
const committedSlot = eventIdToSlot(result.event.eventId);
|
|
2781
|
+
if (committedSlot === null || committedSlot <= params.eventCount + 1) return result;
|
|
2782
|
+
const skippedCount = committedSlot - params.eventCount - 1;
|
|
2783
|
+
const events = (await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
2784
|
+
limit: skippedCount,
|
|
2785
|
+
cursor: params.eventCount > 0 ? slotToEventId(params.eventCount) : void 0,
|
|
2786
|
+
sortOrder: "asc"
|
|
2787
|
+
}, (event) => event.eventId)).data.filter((event) => event.eventId < result.event.eventId);
|
|
2788
|
+
return {
|
|
2789
|
+
...result,
|
|
2790
|
+
events,
|
|
2791
|
+
cursor: null,
|
|
2792
|
+
hasMore: events.length < skippedCount
|
|
2793
|
+
};
|
|
2794
|
+
}
|
|
1594
2795
|
//#endregion
|
|
1595
2796
|
//#region src/retention.ts
|
|
1596
2797
|
const CLEANUP_RECORD_KEY = "retention:cleanup";
|
|
1597
2798
|
const TOMBSTONE_KEY = "retention:tombstone";
|
|
1598
|
-
const INDEX_MARKER_PREFIX = "retention:index:";
|
|
1599
2799
|
const HOOK_MARKER_PREFIX = "retention:hook:";
|
|
2800
|
+
const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
|
|
1600
2801
|
function sortableTimestamp(date) {
|
|
1601
2802
|
return date.getTime().toString().padStart(13, "0");
|
|
1602
2803
|
}
|
|
@@ -1606,12 +2807,6 @@ function workflowRunIndexKey(run) {
|
|
|
1606
2807
|
function globalRunIndexKey(run) {
|
|
1607
2808
|
return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
1608
2809
|
}
|
|
1609
|
-
function correlationIndexKey(correlationId, createdAt, eventId, runId) {
|
|
1610
|
-
return `correlation:${encodeURIComponent(correlationId)}:${sortableTimestamp(createdAt)}:${eventId}:${runId}`;
|
|
1611
|
-
}
|
|
1612
|
-
function indexMarkerKey(indexKey) {
|
|
1613
|
-
return `${INDEX_MARKER_PREFIX}${encodeURIComponent(indexKey)}`;
|
|
1614
|
-
}
|
|
1615
2810
|
function hookMarkerKey(reference) {
|
|
1616
2811
|
return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
|
|
1617
2812
|
}
|
|
@@ -1635,68 +2830,18 @@ function expiredRead(tombstone) {
|
|
|
1635
2830
|
}
|
|
1636
2831
|
//#endregion
|
|
1637
2832
|
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
1638
|
-
const
|
|
1639
|
-
const
|
|
2833
|
+
const STORAGE_BATCH_SIZE = 128;
|
|
2834
|
+
const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
|
|
2835
|
+
const HOOK_MARKER_PAGE_SIZE = 64;
|
|
2836
|
+
const TERMINAL_HOOK_PAGE_SIZE = 64;
|
|
2837
|
+
const TERMINAL_WAIT_PAGE_SIZE = STORAGE_BATCH_SIZE;
|
|
2838
|
+
const STREAMS_PER_CLEANUP_PAGE = 16;
|
|
2839
|
+
const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
2840
|
+
const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
|
|
2841
|
+
const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
|
|
2842
|
+
const QUEUE_SHARDS_PER_CLEANUP_PAGE = 8;
|
|
1640
2843
|
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
1641
|
-
|
|
1642
|
-
* Current schema version for DO storage.
|
|
1643
|
-
* Increment when making breaking changes to the storage layout.
|
|
1644
|
-
* Migrations run lazily on first request after deploy.
|
|
1645
|
-
*
|
|
1646
|
-
* v2: one storage key per event ('event:<eventId>'), step ('step:<stepId>')
|
|
1647
|
-
* and hook ('hook:<hookId>') instead of single aggregate values. This package
|
|
1648
|
-
* is pre-production, so the migration simply drops the legacy aggregates.
|
|
1649
|
-
*/
|
|
1650
|
-
const CURRENT_SCHEMA_VERSION = 3;
|
|
1651
|
-
/**
|
|
1652
|
-
* Idempotent migration functions keyed by target version. The schema version
|
|
1653
|
-
* is written only after the migration completes, so eviction safely retries
|
|
1654
|
-
* an interrupted migration.
|
|
1655
|
-
*/
|
|
1656
|
-
const MIGRATIONS = {
|
|
1657
|
-
2: async (storage) => {
|
|
1658
|
-
await storage.transaction(async (txn) => {
|
|
1659
|
-
await txn.delete("events");
|
|
1660
|
-
await txn.delete("steps");
|
|
1661
|
-
await txn.delete("hooks");
|
|
1662
|
-
});
|
|
1663
|
-
},
|
|
1664
|
-
3: async (storage) => {
|
|
1665
|
-
await backfillCreationIndex(storage, STEP_KEY_PREFIX, STEP_CREATED_KEY_PREFIX, (step) => step.stepId);
|
|
1666
|
-
await backfillCreationIndex(storage, HOOK_KEY_PREFIX, HOOK_CREATED_KEY_PREFIX, (hook) => hook.hookId);
|
|
1667
|
-
}
|
|
1668
|
-
};
|
|
1669
|
-
async function backfillCreationIndex(storage, entityPrefix, indexPrefix, getId) {
|
|
1670
|
-
let startAfter;
|
|
1671
|
-
while (true) {
|
|
1672
|
-
const entities = await storage.list({
|
|
1673
|
-
prefix: entityPrefix,
|
|
1674
|
-
startAfter,
|
|
1675
|
-
limit: 100
|
|
1676
|
-
});
|
|
1677
|
-
if (entities.size === 0) return;
|
|
1678
|
-
for (const [key, entity] of entities) {
|
|
1679
|
-
const id = getId(entity);
|
|
1680
|
-
await storage.put(`${indexPrefix}${entity.createdAt.toISOString()}:${id}`, id);
|
|
1681
|
-
startAfter = key;
|
|
1682
|
-
}
|
|
1683
|
-
if (entities.size < 100) return;
|
|
1684
|
-
}
|
|
1685
|
-
}
|
|
1686
|
-
/**
|
|
1687
|
-
* Ensure the DO storage schema is up to date.
|
|
1688
|
-
* Runs any pending migrations sequentially inside transactions,
|
|
1689
|
-
* persisting the version after each successful migration.
|
|
1690
|
-
* Safe to call on every request -- it's a no-op when already current.
|
|
1691
|
-
*/
|
|
1692
|
-
async function ensureSchemaUpToDate(storage) {
|
|
1693
|
-
const stored = await storage.get("schema_version") ?? 1;
|
|
1694
|
-
for (let v = stored + 1; v <= CURRENT_SCHEMA_VERSION; v++) {
|
|
1695
|
-
const migration = MIGRATIONS[v];
|
|
1696
|
-
if (migration) await migration(storage);
|
|
1697
|
-
await storage.put("schema_version", v);
|
|
1698
|
-
}
|
|
1699
|
-
}
|
|
2844
|
+
const CLEANUP_PROGRESS_KEY = "retention:progress";
|
|
1700
2845
|
/**
|
|
1701
2846
|
* Adapt DO storage (or a transaction) to the {@link EventStore} interface
|
|
1702
2847
|
* shared with the test mocks. No casts: the methods line up structurally,
|
|
@@ -1705,11 +2850,21 @@ async function ensureSchemaUpToDate(storage) {
|
|
|
1705
2850
|
function storeFrom(storage) {
|
|
1706
2851
|
return {
|
|
1707
2852
|
get: (key) => storage.get(key),
|
|
2853
|
+
getMany: (keys) => storage.get(keys),
|
|
1708
2854
|
put: (key, value) => storage.put(key, value),
|
|
1709
2855
|
delete: (key) => storage.delete(key),
|
|
1710
|
-
|
|
2856
|
+
deleteMany: (keys) => storage.delete(keys),
|
|
2857
|
+
list: (options) => storage.list(options),
|
|
2858
|
+
deferTerminalCleanup: true
|
|
1711
2859
|
};
|
|
1712
2860
|
}
|
|
2861
|
+
function hookCreationIndexKey(hook) {
|
|
2862
|
+
return `${HOOK_CREATED_KEY_PREFIX}${hook.createdAt.toISOString()}:${hook.hookId}`;
|
|
2863
|
+
}
|
|
2864
|
+
async function putStorageEntries(storage, entries) {
|
|
2865
|
+
const all = Array.from(entries);
|
|
2866
|
+
for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE)));
|
|
2867
|
+
}
|
|
1713
2868
|
/**
|
|
1714
2869
|
* Durable Object for managing a single workflow run's state.
|
|
1715
2870
|
*
|
|
@@ -1721,37 +2876,55 @@ function storeFrom(storage) {
|
|
|
1721
2876
|
* queue-message dedup claims (see claimInflight/releaseInflight).
|
|
1722
2877
|
*/
|
|
1723
2878
|
var WorkflowRunDO = class extends DurableObject {
|
|
1724
|
-
schemaReady = null;
|
|
1725
2879
|
now() {
|
|
1726
2880
|
const clock = this.env?.clock;
|
|
1727
2881
|
return typeof clock === "function" ? clock() : Date.now();
|
|
1728
2882
|
}
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
return this.schemaReady;
|
|
1732
|
-
}
|
|
1733
|
-
async expiredTombstone(storage, now = this.now()) {
|
|
1734
|
-
const tombstone = await storage.get(TOMBSTONE_KEY);
|
|
2883
|
+
tombstoneFrom(values, now = this.now()) {
|
|
2884
|
+
const tombstone = values.get(TOMBSTONE_KEY);
|
|
1735
2885
|
if (tombstone) return tombstone;
|
|
1736
|
-
const cleanup =
|
|
2886
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
1737
2887
|
if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
|
|
1738
2888
|
return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
|
|
1739
2889
|
}
|
|
1740
|
-
async
|
|
1741
|
-
await
|
|
1742
|
-
const tombstone = await this.expiredTombstone(this.ctx.storage);
|
|
1743
|
-
if (tombstone) return expiredRead(tombstone);
|
|
2890
|
+
async retentionState(storage, now = this.now()) {
|
|
2891
|
+
const values = await storage.get([TOMBSTONE_KEY, CLEANUP_RECORD_KEY]);
|
|
1744
2892
|
return {
|
|
1745
|
-
|
|
1746
|
-
|
|
2893
|
+
cleanup: values.get(CLEANUP_RECORD_KEY),
|
|
2894
|
+
tombstone: this.tombstoneFrom(values, now)
|
|
1747
2895
|
};
|
|
1748
2896
|
}
|
|
2897
|
+
async read(reader) {
|
|
2898
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
2899
|
+
const { tombstone } = await this.retentionState(txn);
|
|
2900
|
+
if (tombstone) return expiredRead(tombstone);
|
|
2901
|
+
return {
|
|
2902
|
+
ok: true,
|
|
2903
|
+
value: await reader(txn)
|
|
2904
|
+
};
|
|
2905
|
+
});
|
|
2906
|
+
}
|
|
2907
|
+
async readKey(key) {
|
|
2908
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
2909
|
+
const values = await txn.get([
|
|
2910
|
+
TOMBSTONE_KEY,
|
|
2911
|
+
CLEANUP_RECORD_KEY,
|
|
2912
|
+
key
|
|
2913
|
+
]);
|
|
2914
|
+
const tombstone = this.tombstoneFrom(values);
|
|
2915
|
+
if (tombstone) return expiredRead(tombstone);
|
|
2916
|
+
return {
|
|
2917
|
+
ok: true,
|
|
2918
|
+
value: values.get(key) ?? null
|
|
2919
|
+
};
|
|
2920
|
+
});
|
|
2921
|
+
}
|
|
1749
2922
|
/** Apply an event and capture all retention metadata in the same transaction. */
|
|
1750
2923
|
async applyEvent(request) {
|
|
1751
|
-
await this.ensureSchema();
|
|
1752
2924
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1753
2925
|
const now = new Date(this.now());
|
|
1754
|
-
const
|
|
2926
|
+
const retention = await this.retentionState(txn, now.getTime());
|
|
2927
|
+
const { tombstone } = retention;
|
|
1755
2928
|
if (tombstone) return {
|
|
1756
2929
|
ok: false,
|
|
1757
2930
|
code: "RUN_EXPIRED",
|
|
@@ -1760,25 +2933,17 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1760
2933
|
let eventSequence = await txn.get("event_sequence") ?? 0;
|
|
1761
2934
|
const outcome = await applyEvent(storeFrom(txn), {
|
|
1762
2935
|
...request,
|
|
1763
|
-
nextEventId: () =>
|
|
2936
|
+
nextEventId: () => slotToEventId(++eventSequence),
|
|
1764
2937
|
now
|
|
1765
2938
|
});
|
|
1766
2939
|
if (!outcome.ok) return outcome;
|
|
1767
2940
|
await txn.put("event_sequence", eventSequence);
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
}
|
|
1772
|
-
if (outcome.hookToIndex) {
|
|
1773
|
-
const reference = {
|
|
1774
|
-
hookId: outcome.hookToIndex.hookId,
|
|
1775
|
-
token: outcome.hookToIndex.token
|
|
1776
|
-
};
|
|
1777
|
-
await txn.put(hookMarkerKey(reference), reference);
|
|
1778
|
-
}
|
|
1779
|
-
for (const reference of outcome.releasedHooks) await txn.put(hookMarkerKey(reference), reference);
|
|
2941
|
+
await putStorageEntries(txn, (outcome.hookToIndex ? [{
|
|
2942
|
+
hookId: outcome.hookToIndex.hookId,
|
|
2943
|
+
token: outcome.hookToIndex.token
|
|
2944
|
+
}, ...outcome.releasedHooks] : outcome.releasedHooks).map((reference) => [hookMarkerKey(reference), reference]));
|
|
1780
2945
|
const retentionMs = request.cleanup?.retentionMs ?? 0;
|
|
1781
|
-
if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !
|
|
2946
|
+
if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !retention.cleanup) {
|
|
1782
2947
|
const terminalRun = outcome.run;
|
|
1783
2948
|
const dueAt = new Date(terminalRun.completedAt.getTime() + retentionMs);
|
|
1784
2949
|
const run = WorkflowRunSchema.parse({
|
|
@@ -1795,6 +2960,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1795
2960
|
dueAt,
|
|
1796
2961
|
queueShards: request.cleanup?.queueShards ?? 1,
|
|
1797
2962
|
phase: "retained",
|
|
2963
|
+
generation: 0,
|
|
1798
2964
|
attempts: 0,
|
|
1799
2965
|
deletedPayloadKeys: 0,
|
|
1800
2966
|
deletedStreams: 0,
|
|
@@ -1805,52 +2971,82 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1805
2971
|
await txn.setAlarm(dueAt);
|
|
1806
2972
|
outcome.run = run;
|
|
1807
2973
|
}
|
|
1808
|
-
|
|
2974
|
+
if (outcome.run && isTerminalWorkflowRunStatus(outcome.run.status)) {
|
|
2975
|
+
if (!await txn.get("terminal:cleanup")) await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
2976
|
+
version: 1,
|
|
2977
|
+
runId: outcome.run.runId,
|
|
2978
|
+
phase: "hooks",
|
|
2979
|
+
generation: 0,
|
|
2980
|
+
attempts: 0
|
|
2981
|
+
});
|
|
2982
|
+
await txn.setAlarm(this.now() + 1);
|
|
2983
|
+
}
|
|
2984
|
+
return finalizeEventPage(storeFrom(txn), outcome, request.params);
|
|
2985
|
+
});
|
|
2986
|
+
}
|
|
2987
|
+
/**
|
|
2988
|
+
* Resolve an ambiguous hook-creation RPC in the same serialization domain as
|
|
2989
|
+
* applyEvent. If the hook is absent, the exact reservation is durably fenced
|
|
2990
|
+
* before the caller releases its token and hook-ID claims.
|
|
2991
|
+
*/
|
|
2992
|
+
async resolveHookTokenClaim(request) {
|
|
2993
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
2994
|
+
if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
|
|
2995
|
+
await txn.put(hookClaimCancellationKey(request.claimId), {
|
|
2996
|
+
hookId: request.hookId,
|
|
2997
|
+
token: request.token,
|
|
2998
|
+
canceledAt: this.now()
|
|
2999
|
+
});
|
|
3000
|
+
return { committed: false };
|
|
1809
3001
|
});
|
|
1810
3002
|
}
|
|
1811
3003
|
async getRun() {
|
|
1812
|
-
return this.
|
|
3004
|
+
return this.readKey("run");
|
|
1813
3005
|
}
|
|
1814
3006
|
async getStep(stepId) {
|
|
1815
|
-
return this.
|
|
3007
|
+
return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
|
|
1816
3008
|
}
|
|
1817
3009
|
async listSteps(params) {
|
|
1818
|
-
return this.read(() => listByCreationTime(storeFrom(
|
|
3010
|
+
return this.read((txn) => listByCreationTime(storeFrom(txn), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
|
|
1819
3011
|
limit: params?.limit ?? 20,
|
|
1820
3012
|
cursor: params?.cursor,
|
|
1821
3013
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1822
3014
|
}));
|
|
1823
3015
|
}
|
|
1824
3016
|
async getEvent(eventId) {
|
|
1825
|
-
return this.
|
|
3017
|
+
return this.readKey(`${EVENT_KEY_PREFIX}${eventId}`);
|
|
1826
3018
|
}
|
|
1827
3019
|
async listEvents(params) {
|
|
1828
|
-
return this.read(() => listByPrefix(storeFrom(
|
|
3020
|
+
return this.read((txn) => listByPrefix(storeFrom(txn), EVENT_KEY_PREFIX, {
|
|
1829
3021
|
limit: params?.limit ?? 100,
|
|
1830
3022
|
cursor: params?.cursor,
|
|
1831
3023
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1832
3024
|
}, (event) => event.eventId));
|
|
1833
3025
|
}
|
|
1834
3026
|
async listHooks(params) {
|
|
1835
|
-
return this.read(() => listByCreationTime(storeFrom(
|
|
3027
|
+
return this.read((txn) => listByCreationTime(storeFrom(txn), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
|
|
1836
3028
|
limit: params?.limit ?? 100,
|
|
1837
3029
|
cursor: params?.cursor,
|
|
1838
3030
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1839
3031
|
}));
|
|
1840
3032
|
}
|
|
1841
3033
|
async getCleanupStatus() {
|
|
1842
|
-
await this.ensureSchema();
|
|
1843
3034
|
return await this.ctx.storage.get("retention:cleanup") ?? null;
|
|
1844
3035
|
}
|
|
1845
3036
|
/** Explicitly schedule an existing terminal run, used for opt-in backfills. */
|
|
1846
3037
|
async scheduleCleanup(request) {
|
|
1847
|
-
await this.ensureSchema();
|
|
1848
3038
|
if (request.retentionMs <= 0) return this.getCleanupStatus();
|
|
1849
3039
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1850
|
-
const
|
|
3040
|
+
const values = await txn.get([
|
|
3041
|
+
CLEANUP_RECORD_KEY,
|
|
3042
|
+
TOMBSTONE_KEY,
|
|
3043
|
+
TERMINAL_CLEANUP_KEY,
|
|
3044
|
+
"run"
|
|
3045
|
+
]);
|
|
3046
|
+
const existing = values.get(CLEANUP_RECORD_KEY);
|
|
1851
3047
|
if (existing) return existing;
|
|
1852
|
-
if (
|
|
1853
|
-
const current =
|
|
3048
|
+
if (values.get("retention:tombstone")) return null;
|
|
3049
|
+
const current = values.get("run");
|
|
1854
3050
|
if (!current || !isTerminalWorkflowRunStatus(current.status)) return null;
|
|
1855
3051
|
const terminalRun = current;
|
|
1856
3052
|
const dueAt = new Date(terminalRun.completedAt.getTime() + request.retentionMs);
|
|
@@ -1868,6 +3064,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1868
3064
|
dueAt,
|
|
1869
3065
|
queueShards: request.queueShards,
|
|
1870
3066
|
phase: "retained",
|
|
3067
|
+
generation: 0,
|
|
1871
3068
|
attempts: 0,
|
|
1872
3069
|
deletedPayloadKeys: 0,
|
|
1873
3070
|
deletedStreams: 0,
|
|
@@ -1875,12 +3072,11 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1875
3072
|
};
|
|
1876
3073
|
await txn.put("run", run);
|
|
1877
3074
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
1878
|
-
await txn.setAlarm(dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
3075
|
+
await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
1879
3076
|
return cleanup;
|
|
1880
3077
|
});
|
|
1881
3078
|
}
|
|
1882
3079
|
async cleanupNow(request) {
|
|
1883
|
-
await this.ensureSchema();
|
|
1884
3080
|
if (!await this.ctx.storage.get("retention:cleanup")) await this.scheduleCleanup({
|
|
1885
3081
|
...request,
|
|
1886
3082
|
retentionMs: Math.max(1, request.retentionMs)
|
|
@@ -1896,137 +3092,334 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1896
3092
|
}));
|
|
1897
3093
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
1898
3094
|
...cleanup,
|
|
1899
|
-
dueAt
|
|
3095
|
+
dueAt,
|
|
3096
|
+
generation: cleanup.generation + 1
|
|
1900
3097
|
});
|
|
1901
3098
|
await txn.setAlarm(this.now() + 1);
|
|
1902
3099
|
});
|
|
1903
|
-
|
|
1904
|
-
await this.executeCleanup();
|
|
1905
|
-
} catch (error) {
|
|
1906
|
-
await this.recordCleanupFailure(error);
|
|
1907
|
-
}
|
|
3100
|
+
await this.executeScheduledCleanupPage();
|
|
1908
3101
|
return this.getCleanupStatus();
|
|
1909
3102
|
}
|
|
1910
3103
|
async rearmCleanup() {
|
|
1911
|
-
await this.
|
|
1912
|
-
const cleanup =
|
|
1913
|
-
if (
|
|
1914
|
-
if (cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
|
|
3104
|
+
const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TERMINAL_CLEANUP_KEY]);
|
|
3105
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3106
|
+
if (values.get("terminal:cleanup")) await this.ctx.storage.setAlarm(this.now() + 1);
|
|
3107
|
+
else if (!cleanup || cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
|
|
1915
3108
|
else await this.ctx.storage.setAlarm(Math.max(this.now() + 1, cleanup.dueAt.getTime()));
|
|
1916
|
-
return cleanup;
|
|
3109
|
+
return cleanup ?? null;
|
|
1917
3110
|
}
|
|
1918
3111
|
async alarm() {
|
|
1919
|
-
await this.
|
|
1920
|
-
try {
|
|
1921
|
-
await this.executeCleanup();
|
|
1922
|
-
} catch (error) {
|
|
1923
|
-
await this.recordCleanupFailure(error);
|
|
1924
|
-
}
|
|
3112
|
+
await this.executeScheduledCleanupPage();
|
|
1925
3113
|
}
|
|
1926
3114
|
namespace(name) {
|
|
1927
3115
|
const namespace = this.env?.[name];
|
|
1928
3116
|
if (!namespace || typeof namespace === "function") throw new Error(`world-celld retention missing binding ${name}`);
|
|
1929
3117
|
return namespace;
|
|
1930
3118
|
}
|
|
1931
|
-
|
|
3119
|
+
workflowIndex() {
|
|
3120
|
+
return createWorkflowIndex({
|
|
3121
|
+
runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
|
|
3122
|
+
runFences: this.namespace("WORKFLOW_RUN_FENCES"),
|
|
3123
|
+
hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
|
|
3124
|
+
hookIds: this.namespace("WORKFLOW_HOOK_IDS")
|
|
3125
|
+
});
|
|
3126
|
+
}
|
|
3127
|
+
async setCleanupPhase(expected, next, updates = {}, continueAt) {
|
|
1932
3128
|
await this.ctx.storage.transaction(async (txn) => {
|
|
1933
3129
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
1934
|
-
if (!cleanup || cleanup.phase !== expected) return;
|
|
3130
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
|
|
1935
3131
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
1936
3132
|
...cleanup,
|
|
1937
3133
|
...updates,
|
|
1938
3134
|
phase: next,
|
|
3135
|
+
generation: cleanup.generation + 1,
|
|
1939
3136
|
attempts: 0,
|
|
1940
3137
|
lastError: void 0
|
|
1941
3138
|
});
|
|
3139
|
+
if (continueAt !== void 0) await txn.setAlarm(continueAt);
|
|
1942
3140
|
});
|
|
1943
3141
|
}
|
|
1944
|
-
async
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
await this.
|
|
1949
|
-
|
|
3142
|
+
async executeScheduledCleanupPage() {
|
|
3143
|
+
const terminalCleanup = await this.ctx.storage.get(TERMINAL_CLEANUP_KEY);
|
|
3144
|
+
if (terminalCleanup) {
|
|
3145
|
+
try {
|
|
3146
|
+
await this.executeTerminalCleanup(terminalCleanup);
|
|
3147
|
+
} catch (error) {
|
|
3148
|
+
await this.recordTerminalCleanupFailure(error, terminalCleanup);
|
|
1950
3149
|
}
|
|
1951
|
-
|
|
1952
|
-
|
|
3150
|
+
return;
|
|
3151
|
+
}
|
|
3152
|
+
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
3153
|
+
if (!cleanup || cleanup.phase === "tombstoned") {
|
|
3154
|
+
await this.ctx.storage.deleteAlarm();
|
|
3155
|
+
return;
|
|
3156
|
+
}
|
|
3157
|
+
try {
|
|
3158
|
+
await this.executeCleanup(cleanup);
|
|
3159
|
+
} catch (error) {
|
|
3160
|
+
await this.recordCleanupFailure(error, cleanup);
|
|
3161
|
+
}
|
|
3162
|
+
}
|
|
3163
|
+
async executeCleanup(cleanup) {
|
|
3164
|
+
if (cleanup.phase === "retained") {
|
|
3165
|
+
if (cleanup.dueAt.getTime() > this.now()) {
|
|
3166
|
+
await this.ctx.storage.setAlarm(cleanup.dueAt);
|
|
1953
3167
|
return;
|
|
1954
3168
|
}
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
3169
|
+
await this.setCleanupPhase(cleanup, "index", {}, this.now() + 1);
|
|
3170
|
+
return;
|
|
3171
|
+
}
|
|
3172
|
+
if (cleanup.phase === "index") {
|
|
3173
|
+
await this.deleteIndexPage(cleanup);
|
|
3174
|
+
return;
|
|
3175
|
+
}
|
|
3176
|
+
if (cleanup.phase === "streams") {
|
|
3177
|
+
await this.deleteStreamPage(cleanup);
|
|
3178
|
+
return;
|
|
3179
|
+
}
|
|
3180
|
+
if (cleanup.phase === "queues") {
|
|
3181
|
+
await this.deleteQueuePage(cleanup);
|
|
3182
|
+
return;
|
|
3183
|
+
}
|
|
3184
|
+
if (cleanup.phase === "payload") await this.deletePayloadBatch();
|
|
3185
|
+
}
|
|
3186
|
+
async executeTerminalCleanup(cleanup) {
|
|
3187
|
+
if (cleanup.phase === "hooks") await this.deleteTerminalHookPage(cleanup);
|
|
3188
|
+
else if (cleanup.phase === "markers") await this.deleteTerminalHookMarkerPage(cleanup);
|
|
3189
|
+
else await this.deleteTerminalWaitPage(cleanup);
|
|
3190
|
+
}
|
|
3191
|
+
async deleteTerminalHookPage(cleanup) {
|
|
3192
|
+
const entries = await this.ctx.storage.list({
|
|
3193
|
+
prefix: HOOK_KEY_PREFIX,
|
|
3194
|
+
limit: 65
|
|
3195
|
+
});
|
|
3196
|
+
const page = Array.from(entries.values()).slice(0, TERMINAL_HOOK_PAGE_SIZE);
|
|
3197
|
+
await this.workflowIndex().releaseHookIndexes({
|
|
3198
|
+
runId: cleanup.runId,
|
|
3199
|
+
hooks: page.map((hook) => ({
|
|
3200
|
+
hookId: hook.hookId,
|
|
3201
|
+
token: hook.token
|
|
3202
|
+
})),
|
|
3203
|
+
terminal: true
|
|
3204
|
+
});
|
|
3205
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3206
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3207
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3208
|
+
const keys = page.flatMap((hook) => [`${HOOK_KEY_PREFIX}${hook.hookId}`, hookCreationIndexKey(hook)]);
|
|
3209
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3210
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3211
|
+
...current,
|
|
3212
|
+
phase: entries.size <= TERMINAL_HOOK_PAGE_SIZE ? "markers" : "hooks",
|
|
3213
|
+
generation: current.generation + 1,
|
|
3214
|
+
attempts: 0,
|
|
3215
|
+
lastError: void 0
|
|
3216
|
+
});
|
|
3217
|
+
await txn.setAlarm(this.now() + 1);
|
|
3218
|
+
});
|
|
3219
|
+
}
|
|
3220
|
+
async deleteTerminalHookMarkerPage(cleanup) {
|
|
3221
|
+
const entries = await this.ctx.storage.list({
|
|
3222
|
+
prefix: HOOK_MARKER_PREFIX,
|
|
3223
|
+
limit: 65
|
|
3224
|
+
});
|
|
3225
|
+
const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
|
|
3226
|
+
await this.workflowIndex().releaseHookIndexes({
|
|
3227
|
+
runId: cleanup.runId,
|
|
3228
|
+
hooks: page.map(([, reference]) => reference),
|
|
3229
|
+
terminal: true
|
|
3230
|
+
});
|
|
3231
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3232
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3233
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3234
|
+
const keys = page.map(([key]) => key);
|
|
3235
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3236
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3237
|
+
...current,
|
|
3238
|
+
phase: entries.size <= HOOK_MARKER_PAGE_SIZE ? "waits" : "markers",
|
|
3239
|
+
generation: current.generation + 1,
|
|
3240
|
+
attempts: 0,
|
|
3241
|
+
lastError: void 0
|
|
3242
|
+
});
|
|
3243
|
+
await txn.setAlarm(this.now() + 1);
|
|
3244
|
+
});
|
|
3245
|
+
}
|
|
3246
|
+
async deleteTerminalWaitPage(cleanup) {
|
|
3247
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3248
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3249
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3250
|
+
const entries = await txn.list({
|
|
3251
|
+
prefix: WAIT_KEY_PREFIX,
|
|
3252
|
+
limit: 129
|
|
3253
|
+
});
|
|
3254
|
+
const keys = Array.from(entries.keys()).slice(0, TERMINAL_WAIT_PAGE_SIZE);
|
|
3255
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3256
|
+
if (entries.size > TERMINAL_WAIT_PAGE_SIZE) {
|
|
3257
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3258
|
+
...current,
|
|
3259
|
+
generation: current.generation + 1,
|
|
3260
|
+
attempts: 0,
|
|
3261
|
+
lastError: void 0
|
|
1975
3262
|
});
|
|
1976
|
-
await this.
|
|
1977
|
-
|
|
3263
|
+
await txn.setAlarm(this.now() + 1);
|
|
3264
|
+
return;
|
|
1978
3265
|
}
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
3266
|
+
await txn.delete(TERMINAL_CLEANUP_KEY);
|
|
3267
|
+
const retention = await txn.get(CLEANUP_RECORD_KEY);
|
|
3268
|
+
if (retention && retention.phase !== "tombstoned") await txn.setAlarm(Math.max(this.now() + 1, retention.dueAt.getTime()));
|
|
3269
|
+
else await txn.deleteAlarm();
|
|
3270
|
+
});
|
|
3271
|
+
}
|
|
3272
|
+
async deleteIndexPage(cleanup) {
|
|
3273
|
+
const hookEntries = await this.ctx.storage.list({
|
|
3274
|
+
prefix: HOOK_MARKER_PREFIX,
|
|
3275
|
+
limit: 65
|
|
3276
|
+
});
|
|
3277
|
+
const hookMarkers = Array.from(hookEntries).slice(0, HOOK_MARKER_PAGE_SIZE);
|
|
3278
|
+
await this.workflowIndex().expireRun({
|
|
3279
|
+
runId: cleanup.runId,
|
|
3280
|
+
keys: [workflowRunIndexKey(cleanup), globalRunIndexKey(cleanup)],
|
|
3281
|
+
hooks: hookMarkers.map(([, value]) => value),
|
|
3282
|
+
expiredAt: cleanup.dueAt.getTime()
|
|
3283
|
+
});
|
|
3284
|
+
const markerKeys = hookMarkers.map(([key]) => key);
|
|
3285
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3286
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3287
|
+
if (current?.phase !== "index" || current.generation !== cleanup.generation) return;
|
|
3288
|
+
if (markerKeys.length > 0) await txn.delete(markerKeys);
|
|
3289
|
+
if (hookEntries.size <= HOOK_MARKER_PAGE_SIZE) await txn.put(CLEANUP_RECORD_KEY, {
|
|
3290
|
+
...current,
|
|
3291
|
+
phase: "streams",
|
|
3292
|
+
generation: current.generation + 1,
|
|
3293
|
+
attempts: 0,
|
|
3294
|
+
lastError: void 0
|
|
3295
|
+
});
|
|
3296
|
+
else await txn.put(CLEANUP_RECORD_KEY, {
|
|
3297
|
+
...current,
|
|
3298
|
+
generation: current.generation + 1,
|
|
3299
|
+
attempts: 0,
|
|
3300
|
+
lastError: void 0
|
|
3301
|
+
});
|
|
3302
|
+
await txn.setAlarm(this.now() + 1);
|
|
3303
|
+
});
|
|
3304
|
+
}
|
|
3305
|
+
async deleteStreamPage(cleanup) {
|
|
3306
|
+
const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
|
|
3307
|
+
const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
|
|
3308
|
+
const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime(), { limit: STREAMS_PER_CLEANUP_PAGE });
|
|
3309
|
+
const completed = [];
|
|
3310
|
+
let remainingChunks = STREAM_CHUNKS_PER_CLEANUP_PAGE;
|
|
3311
|
+
let remainingBytes = STREAM_BYTES_PER_CLEANUP_PAGE;
|
|
3312
|
+
for (const name of streams) {
|
|
3313
|
+
if (remainingChunks <= 0 || remainingBytes <= 0) break;
|
|
3314
|
+
const result = await streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime(), {
|
|
3315
|
+
limit: remainingChunks,
|
|
3316
|
+
byteLimit: remainingBytes
|
|
3317
|
+
});
|
|
3318
|
+
remainingChunks -= result.chunks;
|
|
3319
|
+
remainingBytes -= result.bytes;
|
|
3320
|
+
if (result.done) completed.push(name);
|
|
3321
|
+
}
|
|
3322
|
+
const registryResult = await registry.finalizeRegistry(cleanup.runId, completed);
|
|
3323
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3324
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3325
|
+
if (current?.phase !== "streams" || current.generation !== cleanup.generation) return;
|
|
3326
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3327
|
+
...current,
|
|
3328
|
+
deletedStreams: registryResult.deleted,
|
|
3329
|
+
phase: registryResult.done ? "queues" : "streams",
|
|
3330
|
+
generation: current.generation + 1,
|
|
3331
|
+
attempts: 0,
|
|
3332
|
+
lastError: void 0
|
|
3333
|
+
});
|
|
3334
|
+
await txn.setAlarm(this.now() + 1);
|
|
3335
|
+
});
|
|
3336
|
+
}
|
|
3337
|
+
async deleteQueuePage(cleanup) {
|
|
3338
|
+
const queues = this.namespace("WORKFLOW_QUEUE");
|
|
3339
|
+
let progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3340
|
+
queueShard: 0,
|
|
3341
|
+
queueShardDeleted: 0
|
|
3342
|
+
};
|
|
3343
|
+
let remainingShards = QUEUE_SHARDS_PER_CLEANUP_PAGE;
|
|
3344
|
+
let expectedGeneration = cleanup.generation;
|
|
3345
|
+
while (progress.queueShard < cleanup.queueShards && remainingShards-- > 0) {
|
|
3346
|
+
const requestedShard = progress.queueShard;
|
|
3347
|
+
const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
|
|
3348
|
+
const reconciled = await this.ctx.storage.transaction(async (txn) => {
|
|
3349
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3350
|
+
if (current?.phase !== "queues") return null;
|
|
3351
|
+
if (current.generation !== expectedGeneration) return { stale: true };
|
|
3352
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3353
|
+
queueShard: 0,
|
|
3354
|
+
queueShardDeleted: 0
|
|
3355
|
+
};
|
|
3356
|
+
if (persisted.queueShard !== requestedShard) return { stale: true };
|
|
3357
|
+
if (result.deleted < persisted.queueShardDeleted) return { stale: true };
|
|
3358
|
+
const next = result.done ? {
|
|
3359
|
+
queueShard: persisted.queueShard + 1,
|
|
3360
|
+
queueShardDeleted: 0
|
|
3361
|
+
} : {
|
|
3362
|
+
...persisted,
|
|
3363
|
+
queueShardDeleted: result.deleted
|
|
3364
|
+
};
|
|
3365
|
+
const delta = result.deleted - persisted.queueShardDeleted;
|
|
3366
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3367
|
+
...current,
|
|
3368
|
+
deletedQueueMessages: current.deletedQueueMessages + delta,
|
|
3369
|
+
generation: current.generation + 1,
|
|
3370
|
+
attempts: 0,
|
|
3371
|
+
lastError: void 0
|
|
1997
3372
|
});
|
|
1998
|
-
await
|
|
1999
|
-
await this.
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
if (
|
|
2013
|
-
if (!await this.deletePayloadBatch()) return;
|
|
2014
|
-
}
|
|
3373
|
+
await txn.put(CLEANUP_PROGRESS_KEY, next);
|
|
3374
|
+
await txn.setAlarm(this.now() + 1);
|
|
3375
|
+
return {
|
|
3376
|
+
stale: false,
|
|
3377
|
+
progress: next,
|
|
3378
|
+
generation: current.generation + 1,
|
|
3379
|
+
applied: result.done || next.queueShard !== persisted.queueShard || next.queueShardDeleted !== persisted.queueShardDeleted
|
|
3380
|
+
};
|
|
3381
|
+
});
|
|
3382
|
+
if (!reconciled) return;
|
|
3383
|
+
if (reconciled.stale) return;
|
|
3384
|
+
progress = reconciled.progress;
|
|
3385
|
+
expectedGeneration = reconciled.generation;
|
|
3386
|
+
if (!reconciled.applied && progress.queueShard === requestedShard) break;
|
|
3387
|
+
if (!result.done && progress.queueShard === requestedShard) break;
|
|
2015
3388
|
}
|
|
3389
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3390
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3391
|
+
if (current?.phase !== "queues") return;
|
|
3392
|
+
if ((await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3393
|
+
queueShard: 0,
|
|
3394
|
+
queueShardDeleted: 0
|
|
3395
|
+
}).queueShard >= current.queueShards) {
|
|
3396
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3397
|
+
...current,
|
|
3398
|
+
phase: "payload",
|
|
3399
|
+
generation: current.generation + 1
|
|
3400
|
+
});
|
|
3401
|
+
await txn.delete(CLEANUP_PROGRESS_KEY);
|
|
3402
|
+
await txn.setAlarm(this.now() + 1);
|
|
3403
|
+
}
|
|
3404
|
+
});
|
|
2016
3405
|
}
|
|
2017
3406
|
async deletePayloadBatch() {
|
|
2018
3407
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2019
3408
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2020
3409
|
if (!cleanup || cleanup.phase !== "payload") return true;
|
|
2021
|
-
const entries = await txn.list({
|
|
2022
|
-
|
|
3410
|
+
const entries = await txn.list({
|
|
3411
|
+
limit: 131,
|
|
3412
|
+
noCache: true
|
|
3413
|
+
});
|
|
3414
|
+
const keys = Array.from(entries.keys()).filter((key) => key !== "retention:cleanup" && key !== "retention:tombstone").slice(0, PAYLOAD_DELETE_BATCH);
|
|
2023
3415
|
const now = new Date(this.now());
|
|
2024
3416
|
await txn.put(TOMBSTONE_KEY, cleanupTombstone(cleanup, now));
|
|
2025
|
-
|
|
3417
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
2026
3418
|
if (keys.length > 0) {
|
|
2027
3419
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2028
3420
|
...cleanup,
|
|
2029
3421
|
deletedPayloadKeys: cleanup.deletedPayloadKeys + keys.length,
|
|
3422
|
+
generation: cleanup.generation + 1,
|
|
2030
3423
|
lastError: void 0
|
|
2031
3424
|
});
|
|
2032
3425
|
await txn.setAlarm(this.now() + 1);
|
|
@@ -2035,6 +3428,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2035
3428
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2036
3429
|
...cleanup,
|
|
2037
3430
|
phase: "tombstoned",
|
|
3431
|
+
generation: cleanup.generation + 1,
|
|
2038
3432
|
attempts: 0,
|
|
2039
3433
|
lastError: void 0,
|
|
2040
3434
|
tombstonedAt: now
|
|
@@ -2043,14 +3437,31 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2043
3437
|
return true;
|
|
2044
3438
|
});
|
|
2045
3439
|
}
|
|
2046
|
-
async recordCleanupFailure(error) {
|
|
3440
|
+
async recordCleanupFailure(error, expected) {
|
|
2047
3441
|
await this.ctx.storage.transaction(async (txn) => {
|
|
2048
3442
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2049
|
-
if (!cleanup || cleanup.phase === "tombstoned") return;
|
|
3443
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation || cleanup.phase === "tombstoned") return;
|
|
2050
3444
|
const attempts = cleanup.attempts + 1;
|
|
2051
3445
|
const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
|
|
2052
3446
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2053
3447
|
...cleanup,
|
|
3448
|
+
generation: cleanup.generation + 1,
|
|
3449
|
+
attempts,
|
|
3450
|
+
lastError: error instanceof Error ? error.message : String(error),
|
|
3451
|
+
lastAttemptAt: new Date(this.now())
|
|
3452
|
+
});
|
|
3453
|
+
await txn.setAlarm(this.now() + delay);
|
|
3454
|
+
});
|
|
3455
|
+
}
|
|
3456
|
+
async recordTerminalCleanupFailure(error, expected) {
|
|
3457
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3458
|
+
const cleanup = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3459
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
|
|
3460
|
+
const attempts = cleanup.attempts + 1;
|
|
3461
|
+
const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
|
|
3462
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3463
|
+
...cleanup,
|
|
3464
|
+
generation: cleanup.generation + 1,
|
|
2054
3465
|
attempts,
|
|
2055
3466
|
lastError: error instanceof Error ? error.message : String(error),
|
|
2056
3467
|
lastAttemptAt: new Date(this.now())
|
|
@@ -2059,7 +3470,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2059
3470
|
});
|
|
2060
3471
|
}
|
|
2061
3472
|
async claimInflight(params) {
|
|
2062
|
-
await this.ensureSchema();
|
|
2063
3473
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2064
3474
|
const existing = await txn.get("claim");
|
|
2065
3475
|
const now = this.now();
|
|
@@ -2072,7 +3482,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2072
3482
|
});
|
|
2073
3483
|
}
|
|
2074
3484
|
async releaseInflight() {
|
|
2075
|
-
await this.ensureSchema();
|
|
2076
3485
|
await this.ctx.storage.delete("claim");
|
|
2077
3486
|
}
|
|
2078
3487
|
};
|
|
@@ -2087,6 +3496,6 @@ var worker_default = { async fetch(request, env) {
|
|
|
2087
3496
|
return createRouter(env)(request);
|
|
2088
3497
|
} };
|
|
2089
3498
|
//#endregion
|
|
2090
|
-
export {
|
|
3499
|
+
export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO, RunFenceDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
|
|
2091
3500
|
|
|
2092
3501
|
//# sourceMappingURL=worker.js.map
|