@ewhauser/world-celld 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +49 -19
- package/celld-worker/README.md +5 -3
- package/celld-worker/worker.ts +9 -1
- package/celld-worker/wrangler.jsonc +13 -4
- package/dist/index.d.ts +168 -58
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +432 -124
- package/dist/index.js.map +1 -1
- package/dist/testing.d.ts +47 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +1988 -558
- package/dist/testing.js.map +1 -1
- package/dist/util-ya3V_FTA.js +558 -0
- package/dist/util-ya3V_FTA.js.map +1 -0
- package/dist/worker.d.ts +238 -108
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +2082 -520
- package/dist/worker.js.map +1 -1
- package/package.json +4 -2
- package/dist/retention-bLY1vcOu.js +0 -164
- package/dist/retention-bLY1vcOu.js.map +0 -1
package/dist/worker.js
CHANGED
|
@@ -96,6 +96,334 @@ function rpcStringify(value) {
|
|
|
96
96
|
function rpcParse(text) {
|
|
97
97
|
return JSON.parse(text, rpcReviver);
|
|
98
98
|
}
|
|
99
|
+
const INDEX_PROTOCOL_VERSION = "v1";
|
|
100
|
+
const HOOK_RELEASE_BATCH = 64;
|
|
101
|
+
/** Stable FNV-1a routing hash. Collisions share a shard but never a storage key. */
|
|
102
|
+
function stableIndexHash(value) {
|
|
103
|
+
let hash = 2166136261;
|
|
104
|
+
for (const byte of new TextEncoder().encode(value)) {
|
|
105
|
+
hash ^= byte;
|
|
106
|
+
hash = Math.imul(hash, 16777619);
|
|
107
|
+
}
|
|
108
|
+
return hash >>> 0;
|
|
109
|
+
}
|
|
110
|
+
function shardName(domain, value, cardinality) {
|
|
111
|
+
const width = Math.max(2, (cardinality - 1).toString(16).length);
|
|
112
|
+
const shard = stableIndexHash(value) % cardinality;
|
|
113
|
+
return `${domain}:${INDEX_PROTOCOL_VERSION}:${shard.toString(16).padStart(width, "0")}`;
|
|
114
|
+
}
|
|
115
|
+
function runCatalogShardName(runId) {
|
|
116
|
+
return shardName("run-catalog", runId, 16);
|
|
117
|
+
}
|
|
118
|
+
function hookTokenShardName(token) {
|
|
119
|
+
return shardName("hook-token", token, 32);
|
|
120
|
+
}
|
|
121
|
+
function hookIdShardName(hookId) {
|
|
122
|
+
return shardName("hook-id", hookId, 32);
|
|
123
|
+
}
|
|
124
|
+
function allRunCatalogShardNames() {
|
|
125
|
+
return Array.from({ length: 16 }, (_, shard) => `run-catalog:${INDEX_PROTOCOL_VERSION}:${shard.toString(16).padStart(2, "0")}`);
|
|
126
|
+
}
|
|
127
|
+
function stub(namespace, name) {
|
|
128
|
+
return namespace.get(namespace.idFromName(name));
|
|
129
|
+
}
|
|
130
|
+
function runKeys(run) {
|
|
131
|
+
const timestamp = run.createdAt.getTime().toString().padStart(13, "0");
|
|
132
|
+
return [`run:${run.workflowName}:${timestamp}:${run.runId}`, `runall:${timestamp}:${run.runId}`];
|
|
133
|
+
}
|
|
134
|
+
function groupByShard(values, route) {
|
|
135
|
+
const groups = /* @__PURE__ */ new Map();
|
|
136
|
+
for (const value of values) {
|
|
137
|
+
const name = route(value);
|
|
138
|
+
const group = groups.get(name);
|
|
139
|
+
if (group) group.push(value);
|
|
140
|
+
else groups.set(name, [value]);
|
|
141
|
+
}
|
|
142
|
+
return groups;
|
|
143
|
+
}
|
|
144
|
+
function sameOwner$1(left, right) {
|
|
145
|
+
return left.runId === right.runId && left.hookId === right.hookId;
|
|
146
|
+
}
|
|
147
|
+
function compareBytes(left, right) {
|
|
148
|
+
const length = Math.min(left.byteLength, right.byteLength);
|
|
149
|
+
for (let index = 0; index < length; index++) {
|
|
150
|
+
const difference = left[index] - right[index];
|
|
151
|
+
if (difference !== 0) return difference;
|
|
152
|
+
}
|
|
153
|
+
return left.byteLength - right.byteLength;
|
|
154
|
+
}
|
|
155
|
+
async function inBatches(values, operation) {
|
|
156
|
+
let deleted = 0;
|
|
157
|
+
for (let offset = 0; offset < values.length; offset += HOOK_RELEASE_BATCH) deleted += (await operation(values.slice(offset, offset + HOOK_RELEASE_BATCH))).deleted;
|
|
158
|
+
return deleted;
|
|
159
|
+
}
|
|
160
|
+
function createWorkflowIndex(bindings) {
|
|
161
|
+
const releaseHooks = async (request) => {
|
|
162
|
+
const tokenGroups = groupByShard(request.hooks, (hook) => hookTokenShardName(hook.token));
|
|
163
|
+
const idGroups = groupByShard(request.hooks, (hook) => hookIdShardName(hook.hookId));
|
|
164
|
+
const releases = [];
|
|
165
|
+
for (const [name, hooks] of tokenGroups) {
|
|
166
|
+
const target = stub(bindings.hookTokens, name);
|
|
167
|
+
releases.push(inBatches(hooks.map((hook) => ({
|
|
168
|
+
token: hook.token,
|
|
169
|
+
owner: {
|
|
170
|
+
runId: request.runId,
|
|
171
|
+
hookId: hook.hookId
|
|
172
|
+
}
|
|
173
|
+
})), (batch) => target.releaseBatch(batch)));
|
|
174
|
+
}
|
|
175
|
+
for (const [name, hooks] of idGroups) {
|
|
176
|
+
const target = stub(bindings.hookIds, name);
|
|
177
|
+
releases.push(inBatches(hooks.map((hook) => ({
|
|
178
|
+
hookId: hook.hookId,
|
|
179
|
+
owner: {
|
|
180
|
+
runId: request.runId,
|
|
181
|
+
hookId: hook.hookId
|
|
182
|
+
}
|
|
183
|
+
})), (batch) => target.releaseBatch(batch)));
|
|
184
|
+
}
|
|
185
|
+
return (await Promise.all(releases)).reduce((total, count) => total + count, 0);
|
|
186
|
+
};
|
|
187
|
+
return {
|
|
188
|
+
async commitRun(run, serializedMetadata, publicationExpiresAt) {
|
|
189
|
+
return await stub(bindings.runCatalog, runCatalogShardName(run.runId)).upsertRun(run.runId, runKeys(run), serializedMetadata, publicationExpiresAt);
|
|
190
|
+
},
|
|
191
|
+
async listRuns(options) {
|
|
192
|
+
const suppliedLimit = options?.limit;
|
|
193
|
+
const limit = Math.min(1e3, Math.max(1, typeof suppliedLimit === "number" && Number.isFinite(suppliedLimit) ? Math.floor(suppliedLimit) : 1e3));
|
|
194
|
+
const pages = await Promise.all(allRunCatalogShardNames().map((name) => stub(bindings.runCatalog, name).list({
|
|
195
|
+
...options,
|
|
196
|
+
limit: limit + 1
|
|
197
|
+
})));
|
|
198
|
+
const encoder = new TextEncoder();
|
|
199
|
+
const candidates = pages.flatMap((shardPage) => shardPage.keys.map((entry) => ({
|
|
200
|
+
entry,
|
|
201
|
+
encodedName: encoder.encode(entry.name)
|
|
202
|
+
}))).toSorted((left, right) => {
|
|
203
|
+
const compared = compareBytes(left.encodedName, right.encodedName);
|
|
204
|
+
return options?.reverse ? -compared : compared;
|
|
205
|
+
});
|
|
206
|
+
const page = candidates.slice(0, limit).map(({ entry }) => entry);
|
|
207
|
+
const listComplete = candidates.length <= limit && pages.every((entry) => entry.list_complete);
|
|
208
|
+
return {
|
|
209
|
+
keys: page,
|
|
210
|
+
list_complete: listComplete,
|
|
211
|
+
cursor: listComplete ? void 0 : page.at(-1)?.name
|
|
212
|
+
};
|
|
213
|
+
},
|
|
214
|
+
getHookByToken(token) {
|
|
215
|
+
return stub(bindings.hookTokens, hookTokenShardName(token)).get(token);
|
|
216
|
+
},
|
|
217
|
+
getHookById(hookId) {
|
|
218
|
+
return stub(bindings.hookIds, hookIdShardName(hookId)).get(hookId);
|
|
219
|
+
},
|
|
220
|
+
async reserveHook(token, owner) {
|
|
221
|
+
const proposedClaimId = crypto.randomUUID();
|
|
222
|
+
const tokenTarget = stub(bindings.hookTokens, hookTokenShardName(token));
|
|
223
|
+
const tokenClaim = await tokenTarget.reserve(token, owner, proposedClaimId);
|
|
224
|
+
if (!tokenClaim.claimed && !sameOwner$1(tokenClaim.holder, owner)) return {
|
|
225
|
+
admitted: false,
|
|
226
|
+
holder: tokenClaim.holder
|
|
227
|
+
};
|
|
228
|
+
const tokenClaimId = tokenClaim.claimed ? tokenClaim.claimId : void 0;
|
|
229
|
+
const hookIdTarget = stub(bindings.hookIds, hookIdShardName(owner.hookId));
|
|
230
|
+
const sharedClaimId = tokenClaimId ?? proposedClaimId;
|
|
231
|
+
let hookIdClaim;
|
|
232
|
+
try {
|
|
233
|
+
hookIdClaim = await hookIdTarget.reserve(owner.hookId, owner, sharedClaimId);
|
|
234
|
+
} catch (error) {
|
|
235
|
+
await Promise.allSettled([...tokenClaimId ? [tokenTarget.releaseClaim(token, owner, tokenClaimId)] : [], hookIdTarget.releaseClaim(owner.hookId, owner, sharedClaimId)]);
|
|
236
|
+
throw error;
|
|
237
|
+
}
|
|
238
|
+
if (!hookIdClaim.claimed) {
|
|
239
|
+
if (tokenClaimId) await tokenTarget.releaseClaim(token, owner, tokenClaimId);
|
|
240
|
+
return {
|
|
241
|
+
admitted: false,
|
|
242
|
+
holder: hookIdClaim.holder
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
return {
|
|
246
|
+
admitted: true,
|
|
247
|
+
reservation: {
|
|
248
|
+
claimId: `${tokenClaimId ?? "-"}:${hookIdClaim.claimId}`,
|
|
249
|
+
tokenClaimId,
|
|
250
|
+
hookIdClaimId: hookIdClaim.claimId
|
|
251
|
+
}
|
|
252
|
+
};
|
|
253
|
+
},
|
|
254
|
+
async finalizeHookIndexes(token, hookId, serializedHook, owner, reservation) {
|
|
255
|
+
if (!(await stub(bindings.hookTokens, hookTokenShardName(token)).finalize(token, serializedHook, owner, reservation?.tokenClaimId)).stored) return;
|
|
256
|
+
await stub(bindings.hookIds, hookIdShardName(hookId)).publish(hookId, serializedHook, owner, reservation?.hookIdClaimId);
|
|
257
|
+
},
|
|
258
|
+
async releaseHookReservation(token, owner, reservation) {
|
|
259
|
+
const releases = [];
|
|
260
|
+
if (reservation.tokenClaimId) releases.push(stub(bindings.hookTokens, hookTokenShardName(token)).releaseClaim(token, owner, reservation.tokenClaimId));
|
|
261
|
+
if (reservation.hookIdClaimId) releases.push(stub(bindings.hookIds, hookIdShardName(owner.hookId)).releaseClaim(owner.hookId, owner, reservation.hookIdClaimId));
|
|
262
|
+
await Promise.all(releases);
|
|
263
|
+
},
|
|
264
|
+
async releaseHookIndexes(request) {
|
|
265
|
+
return { deleted: await releaseHooks(request) };
|
|
266
|
+
},
|
|
267
|
+
async expireRun(request) {
|
|
268
|
+
const [catalog, hookDeletes] = await Promise.all([stub(bindings.runCatalog, runCatalogShardName(request.runId)).expireRun(request.runId, request.keys, request.expiredAt), releaseHooks({
|
|
269
|
+
runId: request.runId,
|
|
270
|
+
hooks: request.hooks
|
|
271
|
+
})]);
|
|
272
|
+
return { deleted: catalog.deleted + hookDeletes };
|
|
273
|
+
}
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
//#endregion
|
|
277
|
+
//#region src/stream-protocol.ts
|
|
278
|
+
/**
|
|
279
|
+
* Hard-cutover stream protocol shared by the client, Worker router, and
|
|
280
|
+
* StreamDO. Chunk payloads use a compact binary envelope on the public HTTP
|
|
281
|
+
* boundary and Uint8Array values on the DO/storage boundary.
|
|
282
|
+
*/
|
|
283
|
+
const STREAM_BATCH_CONTENT_TYPE = "application/vnd.world-celld.stream-batch.v1";
|
|
284
|
+
/** A chunk remains below the 2 MiB SQLite-backed DO key/value limit. */
|
|
285
|
+
const MAX_STREAM_CHUNK_BYTES = 1048576;
|
|
286
|
+
/** Keeps serialized RPC bodies well below the 32 MiB Workers RPC limit. */
|
|
287
|
+
const MAX_STREAM_BATCH_BYTES = 8388608;
|
|
288
|
+
const MAX_STREAM_READ_BYTES = 8388608;
|
|
289
|
+
const MAX_STREAM_ERROR_BYTES = 16384;
|
|
290
|
+
/** Must remain below the default 30 second fleet request deadline. */
|
|
291
|
+
const MAX_STREAM_LONG_POLL_MS = 2e4;
|
|
292
|
+
const WRITE_MAGIC = 1464030039;
|
|
293
|
+
const WRITE_ACK_MAGIC = 1464030017;
|
|
294
|
+
const READ_MAGIC = 1464030034;
|
|
295
|
+
const PROTOCOL_VERSION = 1;
|
|
296
|
+
const WRITE_HEADER_BYTES = 8;
|
|
297
|
+
const WRITE_ACK_BYTES = 16;
|
|
298
|
+
const READ_HEADER_BYTES = 28;
|
|
299
|
+
const STATE_TO_WIRE = {
|
|
300
|
+
open: 0,
|
|
301
|
+
closed: 1,
|
|
302
|
+
errored: 2,
|
|
303
|
+
expired: 3
|
|
304
|
+
};
|
|
305
|
+
function protocolError(message) {
|
|
306
|
+
const error = /* @__PURE__ */ new Error(`world-celld: invalid stream protocol: ${message}`);
|
|
307
|
+
error.name = "StreamProtocolError";
|
|
308
|
+
return error;
|
|
309
|
+
}
|
|
310
|
+
function requireSafeIndex(value, field) {
|
|
311
|
+
if (!Number.isSafeInteger(value) || value < -1 || value > 2147483647) throw protocolError(`${field} is out of range`);
|
|
312
|
+
}
|
|
313
|
+
function validateStreamWriteChunks(chunks) {
|
|
314
|
+
if (chunks.length === 0) throw protocolError("batch must contain at least one chunk");
|
|
315
|
+
if (chunks.length > 32) throw protocolError(`batch exceeds 32 chunks`);
|
|
316
|
+
let totalBytes = 0;
|
|
317
|
+
for (const chunk of chunks) {
|
|
318
|
+
if (!(chunk instanceof Uint8Array)) throw protocolError("chunk is not binary data");
|
|
319
|
+
if (chunk.byteLength > 1048576) throw protocolError(`chunk exceeds ${MAX_STREAM_CHUNK_BYTES} bytes`);
|
|
320
|
+
totalBytes += chunk.byteLength;
|
|
321
|
+
if (totalBytes > 8388608) throw protocolError(`batch exceeds ${MAX_STREAM_BATCH_BYTES} bytes`);
|
|
322
|
+
}
|
|
323
|
+
return totalBytes;
|
|
324
|
+
}
|
|
325
|
+
function validateStreamReadRequest(request) {
|
|
326
|
+
if (!request.runId) throw protocolError("runId is required");
|
|
327
|
+
if (!Number.isSafeInteger(request.startIndex) || request.startIndex < 0) throw protocolError("startIndex must be a non-negative safe integer");
|
|
328
|
+
if (!Number.isSafeInteger(request.maxChunks) || request.maxChunks < 0 || request.maxChunks > 32) throw protocolError(`maxChunks must be between 0 and 32`);
|
|
329
|
+
if (!Number.isSafeInteger(request.maxBytes) || request.maxBytes < 1048576 || request.maxBytes > 8388608) throw protocolError(`maxBytes must be between ${MAX_STREAM_CHUNK_BYTES} and ${MAX_STREAM_READ_BYTES}`);
|
|
330
|
+
if (!Number.isSafeInteger(request.waitMs) || request.waitMs < 0 || request.waitMs > 2e4) throw protocolError(`waitMs must be between 0 and ${MAX_STREAM_LONG_POLL_MS}`);
|
|
331
|
+
}
|
|
332
|
+
function decodeStreamWriteBatch(encoded) {
|
|
333
|
+
if (encoded.byteLength < WRITE_HEADER_BYTES) throw protocolError("truncated write header");
|
|
334
|
+
const view = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength);
|
|
335
|
+
if (view.getUint32(0) !== WRITE_MAGIC) throw protocolError("bad write magic");
|
|
336
|
+
if (view.getUint16(4) !== PROTOCOL_VERSION) throw protocolError("unsupported write version");
|
|
337
|
+
const count = view.getUint16(6);
|
|
338
|
+
const chunks = [];
|
|
339
|
+
let offset = WRITE_HEADER_BYTES;
|
|
340
|
+
for (let index = 0; index < count; index++) {
|
|
341
|
+
if (offset + 4 > encoded.byteLength) throw protocolError("truncated chunk length");
|
|
342
|
+
const length = view.getUint32(offset);
|
|
343
|
+
offset += 4;
|
|
344
|
+
if (offset + length > encoded.byteLength) throw protocolError("truncated chunk data");
|
|
345
|
+
chunks.push(new Uint8Array(encoded.subarray(offset, offset + length)));
|
|
346
|
+
offset += length;
|
|
347
|
+
}
|
|
348
|
+
if (offset !== encoded.byteLength) throw protocolError("trailing write data");
|
|
349
|
+
validateStreamWriteChunks(chunks);
|
|
350
|
+
return chunks;
|
|
351
|
+
}
|
|
352
|
+
function encodeStreamWriteResult(result) {
|
|
353
|
+
requireSafeIndex(result.startIndex, "startIndex");
|
|
354
|
+
requireSafeIndex(result.tailIndex, "tailIndex");
|
|
355
|
+
if (!Number.isSafeInteger(result.count) || result.count < 1 || result.count > 32) throw protocolError("write count is out of range");
|
|
356
|
+
const encoded = new Uint8Array(WRITE_ACK_BYTES);
|
|
357
|
+
const view = new DataView(encoded.buffer);
|
|
358
|
+
view.setUint32(0, WRITE_ACK_MAGIC);
|
|
359
|
+
view.setUint16(4, PROTOCOL_VERSION);
|
|
360
|
+
view.setUint16(6, result.count);
|
|
361
|
+
view.setInt32(8, result.startIndex);
|
|
362
|
+
view.setInt32(12, result.tailIndex);
|
|
363
|
+
return encoded;
|
|
364
|
+
}
|
|
365
|
+
function encodeStreamReadResult(result) {
|
|
366
|
+
requireSafeIndex(result.startIndex, "startIndex");
|
|
367
|
+
requireSafeIndex(result.tailIndex, "tailIndex");
|
|
368
|
+
let chunkBytes = 0;
|
|
369
|
+
for (const chunk of result.chunks) chunkBytes += 4 + chunk.byteLength;
|
|
370
|
+
if (result.chunks.length > 32 || chunkBytes > 8388736) throw protocolError("read result exceeds configured bounds");
|
|
371
|
+
const encoder = new TextEncoder();
|
|
372
|
+
const errorName = result.error ? encoder.encode(result.error.name) : /* @__PURE__ */ new Uint8Array();
|
|
373
|
+
const errorMessage = result.error ? encoder.encode(result.error.message) : /* @__PURE__ */ new Uint8Array();
|
|
374
|
+
if (errorName.byteLength + errorMessage.byteLength > 16384) throw protocolError(`stream error exceeds ${MAX_STREAM_ERROR_BYTES} bytes`);
|
|
375
|
+
const encoded = new Uint8Array(READ_HEADER_BYTES + chunkBytes + errorName.byteLength + errorMessage.byteLength);
|
|
376
|
+
const view = new DataView(encoded.buffer);
|
|
377
|
+
view.setUint32(0, READ_MAGIC);
|
|
378
|
+
view.setUint8(4, PROTOCOL_VERSION);
|
|
379
|
+
const state = STATE_TO_WIRE[result.state];
|
|
380
|
+
if (state === void 0) throw protocolError("unknown stream state");
|
|
381
|
+
view.setUint8(5, state);
|
|
382
|
+
view.setUint8(6, result.timedOut ? 1 : 0);
|
|
383
|
+
view.setUint8(7, 0);
|
|
384
|
+
view.setInt32(8, result.startIndex);
|
|
385
|
+
view.setInt32(12, result.tailIndex);
|
|
386
|
+
view.setUint32(16, result.chunks.length);
|
|
387
|
+
view.setUint32(20, errorName.byteLength);
|
|
388
|
+
view.setUint32(24, errorMessage.byteLength);
|
|
389
|
+
let offset = READ_HEADER_BYTES;
|
|
390
|
+
for (const chunk of result.chunks) {
|
|
391
|
+
view.setUint32(offset, chunk.byteLength);
|
|
392
|
+
offset += 4;
|
|
393
|
+
encoded.set(chunk, offset);
|
|
394
|
+
offset += chunk.byteLength;
|
|
395
|
+
}
|
|
396
|
+
encoded.set(errorName, offset);
|
|
397
|
+
offset += errorName.byteLength;
|
|
398
|
+
encoded.set(errorMessage, offset);
|
|
399
|
+
return encoded;
|
|
400
|
+
}
|
|
401
|
+
function normalizeStreamError(error) {
|
|
402
|
+
let normalized;
|
|
403
|
+
if (error instanceof Error) normalized = {
|
|
404
|
+
name: error.name || "Error",
|
|
405
|
+
message: error.message
|
|
406
|
+
};
|
|
407
|
+
else if (typeof error === "string") normalized = {
|
|
408
|
+
name: "Error",
|
|
409
|
+
message: error
|
|
410
|
+
};
|
|
411
|
+
else if (error && typeof error === "object" && typeof error.name === "string" && typeof error.message === "string") normalized = {
|
|
412
|
+
name: error.name,
|
|
413
|
+
message: error.message
|
|
414
|
+
};
|
|
415
|
+
else normalized = {
|
|
416
|
+
name: "Error",
|
|
417
|
+
message: String(error)
|
|
418
|
+
};
|
|
419
|
+
normalized.name = normalized.name.slice(0, 1024);
|
|
420
|
+
normalized.message = normalized.message.slice(0, 4096);
|
|
421
|
+
while (new TextEncoder().encode(normalized.name).byteLength + new TextEncoder().encode(normalized.message).byteLength > MAX_STREAM_ERROR_BYTES) {
|
|
422
|
+
normalized.message = normalized.message.slice(0, Math.max(0, normalized.message.length - 256));
|
|
423
|
+
if (normalized.message.length === 0) normalized.name = normalized.name.slice(0, Math.max(0, normalized.name.length - 256));
|
|
424
|
+
}
|
|
425
|
+
return normalized;
|
|
426
|
+
}
|
|
99
427
|
//#endregion
|
|
100
428
|
//#region src/worker/auth.ts
|
|
101
429
|
/**
|
|
@@ -145,18 +473,28 @@ async function authenticate(request, secret) {
|
|
|
145
473
|
* Routes:
|
|
146
474
|
* GET /v1/health — liveness + spec version (unauthenticated)
|
|
147
475
|
* POST /v1/rpc/{binding}/{name}/{method} — DO RPC (bearer auth)
|
|
476
|
+
* POST /v1/index/{domain}/{method} — cohesive index operation
|
|
477
|
+
* GET /v1/streams/{name}/chunks — bounded binary long-poll read
|
|
478
|
+
* POST /v1/streams/{name}/chunks — bounded binary batch append
|
|
148
479
|
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
* dispatch — everything else (including fetch/alarm) is unreachable.
|
|
480
|
+
* Generic RPC bodies use the tagged JSON codec. Stream chunk bodies use the
|
|
481
|
+
* compact binary stream protocol. Only whitelisted routes/methods dispatch.
|
|
152
482
|
*/
|
|
153
483
|
const WORLD_NAME = "world-celld";
|
|
154
484
|
const WORLD_VERSION = "0.1.0";
|
|
485
|
+
function workflowIndex(env) {
|
|
486
|
+
return createWorkflowIndex({
|
|
487
|
+
runCatalog: env.WORKFLOW_RUN_CATALOG,
|
|
488
|
+
hookTokens: env.WORKFLOW_HOOK_TOKENS,
|
|
489
|
+
hookIds: env.WORKFLOW_HOOK_IDS
|
|
490
|
+
});
|
|
491
|
+
}
|
|
155
492
|
const BINDINGS = {
|
|
156
493
|
runs: {
|
|
157
494
|
env: "WORKFLOW_DB",
|
|
158
495
|
methods: /* @__PURE__ */ new Set([
|
|
159
496
|
"applyEvent",
|
|
497
|
+
"getLifecycleStatus",
|
|
160
498
|
"getRun",
|
|
161
499
|
"getStep",
|
|
162
500
|
"getEvent",
|
|
@@ -166,16 +504,15 @@ const BINDINGS = {
|
|
|
166
504
|
"getCleanupStatus",
|
|
167
505
|
"scheduleCleanup",
|
|
168
506
|
"cleanupNow",
|
|
169
|
-
"rearmCleanup"
|
|
507
|
+
"rearmCleanup",
|
|
508
|
+
"resolveHookTokenClaim"
|
|
170
509
|
])
|
|
171
510
|
},
|
|
172
511
|
streams: {
|
|
173
512
|
env: "WORKFLOW_STREAMS",
|
|
174
513
|
methods: /* @__PURE__ */ new Set([
|
|
175
|
-
"writeChunk",
|
|
176
514
|
"closeStream",
|
|
177
|
-
"
|
|
178
|
-
"getInfo",
|
|
515
|
+
"failStream",
|
|
179
516
|
"registerStream",
|
|
180
517
|
"listStreams",
|
|
181
518
|
"expireRegistry",
|
|
@@ -183,20 +520,13 @@ const BINDINGS = {
|
|
|
183
520
|
"expireStream"
|
|
184
521
|
])
|
|
185
522
|
},
|
|
186
|
-
|
|
187
|
-
env: "
|
|
188
|
-
methods: /* @__PURE__ */ new Set([
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
"expireRun",
|
|
194
|
-
"list",
|
|
195
|
-
"reserveHookToken",
|
|
196
|
-
"finalizeHookIndexes",
|
|
197
|
-
"releaseHookToken",
|
|
198
|
-
"deleteHookIndexes"
|
|
199
|
-
])
|
|
523
|
+
"hook-tokens": {
|
|
524
|
+
env: "WORKFLOW_HOOK_TOKENS",
|
|
525
|
+
methods: /* @__PURE__ */ new Set(["get"])
|
|
526
|
+
},
|
|
527
|
+
"hook-ids": {
|
|
528
|
+
env: "WORKFLOW_HOOK_IDS",
|
|
529
|
+
methods: /* @__PURE__ */ new Set(["get"])
|
|
200
530
|
},
|
|
201
531
|
queue: {
|
|
202
532
|
env: "WORKFLOW_QUEUE",
|
|
@@ -207,12 +537,14 @@ const BINDINGS = {
|
|
|
207
537
|
"redriveDeadLetter",
|
|
208
538
|
"purgeDeadLetters",
|
|
209
539
|
"rearmAlarm",
|
|
210
|
-
"expireRun"
|
|
540
|
+
"expireRun",
|
|
541
|
+
"acknowledgeExpireRun"
|
|
211
542
|
])
|
|
212
543
|
}
|
|
213
544
|
};
|
|
214
545
|
/** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
|
|
215
546
|
const MAX_BODY_BYTES = 33554432;
|
|
547
|
+
const MAX_STREAM_WRITE_BODY_BYTES = 8389632;
|
|
216
548
|
function errorResponse(status, name, message) {
|
|
217
549
|
return Response.json({ error: {
|
|
218
550
|
name,
|
|
@@ -241,6 +573,49 @@ async function readBoundedBody(request) {
|
|
|
241
573
|
reader.releaseLock();
|
|
242
574
|
}
|
|
243
575
|
}
|
|
576
|
+
async function readBoundedBytes(request, maxBytes) {
|
|
577
|
+
if (!request.body) return /* @__PURE__ */ new Uint8Array();
|
|
578
|
+
const reader = request.body.getReader();
|
|
579
|
+
const chunks = [];
|
|
580
|
+
let total = 0;
|
|
581
|
+
try {
|
|
582
|
+
for (;;) {
|
|
583
|
+
const { done, value } = await reader.read();
|
|
584
|
+
if (done) break;
|
|
585
|
+
total += value.byteLength;
|
|
586
|
+
if (total > maxBytes) {
|
|
587
|
+
await reader.cancel("stream body exceeds configured limit").catch(() => void 0);
|
|
588
|
+
return null;
|
|
589
|
+
}
|
|
590
|
+
chunks.push(value);
|
|
591
|
+
}
|
|
592
|
+
} finally {
|
|
593
|
+
reader.releaseLock();
|
|
594
|
+
}
|
|
595
|
+
const body = new Uint8Array(total);
|
|
596
|
+
let offset = 0;
|
|
597
|
+
for (const chunk of chunks) {
|
|
598
|
+
body.set(chunk, offset);
|
|
599
|
+
offset += chunk.byteLength;
|
|
600
|
+
}
|
|
601
|
+
return body;
|
|
602
|
+
}
|
|
603
|
+
function parseBoundedInteger(url, name, minimum, maximum) {
|
|
604
|
+
const raw = url.searchParams.get(name);
|
|
605
|
+
const value = raw === null ? NaN : Number(raw);
|
|
606
|
+
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
|
607
|
+
const error = /* @__PURE__ */ new Error(`${name} must be an integer between ${minimum} and ${maximum}`);
|
|
608
|
+
error.name = "StreamProtocolError";
|
|
609
|
+
throw error;
|
|
610
|
+
}
|
|
611
|
+
return value;
|
|
612
|
+
}
|
|
613
|
+
function streamResponse(body) {
|
|
614
|
+
return new Response(body, {
|
|
615
|
+
status: 200,
|
|
616
|
+
headers: { "content-type": STREAM_BATCH_CONTENT_TYPE }
|
|
617
|
+
});
|
|
618
|
+
}
|
|
244
619
|
function createRouter(env) {
|
|
245
620
|
return async (request) => {
|
|
246
621
|
const url = new URL(request.url);
|
|
@@ -252,9 +627,79 @@ function createRouter(env) {
|
|
|
252
627
|
version: WORLD_VERSION,
|
|
253
628
|
specVersion: SPEC_VERSION_CURRENT
|
|
254
629
|
});
|
|
255
|
-
if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
|
|
256
630
|
const auth = await authenticate(request, env.WORLD_SECRET);
|
|
257
631
|
if (!auth.ok) return auth.response;
|
|
632
|
+
if (parts[1] === "index" && parts.length === 4) {
|
|
633
|
+
if (request.method !== "POST") return errorResponse(405, "MethodNotAllowed", "expected POST");
|
|
634
|
+
if (Number(request.headers.get("content-length") ?? "0") > MAX_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
635
|
+
let args;
|
|
636
|
+
try {
|
|
637
|
+
const text = await readBoundedBody(request);
|
|
638
|
+
if (text === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
639
|
+
const parsed = rpcParse(text);
|
|
640
|
+
if (!Array.isArray(parsed)) return errorResponse(400, "BadRequest", "body must be an argument array");
|
|
641
|
+
args = parsed;
|
|
642
|
+
} catch {
|
|
643
|
+
return errorResponse(400, "BadRequest", "malformed rpc body");
|
|
644
|
+
}
|
|
645
|
+
try {
|
|
646
|
+
const index = workflowIndex(env);
|
|
647
|
+
let result;
|
|
648
|
+
const operation = `${parts[2]}.${parts[3]}`;
|
|
649
|
+
if (operation === "runs.list") result = await index.listRuns(args[0]);
|
|
650
|
+
else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1], args[2]);
|
|
651
|
+
else if (operation === "runs.expire") result = await index.expireRun(args[0]);
|
|
652
|
+
else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
|
|
653
|
+
else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
|
|
654
|
+
else if (operation === "hooks.release-reservation") result = await index.releaseHookReservation(args[0], args[1], args[2]);
|
|
655
|
+
else if (operation === "hooks.release") result = await index.releaseHookIndexes(args[0]);
|
|
656
|
+
else return errorResponse(404, "NotFound", `unknown index operation: ${operation}`);
|
|
657
|
+
return new Response(rpcStringify(result ?? null), {
|
|
658
|
+
status: 200,
|
|
659
|
+
headers: { "content-type": "application/json" }
|
|
660
|
+
});
|
|
661
|
+
} catch (error) {
|
|
662
|
+
const err = error;
|
|
663
|
+
return Response.json({ error: {
|
|
664
|
+
name: err?.name ?? "Error",
|
|
665
|
+
message: err?.message ?? String(error),
|
|
666
|
+
status: typeof err?.status === "number" ? err.status : void 0
|
|
667
|
+
} }, { status: 500 });
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
if (parts[1] === "streams" && parts.length === 4 && parts[3] === "chunks") {
|
|
671
|
+
const namespace = env.WORKFLOW_STREAMS;
|
|
672
|
+
if (!namespace) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_STREAMS");
|
|
673
|
+
const name = decodeURIComponent(parts[2]);
|
|
674
|
+
const runId = url.searchParams.get("runId");
|
|
675
|
+
if (!runId) return errorResponse(400, "BadRequest", "runId is required");
|
|
676
|
+
const stub = namespace.get(namespace.idFromName(name));
|
|
677
|
+
try {
|
|
678
|
+
if (request.method === "POST") {
|
|
679
|
+
if (!request.headers.get("content-type")?.startsWith("application/vnd.world-celld.stream-batch.v1")) return errorResponse(415, "UnsupportedMediaType", STREAM_BATCH_CONTENT_TYPE);
|
|
680
|
+
if (Number(request.headers.get("content-length") ?? "0") > MAX_STREAM_WRITE_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
|
|
681
|
+
const body = await readBoundedBytes(request, MAX_STREAM_WRITE_BODY_BYTES);
|
|
682
|
+
if (body === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
|
|
683
|
+
return streamResponse(encodeStreamWriteResult(await stub.writeChunks(runId, decodeStreamWriteBatch(body))));
|
|
684
|
+
}
|
|
685
|
+
if (request.method === "GET") return streamResponse(encodeStreamReadResult(await stub.readChunks({
|
|
686
|
+
runId,
|
|
687
|
+
startIndex: parseBoundedInteger(url, "startIndex", 0, 2147483647),
|
|
688
|
+
maxChunks: parseBoundedInteger(url, "maxChunks", 0, 32),
|
|
689
|
+
maxBytes: parseBoundedInteger(url, "maxBytes", MAX_STREAM_CHUNK_BYTES, MAX_STREAM_READ_BYTES),
|
|
690
|
+
waitMs: parseBoundedInteger(url, "waitMs", 0, MAX_STREAM_LONG_POLL_MS)
|
|
691
|
+
}, request.signal)));
|
|
692
|
+
return errorResponse(405, "MethodNotAllowed", "expected GET or POST");
|
|
693
|
+
} catch (error) {
|
|
694
|
+
const err = error;
|
|
695
|
+
return Response.json({ error: {
|
|
696
|
+
name: err?.name ?? "Error",
|
|
697
|
+
message: err?.message ?? String(error),
|
|
698
|
+
status: typeof err?.status === "number" ? err.status : void 0
|
|
699
|
+
} }, { status: err?.name === "StreamProtocolError" ? 400 : 500 });
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
|
|
258
703
|
if (request.method !== "POST" || parts.length !== 5) return errorResponse(404, "NotFound", "expected POST /v1/rpc/{binding}/{name}/{method}");
|
|
259
704
|
const [, , bindingKey, encodedName, method] = parts;
|
|
260
705
|
const binding = BINDINGS[bindingKey];
|
|
@@ -299,162 +744,387 @@ function compact(obj) {
|
|
|
299
744
|
return value;
|
|
300
745
|
}
|
|
301
746
|
//#endregion
|
|
302
|
-
//#region src/worker/durable-objects/
|
|
747
|
+
//#region src/worker/durable-objects/hook-index-shared.ts
|
|
303
748
|
function sameOwner(left, right) {
|
|
304
749
|
return left.runId === right.runId && left.hookId === right.hookId;
|
|
305
750
|
}
|
|
306
|
-
function
|
|
751
|
+
function ownerFromSerializedHook(raw) {
|
|
307
752
|
const hook = parse(raw);
|
|
308
753
|
return {
|
|
309
754
|
runId: hook.runId,
|
|
310
755
|
hookId: hook.hookId
|
|
311
756
|
};
|
|
312
757
|
}
|
|
758
|
+
function indexKey(prefix, value) {
|
|
759
|
+
return `${prefix}:${encodeURIComponent(value)}`;
|
|
760
|
+
}
|
|
761
|
+
async function runIsActive(env, runId) {
|
|
762
|
+
const namespace = env.WORKFLOW_DB;
|
|
763
|
+
if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_DB binding");
|
|
764
|
+
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "active";
|
|
765
|
+
}
|
|
766
|
+
//#endregion
|
|
767
|
+
//#region src/lifecycle.ts
|
|
313
768
|
/**
|
|
314
|
-
*
|
|
315
|
-
* uses for run listing and hook token/id lookup.
|
|
769
|
+
* Hard protocol bounds used by lifecycle leases and derivative-fence cleanup.
|
|
316
770
|
*
|
|
317
|
-
*
|
|
318
|
-
*
|
|
319
|
-
* concurrent runs cannot both acquire the same token.
|
|
771
|
+
* These are not retention TTLs. They bound how long a request which was
|
|
772
|
+
* admitted before expiry may still mutate a derivative Durable Object:
|
|
320
773
|
*
|
|
321
|
-
*
|
|
322
|
-
*
|
|
323
|
-
*
|
|
324
|
-
*
|
|
774
|
+
* - fleet calls have at most three 5-minute attempts plus bounded retry delay;
|
|
775
|
+
* - one queue delivery owns its lease for at most 5 minutes plus 30 seconds;
|
|
776
|
+
* - hook admission spans reserve retries, one authoritative apply, and
|
|
777
|
+
* finalize retries before the exact claim must be reacquired.
|
|
325
778
|
*/
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
779
|
+
const MAX_FLEET_RPC_TIMEOUT_MS = 3e5;
|
|
780
|
+
const MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS = 1200900;
|
|
781
|
+
const MAX_QUEUE_DELIVERY_TIMEOUT_MS = 3e5;
|
|
782
|
+
const QUEUE_INFLIGHT_GRACE_MS = 3e4;
|
|
783
|
+
const QUEUE_FENCE_GRACE_MS = Math.max(MAX_FLEET_RPC_TIMEOUT_MS, 33e4);
|
|
784
|
+
const HOOK_CLAIM_LEASE_MS = 2101800;
|
|
785
|
+
const LIFECYCLE_COMPACTION_RETRY_MS = 1e3;
|
|
786
|
+
//#endregion
|
|
787
|
+
//#region src/worker/durable-objects/HookIdDO.ts
|
|
788
|
+
const MAX_RELEASE_BATCH$1 = 64;
|
|
789
|
+
const CLAIM_DEADLINE_PREFIX$1 = "claim-deadline:";
|
|
790
|
+
function pad$3(ms) {
|
|
791
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
792
|
+
}
|
|
793
|
+
function recordKey$1(hookId) {
|
|
794
|
+
return indexKey("hookid", hookId);
|
|
795
|
+
}
|
|
796
|
+
function claimKey$1(hookId) {
|
|
797
|
+
return indexKey("claim", hookId);
|
|
798
|
+
}
|
|
799
|
+
function claimDeadlineKey$1(hookId, expiresAt) {
|
|
800
|
+
return `${CLAIM_DEADLINE_PREFIX$1}${pad$3(expiresAt)}:${encodeURIComponent(hookId)}`;
|
|
801
|
+
}
|
|
802
|
+
/** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
|
|
803
|
+
var HookIdDO = class extends DurableObject {
|
|
804
|
+
now() {
|
|
805
|
+
return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
|
|
329
806
|
}
|
|
330
|
-
async
|
|
331
|
-
await this.ctx.storage.
|
|
807
|
+
async get(hookId) {
|
|
808
|
+
const raw = await this.ctx.storage.get(recordKey$1(hookId));
|
|
809
|
+
if (!raw) return null;
|
|
810
|
+
const owner = ownerFromSerializedHook(raw);
|
|
811
|
+
return await runIsActive(this.env, owner.runId) ? raw : null;
|
|
812
|
+
}
|
|
813
|
+
async reserve(hookId, owner, proposedClaimId) {
|
|
814
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
815
|
+
const record = recordKey$1(hookId);
|
|
816
|
+
const claim = claimKey$1(hookId);
|
|
817
|
+
const values = await txn.get([record, claim]);
|
|
818
|
+
const indexed = values.get(record);
|
|
819
|
+
if (indexed !== void 0) return {
|
|
820
|
+
claimed: false,
|
|
821
|
+
holder: ownerFromSerializedHook(indexed)
|
|
822
|
+
};
|
|
823
|
+
let existing = values.get(claim);
|
|
824
|
+
const now = this.now();
|
|
825
|
+
if (existing && existing.expiresAt <= now) {
|
|
826
|
+
await txn.delete([claim, existing.deadlineKey]);
|
|
827
|
+
existing = void 0;
|
|
828
|
+
}
|
|
829
|
+
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
830
|
+
claimed: false,
|
|
831
|
+
holder: existing.owner
|
|
832
|
+
};
|
|
833
|
+
if (existing === void 0 || existing.claimId !== proposedClaimId) {
|
|
834
|
+
if (existing) await txn.delete(existing.deadlineKey);
|
|
835
|
+
const expiresAt = now + HOOK_CLAIM_LEASE_MS;
|
|
836
|
+
const deadlineKey = claimDeadlineKey$1(hookId, expiresAt);
|
|
837
|
+
await txn.put(claim, {
|
|
838
|
+
owner,
|
|
839
|
+
claimId: proposedClaimId,
|
|
840
|
+
reservedAt: now,
|
|
841
|
+
expiresAt,
|
|
842
|
+
deadlineKey
|
|
843
|
+
});
|
|
844
|
+
await txn.put(deadlineKey, {
|
|
845
|
+
hookId,
|
|
846
|
+
claimId: proposedClaimId,
|
|
847
|
+
expiresAt
|
|
848
|
+
});
|
|
849
|
+
await this.armAtMost(txn, expiresAt);
|
|
850
|
+
return {
|
|
851
|
+
claimed: true,
|
|
852
|
+
claimId: proposedClaimId
|
|
853
|
+
};
|
|
854
|
+
}
|
|
855
|
+
return {
|
|
856
|
+
claimed: true,
|
|
857
|
+
claimId: existing.claimId
|
|
858
|
+
};
|
|
859
|
+
});
|
|
332
860
|
}
|
|
333
|
-
|
|
334
|
-
|
|
861
|
+
async publish(hookId, serializedHook, owner, claimId) {
|
|
862
|
+
if (!await runIsActive(this.env, owner.runId)) {
|
|
863
|
+
await this.releaseBatch([{
|
|
864
|
+
hookId,
|
|
865
|
+
owner
|
|
866
|
+
}]);
|
|
867
|
+
return { stored: false };
|
|
868
|
+
}
|
|
335
869
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
336
|
-
|
|
337
|
-
|
|
870
|
+
const record = recordKey$1(hookId);
|
|
871
|
+
const claim = claimKey$1(hookId);
|
|
872
|
+
const values = await txn.get([record, claim]);
|
|
873
|
+
const indexed = values.get(record);
|
|
874
|
+
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
|
|
875
|
+
const existingClaim = values.get(claim);
|
|
876
|
+
if (existingClaim && existingClaim.expiresAt <= this.now()) {
|
|
877
|
+
await txn.delete([claim, existingClaim.deadlineKey]);
|
|
878
|
+
return { stored: false };
|
|
879
|
+
}
|
|
880
|
+
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
|
|
881
|
+
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
|
|
882
|
+
await txn.put(record, serializedHook);
|
|
883
|
+
if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
|
|
338
884
|
return { stored: true };
|
|
339
885
|
});
|
|
340
886
|
}
|
|
341
|
-
async
|
|
342
|
-
await this.ctx.storage.
|
|
887
|
+
async releaseClaim(hookId, owner, claimId) {
|
|
888
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
889
|
+
const key = claimKey$1(hookId);
|
|
890
|
+
const claim = await txn.get(key);
|
|
891
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
|
|
892
|
+
});
|
|
343
893
|
}
|
|
344
|
-
async
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
894
|
+
async releaseBatch(hooks) {
|
|
895
|
+
if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
|
|
896
|
+
if (hooks.length === 0) return { deleted: 0 };
|
|
897
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
898
|
+
const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
|
|
899
|
+
const values = await txn.get(keys);
|
|
900
|
+
const deletes = [];
|
|
901
|
+
const deadlineDeletes = [];
|
|
902
|
+
for (const { hookId, owner } of hooks) {
|
|
903
|
+
const record = recordKey$1(hookId);
|
|
904
|
+
const claim = claimKey$1(hookId);
|
|
905
|
+
const indexed = values.get(record);
|
|
906
|
+
const existingClaim = values.get(claim);
|
|
907
|
+
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
908
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
|
|
909
|
+
deletes.push(claim);
|
|
910
|
+
deadlineDeletes.push(existingClaim.deadlineKey);
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
|
|
914
|
+
if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
|
|
915
|
+
return { deleted };
|
|
354
916
|
});
|
|
355
|
-
const names = Array.from(entries.keys());
|
|
356
|
-
const page = names.slice(0, limit);
|
|
357
|
-
const listComplete = names.length <= limit;
|
|
358
|
-
return {
|
|
359
|
-
keys: page.map((name) => ({ name })),
|
|
360
|
-
list_complete: listComplete,
|
|
361
|
-
cursor: listComplete ? void 0 : page.at(-1)
|
|
362
|
-
};
|
|
363
917
|
}
|
|
364
|
-
async
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
918
|
+
async alarm() {
|
|
919
|
+
try {
|
|
920
|
+
await this.compactClaims();
|
|
921
|
+
} catch {
|
|
922
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
async armAtMost(storage, at) {
|
|
926
|
+
const current = await storage.getAlarm();
|
|
927
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
928
|
+
}
|
|
929
|
+
async compactClaims() {
|
|
930
|
+
const now = this.now();
|
|
931
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
932
|
+
const due = await txn.list({
|
|
933
|
+
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
934
|
+
end: `${CLAIM_DEADLINE_PREFIX$1}${pad$3(now + 1)}`,
|
|
935
|
+
limit: 129
|
|
936
|
+
});
|
|
937
|
+
const page = Array.from(due).slice(0, 128);
|
|
938
|
+
const claims = await txn.get(page.map(([, item]) => claimKey$1(item.hookId)));
|
|
939
|
+
const deletes = page.flatMap(([deadline, item]) => {
|
|
940
|
+
const key = claimKey$1(item.hookId);
|
|
941
|
+
const claim = claims.get(key);
|
|
942
|
+
return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
|
|
943
|
+
});
|
|
944
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
945
|
+
if (due.size > 128) {
|
|
946
|
+
await txn.setAlarm(now + 1);
|
|
947
|
+
return;
|
|
948
|
+
}
|
|
949
|
+
const nextAt = (await txn.list({
|
|
950
|
+
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
951
|
+
limit: 1
|
|
952
|
+
})).values().next().value?.expiresAt;
|
|
953
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
954
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
955
|
+
});
|
|
956
|
+
}
|
|
957
|
+
};
|
|
958
|
+
//#endregion
|
|
959
|
+
//#region src/worker/durable-objects/HookTokenDO.ts
|
|
960
|
+
const MAX_RELEASE_BATCH = 64;
|
|
961
|
+
const CLAIM_DEADLINE_PREFIX = "claim-deadline:";
|
|
962
|
+
function pad$2(ms) {
|
|
963
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
964
|
+
}
|
|
965
|
+
function recordKey(token) {
|
|
966
|
+
return indexKey("hook", token);
|
|
967
|
+
}
|
|
968
|
+
function claimKey(token) {
|
|
969
|
+
return indexKey("claim", token);
|
|
970
|
+
}
|
|
971
|
+
function claimDeadlineKey(token, expiresAt) {
|
|
972
|
+
return `${CLAIM_DEADLINE_PREFIX}${pad$2(expiresAt)}:${encodeURIComponent(token)}`;
|
|
973
|
+
}
|
|
974
|
+
/** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
|
|
975
|
+
var HookTokenDO = class extends DurableObject {
|
|
976
|
+
now() {
|
|
977
|
+
return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
|
|
978
|
+
}
|
|
979
|
+
async get(token) {
|
|
980
|
+
const raw = await this.ctx.storage.get(recordKey(token));
|
|
981
|
+
if (!raw) return null;
|
|
982
|
+
const owner = ownerFromSerializedHook(raw);
|
|
983
|
+
return await runIsActive(this.env, owner.runId) ? raw : null;
|
|
984
|
+
}
|
|
985
|
+
async reserve(token, owner, proposedClaimId) {
|
|
986
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
987
|
+
const record = recordKey(token);
|
|
988
|
+
const claim = claimKey(token);
|
|
989
|
+
const values = await txn.get([record, claim]);
|
|
990
|
+
const indexed = values.get(record);
|
|
991
|
+
if (indexed !== void 0) return {
|
|
368
992
|
claimed: false,
|
|
369
|
-
holder:
|
|
993
|
+
holder: ownerFromSerializedHook(indexed)
|
|
370
994
|
};
|
|
371
|
-
|
|
372
|
-
const
|
|
373
|
-
if (
|
|
995
|
+
let existing = values.get(claim);
|
|
996
|
+
const now = this.now();
|
|
997
|
+
if (existing && existing.expiresAt <= now) {
|
|
998
|
+
await txn.delete([claim, existing.deadlineKey]);
|
|
999
|
+
existing = void 0;
|
|
1000
|
+
}
|
|
1001
|
+
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
374
1002
|
claimed: false,
|
|
375
|
-
holder:
|
|
1003
|
+
holder: existing.owner
|
|
1004
|
+
};
|
|
1005
|
+
if (existing === void 0) {
|
|
1006
|
+
const expiresAt = now + HOOK_CLAIM_LEASE_MS;
|
|
1007
|
+
const deadlineKey = claimDeadlineKey(token, expiresAt);
|
|
1008
|
+
await txn.put(claim, {
|
|
1009
|
+
owner,
|
|
1010
|
+
claimId: proposedClaimId,
|
|
1011
|
+
reservedAt: now,
|
|
1012
|
+
expiresAt,
|
|
1013
|
+
deadlineKey
|
|
1014
|
+
});
|
|
1015
|
+
await txn.put(deadlineKey, {
|
|
1016
|
+
token,
|
|
1017
|
+
claimId: proposedClaimId,
|
|
1018
|
+
expiresAt
|
|
1019
|
+
});
|
|
1020
|
+
await this.armAtMost(txn, expiresAt);
|
|
1021
|
+
return {
|
|
1022
|
+
claimed: true,
|
|
1023
|
+
claimId: proposedClaimId
|
|
1024
|
+
};
|
|
1025
|
+
}
|
|
1026
|
+
return {
|
|
1027
|
+
claimed: true,
|
|
1028
|
+
claimId: existing.claimId
|
|
376
1029
|
};
|
|
377
|
-
await txn.put(claimKey, {
|
|
378
|
-
owner,
|
|
379
|
-
reservedAt: Date.now()
|
|
380
|
-
});
|
|
381
|
-
return { claimed: true };
|
|
382
1030
|
});
|
|
383
1031
|
}
|
|
384
|
-
async
|
|
385
|
-
await this.
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
1032
|
+
async finalize(token, serializedHook, owner, claimId) {
|
|
1033
|
+
if (!await runIsActive(this.env, owner.runId)) {
|
|
1034
|
+
await this.releaseBatch([{
|
|
1035
|
+
token,
|
|
1036
|
+
owner
|
|
1037
|
+
}]);
|
|
1038
|
+
return { stored: false };
|
|
1039
|
+
}
|
|
1040
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1041
|
+
const record = recordKey(token);
|
|
1042
|
+
const claim = claimKey(token);
|
|
1043
|
+
const values = await txn.get([record, claim]);
|
|
1044
|
+
const indexed = values.get(record);
|
|
1045
|
+
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
|
|
1046
|
+
const existingClaim = values.get(claim);
|
|
1047
|
+
if (existingClaim && existingClaim.expiresAt <= this.now()) {
|
|
1048
|
+
await txn.delete([claim, existingClaim.deadlineKey]);
|
|
1049
|
+
return { stored: false };
|
|
390
1050
|
}
|
|
391
|
-
|
|
392
|
-
if (
|
|
393
|
-
|
|
394
|
-
if (
|
|
395
|
-
|
|
396
|
-
await txn.put(`hookid:${hookId}`, serializedHook);
|
|
397
|
-
if (claim !== void 0) await txn.delete(`hookclaim:${token}`);
|
|
1051
|
+
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
|
|
1052
|
+
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
|
|
1053
|
+
await txn.put(record, serializedHook);
|
|
1054
|
+
if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
|
|
1055
|
+
return { stored: true };
|
|
398
1056
|
});
|
|
399
1057
|
}
|
|
400
|
-
async
|
|
1058
|
+
async releaseClaim(token, owner, claimId) {
|
|
401
1059
|
await this.ctx.storage.transaction(async (txn) => {
|
|
402
|
-
const key =
|
|
1060
|
+
const key = claimKey(token);
|
|
403
1061
|
const claim = await txn.get(key);
|
|
404
|
-
if (claim
|
|
1062
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
|
|
405
1063
|
});
|
|
406
1064
|
}
|
|
407
|
-
async
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
const idKey = `hookid:${hookId}`;
|
|
411
|
-
const claimKey = `hookclaim:${token}`;
|
|
412
|
-
const [byToken, byId, claim] = await Promise.all([
|
|
413
|
-
txn.get(tokenKey),
|
|
414
|
-
txn.get(idKey),
|
|
415
|
-
txn.get(claimKey)
|
|
416
|
-
]);
|
|
417
|
-
if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) await txn.delete(tokenKey);
|
|
418
|
-
if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) await txn.delete(idKey);
|
|
419
|
-
if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(claimKey);
|
|
420
|
-
});
|
|
421
|
-
}
|
|
422
|
-
/**
|
|
423
|
-
* Fence a run and remove all of its known derived indexes atomically.
|
|
424
|
-
* Once the fence exists, putOwned() and hook finalization cannot resurrect
|
|
425
|
-
* entries from a delayed request.
|
|
426
|
-
*/
|
|
427
|
-
async expireRun(request) {
|
|
1065
|
+
async releaseBatch(hooks) {
|
|
1066
|
+
if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
|
|
1067
|
+
if (hooks.length === 0) return { deleted: 0 };
|
|
428
1068
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
const
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
txn.get(idKey),
|
|
443
|
-
txn.get(claimKey)
|
|
444
|
-
]);
|
|
445
|
-
if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) {
|
|
446
|
-
if (await txn.delete(tokenKey)) deleted++;
|
|
447
|
-
}
|
|
448
|
-
if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) {
|
|
449
|
-
if (await txn.delete(idKey)) deleted++;
|
|
450
|
-
}
|
|
451
|
-
if (claim !== void 0 && sameOwner(claim.owner, owner)) {
|
|
452
|
-
if (await txn.delete(claimKey)) deleted++;
|
|
1069
|
+
const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
|
|
1070
|
+
const values = await txn.get(keys);
|
|
1071
|
+
const deletes = [];
|
|
1072
|
+
const deadlineDeletes = [];
|
|
1073
|
+
for (const { token, owner } of hooks) {
|
|
1074
|
+
const record = recordKey(token);
|
|
1075
|
+
const claim = claimKey(token);
|
|
1076
|
+
const indexed = values.get(record);
|
|
1077
|
+
const existingClaim = values.get(claim);
|
|
1078
|
+
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
1079
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
|
|
1080
|
+
deletes.push(claim);
|
|
1081
|
+
deadlineDeletes.push(existingClaim.deadlineKey);
|
|
453
1082
|
}
|
|
454
1083
|
}
|
|
1084
|
+
const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
|
|
1085
|
+
if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
|
|
455
1086
|
return { deleted };
|
|
456
1087
|
});
|
|
457
1088
|
}
|
|
1089
|
+
async alarm() {
|
|
1090
|
+
try {
|
|
1091
|
+
await this.compactClaims();
|
|
1092
|
+
} catch {
|
|
1093
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
async armAtMost(storage, at) {
|
|
1097
|
+
const current = await storage.getAlarm();
|
|
1098
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
1099
|
+
}
|
|
1100
|
+
async compactClaims() {
|
|
1101
|
+
const now = this.now();
|
|
1102
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1103
|
+
const due = await txn.list({
|
|
1104
|
+
prefix: CLAIM_DEADLINE_PREFIX,
|
|
1105
|
+
end: `${CLAIM_DEADLINE_PREFIX}${pad$2(now + 1)}`,
|
|
1106
|
+
limit: 129
|
|
1107
|
+
});
|
|
1108
|
+
const page = Array.from(due).slice(0, 128);
|
|
1109
|
+
const claims = await txn.get(page.map(([, item]) => claimKey(item.token)));
|
|
1110
|
+
const deletes = page.flatMap(([deadline, item]) => {
|
|
1111
|
+
const key = claimKey(item.token);
|
|
1112
|
+
const claim = claims.get(key);
|
|
1113
|
+
return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
|
|
1114
|
+
});
|
|
1115
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
1116
|
+
if (due.size > 128) {
|
|
1117
|
+
await txn.setAlarm(now + 1);
|
|
1118
|
+
return;
|
|
1119
|
+
}
|
|
1120
|
+
const nextAt = (await txn.list({
|
|
1121
|
+
prefix: CLAIM_DEADLINE_PREFIX,
|
|
1122
|
+
limit: 1
|
|
1123
|
+
})).values().next().value?.expiresAt;
|
|
1124
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
1125
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
1126
|
+
});
|
|
1127
|
+
}
|
|
458
1128
|
};
|
|
459
1129
|
//#endregion
|
|
460
1130
|
//#region src/worker/durable-objects/QueueDO.ts
|
|
@@ -467,15 +1137,26 @@ const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
|
|
|
467
1137
|
const DEFAULT_MAX_ATTEMPTS = 5;
|
|
468
1138
|
const DEFAULT_MAX_INFLIGHT = 5;
|
|
469
1139
|
const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
|
|
470
|
-
/** Grace period past the delivery timeout before a claim counts as lost. */
|
|
471
|
-
const INFLIGHT_GRACE_MS = 3e4;
|
|
472
1140
|
/** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
|
|
473
1141
|
const MIN_ALARM_DELAY_MS = 1;
|
|
474
|
-
|
|
1142
|
+
/** Bound expiry recovery so one alarm cannot monopolize the cell. */
|
|
1143
|
+
const EXPIRED_INFLIGHT_BATCH = 128;
|
|
1144
|
+
const STORAGE_BATCH_SIZE$2 = 128;
|
|
1145
|
+
const EXPIRE_RUN_REFERENCE_BATCH = 64;
|
|
1146
|
+
const PURGE_DEAD_LETTER_BATCH = 128;
|
|
1147
|
+
const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
|
|
1148
|
+
const EXPIRED_RUN_GC_PREFIX = "expired-run-gc:";
|
|
1149
|
+
function pad$1(ms) {
|
|
475
1150
|
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
476
1151
|
}
|
|
477
1152
|
function dueKey(atMs, messageId) {
|
|
478
|
-
return `due:${pad(atMs)}:${messageId}`;
|
|
1153
|
+
return `due:${pad$1(atMs)}:${messageId}`;
|
|
1154
|
+
}
|
|
1155
|
+
function inflightKey(deadlineMs, messageId) {
|
|
1156
|
+
return `${INFLIGHT_DEADLINE_PREFIX}${pad$1(deadlineMs)}:${messageId}`;
|
|
1157
|
+
}
|
|
1158
|
+
function deadlineFromInflightKey(key) {
|
|
1159
|
+
return Number.parseInt(key.slice(18, 31), 10);
|
|
479
1160
|
}
|
|
480
1161
|
function runReferenceKey(runId, messageId) {
|
|
481
1162
|
return `run:${runId}:${messageId}`;
|
|
@@ -483,9 +1164,40 @@ function runReferenceKey(runId, messageId) {
|
|
|
483
1164
|
function expiredRunKey(runId) {
|
|
484
1165
|
return `expired-run:${runId}`;
|
|
485
1166
|
}
|
|
1167
|
+
function expiredRunGcKey(runId, compactAt) {
|
|
1168
|
+
return `${EXPIRED_RUN_GC_PREFIX}${pad$1(compactAt)}:${encodeURIComponent(runId)}`;
|
|
1169
|
+
}
|
|
1170
|
+
function boundedLimit$1(value, fallback, maximum) {
|
|
1171
|
+
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
1172
|
+
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
1173
|
+
}
|
|
1174
|
+
async function getMany$1(storage, keys) {
|
|
1175
|
+
const result = /* @__PURE__ */ new Map();
|
|
1176
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$2) {
|
|
1177
|
+
const page = await storage.get(keys.slice(offset, offset + STORAGE_BATCH_SIZE$2));
|
|
1178
|
+
for (const [key, value] of page) result.set(key, value);
|
|
1179
|
+
}
|
|
1180
|
+
return result;
|
|
1181
|
+
}
|
|
1182
|
+
async function putMany(storage, entries) {
|
|
1183
|
+
const all = Array.from(entries);
|
|
1184
|
+
for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE$2) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE$2)));
|
|
1185
|
+
}
|
|
1186
|
+
async function deleteMany$1(storage, keys) {
|
|
1187
|
+
let deleted = 0;
|
|
1188
|
+
const unique = Array.from(new Set(keys));
|
|
1189
|
+
for (let offset = 0; offset < unique.length; offset += STORAGE_BATCH_SIZE$2) deleted += await storage.delete(unique.slice(offset, offset + STORAGE_BATCH_SIZE$2));
|
|
1190
|
+
return deleted;
|
|
1191
|
+
}
|
|
486
1192
|
function backoffSeconds(attempt) {
|
|
487
1193
|
return Math.min(60, 2 ** attempt);
|
|
488
1194
|
}
|
|
1195
|
+
async function cancelUnusedResponseBody(response) {
|
|
1196
|
+
try {
|
|
1197
|
+
await response.body?.cancel();
|
|
1198
|
+
} catch {}
|
|
1199
|
+
}
|
|
1200
|
+
const MAX_QUEUE_INFLIGHT = 128;
|
|
489
1201
|
var QueueDO = class extends DurableObject {
|
|
490
1202
|
#now() {
|
|
491
1203
|
const clock = this.env?.clock;
|
|
@@ -500,15 +1212,34 @@ var QueueDO = class extends DurableObject {
|
|
|
500
1212
|
const parsed = raw ? Number.parseInt(raw, 10) : NaN;
|
|
501
1213
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
502
1214
|
}
|
|
1215
|
+
#maxInflight() {
|
|
1216
|
+
const configured = this.#intVar("QUEUE_MAX_INFLIGHT", DEFAULT_MAX_INFLIGHT);
|
|
1217
|
+
if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
|
|
1218
|
+
return configured;
|
|
1219
|
+
}
|
|
1220
|
+
#deliveryTimeoutMs() {
|
|
1221
|
+
const configured = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
1222
|
+
if (configured > 3e5) throw new Error(`QUEUE_DELIVERY_TIMEOUT_MS must be at most ${MAX_QUEUE_DELIVERY_TIMEOUT_MS}`);
|
|
1223
|
+
return configured;
|
|
1224
|
+
}
|
|
1225
|
+
async #authoritativeRunExpired(runId) {
|
|
1226
|
+
const namespace = this.env?.WORKFLOW_DB;
|
|
1227
|
+
if (!namespace) throw new Error("world-celld queue missing WORKFLOW_DB binding");
|
|
1228
|
+
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "expired";
|
|
1229
|
+
}
|
|
1230
|
+
async #withRunMutation(runId, onExpired, onAllowed) {
|
|
1231
|
+
const expired = await this.#authoritativeRunExpired(runId);
|
|
1232
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1233
|
+
if (await txn.get(expiredRunKey(runId)) || expired) return await onExpired(txn);
|
|
1234
|
+
return await onAllowed(txn);
|
|
1235
|
+
});
|
|
1236
|
+
}
|
|
503
1237
|
async enqueue(request) {
|
|
1238
|
+
this.#maxInflight();
|
|
1239
|
+
this.#deliveryTimeoutMs();
|
|
504
1240
|
const storage = this.ctx.storage;
|
|
505
1241
|
const now = this.#now();
|
|
506
|
-
|
|
507
|
-
if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
|
|
508
|
-
ok: false,
|
|
509
|
-
code: "RUN_EXPIRED",
|
|
510
|
-
message: `Workflow run "${request.runId}" has expired`
|
|
511
|
-
};
|
|
1242
|
+
const persist = async (txn) => {
|
|
512
1243
|
const pinned = await txn.get("cfg");
|
|
513
1244
|
if (!pinned) await txn.put("cfg", {
|
|
514
1245
|
queueShards: request.config.queueShards,
|
|
@@ -560,7 +1291,12 @@ var QueueDO = class extends DurableObject {
|
|
|
560
1291
|
messageId: row.messageId,
|
|
561
1292
|
deduped: false
|
|
562
1293
|
};
|
|
563
|
-
}
|
|
1294
|
+
};
|
|
1295
|
+
return request.runId ? await this.#withRunMutation(request.runId, async () => ({
|
|
1296
|
+
ok: false,
|
|
1297
|
+
code: "RUN_EXPIRED",
|
|
1298
|
+
message: `Workflow run "${request.runId}" has expired`
|
|
1299
|
+
}), persist) : await storage.transaction(persist);
|
|
564
1300
|
}
|
|
565
1301
|
async alarm() {
|
|
566
1302
|
let claimed = [];
|
|
@@ -574,8 +1310,11 @@ var QueueDO = class extends DurableObject {
|
|
|
574
1310
|
};
|
|
575
1311
|
if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
|
|
576
1312
|
else await attempt();
|
|
577
|
-
if (failure !== null)
|
|
578
|
-
|
|
1313
|
+
if (failure !== null) {
|
|
1314
|
+
await this.ctx.storage.setAlarm(this.#now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1315
|
+
throw failure;
|
|
1316
|
+
}
|
|
1317
|
+
for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
|
|
579
1318
|
}
|
|
580
1319
|
/**
|
|
581
1320
|
* Storage-only: recover lost inflight claims, claim due messages up to the
|
|
@@ -584,49 +1323,83 @@ var QueueDO = class extends DurableObject {
|
|
|
584
1323
|
async #claimPhase() {
|
|
585
1324
|
const storage = this.ctx.storage;
|
|
586
1325
|
const now = this.#now();
|
|
587
|
-
const maxInflight = this.#
|
|
1326
|
+
const maxInflight = this.#maxInflight();
|
|
588
1327
|
return await storage.transaction(async (txn) => {
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
1328
|
+
if (await this.#compactExpiredRunFences(txn, now)) {
|
|
1329
|
+
await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
|
|
1330
|
+
return [];
|
|
1331
|
+
}
|
|
1332
|
+
const expiredPage = await txn.list({
|
|
1333
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1334
|
+
end: `${INFLIGHT_DEADLINE_PREFIX}${pad$1(now + 1)}`,
|
|
1335
|
+
limit: 129
|
|
1336
|
+
});
|
|
1337
|
+
const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
|
|
1338
|
+
const expiredRows = await getMany$1(txn, expired.map(([, messageId]) => `msg:${messageId}`));
|
|
1339
|
+
await deleteMany$1(txn, expired.map(([key]) => key));
|
|
1340
|
+
const recoveryWrites = [];
|
|
1341
|
+
for (const [, messageId] of expired) {
|
|
1342
|
+
const row = expiredRows.get(`msg:${messageId}`);
|
|
595
1343
|
if (row) {
|
|
596
1344
|
const scheduleKey = dueKey(now, messageId);
|
|
597
|
-
|
|
598
|
-
if (row.runId)
|
|
1345
|
+
recoveryWrites.push([scheduleKey, messageId]);
|
|
1346
|
+
if (row.runId) recoveryWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
599
1347
|
messageId: row.messageId,
|
|
600
1348
|
dueKey: scheduleKey,
|
|
601
1349
|
idempotencyKey: row.idempotencyKey
|
|
602
|
-
});
|
|
1350
|
+
}]);
|
|
603
1351
|
}
|
|
604
|
-
}
|
|
1352
|
+
}
|
|
1353
|
+
await putMany(txn, recoveryWrites);
|
|
1354
|
+
if (expiredPage.size > EXPIRED_INFLIGHT_BATCH) {
|
|
1355
|
+
await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
|
|
1356
|
+
return [];
|
|
1357
|
+
}
|
|
1358
|
+
const inflightCount = (await txn.list({
|
|
1359
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1360
|
+
limit: maxInflight
|
|
1361
|
+
})).size;
|
|
605
1362
|
const rows = [];
|
|
606
1363
|
if (inflightCount < maxInflight) {
|
|
607
1364
|
const due = await txn.list({
|
|
608
1365
|
prefix: "due:",
|
|
609
|
-
end: `due:${pad(now + 1)}`,
|
|
1366
|
+
end: `due:${pad$1(now + 1)}`,
|
|
610
1367
|
limit: maxInflight - inflightCount
|
|
611
1368
|
});
|
|
612
|
-
const timeoutMs = this.#
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
1369
|
+
const timeoutMs = this.#deliveryTimeoutMs();
|
|
1370
|
+
const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
|
|
1371
|
+
await deleteMany$1(txn, Array.from(due.keys()));
|
|
1372
|
+
const claimWrites = [];
|
|
1373
|
+
for (const [, messageId] of due) {
|
|
1374
|
+
const row = dueRows.get(`msg:${messageId}`);
|
|
616
1375
|
if (!row) continue;
|
|
617
|
-
|
|
618
|
-
|
|
1376
|
+
const claimKey = inflightKey(now + timeoutMs + QUEUE_INFLIGHT_GRACE_MS, messageId);
|
|
1377
|
+
claimWrites.push([claimKey, messageId]);
|
|
1378
|
+
if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
1379
|
+
messageId: row.messageId,
|
|
1380
|
+
inflightKey: claimKey,
|
|
1381
|
+
idempotencyKey: row.idempotencyKey
|
|
1382
|
+
}]);
|
|
1383
|
+
rows.push({
|
|
1384
|
+
row,
|
|
1385
|
+
inflightKey: claimKey
|
|
1386
|
+
});
|
|
619
1387
|
}
|
|
1388
|
+
await putMany(txn, claimWrites);
|
|
620
1389
|
}
|
|
621
1390
|
await this.#scheduleNextAlarm(txn, now);
|
|
622
1391
|
return rows;
|
|
623
1392
|
});
|
|
624
1393
|
}
|
|
625
|
-
async #deliver(row) {
|
|
1394
|
+
async #deliver({ row, inflightKey: claimKey }) {
|
|
626
1395
|
const storage = this.ctx.storage;
|
|
627
|
-
const timeoutMs = this.#
|
|
1396
|
+
const timeoutMs = this.#deliveryTimeoutMs();
|
|
628
1397
|
const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
|
|
629
1398
|
const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
|
|
1399
|
+
if (row.runId && await this.#authoritativeRunExpired(row.runId)) {
|
|
1400
|
+
await storage.transaction((txn) => this.#dropExpiredMessage(txn, row, claimKey, this.#now()));
|
|
1401
|
+
return;
|
|
1402
|
+
}
|
|
630
1403
|
let response = null;
|
|
631
1404
|
let transportError = null;
|
|
632
1405
|
try {
|
|
@@ -646,47 +1419,44 @@ var QueueDO = class extends DurableObject {
|
|
|
646
1419
|
transportError = error;
|
|
647
1420
|
}
|
|
648
1421
|
const now = this.#now();
|
|
649
|
-
if (response?.ok)
|
|
650
|
-
|
|
1422
|
+
if (response?.ok) {
|
|
1423
|
+
await cancelUnusedResponseBody(response);
|
|
1424
|
+
await this.#ack(row, claimKey);
|
|
1425
|
+
} else if (response && response.status === 503) {
|
|
651
1426
|
let timeoutSeconds;
|
|
652
1427
|
try {
|
|
653
1428
|
timeoutSeconds = (await response.json())?.timeoutSeconds;
|
|
654
1429
|
} catch {
|
|
655
1430
|
timeoutSeconds = void 0;
|
|
656
1431
|
}
|
|
657
|
-
if (typeof timeoutSeconds === "number")
|
|
658
|
-
|
|
659
|
-
await txn.delete(
|
|
660
|
-
|
|
661
|
-
await txn.
|
|
662
|
-
if (row.
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
1432
|
+
if (typeof timeoutSeconds === "number") {
|
|
1433
|
+
const reschedule = async (txn) => {
|
|
1434
|
+
if (!await txn.delete(claimKey)) return;
|
|
1435
|
+
const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
|
|
1436
|
+
await txn.put(scheduleKey, row.messageId);
|
|
1437
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1438
|
+
messageId: row.messageId,
|
|
1439
|
+
dueKey: scheduleKey,
|
|
1440
|
+
idempotencyKey: row.idempotencyKey
|
|
1441
|
+
});
|
|
666
1442
|
await this.#scheduleNextAlarm(txn, now);
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
await
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
idempotencyKey: row.idempotencyKey
|
|
676
|
-
});
|
|
677
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
678
|
-
});
|
|
679
|
-
else await this.#retry(row, "HTTP 503");
|
|
680
|
-
} else if (response && PERMANENT_STATUSES.has(response.status)) await this.#ack(row);
|
|
681
|
-
else {
|
|
1443
|
+
};
|
|
1444
|
+
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), reschedule);
|
|
1445
|
+
else await storage.transaction(reschedule);
|
|
1446
|
+
} else await this.#retry(row, claimKey, "HTTP 503");
|
|
1447
|
+
} else if (response && PERMANENT_STATUSES.has(response.status)) {
|
|
1448
|
+
await cancelUnusedResponseBody(response);
|
|
1449
|
+
await this.#ack(row, claimKey);
|
|
1450
|
+
} else {
|
|
682
1451
|
const reason = response ? `HTTP ${response.status}` : `transport error: ${String(transportError)}`;
|
|
683
|
-
await
|
|
1452
|
+
if (response) await cancelUnusedResponseBody(response);
|
|
1453
|
+
await this.#retry(row, claimKey, reason);
|
|
684
1454
|
}
|
|
685
1455
|
}
|
|
686
|
-
async #ack(row) {
|
|
1456
|
+
async #ack(row, claimKey) {
|
|
687
1457
|
await this.ctx.storage.transaction(async (txn) => {
|
|
1458
|
+
if (!await txn.delete(claimKey)) return;
|
|
688
1459
|
await txn.delete(`msg:${row.messageId}`);
|
|
689
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
690
1460
|
if (row.idempotencyKey) {
|
|
691
1461
|
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
692
1462
|
}
|
|
@@ -694,24 +1464,23 @@ var QueueDO = class extends DurableObject {
|
|
|
694
1464
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
695
1465
|
});
|
|
696
1466
|
}
|
|
697
|
-
async #
|
|
1467
|
+
async #dropExpiredMessage(txn, row, claimKey, now) {
|
|
1468
|
+
if (!await txn.delete(claimKey)) return;
|
|
1469
|
+
await txn.delete(`msg:${row.messageId}`);
|
|
1470
|
+
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
1471
|
+
if (row.idempotencyKey) {
|
|
1472
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1473
|
+
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
1474
|
+
}
|
|
1475
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
1476
|
+
}
|
|
1477
|
+
async #retry(row, claimKey, reason) {
|
|
698
1478
|
const storage = this.ctx.storage;
|
|
699
1479
|
const now = this.#now();
|
|
700
1480
|
const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
701
1481
|
const attempt = row.attempt + 1;
|
|
702
|
-
|
|
703
|
-
if (
|
|
704
|
-
await txn.delete(`msg:${row.messageId}`);
|
|
705
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
706
|
-
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
707
|
-
if (row.idempotencyKey) {
|
|
708
|
-
const dedupKey = `key:${row.idempotencyKey}`;
|
|
709
|
-
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
710
|
-
}
|
|
711
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
712
|
-
return;
|
|
713
|
-
}
|
|
714
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
1482
|
+
const persistRetry = async (txn) => {
|
|
1483
|
+
if (!await txn.delete(claimKey)) return;
|
|
715
1484
|
if (attempt >= maxAttempts) {
|
|
716
1485
|
const dead = {
|
|
717
1486
|
...row,
|
|
@@ -719,7 +1488,7 @@ var QueueDO = class extends DurableObject {
|
|
|
719
1488
|
lastError: reason,
|
|
720
1489
|
failedAt: now
|
|
721
1490
|
};
|
|
722
|
-
const deadKey = `dlq:${pad(now)}:${row.messageId}`;
|
|
1491
|
+
const deadKey = `dlq:${pad$1(now)}:${row.messageId}`;
|
|
723
1492
|
await txn.put(deadKey, dead);
|
|
724
1493
|
await txn.delete(`msg:${row.messageId}`);
|
|
725
1494
|
if (row.idempotencyKey) {
|
|
@@ -747,7 +1516,32 @@ var QueueDO = class extends DurableObject {
|
|
|
747
1516
|
idempotencyKey: row.idempotencyKey
|
|
748
1517
|
});
|
|
749
1518
|
await this.#scheduleNextAlarm(txn, now);
|
|
1519
|
+
};
|
|
1520
|
+
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), persistRetry);
|
|
1521
|
+
else await storage.transaction(persistRetry);
|
|
1522
|
+
}
|
|
1523
|
+
/** Return true when another due compaction page must run immediately. */
|
|
1524
|
+
async #compactExpiredRunFences(txn, now) {
|
|
1525
|
+
const due = await txn.list({
|
|
1526
|
+
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
1527
|
+
end: `${EXPIRED_RUN_GC_PREFIX}${pad$1(now + 1)}`,
|
|
1528
|
+
limit: 129
|
|
750
1529
|
});
|
|
1530
|
+
const page = Array.from(due).slice(0, 128);
|
|
1531
|
+
if (page.length === 0) return false;
|
|
1532
|
+
const markerKeys = page.map(([, item]) => expiredRunKey(item.runId));
|
|
1533
|
+
const markers = await getMany$1(txn, markerKeys);
|
|
1534
|
+
const deletes = [];
|
|
1535
|
+
for (let index = 0; index < page.length; index++) {
|
|
1536
|
+
const [gcKey, item] = page[index];
|
|
1537
|
+
const markerKey = markerKeys[index];
|
|
1538
|
+
const marker = markers.get(markerKey);
|
|
1539
|
+
deletes.push(gcKey);
|
|
1540
|
+
if (marker?.compactAt !== item.compactAt) continue;
|
|
1541
|
+
deletes.push(markerKey);
|
|
1542
|
+
}
|
|
1543
|
+
await deleteMany$1(txn, deletes);
|
|
1544
|
+
return due.size > 128;
|
|
751
1545
|
}
|
|
752
1546
|
/** Arm the alarm no later than `atMs` (durable backoff never throws). */
|
|
753
1547
|
async #armAlarmAtMost(storage, atMs, now) {
|
|
@@ -771,8 +1565,19 @@ var QueueDO = class extends DurableObject {
|
|
|
771
1565
|
limit: 1
|
|
772
1566
|
});
|
|
773
1567
|
for (const key of due.keys()) next = Number.parseInt(key.slice(4, 17), 10);
|
|
774
|
-
const inflight = await storage.list({
|
|
775
|
-
|
|
1568
|
+
const inflight = await storage.list({
|
|
1569
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1570
|
+
limit: 1
|
|
1571
|
+
});
|
|
1572
|
+
for (const key of inflight.keys()) {
|
|
1573
|
+
const deadline = deadlineFromInflightKey(key);
|
|
1574
|
+
if (next === null || deadline < next) next = deadline;
|
|
1575
|
+
}
|
|
1576
|
+
const expiryGc = await storage.list({
|
|
1577
|
+
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
1578
|
+
limit: 1
|
|
1579
|
+
});
|
|
1580
|
+
for (const item of expiryGc.values()) if (next === null || item.compactAt < next) next = item.compactAt;
|
|
776
1581
|
if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
|
|
777
1582
|
else await storage.deleteAlarm();
|
|
778
1583
|
}
|
|
@@ -780,7 +1585,7 @@ var QueueDO = class extends DurableObject {
|
|
|
780
1585
|
const storage = this.ctx.storage;
|
|
781
1586
|
const [due, inflight, dlq, alarmAt] = await Promise.all([
|
|
782
1587
|
storage.list({ prefix: "due:" }),
|
|
783
|
-
storage.list({ prefix:
|
|
1588
|
+
storage.list({ prefix: INFLIGHT_DEADLINE_PREFIX }),
|
|
784
1589
|
storage.list({ prefix: "dlq:" }),
|
|
785
1590
|
storage.getAlarm()
|
|
786
1591
|
]);
|
|
@@ -811,47 +1616,55 @@ var QueueDO = class extends DurableObject {
|
|
|
811
1616
|
async redriveDeadLetter(messageId) {
|
|
812
1617
|
const storage = this.ctx.storage;
|
|
813
1618
|
const now = this.#now();
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
await txn.put(
|
|
832
|
-
const scheduleKey = dueKey(now, row.messageId);
|
|
833
|
-
await txn.put(scheduleKey, row.messageId);
|
|
834
|
-
await txn.delete(key);
|
|
835
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
836
|
-
messageId: row.messageId,
|
|
837
|
-
dueKey: scheduleKey,
|
|
838
|
-
idempotencyKey: row.idempotencyKey
|
|
839
|
-
});
|
|
840
|
-
await this.#armAlarmAtMost(txn, now, now);
|
|
841
|
-
return { ok: true };
|
|
1619
|
+
const found = Array.from(await storage.list({ prefix: "dlq:" })).find(([, dead]) => dead.messageId === messageId);
|
|
1620
|
+
if (!found) return { ok: false };
|
|
1621
|
+
const [deadKey, candidate] = found;
|
|
1622
|
+
const redrive = async (txn) => {
|
|
1623
|
+
const dead = await txn.get(deadKey);
|
|
1624
|
+
if (!dead || dead.messageId !== messageId) return { ok: false };
|
|
1625
|
+
const { failedAt: _failedAt, ...rest } = dead;
|
|
1626
|
+
const row = {
|
|
1627
|
+
...rest,
|
|
1628
|
+
attempt: 0,
|
|
1629
|
+
lastError: void 0
|
|
1630
|
+
};
|
|
1631
|
+
if (row.idempotencyKey) {
|
|
1632
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1633
|
+
const holder = await txn.get(dedupKey);
|
|
1634
|
+
if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
|
|
1635
|
+
if (holder && holder !== row.messageId) await txn.delete(dedupKey);
|
|
1636
|
+
await txn.put(dedupKey, row.messageId);
|
|
842
1637
|
}
|
|
1638
|
+
await txn.put(`msg:${row.messageId}`, row);
|
|
1639
|
+
const scheduleKey = dueKey(now, row.messageId);
|
|
1640
|
+
await txn.put(scheduleKey, row.messageId);
|
|
1641
|
+
await txn.delete(deadKey);
|
|
1642
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1643
|
+
messageId: row.messageId,
|
|
1644
|
+
dueKey: scheduleKey,
|
|
1645
|
+
idempotencyKey: row.idempotencyKey
|
|
1646
|
+
});
|
|
1647
|
+
await this.#armAlarmAtMost(txn, now, now);
|
|
1648
|
+
return { ok: true };
|
|
1649
|
+
};
|
|
1650
|
+
const rejectExpired = async (txn) => {
|
|
1651
|
+
if ((await txn.get(deadKey))?.messageId === messageId) await txn.delete([deadKey, runReferenceKey(candidate.runId, messageId)]);
|
|
843
1652
|
return { ok: false };
|
|
844
|
-
}
|
|
1653
|
+
};
|
|
1654
|
+
return candidate.runId ? await this.#withRunMutation(candidate.runId, rejectExpired, redrive) : await storage.transaction(redrive);
|
|
845
1655
|
}
|
|
846
1656
|
async purgeDeadLetters() {
|
|
847
1657
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
848
|
-
const
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
return {
|
|
1658
|
+
const entries = await txn.list({
|
|
1659
|
+
prefix: "dlq:",
|
|
1660
|
+
limit: 129
|
|
1661
|
+
});
|
|
1662
|
+
const page = Array.from(entries).slice(0, PURGE_DEAD_LETTER_BATCH);
|
|
1663
|
+
await deleteMany$1(txn, page.flatMap(([key, dead]) => [key, ...dead.runId ? [runReferenceKey(dead.runId, dead.messageId)] : []]));
|
|
1664
|
+
return {
|
|
1665
|
+
purged: page.length,
|
|
1666
|
+
hasMore: entries.size > PURGE_DEAD_LETTER_BATCH
|
|
1667
|
+
};
|
|
855
1668
|
});
|
|
856
1669
|
}
|
|
857
1670
|
/**
|
|
@@ -863,140 +1676,450 @@ var QueueDO = class extends DurableObject {
|
|
|
863
1676
|
return { alarmAt: await this.ctx.storage.getAlarm() };
|
|
864
1677
|
}
|
|
865
1678
|
/** Fence a run and purge every live, inflight, and dead-letter message it owns. */
|
|
866
|
-
async expireRun(runId, expiredAt) {
|
|
867
|
-
|
|
1679
|
+
async expireRun(runId, expiredAt, options) {
|
|
1680
|
+
const storage = this.ctx.storage;
|
|
1681
|
+
const limit = boundedLimit$1(options?.limit, EXPIRE_RUN_REFERENCE_BATCH, EXPIRE_RUN_REFERENCE_BATCH);
|
|
1682
|
+
return await storage.transaction(async (txn) => {
|
|
868
1683
|
const existing = await txn.get(expiredRunKey(runId));
|
|
869
|
-
const references = await txn.list({
|
|
1684
|
+
const references = await txn.list({
|
|
1685
|
+
prefix: `run:${runId}:`,
|
|
1686
|
+
limit: limit + 1
|
|
1687
|
+
});
|
|
1688
|
+
const page = Array.from(references).slice(0, limit);
|
|
1689
|
+
const dedupHolders = await getMany$1(txn, page.flatMap(([, reference]) => reference.idempotencyKey ? [`key:${reference.idempotencyKey}`] : []));
|
|
1690
|
+
const keys = [];
|
|
870
1691
|
let count = existing?.deleted ?? 0;
|
|
871
|
-
for (const [key, reference] of
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
if (reference.dueKey)
|
|
875
|
-
if (reference.dlqKey)
|
|
1692
|
+
for (const [key, reference] of page) {
|
|
1693
|
+
keys.push(`msg:${reference.messageId}`, key);
|
|
1694
|
+
if (reference.inflightKey) keys.push(reference.inflightKey);
|
|
1695
|
+
if (reference.dueKey) keys.push(reference.dueKey);
|
|
1696
|
+
if (reference.dlqKey) keys.push(reference.dlqKey);
|
|
876
1697
|
if (reference.idempotencyKey) {
|
|
877
1698
|
const dedupKey = `key:${reference.idempotencyKey}`;
|
|
878
|
-
if (
|
|
1699
|
+
if (dedupHolders.get(dedupKey) === reference.messageId) keys.push(dedupKey);
|
|
879
1700
|
}
|
|
880
|
-
await txn.delete(key);
|
|
881
1701
|
count++;
|
|
882
1702
|
}
|
|
1703
|
+
await deleteMany$1(txn, keys);
|
|
1704
|
+
const done = references.size <= limit;
|
|
1705
|
+
const markerExpiredAt = existing?.expiredAt ?? expiredAt;
|
|
883
1706
|
await txn.put(expiredRunKey(runId), {
|
|
884
|
-
expiredAt,
|
|
885
|
-
deleted: count
|
|
1707
|
+
expiredAt: markerExpiredAt,
|
|
1708
|
+
deleted: count,
|
|
1709
|
+
compactAt: existing?.compactAt ?? null
|
|
886
1710
|
});
|
|
887
1711
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
888
|
-
return
|
|
889
|
-
|
|
1712
|
+
return done ? {
|
|
1713
|
+
deleted: count,
|
|
1714
|
+
done: true,
|
|
1715
|
+
receipt: {
|
|
1716
|
+
expiredAt: markerExpiredAt,
|
|
1717
|
+
deleted: count
|
|
1718
|
+
}
|
|
1719
|
+
} : {
|
|
1720
|
+
deleted: count,
|
|
1721
|
+
done: false
|
|
1722
|
+
};
|
|
1723
|
+
});
|
|
1724
|
+
}
|
|
1725
|
+
/** Acknowledge a final cleanup receipt before making its marker compactable. */
|
|
1726
|
+
async acknowledgeExpireRun(runId, receipt) {
|
|
1727
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1728
|
+
const key = expiredRunKey(runId);
|
|
1729
|
+
const marker = await txn.get(key);
|
|
1730
|
+
if (!marker) return { acknowledged: true };
|
|
1731
|
+
if (marker.expiredAt !== receipt.expiredAt || marker.deleted !== receipt.deleted) return { acknowledged: false };
|
|
1732
|
+
if (marker.compactAt === null) {
|
|
1733
|
+
const compactAt = this.#now() + QUEUE_FENCE_GRACE_MS;
|
|
1734
|
+
await txn.put(key, {
|
|
1735
|
+
...marker,
|
|
1736
|
+
compactAt
|
|
1737
|
+
});
|
|
1738
|
+
await txn.put(expiredRunGcKey(runId, compactAt), {
|
|
1739
|
+
runId,
|
|
1740
|
+
compactAt
|
|
1741
|
+
});
|
|
1742
|
+
}
|
|
1743
|
+
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1744
|
+
return { acknowledged: true };
|
|
1745
|
+
});
|
|
1746
|
+
}
|
|
1747
|
+
};
|
|
1748
|
+
//#endregion
|
|
1749
|
+
//#region src/worker/durable-objects/RunCatalogDO.ts
|
|
1750
|
+
const MAX_INDEX_LIST_SIZE = 1001;
|
|
1751
|
+
const EXPIRY_GC_PREFIX = "expiry-gc:";
|
|
1752
|
+
function pad(ms) {
|
|
1753
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
1754
|
+
}
|
|
1755
|
+
function expiredKey(runId) {
|
|
1756
|
+
return `expired:${runId}`;
|
|
1757
|
+
}
|
|
1758
|
+
function expiryGcKey(runId, compactAt) {
|
|
1759
|
+
return `${EXPIRY_GC_PREFIX}${pad(compactAt)}:${encodeURIComponent(runId)}`;
|
|
1760
|
+
}
|
|
1761
|
+
/** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
|
|
1762
|
+
var RunCatalogDO = class extends DurableObject {
|
|
1763
|
+
now() {
|
|
1764
|
+
const clock = this.env?.clock;
|
|
1765
|
+
return typeof clock === "function" ? clock() : Date.now();
|
|
1766
|
+
}
|
|
1767
|
+
async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
|
|
1768
|
+
if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
|
|
1769
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1770
|
+
if (await txn.get(expiredKey(runId)) !== void 0) return { stored: false };
|
|
1771
|
+
const now = this.now();
|
|
1772
|
+
if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
|
|
1773
|
+
await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
|
|
1774
|
+
return { stored: true };
|
|
1775
|
+
});
|
|
1776
|
+
}
|
|
1777
|
+
async list(options = {}) {
|
|
1778
|
+
const suppliedLimit = options.limit;
|
|
1779
|
+
const limit = Math.min(MAX_INDEX_LIST_SIZE, Math.max(1, typeof suppliedLimit === "number" && Number.isFinite(suppliedLimit) ? Math.floor(suppliedLimit) : MAX_INDEX_LIST_SIZE));
|
|
1780
|
+
const entries = await this.ctx.storage.list({
|
|
1781
|
+
prefix: options.prefix ?? "",
|
|
1782
|
+
...options.reverse ? {
|
|
1783
|
+
reverse: true,
|
|
1784
|
+
end: options.cursor
|
|
1785
|
+
} : { startAfter: options.cursor },
|
|
1786
|
+
limit: limit + 1
|
|
1787
|
+
});
|
|
1788
|
+
const page = Array.from(entries).slice(0, limit);
|
|
1789
|
+
const listComplete = entries.size <= limit;
|
|
1790
|
+
return {
|
|
1791
|
+
keys: page.map(([name, value]) => ({
|
|
1792
|
+
name,
|
|
1793
|
+
value
|
|
1794
|
+
})),
|
|
1795
|
+
list_complete: listComplete,
|
|
1796
|
+
cursor: listComplete ? void 0 : page.at(-1)?.[0]
|
|
1797
|
+
};
|
|
1798
|
+
}
|
|
1799
|
+
async expireRun(runId, keys, expiredAt) {
|
|
1800
|
+
if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
|
|
1801
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1802
|
+
const marker = expiredKey(runId);
|
|
1803
|
+
const compactAt = (await txn.get(marker))?.compactAt ?? this.now() + 1200900;
|
|
1804
|
+
await txn.put(marker, {
|
|
1805
|
+
expiredAt,
|
|
1806
|
+
compactAt
|
|
1807
|
+
});
|
|
1808
|
+
await txn.put(expiryGcKey(runId, compactAt), {
|
|
1809
|
+
runId,
|
|
1810
|
+
compactAt
|
|
1811
|
+
});
|
|
1812
|
+
await this.armAtMost(txn, compactAt);
|
|
1813
|
+
return { deleted: await txn.delete(keys) };
|
|
1814
|
+
});
|
|
1815
|
+
}
|
|
1816
|
+
async alarm() {
|
|
1817
|
+
try {
|
|
1818
|
+
await this.compactExpiryFences();
|
|
1819
|
+
} catch {
|
|
1820
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
async armAtMost(storage, at) {
|
|
1824
|
+
const current = await storage.getAlarm();
|
|
1825
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
1826
|
+
}
|
|
1827
|
+
async compactExpiryFences() {
|
|
1828
|
+
const now = this.now();
|
|
1829
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1830
|
+
const due = await txn.list({
|
|
1831
|
+
prefix: EXPIRY_GC_PREFIX,
|
|
1832
|
+
end: `${EXPIRY_GC_PREFIX}${pad(now + 1)}`,
|
|
1833
|
+
limit: 129
|
|
1834
|
+
});
|
|
1835
|
+
const page = Array.from(due).slice(0, 128);
|
|
1836
|
+
const markerKeys = page.map(([, item]) => expiredKey(item.runId));
|
|
1837
|
+
const markers = await txn.get(markerKeys);
|
|
1838
|
+
const deletes = page.flatMap(([key, item], index) => {
|
|
1839
|
+
return markers.get(markerKeys[index])?.compactAt === item.compactAt ? [key, markerKeys[index]] : [key];
|
|
1840
|
+
});
|
|
1841
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
1842
|
+
if (due.size > 128) {
|
|
1843
|
+
await txn.setAlarm(now + 1);
|
|
1844
|
+
return;
|
|
1845
|
+
}
|
|
1846
|
+
const nextAt = (await txn.list({
|
|
1847
|
+
prefix: EXPIRY_GC_PREFIX,
|
|
1848
|
+
limit: 1
|
|
1849
|
+
})).values().next().value?.compactAt;
|
|
1850
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
1851
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
1852
|
+
});
|
|
890
1853
|
}
|
|
891
1854
|
};
|
|
892
1855
|
//#endregion
|
|
893
1856
|
//#region src/worker/durable-objects/StreamDO.ts
|
|
894
1857
|
const META_KEY = "meta";
|
|
895
1858
|
const CHUNK_KEY_PREFIX = "chunk:";
|
|
1859
|
+
const CHUNK_SIZE_KEY_PREFIX = "chunk-size:";
|
|
896
1860
|
/** Registry keys used when this DO instance acts as a per-run stream index. */
|
|
897
1861
|
const STREAM_REGISTRY_PREFIX = "stream:";
|
|
898
1862
|
const REGISTRY_OWNER_KEY = "registry:owner";
|
|
899
1863
|
const REGISTRY_EXPIRED_KEY = "registry:expired";
|
|
900
|
-
/**
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
*/
|
|
1864
|
+
/** Each chunk occupies a payload key and a size key; one delete accepts at most 128 keys. */
|
|
1865
|
+
const DEFAULT_EXPIRE_CHUNK_LIMIT = 64;
|
|
1866
|
+
const MAX_EXPIRE_CHUNK_LIMIT = 64;
|
|
1867
|
+
/** Bound storage deletion work as well as item count; one oversized chunk still makes progress. */
|
|
1868
|
+
const DEFAULT_EXPIRE_BYTE_LIMIT = 16777216;
|
|
1869
|
+
const MAX_EXPIRE_BYTE_LIMIT = DEFAULT_EXPIRE_BYTE_LIMIT;
|
|
1870
|
+
function boundedLimit(value, fallback, maximum) {
|
|
1871
|
+
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
1872
|
+
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
1873
|
+
}
|
|
1874
|
+
function emptyMeta() {
|
|
1875
|
+
return {
|
|
1876
|
+
count: 0,
|
|
1877
|
+
state: "open"
|
|
1878
|
+
};
|
|
1879
|
+
}
|
|
1880
|
+
function validateMeta(meta) {
|
|
1881
|
+
if (!Number.isSafeInteger(meta.count) || meta.count < 0 || meta.count > 2147483647 || ![
|
|
1882
|
+
"open",
|
|
1883
|
+
"closed",
|
|
1884
|
+
"errored",
|
|
1885
|
+
"expired"
|
|
1886
|
+
].includes(meta.state)) throw new Error("Invalid persisted stream metadata");
|
|
1887
|
+
if (meta.state === "errored" && !meta.error) throw new Error("Invalid persisted stream error metadata");
|
|
1888
|
+
return meta;
|
|
1889
|
+
}
|
|
1890
|
+
/** Zero-padding keeps storage.list() results in stream offset order. */
|
|
904
1891
|
function chunkKey(index) {
|
|
905
1892
|
return `${CHUNK_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
|
|
906
1893
|
}
|
|
1894
|
+
function chunkSizeKey(index) {
|
|
1895
|
+
return `${CHUNK_SIZE_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
|
|
1896
|
+
}
|
|
1897
|
+
function indexFromChunkSizeKey(key) {
|
|
1898
|
+
const suffix = key.slice(11);
|
|
1899
|
+
const index = Number(suffix);
|
|
1900
|
+
if (!/^\d{12}$/.test(suffix) || !Number.isSafeInteger(index) || index < 0) throw new Error(`Invalid persisted stream chunk size key "${key}"`);
|
|
1901
|
+
return index;
|
|
1902
|
+
}
|
|
1903
|
+
function validateChunkSize(size, index) {
|
|
1904
|
+
if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0 || size > 1048576) throw new Error(`Invalid persisted stream chunk size at index ${index}`);
|
|
1905
|
+
return size;
|
|
1906
|
+
}
|
|
1907
|
+
function compactChunk(chunk) {
|
|
1908
|
+
return chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength ? chunk : new Uint8Array(chunk);
|
|
1909
|
+
}
|
|
1910
|
+
function abortReason(signal) {
|
|
1911
|
+
return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
|
1912
|
+
}
|
|
907
1913
|
/**
|
|
908
1914
|
* Durable Object backing workflow streams.
|
|
909
1915
|
*
|
|
910
|
-
*
|
|
911
|
-
*
|
|
912
|
-
*
|
|
913
|
-
* each chunk its index; one chunk per storage key.
|
|
914
|
-
* - `run-streams:<runId>`: registry of stream names owned by a run,
|
|
915
|
-
* powering `listStreamsByRunId`.
|
|
1916
|
+
* Stream cells persist binary chunks under monotonic offset keys. Public
|
|
1917
|
+
* reads are bounded long polls, and writes append a bounded ordered batch in
|
|
1918
|
+
* one storage transaction. Per-run registry cells retain their existing role.
|
|
916
1919
|
*/
|
|
917
1920
|
var StreamDO = class extends DurableObject {
|
|
1921
|
+
meta;
|
|
1922
|
+
metaLoad;
|
|
1923
|
+
mutationTail = Promise.resolve();
|
|
1924
|
+
changeVersion = 0;
|
|
1925
|
+
waiters = /* @__PURE__ */ new Set();
|
|
918
1926
|
async getMeta() {
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1927
|
+
if (this.meta) return this.meta;
|
|
1928
|
+
this.metaLoad ??= this.ctx.storage.get(META_KEY).then((meta) => {
|
|
1929
|
+
const loaded = meta ? validateMeta(meta) : emptyMeta();
|
|
1930
|
+
this.meta = loaded;
|
|
1931
|
+
return loaded;
|
|
1932
|
+
});
|
|
1933
|
+
try {
|
|
1934
|
+
return await this.metaLoad;
|
|
1935
|
+
} catch (error) {
|
|
1936
|
+
this.metaLoad = void 0;
|
|
1937
|
+
throw error;
|
|
1938
|
+
}
|
|
923
1939
|
}
|
|
924
1940
|
assertOwner(meta, runId) {
|
|
925
|
-
if (
|
|
1941
|
+
if (!runId) throw new Error("Stream runId is required");
|
|
926
1942
|
if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
|
|
927
1943
|
}
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
1944
|
+
assertWritable(meta, runId) {
|
|
1945
|
+
this.assertOwner(meta, runId);
|
|
1946
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
1947
|
+
if (meta.state === "closed") throw new Error("Cannot write to a closed stream");
|
|
1948
|
+
if (meta.state === "errored") throw new Error("Cannot write to an errored stream");
|
|
1949
|
+
}
|
|
1950
|
+
runMutation(operation) {
|
|
1951
|
+
const result = this.mutationTail.then(operation, operation);
|
|
1952
|
+
this.mutationTail = result.then(() => void 0, () => void 0);
|
|
1953
|
+
return result;
|
|
1954
|
+
}
|
|
1955
|
+
wakeReaders() {
|
|
1956
|
+
this.changeVersion += 1;
|
|
1957
|
+
for (const waiter of Array.from(this.waiters)) waiter.resolve("change");
|
|
1958
|
+
}
|
|
1959
|
+
waitForChange(observedVersion, waitMs, signal) {
|
|
1960
|
+
if (signal?.aborted) return Promise.reject(abortReason(signal));
|
|
1961
|
+
return new Promise((resolve, reject) => {
|
|
1962
|
+
let settled = false;
|
|
1963
|
+
const waiter = {
|
|
1964
|
+
timer: setTimeout(() => finish("timeout"), waitMs),
|
|
1965
|
+
signal,
|
|
1966
|
+
resolve: (reason) => finish(reason)
|
|
1967
|
+
};
|
|
1968
|
+
const cleanup = () => {
|
|
1969
|
+
clearTimeout(waiter.timer);
|
|
1970
|
+
this.waiters.delete(waiter);
|
|
1971
|
+
if (waiter.signal && waiter.onAbort) waiter.signal.removeEventListener("abort", waiter.onAbort);
|
|
937
1972
|
};
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
1973
|
+
const finish = (reason) => {
|
|
1974
|
+
if (settled) return;
|
|
1975
|
+
settled = true;
|
|
1976
|
+
cleanup();
|
|
1977
|
+
resolve(reason);
|
|
1978
|
+
};
|
|
1979
|
+
waiter.onAbort = () => {
|
|
1980
|
+
if (settled) return;
|
|
1981
|
+
settled = true;
|
|
1982
|
+
cleanup();
|
|
1983
|
+
reject(abortReason(signal));
|
|
1984
|
+
};
|
|
1985
|
+
this.waiters.add(waiter);
|
|
1986
|
+
signal?.addEventListener("abort", waiter.onAbort, { once: true });
|
|
1987
|
+
if (this.changeVersion !== observedVersion) finish("change");
|
|
1988
|
+
});
|
|
1989
|
+
}
|
|
1990
|
+
async readSnapshot(request, timedOut) {
|
|
1991
|
+
const meta = await this.getMeta();
|
|
1992
|
+
this.assertOwner(meta, request.runId);
|
|
1993
|
+
const available = meta.state === "expired" ? 0 : Math.max(0, Math.min(request.maxChunks, meta.count - request.startIndex));
|
|
1994
|
+
const chunks = [];
|
|
1995
|
+
if (available > 0) {
|
|
1996
|
+
const indexes = Array.from({ length: available }, (_, offset) => request.startIndex + offset);
|
|
1997
|
+
const sizeKeys = indexes.map(chunkSizeKey);
|
|
1998
|
+
const storedSizes = await this.ctx.storage.get(sizeKeys);
|
|
1999
|
+
const selectedIndexes = [];
|
|
2000
|
+
const selectedSizes = [];
|
|
2001
|
+
let bytes = 0;
|
|
2002
|
+
for (const index of indexes) {
|
|
2003
|
+
const size = validateChunkSize(storedSizes.get(chunkSizeKey(index)), index);
|
|
2004
|
+
if (bytes + size > request.maxBytes) break;
|
|
2005
|
+
selectedIndexes.push(index);
|
|
2006
|
+
selectedSizes.push(size);
|
|
2007
|
+
bytes += size;
|
|
2008
|
+
}
|
|
2009
|
+
const payloadKeys = selectedIndexes.map(chunkKey);
|
|
2010
|
+
const storedChunks = await this.ctx.storage.get(payloadKeys);
|
|
2011
|
+
for (let offset = 0; offset < payloadKeys.length; offset++) {
|
|
2012
|
+
const chunk = storedChunks.get(payloadKeys[offset]);
|
|
2013
|
+
if (!(chunk instanceof Uint8Array) || chunk.byteLength !== selectedSizes[offset]) throw new Error(`Invalid persisted stream chunk at index ${selectedIndexes[offset]}`);
|
|
2014
|
+
chunks.push(chunk);
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
return {
|
|
2018
|
+
startIndex: request.startIndex,
|
|
2019
|
+
tailIndex: meta.count - 1,
|
|
2020
|
+
chunks,
|
|
2021
|
+
state: meta.state,
|
|
2022
|
+
timedOut,
|
|
2023
|
+
error: meta.error
|
|
2024
|
+
};
|
|
2025
|
+
}
|
|
2026
|
+
/** Append one bounded binary batch with contiguous indexes. */
|
|
2027
|
+
async writeChunks(runId, chunks) {
|
|
2028
|
+
validateStreamWriteChunks(chunks);
|
|
2029
|
+
const storedChunks = chunks.map(compactChunk);
|
|
2030
|
+
return await this.runMutation(async () => {
|
|
2031
|
+
const committed = await this.ctx.storage.transaction(async (txn) => {
|
|
2032
|
+
const stored = await txn.get(META_KEY);
|
|
2033
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
2034
|
+
this.assertWritable(meta, runId);
|
|
2035
|
+
const startIndex = meta.count;
|
|
2036
|
+
if (startIndex + chunks.length > 2147483647) throw new Error("Stream offset limit exceeded");
|
|
2037
|
+
const nextMeta = {
|
|
2038
|
+
count: startIndex + chunks.length,
|
|
2039
|
+
state: "open",
|
|
2040
|
+
ownerRunId: runId
|
|
2041
|
+
};
|
|
2042
|
+
const entries = { [META_KEY]: nextMeta };
|
|
2043
|
+
for (let offset = 0; offset < storedChunks.length; offset++) {
|
|
2044
|
+
const index = startIndex + offset;
|
|
2045
|
+
entries[chunkKey(index)] = storedChunks[offset];
|
|
2046
|
+
entries[chunkSizeKey(index)] = storedChunks[offset].byteLength;
|
|
2047
|
+
}
|
|
2048
|
+
await txn.put(entries);
|
|
2049
|
+
return {
|
|
2050
|
+
meta: nextMeta,
|
|
2051
|
+
result: {
|
|
2052
|
+
startIndex,
|
|
2053
|
+
count: chunks.length,
|
|
2054
|
+
tailIndex: nextMeta.count - 1
|
|
2055
|
+
}
|
|
2056
|
+
};
|
|
946
2057
|
});
|
|
947
|
-
|
|
2058
|
+
this.meta = committed.meta;
|
|
2059
|
+
this.metaLoad = Promise.resolve(committed.meta);
|
|
2060
|
+
this.wakeReaders();
|
|
2061
|
+
return committed.result;
|
|
948
2062
|
});
|
|
949
2063
|
}
|
|
950
2064
|
/**
|
|
951
|
-
*
|
|
952
|
-
*
|
|
2065
|
+
* Return available binary chunks and terminal metadata in one bounded
|
|
2066
|
+
* operation, waiting at most waitMs when the requested offset is idle.
|
|
953
2067
|
*/
|
|
2068
|
+
async readChunks(request, signal) {
|
|
2069
|
+
validateStreamReadRequest(request);
|
|
2070
|
+
const observedVersion = this.changeVersion;
|
|
2071
|
+
const immediate = await this.readSnapshot(request, false);
|
|
2072
|
+
if (immediate.chunks.length > 0 || immediate.state !== "open" || request.waitMs === 0 || request.maxChunks === 0) return immediate;
|
|
2073
|
+
const reason = await this.waitForChange(observedVersion, request.waitMs, signal);
|
|
2074
|
+
return await this.readSnapshot(request, reason === "timeout");
|
|
2075
|
+
}
|
|
2076
|
+
/** Close idempotently; the first terminal state wins. */
|
|
954
2077
|
async closeStream(runId) {
|
|
955
|
-
await this.
|
|
956
|
-
const
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
2078
|
+
await this.runMutation(async () => {
|
|
2079
|
+
const nextMeta = await this.ctx.storage.transaction(async (txn) => {
|
|
2080
|
+
const stored = await txn.get(META_KEY);
|
|
2081
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
2082
|
+
this.assertOwner(meta, runId);
|
|
2083
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
2084
|
+
if (meta.state !== "open") return meta;
|
|
2085
|
+
const closed = {
|
|
2086
|
+
...meta,
|
|
2087
|
+
ownerRunId: runId,
|
|
2088
|
+
state: "closed"
|
|
2089
|
+
};
|
|
2090
|
+
await txn.put(META_KEY, closed);
|
|
2091
|
+
return closed;
|
|
965
2092
|
});
|
|
2093
|
+
this.meta = nextMeta;
|
|
2094
|
+
this.metaLoad = Promise.resolve(nextMeta);
|
|
2095
|
+
this.wakeReaders();
|
|
966
2096
|
});
|
|
967
2097
|
}
|
|
968
|
-
/**
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
2098
|
+
/** Persist a terminal stream failure and wake every pending reader. */
|
|
2099
|
+
async failStream(runId, error) {
|
|
2100
|
+
const normalized = normalizeStreamError(error);
|
|
2101
|
+
await this.runMutation(async () => {
|
|
2102
|
+
const nextMeta = await this.ctx.storage.transaction(async (txn) => {
|
|
2103
|
+
const stored = await txn.get(META_KEY);
|
|
2104
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
2105
|
+
this.assertOwner(meta, runId);
|
|
2106
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
2107
|
+
if (meta.state !== "open") return meta;
|
|
2108
|
+
const failed = {
|
|
2109
|
+
...meta,
|
|
2110
|
+
ownerRunId: runId,
|
|
2111
|
+
state: "errored",
|
|
2112
|
+
error: normalized
|
|
2113
|
+
};
|
|
2114
|
+
await txn.put(META_KEY, failed);
|
|
2115
|
+
return failed;
|
|
2116
|
+
});
|
|
2117
|
+
this.meta = nextMeta;
|
|
2118
|
+
this.metaLoad = Promise.resolve(nextMeta);
|
|
2119
|
+
this.wakeReaders();
|
|
979
2120
|
});
|
|
980
|
-
return {
|
|
981
|
-
chunks: Array.from(entries.values()),
|
|
982
|
-
done: meta.closed,
|
|
983
|
-
tailIndex: meta.count - 1
|
|
984
|
-
};
|
|
985
2121
|
}
|
|
986
|
-
/**
|
|
987
|
-
* Lightweight stream metadata: last chunk index and completion flag.
|
|
988
|
-
*/
|
|
989
|
-
async getInfo() {
|
|
990
|
-
const meta = await this.getMeta();
|
|
991
|
-
if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
|
|
992
|
-
return {
|
|
993
|
-
tailIndex: meta.count - 1,
|
|
994
|
-
done: meta.closed
|
|
995
|
-
};
|
|
996
|
-
}
|
|
997
|
-
/**
|
|
998
|
-
* Register a stream name against this per-run registry instance.
|
|
999
|
-
*/
|
|
2122
|
+
/** Register a stream name against this per-run registry instance. */
|
|
1000
2123
|
async registerStream(runId, name) {
|
|
1001
2124
|
await this.ctx.storage.transaction(async (txn) => {
|
|
1002
2125
|
if (await txn.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
|
|
@@ -1006,57 +2129,124 @@ var StreamDO = class extends DurableObject {
|
|
|
1006
2129
|
await txn.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
|
|
1007
2130
|
});
|
|
1008
2131
|
}
|
|
1009
|
-
/**
|
|
1010
|
-
* List stream names registered against this per-run registry instance.
|
|
1011
|
-
*/
|
|
1012
2132
|
async listStreams() {
|
|
1013
2133
|
if (await this.ctx.storage.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error("Workflow run streams have expired");
|
|
1014
2134
|
const entries = await this.ctx.storage.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1015
2135
|
return Array.from(entries.keys()).map((key) => key.slice(7));
|
|
1016
2136
|
}
|
|
1017
2137
|
/** Fence a run's registry before any stream cells are removed. */
|
|
1018
|
-
async expireRegistry(runId, expiredAt) {
|
|
2138
|
+
async expireRegistry(runId, expiredAt, options) {
|
|
2139
|
+
const limit = boundedLimit(options?.limit, 16, 128);
|
|
1019
2140
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1020
|
-
const
|
|
2141
|
+
const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
|
|
2142
|
+
const owner = state.get(REGISTRY_OWNER_KEY);
|
|
1021
2143
|
if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
|
|
1022
|
-
|
|
1023
|
-
await txn.put(
|
|
1024
|
-
|
|
2144
|
+
const expiry = state.get(REGISTRY_EXPIRED_KEY);
|
|
2145
|
+
await txn.put({
|
|
2146
|
+
[REGISTRY_OWNER_KEY]: runId,
|
|
2147
|
+
[REGISTRY_EXPIRED_KEY]: expiry ?? {
|
|
2148
|
+
expiredAt,
|
|
2149
|
+
deleted: 0
|
|
2150
|
+
}
|
|
2151
|
+
});
|
|
2152
|
+
const entries = await txn.list({
|
|
2153
|
+
prefix: STREAM_REGISTRY_PREFIX,
|
|
2154
|
+
limit
|
|
2155
|
+
});
|
|
1025
2156
|
return { streams: Array.from(entries.keys()).map((key) => key.slice(7)) };
|
|
1026
2157
|
});
|
|
1027
2158
|
}
|
|
1028
|
-
/** Remove
|
|
1029
|
-
async finalizeRegistry(runId) {
|
|
1030
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
1031
|
-
const owner = await txn.get(REGISTRY_OWNER_KEY);
|
|
1032
|
-
const expiredAt = await txn.get(REGISTRY_EXPIRED_KEY);
|
|
1033
|
-
if (owner !== runId || expiredAt === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
|
|
1034
|
-
const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1035
|
-
for (const key of entries.keys()) await txn.delete(key);
|
|
1036
|
-
});
|
|
1037
|
-
}
|
|
1038
|
-
/** Delete stream payload while leaving a small ownership tombstone. */
|
|
1039
|
-
async expireStream(runId, expiredAt) {
|
|
2159
|
+
/** Remove completed registry entries in a bounded, retry-safe batch. */
|
|
2160
|
+
async finalizeRegistry(runId, streams) {
|
|
1040
2161
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1041
|
-
const
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
};
|
|
1045
|
-
|
|
1046
|
-
const
|
|
1047
|
-
|
|
1048
|
-
await txn.put(
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
ownerRunId: runId,
|
|
1052
|
-
expiredAt
|
|
2162
|
+
const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
|
|
2163
|
+
const owner = state.get(REGISTRY_OWNER_KEY);
|
|
2164
|
+
const expiry = state.get(REGISTRY_EXPIRED_KEY);
|
|
2165
|
+
if (owner !== runId || expiry === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
|
|
2166
|
+
const keys = Array.from(new Set(streams.map((name) => `${STREAM_REGISTRY_PREFIX}${name}`)));
|
|
2167
|
+
const removed = keys.length === 0 ? 0 : await txn.delete(keys);
|
|
2168
|
+
const deleted = expiry.deleted + removed;
|
|
2169
|
+
await txn.put(REGISTRY_EXPIRED_KEY, {
|
|
2170
|
+
...expiry,
|
|
2171
|
+
deleted
|
|
1053
2172
|
});
|
|
1054
2173
|
return {
|
|
1055
|
-
deleted
|
|
1056
|
-
|
|
2174
|
+
deleted,
|
|
2175
|
+
done: (await txn.list({
|
|
2176
|
+
prefix: STREAM_REGISTRY_PREFIX,
|
|
2177
|
+
limit: 1
|
|
2178
|
+
})).size === 0
|
|
1057
2179
|
};
|
|
1058
2180
|
});
|
|
1059
2181
|
}
|
|
2182
|
+
/**
|
|
2183
|
+
* Fence the stream and delete one bounded page. Listing size keys avoids
|
|
2184
|
+
* loading payloads, and the transaction makes a crash retry resume safely.
|
|
2185
|
+
*/
|
|
2186
|
+
async expireStream(runId, expiredAt, options) {
|
|
2187
|
+
const limit = boundedLimit(options?.limit, DEFAULT_EXPIRE_CHUNK_LIMIT, MAX_EXPIRE_CHUNK_LIMIT);
|
|
2188
|
+
const byteLimit = boundedLimit(options?.byteLimit, DEFAULT_EXPIRE_BYTE_LIMIT, MAX_EXPIRE_BYTE_LIMIT);
|
|
2189
|
+
return await this.runMutation(async () => {
|
|
2190
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
2191
|
+
const stored = await txn.get(META_KEY);
|
|
2192
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
2193
|
+
this.assertOwner(meta, runId);
|
|
2194
|
+
const firstFence = meta.state !== "expired";
|
|
2195
|
+
if (meta.state === "expired" && meta.payloadDeleted) return {
|
|
2196
|
+
meta,
|
|
2197
|
+
result: {
|
|
2198
|
+
deleted: false,
|
|
2199
|
+
chunks: 0,
|
|
2200
|
+
bytes: 0,
|
|
2201
|
+
done: true
|
|
2202
|
+
}
|
|
2203
|
+
};
|
|
2204
|
+
const candidates = await txn.list({
|
|
2205
|
+
prefix: CHUNK_SIZE_KEY_PREFIX,
|
|
2206
|
+
limit: limit + 1
|
|
2207
|
+
});
|
|
2208
|
+
const page = [];
|
|
2209
|
+
let bytes = 0;
|
|
2210
|
+
for (const [key, storedSize] of Array.from(candidates).slice(0, limit)) {
|
|
2211
|
+
const index = indexFromChunkSizeKey(key);
|
|
2212
|
+
const size = validateChunkSize(storedSize, index);
|
|
2213
|
+
if (page.length > 0 && bytes + size > byteLimit) break;
|
|
2214
|
+
page.push({
|
|
2215
|
+
index,
|
|
2216
|
+
size
|
|
2217
|
+
});
|
|
2218
|
+
bytes += size;
|
|
2219
|
+
}
|
|
2220
|
+
const keys = page.flatMap(({ index }) => [chunkKey(index), chunkSizeKey(index)]);
|
|
2221
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
2222
|
+
const done = candidates.size === page.length;
|
|
2223
|
+
const nextMeta = {
|
|
2224
|
+
...meta,
|
|
2225
|
+
count: done ? 0 : meta.count,
|
|
2226
|
+
ownerRunId: runId,
|
|
2227
|
+
state: "expired",
|
|
2228
|
+
error: void 0,
|
|
2229
|
+
expiredAt: meta.expiredAt ?? expiredAt,
|
|
2230
|
+
expiredChunkCount: meta.expiredChunkCount ?? meta.count,
|
|
2231
|
+
payloadDeleted: done
|
|
2232
|
+
};
|
|
2233
|
+
await txn.put(META_KEY, nextMeta);
|
|
2234
|
+
return {
|
|
2235
|
+
meta: nextMeta,
|
|
2236
|
+
result: {
|
|
2237
|
+
deleted: firstFence,
|
|
2238
|
+
chunks: page.length,
|
|
2239
|
+
bytes,
|
|
2240
|
+
done
|
|
2241
|
+
}
|
|
2242
|
+
};
|
|
2243
|
+
});
|
|
2244
|
+
this.meta = result.meta;
|
|
2245
|
+
this.metaLoad = Promise.resolve(result.meta);
|
|
2246
|
+
this.wakeReaders();
|
|
2247
|
+
return result.result;
|
|
2248
|
+
});
|
|
2249
|
+
}
|
|
1060
2250
|
};
|
|
1061
2251
|
//#endregion
|
|
1062
2252
|
//#region src/apply-event.ts
|
|
@@ -1076,10 +2266,36 @@ const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
|
|
|
1076
2266
|
* rejected as a duplicate instead of resurrecting the hook.
|
|
1077
2267
|
*/
|
|
1078
2268
|
const HOOK_EVENT_MARKER_PREFIX = "hookevent:";
|
|
2269
|
+
/** Durable repair record for idempotent post-commit hook-index release. */
|
|
2270
|
+
const HOOK_RELEASE_MARKER_PREFIX = "hookreleased:";
|
|
2271
|
+
/** Exact ambiguous reservations canceled by a serialized RunDO resolution. */
|
|
2272
|
+
const HOOK_CLAIM_CANCELLATION_PREFIX = "hookclaimcancelled:";
|
|
1079
2273
|
/** Workflow-authored attr_set correlation claims. */
|
|
1080
2274
|
const ATTRIBUTE_EVENT_MARKER_PREFIX = "attrevent:";
|
|
1081
2275
|
/** Durable lazy-hook resume claims, scoped by the containing run DO. */
|
|
1082
2276
|
const HOOK_RESUME_CLAIM_PREFIX = "hookresume:";
|
|
2277
|
+
const STORAGE_BATCH_SIZE$1 = 128;
|
|
2278
|
+
const MAX_ENTITY_PAGE_SIZE = 1e3;
|
|
2279
|
+
async function getMany(store, keys) {
|
|
2280
|
+
if (store.getMany) {
|
|
2281
|
+
const result = /* @__PURE__ */ new Map();
|
|
2282
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) {
|
|
2283
|
+
const page = await store.getMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
2284
|
+
for (const entry of page) result.set(...entry);
|
|
2285
|
+
}
|
|
2286
|
+
return result;
|
|
2287
|
+
}
|
|
2288
|
+
const values = await Promise.all(keys.map((key) => store.get(key)));
|
|
2289
|
+
return new Map(keys.flatMap((key, index) => values[index] === void 0 ? [] : [[key, values[index]]]));
|
|
2290
|
+
}
|
|
2291
|
+
async function deleteMany(store, keys) {
|
|
2292
|
+
if (store.deleteMany) {
|
|
2293
|
+
let deleted = 0;
|
|
2294
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) deleted += await store.deleteMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
2295
|
+
return deleted;
|
|
2296
|
+
}
|
|
2297
|
+
return (await Promise.all(keys.map((key) => store.delete(key)))).filter(Boolean).length;
|
|
2298
|
+
}
|
|
1083
2299
|
function failure(code, message, extra) {
|
|
1084
2300
|
return {
|
|
1085
2301
|
ok: false,
|
|
@@ -1088,6 +2304,9 @@ function failure(code, message, extra) {
|
|
|
1088
2304
|
...extra
|
|
1089
2305
|
};
|
|
1090
2306
|
}
|
|
2307
|
+
function hookClaimCancellationKey(claimId) {
|
|
2308
|
+
return `${HOOK_CLAIM_CANCELLATION_PREFIX}${encodeURIComponent(claimId)}`;
|
|
2309
|
+
}
|
|
1091
2310
|
function readStringProp(value, key) {
|
|
1092
2311
|
if (value !== null && typeof value === "object") {
|
|
1093
2312
|
const prop = value[key];
|
|
@@ -1143,7 +2362,9 @@ function creationIndexKey(prefix, createdAt, id) {
|
|
|
1143
2362
|
* requested ordering.
|
|
1144
2363
|
*/
|
|
1145
2364
|
async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
1146
|
-
const
|
|
2365
|
+
const requestedLimit = Number.isFinite(params.limit) ? Math.floor(params.limit) : 1;
|
|
2366
|
+
const limit = Math.min(MAX_ENTITY_PAGE_SIZE, Math.max(1, requestedLimit));
|
|
2367
|
+
const { cursor, sortOrder = "asc" } = params;
|
|
1147
2368
|
const entries = await store.list(sortOrder === "desc" ? {
|
|
1148
2369
|
prefix: indexPrefix,
|
|
1149
2370
|
reverse: true,
|
|
@@ -1155,11 +2376,12 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
|
1155
2376
|
startAfter: cursor ? `${indexPrefix}${cursor}` : void 0
|
|
1156
2377
|
});
|
|
1157
2378
|
const page = Array.from(entries.entries()).slice(0, limit);
|
|
1158
|
-
const
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
2379
|
+
const entityKeys = page.map(([, id]) => `${entityPrefix}${id}`);
|
|
2380
|
+
const entities = await getMany(store, entityKeys);
|
|
2381
|
+
const data = entityKeys.flatMap((key) => {
|
|
2382
|
+
const item = entities.get(key);
|
|
2383
|
+
return item === void 0 ? [] : [item];
|
|
2384
|
+
});
|
|
1163
2385
|
const hasMore = entries.size > limit;
|
|
1164
2386
|
const lastKey = page.at(-1)?.[0];
|
|
1165
2387
|
return {
|
|
@@ -1170,22 +2392,26 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
|
1170
2392
|
}
|
|
1171
2393
|
/** Delete all hook entities for the run, returning the released tokens. */
|
|
1172
2394
|
async function releaseAllHooks(store) {
|
|
2395
|
+
if (store.deferTerminalCleanup) return [];
|
|
1173
2396
|
const hooks = await store.list({ prefix: HOOK_KEY_PREFIX });
|
|
1174
2397
|
const released = [];
|
|
2398
|
+
const keys = [];
|
|
1175
2399
|
for (const hook of hooks.values()) {
|
|
1176
|
-
|
|
1177
|
-
|
|
2400
|
+
keys.push(`${HOOK_KEY_PREFIX}${hook.hookId}`);
|
|
2401
|
+
keys.push(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
|
|
1178
2402
|
released.push({
|
|
1179
2403
|
hookId: hook.hookId,
|
|
1180
2404
|
token: hook.token
|
|
1181
2405
|
});
|
|
1182
2406
|
}
|
|
2407
|
+
await deleteMany(store, keys);
|
|
1183
2408
|
return released;
|
|
1184
2409
|
}
|
|
1185
2410
|
/** Delete all persisted waits once their owning run becomes terminal. */
|
|
1186
2411
|
async function releaseAllWaits(store) {
|
|
2412
|
+
if (store.deferTerminalCleanup) return;
|
|
1187
2413
|
const waits = await store.list({ prefix: WAIT_KEY_PREFIX });
|
|
1188
|
-
|
|
2414
|
+
await deleteMany(store, Array.from(waits.keys()));
|
|
1189
2415
|
}
|
|
1190
2416
|
async function applyEvent(store, ctx) {
|
|
1191
2417
|
const { runId, data, now } = ctx;
|
|
@@ -1305,7 +2531,16 @@ async function applyEvent(store, ctx) {
|
|
|
1305
2531
|
}
|
|
1306
2532
|
}
|
|
1307
2533
|
if ((data.eventType === "hook_disposed" || data.eventType === "hook_received") && data.correlationId) {
|
|
1308
|
-
if (!await store.get(`hook:${data.correlationId}`))
|
|
2534
|
+
if (!await store.get(`hook:${data.correlationId}`)) {
|
|
2535
|
+
if (data.eventType === "hook_disposed") {
|
|
2536
|
+
const released = await store.get(`${HOOK_RELEASE_MARKER_PREFIX}${data.correlationId}`);
|
|
2537
|
+
if (released) return {
|
|
2538
|
+
ok: true,
|
|
2539
|
+
releasedHooks: [released]
|
|
2540
|
+
};
|
|
2541
|
+
}
|
|
2542
|
+
return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
2543
|
+
}
|
|
1309
2544
|
}
|
|
1310
2545
|
if (!skipRunValidation && data.eventType !== "run_started" && !currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1311
2546
|
switch (data.eventType) {
|
|
@@ -1641,6 +2876,7 @@ async function applyEvent(store, ctx) {
|
|
|
1641
2876
|
const holder = ctx.tokenHolder ?? null;
|
|
1642
2877
|
const markerKey = `${HOOK_EVENT_MARKER_PREFIX}${hookId}`;
|
|
1643
2878
|
const hookKey = `${HOOK_KEY_PREFIX}${hookId}`;
|
|
2879
|
+
if (ctx.hookClaimId && await store.get(hookClaimCancellationKey(ctx.hookClaimId)) !== void 0) return failure("HOOK_CLAIM_CANCELLED", `Hook reservation for "${hookId}" was canceled before commit`);
|
|
1644
2880
|
if (holder && (holder.runId !== runId || holder.hookId !== hookId)) {
|
|
1645
2881
|
const event = buildEvent({
|
|
1646
2882
|
eventType: "hook_conflict",
|
|
@@ -1700,6 +2936,10 @@ async function applyEvent(store, ctx) {
|
|
|
1700
2936
|
if (!hook) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
1701
2937
|
await store.delete(hookKey);
|
|
1702
2938
|
await store.delete(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
|
|
2939
|
+
await store.put(`${HOOK_RELEASE_MARKER_PREFIX}${hook.hookId}`, {
|
|
2940
|
+
hookId: hook.hookId,
|
|
2941
|
+
token: hook.token
|
|
2942
|
+
});
|
|
1703
2943
|
const event = buildEvent({ ...data });
|
|
1704
2944
|
await putEvent(event);
|
|
1705
2945
|
return {
|
|
@@ -1817,8 +3057,8 @@ async function finalizeEventPage(store, outcome, params) {
|
|
|
1817
3057
|
//#region src/retention.ts
|
|
1818
3058
|
const CLEANUP_RECORD_KEY = "retention:cleanup";
|
|
1819
3059
|
const TOMBSTONE_KEY = "retention:tombstone";
|
|
1820
|
-
const INDEX_MARKER_PREFIX = "retention:index:";
|
|
1821
3060
|
const HOOK_MARKER_PREFIX = "retention:hook:";
|
|
3061
|
+
const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
|
|
1822
3062
|
function sortableTimestamp(date) {
|
|
1823
3063
|
return date.getTime().toString().padStart(13, "0");
|
|
1824
3064
|
}
|
|
@@ -1828,25 +3068,28 @@ function workflowRunIndexKey(run) {
|
|
|
1828
3068
|
function globalRunIndexKey(run) {
|
|
1829
3069
|
return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
1830
3070
|
}
|
|
1831
|
-
function correlationIndexKey(correlationId, createdAt, eventId, runId) {
|
|
1832
|
-
return `correlation:${encodeURIComponent(correlationId)}:${sortableTimestamp(createdAt)}:${eventId}:${runId}`;
|
|
1833
|
-
}
|
|
1834
|
-
function indexMarkerKey(indexKey) {
|
|
1835
|
-
return `${INDEX_MARKER_PREFIX}${encodeURIComponent(indexKey)}`;
|
|
1836
|
-
}
|
|
1837
3071
|
function hookMarkerKey(reference) {
|
|
1838
3072
|
return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
|
|
1839
3073
|
}
|
|
1840
3074
|
function cleanupTombstone(record, now) {
|
|
3075
|
+
const cleanup = {
|
|
3076
|
+
...record,
|
|
3077
|
+
phase: "tombstoned",
|
|
3078
|
+
tombstonedAt: now
|
|
3079
|
+
};
|
|
1841
3080
|
return {
|
|
1842
3081
|
version: 1,
|
|
1843
3082
|
runId: record.runId,
|
|
1844
3083
|
terminalStatus: record.terminalStatus,
|
|
1845
3084
|
completedAt: record.completedAt,
|
|
1846
3085
|
expiredAt: record.dueAt,
|
|
1847
|
-
tombstonedAt: now
|
|
3086
|
+
tombstonedAt: now,
|
|
3087
|
+
cleanup
|
|
1848
3088
|
};
|
|
1849
3089
|
}
|
|
3090
|
+
function cleanupFromTombstone(tombstone) {
|
|
3091
|
+
return tombstone.cleanup;
|
|
3092
|
+
}
|
|
1850
3093
|
function expiredRead(tombstone) {
|
|
1851
3094
|
return {
|
|
1852
3095
|
ok: false,
|
|
@@ -1857,9 +3100,17 @@ function expiredRead(tombstone) {
|
|
|
1857
3100
|
}
|
|
1858
3101
|
//#endregion
|
|
1859
3102
|
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
1860
|
-
const
|
|
1861
|
-
const
|
|
3103
|
+
const STORAGE_BATCH_SIZE = 128;
|
|
3104
|
+
const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
|
|
3105
|
+
const HOOK_MARKER_PAGE_SIZE = 64;
|
|
3106
|
+
const TERMINAL_HOOK_PAGE_SIZE = 64;
|
|
3107
|
+
const TERMINAL_WAIT_PAGE_SIZE = STORAGE_BATCH_SIZE;
|
|
3108
|
+
const STREAMS_PER_CLEANUP_PAGE = 16;
|
|
3109
|
+
const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
3110
|
+
const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
|
|
3111
|
+
const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
|
|
1862
3112
|
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
3113
|
+
const CLEANUP_PROGRESS_KEY = "retention:progress";
|
|
1863
3114
|
/**
|
|
1864
3115
|
* Adapt DO storage (or a transaction) to the {@link EventStore} interface
|
|
1865
3116
|
* shared with the test mocks. No casts: the methods line up structurally,
|
|
@@ -1868,11 +3119,21 @@ const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
|
1868
3119
|
function storeFrom(storage) {
|
|
1869
3120
|
return {
|
|
1870
3121
|
get: (key) => storage.get(key),
|
|
3122
|
+
getMany: (keys) => storage.get(keys),
|
|
1871
3123
|
put: (key, value) => storage.put(key, value),
|
|
1872
3124
|
delete: (key) => storage.delete(key),
|
|
1873
|
-
|
|
3125
|
+
deleteMany: (keys) => storage.delete(keys),
|
|
3126
|
+
list: (options) => storage.list(options),
|
|
3127
|
+
deferTerminalCleanup: true
|
|
1874
3128
|
};
|
|
1875
3129
|
}
|
|
3130
|
+
function hookCreationIndexKey(hook) {
|
|
3131
|
+
return `${HOOK_CREATED_KEY_PREFIX}${hook.createdAt.toISOString()}:${hook.hookId}`;
|
|
3132
|
+
}
|
|
3133
|
+
async function putStorageEntries(storage, entries) {
|
|
3134
|
+
const all = Array.from(entries);
|
|
3135
|
+
for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE)));
|
|
3136
|
+
}
|
|
1876
3137
|
/**
|
|
1877
3138
|
* Durable Object for managing a single workflow run's state.
|
|
1878
3139
|
*
|
|
@@ -1888,26 +3149,51 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1888
3149
|
const clock = this.env?.clock;
|
|
1889
3150
|
return typeof clock === "function" ? clock() : Date.now();
|
|
1890
3151
|
}
|
|
1891
|
-
|
|
1892
|
-
const tombstone =
|
|
3152
|
+
tombstoneFrom(values, now = this.now()) {
|
|
3153
|
+
const tombstone = values.get(TOMBSTONE_KEY);
|
|
1893
3154
|
if (tombstone) return tombstone;
|
|
1894
|
-
const cleanup =
|
|
3155
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
1895
3156
|
if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
|
|
1896
3157
|
return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
|
|
1897
3158
|
}
|
|
1898
|
-
async
|
|
1899
|
-
const
|
|
1900
|
-
if (tombstone) return expiredRead(tombstone);
|
|
3159
|
+
async retentionState(storage, now = this.now()) {
|
|
3160
|
+
const values = await storage.get([TOMBSTONE_KEY, CLEANUP_RECORD_KEY]);
|
|
1901
3161
|
return {
|
|
1902
|
-
|
|
1903
|
-
|
|
3162
|
+
cleanup: values.get(CLEANUP_RECORD_KEY),
|
|
3163
|
+
tombstone: this.tombstoneFrom(values, now)
|
|
1904
3164
|
};
|
|
1905
3165
|
}
|
|
3166
|
+
async read(reader) {
|
|
3167
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3168
|
+
const { tombstone } = await this.retentionState(txn);
|
|
3169
|
+
if (tombstone) return expiredRead(tombstone);
|
|
3170
|
+
return {
|
|
3171
|
+
ok: true,
|
|
3172
|
+
value: await reader(txn)
|
|
3173
|
+
};
|
|
3174
|
+
});
|
|
3175
|
+
}
|
|
3176
|
+
async readKey(key) {
|
|
3177
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3178
|
+
const values = await txn.get([
|
|
3179
|
+
TOMBSTONE_KEY,
|
|
3180
|
+
CLEANUP_RECORD_KEY,
|
|
3181
|
+
key
|
|
3182
|
+
]);
|
|
3183
|
+
const tombstone = this.tombstoneFrom(values);
|
|
3184
|
+
if (tombstone) return expiredRead(tombstone);
|
|
3185
|
+
return {
|
|
3186
|
+
ok: true,
|
|
3187
|
+
value: values.get(key) ?? null
|
|
3188
|
+
};
|
|
3189
|
+
});
|
|
3190
|
+
}
|
|
1906
3191
|
/** Apply an event and capture all retention metadata in the same transaction. */
|
|
1907
3192
|
async applyEvent(request) {
|
|
1908
3193
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1909
3194
|
const now = new Date(this.now());
|
|
1910
|
-
const
|
|
3195
|
+
const retention = await this.retentionState(txn, now.getTime());
|
|
3196
|
+
const { tombstone } = retention;
|
|
1911
3197
|
if (tombstone) return {
|
|
1912
3198
|
ok: false,
|
|
1913
3199
|
code: "RUN_EXPIRED",
|
|
@@ -1921,20 +3207,12 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1921
3207
|
});
|
|
1922
3208
|
if (!outcome.ok) return outcome;
|
|
1923
3209
|
await txn.put("event_sequence", eventSequence);
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
}
|
|
1928
|
-
if (outcome.hookToIndex) {
|
|
1929
|
-
const reference = {
|
|
1930
|
-
hookId: outcome.hookToIndex.hookId,
|
|
1931
|
-
token: outcome.hookToIndex.token
|
|
1932
|
-
};
|
|
1933
|
-
await txn.put(hookMarkerKey(reference), reference);
|
|
1934
|
-
}
|
|
1935
|
-
for (const reference of outcome.releasedHooks) await txn.put(hookMarkerKey(reference), reference);
|
|
3210
|
+
await putStorageEntries(txn, (outcome.hookToIndex ? [{
|
|
3211
|
+
hookId: outcome.hookToIndex.hookId,
|
|
3212
|
+
token: outcome.hookToIndex.token
|
|
3213
|
+
}, ...outcome.releasedHooks] : outcome.releasedHooks).map((reference) => [hookMarkerKey(reference), reference]));
|
|
1936
3214
|
const retentionMs = request.cleanup?.retentionMs ?? 0;
|
|
1937
|
-
if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !
|
|
3215
|
+
if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !retention.cleanup) {
|
|
1938
3216
|
const terminalRun = outcome.run;
|
|
1939
3217
|
const dueAt = new Date(terminalRun.completedAt.getTime() + retentionMs);
|
|
1940
3218
|
const run = WorkflowRunSchema.parse({
|
|
@@ -1951,6 +3229,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1951
3229
|
dueAt,
|
|
1952
3230
|
queueShards: request.cleanup?.queueShards ?? 1,
|
|
1953
3231
|
phase: "retained",
|
|
3232
|
+
generation: 0,
|
|
1954
3233
|
attempts: 0,
|
|
1955
3234
|
deletedPayloadKeys: 0,
|
|
1956
3235
|
deletedStreams: 0,
|
|
@@ -1961,50 +3240,103 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1961
3240
|
await txn.setAlarm(dueAt);
|
|
1962
3241
|
outcome.run = run;
|
|
1963
3242
|
}
|
|
3243
|
+
if (outcome.run && isTerminalWorkflowRunStatus(outcome.run.status)) {
|
|
3244
|
+
if (!await txn.get("terminal:cleanup")) await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3245
|
+
version: 1,
|
|
3246
|
+
runId: outcome.run.runId,
|
|
3247
|
+
phase: "hooks",
|
|
3248
|
+
generation: 0,
|
|
3249
|
+
attempts: 0
|
|
3250
|
+
});
|
|
3251
|
+
await txn.setAlarm(this.now() + 1);
|
|
3252
|
+
}
|
|
3253
|
+
if (outcome.run) outcome.indexPublicationExpiresAt = now.getTime() + MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS;
|
|
1964
3254
|
return finalizeEventPage(storeFrom(txn), outcome, request.params);
|
|
1965
3255
|
});
|
|
1966
3256
|
}
|
|
3257
|
+
/**
|
|
3258
|
+
* Resolve an ambiguous hook-creation RPC in the same serialization domain as
|
|
3259
|
+
* applyEvent. If the hook is absent, the exact reservation is durably fenced
|
|
3260
|
+
* before the caller releases its token and hook-ID claims.
|
|
3261
|
+
*/
|
|
3262
|
+
async resolveHookTokenClaim(request) {
|
|
3263
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3264
|
+
const { tombstone } = await this.retentionState(txn);
|
|
3265
|
+
if (tombstone) return { committed: false };
|
|
3266
|
+
if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
|
|
3267
|
+
await txn.put(hookClaimCancellationKey(request.claimId), {
|
|
3268
|
+
hookId: request.hookId,
|
|
3269
|
+
token: request.token,
|
|
3270
|
+
canceledAt: this.now()
|
|
3271
|
+
});
|
|
3272
|
+
return { committed: false };
|
|
3273
|
+
});
|
|
3274
|
+
}
|
|
1967
3275
|
async getRun() {
|
|
1968
|
-
return this.
|
|
3276
|
+
return this.readKey("run");
|
|
3277
|
+
}
|
|
3278
|
+
/** Single authoritative lifecycle answer used after derivative compaction. */
|
|
3279
|
+
async getLifecycleStatus() {
|
|
3280
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3281
|
+
const values = await txn.get([
|
|
3282
|
+
TOMBSTONE_KEY,
|
|
3283
|
+
CLEANUP_RECORD_KEY,
|
|
3284
|
+
"run"
|
|
3285
|
+
]);
|
|
3286
|
+
if (this.tombstoneFrom(values)) return "expired";
|
|
3287
|
+
const run = values.get("run");
|
|
3288
|
+
if (!run) return "missing";
|
|
3289
|
+
return isTerminalWorkflowRunStatus(run.status) ? "terminal" : "active";
|
|
3290
|
+
});
|
|
1969
3291
|
}
|
|
1970
3292
|
async getStep(stepId) {
|
|
1971
|
-
return this.
|
|
3293
|
+
return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
|
|
1972
3294
|
}
|
|
1973
3295
|
async listSteps(params) {
|
|
1974
|
-
return this.read(() => listByCreationTime(storeFrom(
|
|
3296
|
+
return this.read((txn) => listByCreationTime(storeFrom(txn), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
|
|
1975
3297
|
limit: params?.limit ?? 20,
|
|
1976
3298
|
cursor: params?.cursor,
|
|
1977
3299
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1978
3300
|
}));
|
|
1979
3301
|
}
|
|
1980
3302
|
async getEvent(eventId) {
|
|
1981
|
-
return this.
|
|
3303
|
+
return this.readKey(`${EVENT_KEY_PREFIX}${eventId}`);
|
|
1982
3304
|
}
|
|
1983
3305
|
async listEvents(params) {
|
|
1984
|
-
return this.read(() => listByPrefix(storeFrom(
|
|
3306
|
+
return this.read((txn) => listByPrefix(storeFrom(txn), EVENT_KEY_PREFIX, {
|
|
1985
3307
|
limit: params?.limit ?? 100,
|
|
1986
3308
|
cursor: params?.cursor,
|
|
1987
3309
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1988
3310
|
}, (event) => event.eventId));
|
|
1989
3311
|
}
|
|
1990
3312
|
async listHooks(params) {
|
|
1991
|
-
return this.read(() => listByCreationTime(storeFrom(
|
|
3313
|
+
return this.read((txn) => listByCreationTime(storeFrom(txn), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
|
|
1992
3314
|
limit: params?.limit ?? 100,
|
|
1993
3315
|
cursor: params?.cursor,
|
|
1994
3316
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1995
3317
|
}));
|
|
1996
3318
|
}
|
|
1997
3319
|
async getCleanupStatus() {
|
|
1998
|
-
|
|
3320
|
+
const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TOMBSTONE_KEY]);
|
|
3321
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3322
|
+
if (cleanup) return cleanup;
|
|
3323
|
+
const tombstone = values.get(TOMBSTONE_KEY);
|
|
3324
|
+
return tombstone ? cleanupFromTombstone(tombstone) : null;
|
|
1999
3325
|
}
|
|
2000
3326
|
/** Explicitly schedule an existing terminal run, used for opt-in backfills. */
|
|
2001
3327
|
async scheduleCleanup(request) {
|
|
2002
3328
|
if (request.retentionMs <= 0) return this.getCleanupStatus();
|
|
2003
3329
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2004
|
-
const
|
|
3330
|
+
const values = await txn.get([
|
|
3331
|
+
CLEANUP_RECORD_KEY,
|
|
3332
|
+
TOMBSTONE_KEY,
|
|
3333
|
+
TERMINAL_CLEANUP_KEY,
|
|
3334
|
+
"run"
|
|
3335
|
+
]);
|
|
3336
|
+
const existing = values.get(CLEANUP_RECORD_KEY);
|
|
2005
3337
|
if (existing) return existing;
|
|
2006
|
-
if (
|
|
2007
|
-
const current =
|
|
3338
|
+
if (values.get("retention:tombstone")) return null;
|
|
3339
|
+
const current = values.get("run");
|
|
2008
3340
|
if (!current || !isTerminalWorkflowRunStatus(current.status)) return null;
|
|
2009
3341
|
const terminalRun = current;
|
|
2010
3342
|
const dueAt = new Date(terminalRun.completedAt.getTime() + request.retentionMs);
|
|
@@ -2022,6 +3354,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2022
3354
|
dueAt,
|
|
2023
3355
|
queueShards: request.queueShards,
|
|
2024
3356
|
phase: "retained",
|
|
3357
|
+
generation: 0,
|
|
2025
3358
|
attempts: 0,
|
|
2026
3359
|
deletedPayloadKeys: 0,
|
|
2027
3360
|
deletedStreams: 0,
|
|
@@ -2029,7 +3362,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2029
3362
|
};
|
|
2030
3363
|
await txn.put("run", run);
|
|
2031
3364
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
2032
|
-
await txn.setAlarm(dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
3365
|
+
await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
2033
3366
|
return cleanup;
|
|
2034
3367
|
});
|
|
2035
3368
|
}
|
|
@@ -2049,159 +3382,388 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2049
3382
|
}));
|
|
2050
3383
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2051
3384
|
...cleanup,
|
|
2052
|
-
dueAt
|
|
3385
|
+
dueAt,
|
|
3386
|
+
generation: cleanup.generation + 1
|
|
2053
3387
|
});
|
|
2054
3388
|
await txn.setAlarm(this.now() + 1);
|
|
2055
3389
|
});
|
|
2056
|
-
|
|
2057
|
-
await this.executeCleanup();
|
|
2058
|
-
} catch (error) {
|
|
2059
|
-
await this.recordCleanupFailure(error);
|
|
2060
|
-
}
|
|
3390
|
+
await this.executeScheduledCleanupPage();
|
|
2061
3391
|
return this.getCleanupStatus();
|
|
2062
3392
|
}
|
|
2063
3393
|
async rearmCleanup() {
|
|
2064
|
-
const
|
|
2065
|
-
|
|
2066
|
-
if (
|
|
3394
|
+
const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TERMINAL_CLEANUP_KEY]);
|
|
3395
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3396
|
+
if (values.get("terminal:cleanup")) await this.ctx.storage.setAlarm(this.now() + 1);
|
|
3397
|
+
else if (!cleanup || cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
|
|
2067
3398
|
else await this.ctx.storage.setAlarm(Math.max(this.now() + 1, cleanup.dueAt.getTime()));
|
|
2068
|
-
return cleanup;
|
|
3399
|
+
return cleanup ?? null;
|
|
2069
3400
|
}
|
|
2070
3401
|
async alarm() {
|
|
2071
|
-
|
|
2072
|
-
await this.executeCleanup();
|
|
2073
|
-
} catch (error) {
|
|
2074
|
-
await this.recordCleanupFailure(error);
|
|
2075
|
-
}
|
|
3402
|
+
await this.executeScheduledCleanupPage();
|
|
2076
3403
|
}
|
|
2077
3404
|
namespace(name) {
|
|
2078
3405
|
const namespace = this.env?.[name];
|
|
2079
3406
|
if (!namespace || typeof namespace === "function") throw new Error(`world-celld retention missing binding ${name}`);
|
|
2080
3407
|
return namespace;
|
|
2081
3408
|
}
|
|
2082
|
-
|
|
3409
|
+
workflowIndex() {
|
|
3410
|
+
return createWorkflowIndex({
|
|
3411
|
+
runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
|
|
3412
|
+
hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
|
|
3413
|
+
hookIds: this.namespace("WORKFLOW_HOOK_IDS")
|
|
3414
|
+
});
|
|
3415
|
+
}
|
|
3416
|
+
async setCleanupPhase(expected, next, updates = {}, continueAt) {
|
|
2083
3417
|
await this.ctx.storage.transaction(async (txn) => {
|
|
2084
3418
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2085
|
-
if (!cleanup || cleanup.phase !== expected) return;
|
|
3419
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
|
|
2086
3420
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2087
3421
|
...cleanup,
|
|
2088
3422
|
...updates,
|
|
2089
3423
|
phase: next,
|
|
3424
|
+
generation: cleanup.generation + 1,
|
|
2090
3425
|
attempts: 0,
|
|
2091
3426
|
lastError: void 0
|
|
2092
3427
|
});
|
|
3428
|
+
if (continueAt !== void 0) await txn.setAlarm(continueAt);
|
|
2093
3429
|
});
|
|
2094
3430
|
}
|
|
2095
|
-
async
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
await this.
|
|
2100
|
-
|
|
3431
|
+
async executeScheduledCleanupPage() {
|
|
3432
|
+
const terminalCleanup = await this.ctx.storage.get(TERMINAL_CLEANUP_KEY);
|
|
3433
|
+
if (terminalCleanup) {
|
|
3434
|
+
try {
|
|
3435
|
+
await this.executeTerminalCleanup(terminalCleanup);
|
|
3436
|
+
} catch (error) {
|
|
3437
|
+
await this.recordTerminalCleanupFailure(error, terminalCleanup);
|
|
2101
3438
|
}
|
|
2102
|
-
|
|
2103
|
-
|
|
3439
|
+
return;
|
|
3440
|
+
}
|
|
3441
|
+
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
3442
|
+
if (!cleanup || cleanup.phase === "tombstoned") {
|
|
3443
|
+
await this.ctx.storage.deleteAlarm();
|
|
3444
|
+
return;
|
|
3445
|
+
}
|
|
3446
|
+
try {
|
|
3447
|
+
await this.executeCleanup(cleanup);
|
|
3448
|
+
} catch (error) {
|
|
3449
|
+
await this.recordCleanupFailure(error, cleanup);
|
|
3450
|
+
}
|
|
3451
|
+
}
|
|
3452
|
+
async executeCleanup(cleanup) {
|
|
3453
|
+
if (cleanup.phase === "retained") {
|
|
3454
|
+
if (cleanup.dueAt.getTime() > this.now()) {
|
|
3455
|
+
await this.ctx.storage.setAlarm(cleanup.dueAt);
|
|
2104
3456
|
return;
|
|
2105
3457
|
}
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
3458
|
+
await this.setCleanupPhase(cleanup, "index", {}, this.now() + 1);
|
|
3459
|
+
return;
|
|
3460
|
+
}
|
|
3461
|
+
if (cleanup.phase === "index") {
|
|
3462
|
+
await this.deleteIndexPage(cleanup);
|
|
3463
|
+
return;
|
|
3464
|
+
}
|
|
3465
|
+
if (cleanup.phase === "streams") {
|
|
3466
|
+
await this.deleteStreamPage(cleanup);
|
|
3467
|
+
return;
|
|
3468
|
+
}
|
|
3469
|
+
if (cleanup.phase === "queues") {
|
|
3470
|
+
await this.deleteQueuePage(cleanup);
|
|
3471
|
+
return;
|
|
3472
|
+
}
|
|
3473
|
+
if (cleanup.phase === "payload") await this.deletePayloadBatch();
|
|
3474
|
+
}
|
|
3475
|
+
async executeTerminalCleanup(cleanup) {
|
|
3476
|
+
if (cleanup.phase === "hooks") await this.deleteTerminalHookPage(cleanup);
|
|
3477
|
+
else if (cleanup.phase === "markers") await this.deleteTerminalHookMarkerPage(cleanup);
|
|
3478
|
+
else await this.deleteTerminalWaitPage(cleanup);
|
|
3479
|
+
}
|
|
3480
|
+
async deleteTerminalHookPage(cleanup) {
|
|
3481
|
+
const entries = await this.ctx.storage.list({
|
|
3482
|
+
prefix: HOOK_KEY_PREFIX,
|
|
3483
|
+
limit: 65
|
|
3484
|
+
});
|
|
3485
|
+
const page = Array.from(entries.values()).slice(0, TERMINAL_HOOK_PAGE_SIZE);
|
|
3486
|
+
await this.workflowIndex().releaseHookIndexes({
|
|
3487
|
+
runId: cleanup.runId,
|
|
3488
|
+
hooks: page.map((hook) => ({
|
|
3489
|
+
hookId: hook.hookId,
|
|
3490
|
+
token: hook.token
|
|
3491
|
+
}))
|
|
3492
|
+
});
|
|
3493
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3494
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3495
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3496
|
+
const keys = page.flatMap((hook) => [`${HOOK_KEY_PREFIX}${hook.hookId}`, hookCreationIndexKey(hook)]);
|
|
3497
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3498
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3499
|
+
...current,
|
|
3500
|
+
phase: entries.size <= TERMINAL_HOOK_PAGE_SIZE ? "markers" : "hooks",
|
|
3501
|
+
generation: current.generation + 1,
|
|
3502
|
+
attempts: 0,
|
|
3503
|
+
lastError: void 0
|
|
3504
|
+
});
|
|
3505
|
+
await txn.setAlarm(this.now() + 1);
|
|
3506
|
+
});
|
|
3507
|
+
}
|
|
3508
|
+
async deleteTerminalHookMarkerPage(cleanup) {
|
|
3509
|
+
const entries = await this.ctx.storage.list({
|
|
3510
|
+
prefix: HOOK_MARKER_PREFIX,
|
|
3511
|
+
limit: 65
|
|
3512
|
+
});
|
|
3513
|
+
const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
|
|
3514
|
+
await this.workflowIndex().releaseHookIndexes({
|
|
3515
|
+
runId: cleanup.runId,
|
|
3516
|
+
hooks: page.map(([, reference]) => reference)
|
|
3517
|
+
});
|
|
3518
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3519
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3520
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3521
|
+
const keys = page.map(([key]) => key);
|
|
3522
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3523
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3524
|
+
...current,
|
|
3525
|
+
phase: entries.size <= HOOK_MARKER_PAGE_SIZE ? "waits" : "markers",
|
|
3526
|
+
generation: current.generation + 1,
|
|
3527
|
+
attempts: 0,
|
|
3528
|
+
lastError: void 0
|
|
3529
|
+
});
|
|
3530
|
+
await txn.setAlarm(this.now() + 1);
|
|
3531
|
+
});
|
|
3532
|
+
}
|
|
3533
|
+
async deleteTerminalWaitPage(cleanup) {
|
|
3534
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3535
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3536
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3537
|
+
const entries = await txn.list({
|
|
3538
|
+
prefix: WAIT_KEY_PREFIX,
|
|
3539
|
+
limit: 129
|
|
3540
|
+
});
|
|
3541
|
+
const keys = Array.from(entries.keys()).slice(0, TERMINAL_WAIT_PAGE_SIZE);
|
|
3542
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3543
|
+
if (entries.size > TERMINAL_WAIT_PAGE_SIZE) {
|
|
3544
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3545
|
+
...current,
|
|
3546
|
+
generation: current.generation + 1,
|
|
3547
|
+
attempts: 0,
|
|
3548
|
+
lastError: void 0
|
|
2126
3549
|
});
|
|
2127
|
-
await this.
|
|
2128
|
-
|
|
3550
|
+
await txn.setAlarm(this.now() + 1);
|
|
3551
|
+
return;
|
|
2129
3552
|
}
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
3553
|
+
await txn.delete(TERMINAL_CLEANUP_KEY);
|
|
3554
|
+
const retention = await txn.get(CLEANUP_RECORD_KEY);
|
|
3555
|
+
if (retention && retention.phase !== "tombstoned") await txn.setAlarm(Math.max(this.now() + 1, retention.dueAt.getTime()));
|
|
3556
|
+
else await txn.deleteAlarm();
|
|
3557
|
+
});
|
|
3558
|
+
}
|
|
3559
|
+
async deleteIndexPage(cleanup) {
|
|
3560
|
+
const hookEntries = await this.ctx.storage.list({
|
|
3561
|
+
prefix: HOOK_MARKER_PREFIX,
|
|
3562
|
+
limit: 65
|
|
3563
|
+
});
|
|
3564
|
+
const hookMarkers = Array.from(hookEntries).slice(0, HOOK_MARKER_PAGE_SIZE);
|
|
3565
|
+
await this.workflowIndex().expireRun({
|
|
3566
|
+
runId: cleanup.runId,
|
|
3567
|
+
keys: [workflowRunIndexKey(cleanup), globalRunIndexKey(cleanup)],
|
|
3568
|
+
hooks: hookMarkers.map(([, value]) => value),
|
|
3569
|
+
expiredAt: cleanup.dueAt.getTime()
|
|
3570
|
+
});
|
|
3571
|
+
const markerKeys = hookMarkers.map(([key]) => key);
|
|
3572
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3573
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3574
|
+
if (current?.phase !== "index" || current.generation !== cleanup.generation) return;
|
|
3575
|
+
if (markerKeys.length > 0) await txn.delete(markerKeys);
|
|
3576
|
+
if (hookEntries.size <= HOOK_MARKER_PAGE_SIZE) await txn.put(CLEANUP_RECORD_KEY, {
|
|
3577
|
+
...current,
|
|
3578
|
+
phase: "streams",
|
|
3579
|
+
generation: current.generation + 1,
|
|
3580
|
+
attempts: 0,
|
|
3581
|
+
lastError: void 0
|
|
3582
|
+
});
|
|
3583
|
+
else await txn.put(CLEANUP_RECORD_KEY, {
|
|
3584
|
+
...current,
|
|
3585
|
+
generation: current.generation + 1,
|
|
3586
|
+
attempts: 0,
|
|
3587
|
+
lastError: void 0
|
|
3588
|
+
});
|
|
3589
|
+
await txn.setAlarm(this.now() + 1);
|
|
3590
|
+
});
|
|
3591
|
+
}
|
|
3592
|
+
async deleteStreamPage(cleanup) {
|
|
3593
|
+
const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
|
|
3594
|
+
const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
|
|
3595
|
+
const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime(), { limit: STREAMS_PER_CLEANUP_PAGE });
|
|
3596
|
+
const completed = [];
|
|
3597
|
+
let remainingChunks = STREAM_CHUNKS_PER_CLEANUP_PAGE;
|
|
3598
|
+
let remainingBytes = STREAM_BYTES_PER_CLEANUP_PAGE;
|
|
3599
|
+
for (const name of streams) {
|
|
3600
|
+
if (remainingChunks <= 0 || remainingBytes <= 0) break;
|
|
3601
|
+
const result = await streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime(), {
|
|
3602
|
+
limit: remainingChunks,
|
|
3603
|
+
byteLimit: remainingBytes
|
|
3604
|
+
});
|
|
3605
|
+
remainingChunks -= result.chunks;
|
|
3606
|
+
remainingBytes -= result.bytes;
|
|
3607
|
+
if (result.done) completed.push(name);
|
|
3608
|
+
}
|
|
3609
|
+
const registryResult = await registry.finalizeRegistry(cleanup.runId, completed);
|
|
3610
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3611
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3612
|
+
if (current?.phase !== "streams" || current.generation !== cleanup.generation) return;
|
|
3613
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3614
|
+
...current,
|
|
3615
|
+
deletedStreams: registryResult.deleted,
|
|
3616
|
+
phase: registryResult.done ? "queues" : "streams",
|
|
3617
|
+
generation: current.generation + 1,
|
|
3618
|
+
attempts: 0,
|
|
3619
|
+
lastError: void 0
|
|
3620
|
+
});
|
|
3621
|
+
await txn.setAlarm(this.now() + 1);
|
|
3622
|
+
});
|
|
3623
|
+
}
|
|
3624
|
+
async deleteQueuePage(cleanup) {
|
|
3625
|
+
const queues = this.namespace("WORKFLOW_QUEUE");
|
|
3626
|
+
const progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3627
|
+
queueShard: 0,
|
|
3628
|
+
queueShardDeleted: 0
|
|
3629
|
+
};
|
|
3630
|
+
if (progress.pendingAck) {
|
|
3631
|
+
const { queueShard, receipt } = progress.pendingAck;
|
|
3632
|
+
if (!(await queues.get(queues.idFromName(`q:${queueShard}`)).acknowledgeExpireRun(cleanup.runId, receipt)).acknowledged) throw new Error(`queue shard ${queueShard} rejected expiry receipt for ${cleanup.runId}`);
|
|
3633
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3634
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3635
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3636
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY);
|
|
3637
|
+
if (persisted?.pendingAck?.queueShard !== queueShard || persisted.pendingAck.receipt.expiredAt !== receipt.expiredAt || persisted.pendingAck.receipt.deleted !== receipt.deleted) return;
|
|
3638
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3639
|
+
...current,
|
|
3640
|
+
generation: current.generation + 1,
|
|
3641
|
+
attempts: 0,
|
|
3642
|
+
lastError: void 0
|
|
2148
3643
|
});
|
|
2149
|
-
await
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
3644
|
+
await txn.put(CLEANUP_PROGRESS_KEY, {
|
|
3645
|
+
queueShard: queueShard + 1,
|
|
3646
|
+
queueShardDeleted: 0
|
|
3647
|
+
});
|
|
3648
|
+
await txn.setAlarm(this.now() + 1);
|
|
3649
|
+
});
|
|
3650
|
+
return;
|
|
3651
|
+
}
|
|
3652
|
+
if (progress.queueShard < cleanup.queueShards) {
|
|
3653
|
+
const requestedShard = progress.queueShard;
|
|
3654
|
+
const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
|
|
3655
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3656
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3657
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3658
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3659
|
+
queueShard: 0,
|
|
3660
|
+
queueShardDeleted: 0
|
|
3661
|
+
};
|
|
3662
|
+
if (persisted.queueShard !== requestedShard || persisted.pendingAck) return;
|
|
3663
|
+
if (result.deleted < persisted.queueShardDeleted) throw new Error(`queue shard ${requestedShard} regressed expiry receipt for ${cleanup.runId}`);
|
|
3664
|
+
const delta = result.deleted - persisted.queueShardDeleted;
|
|
3665
|
+
const next = result.done ? {
|
|
3666
|
+
queueShard: requestedShard,
|
|
3667
|
+
queueShardDeleted: result.deleted,
|
|
3668
|
+
pendingAck: {
|
|
3669
|
+
queueShard: requestedShard,
|
|
3670
|
+
receipt: result.receipt
|
|
3671
|
+
}
|
|
3672
|
+
} : {
|
|
3673
|
+
queueShard: requestedShard,
|
|
3674
|
+
queueShardDeleted: result.deleted
|
|
3675
|
+
};
|
|
3676
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3677
|
+
...current,
|
|
3678
|
+
deletedQueueMessages: current.deletedQueueMessages + delta,
|
|
3679
|
+
generation: current.generation + 1,
|
|
3680
|
+
attempts: 0,
|
|
3681
|
+
lastError: void 0
|
|
3682
|
+
});
|
|
3683
|
+
await txn.put(CLEANUP_PROGRESS_KEY, next);
|
|
3684
|
+
await txn.setAlarm(this.now() + 1);
|
|
3685
|
+
});
|
|
3686
|
+
return;
|
|
2166
3687
|
}
|
|
3688
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3689
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3690
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3691
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3692
|
+
queueShard: 0,
|
|
3693
|
+
queueShardDeleted: 0
|
|
3694
|
+
};
|
|
3695
|
+
if (persisted.queueShard >= current.queueShards && !persisted.pendingAck) {
|
|
3696
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3697
|
+
...current,
|
|
3698
|
+
phase: "payload",
|
|
3699
|
+
generation: current.generation + 1
|
|
3700
|
+
});
|
|
3701
|
+
await txn.delete(CLEANUP_PROGRESS_KEY);
|
|
3702
|
+
await txn.setAlarm(this.now() + 1);
|
|
3703
|
+
}
|
|
3704
|
+
});
|
|
2167
3705
|
}
|
|
2168
3706
|
async deletePayloadBatch() {
|
|
2169
3707
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2170
3708
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2171
3709
|
if (!cleanup || cleanup.phase !== "payload") return true;
|
|
2172
|
-
const entries = await txn.list({
|
|
3710
|
+
const entries = await txn.list({
|
|
3711
|
+
limit: 131,
|
|
3712
|
+
noCache: true
|
|
3713
|
+
});
|
|
2173
3714
|
const keys = Array.from(entries.keys()).filter((key) => key !== "retention:cleanup" && key !== "retention:tombstone").slice(0, PAYLOAD_DELETE_BATCH);
|
|
2174
3715
|
const now = new Date(this.now());
|
|
2175
3716
|
await txn.put(TOMBSTONE_KEY, cleanupTombstone(cleanup, now));
|
|
2176
|
-
|
|
3717
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
2177
3718
|
if (keys.length > 0) {
|
|
2178
3719
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2179
3720
|
...cleanup,
|
|
2180
3721
|
deletedPayloadKeys: cleanup.deletedPayloadKeys + keys.length,
|
|
3722
|
+
generation: cleanup.generation + 1,
|
|
2181
3723
|
lastError: void 0
|
|
2182
3724
|
});
|
|
2183
3725
|
await txn.setAlarm(this.now() + 1);
|
|
2184
3726
|
return false;
|
|
2185
3727
|
}
|
|
2186
|
-
|
|
3728
|
+
const completed = {
|
|
2187
3729
|
...cleanup,
|
|
2188
3730
|
phase: "tombstoned",
|
|
3731
|
+
generation: cleanup.generation + 1,
|
|
2189
3732
|
attempts: 0,
|
|
2190
3733
|
lastError: void 0,
|
|
2191
3734
|
tombstonedAt: now
|
|
2192
|
-
}
|
|
3735
|
+
};
|
|
3736
|
+
await txn.put(TOMBSTONE_KEY, cleanupTombstone(completed, now));
|
|
3737
|
+
await txn.delete(CLEANUP_RECORD_KEY);
|
|
2193
3738
|
await txn.deleteAlarm();
|
|
2194
3739
|
return true;
|
|
2195
3740
|
});
|
|
2196
3741
|
}
|
|
2197
|
-
async recordCleanupFailure(error) {
|
|
3742
|
+
async recordCleanupFailure(error, expected) {
|
|
2198
3743
|
await this.ctx.storage.transaction(async (txn) => {
|
|
2199
3744
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2200
|
-
if (!cleanup || cleanup.phase === "tombstoned") return;
|
|
3745
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation || cleanup.phase === "tombstoned") return;
|
|
2201
3746
|
const attempts = cleanup.attempts + 1;
|
|
2202
3747
|
const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
|
|
2203
3748
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2204
3749
|
...cleanup,
|
|
3750
|
+
generation: cleanup.generation + 1,
|
|
3751
|
+
attempts,
|
|
3752
|
+
lastError: error instanceof Error ? error.message : String(error),
|
|
3753
|
+
lastAttemptAt: new Date(this.now())
|
|
3754
|
+
});
|
|
3755
|
+
await txn.setAlarm(this.now() + delay);
|
|
3756
|
+
});
|
|
3757
|
+
}
|
|
3758
|
+
async recordTerminalCleanupFailure(error, expected) {
|
|
3759
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3760
|
+
const cleanup = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3761
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
|
|
3762
|
+
const attempts = cleanup.attempts + 1;
|
|
3763
|
+
const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
|
|
3764
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3765
|
+
...cleanup,
|
|
3766
|
+
generation: cleanup.generation + 1,
|
|
2205
3767
|
attempts,
|
|
2206
3768
|
lastError: error instanceof Error ? error.message : String(error),
|
|
2207
3769
|
lastAttemptAt: new Date(this.now())
|
|
@@ -2236,6 +3798,6 @@ var worker_default = { async fetch(request, env) {
|
|
|
2236
3798
|
return createRouter(env)(request);
|
|
2237
3799
|
} };
|
|
2238
3800
|
//#endregion
|
|
2239
|
-
export {
|
|
3801
|
+
export { HookIdDO, HookTokenDO, QueueDO, RunCatalogDO, StreamDO, WorkflowRunDO, createRouter, worker_default as default };
|
|
2240
3802
|
|
|
2241
3803
|
//# sourceMappingURL=worker.js.map
|