@ewhauser/world-celld 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/README.md +49 -19
- package/celld-worker/README.md +5 -3
- package/celld-worker/worker.ts +9 -1
- package/celld-worker/wrangler.jsonc +13 -4
- package/dist/index.d.ts +168 -58
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +432 -124
- package/dist/index.js.map +1 -1
- package/dist/testing.d.ts +47 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +1988 -558
- package/dist/testing.js.map +1 -1
- package/dist/util-ya3V_FTA.js +558 -0
- package/dist/util-ya3V_FTA.js.map +1 -0
- package/dist/worker.d.ts +238 -108
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +2082 -520
- package/dist/worker.js.map +1 -1
- package/package.json +4 -2
- package/dist/retention-bLY1vcOu.js +0 -164
- package/dist/retention-bLY1vcOu.js.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { A as validateStreamReadRequest, E as encodeStreamReadResult, O as encodeStreamWriteResult, S as STREAM_BATCH_CONTENT_TYPE, c as HOOK_CLAIM_LEASE_MS, d as MAX_QUEUE_DELIVERY_TIMEOUT_MS, f as MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS, g as MAX_STREAM_CHUNK_BYTES, h as MAX_STREAM_BATCH_BYTES, i as parse, j as validateStreamWriteChunks, k as normalizeStreamError, l as LIFECYCLE_COMPACTION_RETRY_MS, m as QUEUE_INFLIGHT_GRACE_MS, n as rpcParse, o as createWorkflowIndex, p as QUEUE_FENCE_GRACE_MS, r as rpcStringify, t as compact, v as MAX_STREAM_LONG_POLL_MS, w as decodeStreamWriteBatch, y as MAX_STREAM_READ_BYTES } from "./util-ya3V_FTA.js";
|
|
2
2
|
import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
|
|
3
3
|
import http from "node:http";
|
|
4
4
|
//#region src/testing/fake-cell.ts
|
|
@@ -33,40 +33,109 @@ var FakeStorage = class {
|
|
|
33
33
|
clock;
|
|
34
34
|
data = /* @__PURE__ */ new Map();
|
|
35
35
|
alarmAt = null;
|
|
36
|
+
getManyCalls = [];
|
|
37
|
+
operationCounts = {
|
|
38
|
+
get: 0,
|
|
39
|
+
getMany: 0,
|
|
40
|
+
list: 0,
|
|
41
|
+
put: 0,
|
|
42
|
+
putMany: 0,
|
|
43
|
+
delete: 0,
|
|
44
|
+
deleteMany: 0,
|
|
45
|
+
transaction: 0
|
|
46
|
+
};
|
|
47
|
+
transactionTail = Promise.resolve();
|
|
48
|
+
transactionDelayMs = 0;
|
|
49
|
+
/** Deterministic query-shape instrumentation for storage scaling tests. */
|
|
50
|
+
listCalls = [];
|
|
51
|
+
operationCalls = [];
|
|
36
52
|
mutationFailure;
|
|
37
53
|
constructor(clock = () => Date.now()) {
|
|
38
54
|
this.clock = clock;
|
|
39
55
|
}
|
|
40
|
-
async get(
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
56
|
+
async get(keyOrKeys) {
|
|
57
|
+
if (Array.isArray(keyOrKeys)) {
|
|
58
|
+
this.operationCounts.getMany += 1;
|
|
59
|
+
this.getManyCalls.push([...keyOrKeys]);
|
|
60
|
+
this.recordStorageCall("get", keyOrKeys, false);
|
|
61
|
+
return new Map(keyOrKeys.filter((key) => this.data.has(key)).map((key) => [key, this.data.get(key)]));
|
|
62
|
+
}
|
|
63
|
+
this.operationCounts.get += 1;
|
|
64
|
+
this.recordStorageCall("get", [keyOrKeys], false);
|
|
65
|
+
return this.data.get(keyOrKeys);
|
|
66
|
+
}
|
|
67
|
+
async put(keyOrEntries, value) {
|
|
68
|
+
if (typeof keyOrEntries === "string") {
|
|
69
|
+
this.operationCounts.put += 1;
|
|
70
|
+
this.maybeFailMutation({
|
|
71
|
+
operation: "put",
|
|
72
|
+
key: keyOrEntries,
|
|
73
|
+
value
|
|
74
|
+
});
|
|
75
|
+
this.recordStorageCall("put", [keyOrEntries], false);
|
|
76
|
+
this.data.set(keyOrEntries, structuredClone(value));
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
this.operationCounts.putMany += 1;
|
|
80
|
+
for (const [key, entryValue] of Object.entries(keyOrEntries)) this.maybeFailMutation({
|
|
45
81
|
operation: "put",
|
|
46
82
|
key,
|
|
47
|
-
value
|
|
83
|
+
value: entryValue
|
|
48
84
|
});
|
|
49
|
-
this.
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
85
|
+
this.recordStorageCall("put", Object.keys(keyOrEntries), false);
|
|
86
|
+
for (const [key, entryValue] of Object.entries(keyOrEntries)) this.data.set(key, structuredClone(entryValue));
|
|
87
|
+
}
|
|
88
|
+
async delete(keyOrKeys) {
|
|
89
|
+
if (typeof keyOrKeys === "string") {
|
|
90
|
+
this.operationCounts.delete += 1;
|
|
91
|
+
this.maybeFailMutation({
|
|
92
|
+
operation: "delete",
|
|
93
|
+
key: keyOrKeys
|
|
94
|
+
});
|
|
95
|
+
this.recordStorageCall("delete", [keyOrKeys], false);
|
|
96
|
+
return this.data.delete(keyOrKeys);
|
|
97
|
+
}
|
|
98
|
+
this.operationCounts.deleteMany += 1;
|
|
99
|
+
for (const key of keyOrKeys) this.maybeFailMutation({
|
|
53
100
|
operation: "delete",
|
|
54
101
|
key
|
|
55
102
|
});
|
|
56
|
-
|
|
103
|
+
this.recordStorageCall("delete", keyOrKeys, false);
|
|
104
|
+
let deleted = 0;
|
|
105
|
+
for (const key of keyOrKeys) if (this.data.delete(key)) deleted += 1;
|
|
106
|
+
return deleted;
|
|
57
107
|
}
|
|
58
108
|
async deleteAll() {
|
|
59
109
|
this.data.clear();
|
|
60
110
|
}
|
|
61
111
|
async list(options = {}) {
|
|
112
|
+
this.operationCounts.list += 1;
|
|
62
113
|
const keys = applyListOptions(Array.from(this.data.keys()), options);
|
|
114
|
+
this.recordList(options, keys.length, false);
|
|
63
115
|
return new Map(keys.map((k) => [k, this.data.get(k)]));
|
|
64
116
|
}
|
|
65
117
|
async transaction(cb) {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
118
|
+
this.operationCounts.transaction += 1;
|
|
119
|
+
let release;
|
|
120
|
+
const previous = this.transactionTail;
|
|
121
|
+
this.transactionTail = new Promise((resolve) => {
|
|
122
|
+
release = resolve;
|
|
123
|
+
});
|
|
124
|
+
await previous;
|
|
125
|
+
try {
|
|
126
|
+
if (this.transactionDelayMs > 0) await new Promise((resolve) => setTimeout(resolve, this.transactionDelayMs));
|
|
127
|
+
const txn = new FakeTransaction(this);
|
|
128
|
+
const result = await cb(txn);
|
|
129
|
+
txn.commit();
|
|
130
|
+
return result;
|
|
131
|
+
} finally {
|
|
132
|
+
release();
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
/** Add deterministic per-transaction work for contention benchmarks. */
|
|
136
|
+
setTransactionDelay(ms) {
|
|
137
|
+
if (!Number.isFinite(ms) || ms < 0) throw new Error("transaction delay must be non-negative");
|
|
138
|
+
this.transactionDelayMs = ms;
|
|
70
139
|
}
|
|
71
140
|
async getAlarm() {
|
|
72
141
|
return this.alarmAt;
|
|
@@ -80,6 +149,24 @@ var FakeStorage = class {
|
|
|
80
149
|
now() {
|
|
81
150
|
return this.clock();
|
|
82
151
|
}
|
|
152
|
+
resetOperationCounts() {
|
|
153
|
+
for (const key of Object.keys(this.operationCounts)) this.operationCounts[key] = 0;
|
|
154
|
+
this.getManyCalls.length = 0;
|
|
155
|
+
this.operationCalls.length = 0;
|
|
156
|
+
}
|
|
157
|
+
/** @internal Used by FakeTransaction to count the same platform operations. */
|
|
158
|
+
recordOperation(operation) {
|
|
159
|
+
this.operationCounts[operation] += 1;
|
|
160
|
+
}
|
|
161
|
+
/** @internal Shared with FakeTransaction for deterministic key tracing. */
|
|
162
|
+
recordStorageCall(operation, keys, transactional) {
|
|
163
|
+
this.operationCalls.push({
|
|
164
|
+
operation,
|
|
165
|
+
keys: [...keys],
|
|
166
|
+
transactional,
|
|
167
|
+
synchronous: false
|
|
168
|
+
});
|
|
169
|
+
}
|
|
83
170
|
/** Inject one matching write failure, including writes inside transactions. */
|
|
84
171
|
failNextMutation(predicate, error = /* @__PURE__ */ new Error("injected fake storage mutation failure")) {
|
|
85
172
|
this.mutationFailure = {
|
|
@@ -94,6 +181,14 @@ var FakeStorage = class {
|
|
|
94
181
|
this.mutationFailure = void 0;
|
|
95
182
|
throw error;
|
|
96
183
|
}
|
|
184
|
+
/** @internal Shared with FakeTransaction for deterministic query tracing. */
|
|
185
|
+
recordList(options, resultSize, transactional) {
|
|
186
|
+
this.listCalls.push({
|
|
187
|
+
options: { ...options },
|
|
188
|
+
resultSize,
|
|
189
|
+
transactional
|
|
190
|
+
});
|
|
191
|
+
}
|
|
97
192
|
};
|
|
98
193
|
var FakeTransaction = class {
|
|
99
194
|
storage;
|
|
@@ -103,21 +198,64 @@ var FakeTransaction = class {
|
|
|
103
198
|
constructor(storage) {
|
|
104
199
|
this.storage = storage;
|
|
105
200
|
}
|
|
106
|
-
async get(
|
|
201
|
+
async get(keyOrKeys) {
|
|
202
|
+
if (Array.isArray(keyOrKeys)) {
|
|
203
|
+
this.storage.recordOperation("getMany");
|
|
204
|
+
this.storage.getManyCalls.push([...keyOrKeys]);
|
|
205
|
+
this.storage.recordStorageCall("get", keyOrKeys, true);
|
|
206
|
+
const result = /* @__PURE__ */ new Map();
|
|
207
|
+
for (const key of keyOrKeys) {
|
|
208
|
+
if (this.deleted.has(key)) continue;
|
|
209
|
+
if (this.staged.has(key)) result.set(key, this.staged.get(key));
|
|
210
|
+
else if (this.storage.data.has(key)) result.set(key, this.storage.data.get(key));
|
|
211
|
+
}
|
|
212
|
+
return result;
|
|
213
|
+
}
|
|
214
|
+
this.storage.recordOperation("get");
|
|
215
|
+
const key = keyOrKeys;
|
|
216
|
+
this.storage.recordStorageCall("get", [key], true);
|
|
107
217
|
if (this.deleted.has(key)) return void 0;
|
|
108
218
|
if (this.staged.has(key)) return this.staged.get(key);
|
|
109
219
|
return this.storage.data.get(key);
|
|
110
220
|
}
|
|
111
|
-
async put(
|
|
112
|
-
|
|
221
|
+
async put(keyOrEntries, value) {
|
|
222
|
+
if (typeof keyOrEntries === "string") {
|
|
223
|
+
this.storage.recordOperation("put");
|
|
224
|
+
this.storage.maybeFailMutation({
|
|
225
|
+
operation: "put",
|
|
226
|
+
key: keyOrEntries,
|
|
227
|
+
value
|
|
228
|
+
});
|
|
229
|
+
this.storage.recordStorageCall("put", [keyOrEntries], true);
|
|
230
|
+
this.deleted.delete(keyOrEntries);
|
|
231
|
+
this.staged.set(keyOrEntries, structuredClone(value));
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
this.storage.recordOperation("putMany");
|
|
235
|
+
for (const [key, entryValue] of Object.entries(keyOrEntries)) this.storage.maybeFailMutation({
|
|
113
236
|
operation: "put",
|
|
114
237
|
key,
|
|
115
|
-
value
|
|
238
|
+
value: entryValue
|
|
116
239
|
});
|
|
117
|
-
this.
|
|
118
|
-
|
|
240
|
+
this.storage.recordStorageCall("put", Object.keys(keyOrEntries), true);
|
|
241
|
+
for (const [key, entryValue] of Object.entries(keyOrEntries)) {
|
|
242
|
+
this.deleted.delete(key);
|
|
243
|
+
this.staged.set(key, structuredClone(entryValue));
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
async delete(keyOrKeys) {
|
|
247
|
+
if (typeof keyOrKeys === "string") {
|
|
248
|
+
this.storage.recordOperation("delete");
|
|
249
|
+
this.storage.recordStorageCall("delete", [keyOrKeys], true);
|
|
250
|
+
return this.deleteOne(keyOrKeys);
|
|
251
|
+
}
|
|
252
|
+
this.storage.recordOperation("deleteMany");
|
|
253
|
+
this.storage.recordStorageCall("delete", keyOrKeys, true);
|
|
254
|
+
let deleted = 0;
|
|
255
|
+
for (const key of keyOrKeys) if (this.deleteOne(key)) deleted += 1;
|
|
256
|
+
return deleted;
|
|
119
257
|
}
|
|
120
|
-
|
|
258
|
+
deleteOne(key) {
|
|
121
259
|
this.storage.maybeFailMutation({
|
|
122
260
|
operation: "delete",
|
|
123
261
|
key
|
|
@@ -128,10 +266,12 @@ var FakeTransaction = class {
|
|
|
128
266
|
return existed;
|
|
129
267
|
}
|
|
130
268
|
async list(options = {}) {
|
|
269
|
+
this.storage.recordOperation("list");
|
|
131
270
|
const merged = new Map(this.storage.data);
|
|
132
271
|
for (const key of this.deleted) merged.delete(key);
|
|
133
272
|
for (const [key, value] of this.staged) merged.set(key, value);
|
|
134
273
|
const keys = applyListOptions(Array.from(merged.keys()), options);
|
|
274
|
+
this.storage.recordList(options, keys.length, true);
|
|
135
275
|
return new Map(keys.map((k) => [k, merged.get(k)]));
|
|
136
276
|
}
|
|
137
277
|
async getAlarm() {
|
|
@@ -175,28 +315,31 @@ var FakeFleet = class {
|
|
|
175
315
|
get: (id) => this.cell(bindingKey, id.toString()).instance
|
|
176
316
|
};
|
|
177
317
|
}
|
|
318
|
+
instantiate(bindingKey, name, storage, pendingWaits) {
|
|
319
|
+
const CellCtor = this.classes[bindingKey];
|
|
320
|
+
if (!CellCtor) throw new Error(`no cell class registered for binding: ${bindingKey}`);
|
|
321
|
+
return new CellCtor({
|
|
322
|
+
storage,
|
|
323
|
+
id: {
|
|
324
|
+
toString: () => name,
|
|
325
|
+
name
|
|
326
|
+
},
|
|
327
|
+
waitUntil(promise) {
|
|
328
|
+
pendingWaits.push(promise);
|
|
329
|
+
},
|
|
330
|
+
async blockConcurrencyWhile(cb) {
|
|
331
|
+
return cb();
|
|
332
|
+
}
|
|
333
|
+
}, this.cellEnv);
|
|
334
|
+
}
|
|
178
335
|
cell(bindingKey, name) {
|
|
179
336
|
const key = `${bindingKey}\0${name}`;
|
|
180
337
|
let slot = this.cells.get(key);
|
|
181
338
|
if (!slot) {
|
|
182
|
-
const CellCtor = this.classes[bindingKey];
|
|
183
|
-
if (!CellCtor) throw new Error(`no cell class registered for binding: ${bindingKey}`);
|
|
184
339
|
const storage = new FakeStorage(() => this.now);
|
|
185
340
|
const pendingWaits = [];
|
|
186
341
|
slot = {
|
|
187
|
-
instance:
|
|
188
|
-
storage,
|
|
189
|
-
id: {
|
|
190
|
-
toString: () => name,
|
|
191
|
-
name
|
|
192
|
-
},
|
|
193
|
-
waitUntil(promise) {
|
|
194
|
-
pendingWaits.push(promise);
|
|
195
|
-
},
|
|
196
|
-
async blockConcurrencyWhile(cb) {
|
|
197
|
-
return cb();
|
|
198
|
-
}
|
|
199
|
-
}, this.cellEnv),
|
|
342
|
+
instance: this.instantiate(bindingKey, name, storage, pendingWaits),
|
|
200
343
|
storage,
|
|
201
344
|
pendingWaits,
|
|
202
345
|
alarmRetryCount: 0,
|
|
@@ -206,6 +349,12 @@ var FakeFleet = class {
|
|
|
206
349
|
}
|
|
207
350
|
return slot;
|
|
208
351
|
}
|
|
352
|
+
/** Recreate one in-memory DO instance while preserving its durable storage. */
|
|
353
|
+
restartCell(bindingKey, name) {
|
|
354
|
+
const slot = this.cell(bindingKey, name);
|
|
355
|
+
slot.instance = this.instantiate(bindingKey, name, slot.storage, slot.pendingWaits);
|
|
356
|
+
return slot.instance;
|
|
357
|
+
}
|
|
209
358
|
advance(ms) {
|
|
210
359
|
this.now += ms;
|
|
211
360
|
}
|
|
@@ -292,18 +441,28 @@ async function authenticate(request, secret) {
|
|
|
292
441
|
* Routes:
|
|
293
442
|
* GET /v1/health — liveness + spec version (unauthenticated)
|
|
294
443
|
* POST /v1/rpc/{binding}/{name}/{method} — DO RPC (bearer auth)
|
|
444
|
+
* POST /v1/index/{domain}/{method} — cohesive index operation
|
|
445
|
+
* GET /v1/streams/{name}/chunks — bounded binary long-poll read
|
|
446
|
+
* POST /v1/streams/{name}/chunks — bounded binary batch append
|
|
295
447
|
*
|
|
296
|
-
*
|
|
297
|
-
*
|
|
298
|
-
* dispatch — everything else (including fetch/alarm) is unreachable.
|
|
448
|
+
* Generic RPC bodies use the tagged JSON codec. Stream chunk bodies use the
|
|
449
|
+
* compact binary stream protocol. Only whitelisted routes/methods dispatch.
|
|
299
450
|
*/
|
|
300
451
|
const WORLD_NAME = "world-celld";
|
|
301
452
|
const WORLD_VERSION = "0.1.0";
|
|
453
|
+
function workflowIndex(env) {
|
|
454
|
+
return createWorkflowIndex({
|
|
455
|
+
runCatalog: env.WORKFLOW_RUN_CATALOG,
|
|
456
|
+
hookTokens: env.WORKFLOW_HOOK_TOKENS,
|
|
457
|
+
hookIds: env.WORKFLOW_HOOK_IDS
|
|
458
|
+
});
|
|
459
|
+
}
|
|
302
460
|
const BINDINGS = {
|
|
303
461
|
runs: {
|
|
304
462
|
env: "WORKFLOW_DB",
|
|
305
463
|
methods: /* @__PURE__ */ new Set([
|
|
306
464
|
"applyEvent",
|
|
465
|
+
"getLifecycleStatus",
|
|
307
466
|
"getRun",
|
|
308
467
|
"getStep",
|
|
309
468
|
"getEvent",
|
|
@@ -313,16 +472,15 @@ const BINDINGS = {
|
|
|
313
472
|
"getCleanupStatus",
|
|
314
473
|
"scheduleCleanup",
|
|
315
474
|
"cleanupNow",
|
|
316
|
-
"rearmCleanup"
|
|
475
|
+
"rearmCleanup",
|
|
476
|
+
"resolveHookTokenClaim"
|
|
317
477
|
])
|
|
318
478
|
},
|
|
319
479
|
streams: {
|
|
320
480
|
env: "WORKFLOW_STREAMS",
|
|
321
481
|
methods: /* @__PURE__ */ new Set([
|
|
322
|
-
"writeChunk",
|
|
323
482
|
"closeStream",
|
|
324
|
-
"
|
|
325
|
-
"getInfo",
|
|
483
|
+
"failStream",
|
|
326
484
|
"registerStream",
|
|
327
485
|
"listStreams",
|
|
328
486
|
"expireRegistry",
|
|
@@ -330,20 +488,13 @@ const BINDINGS = {
|
|
|
330
488
|
"expireStream"
|
|
331
489
|
])
|
|
332
490
|
},
|
|
333
|
-
|
|
334
|
-
env: "
|
|
335
|
-
methods: /* @__PURE__ */ new Set([
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
"expireRun",
|
|
341
|
-
"list",
|
|
342
|
-
"reserveHookToken",
|
|
343
|
-
"finalizeHookIndexes",
|
|
344
|
-
"releaseHookToken",
|
|
345
|
-
"deleteHookIndexes"
|
|
346
|
-
])
|
|
491
|
+
"hook-tokens": {
|
|
492
|
+
env: "WORKFLOW_HOOK_TOKENS",
|
|
493
|
+
methods: /* @__PURE__ */ new Set(["get"])
|
|
494
|
+
},
|
|
495
|
+
"hook-ids": {
|
|
496
|
+
env: "WORKFLOW_HOOK_IDS",
|
|
497
|
+
methods: /* @__PURE__ */ new Set(["get"])
|
|
347
498
|
},
|
|
348
499
|
queue: {
|
|
349
500
|
env: "WORKFLOW_QUEUE",
|
|
@@ -354,12 +505,14 @@ const BINDINGS = {
|
|
|
354
505
|
"redriveDeadLetter",
|
|
355
506
|
"purgeDeadLetters",
|
|
356
507
|
"rearmAlarm",
|
|
357
|
-
"expireRun"
|
|
508
|
+
"expireRun",
|
|
509
|
+
"acknowledgeExpireRun"
|
|
358
510
|
])
|
|
359
511
|
}
|
|
360
512
|
};
|
|
361
513
|
/** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
|
|
362
514
|
const MAX_BODY_BYTES = 33554432;
|
|
515
|
+
const MAX_STREAM_WRITE_BODY_BYTES = MAX_STREAM_BATCH_BYTES + 1024;
|
|
363
516
|
function errorResponse(status, name, message) {
|
|
364
517
|
return Response.json({ error: {
|
|
365
518
|
name,
|
|
@@ -388,6 +541,49 @@ async function readBoundedBody(request) {
|
|
|
388
541
|
reader.releaseLock();
|
|
389
542
|
}
|
|
390
543
|
}
|
|
544
|
+
async function readBoundedBytes(request, maxBytes) {
|
|
545
|
+
if (!request.body) return /* @__PURE__ */ new Uint8Array();
|
|
546
|
+
const reader = request.body.getReader();
|
|
547
|
+
const chunks = [];
|
|
548
|
+
let total = 0;
|
|
549
|
+
try {
|
|
550
|
+
for (;;) {
|
|
551
|
+
const { done, value } = await reader.read();
|
|
552
|
+
if (done) break;
|
|
553
|
+
total += value.byteLength;
|
|
554
|
+
if (total > maxBytes) {
|
|
555
|
+
await reader.cancel("stream body exceeds configured limit").catch(() => void 0);
|
|
556
|
+
return null;
|
|
557
|
+
}
|
|
558
|
+
chunks.push(value);
|
|
559
|
+
}
|
|
560
|
+
} finally {
|
|
561
|
+
reader.releaseLock();
|
|
562
|
+
}
|
|
563
|
+
const body = new Uint8Array(total);
|
|
564
|
+
let offset = 0;
|
|
565
|
+
for (const chunk of chunks) {
|
|
566
|
+
body.set(chunk, offset);
|
|
567
|
+
offset += chunk.byteLength;
|
|
568
|
+
}
|
|
569
|
+
return body;
|
|
570
|
+
}
|
|
571
|
+
function parseBoundedInteger(url, name, minimum, maximum) {
|
|
572
|
+
const raw = url.searchParams.get(name);
|
|
573
|
+
const value = raw === null ? NaN : Number(raw);
|
|
574
|
+
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
|
575
|
+
const error = /* @__PURE__ */ new Error(`${name} must be an integer between ${minimum} and ${maximum}`);
|
|
576
|
+
error.name = "StreamProtocolError";
|
|
577
|
+
throw error;
|
|
578
|
+
}
|
|
579
|
+
return value;
|
|
580
|
+
}
|
|
581
|
+
function streamResponse(body) {
|
|
582
|
+
return new Response(body, {
|
|
583
|
+
status: 200,
|
|
584
|
+
headers: { "content-type": STREAM_BATCH_CONTENT_TYPE }
|
|
585
|
+
});
|
|
586
|
+
}
|
|
391
587
|
function createRouter(env) {
|
|
392
588
|
return async (request) => {
|
|
393
589
|
const url = new URL(request.url);
|
|
@@ -399,9 +595,83 @@ function createRouter(env) {
|
|
|
399
595
|
version: WORLD_VERSION,
|
|
400
596
|
specVersion: SPEC_VERSION_CURRENT
|
|
401
597
|
});
|
|
402
|
-
if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
|
|
403
598
|
const auth = await authenticate(request, env.WORLD_SECRET);
|
|
404
599
|
if (!auth.ok) return auth.response;
|
|
600
|
+
if (parts[1] === "index" && parts.length === 4) {
|
|
601
|
+
if (request.method !== "POST") return errorResponse(405, "MethodNotAllowed", "expected POST");
|
|
602
|
+
if (Number(request.headers.get("content-length") ?? "0") > MAX_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
603
|
+
let args;
|
|
604
|
+
try {
|
|
605
|
+
const text = await readBoundedBody(request);
|
|
606
|
+
if (text === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
607
|
+
const parsed = rpcParse(text);
|
|
608
|
+
if (!Array.isArray(parsed)) return errorResponse(400, "BadRequest", "body must be an argument array");
|
|
609
|
+
args = parsed;
|
|
610
|
+
} catch {
|
|
611
|
+
return errorResponse(400, "BadRequest", "malformed rpc body");
|
|
612
|
+
}
|
|
613
|
+
try {
|
|
614
|
+
const index = workflowIndex(env);
|
|
615
|
+
let result;
|
|
616
|
+
const operation = `${parts[2]}.${parts[3]}`;
|
|
617
|
+
if (operation === "runs.list") result = await index.listRuns(args[0]);
|
|
618
|
+
else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1], args[2]);
|
|
619
|
+
else if (operation === "runs.expire") result = await index.expireRun(args[0]);
|
|
620
|
+
else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
|
|
621
|
+
else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
|
|
622
|
+
else if (operation === "hooks.release-reservation") result = await index.releaseHookReservation(args[0], args[1], args[2]);
|
|
623
|
+
else if (operation === "hooks.release") result = await index.releaseHookIndexes(args[0]);
|
|
624
|
+
else return errorResponse(404, "NotFound", `unknown index operation: ${operation}`);
|
|
625
|
+
return new Response(rpcStringify(result ?? null), {
|
|
626
|
+
status: 200,
|
|
627
|
+
headers: { "content-type": "application/json" }
|
|
628
|
+
});
|
|
629
|
+
} catch (error) {
|
|
630
|
+
const err = error;
|
|
631
|
+
return Response.json({ error: {
|
|
632
|
+
name: err?.name ?? "Error",
|
|
633
|
+
message: err?.message ?? String(error),
|
|
634
|
+
status: typeof err?.status === "number" ? err.status : void 0
|
|
635
|
+
} }, { status: 500 });
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
if (parts[1] === "streams" && parts.length === 4 && parts[3] === "chunks") {
|
|
639
|
+
const namespace = env.WORKFLOW_STREAMS;
|
|
640
|
+
if (!namespace) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_STREAMS");
|
|
641
|
+
const name = decodeURIComponent(parts[2]);
|
|
642
|
+
const runId = url.searchParams.get("runId");
|
|
643
|
+
if (!runId) return errorResponse(400, "BadRequest", "runId is required");
|
|
644
|
+
const stub = namespace.get(namespace.idFromName(name));
|
|
645
|
+
try {
|
|
646
|
+
if (request.method === "POST") {
|
|
647
|
+
if (!request.headers.get("content-type")?.startsWith("application/vnd.world-celld.stream-batch.v1")) return errorResponse(415, "UnsupportedMediaType", STREAM_BATCH_CONTENT_TYPE);
|
|
648
|
+
if (Number(request.headers.get("content-length") ?? "0") > MAX_STREAM_WRITE_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
|
|
649
|
+
const body = await readBoundedBytes(request, MAX_STREAM_WRITE_BODY_BYTES);
|
|
650
|
+
if (body === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
|
|
651
|
+
const result = await stub.writeChunks(runId, decodeStreamWriteBatch(body));
|
|
652
|
+
return streamResponse(encodeStreamWriteResult(result));
|
|
653
|
+
}
|
|
654
|
+
if (request.method === "GET") {
|
|
655
|
+
const result = await stub.readChunks({
|
|
656
|
+
runId,
|
|
657
|
+
startIndex: parseBoundedInteger(url, "startIndex", 0, 2147483647),
|
|
658
|
+
maxChunks: parseBoundedInteger(url, "maxChunks", 0, 32),
|
|
659
|
+
maxBytes: parseBoundedInteger(url, "maxBytes", MAX_STREAM_CHUNK_BYTES, MAX_STREAM_READ_BYTES),
|
|
660
|
+
waitMs: parseBoundedInteger(url, "waitMs", 0, MAX_STREAM_LONG_POLL_MS)
|
|
661
|
+
}, request.signal);
|
|
662
|
+
return streamResponse(encodeStreamReadResult(result));
|
|
663
|
+
}
|
|
664
|
+
return errorResponse(405, "MethodNotAllowed", "expected GET or POST");
|
|
665
|
+
} catch (error) {
|
|
666
|
+
const err = error;
|
|
667
|
+
return Response.json({ error: {
|
|
668
|
+
name: err?.name ?? "Error",
|
|
669
|
+
message: err?.message ?? String(error),
|
|
670
|
+
status: typeof err?.status === "number" ? err.status : void 0
|
|
671
|
+
} }, { status: err?.name === "StreamProtocolError" ? 400 : 500 });
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
|
|
405
675
|
if (request.method !== "POST" || parts.length !== 5) return errorResponse(404, "NotFound", "expected POST /v1/rpc/{binding}/{name}/{method}");
|
|
406
676
|
const [, , bindingKey, encodedName, method] = parts;
|
|
407
677
|
const binding = BINDINGS[bindingKey];
|
|
@@ -457,162 +727,367 @@ var DurableObject = class {
|
|
|
457
727
|
}
|
|
458
728
|
};
|
|
459
729
|
//#endregion
|
|
460
|
-
//#region src/worker/durable-objects/
|
|
730
|
+
//#region src/worker/durable-objects/hook-index-shared.ts
|
|
461
731
|
function sameOwner(left, right) {
|
|
462
732
|
return left.runId === right.runId && left.hookId === right.hookId;
|
|
463
733
|
}
|
|
464
|
-
function
|
|
734
|
+
function ownerFromSerializedHook(raw) {
|
|
465
735
|
const hook = parse(raw);
|
|
466
736
|
return {
|
|
467
737
|
runId: hook.runId,
|
|
468
738
|
hookId: hook.hookId
|
|
469
739
|
};
|
|
470
740
|
}
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
741
|
+
function indexKey(prefix, value) {
|
|
742
|
+
return `${prefix}:${encodeURIComponent(value)}`;
|
|
743
|
+
}
|
|
744
|
+
async function runIsActive(env, runId) {
|
|
745
|
+
const namespace = env.WORKFLOW_DB;
|
|
746
|
+
if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_DB binding");
|
|
747
|
+
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "active";
|
|
748
|
+
}
|
|
749
|
+
//#endregion
|
|
750
|
+
//#region src/worker/durable-objects/HookIdDO.ts
|
|
751
|
+
const MAX_RELEASE_BATCH$1 = 64;
|
|
752
|
+
const CLAIM_DEADLINE_PREFIX$1 = "claim-deadline:";
|
|
753
|
+
function pad$3(ms) {
|
|
754
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
755
|
+
}
|
|
756
|
+
function recordKey$1(hookId) {
|
|
757
|
+
return indexKey("hookid", hookId);
|
|
758
|
+
}
|
|
759
|
+
function claimKey$1(hookId) {
|
|
760
|
+
return indexKey("claim", hookId);
|
|
761
|
+
}
|
|
762
|
+
function claimDeadlineKey$1(hookId, expiresAt) {
|
|
763
|
+
return `${CLAIM_DEADLINE_PREFIX$1}${pad$3(expiresAt)}:${encodeURIComponent(hookId)}`;
|
|
764
|
+
}
|
|
765
|
+
/** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
|
|
766
|
+
var HookIdDO = class extends DurableObject {
|
|
767
|
+
now() {
|
|
768
|
+
return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
|
|
487
769
|
}
|
|
488
|
-
async
|
|
489
|
-
await this.ctx.storage.
|
|
770
|
+
async get(hookId) {
|
|
771
|
+
const raw = await this.ctx.storage.get(recordKey$1(hookId));
|
|
772
|
+
if (!raw) return null;
|
|
773
|
+
const owner = ownerFromSerializedHook(raw);
|
|
774
|
+
return await runIsActive(this.env, owner.runId) ? raw : null;
|
|
490
775
|
}
|
|
491
|
-
|
|
492
|
-
async putOwned(runId, key, value) {
|
|
776
|
+
async reserve(hookId, owner, proposedClaimId) {
|
|
493
777
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
494
|
-
|
|
495
|
-
|
|
778
|
+
const record = recordKey$1(hookId);
|
|
779
|
+
const claim = claimKey$1(hookId);
|
|
780
|
+
const values = await txn.get([record, claim]);
|
|
781
|
+
const indexed = values.get(record);
|
|
782
|
+
if (indexed !== void 0) return {
|
|
783
|
+
claimed: false,
|
|
784
|
+
holder: ownerFromSerializedHook(indexed)
|
|
785
|
+
};
|
|
786
|
+
let existing = values.get(claim);
|
|
787
|
+
const now = this.now();
|
|
788
|
+
if (existing && existing.expiresAt <= now) {
|
|
789
|
+
await txn.delete([claim, existing.deadlineKey]);
|
|
790
|
+
existing = void 0;
|
|
791
|
+
}
|
|
792
|
+
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
793
|
+
claimed: false,
|
|
794
|
+
holder: existing.owner
|
|
795
|
+
};
|
|
796
|
+
if (existing === void 0 || existing.claimId !== proposedClaimId) {
|
|
797
|
+
if (existing) await txn.delete(existing.deadlineKey);
|
|
798
|
+
const expiresAt = now + HOOK_CLAIM_LEASE_MS;
|
|
799
|
+
const deadlineKey = claimDeadlineKey$1(hookId, expiresAt);
|
|
800
|
+
await txn.put(claim, {
|
|
801
|
+
owner,
|
|
802
|
+
claimId: proposedClaimId,
|
|
803
|
+
reservedAt: now,
|
|
804
|
+
expiresAt,
|
|
805
|
+
deadlineKey
|
|
806
|
+
});
|
|
807
|
+
await txn.put(deadlineKey, {
|
|
808
|
+
hookId,
|
|
809
|
+
claimId: proposedClaimId,
|
|
810
|
+
expiresAt
|
|
811
|
+
});
|
|
812
|
+
await this.armAtMost(txn, expiresAt);
|
|
813
|
+
return {
|
|
814
|
+
claimed: true,
|
|
815
|
+
claimId: proposedClaimId
|
|
816
|
+
};
|
|
817
|
+
}
|
|
818
|
+
return {
|
|
819
|
+
claimed: true,
|
|
820
|
+
claimId: existing.claimId
|
|
821
|
+
};
|
|
822
|
+
});
|
|
823
|
+
}
|
|
824
|
+
async publish(hookId, serializedHook, owner, claimId) {
|
|
825
|
+
if (!await runIsActive(this.env, owner.runId)) {
|
|
826
|
+
await this.releaseBatch([{
|
|
827
|
+
hookId,
|
|
828
|
+
owner
|
|
829
|
+
}]);
|
|
830
|
+
return { stored: false };
|
|
831
|
+
}
|
|
832
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
833
|
+
const record = recordKey$1(hookId);
|
|
834
|
+
const claim = claimKey$1(hookId);
|
|
835
|
+
const values = await txn.get([record, claim]);
|
|
836
|
+
const indexed = values.get(record);
|
|
837
|
+
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
|
|
838
|
+
const existingClaim = values.get(claim);
|
|
839
|
+
if (existingClaim && existingClaim.expiresAt <= this.now()) {
|
|
840
|
+
await txn.delete([claim, existingClaim.deadlineKey]);
|
|
841
|
+
return { stored: false };
|
|
842
|
+
}
|
|
843
|
+
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
|
|
844
|
+
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
|
|
845
|
+
await txn.put(record, serializedHook);
|
|
846
|
+
if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
|
|
496
847
|
return { stored: true };
|
|
497
848
|
});
|
|
498
849
|
}
|
|
499
|
-
async
|
|
500
|
-
await this.ctx.storage.
|
|
850
|
+
async releaseClaim(hookId, owner, claimId) {
|
|
851
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
852
|
+
const key = claimKey$1(hookId);
|
|
853
|
+
const claim = await txn.get(key);
|
|
854
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
|
|
855
|
+
});
|
|
501
856
|
}
|
|
502
|
-
async
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
857
|
+
async releaseBatch(hooks) {
|
|
858
|
+
if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
|
|
859
|
+
if (hooks.length === 0) return { deleted: 0 };
|
|
860
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
861
|
+
const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
|
|
862
|
+
const values = await txn.get(keys);
|
|
863
|
+
const deletes = [];
|
|
864
|
+
const deadlineDeletes = [];
|
|
865
|
+
for (const { hookId, owner } of hooks) {
|
|
866
|
+
const record = recordKey$1(hookId);
|
|
867
|
+
const claim = claimKey$1(hookId);
|
|
868
|
+
const indexed = values.get(record);
|
|
869
|
+
const existingClaim = values.get(claim);
|
|
870
|
+
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
871
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
|
|
872
|
+
deletes.push(claim);
|
|
873
|
+
deadlineDeletes.push(existingClaim.deadlineKey);
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
|
|
877
|
+
if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
|
|
878
|
+
return { deleted };
|
|
512
879
|
});
|
|
513
|
-
const names = Array.from(entries.keys());
|
|
514
|
-
const page = names.slice(0, limit);
|
|
515
|
-
const listComplete = names.length <= limit;
|
|
516
|
-
return {
|
|
517
|
-
keys: page.map((name) => ({ name })),
|
|
518
|
-
list_complete: listComplete,
|
|
519
|
-
cursor: listComplete ? void 0 : page.at(-1)
|
|
520
|
-
};
|
|
521
880
|
}
|
|
522
|
-
async
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
881
|
+
async alarm() {
|
|
882
|
+
try {
|
|
883
|
+
await this.compactClaims();
|
|
884
|
+
} catch {
|
|
885
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
async armAtMost(storage, at) {
|
|
889
|
+
const current = await storage.getAlarm();
|
|
890
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
891
|
+
}
|
|
892
|
+
async compactClaims() {
|
|
893
|
+
const now = this.now();
|
|
894
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
895
|
+
const due = await txn.list({
|
|
896
|
+
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
897
|
+
end: `${CLAIM_DEADLINE_PREFIX$1}${pad$3(now + 1)}`,
|
|
898
|
+
limit: 129
|
|
899
|
+
});
|
|
900
|
+
const page = Array.from(due).slice(0, 128);
|
|
901
|
+
const claims = await txn.get(page.map(([, item]) => claimKey$1(item.hookId)));
|
|
902
|
+
const deletes = page.flatMap(([deadline, item]) => {
|
|
903
|
+
const key = claimKey$1(item.hookId);
|
|
904
|
+
const claim = claims.get(key);
|
|
905
|
+
return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
|
|
906
|
+
});
|
|
907
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
908
|
+
if (due.size > 128) {
|
|
909
|
+
await txn.setAlarm(now + 1);
|
|
910
|
+
return;
|
|
911
|
+
}
|
|
912
|
+
const nextAt = (await txn.list({
|
|
913
|
+
prefix: CLAIM_DEADLINE_PREFIX$1,
|
|
914
|
+
limit: 1
|
|
915
|
+
})).values().next().value?.expiresAt;
|
|
916
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
917
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
};
|
|
921
|
+
//#endregion
|
|
922
|
+
//#region src/worker/durable-objects/HookTokenDO.ts
|
|
923
|
+
const MAX_RELEASE_BATCH = 64;
|
|
924
|
+
const CLAIM_DEADLINE_PREFIX = "claim-deadline:";
|
|
925
|
+
function pad$2(ms) {
|
|
926
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
927
|
+
}
|
|
928
|
+
function recordKey(token) {
|
|
929
|
+
return indexKey("hook", token);
|
|
930
|
+
}
|
|
931
|
+
function claimKey(token) {
|
|
932
|
+
return indexKey("claim", token);
|
|
933
|
+
}
|
|
934
|
+
function claimDeadlineKey(token, expiresAt) {
|
|
935
|
+
return `${CLAIM_DEADLINE_PREFIX}${pad$2(expiresAt)}:${encodeURIComponent(token)}`;
|
|
936
|
+
}
|
|
937
|
+
/** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
|
|
938
|
+
var HookTokenDO = class extends DurableObject {
|
|
939
|
+
now() {
|
|
940
|
+
return typeof this.env.clock === "function" ? this.env.clock() : Date.now();
|
|
941
|
+
}
|
|
942
|
+
async get(token) {
|
|
943
|
+
const raw = await this.ctx.storage.get(recordKey(token));
|
|
944
|
+
if (!raw) return null;
|
|
945
|
+
const owner = ownerFromSerializedHook(raw);
|
|
946
|
+
return await runIsActive(this.env, owner.runId) ? raw : null;
|
|
947
|
+
}
|
|
948
|
+
async reserve(token, owner, proposedClaimId) {
|
|
949
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
950
|
+
const record = recordKey(token);
|
|
951
|
+
const claim = claimKey(token);
|
|
952
|
+
const values = await txn.get([record, claim]);
|
|
953
|
+
const indexed = values.get(record);
|
|
954
|
+
if (indexed !== void 0) return {
|
|
526
955
|
claimed: false,
|
|
527
|
-
holder:
|
|
956
|
+
holder: ownerFromSerializedHook(indexed)
|
|
528
957
|
};
|
|
529
|
-
|
|
530
|
-
const
|
|
531
|
-
if (
|
|
958
|
+
let existing = values.get(claim);
|
|
959
|
+
const now = this.now();
|
|
960
|
+
if (existing && existing.expiresAt <= now) {
|
|
961
|
+
await txn.delete([claim, existing.deadlineKey]);
|
|
962
|
+
existing = void 0;
|
|
963
|
+
}
|
|
964
|
+
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
532
965
|
claimed: false,
|
|
533
|
-
holder:
|
|
966
|
+
holder: existing.owner
|
|
967
|
+
};
|
|
968
|
+
if (existing === void 0) {
|
|
969
|
+
const expiresAt = now + HOOK_CLAIM_LEASE_MS;
|
|
970
|
+
const deadlineKey = claimDeadlineKey(token, expiresAt);
|
|
971
|
+
await txn.put(claim, {
|
|
972
|
+
owner,
|
|
973
|
+
claimId: proposedClaimId,
|
|
974
|
+
reservedAt: now,
|
|
975
|
+
expiresAt,
|
|
976
|
+
deadlineKey
|
|
977
|
+
});
|
|
978
|
+
await txn.put(deadlineKey, {
|
|
979
|
+
token,
|
|
980
|
+
claimId: proposedClaimId,
|
|
981
|
+
expiresAt
|
|
982
|
+
});
|
|
983
|
+
await this.armAtMost(txn, expiresAt);
|
|
984
|
+
return {
|
|
985
|
+
claimed: true,
|
|
986
|
+
claimId: proposedClaimId
|
|
987
|
+
};
|
|
988
|
+
}
|
|
989
|
+
return {
|
|
990
|
+
claimed: true,
|
|
991
|
+
claimId: existing.claimId
|
|
534
992
|
};
|
|
535
|
-
await txn.put(claimKey, {
|
|
536
|
-
owner,
|
|
537
|
-
reservedAt: Date.now()
|
|
538
|
-
});
|
|
539
|
-
return { claimed: true };
|
|
540
993
|
});
|
|
541
994
|
}
|
|
542
|
-
async
|
|
543
|
-
await this.
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
995
|
+
async finalize(token, serializedHook, owner, claimId) {
|
|
996
|
+
if (!await runIsActive(this.env, owner.runId)) {
|
|
997
|
+
await this.releaseBatch([{
|
|
998
|
+
token,
|
|
999
|
+
owner
|
|
1000
|
+
}]);
|
|
1001
|
+
return { stored: false };
|
|
1002
|
+
}
|
|
1003
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1004
|
+
const record = recordKey(token);
|
|
1005
|
+
const claim = claimKey(token);
|
|
1006
|
+
const values = await txn.get([record, claim]);
|
|
1007
|
+
const indexed = values.get(record);
|
|
1008
|
+
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
|
|
1009
|
+
const existingClaim = values.get(claim);
|
|
1010
|
+
if (existingClaim && existingClaim.expiresAt <= this.now()) {
|
|
1011
|
+
await txn.delete([claim, existingClaim.deadlineKey]);
|
|
1012
|
+
return { stored: false };
|
|
548
1013
|
}
|
|
549
|
-
|
|
550
|
-
if (
|
|
551
|
-
|
|
552
|
-
if (
|
|
553
|
-
|
|
554
|
-
await txn.put(`hookid:${hookId}`, serializedHook);
|
|
555
|
-
if (claim !== void 0) await txn.delete(`hookclaim:${token}`);
|
|
1014
|
+
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
|
|
1015
|
+
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
|
|
1016
|
+
await txn.put(record, serializedHook);
|
|
1017
|
+
if (existingClaim !== void 0) await txn.delete([claim, existingClaim.deadlineKey]);
|
|
1018
|
+
return { stored: true };
|
|
556
1019
|
});
|
|
557
1020
|
}
|
|
558
|
-
async
|
|
1021
|
+
async releaseClaim(token, owner, claimId) {
|
|
559
1022
|
await this.ctx.storage.transaction(async (txn) => {
|
|
560
|
-
const key =
|
|
1023
|
+
const key = claimKey(token);
|
|
561
1024
|
const claim = await txn.get(key);
|
|
562
|
-
if (claim
|
|
1025
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete([key, claim.deadlineKey]);
|
|
563
1026
|
});
|
|
564
1027
|
}
|
|
565
|
-
async
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
const idKey = `hookid:${hookId}`;
|
|
569
|
-
const claimKey = `hookclaim:${token}`;
|
|
570
|
-
const [byToken, byId, claim] = await Promise.all([
|
|
571
|
-
txn.get(tokenKey),
|
|
572
|
-
txn.get(idKey),
|
|
573
|
-
txn.get(claimKey)
|
|
574
|
-
]);
|
|
575
|
-
if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) await txn.delete(tokenKey);
|
|
576
|
-
if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) await txn.delete(idKey);
|
|
577
|
-
if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(claimKey);
|
|
578
|
-
});
|
|
579
|
-
}
|
|
580
|
-
/**
|
|
581
|
-
* Fence a run and remove all of its known derived indexes atomically.
|
|
582
|
-
* Once the fence exists, putOwned() and hook finalization cannot resurrect
|
|
583
|
-
* entries from a delayed request.
|
|
584
|
-
*/
|
|
585
|
-
async expireRun(request) {
|
|
1028
|
+
async releaseBatch(hooks) {
|
|
1029
|
+
if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
|
|
1030
|
+
if (hooks.length === 0) return { deleted: 0 };
|
|
586
1031
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
const
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
txn.get(idKey),
|
|
601
|
-
txn.get(claimKey)
|
|
602
|
-
]);
|
|
603
|
-
if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) {
|
|
604
|
-
if (await txn.delete(tokenKey)) deleted++;
|
|
605
|
-
}
|
|
606
|
-
if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) {
|
|
607
|
-
if (await txn.delete(idKey)) deleted++;
|
|
608
|
-
}
|
|
609
|
-
if (claim !== void 0 && sameOwner(claim.owner, owner)) {
|
|
610
|
-
if (await txn.delete(claimKey)) deleted++;
|
|
1032
|
+
const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
|
|
1033
|
+
const values = await txn.get(keys);
|
|
1034
|
+
const deletes = [];
|
|
1035
|
+
const deadlineDeletes = [];
|
|
1036
|
+
for (const { token, owner } of hooks) {
|
|
1037
|
+
const record = recordKey(token);
|
|
1038
|
+
const claim = claimKey(token);
|
|
1039
|
+
const indexed = values.get(record);
|
|
1040
|
+
const existingClaim = values.get(claim);
|
|
1041
|
+
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
1042
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) {
|
|
1043
|
+
deletes.push(claim);
|
|
1044
|
+
deadlineDeletes.push(existingClaim.deadlineKey);
|
|
611
1045
|
}
|
|
612
1046
|
}
|
|
1047
|
+
const deleted = deletes.length === 0 ? 0 : await txn.delete(deletes);
|
|
1048
|
+
if (deadlineDeletes.length > 0) await txn.delete(deadlineDeletes);
|
|
613
1049
|
return { deleted };
|
|
614
1050
|
});
|
|
615
1051
|
}
|
|
1052
|
+
async alarm() {
|
|
1053
|
+
try {
|
|
1054
|
+
await this.compactClaims();
|
|
1055
|
+
} catch {
|
|
1056
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
async armAtMost(storage, at) {
|
|
1060
|
+
const current = await storage.getAlarm();
|
|
1061
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
1062
|
+
}
|
|
1063
|
+
async compactClaims() {
|
|
1064
|
+
const now = this.now();
|
|
1065
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1066
|
+
const due = await txn.list({
|
|
1067
|
+
prefix: CLAIM_DEADLINE_PREFIX,
|
|
1068
|
+
end: `${CLAIM_DEADLINE_PREFIX}${pad$2(now + 1)}`,
|
|
1069
|
+
limit: 129
|
|
1070
|
+
});
|
|
1071
|
+
const page = Array.from(due).slice(0, 128);
|
|
1072
|
+
const claims = await txn.get(page.map(([, item]) => claimKey(item.token)));
|
|
1073
|
+
const deletes = page.flatMap(([deadline, item]) => {
|
|
1074
|
+
const key = claimKey(item.token);
|
|
1075
|
+
const claim = claims.get(key);
|
|
1076
|
+
return claim?.claimId === item.claimId && claim.expiresAt <= now ? [deadline, key] : [deadline];
|
|
1077
|
+
});
|
|
1078
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
1079
|
+
if (due.size > 128) {
|
|
1080
|
+
await txn.setAlarm(now + 1);
|
|
1081
|
+
return;
|
|
1082
|
+
}
|
|
1083
|
+
const nextAt = (await txn.list({
|
|
1084
|
+
prefix: CLAIM_DEADLINE_PREFIX,
|
|
1085
|
+
limit: 1
|
|
1086
|
+
})).values().next().value?.expiresAt;
|
|
1087
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
1088
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
1089
|
+
});
|
|
1090
|
+
}
|
|
616
1091
|
};
|
|
617
1092
|
//#endregion
|
|
618
1093
|
//#region src/worker/durable-objects/QueueDO.ts
|
|
@@ -625,15 +1100,26 @@ const PERMANENT_STATUSES = /* @__PURE__ */ new Set([
|
|
|
625
1100
|
const DEFAULT_MAX_ATTEMPTS = 5;
|
|
626
1101
|
const DEFAULT_MAX_INFLIGHT = 5;
|
|
627
1102
|
const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
|
|
628
|
-
/** Grace period past the delivery timeout before a claim counts as lost. */
|
|
629
|
-
const INFLIGHT_GRACE_MS = 3e4;
|
|
630
1103
|
/** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
|
|
631
1104
|
const MIN_ALARM_DELAY_MS = 1;
|
|
632
|
-
|
|
1105
|
+
/** Bound expiry recovery so one alarm cannot monopolize the cell. */
|
|
1106
|
+
const EXPIRED_INFLIGHT_BATCH = 128;
|
|
1107
|
+
const STORAGE_BATCH_SIZE$2 = 128;
|
|
1108
|
+
const EXPIRE_RUN_REFERENCE_BATCH = 64;
|
|
1109
|
+
const PURGE_DEAD_LETTER_BATCH = 128;
|
|
1110
|
+
const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
|
|
1111
|
+
const EXPIRED_RUN_GC_PREFIX = "expired-run-gc:";
|
|
1112
|
+
function pad$1(ms) {
|
|
633
1113
|
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
634
1114
|
}
|
|
635
1115
|
function dueKey(atMs, messageId) {
|
|
636
|
-
return `due:${pad(atMs)}:${messageId}`;
|
|
1116
|
+
return `due:${pad$1(atMs)}:${messageId}`;
|
|
1117
|
+
}
|
|
1118
|
+
function inflightKey(deadlineMs, messageId) {
|
|
1119
|
+
return `${INFLIGHT_DEADLINE_PREFIX}${pad$1(deadlineMs)}:${messageId}`;
|
|
1120
|
+
}
|
|
1121
|
+
function deadlineFromInflightKey(key) {
|
|
1122
|
+
return Number.parseInt(key.slice(18, 31), 10);
|
|
637
1123
|
}
|
|
638
1124
|
function runReferenceKey(runId, messageId) {
|
|
639
1125
|
return `run:${runId}:${messageId}`;
|
|
@@ -641,9 +1127,40 @@ function runReferenceKey(runId, messageId) {
|
|
|
641
1127
|
function expiredRunKey(runId) {
|
|
642
1128
|
return `expired-run:${runId}`;
|
|
643
1129
|
}
|
|
1130
|
+
function expiredRunGcKey(runId, compactAt) {
|
|
1131
|
+
return `${EXPIRED_RUN_GC_PREFIX}${pad$1(compactAt)}:${encodeURIComponent(runId)}`;
|
|
1132
|
+
}
|
|
1133
|
+
function boundedLimit$1(value, fallback, maximum) {
|
|
1134
|
+
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
1135
|
+
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
1136
|
+
}
|
|
1137
|
+
async function getMany$1(storage, keys) {
|
|
1138
|
+
const result = /* @__PURE__ */ new Map();
|
|
1139
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$2) {
|
|
1140
|
+
const page = await storage.get(keys.slice(offset, offset + STORAGE_BATCH_SIZE$2));
|
|
1141
|
+
for (const [key, value] of page) result.set(key, value);
|
|
1142
|
+
}
|
|
1143
|
+
return result;
|
|
1144
|
+
}
|
|
1145
|
+
async function putMany(storage, entries) {
|
|
1146
|
+
const all = Array.from(entries);
|
|
1147
|
+
for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE$2) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE$2)));
|
|
1148
|
+
}
|
|
1149
|
+
async function deleteMany$1(storage, keys) {
|
|
1150
|
+
let deleted = 0;
|
|
1151
|
+
const unique = Array.from(new Set(keys));
|
|
1152
|
+
for (let offset = 0; offset < unique.length; offset += STORAGE_BATCH_SIZE$2) deleted += await storage.delete(unique.slice(offset, offset + STORAGE_BATCH_SIZE$2));
|
|
1153
|
+
return deleted;
|
|
1154
|
+
}
|
|
644
1155
|
function backoffSeconds(attempt) {
|
|
645
1156
|
return Math.min(60, 2 ** attempt);
|
|
646
1157
|
}
|
|
1158
|
+
async function cancelUnusedResponseBody(response) {
|
|
1159
|
+
try {
|
|
1160
|
+
await response.body?.cancel();
|
|
1161
|
+
} catch {}
|
|
1162
|
+
}
|
|
1163
|
+
const MAX_QUEUE_INFLIGHT = 128;
|
|
647
1164
|
var QueueDO = class extends DurableObject {
|
|
648
1165
|
#now() {
|
|
649
1166
|
const clock = this.env?.clock;
|
|
@@ -658,15 +1175,34 @@ var QueueDO = class extends DurableObject {
|
|
|
658
1175
|
const parsed = raw ? Number.parseInt(raw, 10) : NaN;
|
|
659
1176
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
660
1177
|
}
|
|
1178
|
+
#maxInflight() {
|
|
1179
|
+
const configured = this.#intVar("QUEUE_MAX_INFLIGHT", DEFAULT_MAX_INFLIGHT);
|
|
1180
|
+
if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
|
|
1181
|
+
return configured;
|
|
1182
|
+
}
|
|
1183
|
+
#deliveryTimeoutMs() {
|
|
1184
|
+
const configured = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
1185
|
+
if (configured > 3e5) throw new Error(`QUEUE_DELIVERY_TIMEOUT_MS must be at most ${MAX_QUEUE_DELIVERY_TIMEOUT_MS}`);
|
|
1186
|
+
return configured;
|
|
1187
|
+
}
|
|
1188
|
+
async #authoritativeRunExpired(runId) {
|
|
1189
|
+
const namespace = this.env?.WORKFLOW_DB;
|
|
1190
|
+
if (!namespace) throw new Error("world-celld queue missing WORKFLOW_DB binding");
|
|
1191
|
+
return await namespace.get(namespace.idFromName(runId)).getLifecycleStatus() === "expired";
|
|
1192
|
+
}
|
|
1193
|
+
async #withRunMutation(runId, onExpired, onAllowed) {
|
|
1194
|
+
const expired = await this.#authoritativeRunExpired(runId);
|
|
1195
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1196
|
+
if (await txn.get(expiredRunKey(runId)) || expired) return await onExpired(txn);
|
|
1197
|
+
return await onAllowed(txn);
|
|
1198
|
+
});
|
|
1199
|
+
}
|
|
661
1200
|
async enqueue(request) {
|
|
1201
|
+
this.#maxInflight();
|
|
1202
|
+
this.#deliveryTimeoutMs();
|
|
662
1203
|
const storage = this.ctx.storage;
|
|
663
1204
|
const now = this.#now();
|
|
664
|
-
|
|
665
|
-
if (request.runId && await txn.get(expiredRunKey(request.runId)) !== void 0) return {
|
|
666
|
-
ok: false,
|
|
667
|
-
code: "RUN_EXPIRED",
|
|
668
|
-
message: `Workflow run "${request.runId}" has expired`
|
|
669
|
-
};
|
|
1205
|
+
const persist = async (txn) => {
|
|
670
1206
|
const pinned = await txn.get("cfg");
|
|
671
1207
|
if (!pinned) await txn.put("cfg", {
|
|
672
1208
|
queueShards: request.config.queueShards,
|
|
@@ -718,7 +1254,12 @@ var QueueDO = class extends DurableObject {
|
|
|
718
1254
|
messageId: row.messageId,
|
|
719
1255
|
deduped: false
|
|
720
1256
|
};
|
|
721
|
-
}
|
|
1257
|
+
};
|
|
1258
|
+
return request.runId ? await this.#withRunMutation(request.runId, async () => ({
|
|
1259
|
+
ok: false,
|
|
1260
|
+
code: "RUN_EXPIRED",
|
|
1261
|
+
message: `Workflow run "${request.runId}" has expired`
|
|
1262
|
+
}), persist) : await storage.transaction(persist);
|
|
722
1263
|
}
|
|
723
1264
|
async alarm() {
|
|
724
1265
|
let claimed = [];
|
|
@@ -732,8 +1273,11 @@ var QueueDO = class extends DurableObject {
|
|
|
732
1273
|
};
|
|
733
1274
|
if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
|
|
734
1275
|
else await attempt();
|
|
735
|
-
if (failure !== null)
|
|
736
|
-
|
|
1276
|
+
if (failure !== null) {
|
|
1277
|
+
await this.ctx.storage.setAlarm(this.#now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1278
|
+
throw failure;
|
|
1279
|
+
}
|
|
1280
|
+
for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
|
|
737
1281
|
}
|
|
738
1282
|
/**
|
|
739
1283
|
* Storage-only: recover lost inflight claims, claim due messages up to the
|
|
@@ -742,49 +1286,83 @@ var QueueDO = class extends DurableObject {
|
|
|
742
1286
|
async #claimPhase() {
|
|
743
1287
|
const storage = this.ctx.storage;
|
|
744
1288
|
const now = this.#now();
|
|
745
|
-
const maxInflight = this.#
|
|
1289
|
+
const maxInflight = this.#maxInflight();
|
|
746
1290
|
return await storage.transaction(async (txn) => {
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
1291
|
+
if (await this.#compactExpiredRunFences(txn, now)) {
|
|
1292
|
+
await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
|
|
1293
|
+
return [];
|
|
1294
|
+
}
|
|
1295
|
+
const expiredPage = await txn.list({
|
|
1296
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1297
|
+
end: `${INFLIGHT_DEADLINE_PREFIX}${pad$1(now + 1)}`,
|
|
1298
|
+
limit: 129
|
|
1299
|
+
});
|
|
1300
|
+
const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
|
|
1301
|
+
const expiredRows = await getMany$1(txn, expired.map(([, messageId]) => `msg:${messageId}`));
|
|
1302
|
+
await deleteMany$1(txn, expired.map(([key]) => key));
|
|
1303
|
+
const recoveryWrites = [];
|
|
1304
|
+
for (const [, messageId] of expired) {
|
|
1305
|
+
const row = expiredRows.get(`msg:${messageId}`);
|
|
753
1306
|
if (row) {
|
|
754
1307
|
const scheduleKey = dueKey(now, messageId);
|
|
755
|
-
|
|
756
|
-
if (row.runId)
|
|
1308
|
+
recoveryWrites.push([scheduleKey, messageId]);
|
|
1309
|
+
if (row.runId) recoveryWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
757
1310
|
messageId: row.messageId,
|
|
758
1311
|
dueKey: scheduleKey,
|
|
759
1312
|
idempotencyKey: row.idempotencyKey
|
|
760
|
-
});
|
|
1313
|
+
}]);
|
|
761
1314
|
}
|
|
762
|
-
}
|
|
1315
|
+
}
|
|
1316
|
+
await putMany(txn, recoveryWrites);
|
|
1317
|
+
if (expiredPage.size > EXPIRED_INFLIGHT_BATCH) {
|
|
1318
|
+
await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
|
|
1319
|
+
return [];
|
|
1320
|
+
}
|
|
1321
|
+
const inflightCount = (await txn.list({
|
|
1322
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1323
|
+
limit: maxInflight
|
|
1324
|
+
})).size;
|
|
763
1325
|
const rows = [];
|
|
764
1326
|
if (inflightCount < maxInflight) {
|
|
765
1327
|
const due = await txn.list({
|
|
766
1328
|
prefix: "due:",
|
|
767
|
-
end: `due:${pad(now + 1)}`,
|
|
1329
|
+
end: `due:${pad$1(now + 1)}`,
|
|
768
1330
|
limit: maxInflight - inflightCount
|
|
769
1331
|
});
|
|
770
|
-
const timeoutMs = this.#
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
1332
|
+
const timeoutMs = this.#deliveryTimeoutMs();
|
|
1333
|
+
const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
|
|
1334
|
+
await deleteMany$1(txn, Array.from(due.keys()));
|
|
1335
|
+
const claimWrites = [];
|
|
1336
|
+
for (const [, messageId] of due) {
|
|
1337
|
+
const row = dueRows.get(`msg:${messageId}`);
|
|
774
1338
|
if (!row) continue;
|
|
775
|
-
|
|
776
|
-
|
|
1339
|
+
const claimKey = inflightKey(now + timeoutMs + QUEUE_INFLIGHT_GRACE_MS, messageId);
|
|
1340
|
+
claimWrites.push([claimKey, messageId]);
|
|
1341
|
+
if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
1342
|
+
messageId: row.messageId,
|
|
1343
|
+
inflightKey: claimKey,
|
|
1344
|
+
idempotencyKey: row.idempotencyKey
|
|
1345
|
+
}]);
|
|
1346
|
+
rows.push({
|
|
1347
|
+
row,
|
|
1348
|
+
inflightKey: claimKey
|
|
1349
|
+
});
|
|
777
1350
|
}
|
|
1351
|
+
await putMany(txn, claimWrites);
|
|
778
1352
|
}
|
|
779
1353
|
await this.#scheduleNextAlarm(txn, now);
|
|
780
1354
|
return rows;
|
|
781
1355
|
});
|
|
782
1356
|
}
|
|
783
|
-
async #deliver(row) {
|
|
1357
|
+
async #deliver({ row, inflightKey: claimKey }) {
|
|
784
1358
|
const storage = this.ctx.storage;
|
|
785
|
-
const timeoutMs = this.#
|
|
1359
|
+
const timeoutMs = this.#deliveryTimeoutMs();
|
|
786
1360
|
const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
|
|
787
1361
|
const url = `${row.targetBaseUrl.replace(/\/$/, "")}/.well-known/workflow/v1/${row.pathname}`;
|
|
1362
|
+
if (row.runId && await this.#authoritativeRunExpired(row.runId)) {
|
|
1363
|
+
await storage.transaction((txn) => this.#dropExpiredMessage(txn, row, claimKey, this.#now()));
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
788
1366
|
let response = null;
|
|
789
1367
|
let transportError = null;
|
|
790
1368
|
try {
|
|
@@ -804,47 +1382,44 @@ var QueueDO = class extends DurableObject {
|
|
|
804
1382
|
transportError = error;
|
|
805
1383
|
}
|
|
806
1384
|
const now = this.#now();
|
|
807
|
-
if (response?.ok)
|
|
808
|
-
|
|
1385
|
+
if (response?.ok) {
|
|
1386
|
+
await cancelUnusedResponseBody(response);
|
|
1387
|
+
await this.#ack(row, claimKey);
|
|
1388
|
+
} else if (response && response.status === 503) {
|
|
809
1389
|
let timeoutSeconds;
|
|
810
1390
|
try {
|
|
811
1391
|
timeoutSeconds = (await response.json())?.timeoutSeconds;
|
|
812
1392
|
} catch {
|
|
813
1393
|
timeoutSeconds = void 0;
|
|
814
1394
|
}
|
|
815
|
-
if (typeof timeoutSeconds === "number")
|
|
816
|
-
|
|
817
|
-
await txn.delete(
|
|
818
|
-
|
|
819
|
-
await txn.
|
|
820
|
-
if (row.
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
1395
|
+
if (typeof timeoutSeconds === "number") {
|
|
1396
|
+
const reschedule = async (txn) => {
|
|
1397
|
+
if (!await txn.delete(claimKey)) return;
|
|
1398
|
+
const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
|
|
1399
|
+
await txn.put(scheduleKey, row.messageId);
|
|
1400
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1401
|
+
messageId: row.messageId,
|
|
1402
|
+
dueKey: scheduleKey,
|
|
1403
|
+
idempotencyKey: row.idempotencyKey
|
|
1404
|
+
});
|
|
824
1405
|
await this.#scheduleNextAlarm(txn, now);
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
await
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
idempotencyKey: row.idempotencyKey
|
|
834
|
-
});
|
|
835
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
836
|
-
});
|
|
837
|
-
else await this.#retry(row, "HTTP 503");
|
|
838
|
-
} else if (response && PERMANENT_STATUSES.has(response.status)) await this.#ack(row);
|
|
839
|
-
else {
|
|
1406
|
+
};
|
|
1407
|
+
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), reschedule);
|
|
1408
|
+
else await storage.transaction(reschedule);
|
|
1409
|
+
} else await this.#retry(row, claimKey, "HTTP 503");
|
|
1410
|
+
} else if (response && PERMANENT_STATUSES.has(response.status)) {
|
|
1411
|
+
await cancelUnusedResponseBody(response);
|
|
1412
|
+
await this.#ack(row, claimKey);
|
|
1413
|
+
} else {
|
|
840
1414
|
const reason = response ? `HTTP ${response.status}` : `transport error: ${String(transportError)}`;
|
|
841
|
-
await
|
|
1415
|
+
if (response) await cancelUnusedResponseBody(response);
|
|
1416
|
+
await this.#retry(row, claimKey, reason);
|
|
842
1417
|
}
|
|
843
1418
|
}
|
|
844
|
-
async #ack(row) {
|
|
1419
|
+
async #ack(row, claimKey) {
|
|
845
1420
|
await this.ctx.storage.transaction(async (txn) => {
|
|
1421
|
+
if (!await txn.delete(claimKey)) return;
|
|
846
1422
|
await txn.delete(`msg:${row.messageId}`);
|
|
847
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
848
1423
|
if (row.idempotencyKey) {
|
|
849
1424
|
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
850
1425
|
}
|
|
@@ -852,24 +1427,23 @@ var QueueDO = class extends DurableObject {
|
|
|
852
1427
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
853
1428
|
});
|
|
854
1429
|
}
|
|
855
|
-
async #
|
|
1430
|
+
async #dropExpiredMessage(txn, row, claimKey, now) {
|
|
1431
|
+
if (!await txn.delete(claimKey)) return;
|
|
1432
|
+
await txn.delete(`msg:${row.messageId}`);
|
|
1433
|
+
if (row.runId) await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
1434
|
+
if (row.idempotencyKey) {
|
|
1435
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1436
|
+
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
1437
|
+
}
|
|
1438
|
+
await this.#scheduleNextAlarm(txn, now);
|
|
1439
|
+
}
|
|
1440
|
+
async #retry(row, claimKey, reason) {
|
|
856
1441
|
const storage = this.ctx.storage;
|
|
857
1442
|
const now = this.#now();
|
|
858
1443
|
const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
859
1444
|
const attempt = row.attempt + 1;
|
|
860
|
-
|
|
861
|
-
if (
|
|
862
|
-
await txn.delete(`msg:${row.messageId}`);
|
|
863
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
864
|
-
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
865
|
-
if (row.idempotencyKey) {
|
|
866
|
-
const dedupKey = `key:${row.idempotencyKey}`;
|
|
867
|
-
if (await txn.get(dedupKey) === row.messageId) await txn.delete(dedupKey);
|
|
868
|
-
}
|
|
869
|
-
await this.#scheduleNextAlarm(txn, now);
|
|
870
|
-
return;
|
|
871
|
-
}
|
|
872
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
1445
|
+
const persistRetry = async (txn) => {
|
|
1446
|
+
if (!await txn.delete(claimKey)) return;
|
|
873
1447
|
if (attempt >= maxAttempts) {
|
|
874
1448
|
const dead = {
|
|
875
1449
|
...row,
|
|
@@ -877,7 +1451,7 @@ var QueueDO = class extends DurableObject {
|
|
|
877
1451
|
lastError: reason,
|
|
878
1452
|
failedAt: now
|
|
879
1453
|
};
|
|
880
|
-
const deadKey = `dlq:${pad(now)}:${row.messageId}`;
|
|
1454
|
+
const deadKey = `dlq:${pad$1(now)}:${row.messageId}`;
|
|
881
1455
|
await txn.put(deadKey, dead);
|
|
882
1456
|
await txn.delete(`msg:${row.messageId}`);
|
|
883
1457
|
if (row.idempotencyKey) {
|
|
@@ -905,7 +1479,32 @@ var QueueDO = class extends DurableObject {
|
|
|
905
1479
|
idempotencyKey: row.idempotencyKey
|
|
906
1480
|
});
|
|
907
1481
|
await this.#scheduleNextAlarm(txn, now);
|
|
1482
|
+
};
|
|
1483
|
+
if (row.runId) await this.#withRunMutation(row.runId, (txn) => this.#dropExpiredMessage(txn, row, claimKey, now), persistRetry);
|
|
1484
|
+
else await storage.transaction(persistRetry);
|
|
1485
|
+
}
|
|
1486
|
+
/** Return true when another due compaction page must run immediately. */
|
|
1487
|
+
async #compactExpiredRunFences(txn, now) {
|
|
1488
|
+
const due = await txn.list({
|
|
1489
|
+
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
1490
|
+
end: `${EXPIRED_RUN_GC_PREFIX}${pad$1(now + 1)}`,
|
|
1491
|
+
limit: 129
|
|
908
1492
|
});
|
|
1493
|
+
const page = Array.from(due).slice(0, 128);
|
|
1494
|
+
if (page.length === 0) return false;
|
|
1495
|
+
const markerKeys = page.map(([, item]) => expiredRunKey(item.runId));
|
|
1496
|
+
const markers = await getMany$1(txn, markerKeys);
|
|
1497
|
+
const deletes = [];
|
|
1498
|
+
for (let index = 0; index < page.length; index++) {
|
|
1499
|
+
const [gcKey, item] = page[index];
|
|
1500
|
+
const markerKey = markerKeys[index];
|
|
1501
|
+
const marker = markers.get(markerKey);
|
|
1502
|
+
deletes.push(gcKey);
|
|
1503
|
+
if (marker?.compactAt !== item.compactAt) continue;
|
|
1504
|
+
deletes.push(markerKey);
|
|
1505
|
+
}
|
|
1506
|
+
await deleteMany$1(txn, deletes);
|
|
1507
|
+
return due.size > 128;
|
|
909
1508
|
}
|
|
910
1509
|
/** Arm the alarm no later than `atMs` (durable backoff never throws). */
|
|
911
1510
|
async #armAlarmAtMost(storage, atMs, now) {
|
|
@@ -929,8 +1528,19 @@ var QueueDO = class extends DurableObject {
|
|
|
929
1528
|
limit: 1
|
|
930
1529
|
});
|
|
931
1530
|
for (const key of due.keys()) next = Number.parseInt(key.slice(4, 17), 10);
|
|
932
|
-
const inflight = await storage.list({
|
|
933
|
-
|
|
1531
|
+
const inflight = await storage.list({
|
|
1532
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1533
|
+
limit: 1
|
|
1534
|
+
});
|
|
1535
|
+
for (const key of inflight.keys()) {
|
|
1536
|
+
const deadline = deadlineFromInflightKey(key);
|
|
1537
|
+
if (next === null || deadline < next) next = deadline;
|
|
1538
|
+
}
|
|
1539
|
+
const expiryGc = await storage.list({
|
|
1540
|
+
prefix: EXPIRED_RUN_GC_PREFIX,
|
|
1541
|
+
limit: 1
|
|
1542
|
+
});
|
|
1543
|
+
for (const item of expiryGc.values()) if (next === null || item.compactAt < next) next = item.compactAt;
|
|
934
1544
|
if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
|
|
935
1545
|
else await storage.deleteAlarm();
|
|
936
1546
|
}
|
|
@@ -938,7 +1548,7 @@ var QueueDO = class extends DurableObject {
|
|
|
938
1548
|
const storage = this.ctx.storage;
|
|
939
1549
|
const [due, inflight, dlq, alarmAt] = await Promise.all([
|
|
940
1550
|
storage.list({ prefix: "due:" }),
|
|
941
|
-
storage.list({ prefix:
|
|
1551
|
+
storage.list({ prefix: INFLIGHT_DEADLINE_PREFIX }),
|
|
942
1552
|
storage.list({ prefix: "dlq:" }),
|
|
943
1553
|
storage.getAlarm()
|
|
944
1554
|
]);
|
|
@@ -969,47 +1579,55 @@ var QueueDO = class extends DurableObject {
|
|
|
969
1579
|
async redriveDeadLetter(messageId) {
|
|
970
1580
|
const storage = this.ctx.storage;
|
|
971
1581
|
const now = this.#now();
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
await txn.put(
|
|
990
|
-
const scheduleKey = dueKey(now, row.messageId);
|
|
991
|
-
await txn.put(scheduleKey, row.messageId);
|
|
992
|
-
await txn.delete(key);
|
|
993
|
-
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
994
|
-
messageId: row.messageId,
|
|
995
|
-
dueKey: scheduleKey,
|
|
996
|
-
idempotencyKey: row.idempotencyKey
|
|
997
|
-
});
|
|
998
|
-
await this.#armAlarmAtMost(txn, now, now);
|
|
999
|
-
return { ok: true };
|
|
1582
|
+
const found = Array.from(await storage.list({ prefix: "dlq:" })).find(([, dead]) => dead.messageId === messageId);
|
|
1583
|
+
if (!found) return { ok: false };
|
|
1584
|
+
const [deadKey, candidate] = found;
|
|
1585
|
+
const redrive = async (txn) => {
|
|
1586
|
+
const dead = await txn.get(deadKey);
|
|
1587
|
+
if (!dead || dead.messageId !== messageId) return { ok: false };
|
|
1588
|
+
const { failedAt: _failedAt, ...rest } = dead;
|
|
1589
|
+
const row = {
|
|
1590
|
+
...rest,
|
|
1591
|
+
attempt: 0,
|
|
1592
|
+
lastError: void 0
|
|
1593
|
+
};
|
|
1594
|
+
if (row.idempotencyKey) {
|
|
1595
|
+
const dedupKey = `key:${row.idempotencyKey}`;
|
|
1596
|
+
const holder = await txn.get(dedupKey);
|
|
1597
|
+
if (holder && holder !== row.messageId && await txn.get(`msg:${holder}`)) return { ok: false };
|
|
1598
|
+
if (holder && holder !== row.messageId) await txn.delete(dedupKey);
|
|
1599
|
+
await txn.put(dedupKey, row.messageId);
|
|
1000
1600
|
}
|
|
1601
|
+
await txn.put(`msg:${row.messageId}`, row);
|
|
1602
|
+
const scheduleKey = dueKey(now, row.messageId);
|
|
1603
|
+
await txn.put(scheduleKey, row.messageId);
|
|
1604
|
+
await txn.delete(deadKey);
|
|
1605
|
+
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
1606
|
+
messageId: row.messageId,
|
|
1607
|
+
dueKey: scheduleKey,
|
|
1608
|
+
idempotencyKey: row.idempotencyKey
|
|
1609
|
+
});
|
|
1610
|
+
await this.#armAlarmAtMost(txn, now, now);
|
|
1611
|
+
return { ok: true };
|
|
1612
|
+
};
|
|
1613
|
+
const rejectExpired = async (txn) => {
|
|
1614
|
+
if ((await txn.get(deadKey))?.messageId === messageId) await txn.delete([deadKey, runReferenceKey(candidate.runId, messageId)]);
|
|
1001
1615
|
return { ok: false };
|
|
1002
|
-
}
|
|
1616
|
+
};
|
|
1617
|
+
return candidate.runId ? await this.#withRunMutation(candidate.runId, rejectExpired, redrive) : await storage.transaction(redrive);
|
|
1003
1618
|
}
|
|
1004
1619
|
async purgeDeadLetters() {
|
|
1005
1620
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1006
|
-
const
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
return {
|
|
1621
|
+
const entries = await txn.list({
|
|
1622
|
+
prefix: "dlq:",
|
|
1623
|
+
limit: 129
|
|
1624
|
+
});
|
|
1625
|
+
const page = Array.from(entries).slice(0, PURGE_DEAD_LETTER_BATCH);
|
|
1626
|
+
await deleteMany$1(txn, page.flatMap(([key, dead]) => [key, ...dead.runId ? [runReferenceKey(dead.runId, dead.messageId)] : []]));
|
|
1627
|
+
return {
|
|
1628
|
+
purged: page.length,
|
|
1629
|
+
hasMore: entries.size > PURGE_DEAD_LETTER_BATCH
|
|
1630
|
+
};
|
|
1013
1631
|
});
|
|
1014
1632
|
}
|
|
1015
1633
|
/**
|
|
@@ -1021,140 +1639,450 @@ var QueueDO = class extends DurableObject {
|
|
|
1021
1639
|
return { alarmAt: await this.ctx.storage.getAlarm() };
|
|
1022
1640
|
}
|
|
1023
1641
|
/** Fence a run and purge every live, inflight, and dead-letter message it owns. */
|
|
1024
|
-
async expireRun(runId, expiredAt) {
|
|
1025
|
-
|
|
1642
|
+
async expireRun(runId, expiredAt, options) {
|
|
1643
|
+
const storage = this.ctx.storage;
|
|
1644
|
+
const limit = boundedLimit$1(options?.limit, EXPIRE_RUN_REFERENCE_BATCH, EXPIRE_RUN_REFERENCE_BATCH);
|
|
1645
|
+
return await storage.transaction(async (txn) => {
|
|
1026
1646
|
const existing = await txn.get(expiredRunKey(runId));
|
|
1027
|
-
const references = await txn.list({
|
|
1647
|
+
const references = await txn.list({
|
|
1648
|
+
prefix: `run:${runId}:`,
|
|
1649
|
+
limit: limit + 1
|
|
1650
|
+
});
|
|
1651
|
+
const page = Array.from(references).slice(0, limit);
|
|
1652
|
+
const dedupHolders = await getMany$1(txn, page.flatMap(([, reference]) => reference.idempotencyKey ? [`key:${reference.idempotencyKey}`] : []));
|
|
1653
|
+
const keys = [];
|
|
1028
1654
|
let count = existing?.deleted ?? 0;
|
|
1029
|
-
for (const [key, reference] of
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
if (reference.dueKey)
|
|
1033
|
-
if (reference.dlqKey)
|
|
1655
|
+
for (const [key, reference] of page) {
|
|
1656
|
+
keys.push(`msg:${reference.messageId}`, key);
|
|
1657
|
+
if (reference.inflightKey) keys.push(reference.inflightKey);
|
|
1658
|
+
if (reference.dueKey) keys.push(reference.dueKey);
|
|
1659
|
+
if (reference.dlqKey) keys.push(reference.dlqKey);
|
|
1034
1660
|
if (reference.idempotencyKey) {
|
|
1035
1661
|
const dedupKey = `key:${reference.idempotencyKey}`;
|
|
1036
|
-
if (
|
|
1662
|
+
if (dedupHolders.get(dedupKey) === reference.messageId) keys.push(dedupKey);
|
|
1037
1663
|
}
|
|
1038
|
-
await txn.delete(key);
|
|
1039
1664
|
count++;
|
|
1040
1665
|
}
|
|
1666
|
+
await deleteMany$1(txn, keys);
|
|
1667
|
+
const done = references.size <= limit;
|
|
1668
|
+
const markerExpiredAt = existing?.expiredAt ?? expiredAt;
|
|
1041
1669
|
await txn.put(expiredRunKey(runId), {
|
|
1042
|
-
expiredAt,
|
|
1043
|
-
deleted: count
|
|
1670
|
+
expiredAt: markerExpiredAt,
|
|
1671
|
+
deleted: count,
|
|
1672
|
+
compactAt: existing?.compactAt ?? null
|
|
1044
1673
|
});
|
|
1045
1674
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1046
|
-
return
|
|
1047
|
-
|
|
1675
|
+
return done ? {
|
|
1676
|
+
deleted: count,
|
|
1677
|
+
done: true,
|
|
1678
|
+
receipt: {
|
|
1679
|
+
expiredAt: markerExpiredAt,
|
|
1680
|
+
deleted: count
|
|
1681
|
+
}
|
|
1682
|
+
} : {
|
|
1683
|
+
deleted: count,
|
|
1684
|
+
done: false
|
|
1685
|
+
};
|
|
1686
|
+
});
|
|
1687
|
+
}
|
|
1688
|
+
/** Acknowledge a final cleanup receipt before making its marker compactable. */
|
|
1689
|
+
async acknowledgeExpireRun(runId, receipt) {
|
|
1690
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1691
|
+
const key = expiredRunKey(runId);
|
|
1692
|
+
const marker = await txn.get(key);
|
|
1693
|
+
if (!marker) return { acknowledged: true };
|
|
1694
|
+
if (marker.expiredAt !== receipt.expiredAt || marker.deleted !== receipt.deleted) return { acknowledged: false };
|
|
1695
|
+
if (marker.compactAt === null) {
|
|
1696
|
+
const compactAt = this.#now() + QUEUE_FENCE_GRACE_MS;
|
|
1697
|
+
await txn.put(key, {
|
|
1698
|
+
...marker,
|
|
1699
|
+
compactAt
|
|
1700
|
+
});
|
|
1701
|
+
await txn.put(expiredRunGcKey(runId, compactAt), {
|
|
1702
|
+
runId,
|
|
1703
|
+
compactAt
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1706
|
+
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1707
|
+
return { acknowledged: true };
|
|
1708
|
+
});
|
|
1709
|
+
}
|
|
1710
|
+
};
|
|
1711
|
+
//#endregion
|
|
1712
|
+
//#region src/worker/durable-objects/RunCatalogDO.ts
|
|
1713
|
+
const MAX_INDEX_LIST_SIZE = 1001;
|
|
1714
|
+
const EXPIRY_GC_PREFIX = "expiry-gc:";
|
|
1715
|
+
function pad(ms) {
|
|
1716
|
+
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
1717
|
+
}
|
|
1718
|
+
function expiredKey(runId) {
|
|
1719
|
+
return `expired:${runId}`;
|
|
1720
|
+
}
|
|
1721
|
+
function expiryGcKey(runId, compactAt) {
|
|
1722
|
+
return `${EXPIRY_GC_PREFIX}${pad(compactAt)}:${encodeURIComponent(runId)}`;
|
|
1723
|
+
}
|
|
1724
|
+
/** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
|
|
1725
|
+
var RunCatalogDO = class extends DurableObject {
|
|
1726
|
+
now() {
|
|
1727
|
+
const clock = this.env?.clock;
|
|
1728
|
+
return typeof clock === "function" ? clock() : Date.now();
|
|
1729
|
+
}
|
|
1730
|
+
async upsertRun(runId, keys, serializedMetadata, publicationExpiresAt) {
|
|
1731
|
+
if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
|
|
1732
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1733
|
+
if (await txn.get(expiredKey(runId)) !== void 0) return { stored: false };
|
|
1734
|
+
const now = this.now();
|
|
1735
|
+
if (!Number.isSafeInteger(publicationExpiresAt) || publicationExpiresAt <= now || publicationExpiresAt > now + 1200900) return { stored: false };
|
|
1736
|
+
await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
|
|
1737
|
+
return { stored: true };
|
|
1738
|
+
});
|
|
1739
|
+
}
|
|
1740
|
+
async list(options = {}) {
|
|
1741
|
+
const suppliedLimit = options.limit;
|
|
1742
|
+
const limit = Math.min(MAX_INDEX_LIST_SIZE, Math.max(1, typeof suppliedLimit === "number" && Number.isFinite(suppliedLimit) ? Math.floor(suppliedLimit) : MAX_INDEX_LIST_SIZE));
|
|
1743
|
+
const entries = await this.ctx.storage.list({
|
|
1744
|
+
prefix: options.prefix ?? "",
|
|
1745
|
+
...options.reverse ? {
|
|
1746
|
+
reverse: true,
|
|
1747
|
+
end: options.cursor
|
|
1748
|
+
} : { startAfter: options.cursor },
|
|
1749
|
+
limit: limit + 1
|
|
1750
|
+
});
|
|
1751
|
+
const page = Array.from(entries).slice(0, limit);
|
|
1752
|
+
const listComplete = entries.size <= limit;
|
|
1753
|
+
return {
|
|
1754
|
+
keys: page.map(([name, value]) => ({
|
|
1755
|
+
name,
|
|
1756
|
+
value
|
|
1757
|
+
})),
|
|
1758
|
+
list_complete: listComplete,
|
|
1759
|
+
cursor: listComplete ? void 0 : page.at(-1)?.[0]
|
|
1760
|
+
};
|
|
1761
|
+
}
|
|
1762
|
+
async expireRun(runId, keys, expiredAt) {
|
|
1763
|
+
if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
|
|
1764
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1765
|
+
const marker = expiredKey(runId);
|
|
1766
|
+
const compactAt = (await txn.get(marker))?.compactAt ?? this.now() + 1200900;
|
|
1767
|
+
await txn.put(marker, {
|
|
1768
|
+
expiredAt,
|
|
1769
|
+
compactAt
|
|
1770
|
+
});
|
|
1771
|
+
await txn.put(expiryGcKey(runId, compactAt), {
|
|
1772
|
+
runId,
|
|
1773
|
+
compactAt
|
|
1774
|
+
});
|
|
1775
|
+
await this.armAtMost(txn, compactAt);
|
|
1776
|
+
return { deleted: await txn.delete(keys) };
|
|
1777
|
+
});
|
|
1778
|
+
}
|
|
1779
|
+
async alarm() {
|
|
1780
|
+
try {
|
|
1781
|
+
await this.compactExpiryFences();
|
|
1782
|
+
} catch {
|
|
1783
|
+
await this.ctx.storage.setAlarm(this.now() + LIFECYCLE_COMPACTION_RETRY_MS);
|
|
1784
|
+
}
|
|
1785
|
+
}
|
|
1786
|
+
async armAtMost(storage, at) {
|
|
1787
|
+
const current = await storage.getAlarm();
|
|
1788
|
+
if (current === null || at < current) await storage.setAlarm(at);
|
|
1789
|
+
}
|
|
1790
|
+
async compactExpiryFences() {
|
|
1791
|
+
const now = this.now();
|
|
1792
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
1793
|
+
const due = await txn.list({
|
|
1794
|
+
prefix: EXPIRY_GC_PREFIX,
|
|
1795
|
+
end: `${EXPIRY_GC_PREFIX}${pad(now + 1)}`,
|
|
1796
|
+
limit: 129
|
|
1797
|
+
});
|
|
1798
|
+
const page = Array.from(due).slice(0, 128);
|
|
1799
|
+
const markerKeys = page.map(([, item]) => expiredKey(item.runId));
|
|
1800
|
+
const markers = await txn.get(markerKeys);
|
|
1801
|
+
const deletes = page.flatMap(([key, item], index) => {
|
|
1802
|
+
return markers.get(markerKeys[index])?.compactAt === item.compactAt ? [key, markerKeys[index]] : [key];
|
|
1803
|
+
});
|
|
1804
|
+
for (let offset = 0; offset < deletes.length; offset += 128) await txn.delete(deletes.slice(offset, offset + 128));
|
|
1805
|
+
if (due.size > 128) {
|
|
1806
|
+
await txn.setAlarm(now + 1);
|
|
1807
|
+
return;
|
|
1808
|
+
}
|
|
1809
|
+
const nextAt = (await txn.list({
|
|
1810
|
+
prefix: EXPIRY_GC_PREFIX,
|
|
1811
|
+
limit: 1
|
|
1812
|
+
})).values().next().value?.compactAt;
|
|
1813
|
+
if (nextAt === void 0) await txn.deleteAlarm();
|
|
1814
|
+
else await txn.setAlarm(Math.max(now + 1, nextAt));
|
|
1815
|
+
});
|
|
1048
1816
|
}
|
|
1049
1817
|
};
|
|
1050
1818
|
//#endregion
|
|
1051
1819
|
//#region src/worker/durable-objects/StreamDO.ts
|
|
1052
1820
|
const META_KEY = "meta";
|
|
1053
1821
|
const CHUNK_KEY_PREFIX = "chunk:";
|
|
1822
|
+
const CHUNK_SIZE_KEY_PREFIX = "chunk-size:";
|
|
1054
1823
|
/** Registry keys used when this DO instance acts as a per-run stream index. */
|
|
1055
1824
|
const STREAM_REGISTRY_PREFIX = "stream:";
|
|
1056
1825
|
const REGISTRY_OWNER_KEY = "registry:owner";
|
|
1057
1826
|
const REGISTRY_EXPIRED_KEY = "registry:expired";
|
|
1058
|
-
/**
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
*/
|
|
1827
|
+
/** Each chunk occupies a payload key and a size key; one delete accepts at most 128 keys. */
|
|
1828
|
+
const DEFAULT_EXPIRE_CHUNK_LIMIT = 64;
|
|
1829
|
+
const MAX_EXPIRE_CHUNK_LIMIT = 64;
|
|
1830
|
+
/** Bound storage deletion work as well as item count; one oversized chunk still makes progress. */
|
|
1831
|
+
const DEFAULT_EXPIRE_BYTE_LIMIT = 16777216;
|
|
1832
|
+
const MAX_EXPIRE_BYTE_LIMIT = DEFAULT_EXPIRE_BYTE_LIMIT;
|
|
1833
|
+
function boundedLimit(value, fallback, maximum) {
|
|
1834
|
+
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
1835
|
+
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
1836
|
+
}
|
|
1837
|
+
function emptyMeta() {
|
|
1838
|
+
return {
|
|
1839
|
+
count: 0,
|
|
1840
|
+
state: "open"
|
|
1841
|
+
};
|
|
1842
|
+
}
|
|
1843
|
+
function validateMeta(meta) {
|
|
1844
|
+
if (!Number.isSafeInteger(meta.count) || meta.count < 0 || meta.count > 2147483647 || ![
|
|
1845
|
+
"open",
|
|
1846
|
+
"closed",
|
|
1847
|
+
"errored",
|
|
1848
|
+
"expired"
|
|
1849
|
+
].includes(meta.state)) throw new Error("Invalid persisted stream metadata");
|
|
1850
|
+
if (meta.state === "errored" && !meta.error) throw new Error("Invalid persisted stream error metadata");
|
|
1851
|
+
return meta;
|
|
1852
|
+
}
|
|
1853
|
+
/** Zero-padding keeps storage.list() results in stream offset order. */
|
|
1062
1854
|
function chunkKey(index) {
|
|
1063
1855
|
return `${CHUNK_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
|
|
1064
1856
|
}
|
|
1857
|
+
function chunkSizeKey(index) {
|
|
1858
|
+
return `${CHUNK_SIZE_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
|
|
1859
|
+
}
|
|
1860
|
+
function indexFromChunkSizeKey(key) {
|
|
1861
|
+
const suffix = key.slice(11);
|
|
1862
|
+
const index = Number(suffix);
|
|
1863
|
+
if (!/^\d{12}$/.test(suffix) || !Number.isSafeInteger(index) || index < 0) throw new Error(`Invalid persisted stream chunk size key "${key}"`);
|
|
1864
|
+
return index;
|
|
1865
|
+
}
|
|
1866
|
+
function validateChunkSize(size, index) {
|
|
1867
|
+
if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0 || size > 1048576) throw new Error(`Invalid persisted stream chunk size at index ${index}`);
|
|
1868
|
+
return size;
|
|
1869
|
+
}
|
|
1870
|
+
function compactChunk(chunk) {
|
|
1871
|
+
return chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength ? chunk : new Uint8Array(chunk);
|
|
1872
|
+
}
|
|
1873
|
+
function abortReason(signal) {
|
|
1874
|
+
return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
|
1875
|
+
}
|
|
1065
1876
|
/**
|
|
1066
1877
|
* Durable Object backing workflow streams.
|
|
1067
1878
|
*
|
|
1068
|
-
*
|
|
1069
|
-
*
|
|
1070
|
-
*
|
|
1071
|
-
* each chunk its index; one chunk per storage key.
|
|
1072
|
-
* - `run-streams:<runId>`: registry of stream names owned by a run,
|
|
1073
|
-
* powering `listStreamsByRunId`.
|
|
1879
|
+
* Stream cells persist binary chunks under monotonic offset keys. Public
|
|
1880
|
+
* reads are bounded long polls, and writes append a bounded ordered batch in
|
|
1881
|
+
* one storage transaction. Per-run registry cells retain their existing role.
|
|
1074
1882
|
*/
|
|
1075
1883
|
var StreamDO = class extends DurableObject {
|
|
1884
|
+
meta;
|
|
1885
|
+
metaLoad;
|
|
1886
|
+
mutationTail = Promise.resolve();
|
|
1887
|
+
changeVersion = 0;
|
|
1888
|
+
waiters = /* @__PURE__ */ new Set();
|
|
1076
1889
|
async getMeta() {
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1890
|
+
if (this.meta) return this.meta;
|
|
1891
|
+
this.metaLoad ??= this.ctx.storage.get(META_KEY).then((meta) => {
|
|
1892
|
+
const loaded = meta ? validateMeta(meta) : emptyMeta();
|
|
1893
|
+
this.meta = loaded;
|
|
1894
|
+
return loaded;
|
|
1895
|
+
});
|
|
1896
|
+
try {
|
|
1897
|
+
return await this.metaLoad;
|
|
1898
|
+
} catch (error) {
|
|
1899
|
+
this.metaLoad = void 0;
|
|
1900
|
+
throw error;
|
|
1901
|
+
}
|
|
1081
1902
|
}
|
|
1082
1903
|
assertOwner(meta, runId) {
|
|
1083
|
-
if (
|
|
1904
|
+
if (!runId) throw new Error("Stream runId is required");
|
|
1084
1905
|
if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
|
|
1085
1906
|
}
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1907
|
+
assertWritable(meta, runId) {
|
|
1908
|
+
this.assertOwner(meta, runId);
|
|
1909
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
1910
|
+
if (meta.state === "closed") throw new Error("Cannot write to a closed stream");
|
|
1911
|
+
if (meta.state === "errored") throw new Error("Cannot write to an errored stream");
|
|
1912
|
+
}
|
|
1913
|
+
runMutation(operation) {
|
|
1914
|
+
const result = this.mutationTail.then(operation, operation);
|
|
1915
|
+
this.mutationTail = result.then(() => void 0, () => void 0);
|
|
1916
|
+
return result;
|
|
1917
|
+
}
|
|
1918
|
+
wakeReaders() {
|
|
1919
|
+
this.changeVersion += 1;
|
|
1920
|
+
for (const waiter of Array.from(this.waiters)) waiter.resolve("change");
|
|
1921
|
+
}
|
|
1922
|
+
waitForChange(observedVersion, waitMs, signal) {
|
|
1923
|
+
if (signal?.aborted) return Promise.reject(abortReason(signal));
|
|
1924
|
+
return new Promise((resolve, reject) => {
|
|
1925
|
+
let settled = false;
|
|
1926
|
+
const waiter = {
|
|
1927
|
+
timer: setTimeout(() => finish("timeout"), waitMs),
|
|
1928
|
+
signal,
|
|
1929
|
+
resolve: (reason) => finish(reason)
|
|
1930
|
+
};
|
|
1931
|
+
const cleanup = () => {
|
|
1932
|
+
clearTimeout(waiter.timer);
|
|
1933
|
+
this.waiters.delete(waiter);
|
|
1934
|
+
if (waiter.signal && waiter.onAbort) waiter.signal.removeEventListener("abort", waiter.onAbort);
|
|
1935
|
+
};
|
|
1936
|
+
const finish = (reason) => {
|
|
1937
|
+
if (settled) return;
|
|
1938
|
+
settled = true;
|
|
1939
|
+
cleanup();
|
|
1940
|
+
resolve(reason);
|
|
1941
|
+
};
|
|
1942
|
+
waiter.onAbort = () => {
|
|
1943
|
+
if (settled) return;
|
|
1944
|
+
settled = true;
|
|
1945
|
+
cleanup();
|
|
1946
|
+
reject(abortReason(signal));
|
|
1947
|
+
};
|
|
1948
|
+
this.waiters.add(waiter);
|
|
1949
|
+
signal?.addEventListener("abort", waiter.onAbort, { once: true });
|
|
1950
|
+
if (this.changeVersion !== observedVersion) finish("change");
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
1953
|
+
async readSnapshot(request, timedOut) {
|
|
1954
|
+
const meta = await this.getMeta();
|
|
1955
|
+
this.assertOwner(meta, request.runId);
|
|
1956
|
+
const available = meta.state === "expired" ? 0 : Math.max(0, Math.min(request.maxChunks, meta.count - request.startIndex));
|
|
1957
|
+
const chunks = [];
|
|
1958
|
+
if (available > 0) {
|
|
1959
|
+
const indexes = Array.from({ length: available }, (_, offset) => request.startIndex + offset);
|
|
1960
|
+
const sizeKeys = indexes.map(chunkSizeKey);
|
|
1961
|
+
const storedSizes = await this.ctx.storage.get(sizeKeys);
|
|
1962
|
+
const selectedIndexes = [];
|
|
1963
|
+
const selectedSizes = [];
|
|
1964
|
+
let bytes = 0;
|
|
1965
|
+
for (const index of indexes) {
|
|
1966
|
+
const size = validateChunkSize(storedSizes.get(chunkSizeKey(index)), index);
|
|
1967
|
+
if (bytes + size > request.maxBytes) break;
|
|
1968
|
+
selectedIndexes.push(index);
|
|
1969
|
+
selectedSizes.push(size);
|
|
1970
|
+
bytes += size;
|
|
1971
|
+
}
|
|
1972
|
+
const payloadKeys = selectedIndexes.map(chunkKey);
|
|
1973
|
+
const storedChunks = await this.ctx.storage.get(payloadKeys);
|
|
1974
|
+
for (let offset = 0; offset < payloadKeys.length; offset++) {
|
|
1975
|
+
const chunk = storedChunks.get(payloadKeys[offset]);
|
|
1976
|
+
if (!(chunk instanceof Uint8Array) || chunk.byteLength !== selectedSizes[offset]) throw new Error(`Invalid persisted stream chunk at index ${selectedIndexes[offset]}`);
|
|
1977
|
+
chunks.push(chunk);
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
return {
|
|
1981
|
+
startIndex: request.startIndex,
|
|
1982
|
+
tailIndex: meta.count - 1,
|
|
1983
|
+
chunks,
|
|
1984
|
+
state: meta.state,
|
|
1985
|
+
timedOut,
|
|
1986
|
+
error: meta.error
|
|
1987
|
+
};
|
|
1988
|
+
}
|
|
1989
|
+
/** Append one bounded binary batch with contiguous indexes. */
|
|
1990
|
+
async writeChunks(runId, chunks) {
|
|
1991
|
+
validateStreamWriteChunks(chunks);
|
|
1992
|
+
const storedChunks = chunks.map(compactChunk);
|
|
1993
|
+
return await this.runMutation(async () => {
|
|
1994
|
+
const committed = await this.ctx.storage.transaction(async (txn) => {
|
|
1995
|
+
const stored = await txn.get(META_KEY);
|
|
1996
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
1997
|
+
this.assertWritable(meta, runId);
|
|
1998
|
+
const startIndex = meta.count;
|
|
1999
|
+
if (startIndex + chunks.length > 2147483647) throw new Error("Stream offset limit exceeded");
|
|
2000
|
+
const nextMeta = {
|
|
2001
|
+
count: startIndex + chunks.length,
|
|
2002
|
+
state: "open",
|
|
2003
|
+
ownerRunId: runId
|
|
2004
|
+
};
|
|
2005
|
+
const entries = { [META_KEY]: nextMeta };
|
|
2006
|
+
for (let offset = 0; offset < storedChunks.length; offset++) {
|
|
2007
|
+
const index = startIndex + offset;
|
|
2008
|
+
entries[chunkKey(index)] = storedChunks[offset];
|
|
2009
|
+
entries[chunkSizeKey(index)] = storedChunks[offset].byteLength;
|
|
2010
|
+
}
|
|
2011
|
+
await txn.put(entries);
|
|
2012
|
+
return {
|
|
2013
|
+
meta: nextMeta,
|
|
2014
|
+
result: {
|
|
2015
|
+
startIndex,
|
|
2016
|
+
count: chunks.length,
|
|
2017
|
+
tailIndex: nextMeta.count - 1
|
|
2018
|
+
}
|
|
2019
|
+
};
|
|
1104
2020
|
});
|
|
1105
|
-
|
|
2021
|
+
this.meta = committed.meta;
|
|
2022
|
+
this.metaLoad = Promise.resolve(committed.meta);
|
|
2023
|
+
this.wakeReaders();
|
|
2024
|
+
return committed.result;
|
|
1106
2025
|
});
|
|
1107
2026
|
}
|
|
1108
2027
|
/**
|
|
1109
|
-
*
|
|
1110
|
-
*
|
|
2028
|
+
* Return available binary chunks and terminal metadata in one bounded
|
|
2029
|
+
* operation, waiting at most waitMs when the requested offset is idle.
|
|
1111
2030
|
*/
|
|
2031
|
+
async readChunks(request, signal) {
|
|
2032
|
+
validateStreamReadRequest(request);
|
|
2033
|
+
const observedVersion = this.changeVersion;
|
|
2034
|
+
const immediate = await this.readSnapshot(request, false);
|
|
2035
|
+
if (immediate.chunks.length > 0 || immediate.state !== "open" || request.waitMs === 0 || request.maxChunks === 0) return immediate;
|
|
2036
|
+
const reason = await this.waitForChange(observedVersion, request.waitMs, signal);
|
|
2037
|
+
return await this.readSnapshot(request, reason === "timeout");
|
|
2038
|
+
}
|
|
2039
|
+
/** Close idempotently; the first terminal state wins. */
|
|
1112
2040
|
async closeStream(runId) {
|
|
1113
|
-
await this.
|
|
1114
|
-
const
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
2041
|
+
await this.runMutation(async () => {
|
|
2042
|
+
const nextMeta = await this.ctx.storage.transaction(async (txn) => {
|
|
2043
|
+
const stored = await txn.get(META_KEY);
|
|
2044
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
2045
|
+
this.assertOwner(meta, runId);
|
|
2046
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
2047
|
+
if (meta.state !== "open") return meta;
|
|
2048
|
+
const closed = {
|
|
2049
|
+
...meta,
|
|
2050
|
+
ownerRunId: runId,
|
|
2051
|
+
state: "closed"
|
|
2052
|
+
};
|
|
2053
|
+
await txn.put(META_KEY, closed);
|
|
2054
|
+
return closed;
|
|
1123
2055
|
});
|
|
2056
|
+
this.meta = nextMeta;
|
|
2057
|
+
this.metaLoad = Promise.resolve(nextMeta);
|
|
2058
|
+
this.wakeReaders();
|
|
1124
2059
|
});
|
|
1125
2060
|
}
|
|
1126
|
-
/**
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
2061
|
+
/** Persist a terminal stream failure and wake every pending reader. */
|
|
2062
|
+
async failStream(runId, error) {
|
|
2063
|
+
const normalized = normalizeStreamError(error);
|
|
2064
|
+
await this.runMutation(async () => {
|
|
2065
|
+
const nextMeta = await this.ctx.storage.transaction(async (txn) => {
|
|
2066
|
+
const stored = await txn.get(META_KEY);
|
|
2067
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
2068
|
+
this.assertOwner(meta, runId);
|
|
2069
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
2070
|
+
if (meta.state !== "open") return meta;
|
|
2071
|
+
const failed = {
|
|
2072
|
+
...meta,
|
|
2073
|
+
ownerRunId: runId,
|
|
2074
|
+
state: "errored",
|
|
2075
|
+
error: normalized
|
|
2076
|
+
};
|
|
2077
|
+
await txn.put(META_KEY, failed);
|
|
2078
|
+
return failed;
|
|
2079
|
+
});
|
|
2080
|
+
this.meta = nextMeta;
|
|
2081
|
+
this.metaLoad = Promise.resolve(nextMeta);
|
|
2082
|
+
this.wakeReaders();
|
|
1137
2083
|
});
|
|
1138
|
-
return {
|
|
1139
|
-
chunks: Array.from(entries.values()),
|
|
1140
|
-
done: meta.closed,
|
|
1141
|
-
tailIndex: meta.count - 1
|
|
1142
|
-
};
|
|
1143
|
-
}
|
|
1144
|
-
/**
|
|
1145
|
-
* Lightweight stream metadata: last chunk index and completion flag.
|
|
1146
|
-
*/
|
|
1147
|
-
async getInfo() {
|
|
1148
|
-
const meta = await this.getMeta();
|
|
1149
|
-
if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
|
|
1150
|
-
return {
|
|
1151
|
-
tailIndex: meta.count - 1,
|
|
1152
|
-
done: meta.closed
|
|
1153
|
-
};
|
|
1154
2084
|
}
|
|
1155
|
-
/**
|
|
1156
|
-
* Register a stream name against this per-run registry instance.
|
|
1157
|
-
*/
|
|
2085
|
+
/** Register a stream name against this per-run registry instance. */
|
|
1158
2086
|
async registerStream(runId, name) {
|
|
1159
2087
|
await this.ctx.storage.transaction(async (txn) => {
|
|
1160
2088
|
if (await txn.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
|
|
@@ -1164,57 +2092,124 @@ var StreamDO = class extends DurableObject {
|
|
|
1164
2092
|
await txn.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
|
|
1165
2093
|
});
|
|
1166
2094
|
}
|
|
1167
|
-
/**
|
|
1168
|
-
* List stream names registered against this per-run registry instance.
|
|
1169
|
-
*/
|
|
1170
2095
|
async listStreams() {
|
|
1171
2096
|
if (await this.ctx.storage.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error("Workflow run streams have expired");
|
|
1172
2097
|
const entries = await this.ctx.storage.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1173
2098
|
return Array.from(entries.keys()).map((key) => key.slice(7));
|
|
1174
2099
|
}
|
|
1175
2100
|
/** Fence a run's registry before any stream cells are removed. */
|
|
1176
|
-
async expireRegistry(runId, expiredAt) {
|
|
2101
|
+
async expireRegistry(runId, expiredAt, options) {
|
|
2102
|
+
const limit = boundedLimit(options?.limit, 16, 128);
|
|
1177
2103
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1178
|
-
const
|
|
2104
|
+
const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
|
|
2105
|
+
const owner = state.get(REGISTRY_OWNER_KEY);
|
|
1179
2106
|
if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
|
|
1180
|
-
|
|
1181
|
-
await txn.put(
|
|
1182
|
-
|
|
2107
|
+
const expiry = state.get(REGISTRY_EXPIRED_KEY);
|
|
2108
|
+
await txn.put({
|
|
2109
|
+
[REGISTRY_OWNER_KEY]: runId,
|
|
2110
|
+
[REGISTRY_EXPIRED_KEY]: expiry ?? {
|
|
2111
|
+
expiredAt,
|
|
2112
|
+
deleted: 0
|
|
2113
|
+
}
|
|
2114
|
+
});
|
|
2115
|
+
const entries = await txn.list({
|
|
2116
|
+
prefix: STREAM_REGISTRY_PREFIX,
|
|
2117
|
+
limit
|
|
2118
|
+
});
|
|
1183
2119
|
return { streams: Array.from(entries.keys()).map((key) => key.slice(7)) };
|
|
1184
2120
|
});
|
|
1185
2121
|
}
|
|
1186
|
-
/** Remove
|
|
1187
|
-
async finalizeRegistry(runId) {
|
|
1188
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
1189
|
-
const owner = await txn.get(REGISTRY_OWNER_KEY);
|
|
1190
|
-
const expiredAt = await txn.get(REGISTRY_EXPIRED_KEY);
|
|
1191
|
-
if (owner !== runId || expiredAt === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
|
|
1192
|
-
const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1193
|
-
for (const key of entries.keys()) await txn.delete(key);
|
|
1194
|
-
});
|
|
1195
|
-
}
|
|
1196
|
-
/** Delete stream payload while leaving a small ownership tombstone. */
|
|
1197
|
-
async expireStream(runId, expiredAt) {
|
|
2122
|
+
/** Remove completed registry entries in a bounded, retry-safe batch. */
|
|
2123
|
+
async finalizeRegistry(runId, streams) {
|
|
1198
2124
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1199
|
-
const
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
};
|
|
1203
|
-
|
|
1204
|
-
const
|
|
1205
|
-
|
|
1206
|
-
await txn.put(
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
ownerRunId: runId,
|
|
1210
|
-
expiredAt
|
|
2125
|
+
const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
|
|
2126
|
+
const owner = state.get(REGISTRY_OWNER_KEY);
|
|
2127
|
+
const expiry = state.get(REGISTRY_EXPIRED_KEY);
|
|
2128
|
+
if (owner !== runId || expiry === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
|
|
2129
|
+
const keys = Array.from(new Set(streams.map((name) => `${STREAM_REGISTRY_PREFIX}${name}`)));
|
|
2130
|
+
const removed = keys.length === 0 ? 0 : await txn.delete(keys);
|
|
2131
|
+
const deleted = expiry.deleted + removed;
|
|
2132
|
+
await txn.put(REGISTRY_EXPIRED_KEY, {
|
|
2133
|
+
...expiry,
|
|
2134
|
+
deleted
|
|
1211
2135
|
});
|
|
1212
2136
|
return {
|
|
1213
|
-
deleted
|
|
1214
|
-
|
|
2137
|
+
deleted,
|
|
2138
|
+
done: (await txn.list({
|
|
2139
|
+
prefix: STREAM_REGISTRY_PREFIX,
|
|
2140
|
+
limit: 1
|
|
2141
|
+
})).size === 0
|
|
1215
2142
|
};
|
|
1216
2143
|
});
|
|
1217
2144
|
}
|
|
2145
|
+
/**
|
|
2146
|
+
* Fence the stream and delete one bounded page. Listing size keys avoids
|
|
2147
|
+
* loading payloads, and the transaction makes a crash retry resume safely.
|
|
2148
|
+
*/
|
|
2149
|
+
async expireStream(runId, expiredAt, options) {
|
|
2150
|
+
const limit = boundedLimit(options?.limit, DEFAULT_EXPIRE_CHUNK_LIMIT, MAX_EXPIRE_CHUNK_LIMIT);
|
|
2151
|
+
const byteLimit = boundedLimit(options?.byteLimit, DEFAULT_EXPIRE_BYTE_LIMIT, MAX_EXPIRE_BYTE_LIMIT);
|
|
2152
|
+
return await this.runMutation(async () => {
|
|
2153
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
2154
|
+
const stored = await txn.get(META_KEY);
|
|
2155
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
2156
|
+
this.assertOwner(meta, runId);
|
|
2157
|
+
const firstFence = meta.state !== "expired";
|
|
2158
|
+
if (meta.state === "expired" && meta.payloadDeleted) return {
|
|
2159
|
+
meta,
|
|
2160
|
+
result: {
|
|
2161
|
+
deleted: false,
|
|
2162
|
+
chunks: 0,
|
|
2163
|
+
bytes: 0,
|
|
2164
|
+
done: true
|
|
2165
|
+
}
|
|
2166
|
+
};
|
|
2167
|
+
const candidates = await txn.list({
|
|
2168
|
+
prefix: CHUNK_SIZE_KEY_PREFIX,
|
|
2169
|
+
limit: limit + 1
|
|
2170
|
+
});
|
|
2171
|
+
const page = [];
|
|
2172
|
+
let bytes = 0;
|
|
2173
|
+
for (const [key, storedSize] of Array.from(candidates).slice(0, limit)) {
|
|
2174
|
+
const index = indexFromChunkSizeKey(key);
|
|
2175
|
+
const size = validateChunkSize(storedSize, index);
|
|
2176
|
+
if (page.length > 0 && bytes + size > byteLimit) break;
|
|
2177
|
+
page.push({
|
|
2178
|
+
index,
|
|
2179
|
+
size
|
|
2180
|
+
});
|
|
2181
|
+
bytes += size;
|
|
2182
|
+
}
|
|
2183
|
+
const keys = page.flatMap(({ index }) => [chunkKey(index), chunkSizeKey(index)]);
|
|
2184
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
2185
|
+
const done = candidates.size === page.length;
|
|
2186
|
+
const nextMeta = {
|
|
2187
|
+
...meta,
|
|
2188
|
+
count: done ? 0 : meta.count,
|
|
2189
|
+
ownerRunId: runId,
|
|
2190
|
+
state: "expired",
|
|
2191
|
+
error: void 0,
|
|
2192
|
+
expiredAt: meta.expiredAt ?? expiredAt,
|
|
2193
|
+
expiredChunkCount: meta.expiredChunkCount ?? meta.count,
|
|
2194
|
+
payloadDeleted: done
|
|
2195
|
+
};
|
|
2196
|
+
await txn.put(META_KEY, nextMeta);
|
|
2197
|
+
return {
|
|
2198
|
+
meta: nextMeta,
|
|
2199
|
+
result: {
|
|
2200
|
+
deleted: firstFence,
|
|
2201
|
+
chunks: page.length,
|
|
2202
|
+
bytes,
|
|
2203
|
+
done
|
|
2204
|
+
}
|
|
2205
|
+
};
|
|
2206
|
+
});
|
|
2207
|
+
this.meta = result.meta;
|
|
2208
|
+
this.metaLoad = Promise.resolve(result.meta);
|
|
2209
|
+
this.wakeReaders();
|
|
2210
|
+
return result.result;
|
|
2211
|
+
});
|
|
2212
|
+
}
|
|
1218
2213
|
};
|
|
1219
2214
|
//#endregion
|
|
1220
2215
|
//#region src/apply-event.ts
|
|
@@ -1234,10 +2229,36 @@ const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
|
|
|
1234
2229
|
* rejected as a duplicate instead of resurrecting the hook.
|
|
1235
2230
|
*/
|
|
1236
2231
|
const HOOK_EVENT_MARKER_PREFIX = "hookevent:";
|
|
2232
|
+
/** Durable repair record for idempotent post-commit hook-index release. */
|
|
2233
|
+
const HOOK_RELEASE_MARKER_PREFIX = "hookreleased:";
|
|
2234
|
+
/** Exact ambiguous reservations canceled by a serialized RunDO resolution. */
|
|
2235
|
+
const HOOK_CLAIM_CANCELLATION_PREFIX = "hookclaimcancelled:";
|
|
1237
2236
|
/** Workflow-authored attr_set correlation claims. */
|
|
1238
2237
|
const ATTRIBUTE_EVENT_MARKER_PREFIX = "attrevent:";
|
|
1239
2238
|
/** Durable lazy-hook resume claims, scoped by the containing run DO. */
|
|
1240
2239
|
const HOOK_RESUME_CLAIM_PREFIX = "hookresume:";
|
|
2240
|
+
const STORAGE_BATCH_SIZE$1 = 128;
|
|
2241
|
+
const MAX_ENTITY_PAGE_SIZE = 1e3;
|
|
2242
|
+
async function getMany(store, keys) {
|
|
2243
|
+
if (store.getMany) {
|
|
2244
|
+
const result = /* @__PURE__ */ new Map();
|
|
2245
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) {
|
|
2246
|
+
const page = await store.getMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
2247
|
+
for (const entry of page) result.set(...entry);
|
|
2248
|
+
}
|
|
2249
|
+
return result;
|
|
2250
|
+
}
|
|
2251
|
+
const values = await Promise.all(keys.map((key) => store.get(key)));
|
|
2252
|
+
return new Map(keys.flatMap((key, index) => values[index] === void 0 ? [] : [[key, values[index]]]));
|
|
2253
|
+
}
|
|
2254
|
+
async function deleteMany(store, keys) {
|
|
2255
|
+
if (store.deleteMany) {
|
|
2256
|
+
let deleted = 0;
|
|
2257
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) deleted += await store.deleteMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
2258
|
+
return deleted;
|
|
2259
|
+
}
|
|
2260
|
+
return (await Promise.all(keys.map((key) => store.delete(key)))).filter(Boolean).length;
|
|
2261
|
+
}
|
|
1241
2262
|
function failure(code, message, extra) {
|
|
1242
2263
|
return {
|
|
1243
2264
|
ok: false,
|
|
@@ -1246,6 +2267,9 @@ function failure(code, message, extra) {
|
|
|
1246
2267
|
...extra
|
|
1247
2268
|
};
|
|
1248
2269
|
}
|
|
2270
|
+
function hookClaimCancellationKey(claimId) {
|
|
2271
|
+
return `${HOOK_CLAIM_CANCELLATION_PREFIX}${encodeURIComponent(claimId)}`;
|
|
2272
|
+
}
|
|
1249
2273
|
function readStringProp(value, key) {
|
|
1250
2274
|
if (value !== null && typeof value === "object") {
|
|
1251
2275
|
const prop = value[key];
|
|
@@ -1301,7 +2325,9 @@ function creationIndexKey(prefix, createdAt, id) {
|
|
|
1301
2325
|
* requested ordering.
|
|
1302
2326
|
*/
|
|
1303
2327
|
async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
1304
|
-
const
|
|
2328
|
+
const requestedLimit = Number.isFinite(params.limit) ? Math.floor(params.limit) : 1;
|
|
2329
|
+
const limit = Math.min(MAX_ENTITY_PAGE_SIZE, Math.max(1, requestedLimit));
|
|
2330
|
+
const { cursor, sortOrder = "asc" } = params;
|
|
1305
2331
|
const entries = await store.list(sortOrder === "desc" ? {
|
|
1306
2332
|
prefix: indexPrefix,
|
|
1307
2333
|
reverse: true,
|
|
@@ -1313,11 +2339,12 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
|
1313
2339
|
startAfter: cursor ? `${indexPrefix}${cursor}` : void 0
|
|
1314
2340
|
});
|
|
1315
2341
|
const page = Array.from(entries.entries()).slice(0, limit);
|
|
1316
|
-
const
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
2342
|
+
const entityKeys = page.map(([, id]) => `${entityPrefix}${id}`);
|
|
2343
|
+
const entities = await getMany(store, entityKeys);
|
|
2344
|
+
const data = entityKeys.flatMap((key) => {
|
|
2345
|
+
const item = entities.get(key);
|
|
2346
|
+
return item === void 0 ? [] : [item];
|
|
2347
|
+
});
|
|
1321
2348
|
const hasMore = entries.size > limit;
|
|
1322
2349
|
const lastKey = page.at(-1)?.[0];
|
|
1323
2350
|
return {
|
|
@@ -1328,22 +2355,26 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
|
1328
2355
|
}
|
|
1329
2356
|
/** Delete all hook entities for the run, returning the released tokens. */
|
|
1330
2357
|
async function releaseAllHooks(store) {
|
|
2358
|
+
if (store.deferTerminalCleanup) return [];
|
|
1331
2359
|
const hooks = await store.list({ prefix: HOOK_KEY_PREFIX });
|
|
1332
2360
|
const released = [];
|
|
2361
|
+
const keys = [];
|
|
1333
2362
|
for (const hook of hooks.values()) {
|
|
1334
|
-
|
|
1335
|
-
|
|
2363
|
+
keys.push(`${HOOK_KEY_PREFIX}${hook.hookId}`);
|
|
2364
|
+
keys.push(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
|
|
1336
2365
|
released.push({
|
|
1337
2366
|
hookId: hook.hookId,
|
|
1338
2367
|
token: hook.token
|
|
1339
2368
|
});
|
|
1340
2369
|
}
|
|
2370
|
+
await deleteMany(store, keys);
|
|
1341
2371
|
return released;
|
|
1342
2372
|
}
|
|
1343
2373
|
/** Delete all persisted waits once their owning run becomes terminal. */
|
|
1344
2374
|
async function releaseAllWaits(store) {
|
|
2375
|
+
if (store.deferTerminalCleanup) return;
|
|
1345
2376
|
const waits = await store.list({ prefix: WAIT_KEY_PREFIX });
|
|
1346
|
-
|
|
2377
|
+
await deleteMany(store, Array.from(waits.keys()));
|
|
1347
2378
|
}
|
|
1348
2379
|
async function applyEvent(store, ctx) {
|
|
1349
2380
|
const { runId, data, now } = ctx;
|
|
@@ -1463,7 +2494,16 @@ async function applyEvent(store, ctx) {
|
|
|
1463
2494
|
}
|
|
1464
2495
|
}
|
|
1465
2496
|
if ((data.eventType === "hook_disposed" || data.eventType === "hook_received") && data.correlationId) {
|
|
1466
|
-
if (!await store.get(`hook:${data.correlationId}`))
|
|
2497
|
+
if (!await store.get(`hook:${data.correlationId}`)) {
|
|
2498
|
+
if (data.eventType === "hook_disposed") {
|
|
2499
|
+
const released = await store.get(`${HOOK_RELEASE_MARKER_PREFIX}${data.correlationId}`);
|
|
2500
|
+
if (released) return {
|
|
2501
|
+
ok: true,
|
|
2502
|
+
releasedHooks: [released]
|
|
2503
|
+
};
|
|
2504
|
+
}
|
|
2505
|
+
return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
2506
|
+
}
|
|
1467
2507
|
}
|
|
1468
2508
|
if (!skipRunValidation && data.eventType !== "run_started" && !currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1469
2509
|
switch (data.eventType) {
|
|
@@ -1799,6 +2839,7 @@ async function applyEvent(store, ctx) {
|
|
|
1799
2839
|
const holder = ctx.tokenHolder ?? null;
|
|
1800
2840
|
const markerKey = `${HOOK_EVENT_MARKER_PREFIX}${hookId}`;
|
|
1801
2841
|
const hookKey = `${HOOK_KEY_PREFIX}${hookId}`;
|
|
2842
|
+
if (ctx.hookClaimId && await store.get(hookClaimCancellationKey(ctx.hookClaimId)) !== void 0) return failure("HOOK_CLAIM_CANCELLED", `Hook reservation for "${hookId}" was canceled before commit`);
|
|
1802
2843
|
if (holder && (holder.runId !== runId || holder.hookId !== hookId)) {
|
|
1803
2844
|
const event = buildEvent({
|
|
1804
2845
|
eventType: "hook_conflict",
|
|
@@ -1858,6 +2899,10 @@ async function applyEvent(store, ctx) {
|
|
|
1858
2899
|
if (!hook) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
1859
2900
|
await store.delete(hookKey);
|
|
1860
2901
|
await store.delete(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
|
|
2902
|
+
await store.put(`${HOOK_RELEASE_MARKER_PREFIX}${hook.hookId}`, {
|
|
2903
|
+
hookId: hook.hookId,
|
|
2904
|
+
token: hook.token
|
|
2905
|
+
});
|
|
1861
2906
|
const event = buildEvent({ ...data });
|
|
1862
2907
|
await putEvent(event);
|
|
1863
2908
|
return {
|
|
@@ -1972,10 +3017,63 @@ async function finalizeEventPage(store, outcome, params) {
|
|
|
1972
3017
|
};
|
|
1973
3018
|
}
|
|
1974
3019
|
//#endregion
|
|
3020
|
+
//#region src/retention.ts
|
|
3021
|
+
const CLEANUP_RECORD_KEY = "retention:cleanup";
|
|
3022
|
+
const TOMBSTONE_KEY = "retention:tombstone";
|
|
3023
|
+
const HOOK_MARKER_PREFIX = "retention:hook:";
|
|
3024
|
+
const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
|
|
3025
|
+
function sortableTimestamp(date) {
|
|
3026
|
+
return date.getTime().toString().padStart(13, "0");
|
|
3027
|
+
}
|
|
3028
|
+
function workflowRunIndexKey(run) {
|
|
3029
|
+
return `run:${run.workflowName}:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
3030
|
+
}
|
|
3031
|
+
function globalRunIndexKey(run) {
|
|
3032
|
+
return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
3033
|
+
}
|
|
3034
|
+
function hookMarkerKey(reference) {
|
|
3035
|
+
return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
|
|
3036
|
+
}
|
|
3037
|
+
function cleanupTombstone(record, now) {
|
|
3038
|
+
const cleanup = {
|
|
3039
|
+
...record,
|
|
3040
|
+
phase: "tombstoned",
|
|
3041
|
+
tombstonedAt: now
|
|
3042
|
+
};
|
|
3043
|
+
return {
|
|
3044
|
+
version: 1,
|
|
3045
|
+
runId: record.runId,
|
|
3046
|
+
terminalStatus: record.terminalStatus,
|
|
3047
|
+
completedAt: record.completedAt,
|
|
3048
|
+
expiredAt: record.dueAt,
|
|
3049
|
+
tombstonedAt: now,
|
|
3050
|
+
cleanup
|
|
3051
|
+
};
|
|
3052
|
+
}
|
|
3053
|
+
function cleanupFromTombstone(tombstone) {
|
|
3054
|
+
return tombstone.cleanup;
|
|
3055
|
+
}
|
|
3056
|
+
function expiredRead(tombstone) {
|
|
3057
|
+
return {
|
|
3058
|
+
ok: false,
|
|
3059
|
+
code: "RUN_EXPIRED",
|
|
3060
|
+
message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`,
|
|
3061
|
+
tombstone
|
|
3062
|
+
};
|
|
3063
|
+
}
|
|
3064
|
+
//#endregion
|
|
1975
3065
|
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
1976
|
-
const
|
|
1977
|
-
const
|
|
3066
|
+
const STORAGE_BATCH_SIZE = 128;
|
|
3067
|
+
const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
|
|
3068
|
+
const HOOK_MARKER_PAGE_SIZE = 64;
|
|
3069
|
+
const TERMINAL_HOOK_PAGE_SIZE = 64;
|
|
3070
|
+
const TERMINAL_WAIT_PAGE_SIZE = STORAGE_BATCH_SIZE;
|
|
3071
|
+
const STREAMS_PER_CLEANUP_PAGE = 16;
|
|
3072
|
+
const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
3073
|
+
const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
|
|
3074
|
+
const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
|
|
1978
3075
|
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
3076
|
+
const CLEANUP_PROGRESS_KEY = "retention:progress";
|
|
1979
3077
|
/**
|
|
1980
3078
|
* Adapt DO storage (or a transaction) to the {@link EventStore} interface
|
|
1981
3079
|
* shared with the test mocks. No casts: the methods line up structurally,
|
|
@@ -1984,11 +3082,21 @@ const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
|
1984
3082
|
function storeFrom(storage) {
|
|
1985
3083
|
return {
|
|
1986
3084
|
get: (key) => storage.get(key),
|
|
3085
|
+
getMany: (keys) => storage.get(keys),
|
|
1987
3086
|
put: (key, value) => storage.put(key, value),
|
|
1988
3087
|
delete: (key) => storage.delete(key),
|
|
1989
|
-
|
|
3088
|
+
deleteMany: (keys) => storage.delete(keys),
|
|
3089
|
+
list: (options) => storage.list(options),
|
|
3090
|
+
deferTerminalCleanup: true
|
|
1990
3091
|
};
|
|
1991
3092
|
}
|
|
3093
|
+
function hookCreationIndexKey(hook) {
|
|
3094
|
+
return `${HOOK_CREATED_KEY_PREFIX}${hook.createdAt.toISOString()}:${hook.hookId}`;
|
|
3095
|
+
}
|
|
3096
|
+
async function putStorageEntries(storage, entries) {
|
|
3097
|
+
const all = Array.from(entries);
|
|
3098
|
+
for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE)));
|
|
3099
|
+
}
|
|
1992
3100
|
/**
|
|
1993
3101
|
* Durable Object for managing a single workflow run's state.
|
|
1994
3102
|
*
|
|
@@ -2004,26 +3112,51 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2004
3112
|
const clock = this.env?.clock;
|
|
2005
3113
|
return typeof clock === "function" ? clock() : Date.now();
|
|
2006
3114
|
}
|
|
2007
|
-
|
|
2008
|
-
const tombstone =
|
|
3115
|
+
tombstoneFrom(values, now = this.now()) {
|
|
3116
|
+
const tombstone = values.get(TOMBSTONE_KEY);
|
|
2009
3117
|
if (tombstone) return tombstone;
|
|
2010
|
-
const cleanup =
|
|
3118
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
2011
3119
|
if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
|
|
2012
3120
|
return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
|
|
2013
3121
|
}
|
|
2014
|
-
async
|
|
2015
|
-
const
|
|
2016
|
-
if (tombstone) return expiredRead(tombstone);
|
|
3122
|
+
async retentionState(storage, now = this.now()) {
|
|
3123
|
+
const values = await storage.get([TOMBSTONE_KEY, CLEANUP_RECORD_KEY]);
|
|
2017
3124
|
return {
|
|
2018
|
-
|
|
2019
|
-
|
|
3125
|
+
cleanup: values.get(CLEANUP_RECORD_KEY),
|
|
3126
|
+
tombstone: this.tombstoneFrom(values, now)
|
|
2020
3127
|
};
|
|
2021
3128
|
}
|
|
3129
|
+
async read(reader) {
|
|
3130
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3131
|
+
const { tombstone } = await this.retentionState(txn);
|
|
3132
|
+
if (tombstone) return expiredRead(tombstone);
|
|
3133
|
+
return {
|
|
3134
|
+
ok: true,
|
|
3135
|
+
value: await reader(txn)
|
|
3136
|
+
};
|
|
3137
|
+
});
|
|
3138
|
+
}
|
|
3139
|
+
async readKey(key) {
|
|
3140
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3141
|
+
const values = await txn.get([
|
|
3142
|
+
TOMBSTONE_KEY,
|
|
3143
|
+
CLEANUP_RECORD_KEY,
|
|
3144
|
+
key
|
|
3145
|
+
]);
|
|
3146
|
+
const tombstone = this.tombstoneFrom(values);
|
|
3147
|
+
if (tombstone) return expiredRead(tombstone);
|
|
3148
|
+
return {
|
|
3149
|
+
ok: true,
|
|
3150
|
+
value: values.get(key) ?? null
|
|
3151
|
+
};
|
|
3152
|
+
});
|
|
3153
|
+
}
|
|
2022
3154
|
/** Apply an event and capture all retention metadata in the same transaction. */
|
|
2023
3155
|
async applyEvent(request) {
|
|
2024
3156
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2025
3157
|
const now = new Date(this.now());
|
|
2026
|
-
const
|
|
3158
|
+
const retention = await this.retentionState(txn, now.getTime());
|
|
3159
|
+
const { tombstone } = retention;
|
|
2027
3160
|
if (tombstone) return {
|
|
2028
3161
|
ok: false,
|
|
2029
3162
|
code: "RUN_EXPIRED",
|
|
@@ -2037,20 +3170,12 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2037
3170
|
});
|
|
2038
3171
|
if (!outcome.ok) return outcome;
|
|
2039
3172
|
await txn.put("event_sequence", eventSequence);
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
}
|
|
2044
|
-
if (outcome.hookToIndex) {
|
|
2045
|
-
const reference = {
|
|
2046
|
-
hookId: outcome.hookToIndex.hookId,
|
|
2047
|
-
token: outcome.hookToIndex.token
|
|
2048
|
-
};
|
|
2049
|
-
await txn.put(hookMarkerKey(reference), reference);
|
|
2050
|
-
}
|
|
2051
|
-
for (const reference of outcome.releasedHooks) await txn.put(hookMarkerKey(reference), reference);
|
|
3173
|
+
await putStorageEntries(txn, (outcome.hookToIndex ? [{
|
|
3174
|
+
hookId: outcome.hookToIndex.hookId,
|
|
3175
|
+
token: outcome.hookToIndex.token
|
|
3176
|
+
}, ...outcome.releasedHooks] : outcome.releasedHooks).map((reference) => [hookMarkerKey(reference), reference]));
|
|
2052
3177
|
const retentionMs = request.cleanup?.retentionMs ?? 0;
|
|
2053
|
-
if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !
|
|
3178
|
+
if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !retention.cleanup) {
|
|
2054
3179
|
const terminalRun = outcome.run;
|
|
2055
3180
|
const dueAt = new Date(terminalRun.completedAt.getTime() + retentionMs);
|
|
2056
3181
|
const run = WorkflowRunSchema.parse({
|
|
@@ -2067,6 +3192,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2067
3192
|
dueAt,
|
|
2068
3193
|
queueShards: request.cleanup?.queueShards ?? 1,
|
|
2069
3194
|
phase: "retained",
|
|
3195
|
+
generation: 0,
|
|
2070
3196
|
attempts: 0,
|
|
2071
3197
|
deletedPayloadKeys: 0,
|
|
2072
3198
|
deletedStreams: 0,
|
|
@@ -2077,50 +3203,103 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2077
3203
|
await txn.setAlarm(dueAt);
|
|
2078
3204
|
outcome.run = run;
|
|
2079
3205
|
}
|
|
3206
|
+
if (outcome.run && isTerminalWorkflowRunStatus(outcome.run.status)) {
|
|
3207
|
+
if (!await txn.get("terminal:cleanup")) await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3208
|
+
version: 1,
|
|
3209
|
+
runId: outcome.run.runId,
|
|
3210
|
+
phase: "hooks",
|
|
3211
|
+
generation: 0,
|
|
3212
|
+
attempts: 0
|
|
3213
|
+
});
|
|
3214
|
+
await txn.setAlarm(this.now() + 1);
|
|
3215
|
+
}
|
|
3216
|
+
if (outcome.run) outcome.indexPublicationExpiresAt = now.getTime() + MAX_RUN_INDEX_PUBLICATION_LIFETIME_MS;
|
|
2080
3217
|
return finalizeEventPage(storeFrom(txn), outcome, request.params);
|
|
2081
3218
|
});
|
|
2082
3219
|
}
|
|
3220
|
+
/**
|
|
3221
|
+
* Resolve an ambiguous hook-creation RPC in the same serialization domain as
|
|
3222
|
+
* applyEvent. If the hook is absent, the exact reservation is durably fenced
|
|
3223
|
+
* before the caller releases its token and hook-ID claims.
|
|
3224
|
+
*/
|
|
3225
|
+
async resolveHookTokenClaim(request) {
|
|
3226
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3227
|
+
const { tombstone } = await this.retentionState(txn);
|
|
3228
|
+
if (tombstone) return { committed: false };
|
|
3229
|
+
if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
|
|
3230
|
+
await txn.put(hookClaimCancellationKey(request.claimId), {
|
|
3231
|
+
hookId: request.hookId,
|
|
3232
|
+
token: request.token,
|
|
3233
|
+
canceledAt: this.now()
|
|
3234
|
+
});
|
|
3235
|
+
return { committed: false };
|
|
3236
|
+
});
|
|
3237
|
+
}
|
|
2083
3238
|
async getRun() {
|
|
2084
|
-
return this.
|
|
3239
|
+
return this.readKey("run");
|
|
3240
|
+
}
|
|
3241
|
+
/** Single authoritative lifecycle answer used after derivative compaction. */
|
|
3242
|
+
async getLifecycleStatus() {
|
|
3243
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
3244
|
+
const values = await txn.get([
|
|
3245
|
+
TOMBSTONE_KEY,
|
|
3246
|
+
CLEANUP_RECORD_KEY,
|
|
3247
|
+
"run"
|
|
3248
|
+
]);
|
|
3249
|
+
if (this.tombstoneFrom(values)) return "expired";
|
|
3250
|
+
const run = values.get("run");
|
|
3251
|
+
if (!run) return "missing";
|
|
3252
|
+
return isTerminalWorkflowRunStatus(run.status) ? "terminal" : "active";
|
|
3253
|
+
});
|
|
2085
3254
|
}
|
|
2086
3255
|
async getStep(stepId) {
|
|
2087
|
-
return this.
|
|
3256
|
+
return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
|
|
2088
3257
|
}
|
|
2089
3258
|
async listSteps(params) {
|
|
2090
|
-
return this.read(() => listByCreationTime(storeFrom(
|
|
3259
|
+
return this.read((txn) => listByCreationTime(storeFrom(txn), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
|
|
2091
3260
|
limit: params?.limit ?? 20,
|
|
2092
3261
|
cursor: params?.cursor,
|
|
2093
3262
|
sortOrder: params?.sortOrder ?? "asc"
|
|
2094
3263
|
}));
|
|
2095
3264
|
}
|
|
2096
3265
|
async getEvent(eventId) {
|
|
2097
|
-
return this.
|
|
3266
|
+
return this.readKey(`${EVENT_KEY_PREFIX}${eventId}`);
|
|
2098
3267
|
}
|
|
2099
3268
|
async listEvents(params) {
|
|
2100
|
-
return this.read(() => listByPrefix(storeFrom(
|
|
3269
|
+
return this.read((txn) => listByPrefix(storeFrom(txn), EVENT_KEY_PREFIX, {
|
|
2101
3270
|
limit: params?.limit ?? 100,
|
|
2102
3271
|
cursor: params?.cursor,
|
|
2103
3272
|
sortOrder: params?.sortOrder ?? "asc"
|
|
2104
3273
|
}, (event) => event.eventId));
|
|
2105
3274
|
}
|
|
2106
3275
|
async listHooks(params) {
|
|
2107
|
-
return this.read(() => listByCreationTime(storeFrom(
|
|
3276
|
+
return this.read((txn) => listByCreationTime(storeFrom(txn), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
|
|
2108
3277
|
limit: params?.limit ?? 100,
|
|
2109
3278
|
cursor: params?.cursor,
|
|
2110
3279
|
sortOrder: params?.sortOrder ?? "asc"
|
|
2111
3280
|
}));
|
|
2112
3281
|
}
|
|
2113
3282
|
async getCleanupStatus() {
|
|
2114
|
-
|
|
3283
|
+
const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TOMBSTONE_KEY]);
|
|
3284
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3285
|
+
if (cleanup) return cleanup;
|
|
3286
|
+
const tombstone = values.get(TOMBSTONE_KEY);
|
|
3287
|
+
return tombstone ? cleanupFromTombstone(tombstone) : null;
|
|
2115
3288
|
}
|
|
2116
3289
|
/** Explicitly schedule an existing terminal run, used for opt-in backfills. */
|
|
2117
3290
|
async scheduleCleanup(request) {
|
|
2118
3291
|
if (request.retentionMs <= 0) return this.getCleanupStatus();
|
|
2119
3292
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2120
|
-
const
|
|
3293
|
+
const values = await txn.get([
|
|
3294
|
+
CLEANUP_RECORD_KEY,
|
|
3295
|
+
TOMBSTONE_KEY,
|
|
3296
|
+
TERMINAL_CLEANUP_KEY,
|
|
3297
|
+
"run"
|
|
3298
|
+
]);
|
|
3299
|
+
const existing = values.get(CLEANUP_RECORD_KEY);
|
|
2121
3300
|
if (existing) return existing;
|
|
2122
|
-
if (
|
|
2123
|
-
const current =
|
|
3301
|
+
if (values.get("retention:tombstone")) return null;
|
|
3302
|
+
const current = values.get("run");
|
|
2124
3303
|
if (!current || !isTerminalWorkflowRunStatus(current.status)) return null;
|
|
2125
3304
|
const terminalRun = current;
|
|
2126
3305
|
const dueAt = new Date(terminalRun.completedAt.getTime() + request.retentionMs);
|
|
@@ -2138,6 +3317,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2138
3317
|
dueAt,
|
|
2139
3318
|
queueShards: request.queueShards,
|
|
2140
3319
|
phase: "retained",
|
|
3320
|
+
generation: 0,
|
|
2141
3321
|
attempts: 0,
|
|
2142
3322
|
deletedPayloadKeys: 0,
|
|
2143
3323
|
deletedStreams: 0,
|
|
@@ -2145,7 +3325,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2145
3325
|
};
|
|
2146
3326
|
await txn.put("run", run);
|
|
2147
3327
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
2148
|
-
await txn.setAlarm(dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
3328
|
+
await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
2149
3329
|
return cleanup;
|
|
2150
3330
|
});
|
|
2151
3331
|
}
|
|
@@ -2165,159 +3345,388 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2165
3345
|
}));
|
|
2166
3346
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2167
3347
|
...cleanup,
|
|
2168
|
-
dueAt
|
|
3348
|
+
dueAt,
|
|
3349
|
+
generation: cleanup.generation + 1
|
|
2169
3350
|
});
|
|
2170
3351
|
await txn.setAlarm(this.now() + 1);
|
|
2171
3352
|
});
|
|
2172
|
-
|
|
2173
|
-
await this.executeCleanup();
|
|
2174
|
-
} catch (error) {
|
|
2175
|
-
await this.recordCleanupFailure(error);
|
|
2176
|
-
}
|
|
3353
|
+
await this.executeScheduledCleanupPage();
|
|
2177
3354
|
return this.getCleanupStatus();
|
|
2178
3355
|
}
|
|
2179
3356
|
async rearmCleanup() {
|
|
2180
|
-
const
|
|
2181
|
-
|
|
2182
|
-
if (
|
|
3357
|
+
const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TERMINAL_CLEANUP_KEY]);
|
|
3358
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3359
|
+
if (values.get("terminal:cleanup")) await this.ctx.storage.setAlarm(this.now() + 1);
|
|
3360
|
+
else if (!cleanup || cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
|
|
2183
3361
|
else await this.ctx.storage.setAlarm(Math.max(this.now() + 1, cleanup.dueAt.getTime()));
|
|
2184
|
-
return cleanup;
|
|
3362
|
+
return cleanup ?? null;
|
|
2185
3363
|
}
|
|
2186
3364
|
async alarm() {
|
|
2187
|
-
|
|
2188
|
-
await this.executeCleanup();
|
|
2189
|
-
} catch (error) {
|
|
2190
|
-
await this.recordCleanupFailure(error);
|
|
2191
|
-
}
|
|
3365
|
+
await this.executeScheduledCleanupPage();
|
|
2192
3366
|
}
|
|
2193
3367
|
namespace(name) {
|
|
2194
3368
|
const namespace = this.env?.[name];
|
|
2195
3369
|
if (!namespace || typeof namespace === "function") throw new Error(`world-celld retention missing binding ${name}`);
|
|
2196
3370
|
return namespace;
|
|
2197
3371
|
}
|
|
2198
|
-
|
|
3372
|
+
workflowIndex() {
|
|
3373
|
+
return createWorkflowIndex({
|
|
3374
|
+
runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
|
|
3375
|
+
hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
|
|
3376
|
+
hookIds: this.namespace("WORKFLOW_HOOK_IDS")
|
|
3377
|
+
});
|
|
3378
|
+
}
|
|
3379
|
+
async setCleanupPhase(expected, next, updates = {}, continueAt) {
|
|
2199
3380
|
await this.ctx.storage.transaction(async (txn) => {
|
|
2200
3381
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2201
|
-
if (!cleanup || cleanup.phase !== expected) return;
|
|
3382
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
|
|
2202
3383
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2203
3384
|
...cleanup,
|
|
2204
3385
|
...updates,
|
|
2205
3386
|
phase: next,
|
|
3387
|
+
generation: cleanup.generation + 1,
|
|
2206
3388
|
attempts: 0,
|
|
2207
3389
|
lastError: void 0
|
|
2208
3390
|
});
|
|
3391
|
+
if (continueAt !== void 0) await txn.setAlarm(continueAt);
|
|
2209
3392
|
});
|
|
2210
3393
|
}
|
|
2211
|
-
async
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
await this.
|
|
2216
|
-
|
|
3394
|
+
async executeScheduledCleanupPage() {
|
|
3395
|
+
const terminalCleanup = await this.ctx.storage.get(TERMINAL_CLEANUP_KEY);
|
|
3396
|
+
if (terminalCleanup) {
|
|
3397
|
+
try {
|
|
3398
|
+
await this.executeTerminalCleanup(terminalCleanup);
|
|
3399
|
+
} catch (error) {
|
|
3400
|
+
await this.recordTerminalCleanupFailure(error, terminalCleanup);
|
|
2217
3401
|
}
|
|
2218
|
-
|
|
2219
|
-
|
|
3402
|
+
return;
|
|
3403
|
+
}
|
|
3404
|
+
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
3405
|
+
if (!cleanup || cleanup.phase === "tombstoned") {
|
|
3406
|
+
await this.ctx.storage.deleteAlarm();
|
|
3407
|
+
return;
|
|
3408
|
+
}
|
|
3409
|
+
try {
|
|
3410
|
+
await this.executeCleanup(cleanup);
|
|
3411
|
+
} catch (error) {
|
|
3412
|
+
await this.recordCleanupFailure(error, cleanup);
|
|
3413
|
+
}
|
|
3414
|
+
}
|
|
3415
|
+
async executeCleanup(cleanup) {
|
|
3416
|
+
if (cleanup.phase === "retained") {
|
|
3417
|
+
if (cleanup.dueAt.getTime() > this.now()) {
|
|
3418
|
+
await this.ctx.storage.setAlarm(cleanup.dueAt);
|
|
2220
3419
|
return;
|
|
2221
3420
|
}
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
3421
|
+
await this.setCleanupPhase(cleanup, "index", {}, this.now() + 1);
|
|
3422
|
+
return;
|
|
3423
|
+
}
|
|
3424
|
+
if (cleanup.phase === "index") {
|
|
3425
|
+
await this.deleteIndexPage(cleanup);
|
|
3426
|
+
return;
|
|
3427
|
+
}
|
|
3428
|
+
if (cleanup.phase === "streams") {
|
|
3429
|
+
await this.deleteStreamPage(cleanup);
|
|
3430
|
+
return;
|
|
3431
|
+
}
|
|
3432
|
+
if (cleanup.phase === "queues") {
|
|
3433
|
+
await this.deleteQueuePage(cleanup);
|
|
3434
|
+
return;
|
|
3435
|
+
}
|
|
3436
|
+
if (cleanup.phase === "payload") await this.deletePayloadBatch();
|
|
3437
|
+
}
|
|
3438
|
+
async executeTerminalCleanup(cleanup) {
|
|
3439
|
+
if (cleanup.phase === "hooks") await this.deleteTerminalHookPage(cleanup);
|
|
3440
|
+
else if (cleanup.phase === "markers") await this.deleteTerminalHookMarkerPage(cleanup);
|
|
3441
|
+
else await this.deleteTerminalWaitPage(cleanup);
|
|
3442
|
+
}
|
|
3443
|
+
async deleteTerminalHookPage(cleanup) {
|
|
3444
|
+
const entries = await this.ctx.storage.list({
|
|
3445
|
+
prefix: HOOK_KEY_PREFIX,
|
|
3446
|
+
limit: 65
|
|
3447
|
+
});
|
|
3448
|
+
const page = Array.from(entries.values()).slice(0, TERMINAL_HOOK_PAGE_SIZE);
|
|
3449
|
+
await this.workflowIndex().releaseHookIndexes({
|
|
3450
|
+
runId: cleanup.runId,
|
|
3451
|
+
hooks: page.map((hook) => ({
|
|
3452
|
+
hookId: hook.hookId,
|
|
3453
|
+
token: hook.token
|
|
3454
|
+
}))
|
|
3455
|
+
});
|
|
3456
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3457
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3458
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3459
|
+
const keys = page.flatMap((hook) => [`${HOOK_KEY_PREFIX}${hook.hookId}`, hookCreationIndexKey(hook)]);
|
|
3460
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3461
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3462
|
+
...current,
|
|
3463
|
+
phase: entries.size <= TERMINAL_HOOK_PAGE_SIZE ? "markers" : "hooks",
|
|
3464
|
+
generation: current.generation + 1,
|
|
3465
|
+
attempts: 0,
|
|
3466
|
+
lastError: void 0
|
|
3467
|
+
});
|
|
3468
|
+
await txn.setAlarm(this.now() + 1);
|
|
3469
|
+
});
|
|
3470
|
+
}
|
|
3471
|
+
async deleteTerminalHookMarkerPage(cleanup) {
|
|
3472
|
+
const entries = await this.ctx.storage.list({
|
|
3473
|
+
prefix: HOOK_MARKER_PREFIX,
|
|
3474
|
+
limit: 65
|
|
3475
|
+
});
|
|
3476
|
+
const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
|
|
3477
|
+
await this.workflowIndex().releaseHookIndexes({
|
|
3478
|
+
runId: cleanup.runId,
|
|
3479
|
+
hooks: page.map(([, reference]) => reference)
|
|
3480
|
+
});
|
|
3481
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3482
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3483
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3484
|
+
const keys = page.map(([key]) => key);
|
|
3485
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3486
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3487
|
+
...current,
|
|
3488
|
+
phase: entries.size <= HOOK_MARKER_PAGE_SIZE ? "waits" : "markers",
|
|
3489
|
+
generation: current.generation + 1,
|
|
3490
|
+
attempts: 0,
|
|
3491
|
+
lastError: void 0
|
|
3492
|
+
});
|
|
3493
|
+
await txn.setAlarm(this.now() + 1);
|
|
3494
|
+
});
|
|
3495
|
+
}
|
|
3496
|
+
async deleteTerminalWaitPage(cleanup) {
|
|
3497
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3498
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3499
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3500
|
+
const entries = await txn.list({
|
|
3501
|
+
prefix: WAIT_KEY_PREFIX,
|
|
3502
|
+
limit: 129
|
|
3503
|
+
});
|
|
3504
|
+
const keys = Array.from(entries.keys()).slice(0, TERMINAL_WAIT_PAGE_SIZE);
|
|
3505
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3506
|
+
if (entries.size > TERMINAL_WAIT_PAGE_SIZE) {
|
|
3507
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3508
|
+
...current,
|
|
3509
|
+
generation: current.generation + 1,
|
|
3510
|
+
attempts: 0,
|
|
3511
|
+
lastError: void 0
|
|
2242
3512
|
});
|
|
2243
|
-
await this.
|
|
2244
|
-
|
|
3513
|
+
await txn.setAlarm(this.now() + 1);
|
|
3514
|
+
return;
|
|
2245
3515
|
}
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
3516
|
+
await txn.delete(TERMINAL_CLEANUP_KEY);
|
|
3517
|
+
const retention = await txn.get(CLEANUP_RECORD_KEY);
|
|
3518
|
+
if (retention && retention.phase !== "tombstoned") await txn.setAlarm(Math.max(this.now() + 1, retention.dueAt.getTime()));
|
|
3519
|
+
else await txn.deleteAlarm();
|
|
3520
|
+
});
|
|
3521
|
+
}
|
|
3522
|
+
async deleteIndexPage(cleanup) {
|
|
3523
|
+
const hookEntries = await this.ctx.storage.list({
|
|
3524
|
+
prefix: HOOK_MARKER_PREFIX,
|
|
3525
|
+
limit: 65
|
|
3526
|
+
});
|
|
3527
|
+
const hookMarkers = Array.from(hookEntries).slice(0, HOOK_MARKER_PAGE_SIZE);
|
|
3528
|
+
await this.workflowIndex().expireRun({
|
|
3529
|
+
runId: cleanup.runId,
|
|
3530
|
+
keys: [workflowRunIndexKey(cleanup), globalRunIndexKey(cleanup)],
|
|
3531
|
+
hooks: hookMarkers.map(([, value]) => value),
|
|
3532
|
+
expiredAt: cleanup.dueAt.getTime()
|
|
3533
|
+
});
|
|
3534
|
+
const markerKeys = hookMarkers.map(([key]) => key);
|
|
3535
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3536
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3537
|
+
if (current?.phase !== "index" || current.generation !== cleanup.generation) return;
|
|
3538
|
+
if (markerKeys.length > 0) await txn.delete(markerKeys);
|
|
3539
|
+
if (hookEntries.size <= HOOK_MARKER_PAGE_SIZE) await txn.put(CLEANUP_RECORD_KEY, {
|
|
3540
|
+
...current,
|
|
3541
|
+
phase: "streams",
|
|
3542
|
+
generation: current.generation + 1,
|
|
3543
|
+
attempts: 0,
|
|
3544
|
+
lastError: void 0
|
|
3545
|
+
});
|
|
3546
|
+
else await txn.put(CLEANUP_RECORD_KEY, {
|
|
3547
|
+
...current,
|
|
3548
|
+
generation: current.generation + 1,
|
|
3549
|
+
attempts: 0,
|
|
3550
|
+
lastError: void 0
|
|
3551
|
+
});
|
|
3552
|
+
await txn.setAlarm(this.now() + 1);
|
|
3553
|
+
});
|
|
3554
|
+
}
|
|
3555
|
+
async deleteStreamPage(cleanup) {
|
|
3556
|
+
const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
|
|
3557
|
+
const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
|
|
3558
|
+
const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime(), { limit: STREAMS_PER_CLEANUP_PAGE });
|
|
3559
|
+
const completed = [];
|
|
3560
|
+
let remainingChunks = STREAM_CHUNKS_PER_CLEANUP_PAGE;
|
|
3561
|
+
let remainingBytes = STREAM_BYTES_PER_CLEANUP_PAGE;
|
|
3562
|
+
for (const name of streams) {
|
|
3563
|
+
if (remainingChunks <= 0 || remainingBytes <= 0) break;
|
|
3564
|
+
const result = await streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime(), {
|
|
3565
|
+
limit: remainingChunks,
|
|
3566
|
+
byteLimit: remainingBytes
|
|
3567
|
+
});
|
|
3568
|
+
remainingChunks -= result.chunks;
|
|
3569
|
+
remainingBytes -= result.bytes;
|
|
3570
|
+
if (result.done) completed.push(name);
|
|
3571
|
+
}
|
|
3572
|
+
const registryResult = await registry.finalizeRegistry(cleanup.runId, completed);
|
|
3573
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3574
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3575
|
+
if (current?.phase !== "streams" || current.generation !== cleanup.generation) return;
|
|
3576
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3577
|
+
...current,
|
|
3578
|
+
deletedStreams: registryResult.deleted,
|
|
3579
|
+
phase: registryResult.done ? "queues" : "streams",
|
|
3580
|
+
generation: current.generation + 1,
|
|
3581
|
+
attempts: 0,
|
|
3582
|
+
lastError: void 0
|
|
3583
|
+
});
|
|
3584
|
+
await txn.setAlarm(this.now() + 1);
|
|
3585
|
+
});
|
|
3586
|
+
}
|
|
3587
|
+
async deleteQueuePage(cleanup) {
|
|
3588
|
+
const queues = this.namespace("WORKFLOW_QUEUE");
|
|
3589
|
+
const progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3590
|
+
queueShard: 0,
|
|
3591
|
+
queueShardDeleted: 0
|
|
3592
|
+
};
|
|
3593
|
+
if (progress.pendingAck) {
|
|
3594
|
+
const { queueShard, receipt } = progress.pendingAck;
|
|
3595
|
+
if (!(await queues.get(queues.idFromName(`q:${queueShard}`)).acknowledgeExpireRun(cleanup.runId, receipt)).acknowledged) throw new Error(`queue shard ${queueShard} rejected expiry receipt for ${cleanup.runId}`);
|
|
3596
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3597
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3598
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3599
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY);
|
|
3600
|
+
if (persisted?.pendingAck?.queueShard !== queueShard || persisted.pendingAck.receipt.expiredAt !== receipt.expiredAt || persisted.pendingAck.receipt.deleted !== receipt.deleted) return;
|
|
3601
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3602
|
+
...current,
|
|
3603
|
+
generation: current.generation + 1,
|
|
3604
|
+
attempts: 0,
|
|
3605
|
+
lastError: void 0
|
|
2264
3606
|
});
|
|
2265
|
-
await
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
3607
|
+
await txn.put(CLEANUP_PROGRESS_KEY, {
|
|
3608
|
+
queueShard: queueShard + 1,
|
|
3609
|
+
queueShardDeleted: 0
|
|
3610
|
+
});
|
|
3611
|
+
await txn.setAlarm(this.now() + 1);
|
|
3612
|
+
});
|
|
3613
|
+
return;
|
|
3614
|
+
}
|
|
3615
|
+
if (progress.queueShard < cleanup.queueShards) {
|
|
3616
|
+
const requestedShard = progress.queueShard;
|
|
3617
|
+
const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
|
|
3618
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3619
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3620
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3621
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3622
|
+
queueShard: 0,
|
|
3623
|
+
queueShardDeleted: 0
|
|
3624
|
+
};
|
|
3625
|
+
if (persisted.queueShard !== requestedShard || persisted.pendingAck) return;
|
|
3626
|
+
if (result.deleted < persisted.queueShardDeleted) throw new Error(`queue shard ${requestedShard} regressed expiry receipt for ${cleanup.runId}`);
|
|
3627
|
+
const delta = result.deleted - persisted.queueShardDeleted;
|
|
3628
|
+
const next = result.done ? {
|
|
3629
|
+
queueShard: requestedShard,
|
|
3630
|
+
queueShardDeleted: result.deleted,
|
|
3631
|
+
pendingAck: {
|
|
3632
|
+
queueShard: requestedShard,
|
|
3633
|
+
receipt: result.receipt
|
|
3634
|
+
}
|
|
3635
|
+
} : {
|
|
3636
|
+
queueShard: requestedShard,
|
|
3637
|
+
queueShardDeleted: result.deleted
|
|
3638
|
+
};
|
|
3639
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3640
|
+
...current,
|
|
3641
|
+
deletedQueueMessages: current.deletedQueueMessages + delta,
|
|
3642
|
+
generation: current.generation + 1,
|
|
3643
|
+
attempts: 0,
|
|
3644
|
+
lastError: void 0
|
|
3645
|
+
});
|
|
3646
|
+
await txn.put(CLEANUP_PROGRESS_KEY, next);
|
|
3647
|
+
await txn.setAlarm(this.now() + 1);
|
|
3648
|
+
});
|
|
3649
|
+
return;
|
|
2282
3650
|
}
|
|
3651
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3652
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3653
|
+
if (current?.phase !== "queues" || current.generation !== cleanup.generation) return;
|
|
3654
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3655
|
+
queueShard: 0,
|
|
3656
|
+
queueShardDeleted: 0
|
|
3657
|
+
};
|
|
3658
|
+
if (persisted.queueShard >= current.queueShards && !persisted.pendingAck) {
|
|
3659
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3660
|
+
...current,
|
|
3661
|
+
phase: "payload",
|
|
3662
|
+
generation: current.generation + 1
|
|
3663
|
+
});
|
|
3664
|
+
await txn.delete(CLEANUP_PROGRESS_KEY);
|
|
3665
|
+
await txn.setAlarm(this.now() + 1);
|
|
3666
|
+
}
|
|
3667
|
+
});
|
|
2283
3668
|
}
|
|
2284
3669
|
async deletePayloadBatch() {
|
|
2285
3670
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2286
3671
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2287
3672
|
if (!cleanup || cleanup.phase !== "payload") return true;
|
|
2288
|
-
const entries = await txn.list({
|
|
3673
|
+
const entries = await txn.list({
|
|
3674
|
+
limit: 131,
|
|
3675
|
+
noCache: true
|
|
3676
|
+
});
|
|
2289
3677
|
const keys = Array.from(entries.keys()).filter((key) => key !== "retention:cleanup" && key !== "retention:tombstone").slice(0, PAYLOAD_DELETE_BATCH);
|
|
2290
3678
|
const now = new Date(this.now());
|
|
2291
3679
|
await txn.put(TOMBSTONE_KEY, cleanupTombstone(cleanup, now));
|
|
2292
|
-
|
|
3680
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
2293
3681
|
if (keys.length > 0) {
|
|
2294
3682
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2295
3683
|
...cleanup,
|
|
2296
3684
|
deletedPayloadKeys: cleanup.deletedPayloadKeys + keys.length,
|
|
3685
|
+
generation: cleanup.generation + 1,
|
|
2297
3686
|
lastError: void 0
|
|
2298
3687
|
});
|
|
2299
3688
|
await txn.setAlarm(this.now() + 1);
|
|
2300
3689
|
return false;
|
|
2301
3690
|
}
|
|
2302
|
-
|
|
3691
|
+
const completed = {
|
|
2303
3692
|
...cleanup,
|
|
2304
3693
|
phase: "tombstoned",
|
|
3694
|
+
generation: cleanup.generation + 1,
|
|
2305
3695
|
attempts: 0,
|
|
2306
3696
|
lastError: void 0,
|
|
2307
3697
|
tombstonedAt: now
|
|
2308
|
-
}
|
|
3698
|
+
};
|
|
3699
|
+
await txn.put(TOMBSTONE_KEY, cleanupTombstone(completed, now));
|
|
3700
|
+
await txn.delete(CLEANUP_RECORD_KEY);
|
|
2309
3701
|
await txn.deleteAlarm();
|
|
2310
3702
|
return true;
|
|
2311
3703
|
});
|
|
2312
3704
|
}
|
|
2313
|
-
async recordCleanupFailure(error) {
|
|
3705
|
+
async recordCleanupFailure(error, expected) {
|
|
2314
3706
|
await this.ctx.storage.transaction(async (txn) => {
|
|
2315
3707
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2316
|
-
if (!cleanup || cleanup.phase === "tombstoned") return;
|
|
3708
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation || cleanup.phase === "tombstoned") return;
|
|
2317
3709
|
const attempts = cleanup.attempts + 1;
|
|
2318
3710
|
const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
|
|
2319
3711
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2320
3712
|
...cleanup,
|
|
3713
|
+
generation: cleanup.generation + 1,
|
|
3714
|
+
attempts,
|
|
3715
|
+
lastError: error instanceof Error ? error.message : String(error),
|
|
3716
|
+
lastAttemptAt: new Date(this.now())
|
|
3717
|
+
});
|
|
3718
|
+
await txn.setAlarm(this.now() + delay);
|
|
3719
|
+
});
|
|
3720
|
+
}
|
|
3721
|
+
async recordTerminalCleanupFailure(error, expected) {
|
|
3722
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3723
|
+
const cleanup = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3724
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
|
|
3725
|
+
const attempts = cleanup.attempts + 1;
|
|
3726
|
+
const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
|
|
3727
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3728
|
+
...cleanup,
|
|
3729
|
+
generation: cleanup.generation + 1,
|
|
2321
3730
|
attempts,
|
|
2322
3731
|
lastError: error instanceof Error ? error.message : String(error),
|
|
2323
3732
|
lastAttemptAt: new Date(this.now())
|
|
@@ -2352,7 +3761,9 @@ async function startHarness(options = {}) {
|
|
|
2352
3761
|
const fleet = new FakeFleet({
|
|
2353
3762
|
runs: WorkflowRunDO,
|
|
2354
3763
|
streams: StreamDO,
|
|
2355
|
-
|
|
3764
|
+
"run-catalog": RunCatalogDO,
|
|
3765
|
+
"hook-tokens": HookTokenDO,
|
|
3766
|
+
"hook-ids": HookIdDO,
|
|
2356
3767
|
queue: QueueDO,
|
|
2357
3768
|
...options.extraClasses
|
|
2358
3769
|
}, cellEnv);
|
|
@@ -2360,17 +3771,30 @@ async function startHarness(options = {}) {
|
|
|
2360
3771
|
Object.assign(cellEnv, {
|
|
2361
3772
|
WORKFLOW_DB: fleet.namespace("runs"),
|
|
2362
3773
|
WORKFLOW_STREAMS: fleet.namespace("streams"),
|
|
2363
|
-
|
|
3774
|
+
WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
|
|
3775
|
+
WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
|
|
3776
|
+
WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
|
|
2364
3777
|
WORKFLOW_QUEUE: fleet.namespace("queue")
|
|
2365
3778
|
});
|
|
2366
3779
|
const router = createRouter({
|
|
2367
3780
|
WORKFLOW_DB: fleet.namespace("runs"),
|
|
2368
3781
|
WORKFLOW_STREAMS: fleet.namespace("streams"),
|
|
2369
|
-
|
|
3782
|
+
WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
|
|
3783
|
+
WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
|
|
3784
|
+
WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
|
|
2370
3785
|
WORKFLOW_QUEUE: fleet.namespace("queue"),
|
|
2371
3786
|
WORLD_SECRET: options.secret
|
|
2372
3787
|
});
|
|
2373
3788
|
const server = http.createServer(async (req, res) => {
|
|
3789
|
+
const requestAbort = new AbortController();
|
|
3790
|
+
const abortRequest = () => {
|
|
3791
|
+
if (!requestAbort.signal.aborted) requestAbort.abort(new DOMException("HTTP client disconnected", "AbortError"));
|
|
3792
|
+
};
|
|
3793
|
+
const abortOnPrematureClose = () => {
|
|
3794
|
+
if (!res.writableEnded) abortRequest();
|
|
3795
|
+
};
|
|
3796
|
+
req.once("aborted", abortRequest);
|
|
3797
|
+
res.once("close", abortOnPrematureClose);
|
|
2374
3798
|
try {
|
|
2375
3799
|
const chunks = [];
|
|
2376
3800
|
for await (const chunk of req) chunks.push(chunk);
|
|
@@ -2378,17 +3802,23 @@ async function startHarness(options = {}) {
|
|
|
2378
3802
|
const request = new Request(`http://127.0.0.1${req.url ?? "/"}`, {
|
|
2379
3803
|
method: req.method,
|
|
2380
3804
|
headers: req.headers,
|
|
2381
|
-
body: req.method === "GET" || req.method === "HEAD" || body.length === 0 ? null : body
|
|
3805
|
+
body: req.method === "GET" || req.method === "HEAD" || body.length === 0 ? null : body,
|
|
3806
|
+
signal: requestAbort.signal
|
|
2382
3807
|
});
|
|
2383
3808
|
const response = await router(request);
|
|
3809
|
+
if (requestAbort.signal.aborted || res.destroyed) return;
|
|
2384
3810
|
res.writeHead(response.status, Object.fromEntries(response.headers.entries()));
|
|
2385
3811
|
res.end(Buffer.from(await response.arrayBuffer()));
|
|
2386
3812
|
} catch (error) {
|
|
3813
|
+
if (requestAbort.signal.aborted || res.destroyed) return;
|
|
2387
3814
|
res.writeHead(500, { "content-type": "application/json" });
|
|
2388
3815
|
res.end(JSON.stringify({ error: {
|
|
2389
3816
|
name: "HarnessError",
|
|
2390
3817
|
message: String(error)
|
|
2391
3818
|
} }));
|
|
3819
|
+
} finally {
|
|
3820
|
+
req.off("aborted", abortRequest);
|
|
3821
|
+
res.off("close", abortOnPrematureClose);
|
|
2392
3822
|
}
|
|
2393
3823
|
});
|
|
2394
3824
|
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|