@ewhauser/world-celld 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +47 -19
- package/celld-worker/README.md +5 -3
- package/celld-worker/worker.ts +10 -1
- package/celld-worker/wrangler.jsonc +15 -4
- package/dist/index.d.ts +163 -60
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +524 -197
- package/dist/index.js.map +1 -1
- package/dist/testing.d.ts +47 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +1874 -582
- package/dist/testing.js.map +1 -1
- package/dist/util-DmO0XkFw.js +545 -0
- package/dist/util-DmO0XkFw.js.map +1 -0
- package/dist/worker.d.ts +223 -111
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +1960 -551
- package/dist/worker.js.map +1 -1
- package/package.json +7 -5
- package/dist/retention-bLY1vcOu.js +0 -164
- package/dist/retention-bLY1vcOu.js.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WorkflowRunSchema,
|
|
1
|
+
import { C as validateStreamReadRequest, S as normalizeStreamError, _ as decodeStreamWriteBatch, c as MAX_STREAM_BATCH_BYTES, d as MAX_STREAM_LONG_POLL_MS, f as MAX_STREAM_READ_BYTES, h as STREAM_BATCH_CONTENT_TYPE, i as parse, l as MAX_STREAM_CHUNK_BYTES, n as rpcParse, o as createWorkflowIndex, r as rpcStringify, s as runFenceCellName, t as compact, w as validateStreamWriteChunks, x as encodeStreamWriteResult, y as encodeStreamReadResult } from "./util-DmO0XkFw.js";
|
|
2
|
+
import { EventSchema, HookSchema, SPEC_VERSION_CURRENT, StepSchema, WaitSchema, WorkflowRunSchema, applyAttributeChanges, eventIdToSlot, getMaxEventsPerRun, isChildEntityCreationEvent, isTerminalStepStatus, isTerminalWorkflowRunStatus, slotToEventId, validateAttributeChanges } from "@workflow/world";
|
|
3
3
|
import http from "node:http";
|
|
4
4
|
//#region src/testing/fake-cell.ts
|
|
5
5
|
function applyListOptions(keys, options) {
|
|
@@ -33,40 +33,109 @@ var FakeStorage = class {
|
|
|
33
33
|
clock;
|
|
34
34
|
data = /* @__PURE__ */ new Map();
|
|
35
35
|
alarmAt = null;
|
|
36
|
+
getManyCalls = [];
|
|
37
|
+
operationCounts = {
|
|
38
|
+
get: 0,
|
|
39
|
+
getMany: 0,
|
|
40
|
+
list: 0,
|
|
41
|
+
put: 0,
|
|
42
|
+
putMany: 0,
|
|
43
|
+
delete: 0,
|
|
44
|
+
deleteMany: 0,
|
|
45
|
+
transaction: 0
|
|
46
|
+
};
|
|
47
|
+
transactionTail = Promise.resolve();
|
|
48
|
+
transactionDelayMs = 0;
|
|
49
|
+
/** Deterministic query-shape instrumentation for storage scaling tests. */
|
|
50
|
+
listCalls = [];
|
|
51
|
+
operationCalls = [];
|
|
36
52
|
mutationFailure;
|
|
37
53
|
constructor(clock = () => Date.now()) {
|
|
38
54
|
this.clock = clock;
|
|
39
55
|
}
|
|
40
|
-
async get(
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
56
|
+
async get(keyOrKeys) {
|
|
57
|
+
if (Array.isArray(keyOrKeys)) {
|
|
58
|
+
this.operationCounts.getMany += 1;
|
|
59
|
+
this.getManyCalls.push([...keyOrKeys]);
|
|
60
|
+
this.recordStorageCall("get", keyOrKeys, false);
|
|
61
|
+
return new Map(keyOrKeys.filter((key) => this.data.has(key)).map((key) => [key, this.data.get(key)]));
|
|
62
|
+
}
|
|
63
|
+
this.operationCounts.get += 1;
|
|
64
|
+
this.recordStorageCall("get", [keyOrKeys], false);
|
|
65
|
+
return this.data.get(keyOrKeys);
|
|
66
|
+
}
|
|
67
|
+
async put(keyOrEntries, value) {
|
|
68
|
+
if (typeof keyOrEntries === "string") {
|
|
69
|
+
this.operationCounts.put += 1;
|
|
70
|
+
this.maybeFailMutation({
|
|
71
|
+
operation: "put",
|
|
72
|
+
key: keyOrEntries,
|
|
73
|
+
value
|
|
74
|
+
});
|
|
75
|
+
this.recordStorageCall("put", [keyOrEntries], false);
|
|
76
|
+
this.data.set(keyOrEntries, structuredClone(value));
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
this.operationCounts.putMany += 1;
|
|
80
|
+
for (const [key, entryValue] of Object.entries(keyOrEntries)) this.maybeFailMutation({
|
|
45
81
|
operation: "put",
|
|
46
82
|
key,
|
|
47
|
-
value
|
|
83
|
+
value: entryValue
|
|
48
84
|
});
|
|
49
|
-
this.
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
85
|
+
this.recordStorageCall("put", Object.keys(keyOrEntries), false);
|
|
86
|
+
for (const [key, entryValue] of Object.entries(keyOrEntries)) this.data.set(key, structuredClone(entryValue));
|
|
87
|
+
}
|
|
88
|
+
async delete(keyOrKeys) {
|
|
89
|
+
if (typeof keyOrKeys === "string") {
|
|
90
|
+
this.operationCounts.delete += 1;
|
|
91
|
+
this.maybeFailMutation({
|
|
92
|
+
operation: "delete",
|
|
93
|
+
key: keyOrKeys
|
|
94
|
+
});
|
|
95
|
+
this.recordStorageCall("delete", [keyOrKeys], false);
|
|
96
|
+
return this.data.delete(keyOrKeys);
|
|
97
|
+
}
|
|
98
|
+
this.operationCounts.deleteMany += 1;
|
|
99
|
+
for (const key of keyOrKeys) this.maybeFailMutation({
|
|
53
100
|
operation: "delete",
|
|
54
101
|
key
|
|
55
102
|
});
|
|
56
|
-
|
|
103
|
+
this.recordStorageCall("delete", keyOrKeys, false);
|
|
104
|
+
let deleted = 0;
|
|
105
|
+
for (const key of keyOrKeys) if (this.data.delete(key)) deleted += 1;
|
|
106
|
+
return deleted;
|
|
57
107
|
}
|
|
58
108
|
async deleteAll() {
|
|
59
109
|
this.data.clear();
|
|
60
110
|
}
|
|
61
111
|
async list(options = {}) {
|
|
112
|
+
this.operationCounts.list += 1;
|
|
62
113
|
const keys = applyListOptions(Array.from(this.data.keys()), options);
|
|
114
|
+
this.recordList(options, keys.length, false);
|
|
63
115
|
return new Map(keys.map((k) => [k, this.data.get(k)]));
|
|
64
116
|
}
|
|
65
117
|
async transaction(cb) {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
118
|
+
this.operationCounts.transaction += 1;
|
|
119
|
+
let release;
|
|
120
|
+
const previous = this.transactionTail;
|
|
121
|
+
this.transactionTail = new Promise((resolve) => {
|
|
122
|
+
release = resolve;
|
|
123
|
+
});
|
|
124
|
+
await previous;
|
|
125
|
+
try {
|
|
126
|
+
if (this.transactionDelayMs > 0) await new Promise((resolve) => setTimeout(resolve, this.transactionDelayMs));
|
|
127
|
+
const txn = new FakeTransaction(this);
|
|
128
|
+
const result = await cb(txn);
|
|
129
|
+
txn.commit();
|
|
130
|
+
return result;
|
|
131
|
+
} finally {
|
|
132
|
+
release();
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
/** Add deterministic per-transaction work for contention benchmarks. */
|
|
136
|
+
setTransactionDelay(ms) {
|
|
137
|
+
if (!Number.isFinite(ms) || ms < 0) throw new Error("transaction delay must be non-negative");
|
|
138
|
+
this.transactionDelayMs = ms;
|
|
70
139
|
}
|
|
71
140
|
async getAlarm() {
|
|
72
141
|
return this.alarmAt;
|
|
@@ -80,6 +149,24 @@ var FakeStorage = class {
|
|
|
80
149
|
now() {
|
|
81
150
|
return this.clock();
|
|
82
151
|
}
|
|
152
|
+
resetOperationCounts() {
|
|
153
|
+
for (const key of Object.keys(this.operationCounts)) this.operationCounts[key] = 0;
|
|
154
|
+
this.getManyCalls.length = 0;
|
|
155
|
+
this.operationCalls.length = 0;
|
|
156
|
+
}
|
|
157
|
+
/** @internal Used by FakeTransaction to count the same platform operations. */
|
|
158
|
+
recordOperation(operation) {
|
|
159
|
+
this.operationCounts[operation] += 1;
|
|
160
|
+
}
|
|
161
|
+
/** @internal Shared with FakeTransaction for deterministic key tracing. */
|
|
162
|
+
recordStorageCall(operation, keys, transactional) {
|
|
163
|
+
this.operationCalls.push({
|
|
164
|
+
operation,
|
|
165
|
+
keys: [...keys],
|
|
166
|
+
transactional,
|
|
167
|
+
synchronous: false
|
|
168
|
+
});
|
|
169
|
+
}
|
|
83
170
|
/** Inject one matching write failure, including writes inside transactions. */
|
|
84
171
|
failNextMutation(predicate, error = /* @__PURE__ */ new Error("injected fake storage mutation failure")) {
|
|
85
172
|
this.mutationFailure = {
|
|
@@ -94,6 +181,14 @@ var FakeStorage = class {
|
|
|
94
181
|
this.mutationFailure = void 0;
|
|
95
182
|
throw error;
|
|
96
183
|
}
|
|
184
|
+
/** @internal Shared with FakeTransaction for deterministic query tracing. */
|
|
185
|
+
recordList(options, resultSize, transactional) {
|
|
186
|
+
this.listCalls.push({
|
|
187
|
+
options: { ...options },
|
|
188
|
+
resultSize,
|
|
189
|
+
transactional
|
|
190
|
+
});
|
|
191
|
+
}
|
|
97
192
|
};
|
|
98
193
|
var FakeTransaction = class {
|
|
99
194
|
storage;
|
|
@@ -103,21 +198,64 @@ var FakeTransaction = class {
|
|
|
103
198
|
constructor(storage) {
|
|
104
199
|
this.storage = storage;
|
|
105
200
|
}
|
|
106
|
-
async get(
|
|
201
|
+
async get(keyOrKeys) {
|
|
202
|
+
if (Array.isArray(keyOrKeys)) {
|
|
203
|
+
this.storage.recordOperation("getMany");
|
|
204
|
+
this.storage.getManyCalls.push([...keyOrKeys]);
|
|
205
|
+
this.storage.recordStorageCall("get", keyOrKeys, true);
|
|
206
|
+
const result = /* @__PURE__ */ new Map();
|
|
207
|
+
for (const key of keyOrKeys) {
|
|
208
|
+
if (this.deleted.has(key)) continue;
|
|
209
|
+
if (this.staged.has(key)) result.set(key, this.staged.get(key));
|
|
210
|
+
else if (this.storage.data.has(key)) result.set(key, this.storage.data.get(key));
|
|
211
|
+
}
|
|
212
|
+
return result;
|
|
213
|
+
}
|
|
214
|
+
this.storage.recordOperation("get");
|
|
215
|
+
const key = keyOrKeys;
|
|
216
|
+
this.storage.recordStorageCall("get", [key], true);
|
|
107
217
|
if (this.deleted.has(key)) return void 0;
|
|
108
218
|
if (this.staged.has(key)) return this.staged.get(key);
|
|
109
219
|
return this.storage.data.get(key);
|
|
110
220
|
}
|
|
111
|
-
async put(
|
|
112
|
-
|
|
221
|
+
async put(keyOrEntries, value) {
|
|
222
|
+
if (typeof keyOrEntries === "string") {
|
|
223
|
+
this.storage.recordOperation("put");
|
|
224
|
+
this.storage.maybeFailMutation({
|
|
225
|
+
operation: "put",
|
|
226
|
+
key: keyOrEntries,
|
|
227
|
+
value
|
|
228
|
+
});
|
|
229
|
+
this.storage.recordStorageCall("put", [keyOrEntries], true);
|
|
230
|
+
this.deleted.delete(keyOrEntries);
|
|
231
|
+
this.staged.set(keyOrEntries, structuredClone(value));
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
this.storage.recordOperation("putMany");
|
|
235
|
+
for (const [key, entryValue] of Object.entries(keyOrEntries)) this.storage.maybeFailMutation({
|
|
113
236
|
operation: "put",
|
|
114
237
|
key,
|
|
115
|
-
value
|
|
238
|
+
value: entryValue
|
|
116
239
|
});
|
|
117
|
-
this.
|
|
118
|
-
|
|
240
|
+
this.storage.recordStorageCall("put", Object.keys(keyOrEntries), true);
|
|
241
|
+
for (const [key, entryValue] of Object.entries(keyOrEntries)) {
|
|
242
|
+
this.deleted.delete(key);
|
|
243
|
+
this.staged.set(key, structuredClone(entryValue));
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
async delete(keyOrKeys) {
|
|
247
|
+
if (typeof keyOrKeys === "string") {
|
|
248
|
+
this.storage.recordOperation("delete");
|
|
249
|
+
this.storage.recordStorageCall("delete", [keyOrKeys], true);
|
|
250
|
+
return this.deleteOne(keyOrKeys);
|
|
251
|
+
}
|
|
252
|
+
this.storage.recordOperation("deleteMany");
|
|
253
|
+
this.storage.recordStorageCall("delete", keyOrKeys, true);
|
|
254
|
+
let deleted = 0;
|
|
255
|
+
for (const key of keyOrKeys) if (this.deleteOne(key)) deleted += 1;
|
|
256
|
+
return deleted;
|
|
119
257
|
}
|
|
120
|
-
|
|
258
|
+
deleteOne(key) {
|
|
121
259
|
this.storage.maybeFailMutation({
|
|
122
260
|
operation: "delete",
|
|
123
261
|
key
|
|
@@ -128,10 +266,12 @@ var FakeTransaction = class {
|
|
|
128
266
|
return existed;
|
|
129
267
|
}
|
|
130
268
|
async list(options = {}) {
|
|
269
|
+
this.storage.recordOperation("list");
|
|
131
270
|
const merged = new Map(this.storage.data);
|
|
132
271
|
for (const key of this.deleted) merged.delete(key);
|
|
133
272
|
for (const [key, value] of this.staged) merged.set(key, value);
|
|
134
273
|
const keys = applyListOptions(Array.from(merged.keys()), options);
|
|
274
|
+
this.storage.recordList(options, keys.length, true);
|
|
135
275
|
return new Map(keys.map((k) => [k, merged.get(k)]));
|
|
136
276
|
}
|
|
137
277
|
async getAlarm() {
|
|
@@ -175,28 +315,31 @@ var FakeFleet = class {
|
|
|
175
315
|
get: (id) => this.cell(bindingKey, id.toString()).instance
|
|
176
316
|
};
|
|
177
317
|
}
|
|
318
|
+
instantiate(bindingKey, name, storage, pendingWaits) {
|
|
319
|
+
const CellCtor = this.classes[bindingKey];
|
|
320
|
+
if (!CellCtor) throw new Error(`no cell class registered for binding: ${bindingKey}`);
|
|
321
|
+
return new CellCtor({
|
|
322
|
+
storage,
|
|
323
|
+
id: {
|
|
324
|
+
toString: () => name,
|
|
325
|
+
name
|
|
326
|
+
},
|
|
327
|
+
waitUntil(promise) {
|
|
328
|
+
pendingWaits.push(promise);
|
|
329
|
+
},
|
|
330
|
+
async blockConcurrencyWhile(cb) {
|
|
331
|
+
return cb();
|
|
332
|
+
}
|
|
333
|
+
}, this.cellEnv);
|
|
334
|
+
}
|
|
178
335
|
cell(bindingKey, name) {
|
|
179
336
|
const key = `${bindingKey}\0${name}`;
|
|
180
337
|
let slot = this.cells.get(key);
|
|
181
338
|
if (!slot) {
|
|
182
|
-
const CellCtor = this.classes[bindingKey];
|
|
183
|
-
if (!CellCtor) throw new Error(`no cell class registered for binding: ${bindingKey}`);
|
|
184
339
|
const storage = new FakeStorage(() => this.now);
|
|
185
340
|
const pendingWaits = [];
|
|
186
341
|
slot = {
|
|
187
|
-
instance:
|
|
188
|
-
storage,
|
|
189
|
-
id: {
|
|
190
|
-
toString: () => name,
|
|
191
|
-
name
|
|
192
|
-
},
|
|
193
|
-
waitUntil(promise) {
|
|
194
|
-
pendingWaits.push(promise);
|
|
195
|
-
},
|
|
196
|
-
async blockConcurrencyWhile(cb) {
|
|
197
|
-
return cb();
|
|
198
|
-
}
|
|
199
|
-
}, this.cellEnv),
|
|
342
|
+
instance: this.instantiate(bindingKey, name, storage, pendingWaits),
|
|
200
343
|
storage,
|
|
201
344
|
pendingWaits,
|
|
202
345
|
alarmRetryCount: 0,
|
|
@@ -206,6 +349,12 @@ var FakeFleet = class {
|
|
|
206
349
|
}
|
|
207
350
|
return slot;
|
|
208
351
|
}
|
|
352
|
+
/** Recreate one in-memory DO instance while preserving its durable storage. */
|
|
353
|
+
restartCell(bindingKey, name) {
|
|
354
|
+
const slot = this.cell(bindingKey, name);
|
|
355
|
+
slot.instance = this.instantiate(bindingKey, name, slot.storage, slot.pendingWaits);
|
|
356
|
+
return slot.instance;
|
|
357
|
+
}
|
|
209
358
|
advance(ms) {
|
|
210
359
|
this.now += ms;
|
|
211
360
|
}
|
|
@@ -292,13 +441,23 @@ async function authenticate(request, secret) {
|
|
|
292
441
|
* Routes:
|
|
293
442
|
* GET /v1/health — liveness + spec version (unauthenticated)
|
|
294
443
|
* POST /v1/rpc/{binding}/{name}/{method} — DO RPC (bearer auth)
|
|
444
|
+
* POST /v1/index/{domain}/{method} — cohesive index operation
|
|
445
|
+
* GET /v1/streams/{name}/chunks — bounded binary long-poll read
|
|
446
|
+
* POST /v1/streams/{name}/chunks — bounded binary batch append
|
|
295
447
|
*
|
|
296
|
-
*
|
|
297
|
-
*
|
|
298
|
-
* dispatch — everything else (including fetch/alarm) is unreachable.
|
|
448
|
+
* Generic RPC bodies use the tagged JSON codec. Stream chunk bodies use the
|
|
449
|
+
* compact binary stream protocol. Only whitelisted routes/methods dispatch.
|
|
299
450
|
*/
|
|
300
451
|
const WORLD_NAME = "world-celld";
|
|
301
452
|
const WORLD_VERSION = "0.1.0";
|
|
453
|
+
function workflowIndex(env) {
|
|
454
|
+
return createWorkflowIndex({
|
|
455
|
+
runCatalog: env.WORKFLOW_RUN_CATALOG,
|
|
456
|
+
runFences: env.WORKFLOW_RUN_FENCES,
|
|
457
|
+
hookTokens: env.WORKFLOW_HOOK_TOKENS,
|
|
458
|
+
hookIds: env.WORKFLOW_HOOK_IDS
|
|
459
|
+
});
|
|
460
|
+
}
|
|
302
461
|
const BINDINGS = {
|
|
303
462
|
runs: {
|
|
304
463
|
env: "WORKFLOW_DB",
|
|
@@ -313,16 +472,15 @@ const BINDINGS = {
|
|
|
313
472
|
"getCleanupStatus",
|
|
314
473
|
"scheduleCleanup",
|
|
315
474
|
"cleanupNow",
|
|
316
|
-
"rearmCleanup"
|
|
475
|
+
"rearmCleanup",
|
|
476
|
+
"resolveHookTokenClaim"
|
|
317
477
|
])
|
|
318
478
|
},
|
|
319
479
|
streams: {
|
|
320
480
|
env: "WORKFLOW_STREAMS",
|
|
321
481
|
methods: /* @__PURE__ */ new Set([
|
|
322
|
-
"writeChunk",
|
|
323
482
|
"closeStream",
|
|
324
|
-
"
|
|
325
|
-
"getInfo",
|
|
483
|
+
"failStream",
|
|
326
484
|
"registerStream",
|
|
327
485
|
"listStreams",
|
|
328
486
|
"expireRegistry",
|
|
@@ -330,20 +488,13 @@ const BINDINGS = {
|
|
|
330
488
|
"expireStream"
|
|
331
489
|
])
|
|
332
490
|
},
|
|
333
|
-
|
|
334
|
-
env: "
|
|
335
|
-
methods: /* @__PURE__ */ new Set([
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
"expireRun",
|
|
341
|
-
"list",
|
|
342
|
-
"reserveHookToken",
|
|
343
|
-
"finalizeHookIndexes",
|
|
344
|
-
"releaseHookToken",
|
|
345
|
-
"deleteHookIndexes"
|
|
346
|
-
])
|
|
491
|
+
"hook-tokens": {
|
|
492
|
+
env: "WORKFLOW_HOOK_TOKENS",
|
|
493
|
+
methods: /* @__PURE__ */ new Set(["get"])
|
|
494
|
+
},
|
|
495
|
+
"hook-ids": {
|
|
496
|
+
env: "WORKFLOW_HOOK_IDS",
|
|
497
|
+
methods: /* @__PURE__ */ new Set(["get"])
|
|
347
498
|
},
|
|
348
499
|
queue: {
|
|
349
500
|
env: "WORKFLOW_QUEUE",
|
|
@@ -360,6 +511,7 @@ const BINDINGS = {
|
|
|
360
511
|
};
|
|
361
512
|
/** Request body cap: oversize payloads get a clear 413 instead of an OOM. */
|
|
362
513
|
const MAX_BODY_BYTES = 33554432;
|
|
514
|
+
const MAX_STREAM_WRITE_BODY_BYTES = MAX_STREAM_BATCH_BYTES + 1024;
|
|
363
515
|
function errorResponse(status, name, message) {
|
|
364
516
|
return Response.json({ error: {
|
|
365
517
|
name,
|
|
@@ -388,6 +540,49 @@ async function readBoundedBody(request) {
|
|
|
388
540
|
reader.releaseLock();
|
|
389
541
|
}
|
|
390
542
|
}
|
|
543
|
+
async function readBoundedBytes(request, maxBytes) {
|
|
544
|
+
if (!request.body) return /* @__PURE__ */ new Uint8Array();
|
|
545
|
+
const reader = request.body.getReader();
|
|
546
|
+
const chunks = [];
|
|
547
|
+
let total = 0;
|
|
548
|
+
try {
|
|
549
|
+
for (;;) {
|
|
550
|
+
const { done, value } = await reader.read();
|
|
551
|
+
if (done) break;
|
|
552
|
+
total += value.byteLength;
|
|
553
|
+
if (total > maxBytes) {
|
|
554
|
+
await reader.cancel("stream body exceeds configured limit").catch(() => void 0);
|
|
555
|
+
return null;
|
|
556
|
+
}
|
|
557
|
+
chunks.push(value);
|
|
558
|
+
}
|
|
559
|
+
} finally {
|
|
560
|
+
reader.releaseLock();
|
|
561
|
+
}
|
|
562
|
+
const body = new Uint8Array(total);
|
|
563
|
+
let offset = 0;
|
|
564
|
+
for (const chunk of chunks) {
|
|
565
|
+
body.set(chunk, offset);
|
|
566
|
+
offset += chunk.byteLength;
|
|
567
|
+
}
|
|
568
|
+
return body;
|
|
569
|
+
}
|
|
570
|
+
function parseBoundedInteger(url, name, minimum, maximum) {
|
|
571
|
+
const raw = url.searchParams.get(name);
|
|
572
|
+
const value = raw === null ? NaN : Number(raw);
|
|
573
|
+
if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
|
|
574
|
+
const error = /* @__PURE__ */ new Error(`${name} must be an integer between ${minimum} and ${maximum}`);
|
|
575
|
+
error.name = "StreamProtocolError";
|
|
576
|
+
throw error;
|
|
577
|
+
}
|
|
578
|
+
return value;
|
|
579
|
+
}
|
|
580
|
+
function streamResponse(body) {
|
|
581
|
+
return new Response(body, {
|
|
582
|
+
status: 200,
|
|
583
|
+
headers: { "content-type": STREAM_BATCH_CONTENT_TYPE }
|
|
584
|
+
});
|
|
585
|
+
}
|
|
391
586
|
function createRouter(env) {
|
|
392
587
|
return async (request) => {
|
|
393
588
|
const url = new URL(request.url);
|
|
@@ -399,9 +594,83 @@ function createRouter(env) {
|
|
|
399
594
|
version: WORLD_VERSION,
|
|
400
595
|
specVersion: SPEC_VERSION_CURRENT
|
|
401
596
|
});
|
|
402
|
-
if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
|
|
403
597
|
const auth = await authenticate(request, env.WORLD_SECRET);
|
|
404
598
|
if (!auth.ok) return auth.response;
|
|
599
|
+
if (parts[1] === "index" && parts.length === 4) {
|
|
600
|
+
if (request.method !== "POST") return errorResponse(405, "MethodNotAllowed", "expected POST");
|
|
601
|
+
if (Number(request.headers.get("content-length") ?? "0") > MAX_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
602
|
+
let args;
|
|
603
|
+
try {
|
|
604
|
+
const text = await readBoundedBody(request);
|
|
605
|
+
if (text === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_BODY_BYTES} bytes`);
|
|
606
|
+
const parsed = rpcParse(text);
|
|
607
|
+
if (!Array.isArray(parsed)) return errorResponse(400, "BadRequest", "body must be an argument array");
|
|
608
|
+
args = parsed;
|
|
609
|
+
} catch {
|
|
610
|
+
return errorResponse(400, "BadRequest", "malformed rpc body");
|
|
611
|
+
}
|
|
612
|
+
try {
|
|
613
|
+
const index = workflowIndex(env);
|
|
614
|
+
let result;
|
|
615
|
+
const operation = `${parts[2]}.${parts[3]}`;
|
|
616
|
+
if (operation === "runs.list") result = await index.listRuns(args[0]);
|
|
617
|
+
else if (operation === "runs.commit") result = await index.commitRun(args[0], args[1]);
|
|
618
|
+
else if (operation === "runs.expire") result = await index.expireRun(args[0]);
|
|
619
|
+
else if (operation === "hooks.reserve") result = await index.reserveHook(args[0], args[1]);
|
|
620
|
+
else if (operation === "hooks.finalize") result = await index.finalizeHookIndexes(args[0], args[1], args[2], args[3], args[4]);
|
|
621
|
+
else if (operation === "hooks.release-reservation") result = await index.releaseHookReservation(args[0], args[1], args[2]);
|
|
622
|
+
else if (operation === "hooks.release") result = await index.releaseHookIndexes(args[0]);
|
|
623
|
+
else return errorResponse(404, "NotFound", `unknown index operation: ${operation}`);
|
|
624
|
+
return new Response(rpcStringify(result ?? null), {
|
|
625
|
+
status: 200,
|
|
626
|
+
headers: { "content-type": "application/json" }
|
|
627
|
+
});
|
|
628
|
+
} catch (error) {
|
|
629
|
+
const err = error;
|
|
630
|
+
return Response.json({ error: {
|
|
631
|
+
name: err?.name ?? "Error",
|
|
632
|
+
message: err?.message ?? String(error),
|
|
633
|
+
status: typeof err?.status === "number" ? err.status : void 0
|
|
634
|
+
} }, { status: 500 });
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
if (parts[1] === "streams" && parts.length === 4 && parts[3] === "chunks") {
|
|
638
|
+
const namespace = env.WORKFLOW_STREAMS;
|
|
639
|
+
if (!namespace) return errorResponse(500, "WorldMisconfigured", "missing binding: WORKFLOW_STREAMS");
|
|
640
|
+
const name = decodeURIComponent(parts[2]);
|
|
641
|
+
const runId = url.searchParams.get("runId");
|
|
642
|
+
if (!runId) return errorResponse(400, "BadRequest", "runId is required");
|
|
643
|
+
const stub = namespace.get(namespace.idFromName(name));
|
|
644
|
+
try {
|
|
645
|
+
if (request.method === "POST") {
|
|
646
|
+
if (!request.headers.get("content-type")?.startsWith("application/vnd.world-celld.stream-batch.v1")) return errorResponse(415, "UnsupportedMediaType", STREAM_BATCH_CONTENT_TYPE);
|
|
647
|
+
if (Number(request.headers.get("content-length") ?? "0") > MAX_STREAM_WRITE_BODY_BYTES) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
|
|
648
|
+
const body = await readBoundedBytes(request, MAX_STREAM_WRITE_BODY_BYTES);
|
|
649
|
+
if (body === null) return errorResponse(413, "PayloadTooLarge", `body exceeds ${MAX_STREAM_WRITE_BODY_BYTES} bytes`);
|
|
650
|
+
const result = await stub.writeChunks(runId, decodeStreamWriteBatch(body));
|
|
651
|
+
return streamResponse(encodeStreamWriteResult(result));
|
|
652
|
+
}
|
|
653
|
+
if (request.method === "GET") {
|
|
654
|
+
const result = await stub.readChunks({
|
|
655
|
+
runId,
|
|
656
|
+
startIndex: parseBoundedInteger(url, "startIndex", 0, 2147483647),
|
|
657
|
+
maxChunks: parseBoundedInteger(url, "maxChunks", 0, 32),
|
|
658
|
+
maxBytes: parseBoundedInteger(url, "maxBytes", MAX_STREAM_CHUNK_BYTES, MAX_STREAM_READ_BYTES),
|
|
659
|
+
waitMs: parseBoundedInteger(url, "waitMs", 0, MAX_STREAM_LONG_POLL_MS)
|
|
660
|
+
}, request.signal);
|
|
661
|
+
return streamResponse(encodeStreamReadResult(result));
|
|
662
|
+
}
|
|
663
|
+
return errorResponse(405, "MethodNotAllowed", "expected GET or POST");
|
|
664
|
+
} catch (error) {
|
|
665
|
+
const err = error;
|
|
666
|
+
return Response.json({ error: {
|
|
667
|
+
name: err?.name ?? "Error",
|
|
668
|
+
message: err?.message ?? String(error),
|
|
669
|
+
status: typeof err?.status === "number" ? err.status : void 0
|
|
670
|
+
} }, { status: err?.name === "StreamProtocolError" ? 400 : 500 });
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
if (parts[1] !== "rpc") return errorResponse(404, "NotFound", `unknown path: ${url.pathname}`);
|
|
405
674
|
if (request.method !== "POST" || parts.length !== 5) return errorResponse(404, "NotFound", "expected POST /v1/rpc/{binding}/{name}/{method}");
|
|
406
675
|
const [, , bindingKey, encodedName, method] = parts;
|
|
407
676
|
const binding = BINDINGS[bindingKey];
|
|
@@ -457,160 +726,242 @@ var DurableObject = class {
|
|
|
457
726
|
}
|
|
458
727
|
};
|
|
459
728
|
//#endregion
|
|
460
|
-
//#region src/worker/durable-objects/
|
|
729
|
+
//#region src/worker/durable-objects/hook-index-shared.ts
|
|
461
730
|
function sameOwner(left, right) {
|
|
462
731
|
return left.runId === right.runId && left.hookId === right.hookId;
|
|
463
732
|
}
|
|
464
|
-
function
|
|
733
|
+
function ownerFromSerializedHook(raw) {
|
|
465
734
|
const hook = parse(raw);
|
|
466
735
|
return {
|
|
467
736
|
runId: hook.runId,
|
|
468
737
|
hookId: hook.hookId
|
|
469
738
|
};
|
|
470
739
|
}
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
740
|
+
function indexKey(prefix, value) {
|
|
741
|
+
return `${prefix}:${encodeURIComponent(value)}`;
|
|
742
|
+
}
|
|
743
|
+
/** Compact shard-local fence for every hook owned by one terminal or expired run. */
|
|
744
|
+
function shardRunFenceKey(runId) {
|
|
745
|
+
return `runfence:${encodeURIComponent(runId)}`;
|
|
746
|
+
}
|
|
747
|
+
async function runIsActive(env, runId) {
|
|
748
|
+
const namespace = env.WORKFLOW_RUN_FENCES;
|
|
749
|
+
if (!namespace) throw new Error("world-celld hook index missing WORKFLOW_RUN_FENCES binding");
|
|
750
|
+
return await namespace.get(namespace.idFromName(runFenceCellName(runId))).getStatus() === "active";
|
|
751
|
+
}
|
|
752
|
+
//#endregion
|
|
753
|
+
//#region src/worker/durable-objects/HookIdDO.ts
|
|
754
|
+
const MAX_RELEASE_BATCH$1 = 64;
|
|
755
|
+
function recordKey$1(hookId) {
|
|
756
|
+
return indexKey("hookid", hookId);
|
|
757
|
+
}
|
|
758
|
+
function claimKey$1(hookId) {
|
|
759
|
+
return indexKey("claim", hookId);
|
|
760
|
+
}
|
|
761
|
+
/** Hook-ID ownership and lookup shard; every hook ID has one transaction domain. */
|
|
762
|
+
var HookIdDO = class extends DurableObject {
|
|
763
|
+
async get(hookId) {
|
|
764
|
+
const raw = await this.ctx.storage.get(recordKey$1(hookId));
|
|
765
|
+
if (!raw) return null;
|
|
766
|
+
const owner = ownerFromSerializedHook(raw);
|
|
767
|
+
return await runIsActive(this.env, owner.runId) ? raw : null;
|
|
768
|
+
}
|
|
769
|
+
async reserve(hookId, owner, proposedClaimId) {
|
|
770
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
771
|
+
const record = recordKey$1(hookId);
|
|
772
|
+
const claim = claimKey$1(hookId);
|
|
773
|
+
const values = await txn.get([record, claim]);
|
|
774
|
+
const indexed = values.get(record);
|
|
775
|
+
if (indexed !== void 0) return {
|
|
776
|
+
claimed: false,
|
|
777
|
+
holder: ownerFromSerializedHook(indexed)
|
|
778
|
+
};
|
|
779
|
+
const existing = values.get(claim);
|
|
780
|
+
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
781
|
+
claimed: false,
|
|
782
|
+
holder: existing.owner
|
|
783
|
+
};
|
|
784
|
+
if (existing === void 0 || existing.claimId !== proposedClaimId) {
|
|
785
|
+
await txn.put(claim, {
|
|
786
|
+
owner,
|
|
787
|
+
claimId: proposedClaimId,
|
|
788
|
+
reservedAt: Date.now()
|
|
789
|
+
});
|
|
790
|
+
return {
|
|
791
|
+
claimed: true,
|
|
792
|
+
claimId: proposedClaimId
|
|
793
|
+
};
|
|
794
|
+
}
|
|
795
|
+
return {
|
|
796
|
+
claimed: true,
|
|
797
|
+
claimId: existing.claimId
|
|
798
|
+
};
|
|
799
|
+
});
|
|
490
800
|
}
|
|
491
|
-
|
|
492
|
-
|
|
801
|
+
async publish(hookId, serializedHook, owner, claimId) {
|
|
802
|
+
if (!await runIsActive(this.env, owner.runId)) {
|
|
803
|
+
await this.releaseBatch([{
|
|
804
|
+
hookId,
|
|
805
|
+
owner
|
|
806
|
+
}], Date.now(), true);
|
|
807
|
+
return { stored: false };
|
|
808
|
+
}
|
|
493
809
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
494
|
-
|
|
495
|
-
|
|
810
|
+
const record = recordKey$1(hookId);
|
|
811
|
+
const claim = claimKey$1(hookId);
|
|
812
|
+
const runFence = shardRunFenceKey(owner.runId);
|
|
813
|
+
const values = await txn.get([
|
|
814
|
+
record,
|
|
815
|
+
claim,
|
|
816
|
+
runFence
|
|
817
|
+
]);
|
|
818
|
+
if (values.has(runFence)) {
|
|
819
|
+
const existingClaim = values.get(claim);
|
|
820
|
+
if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
|
|
821
|
+
return { stored: false };
|
|
822
|
+
}
|
|
823
|
+
const indexed = values.get(record);
|
|
824
|
+
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook id ${hookId} is owned by another hook`);
|
|
825
|
+
const existingClaim = values.get(claim);
|
|
826
|
+
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook id ${hookId} is reserved by another hook`);
|
|
827
|
+
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook id ${hookId} has no active reservation`);
|
|
828
|
+
await txn.put(record, serializedHook);
|
|
829
|
+
if (existingClaim !== void 0) await txn.delete(claim);
|
|
496
830
|
return { stored: true };
|
|
497
831
|
});
|
|
498
832
|
}
|
|
499
|
-
async
|
|
500
|
-
await this.ctx.storage.
|
|
833
|
+
async releaseClaim(hookId, owner, claimId) {
|
|
834
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
835
|
+
const key = claimKey$1(hookId);
|
|
836
|
+
const claim = await txn.get(key);
|
|
837
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
|
|
838
|
+
});
|
|
501
839
|
}
|
|
502
|
-
async
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
840
|
+
async releaseBatch(hooks, fencedAt, runFenced) {
|
|
841
|
+
if (hooks.length > MAX_RELEASE_BATCH$1) throw new Error("Hook id release batch is too large");
|
|
842
|
+
if (hooks.length === 0) return { deleted: 0 };
|
|
843
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
844
|
+
const keys = hooks.flatMap(({ hookId }) => [recordKey$1(hookId), claimKey$1(hookId)]);
|
|
845
|
+
const values = await txn.get(keys);
|
|
846
|
+
const deletes = [];
|
|
847
|
+
for (const { hookId, owner } of hooks) {
|
|
848
|
+
const record = recordKey$1(hookId);
|
|
849
|
+
const claim = claimKey$1(hookId);
|
|
850
|
+
const indexed = values.get(record);
|
|
851
|
+
const existingClaim = values.get(claim);
|
|
852
|
+
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
853
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
|
|
854
|
+
}
|
|
855
|
+
if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
|
|
856
|
+
return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
|
|
512
857
|
});
|
|
513
|
-
const names = Array.from(entries.keys());
|
|
514
|
-
const page = names.slice(0, limit);
|
|
515
|
-
const listComplete = names.length <= limit;
|
|
516
|
-
return {
|
|
517
|
-
keys: page.map((name) => ({ name })),
|
|
518
|
-
list_complete: listComplete,
|
|
519
|
-
cursor: listComplete ? void 0 : page.at(-1)
|
|
520
|
-
};
|
|
521
858
|
}
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
859
|
+
};
|
|
860
|
+
//#endregion
|
|
861
|
+
//#region src/worker/durable-objects/HookTokenDO.ts
|
|
862
|
+
const MAX_RELEASE_BATCH = 64;
|
|
863
|
+
function recordKey(token) {
|
|
864
|
+
return indexKey("hook", token);
|
|
865
|
+
}
|
|
866
|
+
function claimKey(token) {
|
|
867
|
+
return indexKey("claim", token);
|
|
868
|
+
}
|
|
869
|
+
/** Token-ownership shard. A token hashes to exactly one shard and transaction domain. */
|
|
870
|
+
var HookTokenDO = class extends DurableObject {
|
|
871
|
+
async get(token) {
|
|
872
|
+
const raw = await this.ctx.storage.get(recordKey(token));
|
|
873
|
+
if (!raw) return null;
|
|
874
|
+
const owner = ownerFromSerializedHook(raw);
|
|
875
|
+
return await runIsActive(this.env, owner.runId) ? raw : null;
|
|
876
|
+
}
|
|
877
|
+
async reserve(token, owner, proposedClaimId) {
|
|
878
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
879
|
+
const record = recordKey(token);
|
|
880
|
+
const claim = claimKey(token);
|
|
881
|
+
const values = await txn.get([record, claim]);
|
|
882
|
+
const indexed = values.get(record);
|
|
883
|
+
if (indexed !== void 0) return {
|
|
526
884
|
claimed: false,
|
|
527
|
-
holder:
|
|
885
|
+
holder: ownerFromSerializedHook(indexed)
|
|
528
886
|
};
|
|
529
|
-
const
|
|
530
|
-
|
|
531
|
-
if (claim !== void 0 && !sameOwner(claim.owner, owner)) return {
|
|
887
|
+
const existing = values.get(claim);
|
|
888
|
+
if (existing !== void 0 && !sameOwner(existing.owner, owner)) return {
|
|
532
889
|
claimed: false,
|
|
533
|
-
holder:
|
|
890
|
+
holder: existing.owner
|
|
891
|
+
};
|
|
892
|
+
if (existing === void 0) {
|
|
893
|
+
await txn.put(claim, {
|
|
894
|
+
owner,
|
|
895
|
+
claimId: proposedClaimId,
|
|
896
|
+
reservedAt: Date.now()
|
|
897
|
+
});
|
|
898
|
+
return {
|
|
899
|
+
claimed: true,
|
|
900
|
+
claimId: proposedClaimId
|
|
901
|
+
};
|
|
902
|
+
}
|
|
903
|
+
return {
|
|
904
|
+
claimed: true,
|
|
905
|
+
claimId: existing.claimId
|
|
534
906
|
};
|
|
535
|
-
await txn.put(claimKey, {
|
|
536
|
-
owner,
|
|
537
|
-
reservedAt: Date.now()
|
|
538
|
-
});
|
|
539
|
-
return { claimed: true };
|
|
540
907
|
});
|
|
541
908
|
}
|
|
542
|
-
async
|
|
543
|
-
await this.
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
909
|
+
async finalize(token, serializedHook, owner, claimId) {
|
|
910
|
+
if (!await runIsActive(this.env, owner.runId)) {
|
|
911
|
+
await this.releaseBatch([{
|
|
912
|
+
token,
|
|
913
|
+
owner
|
|
914
|
+
}], Date.now(), true);
|
|
915
|
+
return { stored: false };
|
|
916
|
+
}
|
|
917
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
918
|
+
const record = recordKey(token);
|
|
919
|
+
const claim = claimKey(token);
|
|
920
|
+
const runFence = shardRunFenceKey(owner.runId);
|
|
921
|
+
const values = await txn.get([
|
|
922
|
+
record,
|
|
923
|
+
claim,
|
|
924
|
+
runFence
|
|
925
|
+
]);
|
|
926
|
+
if (values.has(runFence)) {
|
|
927
|
+
const existingClaim = values.get(claim);
|
|
928
|
+
if (existingClaim && sameOwner(existingClaim.owner, owner)) await txn.delete(claim);
|
|
929
|
+
return { stored: false };
|
|
548
930
|
}
|
|
549
|
-
const
|
|
550
|
-
if (
|
|
551
|
-
const
|
|
552
|
-
if (
|
|
553
|
-
|
|
554
|
-
await txn.put(
|
|
555
|
-
if (
|
|
931
|
+
const indexed = values.get(record);
|
|
932
|
+
if (indexed !== void 0 && !sameOwner(ownerFromSerializedHook(indexed), owner)) throw new Error(`Hook token ${token} is owned by another hook`);
|
|
933
|
+
const existingClaim = values.get(claim);
|
|
934
|
+
if (existingClaim !== void 0 && (!sameOwner(existingClaim.owner, owner) || existingClaim.claimId !== claimId)) throw new Error(`Hook token ${token} is reserved by another hook`);
|
|
935
|
+
if (indexed === void 0 && existingClaim === void 0) throw new Error(`Hook token ${token} has no active reservation`);
|
|
936
|
+
await txn.put(record, serializedHook);
|
|
937
|
+
if (existingClaim !== void 0) await txn.delete(claim);
|
|
938
|
+
return { stored: true };
|
|
556
939
|
});
|
|
557
940
|
}
|
|
558
|
-
async
|
|
941
|
+
async releaseClaim(token, owner, claimId) {
|
|
559
942
|
await this.ctx.storage.transaction(async (txn) => {
|
|
560
|
-
const key =
|
|
943
|
+
const key = claimKey(token);
|
|
561
944
|
const claim = await txn.get(key);
|
|
562
|
-
if (claim
|
|
563
|
-
});
|
|
564
|
-
}
|
|
565
|
-
async deleteHookIndexes(token, hookId, owner) {
|
|
566
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
567
|
-
const tokenKey = `hook:${token}`;
|
|
568
|
-
const idKey = `hookid:${hookId}`;
|
|
569
|
-
const claimKey = `hookclaim:${token}`;
|
|
570
|
-
const [byToken, byId, claim] = await Promise.all([
|
|
571
|
-
txn.get(tokenKey),
|
|
572
|
-
txn.get(idKey),
|
|
573
|
-
txn.get(claimKey)
|
|
574
|
-
]);
|
|
575
|
-
if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) await txn.delete(tokenKey);
|
|
576
|
-
if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) await txn.delete(idKey);
|
|
577
|
-
if (claim !== void 0 && sameOwner(claim.owner, owner)) await txn.delete(claimKey);
|
|
945
|
+
if (claim && sameOwner(claim.owner, owner) && claim.claimId === claimId) await txn.delete(key);
|
|
578
946
|
});
|
|
579
947
|
}
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
* entries from a delayed request.
|
|
584
|
-
*/
|
|
585
|
-
async expireRun(request) {
|
|
948
|
+
async releaseBatch(hooks, fencedAt, runFenced) {
|
|
949
|
+
if (hooks.length > MAX_RELEASE_BATCH) throw new Error("Hook token release batch is too large");
|
|
950
|
+
if (hooks.length === 0) return { deleted: 0 };
|
|
586
951
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
for (const
|
|
591
|
-
const
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
const claimKey = `hookclaim:${hook.token}`;
|
|
598
|
-
const [byToken, byId, claim] = await Promise.all([
|
|
599
|
-
txn.get(tokenKey),
|
|
600
|
-
txn.get(idKey),
|
|
601
|
-
txn.get(claimKey)
|
|
602
|
-
]);
|
|
603
|
-
if (byToken !== void 0 && sameOwner(ownerFromHook(byToken), owner)) {
|
|
604
|
-
if (await txn.delete(tokenKey)) deleted++;
|
|
605
|
-
}
|
|
606
|
-
if (byId !== void 0 && sameOwner(ownerFromHook(byId), owner)) {
|
|
607
|
-
if (await txn.delete(idKey)) deleted++;
|
|
608
|
-
}
|
|
609
|
-
if (claim !== void 0 && sameOwner(claim.owner, owner)) {
|
|
610
|
-
if (await txn.delete(claimKey)) deleted++;
|
|
611
|
-
}
|
|
952
|
+
const keys = hooks.flatMap(({ token }) => [recordKey(token), claimKey(token)]);
|
|
953
|
+
const values = await txn.get(keys);
|
|
954
|
+
const deletes = [];
|
|
955
|
+
for (const { token, owner } of hooks) {
|
|
956
|
+
const record = recordKey(token);
|
|
957
|
+
const claim = claimKey(token);
|
|
958
|
+
const indexed = values.get(record);
|
|
959
|
+
const existingClaim = values.get(claim);
|
|
960
|
+
if (indexed !== void 0 && sameOwner(ownerFromSerializedHook(indexed), owner)) deletes.push(record);
|
|
961
|
+
if (existingClaim !== void 0 && sameOwner(existingClaim.owner, owner)) deletes.push(claim);
|
|
612
962
|
}
|
|
613
|
-
|
|
963
|
+
if (runFenced) await txn.put(shardRunFenceKey(hooks[0].owner.runId), fencedAt);
|
|
964
|
+
return { deleted: deletes.length === 0 ? 0 : await txn.delete(deletes) };
|
|
614
965
|
});
|
|
615
966
|
}
|
|
616
967
|
};
|
|
@@ -629,21 +980,61 @@ const DEFAULT_DELIVERY_TIMEOUT_MS = 3e5;
|
|
|
629
980
|
const INFLIGHT_GRACE_MS = 3e4;
|
|
630
981
|
/** Move overdue alarms to a new timestamp so celld observes a fresh edge. */
|
|
631
982
|
const MIN_ALARM_DELAY_MS = 1;
|
|
983
|
+
/** Bound expiry recovery so one alarm cannot monopolize the cell. */
|
|
984
|
+
const EXPIRED_INFLIGHT_BATCH = 128;
|
|
985
|
+
const STORAGE_BATCH_SIZE$2 = 128;
|
|
986
|
+
const EXPIRE_RUN_REFERENCE_BATCH = 64;
|
|
987
|
+
const PURGE_DEAD_LETTER_BATCH = 128;
|
|
988
|
+
const INFLIGHT_DEADLINE_PREFIX = "inflight-deadline:";
|
|
632
989
|
function pad(ms) {
|
|
633
990
|
return String(Math.max(0, Math.floor(ms))).padStart(13, "0");
|
|
634
991
|
}
|
|
635
992
|
function dueKey(atMs, messageId) {
|
|
636
993
|
return `due:${pad(atMs)}:${messageId}`;
|
|
637
994
|
}
|
|
995
|
+
function inflightKey(deadlineMs, messageId) {
|
|
996
|
+
return `${INFLIGHT_DEADLINE_PREFIX}${pad(deadlineMs)}:${messageId}`;
|
|
997
|
+
}
|
|
998
|
+
function deadlineFromInflightKey(key) {
|
|
999
|
+
return Number.parseInt(key.slice(18, 31), 10);
|
|
1000
|
+
}
|
|
638
1001
|
function runReferenceKey(runId, messageId) {
|
|
639
1002
|
return `run:${runId}:${messageId}`;
|
|
640
1003
|
}
|
|
641
1004
|
function expiredRunKey(runId) {
|
|
642
1005
|
return `expired-run:${runId}`;
|
|
643
1006
|
}
|
|
1007
|
+
function boundedLimit$1(value, fallback, maximum) {
|
|
1008
|
+
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
1009
|
+
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
1010
|
+
}
|
|
1011
|
+
async function getMany$1(storage, keys) {
|
|
1012
|
+
const result = /* @__PURE__ */ new Map();
|
|
1013
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$2) {
|
|
1014
|
+
const page = await storage.get(keys.slice(offset, offset + STORAGE_BATCH_SIZE$2));
|
|
1015
|
+
for (const [key, value] of page) result.set(key, value);
|
|
1016
|
+
}
|
|
1017
|
+
return result;
|
|
1018
|
+
}
|
|
1019
|
+
async function putMany(storage, entries) {
|
|
1020
|
+
const all = Array.from(entries);
|
|
1021
|
+
for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE$2) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE$2)));
|
|
1022
|
+
}
|
|
1023
|
+
async function deleteMany$1(storage, keys) {
|
|
1024
|
+
let deleted = 0;
|
|
1025
|
+
const unique = Array.from(new Set(keys));
|
|
1026
|
+
for (let offset = 0; offset < unique.length; offset += STORAGE_BATCH_SIZE$2) deleted += await storage.delete(unique.slice(offset, offset + STORAGE_BATCH_SIZE$2));
|
|
1027
|
+
return deleted;
|
|
1028
|
+
}
|
|
644
1029
|
function backoffSeconds(attempt) {
|
|
645
1030
|
return Math.min(60, 2 ** attempt);
|
|
646
1031
|
}
|
|
1032
|
+
async function cancelUnusedResponseBody(response) {
|
|
1033
|
+
try {
|
|
1034
|
+
await response.body?.cancel();
|
|
1035
|
+
} catch {}
|
|
1036
|
+
}
|
|
1037
|
+
const MAX_QUEUE_INFLIGHT = 128;
|
|
647
1038
|
var QueueDO = class extends DurableObject {
|
|
648
1039
|
#now() {
|
|
649
1040
|
const clock = this.env?.clock;
|
|
@@ -658,7 +1049,13 @@ var QueueDO = class extends DurableObject {
|
|
|
658
1049
|
const parsed = raw ? Number.parseInt(raw, 10) : NaN;
|
|
659
1050
|
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
660
1051
|
}
|
|
1052
|
+
#maxInflight() {
|
|
1053
|
+
const configured = this.#intVar("QUEUE_MAX_INFLIGHT", DEFAULT_MAX_INFLIGHT);
|
|
1054
|
+
if (configured > MAX_QUEUE_INFLIGHT) throw new Error(`QUEUE_MAX_INFLIGHT must be at most ${MAX_QUEUE_INFLIGHT}`);
|
|
1055
|
+
return configured;
|
|
1056
|
+
}
|
|
661
1057
|
async enqueue(request) {
|
|
1058
|
+
this.#maxInflight();
|
|
662
1059
|
const storage = this.ctx.storage;
|
|
663
1060
|
const now = this.#now();
|
|
664
1061
|
return await storage.transaction(async (txn) => {
|
|
@@ -733,7 +1130,7 @@ var QueueDO = class extends DurableObject {
|
|
|
733
1130
|
if (typeof this.ctx.blockConcurrencyWhile === "function") await this.ctx.blockConcurrencyWhile(attempt);
|
|
734
1131
|
else await attempt();
|
|
735
1132
|
if (failure !== null) throw failure;
|
|
736
|
-
for (const
|
|
1133
|
+
for (const claim of claimed) this.ctx.waitUntil(this.#deliver(claim));
|
|
737
1134
|
}
|
|
738
1135
|
/**
|
|
739
1136
|
* Storage-only: recover lost inflight claims, claim due messages up to the
|
|
@@ -742,24 +1139,38 @@ var QueueDO = class extends DurableObject {
|
|
|
742
1139
|
async #claimPhase() {
|
|
743
1140
|
const storage = this.ctx.storage;
|
|
744
1141
|
const now = this.#now();
|
|
745
|
-
const maxInflight = this.#
|
|
1142
|
+
const maxInflight = this.#maxInflight();
|
|
746
1143
|
return await storage.transaction(async (txn) => {
|
|
747
|
-
const
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
1144
|
+
const expiredPage = await txn.list({
|
|
1145
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1146
|
+
end: `${INFLIGHT_DEADLINE_PREFIX}${pad(now + 1)}`,
|
|
1147
|
+
limit: 129
|
|
1148
|
+
});
|
|
1149
|
+
const expired = Array.from(expiredPage.entries()).slice(0, EXPIRED_INFLIGHT_BATCH);
|
|
1150
|
+
const expiredRows = await getMany$1(txn, expired.map(([, messageId]) => `msg:${messageId}`));
|
|
1151
|
+
await deleteMany$1(txn, expired.map(([key]) => key));
|
|
1152
|
+
const recoveryWrites = [];
|
|
1153
|
+
for (const [, messageId] of expired) {
|
|
1154
|
+
const row = expiredRows.get(`msg:${messageId}`);
|
|
753
1155
|
if (row) {
|
|
754
1156
|
const scheduleKey = dueKey(now, messageId);
|
|
755
|
-
|
|
756
|
-
if (row.runId)
|
|
1157
|
+
recoveryWrites.push([scheduleKey, messageId]);
|
|
1158
|
+
if (row.runId) recoveryWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
757
1159
|
messageId: row.messageId,
|
|
758
1160
|
dueKey: scheduleKey,
|
|
759
1161
|
idempotencyKey: row.idempotencyKey
|
|
760
|
-
});
|
|
1162
|
+
}]);
|
|
761
1163
|
}
|
|
762
|
-
}
|
|
1164
|
+
}
|
|
1165
|
+
await putMany(txn, recoveryWrites);
|
|
1166
|
+
if (expiredPage.size > EXPIRED_INFLIGHT_BATCH) {
|
|
1167
|
+
await txn.setAlarm(now + MIN_ALARM_DELAY_MS);
|
|
1168
|
+
return [];
|
|
1169
|
+
}
|
|
1170
|
+
const inflightCount = (await txn.list({
|
|
1171
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1172
|
+
limit: maxInflight
|
|
1173
|
+
})).size;
|
|
763
1174
|
const rows = [];
|
|
764
1175
|
if (inflightCount < maxInflight) {
|
|
765
1176
|
const due = await txn.list({
|
|
@@ -768,19 +1179,31 @@ var QueueDO = class extends DurableObject {
|
|
|
768
1179
|
limit: maxInflight - inflightCount
|
|
769
1180
|
});
|
|
770
1181
|
const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
1182
|
+
const dueRows = await getMany$1(txn, Array.from(due.values(), (messageId) => `msg:${messageId}`));
|
|
1183
|
+
await deleteMany$1(txn, Array.from(due.keys()));
|
|
1184
|
+
const claimWrites = [];
|
|
1185
|
+
for (const [, messageId] of due) {
|
|
1186
|
+
const row = dueRows.get(`msg:${messageId}`);
|
|
774
1187
|
if (!row) continue;
|
|
775
|
-
|
|
776
|
-
|
|
1188
|
+
const claimKey = inflightKey(now + timeoutMs + INFLIGHT_GRACE_MS, messageId);
|
|
1189
|
+
claimWrites.push([claimKey, messageId]);
|
|
1190
|
+
if (row.runId) claimWrites.push([runReferenceKey(row.runId, row.messageId), {
|
|
1191
|
+
messageId: row.messageId,
|
|
1192
|
+
inflightKey: claimKey,
|
|
1193
|
+
idempotencyKey: row.idempotencyKey
|
|
1194
|
+
}]);
|
|
1195
|
+
rows.push({
|
|
1196
|
+
row,
|
|
1197
|
+
inflightKey: claimKey
|
|
1198
|
+
});
|
|
777
1199
|
}
|
|
1200
|
+
await putMany(txn, claimWrites);
|
|
778
1201
|
}
|
|
779
1202
|
await this.#scheduleNextAlarm(txn, now);
|
|
780
1203
|
return rows;
|
|
781
1204
|
});
|
|
782
1205
|
}
|
|
783
|
-
async #deliver(row) {
|
|
1206
|
+
async #deliver({ row, inflightKey: claimKey }) {
|
|
784
1207
|
const storage = this.ctx.storage;
|
|
785
1208
|
const timeoutMs = this.#intVar("QUEUE_DELIVERY_TIMEOUT_MS", DEFAULT_DELIVERY_TIMEOUT_MS);
|
|
786
1209
|
const callbackSecret = this.env?.WORKFLOW_CALLBACK_SECRET;
|
|
@@ -804,8 +1227,10 @@ var QueueDO = class extends DurableObject {
|
|
|
804
1227
|
transportError = error;
|
|
805
1228
|
}
|
|
806
1229
|
const now = this.#now();
|
|
807
|
-
if (response?.ok)
|
|
808
|
-
|
|
1230
|
+
if (response?.ok) {
|
|
1231
|
+
await cancelUnusedResponseBody(response);
|
|
1232
|
+
await this.#ack(row, claimKey);
|
|
1233
|
+
} else if (response && response.status === 503) {
|
|
809
1234
|
let timeoutSeconds;
|
|
810
1235
|
try {
|
|
811
1236
|
timeoutSeconds = (await response.json())?.timeoutSeconds;
|
|
@@ -813,9 +1238,9 @@ var QueueDO = class extends DurableObject {
|
|
|
813
1238
|
timeoutSeconds = void 0;
|
|
814
1239
|
}
|
|
815
1240
|
if (typeof timeoutSeconds === "number") await storage.transaction(async (txn) => {
|
|
1241
|
+
if (!await txn.delete(claimKey)) return;
|
|
816
1242
|
if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
|
|
817
1243
|
await txn.delete(`msg:${row.messageId}`);
|
|
818
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
819
1244
|
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
820
1245
|
if (row.idempotencyKey) {
|
|
821
1246
|
const dedupKey = `key:${row.idempotencyKey}`;
|
|
@@ -824,7 +1249,6 @@ var QueueDO = class extends DurableObject {
|
|
|
824
1249
|
await this.#scheduleNextAlarm(txn, now);
|
|
825
1250
|
return;
|
|
826
1251
|
}
|
|
827
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
828
1252
|
const scheduleKey = dueKey(now + timeoutSeconds * 1e3, row.messageId);
|
|
829
1253
|
await txn.put(scheduleKey, row.messageId);
|
|
830
1254
|
if (row.runId) await txn.put(runReferenceKey(row.runId, row.messageId), {
|
|
@@ -834,17 +1258,20 @@ var QueueDO = class extends DurableObject {
|
|
|
834
1258
|
});
|
|
835
1259
|
await this.#scheduleNextAlarm(txn, now);
|
|
836
1260
|
});
|
|
837
|
-
else await this.#retry(row, "HTTP 503");
|
|
838
|
-
} else if (response && PERMANENT_STATUSES.has(response.status))
|
|
839
|
-
|
|
1261
|
+
else await this.#retry(row, claimKey, "HTTP 503");
|
|
1262
|
+
} else if (response && PERMANENT_STATUSES.has(response.status)) {
|
|
1263
|
+
await cancelUnusedResponseBody(response);
|
|
1264
|
+
await this.#ack(row, claimKey);
|
|
1265
|
+
} else {
|
|
840
1266
|
const reason = response ? `HTTP ${response.status}` : `transport error: ${String(transportError)}`;
|
|
841
|
-
await
|
|
1267
|
+
if (response) await cancelUnusedResponseBody(response);
|
|
1268
|
+
await this.#retry(row, claimKey, reason);
|
|
842
1269
|
}
|
|
843
1270
|
}
|
|
844
|
-
async #ack(row) {
|
|
1271
|
+
async #ack(row, claimKey) {
|
|
845
1272
|
await this.ctx.storage.transaction(async (txn) => {
|
|
1273
|
+
if (!await txn.delete(claimKey)) return;
|
|
846
1274
|
await txn.delete(`msg:${row.messageId}`);
|
|
847
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
848
1275
|
if (row.idempotencyKey) {
|
|
849
1276
|
if (await txn.get(`key:${row.idempotencyKey}`) === row.messageId) await txn.delete(`key:${row.idempotencyKey}`);
|
|
850
1277
|
}
|
|
@@ -852,15 +1279,15 @@ var QueueDO = class extends DurableObject {
|
|
|
852
1279
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
853
1280
|
});
|
|
854
1281
|
}
|
|
855
|
-
async #retry(row, reason) {
|
|
1282
|
+
async #retry(row, claimKey, reason) {
|
|
856
1283
|
const storage = this.ctx.storage;
|
|
857
1284
|
const now = this.#now();
|
|
858
1285
|
const maxAttempts = this.#intVar("QUEUE_MAX_ATTEMPTS", DEFAULT_MAX_ATTEMPTS);
|
|
859
1286
|
const attempt = row.attempt + 1;
|
|
860
1287
|
await storage.transaction(async (txn) => {
|
|
1288
|
+
if (!await txn.delete(claimKey)) return;
|
|
861
1289
|
if (row.runId && await txn.get(expiredRunKey(row.runId)) !== void 0) {
|
|
862
1290
|
await txn.delete(`msg:${row.messageId}`);
|
|
863
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
864
1291
|
await txn.delete(runReferenceKey(row.runId, row.messageId));
|
|
865
1292
|
if (row.idempotencyKey) {
|
|
866
1293
|
const dedupKey = `key:${row.idempotencyKey}`;
|
|
@@ -869,7 +1296,6 @@ var QueueDO = class extends DurableObject {
|
|
|
869
1296
|
await this.#scheduleNextAlarm(txn, now);
|
|
870
1297
|
return;
|
|
871
1298
|
}
|
|
872
|
-
await txn.delete(`inflight:${row.messageId}`);
|
|
873
1299
|
if (attempt >= maxAttempts) {
|
|
874
1300
|
const dead = {
|
|
875
1301
|
...row,
|
|
@@ -929,8 +1355,14 @@ var QueueDO = class extends DurableObject {
|
|
|
929
1355
|
limit: 1
|
|
930
1356
|
});
|
|
931
1357
|
for (const key of due.keys()) next = Number.parseInt(key.slice(4, 17), 10);
|
|
932
|
-
const inflight = await storage.list({
|
|
933
|
-
|
|
1358
|
+
const inflight = await storage.list({
|
|
1359
|
+
prefix: INFLIGHT_DEADLINE_PREFIX,
|
|
1360
|
+
limit: 1
|
|
1361
|
+
});
|
|
1362
|
+
for (const key of inflight.keys()) {
|
|
1363
|
+
const deadline = deadlineFromInflightKey(key);
|
|
1364
|
+
if (next === null || deadline < next) next = deadline;
|
|
1365
|
+
}
|
|
934
1366
|
if (next !== null) await storage.setAlarm(next <= now ? now + MIN_ALARM_DELAY_MS : next);
|
|
935
1367
|
else await storage.deleteAlarm();
|
|
936
1368
|
}
|
|
@@ -938,7 +1370,7 @@ var QueueDO = class extends DurableObject {
|
|
|
938
1370
|
const storage = this.ctx.storage;
|
|
939
1371
|
const [due, inflight, dlq, alarmAt] = await Promise.all([
|
|
940
1372
|
storage.list({ prefix: "due:" }),
|
|
941
|
-
storage.list({ prefix:
|
|
1373
|
+
storage.list({ prefix: INFLIGHT_DEADLINE_PREFIX }),
|
|
942
1374
|
storage.list({ prefix: "dlq:" }),
|
|
943
1375
|
storage.getAlarm()
|
|
944
1376
|
]);
|
|
@@ -1003,13 +1435,16 @@ var QueueDO = class extends DurableObject {
|
|
|
1003
1435
|
}
|
|
1004
1436
|
async purgeDeadLetters() {
|
|
1005
1437
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1006
|
-
const
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
return {
|
|
1438
|
+
const entries = await txn.list({
|
|
1439
|
+
prefix: "dlq:",
|
|
1440
|
+
limit: 129
|
|
1441
|
+
});
|
|
1442
|
+
const page = Array.from(entries).slice(0, PURGE_DEAD_LETTER_BATCH);
|
|
1443
|
+
await deleteMany$1(txn, page.flatMap(([key, dead]) => [key, ...dead.runId ? [runReferenceKey(dead.runId, dead.messageId)] : []]));
|
|
1444
|
+
return {
|
|
1445
|
+
purged: page.length,
|
|
1446
|
+
hasMore: entries.size > PURGE_DEAD_LETTER_BATCH
|
|
1447
|
+
};
|
|
1013
1448
|
});
|
|
1014
1449
|
}
|
|
1015
1450
|
/**
|
|
@@ -1021,140 +1456,384 @@ var QueueDO = class extends DurableObject {
|
|
|
1021
1456
|
return { alarmAt: await this.ctx.storage.getAlarm() };
|
|
1022
1457
|
}
|
|
1023
1458
|
/** Fence a run and purge every live, inflight, and dead-letter message it owns. */
|
|
1024
|
-
async expireRun(runId, expiredAt) {
|
|
1025
|
-
|
|
1459
|
+
async expireRun(runId, expiredAt, options) {
|
|
1460
|
+
const storage = this.ctx.storage;
|
|
1461
|
+
const limit = boundedLimit$1(options?.limit, EXPIRE_RUN_REFERENCE_BATCH, EXPIRE_RUN_REFERENCE_BATCH);
|
|
1462
|
+
return await storage.transaction(async (txn) => {
|
|
1026
1463
|
const existing = await txn.get(expiredRunKey(runId));
|
|
1027
|
-
const references = await txn.list({
|
|
1464
|
+
const references = await txn.list({
|
|
1465
|
+
prefix: `run:${runId}:`,
|
|
1466
|
+
limit: limit + 1
|
|
1467
|
+
});
|
|
1468
|
+
const page = Array.from(references).slice(0, limit);
|
|
1469
|
+
const dedupHolders = await getMany$1(txn, page.flatMap(([, reference]) => reference.idempotencyKey ? [`key:${reference.idempotencyKey}`] : []));
|
|
1470
|
+
const keys = [];
|
|
1028
1471
|
let count = existing?.deleted ?? 0;
|
|
1029
|
-
for (const [key, reference] of
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
if (reference.dueKey)
|
|
1033
|
-
if (reference.dlqKey)
|
|
1472
|
+
for (const [key, reference] of page) {
|
|
1473
|
+
keys.push(`msg:${reference.messageId}`, key);
|
|
1474
|
+
if (reference.inflightKey) keys.push(reference.inflightKey);
|
|
1475
|
+
if (reference.dueKey) keys.push(reference.dueKey);
|
|
1476
|
+
if (reference.dlqKey) keys.push(reference.dlqKey);
|
|
1034
1477
|
if (reference.idempotencyKey) {
|
|
1035
1478
|
const dedupKey = `key:${reference.idempotencyKey}`;
|
|
1036
|
-
if (
|
|
1479
|
+
if (dedupHolders.get(dedupKey) === reference.messageId) keys.push(dedupKey);
|
|
1037
1480
|
}
|
|
1038
|
-
await txn.delete(key);
|
|
1039
1481
|
count++;
|
|
1040
1482
|
}
|
|
1483
|
+
await deleteMany$1(txn, keys);
|
|
1041
1484
|
await txn.put(expiredRunKey(runId), {
|
|
1042
1485
|
expiredAt,
|
|
1043
1486
|
deleted: count
|
|
1044
1487
|
});
|
|
1045
1488
|
await this.#scheduleNextAlarm(txn, this.#now());
|
|
1046
|
-
return
|
|
1047
|
-
|
|
1489
|
+
return {
|
|
1490
|
+
deleted: count,
|
|
1491
|
+
done: references.size <= limit
|
|
1492
|
+
};
|
|
1493
|
+
});
|
|
1494
|
+
}
|
|
1495
|
+
};
|
|
1496
|
+
//#endregion
|
|
1497
|
+
//#region src/worker/durable-objects/RunCatalogDO.ts
|
|
1498
|
+
const MAX_INDEX_LIST_SIZE = 1001;
|
|
1499
|
+
/** One of a fixed set of run-catalog shards, routed by the stable run ID hash. */
|
|
1500
|
+
var RunCatalogDO = class extends DurableObject {
|
|
1501
|
+
async upsertRun(runId, keys, serializedMetadata) {
|
|
1502
|
+
if (keys.length !== 2) throw new Error("Run catalog commit requires exactly two keys");
|
|
1503
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1504
|
+
if (await txn.get(`expired:${runId}`) !== void 0) return { stored: false };
|
|
1505
|
+
await txn.put(Object.fromEntries(keys.map((key) => [key, serializedMetadata])));
|
|
1506
|
+
return { stored: true };
|
|
1507
|
+
});
|
|
1508
|
+
}
|
|
1509
|
+
async list(options = {}) {
|
|
1510
|
+
const suppliedLimit = options.limit;
|
|
1511
|
+
const limit = Math.min(MAX_INDEX_LIST_SIZE, Math.max(1, typeof suppliedLimit === "number" && Number.isFinite(suppliedLimit) ? Math.floor(suppliedLimit) : MAX_INDEX_LIST_SIZE));
|
|
1512
|
+
const entries = await this.ctx.storage.list({
|
|
1513
|
+
prefix: options.prefix ?? "",
|
|
1514
|
+
...options.reverse ? {
|
|
1515
|
+
reverse: true,
|
|
1516
|
+
end: options.cursor
|
|
1517
|
+
} : { startAfter: options.cursor },
|
|
1518
|
+
limit: limit + 1
|
|
1519
|
+
});
|
|
1520
|
+
const page = Array.from(entries).slice(0, limit);
|
|
1521
|
+
const listComplete = entries.size <= limit;
|
|
1522
|
+
return {
|
|
1523
|
+
keys: page.map(([name, value]) => ({
|
|
1524
|
+
name,
|
|
1525
|
+
value
|
|
1526
|
+
})),
|
|
1527
|
+
list_complete: listComplete,
|
|
1528
|
+
cursor: listComplete ? void 0 : page.at(-1)?.[0]
|
|
1529
|
+
};
|
|
1530
|
+
}
|
|
1531
|
+
async expireRun(runId, keys, expiredAt) {
|
|
1532
|
+
if (keys.length !== 2) throw new Error("Run catalog expiry requires exactly two keys");
|
|
1533
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1534
|
+
await txn.put(`expired:${runId}`, expiredAt);
|
|
1535
|
+
return { deleted: await txn.delete(keys) };
|
|
1536
|
+
});
|
|
1537
|
+
}
|
|
1538
|
+
};
|
|
1539
|
+
//#endregion
|
|
1540
|
+
//#region src/worker/durable-objects/RunFenceDO.ts
|
|
1541
|
+
/** Per-run lifecycle fence used by hook shards without a global lookup cell. */
|
|
1542
|
+
var RunFenceDO = class extends DurableObject {
|
|
1543
|
+
async getStatus() {
|
|
1544
|
+
return (await this.ctx.storage.get("fence"))?.status ?? "active";
|
|
1545
|
+
}
|
|
1546
|
+
async fenceTerminal(at) {
|
|
1547
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1548
|
+
const current = await txn.get("fence");
|
|
1549
|
+
if (current) return current.status;
|
|
1550
|
+
await txn.put("fence", {
|
|
1551
|
+
status: "terminal",
|
|
1552
|
+
at
|
|
1553
|
+
});
|
|
1554
|
+
return "terminal";
|
|
1555
|
+
});
|
|
1556
|
+
}
|
|
1557
|
+
async fenceExpired(at) {
|
|
1558
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
1559
|
+
const current = await txn.get("fence");
|
|
1560
|
+
if (current?.status === "expired") return current.status;
|
|
1561
|
+
await txn.put("fence", {
|
|
1562
|
+
status: "expired",
|
|
1563
|
+
at
|
|
1564
|
+
});
|
|
1565
|
+
return "expired";
|
|
1566
|
+
});
|
|
1048
1567
|
}
|
|
1049
1568
|
};
|
|
1050
1569
|
//#endregion
|
|
1051
1570
|
//#region src/worker/durable-objects/StreamDO.ts
|
|
1052
1571
|
const META_KEY = "meta";
|
|
1053
1572
|
const CHUNK_KEY_PREFIX = "chunk:";
|
|
1573
|
+
const CHUNK_SIZE_KEY_PREFIX = "chunk-size:";
|
|
1054
1574
|
/** Registry keys used when this DO instance acts as a per-run stream index. */
|
|
1055
1575
|
const STREAM_REGISTRY_PREFIX = "stream:";
|
|
1056
1576
|
const REGISTRY_OWNER_KEY = "registry:owner";
|
|
1057
1577
|
const REGISTRY_EXPIRED_KEY = "registry:expired";
|
|
1058
|
-
/**
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
*/
|
|
1578
|
+
/** Each chunk occupies a payload key and a size key; one delete accepts at most 128 keys. */
|
|
1579
|
+
const DEFAULT_EXPIRE_CHUNK_LIMIT = 64;
|
|
1580
|
+
const MAX_EXPIRE_CHUNK_LIMIT = 64;
|
|
1581
|
+
/** Bound storage deletion work as well as item count; one oversized chunk still makes progress. */
|
|
1582
|
+
const DEFAULT_EXPIRE_BYTE_LIMIT = 16777216;
|
|
1583
|
+
const MAX_EXPIRE_BYTE_LIMIT = DEFAULT_EXPIRE_BYTE_LIMIT;
|
|
1584
|
+
function boundedLimit(value, fallback, maximum) {
|
|
1585
|
+
if (value === void 0 || !Number.isFinite(value)) return fallback;
|
|
1586
|
+
return Math.max(1, Math.min(maximum, Math.floor(value)));
|
|
1587
|
+
}
|
|
1588
|
+
function emptyMeta() {
|
|
1589
|
+
return {
|
|
1590
|
+
count: 0,
|
|
1591
|
+
state: "open"
|
|
1592
|
+
};
|
|
1593
|
+
}
|
|
1594
|
+
function validateMeta(meta) {
|
|
1595
|
+
if (!Number.isSafeInteger(meta.count) || meta.count < 0 || meta.count > 2147483647 || ![
|
|
1596
|
+
"open",
|
|
1597
|
+
"closed",
|
|
1598
|
+
"errored",
|
|
1599
|
+
"expired"
|
|
1600
|
+
].includes(meta.state)) throw new Error("Invalid persisted stream metadata");
|
|
1601
|
+
if (meta.state === "errored" && !meta.error) throw new Error("Invalid persisted stream error metadata");
|
|
1602
|
+
return meta;
|
|
1603
|
+
}
|
|
1604
|
+
/** Zero-padding keeps storage.list() results in stream offset order. */
|
|
1062
1605
|
function chunkKey(index) {
|
|
1063
1606
|
return `${CHUNK_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
|
|
1064
1607
|
}
|
|
1608
|
+
function chunkSizeKey(index) {
|
|
1609
|
+
return `${CHUNK_SIZE_KEY_PREFIX}${index.toString().padStart(12, "0")}`;
|
|
1610
|
+
}
|
|
1611
|
+
function indexFromChunkSizeKey(key) {
|
|
1612
|
+
const suffix = key.slice(11);
|
|
1613
|
+
const index = Number(suffix);
|
|
1614
|
+
if (!/^\d{12}$/.test(suffix) || !Number.isSafeInteger(index) || index < 0) throw new Error(`Invalid persisted stream chunk size key "${key}"`);
|
|
1615
|
+
return index;
|
|
1616
|
+
}
|
|
1617
|
+
function validateChunkSize(size, index) {
|
|
1618
|
+
if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0 || size > 1048576) throw new Error(`Invalid persisted stream chunk size at index ${index}`);
|
|
1619
|
+
return size;
|
|
1620
|
+
}
|
|
1621
|
+
function compactChunk(chunk) {
|
|
1622
|
+
return chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength ? chunk : new Uint8Array(chunk);
|
|
1623
|
+
}
|
|
1624
|
+
function abortReason(signal) {
|
|
1625
|
+
return signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
|
1626
|
+
}
|
|
1065
1627
|
/**
|
|
1066
1628
|
* Durable Object backing workflow streams.
|
|
1067
1629
|
*
|
|
1068
|
-
*
|
|
1069
|
-
*
|
|
1070
|
-
*
|
|
1071
|
-
* each chunk its index; one chunk per storage key.
|
|
1072
|
-
* - `run-streams:<runId>`: registry of stream names owned by a run,
|
|
1073
|
-
* powering `listStreamsByRunId`.
|
|
1630
|
+
* Stream cells persist binary chunks under monotonic offset keys. Public
|
|
1631
|
+
* reads are bounded long polls, and writes append a bounded ordered batch in
|
|
1632
|
+
* one storage transaction. Per-run registry cells retain their existing role.
|
|
1074
1633
|
*/
|
|
1075
1634
|
var StreamDO = class extends DurableObject {
|
|
1635
|
+
meta;
|
|
1636
|
+
metaLoad;
|
|
1637
|
+
mutationTail = Promise.resolve();
|
|
1638
|
+
changeVersion = 0;
|
|
1639
|
+
waiters = /* @__PURE__ */ new Set();
|
|
1076
1640
|
async getMeta() {
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1641
|
+
if (this.meta) return this.meta;
|
|
1642
|
+
this.metaLoad ??= this.ctx.storage.get(META_KEY).then((meta) => {
|
|
1643
|
+
const loaded = meta ? validateMeta(meta) : emptyMeta();
|
|
1644
|
+
this.meta = loaded;
|
|
1645
|
+
return loaded;
|
|
1646
|
+
});
|
|
1647
|
+
try {
|
|
1648
|
+
return await this.metaLoad;
|
|
1649
|
+
} catch (error) {
|
|
1650
|
+
this.metaLoad = void 0;
|
|
1651
|
+
throw error;
|
|
1652
|
+
}
|
|
1081
1653
|
}
|
|
1082
1654
|
assertOwner(meta, runId) {
|
|
1083
|
-
if (
|
|
1655
|
+
if (!runId) throw new Error("Stream runId is required");
|
|
1084
1656
|
if (meta.ownerRunId !== void 0 && meta.ownerRunId !== runId) throw new Error(`Stream is owned by workflow run "${meta.ownerRunId}"`);
|
|
1085
1657
|
}
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1658
|
+
assertWritable(meta, runId) {
|
|
1659
|
+
this.assertOwner(meta, runId);
|
|
1660
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
1661
|
+
if (meta.state === "closed") throw new Error("Cannot write to a closed stream");
|
|
1662
|
+
if (meta.state === "errored") throw new Error("Cannot write to an errored stream");
|
|
1663
|
+
}
|
|
1664
|
+
runMutation(operation) {
|
|
1665
|
+
const result = this.mutationTail.then(operation, operation);
|
|
1666
|
+
this.mutationTail = result.then(() => void 0, () => void 0);
|
|
1667
|
+
return result;
|
|
1668
|
+
}
|
|
1669
|
+
wakeReaders() {
|
|
1670
|
+
this.changeVersion += 1;
|
|
1671
|
+
for (const waiter of Array.from(this.waiters)) waiter.resolve("change");
|
|
1672
|
+
}
|
|
1673
|
+
waitForChange(observedVersion, waitMs, signal) {
|
|
1674
|
+
if (signal?.aborted) return Promise.reject(abortReason(signal));
|
|
1675
|
+
return new Promise((resolve, reject) => {
|
|
1676
|
+
let settled = false;
|
|
1677
|
+
const waiter = {
|
|
1678
|
+
timer: setTimeout(() => finish("timeout"), waitMs),
|
|
1679
|
+
signal,
|
|
1680
|
+
resolve: (reason) => finish(reason)
|
|
1681
|
+
};
|
|
1682
|
+
const cleanup = () => {
|
|
1683
|
+
clearTimeout(waiter.timer);
|
|
1684
|
+
this.waiters.delete(waiter);
|
|
1685
|
+
if (waiter.signal && waiter.onAbort) waiter.signal.removeEventListener("abort", waiter.onAbort);
|
|
1686
|
+
};
|
|
1687
|
+
const finish = (reason) => {
|
|
1688
|
+
if (settled) return;
|
|
1689
|
+
settled = true;
|
|
1690
|
+
cleanup();
|
|
1691
|
+
resolve(reason);
|
|
1095
1692
|
};
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1693
|
+
waiter.onAbort = () => {
|
|
1694
|
+
if (settled) return;
|
|
1695
|
+
settled = true;
|
|
1696
|
+
cleanup();
|
|
1697
|
+
reject(abortReason(signal));
|
|
1698
|
+
};
|
|
1699
|
+
this.waiters.add(waiter);
|
|
1700
|
+
signal?.addEventListener("abort", waiter.onAbort, { once: true });
|
|
1701
|
+
if (this.changeVersion !== observedVersion) finish("change");
|
|
1702
|
+
});
|
|
1703
|
+
}
|
|
1704
|
+
async readSnapshot(request, timedOut) {
|
|
1705
|
+
const meta = await this.getMeta();
|
|
1706
|
+
this.assertOwner(meta, request.runId);
|
|
1707
|
+
const available = meta.state === "expired" ? 0 : Math.max(0, Math.min(request.maxChunks, meta.count - request.startIndex));
|
|
1708
|
+
const chunks = [];
|
|
1709
|
+
if (available > 0) {
|
|
1710
|
+
const indexes = Array.from({ length: available }, (_, offset) => request.startIndex + offset);
|
|
1711
|
+
const sizeKeys = indexes.map(chunkSizeKey);
|
|
1712
|
+
const storedSizes = await this.ctx.storage.get(sizeKeys);
|
|
1713
|
+
const selectedIndexes = [];
|
|
1714
|
+
const selectedSizes = [];
|
|
1715
|
+
let bytes = 0;
|
|
1716
|
+
for (const index of indexes) {
|
|
1717
|
+
const size = validateChunkSize(storedSizes.get(chunkSizeKey(index)), index);
|
|
1718
|
+
if (bytes + size > request.maxBytes) break;
|
|
1719
|
+
selectedIndexes.push(index);
|
|
1720
|
+
selectedSizes.push(size);
|
|
1721
|
+
bytes += size;
|
|
1722
|
+
}
|
|
1723
|
+
const payloadKeys = selectedIndexes.map(chunkKey);
|
|
1724
|
+
const storedChunks = await this.ctx.storage.get(payloadKeys);
|
|
1725
|
+
for (let offset = 0; offset < payloadKeys.length; offset++) {
|
|
1726
|
+
const chunk = storedChunks.get(payloadKeys[offset]);
|
|
1727
|
+
if (!(chunk instanceof Uint8Array) || chunk.byteLength !== selectedSizes[offset]) throw new Error(`Invalid persisted stream chunk at index ${selectedIndexes[offset]}`);
|
|
1728
|
+
chunks.push(chunk);
|
|
1729
|
+
}
|
|
1730
|
+
}
|
|
1731
|
+
return {
|
|
1732
|
+
startIndex: request.startIndex,
|
|
1733
|
+
tailIndex: meta.count - 1,
|
|
1734
|
+
chunks,
|
|
1735
|
+
state: meta.state,
|
|
1736
|
+
timedOut,
|
|
1737
|
+
error: meta.error
|
|
1738
|
+
};
|
|
1739
|
+
}
|
|
1740
|
+
/** Append one bounded binary batch with contiguous indexes. */
|
|
1741
|
+
async writeChunks(runId, chunks) {
|
|
1742
|
+
validateStreamWriteChunks(chunks);
|
|
1743
|
+
const storedChunks = chunks.map(compactChunk);
|
|
1744
|
+
return await this.runMutation(async () => {
|
|
1745
|
+
const committed = await this.ctx.storage.transaction(async (txn) => {
|
|
1746
|
+
const stored = await txn.get(META_KEY);
|
|
1747
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
1748
|
+
this.assertWritable(meta, runId);
|
|
1749
|
+
const startIndex = meta.count;
|
|
1750
|
+
if (startIndex + chunks.length > 2147483647) throw new Error("Stream offset limit exceeded");
|
|
1751
|
+
const nextMeta = {
|
|
1752
|
+
count: startIndex + chunks.length,
|
|
1753
|
+
state: "open",
|
|
1754
|
+
ownerRunId: runId
|
|
1755
|
+
};
|
|
1756
|
+
const entries = { [META_KEY]: nextMeta };
|
|
1757
|
+
for (let offset = 0; offset < storedChunks.length; offset++) {
|
|
1758
|
+
const index = startIndex + offset;
|
|
1759
|
+
entries[chunkKey(index)] = storedChunks[offset];
|
|
1760
|
+
entries[chunkSizeKey(index)] = storedChunks[offset].byteLength;
|
|
1761
|
+
}
|
|
1762
|
+
await txn.put(entries);
|
|
1763
|
+
return {
|
|
1764
|
+
meta: nextMeta,
|
|
1765
|
+
result: {
|
|
1766
|
+
startIndex,
|
|
1767
|
+
count: chunks.length,
|
|
1768
|
+
tailIndex: nextMeta.count - 1
|
|
1769
|
+
}
|
|
1770
|
+
};
|
|
1104
1771
|
});
|
|
1105
|
-
|
|
1772
|
+
this.meta = committed.meta;
|
|
1773
|
+
this.metaLoad = Promise.resolve(committed.meta);
|
|
1774
|
+
this.wakeReaders();
|
|
1775
|
+
return committed.result;
|
|
1106
1776
|
});
|
|
1107
1777
|
}
|
|
1108
1778
|
/**
|
|
1109
|
-
*
|
|
1110
|
-
*
|
|
1779
|
+
* Return available binary chunks and terminal metadata in one bounded
|
|
1780
|
+
* operation, waiting at most waitMs when the requested offset is idle.
|
|
1111
1781
|
*/
|
|
1782
|
+
async readChunks(request, signal) {
|
|
1783
|
+
validateStreamReadRequest(request);
|
|
1784
|
+
const observedVersion = this.changeVersion;
|
|
1785
|
+
const immediate = await this.readSnapshot(request, false);
|
|
1786
|
+
if (immediate.chunks.length > 0 || immediate.state !== "open" || request.waitMs === 0 || request.maxChunks === 0) return immediate;
|
|
1787
|
+
const reason = await this.waitForChange(observedVersion, request.waitMs, signal);
|
|
1788
|
+
return await this.readSnapshot(request, reason === "timeout");
|
|
1789
|
+
}
|
|
1790
|
+
/** Close idempotently; the first terminal state wins. */
|
|
1112
1791
|
async closeStream(runId) {
|
|
1113
|
-
await this.
|
|
1114
|
-
const
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1792
|
+
await this.runMutation(async () => {
|
|
1793
|
+
const nextMeta = await this.ctx.storage.transaction(async (txn) => {
|
|
1794
|
+
const stored = await txn.get(META_KEY);
|
|
1795
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
1796
|
+
this.assertOwner(meta, runId);
|
|
1797
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
1798
|
+
if (meta.state !== "open") return meta;
|
|
1799
|
+
const closed = {
|
|
1800
|
+
...meta,
|
|
1801
|
+
ownerRunId: runId,
|
|
1802
|
+
state: "closed"
|
|
1803
|
+
};
|
|
1804
|
+
await txn.put(META_KEY, closed);
|
|
1805
|
+
return closed;
|
|
1123
1806
|
});
|
|
1807
|
+
this.meta = nextMeta;
|
|
1808
|
+
this.metaLoad = Promise.resolve(nextMeta);
|
|
1809
|
+
this.wakeReaders();
|
|
1124
1810
|
});
|
|
1125
1811
|
}
|
|
1126
|
-
/**
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1812
|
+
/** Persist a terminal stream failure and wake every pending reader. */
|
|
1813
|
+
async failStream(runId, error) {
|
|
1814
|
+
const normalized = normalizeStreamError(error);
|
|
1815
|
+
await this.runMutation(async () => {
|
|
1816
|
+
const nextMeta = await this.ctx.storage.transaction(async (txn) => {
|
|
1817
|
+
const stored = await txn.get(META_KEY);
|
|
1818
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
1819
|
+
this.assertOwner(meta, runId);
|
|
1820
|
+
if (meta.state === "expired") throw new Error(`Workflow run "${runId}" has expired`);
|
|
1821
|
+
if (meta.state !== "open") return meta;
|
|
1822
|
+
const failed = {
|
|
1823
|
+
...meta,
|
|
1824
|
+
ownerRunId: runId,
|
|
1825
|
+
state: "errored",
|
|
1826
|
+
error: normalized
|
|
1827
|
+
};
|
|
1828
|
+
await txn.put(META_KEY, failed);
|
|
1829
|
+
return failed;
|
|
1830
|
+
});
|
|
1831
|
+
this.meta = nextMeta;
|
|
1832
|
+
this.metaLoad = Promise.resolve(nextMeta);
|
|
1833
|
+
this.wakeReaders();
|
|
1137
1834
|
});
|
|
1138
|
-
return {
|
|
1139
|
-
chunks: Array.from(entries.values()),
|
|
1140
|
-
done: meta.closed,
|
|
1141
|
-
tailIndex: meta.count - 1
|
|
1142
|
-
};
|
|
1143
1835
|
}
|
|
1144
|
-
/**
|
|
1145
|
-
* Lightweight stream metadata: last chunk index and completion flag.
|
|
1146
|
-
*/
|
|
1147
|
-
async getInfo() {
|
|
1148
|
-
const meta = await this.getMeta();
|
|
1149
|
-
if (meta.expiredAt !== void 0) throw new Error("Stream has expired");
|
|
1150
|
-
return {
|
|
1151
|
-
tailIndex: meta.count - 1,
|
|
1152
|
-
done: meta.closed
|
|
1153
|
-
};
|
|
1154
|
-
}
|
|
1155
|
-
/**
|
|
1156
|
-
* Register a stream name against this per-run registry instance.
|
|
1157
|
-
*/
|
|
1836
|
+
/** Register a stream name against this per-run registry instance. */
|
|
1158
1837
|
async registerStream(runId, name) {
|
|
1159
1838
|
await this.ctx.storage.transaction(async (txn) => {
|
|
1160
1839
|
if (await txn.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error(`Workflow run "${runId}" has expired`);
|
|
@@ -1164,57 +1843,124 @@ var StreamDO = class extends DurableObject {
|
|
|
1164
1843
|
await txn.put(`${STREAM_REGISTRY_PREFIX}${name}`, true);
|
|
1165
1844
|
});
|
|
1166
1845
|
}
|
|
1167
|
-
/**
|
|
1168
|
-
* List stream names registered against this per-run registry instance.
|
|
1169
|
-
*/
|
|
1170
1846
|
async listStreams() {
|
|
1171
1847
|
if (await this.ctx.storage.get(REGISTRY_EXPIRED_KEY) !== void 0) throw new Error("Workflow run streams have expired");
|
|
1172
1848
|
const entries = await this.ctx.storage.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1173
1849
|
return Array.from(entries.keys()).map((key) => key.slice(7));
|
|
1174
1850
|
}
|
|
1175
1851
|
/** Fence a run's registry before any stream cells are removed. */
|
|
1176
|
-
async expireRegistry(runId, expiredAt) {
|
|
1852
|
+
async expireRegistry(runId, expiredAt, options) {
|
|
1853
|
+
const limit = boundedLimit(options?.limit, 16, 128);
|
|
1177
1854
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1178
|
-
const
|
|
1855
|
+
const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
|
|
1856
|
+
const owner = state.get(REGISTRY_OWNER_KEY);
|
|
1179
1857
|
if (owner !== void 0 && owner !== runId) throw new Error(`Stream registry is owned by workflow run "${owner}"`);
|
|
1180
|
-
|
|
1181
|
-
await txn.put(
|
|
1182
|
-
|
|
1858
|
+
const expiry = state.get(REGISTRY_EXPIRED_KEY);
|
|
1859
|
+
await txn.put({
|
|
1860
|
+
[REGISTRY_OWNER_KEY]: runId,
|
|
1861
|
+
[REGISTRY_EXPIRED_KEY]: expiry ?? {
|
|
1862
|
+
expiredAt,
|
|
1863
|
+
deleted: 0
|
|
1864
|
+
}
|
|
1865
|
+
});
|
|
1866
|
+
const entries = await txn.list({
|
|
1867
|
+
prefix: STREAM_REGISTRY_PREFIX,
|
|
1868
|
+
limit
|
|
1869
|
+
});
|
|
1183
1870
|
return { streams: Array.from(entries.keys()).map((key) => key.slice(7)) };
|
|
1184
1871
|
});
|
|
1185
1872
|
}
|
|
1186
|
-
/** Remove
|
|
1187
|
-
async finalizeRegistry(runId) {
|
|
1188
|
-
await this.ctx.storage.transaction(async (txn) => {
|
|
1189
|
-
const owner = await txn.get(REGISTRY_OWNER_KEY);
|
|
1190
|
-
const expiredAt = await txn.get(REGISTRY_EXPIRED_KEY);
|
|
1191
|
-
if (owner !== runId || expiredAt === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
|
|
1192
|
-
const entries = await txn.list({ prefix: STREAM_REGISTRY_PREFIX });
|
|
1193
|
-
for (const key of entries.keys()) await txn.delete(key);
|
|
1194
|
-
});
|
|
1195
|
-
}
|
|
1196
|
-
/** Delete stream payload while leaving a small ownership tombstone. */
|
|
1197
|
-
async expireStream(runId, expiredAt) {
|
|
1873
|
+
/** Remove completed registry entries in a bounded, retry-safe batch. */
|
|
1874
|
+
async finalizeRegistry(runId, streams) {
|
|
1198
1875
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1199
|
-
const
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
};
|
|
1203
|
-
|
|
1204
|
-
const
|
|
1205
|
-
|
|
1206
|
-
await txn.put(
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
ownerRunId: runId,
|
|
1210
|
-
expiredAt
|
|
1876
|
+
const state = await txn.get([REGISTRY_OWNER_KEY, REGISTRY_EXPIRED_KEY]);
|
|
1877
|
+
const owner = state.get(REGISTRY_OWNER_KEY);
|
|
1878
|
+
const expiry = state.get(REGISTRY_EXPIRED_KEY);
|
|
1879
|
+
if (owner !== runId || expiry === void 0) throw new Error(`Stream registry for workflow run "${runId}" is not expired`);
|
|
1880
|
+
const keys = Array.from(new Set(streams.map((name) => `${STREAM_REGISTRY_PREFIX}${name}`)));
|
|
1881
|
+
const removed = keys.length === 0 ? 0 : await txn.delete(keys);
|
|
1882
|
+
const deleted = expiry.deleted + removed;
|
|
1883
|
+
await txn.put(REGISTRY_EXPIRED_KEY, {
|
|
1884
|
+
...expiry,
|
|
1885
|
+
deleted
|
|
1211
1886
|
});
|
|
1212
1887
|
return {
|
|
1213
|
-
deleted
|
|
1214
|
-
|
|
1888
|
+
deleted,
|
|
1889
|
+
done: (await txn.list({
|
|
1890
|
+
prefix: STREAM_REGISTRY_PREFIX,
|
|
1891
|
+
limit: 1
|
|
1892
|
+
})).size === 0
|
|
1215
1893
|
};
|
|
1216
1894
|
});
|
|
1217
1895
|
}
|
|
1896
|
+
/**
|
|
1897
|
+
* Fence the stream and delete one bounded page. Listing size keys avoids
|
|
1898
|
+
* loading payloads, and the transaction makes a crash retry resume safely.
|
|
1899
|
+
*/
|
|
1900
|
+
async expireStream(runId, expiredAt, options) {
|
|
1901
|
+
const limit = boundedLimit(options?.limit, DEFAULT_EXPIRE_CHUNK_LIMIT, MAX_EXPIRE_CHUNK_LIMIT);
|
|
1902
|
+
const byteLimit = boundedLimit(options?.byteLimit, DEFAULT_EXPIRE_BYTE_LIMIT, MAX_EXPIRE_BYTE_LIMIT);
|
|
1903
|
+
return await this.runMutation(async () => {
|
|
1904
|
+
const result = await this.ctx.storage.transaction(async (txn) => {
|
|
1905
|
+
const stored = await txn.get(META_KEY);
|
|
1906
|
+
const meta = stored ? validateMeta(stored) : emptyMeta();
|
|
1907
|
+
this.assertOwner(meta, runId);
|
|
1908
|
+
const firstFence = meta.state !== "expired";
|
|
1909
|
+
if (meta.state === "expired" && meta.payloadDeleted) return {
|
|
1910
|
+
meta,
|
|
1911
|
+
result: {
|
|
1912
|
+
deleted: false,
|
|
1913
|
+
chunks: 0,
|
|
1914
|
+
bytes: 0,
|
|
1915
|
+
done: true
|
|
1916
|
+
}
|
|
1917
|
+
};
|
|
1918
|
+
const candidates = await txn.list({
|
|
1919
|
+
prefix: CHUNK_SIZE_KEY_PREFIX,
|
|
1920
|
+
limit: limit + 1
|
|
1921
|
+
});
|
|
1922
|
+
const page = [];
|
|
1923
|
+
let bytes = 0;
|
|
1924
|
+
for (const [key, storedSize] of Array.from(candidates).slice(0, limit)) {
|
|
1925
|
+
const index = indexFromChunkSizeKey(key);
|
|
1926
|
+
const size = validateChunkSize(storedSize, index);
|
|
1927
|
+
if (page.length > 0 && bytes + size > byteLimit) break;
|
|
1928
|
+
page.push({
|
|
1929
|
+
index,
|
|
1930
|
+
size
|
|
1931
|
+
});
|
|
1932
|
+
bytes += size;
|
|
1933
|
+
}
|
|
1934
|
+
const keys = page.flatMap(({ index }) => [chunkKey(index), chunkSizeKey(index)]);
|
|
1935
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
1936
|
+
const done = candidates.size === page.length;
|
|
1937
|
+
const nextMeta = {
|
|
1938
|
+
...meta,
|
|
1939
|
+
count: done ? 0 : meta.count,
|
|
1940
|
+
ownerRunId: runId,
|
|
1941
|
+
state: "expired",
|
|
1942
|
+
error: void 0,
|
|
1943
|
+
expiredAt: meta.expiredAt ?? expiredAt,
|
|
1944
|
+
expiredChunkCount: meta.expiredChunkCount ?? meta.count,
|
|
1945
|
+
payloadDeleted: done
|
|
1946
|
+
};
|
|
1947
|
+
await txn.put(META_KEY, nextMeta);
|
|
1948
|
+
return {
|
|
1949
|
+
meta: nextMeta,
|
|
1950
|
+
result: {
|
|
1951
|
+
deleted: firstFence,
|
|
1952
|
+
chunks: page.length,
|
|
1953
|
+
bytes,
|
|
1954
|
+
done
|
|
1955
|
+
}
|
|
1956
|
+
};
|
|
1957
|
+
});
|
|
1958
|
+
this.meta = result.meta;
|
|
1959
|
+
this.metaLoad = Promise.resolve(result.meta);
|
|
1960
|
+
this.wakeReaders();
|
|
1961
|
+
return result.result;
|
|
1962
|
+
});
|
|
1963
|
+
}
|
|
1218
1964
|
};
|
|
1219
1965
|
//#endregion
|
|
1220
1966
|
//#region src/apply-event.ts
|
|
@@ -1224,6 +1970,7 @@ const RUN_KEY = "run";
|
|
|
1224
1970
|
const EVENT_KEY_PREFIX = "event:";
|
|
1225
1971
|
const STEP_KEY_PREFIX = "step:";
|
|
1226
1972
|
const HOOK_KEY_PREFIX = "hook:";
|
|
1973
|
+
const WAIT_KEY_PREFIX = "wait:";
|
|
1227
1974
|
const STEP_CREATED_KEY_PREFIX = "stepcreated:";
|
|
1228
1975
|
const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
|
|
1229
1976
|
/**
|
|
@@ -1233,6 +1980,36 @@ const HOOK_CREATED_KEY_PREFIX = "hookcreated:";
|
|
|
1233
1980
|
* rejected as a duplicate instead of resurrecting the hook.
|
|
1234
1981
|
*/
|
|
1235
1982
|
const HOOK_EVENT_MARKER_PREFIX = "hookevent:";
|
|
1983
|
+
/** Durable repair record for idempotent post-commit hook-index release. */
|
|
1984
|
+
const HOOK_RELEASE_MARKER_PREFIX = "hookreleased:";
|
|
1985
|
+
/** Exact ambiguous reservations canceled by a serialized RunDO resolution. */
|
|
1986
|
+
const HOOK_CLAIM_CANCELLATION_PREFIX = "hookclaimcancelled:";
|
|
1987
|
+
/** Workflow-authored attr_set correlation claims. */
|
|
1988
|
+
const ATTRIBUTE_EVENT_MARKER_PREFIX = "attrevent:";
|
|
1989
|
+
/** Durable lazy-hook resume claims, scoped by the containing run DO. */
|
|
1990
|
+
const HOOK_RESUME_CLAIM_PREFIX = "hookresume:";
|
|
1991
|
+
const STORAGE_BATCH_SIZE$1 = 128;
|
|
1992
|
+
const MAX_ENTITY_PAGE_SIZE = 1e3;
|
|
1993
|
+
async function getMany(store, keys) {
|
|
1994
|
+
if (store.getMany) {
|
|
1995
|
+
const result = /* @__PURE__ */ new Map();
|
|
1996
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) {
|
|
1997
|
+
const page = await store.getMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
1998
|
+
for (const entry of page) result.set(...entry);
|
|
1999
|
+
}
|
|
2000
|
+
return result;
|
|
2001
|
+
}
|
|
2002
|
+
const values = await Promise.all(keys.map((key) => store.get(key)));
|
|
2003
|
+
return new Map(keys.flatMap((key, index) => values[index] === void 0 ? [] : [[key, values[index]]]));
|
|
2004
|
+
}
|
|
2005
|
+
async function deleteMany(store, keys) {
|
|
2006
|
+
if (store.deleteMany) {
|
|
2007
|
+
let deleted = 0;
|
|
2008
|
+
for (let offset = 0; offset < keys.length; offset += STORAGE_BATCH_SIZE$1) deleted += await store.deleteMany(keys.slice(offset, offset + STORAGE_BATCH_SIZE$1));
|
|
2009
|
+
return deleted;
|
|
2010
|
+
}
|
|
2011
|
+
return (await Promise.all(keys.map((key) => store.delete(key)))).filter(Boolean).length;
|
|
2012
|
+
}
|
|
1236
2013
|
function failure(code, message, extra) {
|
|
1237
2014
|
return {
|
|
1238
2015
|
ok: false,
|
|
@@ -1241,6 +2018,9 @@ function failure(code, message, extra) {
|
|
|
1241
2018
|
...extra
|
|
1242
2019
|
};
|
|
1243
2020
|
}
|
|
2021
|
+
function hookClaimCancellationKey(claimId) {
|
|
2022
|
+
return `${HOOK_CLAIM_CANCELLATION_PREFIX}${encodeURIComponent(claimId)}`;
|
|
2023
|
+
}
|
|
1244
2024
|
function readStringProp(value, key) {
|
|
1245
2025
|
if (value !== null && typeof value === "object") {
|
|
1246
2026
|
const prop = value[key];
|
|
@@ -1296,7 +2076,9 @@ function creationIndexKey(prefix, createdAt, id) {
|
|
|
1296
2076
|
* requested ordering.
|
|
1297
2077
|
*/
|
|
1298
2078
|
async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
1299
|
-
const
|
|
2079
|
+
const requestedLimit = Number.isFinite(params.limit) ? Math.floor(params.limit) : 1;
|
|
2080
|
+
const limit = Math.min(MAX_ENTITY_PAGE_SIZE, Math.max(1, requestedLimit));
|
|
2081
|
+
const { cursor, sortOrder = "asc" } = params;
|
|
1300
2082
|
const entries = await store.list(sortOrder === "desc" ? {
|
|
1301
2083
|
prefix: indexPrefix,
|
|
1302
2084
|
reverse: true,
|
|
@@ -1308,11 +2090,12 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
|
1308
2090
|
startAfter: cursor ? `${indexPrefix}${cursor}` : void 0
|
|
1309
2091
|
});
|
|
1310
2092
|
const page = Array.from(entries.entries()).slice(0, limit);
|
|
1311
|
-
const
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
2093
|
+
const entityKeys = page.map(([, id]) => `${entityPrefix}${id}`);
|
|
2094
|
+
const entities = await getMany(store, entityKeys);
|
|
2095
|
+
const data = entityKeys.flatMap((key) => {
|
|
2096
|
+
const item = entities.get(key);
|
|
2097
|
+
return item === void 0 ? [] : [item];
|
|
2098
|
+
});
|
|
1316
2099
|
const hasMore = entries.size > limit;
|
|
1317
2100
|
const lastKey = page.at(-1)?.[0];
|
|
1318
2101
|
return {
|
|
@@ -1323,21 +2106,31 @@ async function listByCreationTime(store, indexPrefix, entityPrefix, params) {
|
|
|
1323
2106
|
}
|
|
1324
2107
|
/** Delete all hook entities for the run, returning the released tokens. */
|
|
1325
2108
|
async function releaseAllHooks(store) {
|
|
2109
|
+
if (store.deferTerminalCleanup) return [];
|
|
1326
2110
|
const hooks = await store.list({ prefix: HOOK_KEY_PREFIX });
|
|
1327
2111
|
const released = [];
|
|
2112
|
+
const keys = [];
|
|
1328
2113
|
for (const hook of hooks.values()) {
|
|
1329
|
-
|
|
1330
|
-
|
|
2114
|
+
keys.push(`${HOOK_KEY_PREFIX}${hook.hookId}`);
|
|
2115
|
+
keys.push(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
|
|
1331
2116
|
released.push({
|
|
1332
2117
|
hookId: hook.hookId,
|
|
1333
2118
|
token: hook.token
|
|
1334
2119
|
});
|
|
1335
2120
|
}
|
|
2121
|
+
await deleteMany(store, keys);
|
|
1336
2122
|
return released;
|
|
1337
2123
|
}
|
|
2124
|
+
/** Delete all persisted waits once their owning run becomes terminal. */
|
|
2125
|
+
async function releaseAllWaits(store) {
|
|
2126
|
+
if (store.deferTerminalCleanup) return;
|
|
2127
|
+
const waits = await store.list({ prefix: WAIT_KEY_PREFIX });
|
|
2128
|
+
await deleteMany(store, Array.from(waits.keys()));
|
|
2129
|
+
}
|
|
1338
2130
|
async function applyEvent(store, ctx) {
|
|
1339
2131
|
const { runId, data, now } = ctx;
|
|
1340
2132
|
const specVersion = data.specVersion ?? SPEC_VERSION_CURRENT;
|
|
2133
|
+
if (specVersion !== SPEC_VERSION_CURRENT) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" uses unsupported specVersion ${specVersion}; this world requires ${SPEC_VERSION_CURRENT}`, { runSpecVersion: specVersion });
|
|
1341
2134
|
/** Build and validate the stored event BEFORE anything is persisted. */
|
|
1342
2135
|
const buildEvent = (record) => {
|
|
1343
2136
|
const withMeta = {
|
|
@@ -1345,9 +2138,16 @@ async function applyEvent(store, ctx) {
|
|
|
1345
2138
|
runId,
|
|
1346
2139
|
eventId: ctx.nextEventId(),
|
|
1347
2140
|
createdAt: now,
|
|
2141
|
+
occurredAt: ctx.params?.occurredAt,
|
|
2142
|
+
resumeId: ctx.params?.resumeId,
|
|
1348
2143
|
specVersion
|
|
1349
2144
|
};
|
|
2145
|
+
if (record.eventType !== "hook_received") delete withMeta.resumeId;
|
|
1350
2146
|
if (record.eventType === "run_started") delete withMeta.eventData;
|
|
2147
|
+
if (record.eventType === "step_started" && withMeta.eventData !== null && typeof withMeta.eventData === "object") {
|
|
2148
|
+
const { input: _input, ...eventData } = withMeta.eventData;
|
|
2149
|
+
withMeta.eventData = eventData;
|
|
2150
|
+
}
|
|
1351
2151
|
return EventSchema.parse(compact(withMeta));
|
|
1352
2152
|
};
|
|
1353
2153
|
const putEvent = async (event) => {
|
|
@@ -1358,14 +2158,21 @@ async function applyEvent(store, ctx) {
|
|
|
1358
2158
|
if (!skipRunValidation) currentRun = await store.get(RUN_KEY);
|
|
1359
2159
|
let bootstrapped;
|
|
1360
2160
|
if (data.eventType === "run_started" && !currentRun && data.eventData) {
|
|
1361
|
-
const { deploymentId, workflowName, input, executionContext } = data.eventData;
|
|
2161
|
+
const { deploymentId, workflowName, input, executionContext, attributes, allowReservedAttributes, encryptionPublicKey } = data.eventData;
|
|
1362
2162
|
if (deploymentId && workflowName && input !== void 0) {
|
|
2163
|
+
const attributeChanges = Object.entries(attributes ?? {}).map(([key, value]) => ({
|
|
2164
|
+
key,
|
|
2165
|
+
value
|
|
2166
|
+
}));
|
|
2167
|
+
validateAttributeChanges(attributeChanges, { allowReservedAttributes });
|
|
1363
2168
|
const createdRun = WorkflowRunSchema.parse(compact({
|
|
1364
2169
|
runId,
|
|
1365
2170
|
deploymentId,
|
|
1366
2171
|
workflowName,
|
|
1367
2172
|
specVersion,
|
|
1368
2173
|
executionContext,
|
|
2174
|
+
attributes,
|
|
2175
|
+
encryptionPublicKey,
|
|
1369
2176
|
input,
|
|
1370
2177
|
status: "pending",
|
|
1371
2178
|
output: void 0,
|
|
@@ -1381,7 +2188,10 @@ async function applyEvent(store, ctx) {
|
|
|
1381
2188
|
deploymentId,
|
|
1382
2189
|
workflowName,
|
|
1383
2190
|
input,
|
|
1384
|
-
executionContext
|
|
2191
|
+
executionContext,
|
|
2192
|
+
attributes,
|
|
2193
|
+
allowReservedAttributes,
|
|
2194
|
+
encryptionPublicKey
|
|
1385
2195
|
}
|
|
1386
2196
|
});
|
|
1387
2197
|
await store.put(RUN_KEY, createdRun);
|
|
@@ -1393,10 +2203,7 @@ async function applyEvent(store, ctx) {
|
|
|
1393
2203
|
};
|
|
1394
2204
|
}
|
|
1395
2205
|
}
|
|
1396
|
-
if (currentRun) {
|
|
1397
|
-
if (requiresNewerWorld(currentRun.specVersion)) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" requires a newer world (run specVersion ${currentRun.specVersion}, world ${SPEC_VERSION_CURRENT})`, { runSpecVersion: currentRun.specVersion });
|
|
1398
|
-
if (isLegacySpecVersion(currentRun.specVersion)) return failure("LEGACY_RUN_NOT_SUPPORTED", `Run "${runId}" was created with a legacy spec version (${currentRun.specVersion ?? "undefined"}) not supported by this world`);
|
|
1399
|
-
}
|
|
2206
|
+
if (currentRun && currentRun.specVersion !== SPEC_VERSION_CURRENT) return failure("RUN_NOT_SUPPORTED", `Run "${runId}" uses unsupported specVersion ${currentRun.specVersion}; this world requires ${SPEC_VERSION_CURRENT}`, { runSpecVersion: currentRun.specVersion });
|
|
1400
2207
|
if (currentRun && isTerminalWorkflowRunStatus(currentRun.status)) {
|
|
1401
2208
|
if (data.eventType === "run_cancelled" && currentRun.status === "cancelled") {
|
|
1402
2209
|
const event = buildEvent({ ...data });
|
|
@@ -1410,17 +2217,44 @@ async function applyEvent(store, ctx) {
|
|
|
1410
2217
|
}
|
|
1411
2218
|
if (data.eventType === "run_started") return failure("RUN_EXPIRED", `Workflow run "${runId}" is already in terminal state "${currentRun.status}"`);
|
|
1412
2219
|
if (data.eventType === "run_completed" || data.eventType === "run_failed" || data.eventType === "run_cancelled") return failure("ENTITY_CONFLICT", `Cannot transition run from terminal state "${currentRun.status}"`);
|
|
1413
|
-
if (data
|
|
2220
|
+
if (isChildEntityCreationEvent(data)) return failure("ENTITY_CONFLICT", `Cannot create new entities on run in terminal state "${currentRun.status}"`);
|
|
2221
|
+
if (data.eventType === "attr_set") return failure("ENTITY_CONFLICT", `Cannot set attributes on run in terminal state "${currentRun.status}"`);
|
|
1414
2222
|
}
|
|
1415
2223
|
let validatedStep;
|
|
1416
2224
|
if (data.eventType === "step_started" || data.eventType === "step_completed" || data.eventType === "step_failed" || data.eventType === "step_retrying") {
|
|
2225
|
+
const lazyStepStart = data.eventType === "step_started" && isChildEntityCreationEvent(data);
|
|
1417
2226
|
validatedStep = await store.get(`${STEP_KEY_PREFIX}${data.correlationId}`);
|
|
1418
|
-
if (!validatedStep)
|
|
1419
|
-
|
|
1420
|
-
|
|
2227
|
+
if (!validatedStep) {
|
|
2228
|
+
if (!lazyStepStart) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
2229
|
+
} else if (lazyStepStart) return failure("ENTITY_CONFLICT", `Step "${data.correlationId}" already exists`);
|
|
2230
|
+
if (validatedStep && isTerminalStepStatus(validatedStep.status)) return failure("ENTITY_CONFLICT", `Cannot modify step in terminal state "${validatedStep.status}"`);
|
|
2231
|
+
if (currentRun && isTerminalWorkflowRunStatus(currentRun.status) && validatedStep && validatedStep.status !== "running") return failure("RUN_EXPIRED", `Cannot modify non-running step on run in terminal state "${currentRun.status}"`);
|
|
2232
|
+
}
|
|
2233
|
+
if (data.eventType === "hook_received" && ctx.params?.resumeId) {
|
|
2234
|
+
const claim = await store.get(`${HOOK_RESUME_CLAIM_PREFIX}${ctx.params.resumeId}`);
|
|
2235
|
+
if (claim) {
|
|
2236
|
+
if (claim.hookId !== data.correlationId) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" already recorded for a different hook`);
|
|
2237
|
+
if (claim.payloadDigest && ctx.params.resumePayloadDigest && claim.payloadDigest !== ctx.params.resumePayloadDigest) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" already recorded with a different payload`);
|
|
2238
|
+
const committed = await store.get(`${EVENT_KEY_PREFIX}${claim.eventId}`);
|
|
2239
|
+
if (committed?.eventType === "hook_received" && committed.correlationId === claim.hookId && committed.resumeId === ctx.params.resumeId) return {
|
|
2240
|
+
ok: true,
|
|
2241
|
+
event: committed,
|
|
2242
|
+
releasedHooks: []
|
|
2243
|
+
};
|
|
2244
|
+
if (committed) return failure("ENTITY_CONFLICT", `hook_received resumeId "${ctx.params.resumeId}" points to a different event`);
|
|
2245
|
+
}
|
|
1421
2246
|
}
|
|
1422
2247
|
if ((data.eventType === "hook_disposed" || data.eventType === "hook_received") && data.correlationId) {
|
|
1423
|
-
if (!await store.get(`hook:${data.correlationId}`))
|
|
2248
|
+
if (!await store.get(`hook:${data.correlationId}`)) {
|
|
2249
|
+
if (data.eventType === "hook_disposed") {
|
|
2250
|
+
const released = await store.get(`${HOOK_RELEASE_MARKER_PREFIX}${data.correlationId}`);
|
|
2251
|
+
if (released) return {
|
|
2252
|
+
ok: true,
|
|
2253
|
+
releasedHooks: [released]
|
|
2254
|
+
};
|
|
2255
|
+
}
|
|
2256
|
+
return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
2257
|
+
}
|
|
1424
2258
|
}
|
|
1425
2259
|
if (!skipRunValidation && data.eventType !== "run_started" && !currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1426
2260
|
switch (data.eventType) {
|
|
@@ -1431,12 +2265,19 @@ async function applyEvent(store, ctx) {
|
|
|
1431
2265
|
run: existing,
|
|
1432
2266
|
releasedHooks: []
|
|
1433
2267
|
};
|
|
2268
|
+
const attributeChanges = Object.entries(data.eventData.attributes ?? {}).map(([key, value]) => ({
|
|
2269
|
+
key,
|
|
2270
|
+
value
|
|
2271
|
+
}));
|
|
2272
|
+
validateAttributeChanges(attributeChanges, { allowReservedAttributes: data.eventData.allowReservedAttributes });
|
|
1434
2273
|
const run = WorkflowRunSchema.parse(compact({
|
|
1435
2274
|
runId,
|
|
1436
2275
|
deploymentId: data.eventData.deploymentId,
|
|
1437
2276
|
workflowName: data.eventData.workflowName,
|
|
1438
2277
|
specVersion,
|
|
1439
2278
|
executionContext: data.eventData.executionContext,
|
|
2279
|
+
attributes: data.eventData.attributes,
|
|
2280
|
+
encryptionPublicKey: data.eventData.encryptionPublicKey,
|
|
1440
2281
|
input: data.eventData.input,
|
|
1441
2282
|
status: "pending",
|
|
1442
2283
|
output: void 0,
|
|
@@ -1462,11 +2303,25 @@ async function applyEvent(store, ctx) {
|
|
|
1462
2303
|
}
|
|
1463
2304
|
case "run_started": {
|
|
1464
2305
|
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
1465
|
-
if (currentRun.status === "running")
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
2306
|
+
if (currentRun.status === "running") {
|
|
2307
|
+
const result = {
|
|
2308
|
+
ok: true,
|
|
2309
|
+
run: currentRun,
|
|
2310
|
+
releasedHooks: [],
|
|
2311
|
+
maxEvents: getMaxEventsPerRun()
|
|
2312
|
+
};
|
|
2313
|
+
if (ctx.params?.skipPreload) return result;
|
|
2314
|
+
const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
2315
|
+
limit: getMaxEventsPerRun(),
|
|
2316
|
+
sortOrder: "asc"
|
|
2317
|
+
}, (event) => event.eventId);
|
|
2318
|
+
return {
|
|
2319
|
+
...result,
|
|
2320
|
+
events: allEvents.data,
|
|
2321
|
+
cursor: allEvents.cursor,
|
|
2322
|
+
hasMore: allEvents.hasMore
|
|
2323
|
+
};
|
|
2324
|
+
}
|
|
1470
2325
|
const run = WorkflowRunSchema.parse(compact({
|
|
1471
2326
|
...currentRun,
|
|
1472
2327
|
status: "running",
|
|
@@ -1479,17 +2334,24 @@ async function applyEvent(store, ctx) {
|
|
|
1479
2334
|
const event = buildEvent({ ...data });
|
|
1480
2335
|
await store.put(RUN_KEY, run);
|
|
1481
2336
|
await putEvent(event);
|
|
1482
|
-
const
|
|
1483
|
-
limit: 1e3,
|
|
1484
|
-
sortOrder: "asc"
|
|
1485
|
-
}, (e) => e.eventId);
|
|
1486
|
-
return {
|
|
2337
|
+
const result = {
|
|
1487
2338
|
ok: true,
|
|
1488
2339
|
event,
|
|
1489
2340
|
run,
|
|
1490
2341
|
releasedHooks: [],
|
|
1491
2342
|
runCreated: bootstrapped,
|
|
1492
|
-
|
|
2343
|
+
maxEvents: getMaxEventsPerRun()
|
|
2344
|
+
};
|
|
2345
|
+
if (ctx.params?.skipPreload) return result;
|
|
2346
|
+
const allEvents = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
2347
|
+
limit: getMaxEventsPerRun(),
|
|
2348
|
+
sortOrder: "asc"
|
|
2349
|
+
}, (e) => e.eventId);
|
|
2350
|
+
return {
|
|
2351
|
+
...result,
|
|
2352
|
+
events: allEvents.data,
|
|
2353
|
+
cursor: allEvents.cursor,
|
|
2354
|
+
hasMore: allEvents.hasMore
|
|
1493
2355
|
};
|
|
1494
2356
|
}
|
|
1495
2357
|
case "run_completed": {
|
|
@@ -1505,11 +2367,13 @@ async function applyEvent(store, ctx) {
|
|
|
1505
2367
|
const event = buildEvent({ ...data });
|
|
1506
2368
|
await store.put(RUN_KEY, run);
|
|
1507
2369
|
await putEvent(event);
|
|
2370
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
2371
|
+
await releaseAllWaits(store);
|
|
1508
2372
|
return {
|
|
1509
2373
|
ok: true,
|
|
1510
2374
|
event,
|
|
1511
2375
|
run,
|
|
1512
|
-
releasedHooks
|
|
2376
|
+
releasedHooks
|
|
1513
2377
|
};
|
|
1514
2378
|
}
|
|
1515
2379
|
case "run_failed": {
|
|
@@ -1525,11 +2389,13 @@ async function applyEvent(store, ctx) {
|
|
|
1525
2389
|
const event = buildEvent({ ...data });
|
|
1526
2390
|
await store.put(RUN_KEY, run);
|
|
1527
2391
|
await putEvent(event);
|
|
2392
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
2393
|
+
await releaseAllWaits(store);
|
|
1528
2394
|
return {
|
|
1529
2395
|
ok: true,
|
|
1530
2396
|
event,
|
|
1531
2397
|
run,
|
|
1532
|
-
releasedHooks
|
|
2398
|
+
releasedHooks
|
|
1533
2399
|
};
|
|
1534
2400
|
}
|
|
1535
2401
|
case "run_cancelled": {
|
|
@@ -1545,11 +2411,37 @@ async function applyEvent(store, ctx) {
|
|
|
1545
2411
|
const event = buildEvent({ ...data });
|
|
1546
2412
|
await store.put(RUN_KEY, run);
|
|
1547
2413
|
await putEvent(event);
|
|
2414
|
+
const releasedHooks = await releaseAllHooks(store);
|
|
2415
|
+
await releaseAllWaits(store);
|
|
1548
2416
|
return {
|
|
1549
2417
|
ok: true,
|
|
1550
2418
|
event,
|
|
1551
2419
|
run,
|
|
1552
|
-
releasedHooks
|
|
2420
|
+
releasedHooks
|
|
2421
|
+
};
|
|
2422
|
+
}
|
|
2423
|
+
case "attr_set": {
|
|
2424
|
+
if (!currentRun) return failure("RUN_NOT_FOUND", `Workflow run "${runId}" not found`);
|
|
2425
|
+
validateAttributeChanges(data.eventData.changes, {
|
|
2426
|
+
existingKeys: Object.keys(currentRun.attributes),
|
|
2427
|
+
allowReservedAttributes: data.eventData.allowReservedAttributes
|
|
2428
|
+
});
|
|
2429
|
+
const markerKey = data.correlationId && data.eventData.writer.type === "workflow" ? `${ATTRIBUTE_EVENT_MARKER_PREFIX}${data.correlationId}` : void 0;
|
|
2430
|
+
if (markerKey && await store.get(markerKey) !== void 0) return failure("ENTITY_CONFLICT", `Attribute event "${data.correlationId}" already exists`);
|
|
2431
|
+
const run = WorkflowRunSchema.parse(compact({
|
|
2432
|
+
...currentRun,
|
|
2433
|
+
attributes: applyAttributeChanges(currentRun.attributes, data.eventData.changes),
|
|
2434
|
+
updatedAt: now
|
|
2435
|
+
}));
|
|
2436
|
+
const event = buildEvent({ ...data });
|
|
2437
|
+
await store.put(RUN_KEY, run);
|
|
2438
|
+
if (markerKey) await store.put(markerKey, event.eventId);
|
|
2439
|
+
await putEvent(event);
|
|
2440
|
+
return {
|
|
2441
|
+
ok: true,
|
|
2442
|
+
event,
|
|
2443
|
+
run,
|
|
2444
|
+
releasedHooks: []
|
|
1553
2445
|
};
|
|
1554
2446
|
}
|
|
1555
2447
|
case "step_created": {
|
|
@@ -1583,7 +2475,34 @@ async function applyEvent(store, ctx) {
|
|
|
1583
2475
|
};
|
|
1584
2476
|
}
|
|
1585
2477
|
case "step_started": {
|
|
1586
|
-
|
|
2478
|
+
let step = validatedStep;
|
|
2479
|
+
let stepCreated = false;
|
|
2480
|
+
if (!step && data.eventData?.input !== void 0 && data.eventData.stepName) {
|
|
2481
|
+
step = StepSchema.parse(compact({
|
|
2482
|
+
runId,
|
|
2483
|
+
stepId: data.correlationId,
|
|
2484
|
+
stepName: data.eventData.stepName,
|
|
2485
|
+
status: "pending",
|
|
2486
|
+
input: data.eventData.input,
|
|
2487
|
+
attempt: 0,
|
|
2488
|
+
createdAt: now,
|
|
2489
|
+
updatedAt: now,
|
|
2490
|
+
specVersion
|
|
2491
|
+
}));
|
|
2492
|
+
const createdEvent = buildEvent({
|
|
2493
|
+
eventType: "step_created",
|
|
2494
|
+
correlationId: data.correlationId,
|
|
2495
|
+
eventData: {
|
|
2496
|
+
stepName: data.eventData.stepName,
|
|
2497
|
+
workflowName: data.eventData.workflowName,
|
|
2498
|
+
input: data.eventData.input
|
|
2499
|
+
}
|
|
2500
|
+
});
|
|
2501
|
+
await store.put(`${STEP_KEY_PREFIX}${data.correlationId}`, step);
|
|
2502
|
+
await store.put(creationIndexKey(STEP_CREATED_KEY_PREFIX, step.createdAt, step.stepId), step.stepId);
|
|
2503
|
+
await putEvent(createdEvent);
|
|
2504
|
+
stepCreated = true;
|
|
2505
|
+
}
|
|
1587
2506
|
if (!step) return failure("STEP_NOT_FOUND", `Step "${data.correlationId}" not found`);
|
|
1588
2507
|
if (step.retryAfter && step.retryAfter.getTime() > now.getTime()) return failure("TOO_EARLY", `Cannot start step "${data.correlationId}": retryAfter timestamp has not been reached yet`, { retryAfterSeconds: Math.ceil((step.retryAfter.getTime() - now.getTime()) / 1e3) });
|
|
1589
2508
|
const updated = StepSchema.parse(compact({
|
|
@@ -1601,7 +2520,8 @@ async function applyEvent(store, ctx) {
|
|
|
1601
2520
|
ok: true,
|
|
1602
2521
|
event,
|
|
1603
2522
|
step: updated,
|
|
1604
|
-
releasedHooks: []
|
|
2523
|
+
releasedHooks: [],
|
|
2524
|
+
...stepCreated ? { stepCreated: true } : {}
|
|
1605
2525
|
};
|
|
1606
2526
|
}
|
|
1607
2527
|
case "step_completed": {
|
|
@@ -1630,7 +2550,7 @@ async function applyEvent(store, ctx) {
|
|
|
1630
2550
|
const updated = StepSchema.parse(compact({
|
|
1631
2551
|
...step,
|
|
1632
2552
|
status: "failed",
|
|
1633
|
-
error: toStructuredError(data.eventData.error
|
|
2553
|
+
error: toStructuredError(data.eventData.error),
|
|
1634
2554
|
completedAt: now,
|
|
1635
2555
|
updatedAt: now
|
|
1636
2556
|
}));
|
|
@@ -1650,7 +2570,7 @@ async function applyEvent(store, ctx) {
|
|
|
1650
2570
|
const updated = StepSchema.parse(compact({
|
|
1651
2571
|
...step,
|
|
1652
2572
|
status: "pending",
|
|
1653
|
-
error: toStructuredError(data.eventData.error
|
|
2573
|
+
error: toStructuredError(data.eventData.error),
|
|
1654
2574
|
retryAfter: data.eventData.retryAfter ? new Date(data.eventData.retryAfter) : void 0,
|
|
1655
2575
|
updatedAt: now
|
|
1656
2576
|
}));
|
|
@@ -1670,6 +2590,7 @@ async function applyEvent(store, ctx) {
|
|
|
1670
2590
|
const holder = ctx.tokenHolder ?? null;
|
|
1671
2591
|
const markerKey = `${HOOK_EVENT_MARKER_PREFIX}${hookId}`;
|
|
1672
2592
|
const hookKey = `${HOOK_KEY_PREFIX}${hookId}`;
|
|
2593
|
+
if (ctx.hookClaimId && await store.get(hookClaimCancellationKey(ctx.hookClaimId)) !== void 0) return failure("HOOK_CLAIM_CANCELLED", `Hook reservation for "${hookId}" was canceled before commit`);
|
|
1673
2594
|
if (holder && (holder.runId !== runId || holder.hookId !== hookId)) {
|
|
1674
2595
|
const event = buildEvent({
|
|
1675
2596
|
eventType: "hook_conflict",
|
|
@@ -1706,7 +2627,9 @@ async function applyEvent(store, ctx) {
|
|
|
1706
2627
|
environment: "",
|
|
1707
2628
|
createdAt: now,
|
|
1708
2629
|
specVersion,
|
|
1709
|
-
isWebhook: data.eventData.isWebhook ?? false
|
|
2630
|
+
isWebhook: data.eventData.isWebhook ?? false,
|
|
2631
|
+
isSystem: data.eventData.isSystem,
|
|
2632
|
+
tokenRetentionUntil: data.eventData.tokenRetentionUntil
|
|
1710
2633
|
}));
|
|
1711
2634
|
const event = buildEvent({ ...data });
|
|
1712
2635
|
await store.put(hookKey, hook);
|
|
@@ -1727,6 +2650,10 @@ async function applyEvent(store, ctx) {
|
|
|
1727
2650
|
if (!hook) return failure("HOOK_NOT_FOUND", `Hook "${data.correlationId}" not found`);
|
|
1728
2651
|
await store.delete(hookKey);
|
|
1729
2652
|
await store.delete(creationIndexKey(HOOK_CREATED_KEY_PREFIX, hook.createdAt, hook.hookId));
|
|
2653
|
+
await store.put(`${HOOK_RELEASE_MARKER_PREFIX}${hook.hookId}`, {
|
|
2654
|
+
hookId: hook.hookId,
|
|
2655
|
+
token: hook.token
|
|
2656
|
+
});
|
|
1730
2657
|
const event = buildEvent({ ...data });
|
|
1731
2658
|
await putEvent(event);
|
|
1732
2659
|
return {
|
|
@@ -1738,9 +2665,58 @@ async function applyEvent(store, ctx) {
|
|
|
1738
2665
|
}]
|
|
1739
2666
|
};
|
|
1740
2667
|
}
|
|
2668
|
+
case "wait_created": {
|
|
2669
|
+
const waitKey = `${WAIT_KEY_PREFIX}${data.correlationId}`;
|
|
2670
|
+
if (await store.get(waitKey) !== void 0) return failure("ENTITY_CONFLICT", `Wait "${data.correlationId}" already exists`);
|
|
2671
|
+
const wait = WaitSchema.parse({
|
|
2672
|
+
waitId: `${runId}-${data.correlationId}`,
|
|
2673
|
+
runId,
|
|
2674
|
+
status: "waiting",
|
|
2675
|
+
resumeAt: data.eventData.resumeAt,
|
|
2676
|
+
createdAt: now,
|
|
2677
|
+
updatedAt: now,
|
|
2678
|
+
specVersion
|
|
2679
|
+
});
|
|
2680
|
+
const event = buildEvent({ ...data });
|
|
2681
|
+
await store.put(waitKey, wait);
|
|
2682
|
+
await putEvent(event);
|
|
2683
|
+
return {
|
|
2684
|
+
ok: true,
|
|
2685
|
+
event,
|
|
2686
|
+
wait,
|
|
2687
|
+
releasedHooks: []
|
|
2688
|
+
};
|
|
2689
|
+
}
|
|
2690
|
+
case "wait_completed": {
|
|
2691
|
+
const waitKey = `${WAIT_KEY_PREFIX}${data.correlationId}`;
|
|
2692
|
+
const existing = await store.get(waitKey);
|
|
2693
|
+
if (!existing) return failure("WAIT_NOT_FOUND", `Wait "${data.correlationId}" not found`);
|
|
2694
|
+
if (existing.status === "completed") return failure("ENTITY_CONFLICT", `Wait "${data.correlationId}" already completed`);
|
|
2695
|
+
const event = buildEvent({ ...data });
|
|
2696
|
+
const wait = WaitSchema.parse({
|
|
2697
|
+
...existing,
|
|
2698
|
+
status: "completed",
|
|
2699
|
+
resumeAt: data.eventData?.resumeAt ?? existing.resumeAt,
|
|
2700
|
+
completedAt: now,
|
|
2701
|
+
updatedAt: now
|
|
2702
|
+
});
|
|
2703
|
+
await store.put(waitKey, wait);
|
|
2704
|
+
await putEvent(event);
|
|
2705
|
+
return {
|
|
2706
|
+
ok: true,
|
|
2707
|
+
event,
|
|
2708
|
+
wait,
|
|
2709
|
+
releasedHooks: []
|
|
2710
|
+
};
|
|
2711
|
+
}
|
|
1741
2712
|
default: {
|
|
1742
2713
|
const event = buildEvent({ ...data });
|
|
1743
2714
|
await putEvent(event);
|
|
2715
|
+
if (data.eventType === "hook_received" && ctx.params?.resumeId) await store.put(`${HOOK_RESUME_CLAIM_PREFIX}${ctx.params.resumeId}`, {
|
|
2716
|
+
hookId: data.correlationId,
|
|
2717
|
+
eventId: event.eventId,
|
|
2718
|
+
...ctx.params.resumePayloadDigest ? { payloadDigest: ctx.params.resumePayloadDigest } : {}
|
|
2719
|
+
});
|
|
1744
2720
|
return {
|
|
1745
2721
|
ok: true,
|
|
1746
2722
|
event,
|
|
@@ -1749,70 +2725,98 @@ async function applyEvent(store, ctx) {
|
|
|
1749
2725
|
}
|
|
1750
2726
|
}
|
|
1751
2727
|
}
|
|
1752
|
-
//#endregion
|
|
1753
|
-
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
1754
|
-
const PAYLOAD_DELETE_BATCH = 256;
|
|
1755
|
-
const STREAM_DELETE_CONCURRENCY = 16;
|
|
1756
|
-
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
1757
2728
|
/**
|
|
1758
|
-
*
|
|
1759
|
-
*
|
|
1760
|
-
*
|
|
1761
|
-
*
|
|
1762
|
-
* v2: one storage key per event ('event:<eventId>'), step ('step:<stepId>')
|
|
1763
|
-
* and hook ('hook:<hookId>') instead of single aggregate values. This package
|
|
1764
|
-
* is pre-production, so the migration simply drops the legacy aggregates.
|
|
1765
|
-
*/
|
|
1766
|
-
const CURRENT_SCHEMA_VERSION = 3;
|
|
1767
|
-
/**
|
|
1768
|
-
* Idempotent migration functions keyed by target version. The schema version
|
|
1769
|
-
* is written only after the migration completes, so eviction safely retries
|
|
1770
|
-
* an interrupted migration.
|
|
2729
|
+
* Complete Workflow 5's event-page response after an atomic apply. A celld
|
|
2730
|
+
* run is a single serialized writer, so the committed sequence is dense; a
|
|
2731
|
+
* stale eventCount therefore needs only the slots between the requested
|
|
2732
|
+
* position and the event this call committed.
|
|
1771
2733
|
*/
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
};
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
limit: 100
|
|
1792
|
-
});
|
|
1793
|
-
if (entities.size === 0) return;
|
|
1794
|
-
for (const [key, entity] of entities) {
|
|
1795
|
-
const id = getId(entity);
|
|
1796
|
-
await storage.put(`${indexPrefix}${entity.createdAt.toISOString()}:${id}`, id);
|
|
1797
|
-
startAfter = key;
|
|
1798
|
-
}
|
|
1799
|
-
if (entities.size < 100) return;
|
|
2734
|
+
async function finalizeEventPage(store, outcome, params) {
|
|
2735
|
+
if (!outcome.ok) return outcome;
|
|
2736
|
+
const result = outcome.run && outcome.maxEvents === void 0 ? {
|
|
2737
|
+
...outcome,
|
|
2738
|
+
maxEvents: getMaxEventsPerRun()
|
|
2739
|
+
} : outcome;
|
|
2740
|
+
if (!result.event) return result;
|
|
2741
|
+
if (typeof params?.sinceCursor === "string") {
|
|
2742
|
+
const page = await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
2743
|
+
limit: 100,
|
|
2744
|
+
cursor: params.sinceCursor,
|
|
2745
|
+
sortOrder: "asc"
|
|
2746
|
+
}, (event) => event.eventId);
|
|
2747
|
+
return {
|
|
2748
|
+
...result,
|
|
2749
|
+
events: page.data,
|
|
2750
|
+
cursor: page.cursor,
|
|
2751
|
+
hasMore: page.hasMore
|
|
2752
|
+
};
|
|
1800
2753
|
}
|
|
2754
|
+
if (params?.eventCount === void 0) return result;
|
|
2755
|
+
const committedSlot = eventIdToSlot(result.event.eventId);
|
|
2756
|
+
if (committedSlot === null || committedSlot <= params.eventCount + 1) return result;
|
|
2757
|
+
const skippedCount = committedSlot - params.eventCount - 1;
|
|
2758
|
+
const events = (await listByPrefix(store, EVENT_KEY_PREFIX, {
|
|
2759
|
+
limit: skippedCount,
|
|
2760
|
+
cursor: params.eventCount > 0 ? slotToEventId(params.eventCount) : void 0,
|
|
2761
|
+
sortOrder: "asc"
|
|
2762
|
+
}, (event) => event.eventId)).data.filter((event) => event.eventId < result.event.eventId);
|
|
2763
|
+
return {
|
|
2764
|
+
...result,
|
|
2765
|
+
events,
|
|
2766
|
+
cursor: null,
|
|
2767
|
+
hasMore: events.length < skippedCount
|
|
2768
|
+
};
|
|
1801
2769
|
}
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
2770
|
+
//#endregion
|
|
2771
|
+
//#region src/retention.ts
|
|
2772
|
+
const CLEANUP_RECORD_KEY = "retention:cleanup";
|
|
2773
|
+
const TOMBSTONE_KEY = "retention:tombstone";
|
|
2774
|
+
const HOOK_MARKER_PREFIX = "retention:hook:";
|
|
2775
|
+
const TERMINAL_CLEANUP_KEY = "terminal:cleanup";
|
|
2776
|
+
function sortableTimestamp(date) {
|
|
2777
|
+
return date.getTime().toString().padStart(13, "0");
|
|
2778
|
+
}
|
|
2779
|
+
function workflowRunIndexKey(run) {
|
|
2780
|
+
return `run:${run.workflowName}:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
2781
|
+
}
|
|
2782
|
+
function globalRunIndexKey(run) {
|
|
2783
|
+
return `runall:${sortableTimestamp(run.createdAt)}:${run.runId}`;
|
|
2784
|
+
}
|
|
2785
|
+
function hookMarkerKey(reference) {
|
|
2786
|
+
return `${HOOK_MARKER_PREFIX}${encodeURIComponent(reference.hookId)}:${encodeURIComponent(reference.token)}`;
|
|
2787
|
+
}
|
|
2788
|
+
function cleanupTombstone(record, now) {
|
|
2789
|
+
return {
|
|
2790
|
+
version: 1,
|
|
2791
|
+
runId: record.runId,
|
|
2792
|
+
terminalStatus: record.terminalStatus,
|
|
2793
|
+
completedAt: record.completedAt,
|
|
2794
|
+
expiredAt: record.dueAt,
|
|
2795
|
+
tombstonedAt: now
|
|
2796
|
+
};
|
|
2797
|
+
}
|
|
2798
|
+
function expiredRead(tombstone) {
|
|
2799
|
+
return {
|
|
2800
|
+
ok: false,
|
|
2801
|
+
code: "RUN_EXPIRED",
|
|
2802
|
+
message: `Workflow run "${tombstone.runId}" expired at ${tombstone.expiredAt.toISOString()}`,
|
|
2803
|
+
tombstone
|
|
2804
|
+
};
|
|
1815
2805
|
}
|
|
2806
|
+
//#endregion
|
|
2807
|
+
//#region src/worker/durable-objects/WorkflowRunDO.ts
|
|
2808
|
+
const STORAGE_BATCH_SIZE = 128;
|
|
2809
|
+
const PAYLOAD_DELETE_BATCH = STORAGE_BATCH_SIZE;
|
|
2810
|
+
const HOOK_MARKER_PAGE_SIZE = 64;
|
|
2811
|
+
const TERMINAL_HOOK_PAGE_SIZE = 64;
|
|
2812
|
+
const TERMINAL_WAIT_PAGE_SIZE = STORAGE_BATCH_SIZE;
|
|
2813
|
+
const STREAMS_PER_CLEANUP_PAGE = 16;
|
|
2814
|
+
const STREAM_CHUNKS_PER_CLEANUP_PAGE = STORAGE_BATCH_SIZE;
|
|
2815
|
+
const STREAM_BYTES_PER_CLEANUP_PAGE = 16777216;
|
|
2816
|
+
const QUEUE_REFERENCES_PER_CLEANUP_PAGE = 64;
|
|
2817
|
+
const QUEUE_SHARDS_PER_CLEANUP_PAGE = 8;
|
|
2818
|
+
const CLEANUP_RETRY_MAX_MS = 36e5;
|
|
2819
|
+
const CLEANUP_PROGRESS_KEY = "retention:progress";
|
|
1816
2820
|
/**
|
|
1817
2821
|
* Adapt DO storage (or a transaction) to the {@link EventStore} interface
|
|
1818
2822
|
* shared with the test mocks. No casts: the methods line up structurally,
|
|
@@ -1821,11 +2825,21 @@ async function ensureSchemaUpToDate(storage) {
|
|
|
1821
2825
|
function storeFrom(storage) {
|
|
1822
2826
|
return {
|
|
1823
2827
|
get: (key) => storage.get(key),
|
|
2828
|
+
getMany: (keys) => storage.get(keys),
|
|
1824
2829
|
put: (key, value) => storage.put(key, value),
|
|
1825
2830
|
delete: (key) => storage.delete(key),
|
|
1826
|
-
|
|
2831
|
+
deleteMany: (keys) => storage.delete(keys),
|
|
2832
|
+
list: (options) => storage.list(options),
|
|
2833
|
+
deferTerminalCleanup: true
|
|
1827
2834
|
};
|
|
1828
2835
|
}
|
|
2836
|
+
function hookCreationIndexKey(hook) {
|
|
2837
|
+
return `${HOOK_CREATED_KEY_PREFIX}${hook.createdAt.toISOString()}:${hook.hookId}`;
|
|
2838
|
+
}
|
|
2839
|
+
async function putStorageEntries(storage, entries) {
|
|
2840
|
+
const all = Array.from(entries);
|
|
2841
|
+
for (let offset = 0; offset < all.length; offset += STORAGE_BATCH_SIZE) await storage.put(Object.fromEntries(all.slice(offset, offset + STORAGE_BATCH_SIZE)));
|
|
2842
|
+
}
|
|
1829
2843
|
/**
|
|
1830
2844
|
* Durable Object for managing a single workflow run's state.
|
|
1831
2845
|
*
|
|
@@ -1837,37 +2851,55 @@ function storeFrom(storage) {
|
|
|
1837
2851
|
* queue-message dedup claims (see claimInflight/releaseInflight).
|
|
1838
2852
|
*/
|
|
1839
2853
|
var WorkflowRunDO = class extends DurableObject {
|
|
1840
|
-
schemaReady = null;
|
|
1841
2854
|
now() {
|
|
1842
2855
|
const clock = this.env?.clock;
|
|
1843
2856
|
return typeof clock === "function" ? clock() : Date.now();
|
|
1844
2857
|
}
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
return this.schemaReady;
|
|
1848
|
-
}
|
|
1849
|
-
async expiredTombstone(storage, now = this.now()) {
|
|
1850
|
-
const tombstone = await storage.get(TOMBSTONE_KEY);
|
|
2858
|
+
tombstoneFrom(values, now = this.now()) {
|
|
2859
|
+
const tombstone = values.get(TOMBSTONE_KEY);
|
|
1851
2860
|
if (tombstone) return tombstone;
|
|
1852
|
-
const cleanup =
|
|
2861
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
1853
2862
|
if (!cleanup || cleanup.phase === "retained" && cleanup.dueAt.getTime() > now) return null;
|
|
1854
2863
|
return cleanupTombstone(cleanup, cleanup.tombstonedAt ?? cleanup.dueAt);
|
|
1855
2864
|
}
|
|
1856
|
-
async
|
|
1857
|
-
await
|
|
1858
|
-
const tombstone = await this.expiredTombstone(this.ctx.storage);
|
|
1859
|
-
if (tombstone) return expiredRead(tombstone);
|
|
2865
|
+
async retentionState(storage, now = this.now()) {
|
|
2866
|
+
const values = await storage.get([TOMBSTONE_KEY, CLEANUP_RECORD_KEY]);
|
|
1860
2867
|
return {
|
|
1861
|
-
|
|
1862
|
-
|
|
2868
|
+
cleanup: values.get(CLEANUP_RECORD_KEY),
|
|
2869
|
+
tombstone: this.tombstoneFrom(values, now)
|
|
1863
2870
|
};
|
|
1864
2871
|
}
|
|
2872
|
+
async read(reader) {
|
|
2873
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
2874
|
+
const { tombstone } = await this.retentionState(txn);
|
|
2875
|
+
if (tombstone) return expiredRead(tombstone);
|
|
2876
|
+
return {
|
|
2877
|
+
ok: true,
|
|
2878
|
+
value: await reader(txn)
|
|
2879
|
+
};
|
|
2880
|
+
});
|
|
2881
|
+
}
|
|
2882
|
+
async readKey(key) {
|
|
2883
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
2884
|
+
const values = await txn.get([
|
|
2885
|
+
TOMBSTONE_KEY,
|
|
2886
|
+
CLEANUP_RECORD_KEY,
|
|
2887
|
+
key
|
|
2888
|
+
]);
|
|
2889
|
+
const tombstone = this.tombstoneFrom(values);
|
|
2890
|
+
if (tombstone) return expiredRead(tombstone);
|
|
2891
|
+
return {
|
|
2892
|
+
ok: true,
|
|
2893
|
+
value: values.get(key) ?? null
|
|
2894
|
+
};
|
|
2895
|
+
});
|
|
2896
|
+
}
|
|
1865
2897
|
/** Apply an event and capture all retention metadata in the same transaction. */
|
|
1866
2898
|
async applyEvent(request) {
|
|
1867
|
-
await this.ensureSchema();
|
|
1868
2899
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1869
2900
|
const now = new Date(this.now());
|
|
1870
|
-
const
|
|
2901
|
+
const retention = await this.retentionState(txn, now.getTime());
|
|
2902
|
+
const { tombstone } = retention;
|
|
1871
2903
|
if (tombstone) return {
|
|
1872
2904
|
ok: false,
|
|
1873
2905
|
code: "RUN_EXPIRED",
|
|
@@ -1876,25 +2908,17 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1876
2908
|
let eventSequence = await txn.get("event_sequence") ?? 0;
|
|
1877
2909
|
const outcome = await applyEvent(storeFrom(txn), {
|
|
1878
2910
|
...request,
|
|
1879
|
-
nextEventId: () =>
|
|
2911
|
+
nextEventId: () => slotToEventId(++eventSequence),
|
|
1880
2912
|
now
|
|
1881
2913
|
});
|
|
1882
2914
|
if (!outcome.ok) return outcome;
|
|
1883
2915
|
await txn.put("event_sequence", eventSequence);
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
}
|
|
1888
|
-
if (outcome.hookToIndex) {
|
|
1889
|
-
const reference = {
|
|
1890
|
-
hookId: outcome.hookToIndex.hookId,
|
|
1891
|
-
token: outcome.hookToIndex.token
|
|
1892
|
-
};
|
|
1893
|
-
await txn.put(hookMarkerKey(reference), reference);
|
|
1894
|
-
}
|
|
1895
|
-
for (const reference of outcome.releasedHooks) await txn.put(hookMarkerKey(reference), reference);
|
|
2916
|
+
await putStorageEntries(txn, (outcome.hookToIndex ? [{
|
|
2917
|
+
hookId: outcome.hookToIndex.hookId,
|
|
2918
|
+
token: outcome.hookToIndex.token
|
|
2919
|
+
}, ...outcome.releasedHooks] : outcome.releasedHooks).map((reference) => [hookMarkerKey(reference), reference]));
|
|
1896
2920
|
const retentionMs = request.cleanup?.retentionMs ?? 0;
|
|
1897
|
-
if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !
|
|
2921
|
+
if (retentionMs > 0 && outcome.run && isTerminalWorkflowRunStatus(outcome.run.status) && !retention.cleanup) {
|
|
1898
2922
|
const terminalRun = outcome.run;
|
|
1899
2923
|
const dueAt = new Date(terminalRun.completedAt.getTime() + retentionMs);
|
|
1900
2924
|
const run = WorkflowRunSchema.parse({
|
|
@@ -1911,6 +2935,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1911
2935
|
dueAt,
|
|
1912
2936
|
queueShards: request.cleanup?.queueShards ?? 1,
|
|
1913
2937
|
phase: "retained",
|
|
2938
|
+
generation: 0,
|
|
1914
2939
|
attempts: 0,
|
|
1915
2940
|
deletedPayloadKeys: 0,
|
|
1916
2941
|
deletedStreams: 0,
|
|
@@ -1921,52 +2946,82 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1921
2946
|
await txn.setAlarm(dueAt);
|
|
1922
2947
|
outcome.run = run;
|
|
1923
2948
|
}
|
|
1924
|
-
|
|
2949
|
+
if (outcome.run && isTerminalWorkflowRunStatus(outcome.run.status)) {
|
|
2950
|
+
if (!await txn.get("terminal:cleanup")) await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
2951
|
+
version: 1,
|
|
2952
|
+
runId: outcome.run.runId,
|
|
2953
|
+
phase: "hooks",
|
|
2954
|
+
generation: 0,
|
|
2955
|
+
attempts: 0
|
|
2956
|
+
});
|
|
2957
|
+
await txn.setAlarm(this.now() + 1);
|
|
2958
|
+
}
|
|
2959
|
+
return finalizeEventPage(storeFrom(txn), outcome, request.params);
|
|
2960
|
+
});
|
|
2961
|
+
}
|
|
2962
|
+
/**
|
|
2963
|
+
* Resolve an ambiguous hook-creation RPC in the same serialization domain as
|
|
2964
|
+
* applyEvent. If the hook is absent, the exact reservation is durably fenced
|
|
2965
|
+
* before the caller releases its token and hook-ID claims.
|
|
2966
|
+
*/
|
|
2967
|
+
async resolveHookTokenClaim(request) {
|
|
2968
|
+
return await this.ctx.storage.transaction(async (txn) => {
|
|
2969
|
+
if ((await txn.get(`hook:${request.hookId}`))?.token === request.token) return { committed: true };
|
|
2970
|
+
await txn.put(hookClaimCancellationKey(request.claimId), {
|
|
2971
|
+
hookId: request.hookId,
|
|
2972
|
+
token: request.token,
|
|
2973
|
+
canceledAt: this.now()
|
|
2974
|
+
});
|
|
2975
|
+
return { committed: false };
|
|
1925
2976
|
});
|
|
1926
2977
|
}
|
|
1927
2978
|
async getRun() {
|
|
1928
|
-
return this.
|
|
2979
|
+
return this.readKey("run");
|
|
1929
2980
|
}
|
|
1930
2981
|
async getStep(stepId) {
|
|
1931
|
-
return this.
|
|
2982
|
+
return this.readKey(`${STEP_KEY_PREFIX}${stepId}`);
|
|
1932
2983
|
}
|
|
1933
2984
|
async listSteps(params) {
|
|
1934
|
-
return this.read(() => listByCreationTime(storeFrom(
|
|
2985
|
+
return this.read((txn) => listByCreationTime(storeFrom(txn), STEP_CREATED_KEY_PREFIX, STEP_KEY_PREFIX, {
|
|
1935
2986
|
limit: params?.limit ?? 20,
|
|
1936
2987
|
cursor: params?.cursor,
|
|
1937
2988
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1938
2989
|
}));
|
|
1939
2990
|
}
|
|
1940
2991
|
async getEvent(eventId) {
|
|
1941
|
-
return this.
|
|
2992
|
+
return this.readKey(`${EVENT_KEY_PREFIX}${eventId}`);
|
|
1942
2993
|
}
|
|
1943
2994
|
async listEvents(params) {
|
|
1944
|
-
return this.read(() => listByPrefix(storeFrom(
|
|
2995
|
+
return this.read((txn) => listByPrefix(storeFrom(txn), EVENT_KEY_PREFIX, {
|
|
1945
2996
|
limit: params?.limit ?? 100,
|
|
1946
2997
|
cursor: params?.cursor,
|
|
1947
2998
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1948
2999
|
}, (event) => event.eventId));
|
|
1949
3000
|
}
|
|
1950
3001
|
async listHooks(params) {
|
|
1951
|
-
return this.read(() => listByCreationTime(storeFrom(
|
|
3002
|
+
return this.read((txn) => listByCreationTime(storeFrom(txn), HOOK_CREATED_KEY_PREFIX, HOOK_KEY_PREFIX, {
|
|
1952
3003
|
limit: params?.limit ?? 100,
|
|
1953
3004
|
cursor: params?.cursor,
|
|
1954
3005
|
sortOrder: params?.sortOrder ?? "asc"
|
|
1955
3006
|
}));
|
|
1956
3007
|
}
|
|
1957
3008
|
async getCleanupStatus() {
|
|
1958
|
-
await this.ensureSchema();
|
|
1959
3009
|
return await this.ctx.storage.get("retention:cleanup") ?? null;
|
|
1960
3010
|
}
|
|
1961
3011
|
/** Explicitly schedule an existing terminal run, used for opt-in backfills. */
|
|
1962
3012
|
async scheduleCleanup(request) {
|
|
1963
|
-
await this.ensureSchema();
|
|
1964
3013
|
if (request.retentionMs <= 0) return this.getCleanupStatus();
|
|
1965
3014
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
1966
|
-
const
|
|
3015
|
+
const values = await txn.get([
|
|
3016
|
+
CLEANUP_RECORD_KEY,
|
|
3017
|
+
TOMBSTONE_KEY,
|
|
3018
|
+
TERMINAL_CLEANUP_KEY,
|
|
3019
|
+
"run"
|
|
3020
|
+
]);
|
|
3021
|
+
const existing = values.get(CLEANUP_RECORD_KEY);
|
|
1967
3022
|
if (existing) return existing;
|
|
1968
|
-
if (
|
|
1969
|
-
const current =
|
|
3023
|
+
if (values.get("retention:tombstone")) return null;
|
|
3024
|
+
const current = values.get("run");
|
|
1970
3025
|
if (!current || !isTerminalWorkflowRunStatus(current.status)) return null;
|
|
1971
3026
|
const terminalRun = current;
|
|
1972
3027
|
const dueAt = new Date(terminalRun.completedAt.getTime() + request.retentionMs);
|
|
@@ -1984,6 +3039,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1984
3039
|
dueAt,
|
|
1985
3040
|
queueShards: request.queueShards,
|
|
1986
3041
|
phase: "retained",
|
|
3042
|
+
generation: 0,
|
|
1987
3043
|
attempts: 0,
|
|
1988
3044
|
deletedPayloadKeys: 0,
|
|
1989
3045
|
deletedStreams: 0,
|
|
@@ -1991,12 +3047,11 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
1991
3047
|
};
|
|
1992
3048
|
await txn.put("run", run);
|
|
1993
3049
|
await txn.put(CLEANUP_RECORD_KEY, cleanup);
|
|
1994
|
-
await txn.setAlarm(dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
3050
|
+
await txn.setAlarm(values.get("terminal:cleanup") || dueAt.getTime() <= this.now() ? this.now() + 1 : dueAt);
|
|
1995
3051
|
return cleanup;
|
|
1996
3052
|
});
|
|
1997
3053
|
}
|
|
1998
3054
|
async cleanupNow(request) {
|
|
1999
|
-
await this.ensureSchema();
|
|
2000
3055
|
if (!await this.ctx.storage.get("retention:cleanup")) await this.scheduleCleanup({
|
|
2001
3056
|
...request,
|
|
2002
3057
|
retentionMs: Math.max(1, request.retentionMs)
|
|
@@ -2012,137 +3067,334 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2012
3067
|
}));
|
|
2013
3068
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2014
3069
|
...cleanup,
|
|
2015
|
-
dueAt
|
|
3070
|
+
dueAt,
|
|
3071
|
+
generation: cleanup.generation + 1
|
|
2016
3072
|
});
|
|
2017
3073
|
await txn.setAlarm(this.now() + 1);
|
|
2018
3074
|
});
|
|
2019
|
-
|
|
2020
|
-
await this.executeCleanup();
|
|
2021
|
-
} catch (error) {
|
|
2022
|
-
await this.recordCleanupFailure(error);
|
|
2023
|
-
}
|
|
3075
|
+
await this.executeScheduledCleanupPage();
|
|
2024
3076
|
return this.getCleanupStatus();
|
|
2025
3077
|
}
|
|
2026
3078
|
async rearmCleanup() {
|
|
2027
|
-
await this.
|
|
2028
|
-
const cleanup =
|
|
2029
|
-
if (
|
|
2030
|
-
if (cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
|
|
3079
|
+
const values = await this.ctx.storage.get([CLEANUP_RECORD_KEY, TERMINAL_CLEANUP_KEY]);
|
|
3080
|
+
const cleanup = values.get(CLEANUP_RECORD_KEY);
|
|
3081
|
+
if (values.get("terminal:cleanup")) await this.ctx.storage.setAlarm(this.now() + 1);
|
|
3082
|
+
else if (!cleanup || cleanup.phase === "tombstoned") await this.ctx.storage.deleteAlarm();
|
|
2031
3083
|
else await this.ctx.storage.setAlarm(Math.max(this.now() + 1, cleanup.dueAt.getTime()));
|
|
2032
|
-
return cleanup;
|
|
3084
|
+
return cleanup ?? null;
|
|
2033
3085
|
}
|
|
2034
3086
|
async alarm() {
|
|
2035
|
-
await this.
|
|
2036
|
-
try {
|
|
2037
|
-
await this.executeCleanup();
|
|
2038
|
-
} catch (error) {
|
|
2039
|
-
await this.recordCleanupFailure(error);
|
|
2040
|
-
}
|
|
3087
|
+
await this.executeScheduledCleanupPage();
|
|
2041
3088
|
}
|
|
2042
3089
|
namespace(name) {
|
|
2043
3090
|
const namespace = this.env?.[name];
|
|
2044
3091
|
if (!namespace || typeof namespace === "function") throw new Error(`world-celld retention missing binding ${name}`);
|
|
2045
3092
|
return namespace;
|
|
2046
3093
|
}
|
|
2047
|
-
|
|
3094
|
+
workflowIndex() {
|
|
3095
|
+
return createWorkflowIndex({
|
|
3096
|
+
runCatalog: this.namespace("WORKFLOW_RUN_CATALOG"),
|
|
3097
|
+
runFences: this.namespace("WORKFLOW_RUN_FENCES"),
|
|
3098
|
+
hookTokens: this.namespace("WORKFLOW_HOOK_TOKENS"),
|
|
3099
|
+
hookIds: this.namespace("WORKFLOW_HOOK_IDS")
|
|
3100
|
+
});
|
|
3101
|
+
}
|
|
3102
|
+
async setCleanupPhase(expected, next, updates = {}, continueAt) {
|
|
2048
3103
|
await this.ctx.storage.transaction(async (txn) => {
|
|
2049
3104
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2050
|
-
if (!cleanup || cleanup.phase !== expected) return;
|
|
3105
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
|
|
2051
3106
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2052
3107
|
...cleanup,
|
|
2053
3108
|
...updates,
|
|
2054
3109
|
phase: next,
|
|
3110
|
+
generation: cleanup.generation + 1,
|
|
2055
3111
|
attempts: 0,
|
|
2056
3112
|
lastError: void 0
|
|
2057
3113
|
});
|
|
3114
|
+
if (continueAt !== void 0) await txn.setAlarm(continueAt);
|
|
2058
3115
|
});
|
|
2059
3116
|
}
|
|
2060
|
-
async
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
await this.
|
|
2065
|
-
|
|
3117
|
+
async executeScheduledCleanupPage() {
|
|
3118
|
+
const terminalCleanup = await this.ctx.storage.get(TERMINAL_CLEANUP_KEY);
|
|
3119
|
+
if (terminalCleanup) {
|
|
3120
|
+
try {
|
|
3121
|
+
await this.executeTerminalCleanup(terminalCleanup);
|
|
3122
|
+
} catch (error) {
|
|
3123
|
+
await this.recordTerminalCleanupFailure(error, terminalCleanup);
|
|
2066
3124
|
}
|
|
2067
|
-
|
|
2068
|
-
|
|
3125
|
+
return;
|
|
3126
|
+
}
|
|
3127
|
+
const cleanup = await this.ctx.storage.get(CLEANUP_RECORD_KEY);
|
|
3128
|
+
if (!cleanup || cleanup.phase === "tombstoned") {
|
|
3129
|
+
await this.ctx.storage.deleteAlarm();
|
|
3130
|
+
return;
|
|
3131
|
+
}
|
|
3132
|
+
try {
|
|
3133
|
+
await this.executeCleanup(cleanup);
|
|
3134
|
+
} catch (error) {
|
|
3135
|
+
await this.recordCleanupFailure(error, cleanup);
|
|
3136
|
+
}
|
|
3137
|
+
}
|
|
3138
|
+
async executeCleanup(cleanup) {
|
|
3139
|
+
if (cleanup.phase === "retained") {
|
|
3140
|
+
if (cleanup.dueAt.getTime() > this.now()) {
|
|
3141
|
+
await this.ctx.storage.setAlarm(cleanup.dueAt);
|
|
2069
3142
|
return;
|
|
2070
3143
|
}
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
3144
|
+
await this.setCleanupPhase(cleanup, "index", {}, this.now() + 1);
|
|
3145
|
+
return;
|
|
3146
|
+
}
|
|
3147
|
+
if (cleanup.phase === "index") {
|
|
3148
|
+
await this.deleteIndexPage(cleanup);
|
|
3149
|
+
return;
|
|
3150
|
+
}
|
|
3151
|
+
if (cleanup.phase === "streams") {
|
|
3152
|
+
await this.deleteStreamPage(cleanup);
|
|
3153
|
+
return;
|
|
3154
|
+
}
|
|
3155
|
+
if (cleanup.phase === "queues") {
|
|
3156
|
+
await this.deleteQueuePage(cleanup);
|
|
3157
|
+
return;
|
|
3158
|
+
}
|
|
3159
|
+
if (cleanup.phase === "payload") await this.deletePayloadBatch();
|
|
3160
|
+
}
|
|
3161
|
+
async executeTerminalCleanup(cleanup) {
|
|
3162
|
+
if (cleanup.phase === "hooks") await this.deleteTerminalHookPage(cleanup);
|
|
3163
|
+
else if (cleanup.phase === "markers") await this.deleteTerminalHookMarkerPage(cleanup);
|
|
3164
|
+
else await this.deleteTerminalWaitPage(cleanup);
|
|
3165
|
+
}
|
|
3166
|
+
async deleteTerminalHookPage(cleanup) {
|
|
3167
|
+
const entries = await this.ctx.storage.list({
|
|
3168
|
+
prefix: HOOK_KEY_PREFIX,
|
|
3169
|
+
limit: 65
|
|
3170
|
+
});
|
|
3171
|
+
const page = Array.from(entries.values()).slice(0, TERMINAL_HOOK_PAGE_SIZE);
|
|
3172
|
+
await this.workflowIndex().releaseHookIndexes({
|
|
3173
|
+
runId: cleanup.runId,
|
|
3174
|
+
hooks: page.map((hook) => ({
|
|
3175
|
+
hookId: hook.hookId,
|
|
3176
|
+
token: hook.token
|
|
3177
|
+
})),
|
|
3178
|
+
terminal: true
|
|
3179
|
+
});
|
|
3180
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3181
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3182
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3183
|
+
const keys = page.flatMap((hook) => [`${HOOK_KEY_PREFIX}${hook.hookId}`, hookCreationIndexKey(hook)]);
|
|
3184
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3185
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3186
|
+
...current,
|
|
3187
|
+
phase: entries.size <= TERMINAL_HOOK_PAGE_SIZE ? "markers" : "hooks",
|
|
3188
|
+
generation: current.generation + 1,
|
|
3189
|
+
attempts: 0,
|
|
3190
|
+
lastError: void 0
|
|
3191
|
+
});
|
|
3192
|
+
await txn.setAlarm(this.now() + 1);
|
|
3193
|
+
});
|
|
3194
|
+
}
|
|
3195
|
+
async deleteTerminalHookMarkerPage(cleanup) {
|
|
3196
|
+
const entries = await this.ctx.storage.list({
|
|
3197
|
+
prefix: HOOK_MARKER_PREFIX,
|
|
3198
|
+
limit: 65
|
|
3199
|
+
});
|
|
3200
|
+
const page = Array.from(entries).slice(0, HOOK_MARKER_PAGE_SIZE);
|
|
3201
|
+
await this.workflowIndex().releaseHookIndexes({
|
|
3202
|
+
runId: cleanup.runId,
|
|
3203
|
+
hooks: page.map(([, reference]) => reference),
|
|
3204
|
+
terminal: true
|
|
3205
|
+
});
|
|
3206
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3207
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3208
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3209
|
+
const keys = page.map(([key]) => key);
|
|
3210
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3211
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3212
|
+
...current,
|
|
3213
|
+
phase: entries.size <= HOOK_MARKER_PAGE_SIZE ? "waits" : "markers",
|
|
3214
|
+
generation: current.generation + 1,
|
|
3215
|
+
attempts: 0,
|
|
3216
|
+
lastError: void 0
|
|
3217
|
+
});
|
|
3218
|
+
await txn.setAlarm(this.now() + 1);
|
|
3219
|
+
});
|
|
3220
|
+
}
|
|
3221
|
+
async deleteTerminalWaitPage(cleanup) {
|
|
3222
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3223
|
+
const current = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3224
|
+
if (!current || current.phase !== cleanup.phase || current.generation !== cleanup.generation) return;
|
|
3225
|
+
const entries = await txn.list({
|
|
3226
|
+
prefix: WAIT_KEY_PREFIX,
|
|
3227
|
+
limit: 129
|
|
3228
|
+
});
|
|
3229
|
+
const keys = Array.from(entries.keys()).slice(0, TERMINAL_WAIT_PAGE_SIZE);
|
|
3230
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
3231
|
+
if (entries.size > TERMINAL_WAIT_PAGE_SIZE) {
|
|
3232
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3233
|
+
...current,
|
|
3234
|
+
generation: current.generation + 1,
|
|
3235
|
+
attempts: 0,
|
|
3236
|
+
lastError: void 0
|
|
2091
3237
|
});
|
|
2092
|
-
await this.
|
|
2093
|
-
|
|
3238
|
+
await txn.setAlarm(this.now() + 1);
|
|
3239
|
+
return;
|
|
2094
3240
|
}
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
3241
|
+
await txn.delete(TERMINAL_CLEANUP_KEY);
|
|
3242
|
+
const retention = await txn.get(CLEANUP_RECORD_KEY);
|
|
3243
|
+
if (retention && retention.phase !== "tombstoned") await txn.setAlarm(Math.max(this.now() + 1, retention.dueAt.getTime()));
|
|
3244
|
+
else await txn.deleteAlarm();
|
|
3245
|
+
});
|
|
3246
|
+
}
|
|
3247
|
+
async deleteIndexPage(cleanup) {
|
|
3248
|
+
const hookEntries = await this.ctx.storage.list({
|
|
3249
|
+
prefix: HOOK_MARKER_PREFIX,
|
|
3250
|
+
limit: 65
|
|
3251
|
+
});
|
|
3252
|
+
const hookMarkers = Array.from(hookEntries).slice(0, HOOK_MARKER_PAGE_SIZE);
|
|
3253
|
+
await this.workflowIndex().expireRun({
|
|
3254
|
+
runId: cleanup.runId,
|
|
3255
|
+
keys: [workflowRunIndexKey(cleanup), globalRunIndexKey(cleanup)],
|
|
3256
|
+
hooks: hookMarkers.map(([, value]) => value),
|
|
3257
|
+
expiredAt: cleanup.dueAt.getTime()
|
|
3258
|
+
});
|
|
3259
|
+
const markerKeys = hookMarkers.map(([key]) => key);
|
|
3260
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3261
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3262
|
+
if (current?.phase !== "index" || current.generation !== cleanup.generation) return;
|
|
3263
|
+
if (markerKeys.length > 0) await txn.delete(markerKeys);
|
|
3264
|
+
if (hookEntries.size <= HOOK_MARKER_PAGE_SIZE) await txn.put(CLEANUP_RECORD_KEY, {
|
|
3265
|
+
...current,
|
|
3266
|
+
phase: "streams",
|
|
3267
|
+
generation: current.generation + 1,
|
|
3268
|
+
attempts: 0,
|
|
3269
|
+
lastError: void 0
|
|
3270
|
+
});
|
|
3271
|
+
else await txn.put(CLEANUP_RECORD_KEY, {
|
|
3272
|
+
...current,
|
|
3273
|
+
generation: current.generation + 1,
|
|
3274
|
+
attempts: 0,
|
|
3275
|
+
lastError: void 0
|
|
3276
|
+
});
|
|
3277
|
+
await txn.setAlarm(this.now() + 1);
|
|
3278
|
+
});
|
|
3279
|
+
}
|
|
3280
|
+
async deleteStreamPage(cleanup) {
|
|
3281
|
+
const streamsNamespace = this.namespace("WORKFLOW_STREAMS");
|
|
3282
|
+
const registry = streamsNamespace.get(streamsNamespace.idFromName(`run-streams:${cleanup.runId}`));
|
|
3283
|
+
const { streams } = await registry.expireRegistry(cleanup.runId, cleanup.dueAt.getTime(), { limit: STREAMS_PER_CLEANUP_PAGE });
|
|
3284
|
+
const completed = [];
|
|
3285
|
+
let remainingChunks = STREAM_CHUNKS_PER_CLEANUP_PAGE;
|
|
3286
|
+
let remainingBytes = STREAM_BYTES_PER_CLEANUP_PAGE;
|
|
3287
|
+
for (const name of streams) {
|
|
3288
|
+
if (remainingChunks <= 0 || remainingBytes <= 0) break;
|
|
3289
|
+
const result = await streamsNamespace.get(streamsNamespace.idFromName(`stream:${name}`)).expireStream(cleanup.runId, cleanup.dueAt.getTime(), {
|
|
3290
|
+
limit: remainingChunks,
|
|
3291
|
+
byteLimit: remainingBytes
|
|
3292
|
+
});
|
|
3293
|
+
remainingChunks -= result.chunks;
|
|
3294
|
+
remainingBytes -= result.bytes;
|
|
3295
|
+
if (result.done) completed.push(name);
|
|
3296
|
+
}
|
|
3297
|
+
const registryResult = await registry.finalizeRegistry(cleanup.runId, completed);
|
|
3298
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3299
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3300
|
+
if (current?.phase !== "streams" || current.generation !== cleanup.generation) return;
|
|
3301
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3302
|
+
...current,
|
|
3303
|
+
deletedStreams: registryResult.deleted,
|
|
3304
|
+
phase: registryResult.done ? "queues" : "streams",
|
|
3305
|
+
generation: current.generation + 1,
|
|
3306
|
+
attempts: 0,
|
|
3307
|
+
lastError: void 0
|
|
3308
|
+
});
|
|
3309
|
+
await txn.setAlarm(this.now() + 1);
|
|
3310
|
+
});
|
|
3311
|
+
}
|
|
3312
|
+
async deleteQueuePage(cleanup) {
|
|
3313
|
+
const queues = this.namespace("WORKFLOW_QUEUE");
|
|
3314
|
+
let progress = await this.ctx.storage.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3315
|
+
queueShard: 0,
|
|
3316
|
+
queueShardDeleted: 0
|
|
3317
|
+
};
|
|
3318
|
+
let remainingShards = QUEUE_SHARDS_PER_CLEANUP_PAGE;
|
|
3319
|
+
let expectedGeneration = cleanup.generation;
|
|
3320
|
+
while (progress.queueShard < cleanup.queueShards && remainingShards-- > 0) {
|
|
3321
|
+
const requestedShard = progress.queueShard;
|
|
3322
|
+
const result = await queues.get(queues.idFromName(`q:${requestedShard}`)).expireRun(cleanup.runId, cleanup.dueAt.getTime(), { limit: QUEUE_REFERENCES_PER_CLEANUP_PAGE });
|
|
3323
|
+
const reconciled = await this.ctx.storage.transaction(async (txn) => {
|
|
3324
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3325
|
+
if (current?.phase !== "queues") return null;
|
|
3326
|
+
if (current.generation !== expectedGeneration) return { stale: true };
|
|
3327
|
+
const persisted = await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3328
|
+
queueShard: 0,
|
|
3329
|
+
queueShardDeleted: 0
|
|
3330
|
+
};
|
|
3331
|
+
if (persisted.queueShard !== requestedShard) return { stale: true };
|
|
3332
|
+
if (result.deleted < persisted.queueShardDeleted) return { stale: true };
|
|
3333
|
+
const next = result.done ? {
|
|
3334
|
+
queueShard: persisted.queueShard + 1,
|
|
3335
|
+
queueShardDeleted: 0
|
|
3336
|
+
} : {
|
|
3337
|
+
...persisted,
|
|
3338
|
+
queueShardDeleted: result.deleted
|
|
3339
|
+
};
|
|
3340
|
+
const delta = result.deleted - persisted.queueShardDeleted;
|
|
3341
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3342
|
+
...current,
|
|
3343
|
+
deletedQueueMessages: current.deletedQueueMessages + delta,
|
|
3344
|
+
generation: current.generation + 1,
|
|
3345
|
+
attempts: 0,
|
|
3346
|
+
lastError: void 0
|
|
2113
3347
|
});
|
|
2114
|
-
await
|
|
2115
|
-
await this.
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
if (
|
|
2129
|
-
if (!await this.deletePayloadBatch()) return;
|
|
2130
|
-
}
|
|
3348
|
+
await txn.put(CLEANUP_PROGRESS_KEY, next);
|
|
3349
|
+
await txn.setAlarm(this.now() + 1);
|
|
3350
|
+
return {
|
|
3351
|
+
stale: false,
|
|
3352
|
+
progress: next,
|
|
3353
|
+
generation: current.generation + 1,
|
|
3354
|
+
applied: result.done || next.queueShard !== persisted.queueShard || next.queueShardDeleted !== persisted.queueShardDeleted
|
|
3355
|
+
};
|
|
3356
|
+
});
|
|
3357
|
+
if (!reconciled) return;
|
|
3358
|
+
if (reconciled.stale) return;
|
|
3359
|
+
progress = reconciled.progress;
|
|
3360
|
+
expectedGeneration = reconciled.generation;
|
|
3361
|
+
if (!reconciled.applied && progress.queueShard === requestedShard) break;
|
|
3362
|
+
if (!result.done && progress.queueShard === requestedShard) break;
|
|
2131
3363
|
}
|
|
3364
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3365
|
+
const current = await txn.get(CLEANUP_RECORD_KEY);
|
|
3366
|
+
if (current?.phase !== "queues") return;
|
|
3367
|
+
if ((await txn.get(CLEANUP_PROGRESS_KEY) ?? {
|
|
3368
|
+
queueShard: 0,
|
|
3369
|
+
queueShardDeleted: 0
|
|
3370
|
+
}).queueShard >= current.queueShards) {
|
|
3371
|
+
await txn.put(CLEANUP_RECORD_KEY, {
|
|
3372
|
+
...current,
|
|
3373
|
+
phase: "payload",
|
|
3374
|
+
generation: current.generation + 1
|
|
3375
|
+
});
|
|
3376
|
+
await txn.delete(CLEANUP_PROGRESS_KEY);
|
|
3377
|
+
await txn.setAlarm(this.now() + 1);
|
|
3378
|
+
}
|
|
3379
|
+
});
|
|
2132
3380
|
}
|
|
2133
3381
|
async deletePayloadBatch() {
|
|
2134
3382
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2135
3383
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2136
3384
|
if (!cleanup || cleanup.phase !== "payload") return true;
|
|
2137
|
-
const entries = await txn.list({
|
|
2138
|
-
|
|
3385
|
+
const entries = await txn.list({
|
|
3386
|
+
limit: 131,
|
|
3387
|
+
noCache: true
|
|
3388
|
+
});
|
|
3389
|
+
const keys = Array.from(entries.keys()).filter((key) => key !== "retention:cleanup" && key !== "retention:tombstone").slice(0, PAYLOAD_DELETE_BATCH);
|
|
2139
3390
|
const now = new Date(this.now());
|
|
2140
3391
|
await txn.put(TOMBSTONE_KEY, cleanupTombstone(cleanup, now));
|
|
2141
|
-
|
|
3392
|
+
if (keys.length > 0) await txn.delete(keys);
|
|
2142
3393
|
if (keys.length > 0) {
|
|
2143
3394
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2144
3395
|
...cleanup,
|
|
2145
3396
|
deletedPayloadKeys: cleanup.deletedPayloadKeys + keys.length,
|
|
3397
|
+
generation: cleanup.generation + 1,
|
|
2146
3398
|
lastError: void 0
|
|
2147
3399
|
});
|
|
2148
3400
|
await txn.setAlarm(this.now() + 1);
|
|
@@ -2151,6 +3403,7 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2151
3403
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2152
3404
|
...cleanup,
|
|
2153
3405
|
phase: "tombstoned",
|
|
3406
|
+
generation: cleanup.generation + 1,
|
|
2154
3407
|
attempts: 0,
|
|
2155
3408
|
lastError: void 0,
|
|
2156
3409
|
tombstonedAt: now
|
|
@@ -2159,14 +3412,31 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2159
3412
|
return true;
|
|
2160
3413
|
});
|
|
2161
3414
|
}
|
|
2162
|
-
async recordCleanupFailure(error) {
|
|
3415
|
+
async recordCleanupFailure(error, expected) {
|
|
2163
3416
|
await this.ctx.storage.transaction(async (txn) => {
|
|
2164
3417
|
const cleanup = await txn.get(CLEANUP_RECORD_KEY);
|
|
2165
|
-
if (!cleanup || cleanup.phase === "tombstoned") return;
|
|
3418
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation || cleanup.phase === "tombstoned") return;
|
|
2166
3419
|
const attempts = cleanup.attempts + 1;
|
|
2167
3420
|
const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
|
|
2168
3421
|
await txn.put(CLEANUP_RECORD_KEY, {
|
|
2169
3422
|
...cleanup,
|
|
3423
|
+
generation: cleanup.generation + 1,
|
|
3424
|
+
attempts,
|
|
3425
|
+
lastError: error instanceof Error ? error.message : String(error),
|
|
3426
|
+
lastAttemptAt: new Date(this.now())
|
|
3427
|
+
});
|
|
3428
|
+
await txn.setAlarm(this.now() + delay);
|
|
3429
|
+
});
|
|
3430
|
+
}
|
|
3431
|
+
async recordTerminalCleanupFailure(error, expected) {
|
|
3432
|
+
await this.ctx.storage.transaction(async (txn) => {
|
|
3433
|
+
const cleanup = await txn.get(TERMINAL_CLEANUP_KEY);
|
|
3434
|
+
if (!cleanup || cleanup.phase !== expected.phase || cleanup.generation !== expected.generation) return;
|
|
3435
|
+
const attempts = cleanup.attempts + 1;
|
|
3436
|
+
const delay = Math.min(CLEANUP_RETRY_MAX_MS, 1e3 * 2 ** Math.min(attempts - 1, 12));
|
|
3437
|
+
await txn.put(TERMINAL_CLEANUP_KEY, {
|
|
3438
|
+
...cleanup,
|
|
3439
|
+
generation: cleanup.generation + 1,
|
|
2170
3440
|
attempts,
|
|
2171
3441
|
lastError: error instanceof Error ? error.message : String(error),
|
|
2172
3442
|
lastAttemptAt: new Date(this.now())
|
|
@@ -2175,7 +3445,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2175
3445
|
});
|
|
2176
3446
|
}
|
|
2177
3447
|
async claimInflight(params) {
|
|
2178
|
-
await this.ensureSchema();
|
|
2179
3448
|
return await this.ctx.storage.transaction(async (txn) => {
|
|
2180
3449
|
const existing = await txn.get("claim");
|
|
2181
3450
|
const now = this.now();
|
|
@@ -2188,7 +3457,6 @@ var WorkflowRunDO = class extends DurableObject {
|
|
|
2188
3457
|
});
|
|
2189
3458
|
}
|
|
2190
3459
|
async releaseInflight() {
|
|
2191
|
-
await this.ensureSchema();
|
|
2192
3460
|
await this.ctx.storage.delete("claim");
|
|
2193
3461
|
}
|
|
2194
3462
|
};
|
|
@@ -2203,7 +3471,10 @@ async function startHarness(options = {}) {
|
|
|
2203
3471
|
const fleet = new FakeFleet({
|
|
2204
3472
|
runs: WorkflowRunDO,
|
|
2205
3473
|
streams: StreamDO,
|
|
2206
|
-
|
|
3474
|
+
"run-catalog": RunCatalogDO,
|
|
3475
|
+
"run-fences": RunFenceDO,
|
|
3476
|
+
"hook-tokens": HookTokenDO,
|
|
3477
|
+
"hook-ids": HookIdDO,
|
|
2207
3478
|
queue: QueueDO,
|
|
2208
3479
|
...options.extraClasses
|
|
2209
3480
|
}, cellEnv);
|
|
@@ -2211,17 +3482,32 @@ async function startHarness(options = {}) {
|
|
|
2211
3482
|
Object.assign(cellEnv, {
|
|
2212
3483
|
WORKFLOW_DB: fleet.namespace("runs"),
|
|
2213
3484
|
WORKFLOW_STREAMS: fleet.namespace("streams"),
|
|
2214
|
-
|
|
3485
|
+
WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
|
|
3486
|
+
WORKFLOW_RUN_FENCES: fleet.namespace("run-fences"),
|
|
3487
|
+
WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
|
|
3488
|
+
WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
|
|
2215
3489
|
WORKFLOW_QUEUE: fleet.namespace("queue")
|
|
2216
3490
|
});
|
|
2217
3491
|
const router = createRouter({
|
|
2218
3492
|
WORKFLOW_DB: fleet.namespace("runs"),
|
|
2219
3493
|
WORKFLOW_STREAMS: fleet.namespace("streams"),
|
|
2220
|
-
|
|
3494
|
+
WORKFLOW_RUN_CATALOG: fleet.namespace("run-catalog"),
|
|
3495
|
+
WORKFLOW_RUN_FENCES: fleet.namespace("run-fences"),
|
|
3496
|
+
WORKFLOW_HOOK_TOKENS: fleet.namespace("hook-tokens"),
|
|
3497
|
+
WORKFLOW_HOOK_IDS: fleet.namespace("hook-ids"),
|
|
2221
3498
|
WORKFLOW_QUEUE: fleet.namespace("queue"),
|
|
2222
3499
|
WORLD_SECRET: options.secret
|
|
2223
3500
|
});
|
|
2224
3501
|
const server = http.createServer(async (req, res) => {
|
|
3502
|
+
const requestAbort = new AbortController();
|
|
3503
|
+
const abortRequest = () => {
|
|
3504
|
+
if (!requestAbort.signal.aborted) requestAbort.abort(new DOMException("HTTP client disconnected", "AbortError"));
|
|
3505
|
+
};
|
|
3506
|
+
const abortOnPrematureClose = () => {
|
|
3507
|
+
if (!res.writableEnded) abortRequest();
|
|
3508
|
+
};
|
|
3509
|
+
req.once("aborted", abortRequest);
|
|
3510
|
+
res.once("close", abortOnPrematureClose);
|
|
2225
3511
|
try {
|
|
2226
3512
|
const chunks = [];
|
|
2227
3513
|
for await (const chunk of req) chunks.push(chunk);
|
|
@@ -2229,17 +3515,23 @@ async function startHarness(options = {}) {
|
|
|
2229
3515
|
const request = new Request(`http://127.0.0.1${req.url ?? "/"}`, {
|
|
2230
3516
|
method: req.method,
|
|
2231
3517
|
headers: req.headers,
|
|
2232
|
-
body: req.method === "GET" || req.method === "HEAD" || body.length === 0 ? null : body
|
|
3518
|
+
body: req.method === "GET" || req.method === "HEAD" || body.length === 0 ? null : body,
|
|
3519
|
+
signal: requestAbort.signal
|
|
2233
3520
|
});
|
|
2234
3521
|
const response = await router(request);
|
|
3522
|
+
if (requestAbort.signal.aborted || res.destroyed) return;
|
|
2235
3523
|
res.writeHead(response.status, Object.fromEntries(response.headers.entries()));
|
|
2236
3524
|
res.end(Buffer.from(await response.arrayBuffer()));
|
|
2237
3525
|
} catch (error) {
|
|
3526
|
+
if (requestAbort.signal.aborted || res.destroyed) return;
|
|
2238
3527
|
res.writeHead(500, { "content-type": "application/json" });
|
|
2239
3528
|
res.end(JSON.stringify({ error: {
|
|
2240
3529
|
name: "HarnessError",
|
|
2241
3530
|
message: String(error)
|
|
2242
3531
|
} }));
|
|
3532
|
+
} finally {
|
|
3533
|
+
req.off("aborted", abortRequest);
|
|
3534
|
+
res.off("close", abortOnPrematureClose);
|
|
2243
3535
|
}
|
|
2244
3536
|
});
|
|
2245
3537
|
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|