@frockbot/kernel-do 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/authority.ts +468 -13
- package/src/memory-storage.fixture.ts +14 -1
- package/src/run-records.ts +132 -4
- package/src/run-terminal.ts +109 -2
- package/src/storage-keys.ts +8 -0
- package/src/turn-supersede.test.ts +578 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@frockbot/kernel-do",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.3",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
"typecheck": "tsc --noEmit -p tsconfig.json"
|
|
13
13
|
},
|
|
14
14
|
"dependencies": {
|
|
15
|
-
"@frockbot/kernel-composition": "0.3.
|
|
16
|
-
"@frockbot/kernel-contracts": "0.3.
|
|
15
|
+
"@frockbot/kernel-composition": "0.3.3",
|
|
16
|
+
"@frockbot/kernel-contracts": "0.3.3",
|
|
17
17
|
"cordis": "4.0.0-rc.8"
|
|
18
18
|
},
|
|
19
19
|
"devDependencies": {
|
package/src/authority.ts
CHANGED
|
@@ -13,6 +13,7 @@ import { DurableCompositionStore } from "./composition-store.js";
|
|
|
13
13
|
import { DurableCompositionFailureLog } from "./composition-failures.js";
|
|
14
14
|
import {
|
|
15
15
|
botTurnCommandFingerprintV1,
|
|
16
|
+
defaultRunLaneV1,
|
|
16
17
|
storedRunAdmissionV1,
|
|
17
18
|
storedRunSubagentRoleV1,
|
|
18
19
|
storedRunTurnTypeV1,
|
|
@@ -25,6 +26,7 @@ import {
|
|
|
25
26
|
import {
|
|
26
27
|
completeStoredRun,
|
|
27
28
|
type TerminalPackageRecords,
|
|
29
|
+
type SupersededPackageRecords,
|
|
28
30
|
failStoredRun,
|
|
29
31
|
requireStoredRunReconciliation,
|
|
30
32
|
} from "./run-terminal.js";
|
|
@@ -40,6 +42,7 @@ import {
|
|
|
40
42
|
} from "./turn-errors.js";
|
|
41
43
|
import {
|
|
42
44
|
ACTIVE_RUN_KEY,
|
|
45
|
+
PENDING_RUN_KEY,
|
|
43
46
|
IDENTITY_KEY,
|
|
44
47
|
LATEST_EVENTS_KEY,
|
|
45
48
|
MAX_RUN_ADMISSION_FENCES,
|
|
@@ -122,8 +125,31 @@ export interface BotDurableAuthorityHooks<Snapshot> {
|
|
|
122
125
|
deferScheduledWork(transaction: DurableObjectTransaction): Promise<void>;
|
|
123
126
|
/** Settle Package deadlines when the alarm fires idle. */
|
|
124
127
|
settleScheduledWork(): Promise<void>;
|
|
128
|
+
/**
|
|
129
|
+
* Advisory interrupt of the exact Turn named, after the durable intent that
|
|
130
|
+
* justifies it is already written. The reason is an opaque bounded string
|
|
131
|
+
* the kernel records and never reads; a Package that holds no resident Agent
|
|
132
|
+
* needs no implementation, because the durable effect fence stops the Turn
|
|
133
|
+
* either way.
|
|
134
|
+
*/
|
|
135
|
+
interruptTurn?(runId: string, reason: string): void;
|
|
136
|
+
/**
|
|
137
|
+
* Package records written in the same transaction that settles a Turn as
|
|
138
|
+
* `superseded`. Same contract as `terminalRecords`: the kernel writes the
|
|
139
|
+
* returned keys without reading them.
|
|
140
|
+
*/
|
|
141
|
+
supersededRecords?(input: {
|
|
142
|
+
run: StoredRunV1<Snapshot>;
|
|
143
|
+
read<T>(key: string): Promise<T | undefined>;
|
|
144
|
+
}): Promise<Record<string, unknown>>;
|
|
125
145
|
}
|
|
126
146
|
|
|
147
|
+
/** What a `turn/end` records when a later user message took a Turn's place. */
|
|
148
|
+
export const SUPERSEDED_TURN_REASON_V1 = "superseded by a new user message";
|
|
149
|
+
|
|
150
|
+
/** How many times a queued Turn retries the object before giving up. */
|
|
151
|
+
const MAX_QUEUED_RUN_START_ATTEMPTS = 8;
|
|
152
|
+
|
|
127
153
|
export interface BotDurableAuthorityOptions<Snapshot> {
|
|
128
154
|
state: DurableObjectState;
|
|
129
155
|
codec: StoredRunCodecV1<Snapshot>;
|
|
@@ -139,6 +165,18 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
139
165
|
/** Why a generation failed to activate, and whether it is quarantined. */
|
|
140
166
|
readonly compositionFailures: DurableCompositionFailureLog;
|
|
141
167
|
private executingRunId: string | undefined;
|
|
168
|
+
/**
|
|
169
|
+
* The in-process settlement of the executing run. A supersede has to wait
|
|
170
|
+
* for the Turn it interrupted to reach its durable terminal state before the
|
|
171
|
+
* Turn that replaced it can start, and while this object is resident that
|
|
172
|
+
* settlement is a promise rather than an alarm.
|
|
173
|
+
*/
|
|
174
|
+
private executingActivity: Promise<unknown> | undefined;
|
|
175
|
+
/**
|
|
176
|
+
* Queued runs a caller in this object is already waiting to start. Recovery
|
|
177
|
+
* leaves them alone, so a queued Turn is promoted by exactly one path.
|
|
178
|
+
*/
|
|
179
|
+
private readonly queuedWaiters = new Set<string>();
|
|
142
180
|
|
|
143
181
|
constructor(options: BotDurableAuthorityOptions<Snapshot>) {
|
|
144
182
|
this.ctx = options.state;
|
|
@@ -156,9 +194,20 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
156
194
|
async run(command: OwnedBotTurnCommand): Promise<BotTurnCompletion> {
|
|
157
195
|
await this.assertMatchingRunCommand(command);
|
|
158
196
|
await this.recoverActiveRun();
|
|
159
|
-
const replay = await this.
|
|
197
|
+
const replay = await this.settledRunResult(command);
|
|
160
198
|
if (replay) return replay;
|
|
161
199
|
const admission = await this.acceptRun(command);
|
|
200
|
+
if (admission.kind === "queued") {
|
|
201
|
+
// Durably admitted, waiting for the object. The interrupt is advisory
|
|
202
|
+
// and always follows the intent that is already written.
|
|
203
|
+
if (admission.interrupt) {
|
|
204
|
+
this.hooks.interruptTurn?.(
|
|
205
|
+
admission.interrupt.runId,
|
|
206
|
+
SUPERSEDED_TURN_REASON_V1,
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
return this.runQueuedRun(command);
|
|
210
|
+
}
|
|
162
211
|
return this.executeAcceptedRun(
|
|
163
212
|
command,
|
|
164
213
|
admission.previous,
|
|
@@ -167,6 +216,139 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
167
216
|
);
|
|
168
217
|
}
|
|
169
218
|
|
|
219
|
+
/**
|
|
220
|
+
* Drives one durably queued Turn to its own terminal state.
|
|
221
|
+
*
|
|
222
|
+
* The Turn ahead of it settles first — it is either finishing on its own or
|
|
223
|
+
* has just been fenced by the supersede intent — and only then does this one
|
|
224
|
+
* become the active run. Eviction anywhere in here is safe: the queued run is
|
|
225
|
+
* durable, and the recovery alarm promotes it exactly as this does.
|
|
226
|
+
*/
|
|
227
|
+
private async runQueuedRun(
|
|
228
|
+
command: OwnedBotTurnCommand,
|
|
229
|
+
): Promise<BotTurnCompletion> {
|
|
230
|
+
this.queuedWaiters.add(command.runId);
|
|
231
|
+
try {
|
|
232
|
+
for (
|
|
233
|
+
let attempt = 0;
|
|
234
|
+
attempt < MAX_QUEUED_RUN_START_ATTEMPTS;
|
|
235
|
+
attempt++
|
|
236
|
+
) {
|
|
237
|
+
await this.settleExecutingActivity();
|
|
238
|
+
const settled = await this.terminalRunResult(command.runId);
|
|
239
|
+
if (settled) return settled;
|
|
240
|
+
const promoted = await this.promoteQueuedRun(command.runId);
|
|
241
|
+
if (promoted === "blocked") {
|
|
242
|
+
// Another Turn holds the object. Recovery drives it to its own
|
|
243
|
+
// durable terminal or resumable state, and this one tries again.
|
|
244
|
+
await this.recoverActiveRun();
|
|
245
|
+
// Unless what holds the object is an uncertain effect. That is
|
|
246
|
+
// settled by an explicit reconciliation the User asks for, on their
|
|
247
|
+
// own clock, and retrying against it would only burn this caller's
|
|
248
|
+
// attempts and end by failing a Turn the User is owed. The queued
|
|
249
|
+
// run is durable: it stays queued, and the reconciliation's own
|
|
250
|
+
// settlement — or the recovery alarm — starts it.
|
|
251
|
+
if (await this.activeRunAwaitsReconciliation()) {
|
|
252
|
+
return {
|
|
253
|
+
runId: command.runId,
|
|
254
|
+
text: "",
|
|
255
|
+
events: [],
|
|
256
|
+
} satisfies BotTurnCompletion;
|
|
257
|
+
}
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
if (promoted === "not-queued") {
|
|
261
|
+
const terminal = await this.terminalRunResult(command.runId);
|
|
262
|
+
if (terminal) return terminal;
|
|
263
|
+
const current = await this.readRun(command.runId);
|
|
264
|
+
throw new Error(
|
|
265
|
+
`run "${command.runId}" left the queue with status ${
|
|
266
|
+
current?.status ?? "missing"
|
|
267
|
+
}`,
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
return this.executeAcceptedRun(
|
|
271
|
+
command,
|
|
272
|
+
promoted.previous,
|
|
273
|
+
promoted.settings,
|
|
274
|
+
promoted.compositionGenerationId,
|
|
275
|
+
);
|
|
276
|
+
}
|
|
277
|
+
throw new Error(`run "${command.runId}" could not start`);
|
|
278
|
+
} finally {
|
|
279
|
+
this.queuedWaiters.delete(command.runId);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** True while the active run is holding an effect only a User can settle. */
|
|
284
|
+
private async activeRunAwaitsReconciliation(): Promise<boolean> {
|
|
285
|
+
const activeRunId = await this.ctx.storage.get<string>(ACTIVE_RUN_KEY);
|
|
286
|
+
if (!activeRunId) return false;
|
|
287
|
+
const run = await this.readRun(activeRunId);
|
|
288
|
+
return run?.status === "reconciliation-required";
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** Waits out whatever this object is currently running, failures included. */
|
|
292
|
+
private async settleExecutingActivity(): Promise<void> {
|
|
293
|
+
for (let guard = 0; guard < 64; guard += 1) {
|
|
294
|
+
const activity = this.executingActivity;
|
|
295
|
+
if (!activity) return;
|
|
296
|
+
await activity.catch(() => undefined);
|
|
297
|
+
if (this.executingActivity === activity) return;
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Makes the durably queued run the active one, recomputing the history it
|
|
303
|
+
* starts from: the Turn it waited behind appended events, and the queued
|
|
304
|
+
* Turn's model request derives from everything that is durable now.
|
|
305
|
+
*/
|
|
306
|
+
private async promoteQueuedRun(runId: string): Promise<
|
|
307
|
+
| "not-queued"
|
|
308
|
+
| "blocked"
|
|
309
|
+
| {
|
|
310
|
+
previous: SessionEvent[];
|
|
311
|
+
settings: Snapshot;
|
|
312
|
+
compositionGenerationId: string;
|
|
313
|
+
}
|
|
314
|
+
> {
|
|
315
|
+
const key = `${RUN_PREFIX}${runId}`;
|
|
316
|
+
return this.ctx.storage.transaction(async (transaction) => {
|
|
317
|
+
const pendingRunId = await transaction.get<string>(PENDING_RUN_KEY);
|
|
318
|
+
const run = this.codec.optional(await transaction.get<unknown>(key));
|
|
319
|
+
if (
|
|
320
|
+
pendingRunId !== runId ||
|
|
321
|
+
!run ||
|
|
322
|
+
run.status !== "running" ||
|
|
323
|
+
run.phase !== "queued"
|
|
324
|
+
) {
|
|
325
|
+
return "not-queued" as const;
|
|
326
|
+
}
|
|
327
|
+
if (await transaction.get<string>(ACTIVE_RUN_KEY)) {
|
|
328
|
+
return "blocked" as const;
|
|
329
|
+
}
|
|
330
|
+
const latestEvents = (
|
|
331
|
+
(await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
|
|
332
|
+
).map(decodeSessionEvent);
|
|
333
|
+
const promoted = this.codec.require({
|
|
334
|
+
...run,
|
|
335
|
+
phase: "admitted",
|
|
336
|
+
previousEventCount: latestEvents.length,
|
|
337
|
+
} satisfies StoredRunV1<Snapshot>);
|
|
338
|
+
await transaction.put({
|
|
339
|
+
[key]: structuredClone(promoted),
|
|
340
|
+
[ACTIVE_RUN_KEY]: runId,
|
|
341
|
+
});
|
|
342
|
+
await transaction.delete(PENDING_RUN_KEY);
|
|
343
|
+
await this.refreshRecoveryAlarm(transaction);
|
|
344
|
+
return {
|
|
345
|
+
previous: latestEvents,
|
|
346
|
+
settings: promoted.configurationSnapshot,
|
|
347
|
+
compositionGenerationId: promoted.compositionGenerationId,
|
|
348
|
+
};
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
|
|
170
352
|
async reconcileRun(
|
|
171
353
|
identity: BotIdentity,
|
|
172
354
|
runId: string,
|
|
@@ -235,6 +417,28 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
235
417
|
previous: SessionEvent[],
|
|
236
418
|
settings: Snapshot,
|
|
237
419
|
compositionGenerationId: string,
|
|
420
|
+
): Promise<BotTurnCompletion> {
|
|
421
|
+
const activity = this.executeAdmittedRun(
|
|
422
|
+
command,
|
|
423
|
+
previous,
|
|
424
|
+
settings,
|
|
425
|
+
compositionGenerationId,
|
|
426
|
+
);
|
|
427
|
+
this.executingActivity = activity;
|
|
428
|
+
try {
|
|
429
|
+
return await activity;
|
|
430
|
+
} finally {
|
|
431
|
+
if (this.executingActivity === activity) {
|
|
432
|
+
this.executingActivity = undefined;
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
private async executeAdmittedRun(
|
|
438
|
+
command: OwnedBotTurnCommand,
|
|
439
|
+
previous: SessionEvent[],
|
|
440
|
+
settings: Snapshot,
|
|
441
|
+
compositionGenerationId: string,
|
|
238
442
|
): Promise<BotTurnCompletion> {
|
|
239
443
|
this.executingRunId = command.runId;
|
|
240
444
|
try {
|
|
@@ -290,6 +494,8 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
290
494
|
throw new Error(message);
|
|
291
495
|
}
|
|
292
496
|
await this.failRun(command.runId, previous, events, message);
|
|
497
|
+
const settled = await this.supersededRunResult(command.runId);
|
|
498
|
+
if (settled) return settled;
|
|
293
499
|
throw new Error(message);
|
|
294
500
|
} finally {
|
|
295
501
|
if (this.executingRunId === command.runId) {
|
|
@@ -298,6 +504,53 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
298
504
|
}
|
|
299
505
|
}
|
|
300
506
|
|
|
507
|
+
/**
|
|
508
|
+
* The completion a Turn another user message replaced reports. It is not a
|
|
509
|
+
* failure: the Turn settled durably, keeping everything it had already sent,
|
|
510
|
+
* and its caller reads the rest of the conversation from durable state.
|
|
511
|
+
*/
|
|
512
|
+
private async supersededRunResult(
|
|
513
|
+
runId: string,
|
|
514
|
+
): Promise<BotTurnCompletion | undefined> {
|
|
515
|
+
const run = this.codec.optional(
|
|
516
|
+
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
517
|
+
);
|
|
518
|
+
if (run?.status !== "superseded") return undefined;
|
|
519
|
+
return { runId, text: "", events: structuredClone(run.events) };
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/**
|
|
523
|
+
* The completion a run that has already settled reports, or `undefined`
|
|
524
|
+
* while it is still going. Unlike the replay check this asks no questions
|
|
525
|
+
* about the command that produced it: the caller is the run's own waiter.
|
|
526
|
+
*/
|
|
527
|
+
private async terminalRunResult(
|
|
528
|
+
runId: string,
|
|
529
|
+
): Promise<BotTurnCompletion | undefined> {
|
|
530
|
+
const run = this.codec.optional(
|
|
531
|
+
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
532
|
+
);
|
|
533
|
+
if (run?.status === "superseded") {
|
|
534
|
+
return { runId, text: "", events: structuredClone(run.events) };
|
|
535
|
+
}
|
|
536
|
+
if (run?.status !== "completed") return undefined;
|
|
537
|
+
return {
|
|
538
|
+
runId,
|
|
539
|
+
text: run.responseText ?? "",
|
|
540
|
+
events: structuredClone(run.events),
|
|
541
|
+
...(await this.storedNotification(runId)),
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
private async storedNotification(
|
|
546
|
+
runId: string,
|
|
547
|
+
): Promise<{ notification?: BotNotificationIntent }> {
|
|
548
|
+
const notification = await this.ctx.storage.get<BotNotificationIntent>(
|
|
549
|
+
`${NOTIFICATION_PREFIX}${runId}`,
|
|
550
|
+
);
|
|
551
|
+
return notification ? { notification } : {};
|
|
552
|
+
}
|
|
553
|
+
|
|
301
554
|
private async executeResumedRun(
|
|
302
555
|
identity: BotIdentity,
|
|
303
556
|
run: StoredRunV1<Snapshot>,
|
|
@@ -373,6 +626,8 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
373
626
|
throw new Error(message);
|
|
374
627
|
}
|
|
375
628
|
await this.failRun(run.runId, previous, events, message);
|
|
629
|
+
const settled = await this.supersededRunResult(run.runId);
|
|
630
|
+
if (settled) return settled;
|
|
376
631
|
throw new Error(message);
|
|
377
632
|
} finally {
|
|
378
633
|
if (this.executingRunId === run.runId) this.executingRunId = undefined;
|
|
@@ -395,7 +650,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
395
650
|
});
|
|
396
651
|
}
|
|
397
652
|
|
|
398
|
-
private async
|
|
653
|
+
private async settledRunResult(
|
|
399
654
|
command: OwnedBotTurnCommand,
|
|
400
655
|
): Promise<BotTurnCompletion | undefined> {
|
|
401
656
|
const { runId } = command;
|
|
@@ -408,6 +663,16 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
408
663
|
`Turn idempotency key "${runId}" was reused for a different command`,
|
|
409
664
|
);
|
|
410
665
|
}
|
|
666
|
+
// A Turn another user message took the place of is an ordinary outcome,
|
|
667
|
+
// not a failure: it settled durably, said whatever it had already said,
|
|
668
|
+
// and the caller reads the rest from durable state.
|
|
669
|
+
if (run.status === "superseded") {
|
|
670
|
+
return {
|
|
671
|
+
runId,
|
|
672
|
+
text: "",
|
|
673
|
+
events: structuredClone(run.events),
|
|
674
|
+
};
|
|
675
|
+
}
|
|
411
676
|
if (run.status !== "completed") {
|
|
412
677
|
throw new Error(
|
|
413
678
|
`run "${runId}" already exists with status ${run.status}`,
|
|
@@ -627,6 +892,13 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
627
892
|
(!activeRun || activeRun.status !== "reconciliation-required")
|
|
628
893
|
) {
|
|
629
894
|
deadlines.push(Date.now() + RECOVERY_ALARM_DELAY_MS);
|
|
895
|
+
} else if (
|
|
896
|
+
!activeRunId &&
|
|
897
|
+
(await transaction.get<string>(PENDING_RUN_KEY))
|
|
898
|
+
) {
|
|
899
|
+
// A Turn admitted and waiting is work this object owes, so it keeps the
|
|
900
|
+
// recovery alarm even with nothing running.
|
|
901
|
+
deadlines.push(Date.now() + RECOVERY_ALARM_DELAY_MS);
|
|
630
902
|
}
|
|
631
903
|
if (deadlines.length === 0) await transaction.deleteAlarm();
|
|
632
904
|
else await transaction.setAlarm(Math.min(...deadlines));
|
|
@@ -643,11 +915,15 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
643
915
|
if (!existing) await this.ctx.storage.put(IDENTITY_KEY, identity);
|
|
644
916
|
}
|
|
645
917
|
|
|
646
|
-
private async acceptRun(command: OwnedBotTurnCommand): Promise<
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
918
|
+
private async acceptRun(command: OwnedBotTurnCommand): Promise<
|
|
919
|
+
| {
|
|
920
|
+
kind: "active";
|
|
921
|
+
previous: SessionEvent[];
|
|
922
|
+
settings: Snapshot;
|
|
923
|
+
compositionGenerationId: string;
|
|
924
|
+
}
|
|
925
|
+
| { kind: "queued"; interrupt?: { runId: string } }
|
|
926
|
+
> {
|
|
651
927
|
const fenceKey = `${RUN_ADMISSION_FENCE_PREFIX}${command.runId}`;
|
|
652
928
|
const fences = storedRunAdmissionFences(
|
|
653
929
|
await this.ctx.storage.get<unknown>(RUN_ADMISSION_FENCE_INDEX_KEY),
|
|
@@ -690,9 +966,10 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
690
966
|
) {
|
|
691
967
|
throw new Error("Bot authority does not match its durable identity");
|
|
692
968
|
}
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
969
|
+
const activeRunId = await transaction.get<string>(ACTIVE_RUN_KEY);
|
|
970
|
+
const supersede = activeRunId
|
|
971
|
+
? await this.planSupersede(transaction, command, activeRunId)
|
|
972
|
+
: undefined;
|
|
696
973
|
const latestEvents = (
|
|
697
974
|
(await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
|
|
698
975
|
).map(decodeSessionEvent);
|
|
@@ -710,7 +987,10 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
710
987
|
events: [],
|
|
711
988
|
effectAdmissions: [],
|
|
712
989
|
status: "running",
|
|
713
|
-
|
|
990
|
+
// A queued Turn is admitted — durable, ordered, and owed a terminal
|
|
991
|
+
// state — but has not started. Its `previousEventCount` is recomputed
|
|
992
|
+
// when it is promoted, because the Turn ahead of it is still writing.
|
|
993
|
+
phase: activeRunId ? "queued" : "admitted",
|
|
714
994
|
compositionGenerationId: pin.generationId,
|
|
715
995
|
configurationSnapshot: structuredClone(admittedSettings),
|
|
716
996
|
previousEventCount: latestEvents.length,
|
|
@@ -718,6 +998,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
718
998
|
command.turnType,
|
|
719
999
|
command.origin,
|
|
720
1000
|
command.subagentRole,
|
|
1001
|
+
command.lane,
|
|
721
1002
|
),
|
|
722
1003
|
...(command.directTool
|
|
723
1004
|
? { directTool: structuredClone(command.directTool) }
|
|
@@ -726,14 +1007,27 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
726
1007
|
await transaction.put({
|
|
727
1008
|
[key]: admittedRun,
|
|
728
1009
|
[runIndexKey(command.acceptedAt, command.runId)]: command.runId,
|
|
729
|
-
|
|
1010
|
+
...(activeRunId
|
|
1011
|
+
? { [PENDING_RUN_KEY]: command.runId }
|
|
1012
|
+
: { [ACTIVE_RUN_KEY]: command.runId }),
|
|
730
1013
|
[IDENTITY_KEY]: identity ?? {
|
|
731
1014
|
userId: command.userId,
|
|
732
1015
|
botId: command.botId,
|
|
733
1016
|
},
|
|
734
1017
|
});
|
|
1018
|
+
const interrupted = supersede ? await supersede(command.runId) : false;
|
|
735
1019
|
await this.refreshRecoveryAlarm(transaction);
|
|
1020
|
+
if (activeRunId) {
|
|
1021
|
+
return {
|
|
1022
|
+
kind: "queued" as const,
|
|
1023
|
+
// Only a Turn whose supersede intent was actually recorded is
|
|
1024
|
+
// interrupted. One that had not dispatched a model request is left
|
|
1025
|
+
// to finish, and the new message simply waits behind it.
|
|
1026
|
+
...(interrupted ? { interrupt: { runId: activeRunId } } : {}),
|
|
1027
|
+
};
|
|
1028
|
+
}
|
|
736
1029
|
return {
|
|
1030
|
+
kind: "active" as const,
|
|
737
1031
|
previous: latestEvents,
|
|
738
1032
|
settings: admittedSettings,
|
|
739
1033
|
compositionGenerationId: pin.generationId,
|
|
@@ -741,6 +1035,104 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
741
1035
|
});
|
|
742
1036
|
}
|
|
743
1037
|
|
|
1038
|
+
/**
|
|
1039
|
+
* Decides whether one new command may take the place of what is running.
|
|
1040
|
+
*
|
|
1041
|
+
* The rule is the lane's: a user-lane admission carrying explicit supersede
|
|
1042
|
+
* intent replaces the active run and any run already waiting behind it; a
|
|
1043
|
+
* background admission never supersedes and is refused exactly as a second
|
|
1044
|
+
* command always was, so a Routine firing waits for its own next schedule
|
|
1045
|
+
* rather than interrupting a person mid-sentence.
|
|
1046
|
+
*
|
|
1047
|
+
* Returns the writes the admission performs, which report whether the active
|
|
1048
|
+
* Turn was actually interrupted. A Turn that has not dispatched a model
|
|
1049
|
+
* request is left alone — there is nothing durable to lose — and the new
|
|
1050
|
+
* message simply queues behind it.
|
|
1051
|
+
*/
|
|
1052
|
+
private async planSupersede(
|
|
1053
|
+
transaction: DurableObjectTransaction,
|
|
1054
|
+
command: OwnedBotTurnCommand,
|
|
1055
|
+
activeRunId: string,
|
|
1056
|
+
): Promise<((supersededBy: string) => Promise<boolean>) | undefined> {
|
|
1057
|
+
const lane = command.lane ?? defaultRunLaneV1(command.turnType ?? "chat");
|
|
1058
|
+
// The intent is the whole of the decision, and it is the *presence* of the
|
|
1059
|
+
// field that carries it. `supersedes: {}` — a composer that had observed
|
|
1060
|
+
// no run when the person pressed send — supersedes exactly as a named one
|
|
1061
|
+
// does; only an absent field is "no intent", and that is still refused.
|
|
1062
|
+
if (lane !== "user" || !command.supersedes) {
|
|
1063
|
+
throw new Error("bot already has an active run");
|
|
1064
|
+
}
|
|
1065
|
+
const active = this.codec.optional(
|
|
1066
|
+
await transaction.get<unknown>(`${RUN_PREFIX}${activeRunId}`),
|
|
1067
|
+
);
|
|
1068
|
+
if (!active) throw new Error("bot already has an active run");
|
|
1069
|
+
if (active.status === "reconciliation-required") {
|
|
1070
|
+
// An uncertain external effect is never abandoned to admit something
|
|
1071
|
+
// else: the outcome has to be retrieved before this object runs again.
|
|
1072
|
+
throw new Error(
|
|
1073
|
+
`run "${activeRunId}" requires reconciliation before another Turn can be admitted`,
|
|
1074
|
+
);
|
|
1075
|
+
}
|
|
1076
|
+
if (active.status !== "running") {
|
|
1077
|
+
throw new Error("bot already has an active run");
|
|
1078
|
+
}
|
|
1079
|
+
const pendingRunId = await transaction.get<string>(PENDING_RUN_KEY);
|
|
1080
|
+
// A Turn that has not dispatched a model request has no durable work to
|
|
1081
|
+
// lose, so it is left to finish and the new message queues behind it.
|
|
1082
|
+
// GrokBot draws the same line, and for the same reason: nothing may be
|
|
1083
|
+
// stranded before its first durable checkpoint.
|
|
1084
|
+
const dispatched = active.events.some(
|
|
1085
|
+
(event) => event.type === "model/request",
|
|
1086
|
+
);
|
|
1087
|
+
return async (supersededBy: string) => {
|
|
1088
|
+
if (pendingRunId && pendingRunId !== supersededBy) {
|
|
1089
|
+
await this.supersedeQueuedRun(transaction, pendingRunId, supersededBy);
|
|
1090
|
+
}
|
|
1091
|
+
if (!dispatched) return false;
|
|
1092
|
+
if (active.supersededAt) return true;
|
|
1093
|
+
await transaction.put(
|
|
1094
|
+
`${RUN_PREFIX}${activeRunId}`,
|
|
1095
|
+
structuredClone(
|
|
1096
|
+
this.codec.require({
|
|
1097
|
+
...active,
|
|
1098
|
+
supersededAt: new Date().toISOString(),
|
|
1099
|
+
supersededBy,
|
|
1100
|
+
} satisfies StoredRunV1<Snapshot>),
|
|
1101
|
+
),
|
|
1102
|
+
);
|
|
1103
|
+
return true;
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
/**
|
|
1108
|
+
* Settles a Turn that was superseded before it ever started. It appended no
|
|
1109
|
+
* event and spoke to nobody, so it settles as a record on its own.
|
|
1110
|
+
*/
|
|
1111
|
+
private async supersedeQueuedRun(
|
|
1112
|
+
transaction: DurableObjectTransaction,
|
|
1113
|
+
runId: string,
|
|
1114
|
+
supersededBy: string,
|
|
1115
|
+
): Promise<void> {
|
|
1116
|
+
const key = `${RUN_PREFIX}${runId}`;
|
|
1117
|
+
const queued = this.codec.optional(await transaction.get<unknown>(key));
|
|
1118
|
+
if (!queued || queued.status !== "running" || queued.phase !== "queued") {
|
|
1119
|
+
return;
|
|
1120
|
+
}
|
|
1121
|
+
const { responseText: _text, failure: _failure, ...settled } = queued;
|
|
1122
|
+
await transaction.put(
|
|
1123
|
+
key,
|
|
1124
|
+
structuredClone(
|
|
1125
|
+
this.codec.require({
|
|
1126
|
+
...settled,
|
|
1127
|
+
status: "superseded",
|
|
1128
|
+
phase: "admitted",
|
|
1129
|
+
supersededAt: new Date().toISOString(),
|
|
1130
|
+
supersededBy,
|
|
1131
|
+
} satisfies StoredRunV1<Snapshot>),
|
|
1132
|
+
),
|
|
1133
|
+
);
|
|
1134
|
+
}
|
|
1135
|
+
|
|
744
1136
|
private async persistRunEvents(
|
|
745
1137
|
runId: string,
|
|
746
1138
|
events: readonly SessionEvent[],
|
|
@@ -792,6 +1184,14 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
792
1184
|
});
|
|
793
1185
|
}
|
|
794
1186
|
|
|
1187
|
+
/** The Package's superseded-record hook, or `undefined` when it has none. */
|
|
1188
|
+
private supersededPackageRecords():
|
|
1189
|
+
SupersededPackageRecords<Snapshot> | undefined {
|
|
1190
|
+
const hook = this.hooks.supersededRecords;
|
|
1191
|
+
if (!hook) return undefined;
|
|
1192
|
+
return (input) => hook.call(this.hooks, input);
|
|
1193
|
+
}
|
|
1194
|
+
|
|
795
1195
|
private terminalKeys(runId: string) {
|
|
796
1196
|
return {
|
|
797
1197
|
run: `${RUN_PREFIX}${runId}`,
|
|
@@ -816,6 +1216,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
816
1216
|
previous,
|
|
817
1217
|
result,
|
|
818
1218
|
this.terminalPackageRecords(snapshot),
|
|
1219
|
+
this.supersededPackageRecords(),
|
|
819
1220
|
);
|
|
820
1221
|
await this.refreshRecoveryAlarm(transaction);
|
|
821
1222
|
});
|
|
@@ -836,6 +1237,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
836
1237
|
previous,
|
|
837
1238
|
events,
|
|
838
1239
|
failure,
|
|
1240
|
+
this.supersededPackageRecords(),
|
|
839
1241
|
);
|
|
840
1242
|
await this.refreshRecoveryAlarm(transaction);
|
|
841
1243
|
});
|
|
@@ -861,9 +1263,60 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
861
1263
|
});
|
|
862
1264
|
}
|
|
863
1265
|
|
|
1266
|
+
/**
|
|
1267
|
+
* Starts the Turn that was waiting when the object last stopped.
|
|
1268
|
+
*
|
|
1269
|
+
* "Every admitted Turn reaches a durable terminal or resumable state" covers
|
|
1270
|
+
* a Turn that was admitted and never started too: the object can be evicted
|
|
1271
|
+
* between the Turn it superseded terminalizing and its own first step, and
|
|
1272
|
+
* this is what picks it up. It runs exactly once — the promotion is a
|
|
1273
|
+
* transaction, and a caller in this object already waiting for it is left to
|
|
1274
|
+
* do the promoting itself.
|
|
1275
|
+
*/
|
|
1276
|
+
private async recoverQueuedRun(): Promise<void> {
|
|
1277
|
+
const pendingRunId = await this.ctx.storage.get<string>(PENDING_RUN_KEY);
|
|
1278
|
+
if (!pendingRunId || this.queuedWaiters.has(pendingRunId)) return;
|
|
1279
|
+
if (pendingRunId === this.executingRunId) return;
|
|
1280
|
+
const durableIdentity =
|
|
1281
|
+
await this.ctx.storage.get<BotIdentity>(IDENTITY_KEY);
|
|
1282
|
+
const promoted = await this.promoteQueuedRun(pendingRunId);
|
|
1283
|
+
if (typeof promoted === "string") return;
|
|
1284
|
+
if (!durableIdentity) throw new Error("Bot identity is unavailable");
|
|
1285
|
+
const run = await this.readRun(pendingRunId);
|
|
1286
|
+
if (!run) throw new Error(`run "${pendingRunId}" was not accepted`);
|
|
1287
|
+
await this.executeAcceptedRun(
|
|
1288
|
+
this.recoveredCommand(durableIdentity, run),
|
|
1289
|
+
promoted.previous,
|
|
1290
|
+
promoted.settings,
|
|
1291
|
+
promoted.compositionGenerationId,
|
|
1292
|
+
);
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1295
|
+
/** The command a durable run record replays as after eviction. */
|
|
1296
|
+
private recoveredCommand(
|
|
1297
|
+
identity: BotIdentity,
|
|
1298
|
+
run: StoredRunV1<Snapshot>,
|
|
1299
|
+
): OwnedBotTurnCommand {
|
|
1300
|
+
return {
|
|
1301
|
+
userId: identity.userId,
|
|
1302
|
+
botId: identity.botId,
|
|
1303
|
+
runId: run.runId,
|
|
1304
|
+
sessionId: run.sessionId,
|
|
1305
|
+
acceptedAt: run.acceptedAt,
|
|
1306
|
+
text: run.input,
|
|
1307
|
+
turnType: storedRunTurnTypeV1(run),
|
|
1308
|
+
...(storedRunSubagentRoleV1(run)
|
|
1309
|
+
? { subagentRole: storedRunSubagentRoleV1(run) }
|
|
1310
|
+
: {}),
|
|
1311
|
+
...(run.admission?.origin ? { origin: run.admission.origin } : {}),
|
|
1312
|
+
...(run.directTool ? { directTool: run.directTool } : {}),
|
|
1313
|
+
};
|
|
1314
|
+
}
|
|
1315
|
+
|
|
864
1316
|
async recoverActiveRun(): Promise<void> {
|
|
865
1317
|
const activeRunId = await this.ctx.storage.get<string>(ACTIVE_RUN_KEY);
|
|
866
|
-
if (!activeRunId
|
|
1318
|
+
if (!activeRunId) return this.recoverQueuedRun();
|
|
1319
|
+
if (activeRunId === this.executingRunId) return;
|
|
867
1320
|
const durableIdentity =
|
|
868
1321
|
await this.ctx.storage.get<BotIdentity>(IDENTITY_KEY);
|
|
869
1322
|
const key = `${RUN_PREFIX}${activeRunId}`;
|
|
@@ -901,6 +1354,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
901
1354
|
latest.slice(0, run.previousEventCount),
|
|
902
1355
|
completed,
|
|
903
1356
|
this.terminalPackageRecords(run.configurationSnapshot),
|
|
1357
|
+
this.supersededPackageRecords(),
|
|
904
1358
|
);
|
|
905
1359
|
await this.refreshRecoveryAlarm(transaction);
|
|
906
1360
|
return undefined;
|
|
@@ -914,6 +1368,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
914
1368
|
latest.slice(0, run.previousEventCount),
|
|
915
1369
|
run.events,
|
|
916
1370
|
plan.failure,
|
|
1371
|
+
this.supersededPackageRecords(),
|
|
917
1372
|
);
|
|
918
1373
|
await this.refreshRecoveryAlarm(transaction);
|
|
919
1374
|
return undefined;
|