@frockbot/kernel-do 0.3.4 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/authority.ts +141 -42
- package/src/run-recovery.ts +40 -0
- package/src/turn-errors.ts +44 -0
- package/src/turn-supersede.test.ts +253 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@frockbot/kernel-do",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.5",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
"typecheck": "tsc --noEmit -p tsconfig.json"
|
|
13
13
|
},
|
|
14
14
|
"dependencies": {
|
|
15
|
-
"@frockbot/kernel-composition": "0.3.
|
|
16
|
-
"@frockbot/kernel-contracts": "0.3.
|
|
15
|
+
"@frockbot/kernel-composition": "0.3.5",
|
|
16
|
+
"@frockbot/kernel-contracts": "0.3.5",
|
|
17
17
|
"cordis": "4.0.0-rc.8"
|
|
18
18
|
},
|
|
19
19
|
"devDependencies": {
|
package/src/authority.ts
CHANGED
|
@@ -34,11 +34,13 @@ import {
|
|
|
34
34
|
eventsForFailedRun,
|
|
35
35
|
latestModelRequestJournalState,
|
|
36
36
|
planBotRunRecovery,
|
|
37
|
+
repairOrphanedOpenTurnV1,
|
|
37
38
|
unresolvedModelRequestFailure,
|
|
38
39
|
} from "./run-recovery.js";
|
|
39
40
|
import {
|
|
40
41
|
BotTurnReconciliationRequiredError,
|
|
41
42
|
BotTurnRecoveryRequiredError,
|
|
43
|
+
BotTurnRefusedError,
|
|
42
44
|
} from "./turn-errors.js";
|
|
43
45
|
import {
|
|
44
46
|
ACTIVE_RUN_KEY,
|
|
@@ -150,6 +152,22 @@ export const SUPERSEDED_TURN_REASON_V1 = "superseded by a new user message";
|
|
|
150
152
|
/** How many times a queued Turn retries the object before giving up. */
|
|
151
153
|
const MAX_QUEUED_RUN_START_ATTEMPTS = 8;
|
|
152
154
|
|
|
155
|
+
/**
|
|
156
|
+
* True when this object has already durably decided to throw the Turn away.
|
|
157
|
+
*
|
|
158
|
+
* Reconciliation exists to retrieve an external outcome the Turn still needs.
|
|
159
|
+
* A Turn a Stop or a supersede has already discarded needs nothing: its
|
|
160
|
+
* provider outcome cannot change what it settles as, and parking it would keep
|
|
161
|
+
* the active-run marker — and so refuse every later message — over an answer
|
|
162
|
+
* nobody is waiting for. The intent the User expressed wins, and the run
|
|
163
|
+
* settles `cancelled` or `superseded` with everything it had already said.
|
|
164
|
+
*/
|
|
165
|
+
function runWasDiscardedV1(
|
|
166
|
+
run: { stopRequestedAt?: string; supersededAt?: string } | undefined,
|
|
167
|
+
): boolean {
|
|
168
|
+
return Boolean(run?.stopRequestedAt || run?.supersededAt);
|
|
169
|
+
}
|
|
170
|
+
|
|
153
171
|
export interface BotDurableAuthorityOptions<Snapshot> {
|
|
154
172
|
state: DurableObjectState;
|
|
155
173
|
codec: StoredRunCodecV1<Snapshot>;
|
|
@@ -193,7 +211,14 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
193
211
|
|
|
194
212
|
async run(command: OwnedBotTurnCommand): Promise<BotTurnCompletion> {
|
|
195
213
|
await this.assertMatchingRunCommand(command);
|
|
196
|
-
|
|
214
|
+
// Recovering whatever this object was left holding must never decide the
|
|
215
|
+
// fate of a new command. `recoverActiveRun` executes the *previous* Turn
|
|
216
|
+
// inline and rethrows, so a recovery that failed — an uncertain effect, a
|
|
217
|
+
// mount failure, a provider that was down — threw before the new message
|
|
218
|
+
// was ever admitted, and the person's message was simply lost. The old
|
|
219
|
+
// Turn is durable either way and the alarm retries it; admission now
|
|
220
|
+
// refuses or supersedes on its own terms.
|
|
221
|
+
await this.recoverActiveRun().catch(() => undefined);
|
|
197
222
|
const replay = await this.settledRunResult(command);
|
|
198
223
|
if (replay) return replay;
|
|
199
224
|
const admission = await this.acceptRun(command);
|
|
@@ -240,8 +265,10 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
240
265
|
const promoted = await this.promoteQueuedRun(command.runId);
|
|
241
266
|
if (promoted === "blocked") {
|
|
242
267
|
// Another Turn holds the object. Recovery drives it to its own
|
|
243
|
-
// durable terminal or resumable state, and this one tries again
|
|
244
|
-
|
|
268
|
+
// durable terminal or resumable state, and this one tries again —
|
|
269
|
+
// including when that recovery fails, which is the other Turn's
|
|
270
|
+
// problem and not this one's.
|
|
271
|
+
await this.recoverActiveRun().catch(() => undefined);
|
|
245
272
|
// Unless what holds the object is an uncertain effect. That is
|
|
246
273
|
// settled by an explicit reconciliation the User asks for, on their
|
|
247
274
|
// own clock, and retrying against it would only burn this caller's
|
|
@@ -249,11 +276,14 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
249
276
|
// run is durable: it stays queued, and the reconciliation's own
|
|
250
277
|
// settlement — or the recovery alarm — starts it.
|
|
251
278
|
if (await this.activeRunAwaitsReconciliation()) {
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
279
|
+
// A Turn that has not run is not a completed Turn. Answering with
|
|
280
|
+
// an empty completion made the browser render the person's new
|
|
281
|
+
// message as answered with silence; the durable queue entry stays,
|
|
282
|
+
// and the refusal says why nothing has happened yet.
|
|
283
|
+
throw new BotTurnRefusedError(
|
|
284
|
+
"reconciliation-required",
|
|
285
|
+
`run "${command.runId}" is queued: the active run requires reconciliation before another Turn can be admitted`,
|
|
286
|
+
);
|
|
257
287
|
}
|
|
258
288
|
continue;
|
|
259
289
|
}
|
|
@@ -369,12 +399,22 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
369
399
|
(await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
|
|
370
400
|
).map(decodeSessionEvent);
|
|
371
401
|
const settings = run.configurationSnapshot;
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
}
|
|
402
|
+
// The failure is *removed*, not set to `undefined`: a running run that
|
|
403
|
+
// carries a `failure` key is a shape the run record does not allow, and
|
|
404
|
+
// writing one turned "resolve this Turn" into a record nothing could
|
|
405
|
+
// read afterwards. `require` checks it here, where the write is, rather
|
|
406
|
+
// than leaving the projector to fail on every later read.
|
|
407
|
+
const { failure: _failure, ...resumed } = run;
|
|
408
|
+
await transaction.put(
|
|
409
|
+
key,
|
|
410
|
+
structuredClone(
|
|
411
|
+
this.codec.require({
|
|
412
|
+
...resumed,
|
|
413
|
+
status: "running",
|
|
414
|
+
phase: "executing",
|
|
415
|
+
} satisfies StoredRunV1<Snapshot>),
|
|
416
|
+
),
|
|
417
|
+
);
|
|
378
418
|
await this.refreshRecoveryAlarm(transaction);
|
|
379
419
|
return { run, latest, settings };
|
|
380
420
|
});
|
|
@@ -480,8 +520,9 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
480
520
|
}
|
|
481
521
|
const modelState = latestModelRequestJournalState(events);
|
|
482
522
|
if (
|
|
483
|
-
error instanceof BotTurnReconciliationRequiredError ||
|
|
484
|
-
|
|
523
|
+
(error instanceof BotTurnReconciliationRequiredError ||
|
|
524
|
+
modelState.status === "unresolved") &&
|
|
525
|
+
!runWasDiscardedV1(durableRun)
|
|
485
526
|
) {
|
|
486
527
|
await this.requireRunReconciliation(
|
|
487
528
|
command.runId,
|
|
@@ -612,8 +653,9 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
612
653
|
}
|
|
613
654
|
const modelState = latestModelRequestJournalState(events);
|
|
614
655
|
if (
|
|
615
|
-
error instanceof BotTurnReconciliationRequiredError ||
|
|
616
|
-
|
|
656
|
+
(error instanceof BotTurnReconciliationRequiredError ||
|
|
657
|
+
modelState.status === "unresolved") &&
|
|
658
|
+
!runWasDiscardedV1(durableRun)
|
|
617
659
|
) {
|
|
618
660
|
await this.requireRunReconciliation(
|
|
619
661
|
run.runId,
|
|
@@ -659,7 +701,8 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
659
701
|
);
|
|
660
702
|
if (!run) return undefined;
|
|
661
703
|
if (run.commandFingerprint !== botTurnCommandFingerprintV1(command)) {
|
|
662
|
-
throw new
|
|
704
|
+
throw new BotTurnRefusedError(
|
|
705
|
+
"duplicate",
|
|
663
706
|
`Turn idempotency key "${runId}" was reused for a different command`,
|
|
664
707
|
);
|
|
665
708
|
}
|
|
@@ -674,7 +717,8 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
674
717
|
};
|
|
675
718
|
}
|
|
676
719
|
if (run.status !== "completed") {
|
|
677
|
-
throw new
|
|
720
|
+
throw new BotTurnRefusedError(
|
|
721
|
+
"duplicate",
|
|
678
722
|
`run "${runId}" already exists with status ${run.status}`,
|
|
679
723
|
);
|
|
680
724
|
}
|
|
@@ -702,7 +746,8 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
702
746
|
run &&
|
|
703
747
|
run.commandFingerprint !== botTurnCommandFingerprintV1(command)
|
|
704
748
|
) {
|
|
705
|
-
throw new
|
|
749
|
+
throw new BotTurnRefusedError(
|
|
750
|
+
"duplicate",
|
|
706
751
|
`Turn idempotency key "${command.runId}" was reused for a different command`,
|
|
707
752
|
);
|
|
708
753
|
}
|
|
@@ -788,7 +833,16 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
788
833
|
this.ctx.storage.get<BotIdentity>(IDENTITY_KEY),
|
|
789
834
|
]);
|
|
790
835
|
const run = this.codec.optional(storedRun);
|
|
791
|
-
if (run?.status === "reconciliation-required" && identity)
|
|
836
|
+
if (run?.status === "reconciliation-required" && identity) {
|
|
837
|
+
// A parked run is not this alarm's to settle — only an explicit
|
|
838
|
+
// reconciliation settles it — but returning without rescheduling
|
|
839
|
+
// dropped the object's *other* deadlines with it: a Routine due while
|
|
840
|
+
// a Bot sat parked never fired, and nothing set the alarm again.
|
|
841
|
+
await this.ctx.storage.transaction((transaction) =>
|
|
842
|
+
this.refreshRecoveryAlarm(transaction),
|
|
843
|
+
);
|
|
844
|
+
return;
|
|
845
|
+
}
|
|
792
846
|
}
|
|
793
847
|
await this.recoverActiveRun();
|
|
794
848
|
}
|
|
@@ -855,17 +909,15 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
855
909
|
const storedFences = storedRunAdmissionFences(
|
|
856
910
|
await transaction.get<unknown>(RUN_ADMISSION_FENCE_INDEX_KEY),
|
|
857
911
|
);
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
912
|
+
// A bounded FIFO, not a cliff. Nothing ever evicted an entry, so a Bot
|
|
913
|
+
// that had refused 256 sends over its life answered every later fence
|
|
914
|
+
// with a 500 and left the client retrying "Turn admission lookup
|
|
915
|
+
// failed" forever. A run id old enough to age out here can no longer
|
|
916
|
+
// be admitted by any live caller.
|
|
917
|
+
const kept = storedFences.filter((fenced) => fenced !== runId);
|
|
918
|
+
while (kept.length >= MAX_RUN_ADMISSION_FENCES) kept.shift();
|
|
864
919
|
await transaction.put({
|
|
865
|
-
[RUN_ADMISSION_FENCE_INDEX_KEY]: [
|
|
866
|
-
...storedFences.filter((fenced) => fenced !== runId),
|
|
867
|
-
runId,
|
|
868
|
-
],
|
|
920
|
+
[RUN_ADMISSION_FENCE_INDEX_KEY]: [...kept, runId],
|
|
869
921
|
[IDENTITY_KEY]: durableIdentity ?? identity,
|
|
870
922
|
});
|
|
871
923
|
await transaction.delete(`${RUN_ADMISSION_FENCE_PREFIX}${runId}`);
|
|
@@ -932,7 +984,10 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
932
984
|
fences.includes(command.runId) ||
|
|
933
985
|
(await this.ctx.storage.get(fenceKey))
|
|
934
986
|
) {
|
|
935
|
-
throw new
|
|
987
|
+
throw new BotTurnRefusedError(
|
|
988
|
+
"fenced",
|
|
989
|
+
`run "${command.runId}" admission was fenced`,
|
|
990
|
+
);
|
|
936
991
|
}
|
|
937
992
|
const settings = await this.hooks.resolveAdmissionSnapshot(command);
|
|
938
993
|
// Materialized before the transaction; the pin itself is read inside it.
|
|
@@ -944,20 +999,30 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
944
999
|
if (
|
|
945
1000
|
existing.commandFingerprint !== botTurnCommandFingerprintV1(command)
|
|
946
1001
|
) {
|
|
947
|
-
throw new
|
|
1002
|
+
throw new BotTurnRefusedError(
|
|
1003
|
+
"duplicate",
|
|
948
1004
|
`Turn idempotency key "${command.runId}" was reused for a different command`,
|
|
949
1005
|
);
|
|
950
1006
|
}
|
|
951
1007
|
if (existing.status === "completed") {
|
|
952
|
-
throw new
|
|
1008
|
+
throw new BotTurnRefusedError(
|
|
1009
|
+
"duplicate",
|
|
1010
|
+
`run "${command.runId}" already completed`,
|
|
1011
|
+
);
|
|
953
1012
|
}
|
|
954
|
-
throw new
|
|
1013
|
+
throw new BotTurnRefusedError(
|
|
1014
|
+
"duplicate",
|
|
1015
|
+
`run "${command.runId}" already exists`,
|
|
1016
|
+
);
|
|
955
1017
|
}
|
|
956
1018
|
const fences = storedRunAdmissionFences(
|
|
957
1019
|
await transaction.get<unknown>(RUN_ADMISSION_FENCE_INDEX_KEY),
|
|
958
1020
|
);
|
|
959
1021
|
if (fences.includes(command.runId) || (await transaction.get(fenceKey))) {
|
|
960
|
-
throw new
|
|
1022
|
+
throw new BotTurnRefusedError(
|
|
1023
|
+
"fenced",
|
|
1024
|
+
`run "${command.runId}" admission was fenced`,
|
|
1025
|
+
);
|
|
961
1026
|
}
|
|
962
1027
|
const identity = await transaction.get<BotIdentity>(IDENTITY_KEY);
|
|
963
1028
|
if (
|
|
@@ -970,9 +1035,25 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
970
1035
|
const supersede = activeRunId
|
|
971
1036
|
? await this.planSupersede(transaction, command, activeRunId)
|
|
972
1037
|
: undefined;
|
|
973
|
-
const
|
|
1038
|
+
const storedEvents = (
|
|
974
1039
|
(await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
|
|
975
1040
|
).map(decodeSessionEvent);
|
|
1041
|
+
// A Turn that died between `turn/start` and `turn/end` — an event the
|
|
1042
|
+
// encoder refused, a durable write that failed — left the log open, and
|
|
1043
|
+
// every later Turn failed validation with "turn N started while turn
|
|
1044
|
+
// N-1 is open". Nothing owned that repair, because the run that would
|
|
1045
|
+
// have closed it is already terminal, so admission does: with nothing
|
|
1046
|
+
// executing, an open Turn is one nobody is going to finish.
|
|
1047
|
+
const repairs = activeRunId
|
|
1048
|
+
? []
|
|
1049
|
+
: repairOrphanedOpenTurnV1(command.sessionId, storedEvents);
|
|
1050
|
+
const latestEvents = [...storedEvents, ...repairs];
|
|
1051
|
+
if (repairs.length > 0) {
|
|
1052
|
+
await transaction.put(
|
|
1053
|
+
LATEST_EVENTS_KEY,
|
|
1054
|
+
structuredClone(latestEvents.map(decodeSessionEvent)),
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
976
1057
|
const admittedSettings = await this.hooks.admittedSnapshot(
|
|
977
1058
|
transaction,
|
|
978
1059
|
settings,
|
|
@@ -1060,21 +1141,23 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1060
1141
|
// no run when the person pressed send — supersedes exactly as a named one
|
|
1061
1142
|
// does; only an absent field is "no intent", and that is still refused.
|
|
1062
1143
|
if (lane !== "user" || !command.supersedes) {
|
|
1063
|
-
throw new
|
|
1144
|
+
throw new BotTurnRefusedError("busy", "bot already has an active run");
|
|
1064
1145
|
}
|
|
1065
1146
|
const active = this.codec.optional(
|
|
1066
1147
|
await transaction.get<unknown>(`${RUN_PREFIX}${activeRunId}`),
|
|
1067
1148
|
);
|
|
1068
|
-
if (!active)
|
|
1149
|
+
if (!active)
|
|
1150
|
+
throw new BotTurnRefusedError("busy", "bot already has an active run");
|
|
1069
1151
|
if (active.status === "reconciliation-required") {
|
|
1070
1152
|
// An uncertain external effect is never abandoned to admit something
|
|
1071
1153
|
// else: the outcome has to be retrieved before this object runs again.
|
|
1072
|
-
throw new
|
|
1154
|
+
throw new BotTurnRefusedError(
|
|
1155
|
+
"reconciliation-required",
|
|
1073
1156
|
`run "${activeRunId}" requires reconciliation before another Turn can be admitted`,
|
|
1074
1157
|
);
|
|
1075
1158
|
}
|
|
1076
1159
|
if (active.status !== "running") {
|
|
1077
|
-
throw new
|
|
1160
|
+
throw new BotTurnRefusedError("busy", "bot already has an active run");
|
|
1078
1161
|
}
|
|
1079
1162
|
const pendingRunId = await transaction.get<string>(PENDING_RUN_KEY);
|
|
1080
1163
|
// A Turn that has not dispatched a model request has no durable work to
|
|
@@ -1400,6 +1483,22 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1400
1483
|
await this.refreshRecoveryAlarm(transaction);
|
|
1401
1484
|
return { kind: "resume" as const, run, latest, settings };
|
|
1402
1485
|
}
|
|
1486
|
+
if (runWasDiscardedV1(run)) {
|
|
1487
|
+
// Recovery of a Turn Stop or supersede already discarded settles it on
|
|
1488
|
+
// that intent rather than parking it: nothing is owed the answer.
|
|
1489
|
+
await failStoredRun(
|
|
1490
|
+
this.codec,
|
|
1491
|
+
transaction,
|
|
1492
|
+
this.terminalKeys(run.runId),
|
|
1493
|
+
run.runId,
|
|
1494
|
+
latest.slice(0, run.previousEventCount),
|
|
1495
|
+
[...run.events, ...plan.repairs],
|
|
1496
|
+
"Execution outcome requires reconciliation before it can resume",
|
|
1497
|
+
this.supersededPackageRecords(),
|
|
1498
|
+
);
|
|
1499
|
+
await this.refreshRecoveryAlarm(transaction);
|
|
1500
|
+
return undefined;
|
|
1501
|
+
}
|
|
1403
1502
|
await transaction.put({
|
|
1404
1503
|
[key]: {
|
|
1405
1504
|
...run,
|
package/src/run-recovery.ts
CHANGED
|
@@ -182,6 +182,46 @@ export function planBotRunRecovery<Snapshot>(
|
|
|
182
182
|
return { kind: "reconcile", repairs: session.reconcileForResume() };
|
|
183
183
|
}
|
|
184
184
|
|
|
185
|
+
/** True when the durable log ends inside a Turn nothing is going to finish. */
|
|
186
|
+
export function hasOrphanedOpenTurnV1(
|
|
187
|
+
events: readonly SessionEvent[],
|
|
188
|
+
): boolean {
|
|
189
|
+
let openTurn: number | undefined;
|
|
190
|
+
for (const event of events) {
|
|
191
|
+
if (event.type === "turn/start") openTurn = event.turn;
|
|
192
|
+
if (event.type === "turn/end" && event.turn === openTurn) {
|
|
193
|
+
openTurn = undefined;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return openTurn !== undefined;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Closes a Turn the log was left inside, so the next one can start.
|
|
201
|
+
*
|
|
202
|
+
* A Turn that threw between `turn/start` and `turn/end` — an event the
|
|
203
|
+
* encoder refused, a durable write that failed — leaves an open turn in the
|
|
204
|
+
* durable log, and the next Turn on that Bot fails validation with "turn N
|
|
205
|
+
* started while turn N-1 is open". Forever: nothing owned the repair, because
|
|
206
|
+
* the run that would have written the `turn/end` is already terminal. This is
|
|
207
|
+
* that repair, applied when no run is executing, so an interrupted Turn is
|
|
208
|
+
* recorded as interrupted rather than wedging the Bot.
|
|
209
|
+
*
|
|
210
|
+
* A log too malformed to reconcile is left exactly as it is: repairing it
|
|
211
|
+
* blindly would invent history.
|
|
212
|
+
*/
|
|
213
|
+
export function repairOrphanedOpenTurnV1(
|
|
214
|
+
sessionId: string,
|
|
215
|
+
latest: readonly SessionEvent[],
|
|
216
|
+
): SessionEvent[] {
|
|
217
|
+
if (!hasOrphanedOpenTurnV1(latest)) return [];
|
|
218
|
+
try {
|
|
219
|
+
return new Session(sessionId, () => {}, latest).reconcileInterrupted();
|
|
220
|
+
} catch {
|
|
221
|
+
return [];
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
185
225
|
export function eventsForFailedRun(
|
|
186
226
|
durableRun: { events: SessionEvent[] } | undefined,
|
|
187
227
|
error: unknown,
|
package/src/turn-errors.ts
CHANGED
|
@@ -25,6 +25,50 @@ export class BotTurnReconciliationRequiredError extends Error {
|
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
/** Why the Bot declined to admit a Turn. */
|
|
29
|
+
export type BotTurnRefusalCodeV1 =
|
|
30
|
+
"busy" | "reconciliation-required" | "fenced" | "duplicate";
|
|
31
|
+
|
|
32
|
+
const BOT_TURN_REFUSAL_PREFIX_V1 = "BotTurnRefusedError:";
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* An admission the Bot declined, as a typed error rather than a sentence.
|
|
36
|
+
*
|
|
37
|
+
* A refusal crosses a Durable Object RPC boundary, which preserves an error's
|
|
38
|
+
* `name` and `message` and drops everything else — so the code rides on the
|
|
39
|
+
* name, the way the gateway already relies on `name` for `BotNotFoundError`.
|
|
40
|
+
* Classifying these by matching prose against `error.message` meant any
|
|
41
|
+
* reword silently turned an ordinary 409 into a 500, and two real messages
|
|
42
|
+
* already fell through.
|
|
43
|
+
*/
|
|
44
|
+
export class BotTurnRefusedError extends Error {
|
|
45
|
+
constructor(
|
|
46
|
+
readonly code: BotTurnRefusalCodeV1,
|
|
47
|
+
message: string,
|
|
48
|
+
) {
|
|
49
|
+
super(message);
|
|
50
|
+
this.name = `${BOT_TURN_REFUSAL_PREFIX_V1}${code}`;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The refusal an error carries, or `undefined` when it is not one. */
|
|
55
|
+
export function botTurnRefusalCodeV1(
|
|
56
|
+
error: unknown,
|
|
57
|
+
): BotTurnRefusalCodeV1 | undefined {
|
|
58
|
+
const name =
|
|
59
|
+
typeof error === "object" && error !== null && "name" in error
|
|
60
|
+
? String((error as { name: unknown }).name)
|
|
61
|
+
: "";
|
|
62
|
+
if (!name.startsWith(BOT_TURN_REFUSAL_PREFIX_V1)) return undefined;
|
|
63
|
+
const code = name.slice(BOT_TURN_REFUSAL_PREFIX_V1.length);
|
|
64
|
+
return code === "busy" ||
|
|
65
|
+
code === "reconciliation-required" ||
|
|
66
|
+
code === "fenced" ||
|
|
67
|
+
code === "duplicate"
|
|
68
|
+
? code
|
|
69
|
+
: undefined;
|
|
70
|
+
}
|
|
71
|
+
|
|
28
72
|
export class BotTurnRecoveryRequiredError extends Error {
|
|
29
73
|
constructor(readonly events: SessionEvent[]) {
|
|
30
74
|
super("Bot turn has a durable outcome settlement pending");
|
|
@@ -12,6 +12,10 @@ import {
|
|
|
12
12
|
type OwnedBotTurnCommand,
|
|
13
13
|
} from "./authority.ts";
|
|
14
14
|
import { MemoryStorage } from "./memory-storage.fixture.ts";
|
|
15
|
+
import {
|
|
16
|
+
BotTurnReconciliationRequiredError,
|
|
17
|
+
BotTurnRecoveryRequiredError,
|
|
18
|
+
} from "./turn-errors.ts";
|
|
15
19
|
import {
|
|
16
20
|
createStoredRunCodecV1,
|
|
17
21
|
storedRunLaneV1,
|
|
@@ -108,7 +112,23 @@ function deferred<T>(): Deferred<T> {
|
|
|
108
112
|
*/
|
|
109
113
|
function createAuthority(
|
|
110
114
|
storage: MemoryStorage,
|
|
111
|
-
options: {
|
|
115
|
+
options: {
|
|
116
|
+
dispatch?(runId: string): boolean;
|
|
117
|
+
/**
|
|
118
|
+
* Ends the named Turn the way the Agent loop ends one whose model stream
|
|
119
|
+
* was aborted mid-flight: a journaled `model/request` with no durable
|
|
120
|
+
* provider outcome, and a reconciliation demand.
|
|
121
|
+
*/
|
|
122
|
+
uncertain?(runId: string): boolean;
|
|
123
|
+
/**
|
|
124
|
+
* Parks the named Turn when it is released: the provider call it had
|
|
125
|
+
* dispatched by then has no durable outcome, and nothing but an explicit
|
|
126
|
+
* reconciliation can settle it.
|
|
127
|
+
*/
|
|
128
|
+
parkOnRelease?(runId: string): boolean;
|
|
129
|
+
/** Fails the recovery of an evicted Turn, leaving it active and owed. */
|
|
130
|
+
failRecovery?(runId: string): boolean;
|
|
131
|
+
} = {},
|
|
112
132
|
): Probe {
|
|
113
133
|
const observed: BotTurnExecutionInput<undefined>[] = [];
|
|
114
134
|
const interrupts: { runId: string; reason: string }[] = [];
|
|
@@ -138,6 +158,9 @@ function createAuthority(
|
|
|
138
158
|
executeTurn: async (input) => {
|
|
139
159
|
observed.push(input);
|
|
140
160
|
const runId = input.command.runId;
|
|
161
|
+
if (input.resume && options.failRecovery?.(runId)) {
|
|
162
|
+
throw new BotTurnRecoveryRequiredError([]);
|
|
163
|
+
}
|
|
141
164
|
const turn = observed.length;
|
|
142
165
|
const handle = handleFor(runId);
|
|
143
166
|
let seq = input.previousEvents.length;
|
|
@@ -166,7 +189,22 @@ function createAuthority(
|
|
|
166
189
|
text: input.command.text,
|
|
167
190
|
} as never,
|
|
168
191
|
);
|
|
169
|
-
|
|
192
|
+
const uncertain = options.uncertain?.(runId) ?? false;
|
|
193
|
+
if (uncertain) {
|
|
194
|
+
await persist({
|
|
195
|
+
type: "model/request",
|
|
196
|
+
turn,
|
|
197
|
+
step: 1,
|
|
198
|
+
request: {
|
|
199
|
+
requestId: `request-${runId}`,
|
|
200
|
+
provider: "foundation",
|
|
201
|
+
model: "foundation-model",
|
|
202
|
+
system: "system",
|
|
203
|
+
messages: [{ role: "user", content: input.command.text }],
|
|
204
|
+
tools: [],
|
|
205
|
+
},
|
|
206
|
+
} as never);
|
|
207
|
+
} else if (options.dispatch?.(runId) ?? true) {
|
|
170
208
|
await persist(
|
|
171
209
|
{
|
|
172
210
|
type: "model/request",
|
|
@@ -193,6 +231,43 @@ function createAuthority(
|
|
|
193
231
|
}
|
|
194
232
|
handle.started.resolve();
|
|
195
233
|
const outcome = await handle.settled.promise;
|
|
234
|
+
if (outcome.interrupted === undefined && options.parkOnRelease?.(runId)) {
|
|
235
|
+
const reason = `Model request "request-${runId}" has no durable provider outcome`;
|
|
236
|
+
await persist(
|
|
237
|
+
{
|
|
238
|
+
type: "model/request",
|
|
239
|
+
turn,
|
|
240
|
+
step: 1,
|
|
241
|
+
request: {
|
|
242
|
+
requestId: `request-${runId}`,
|
|
243
|
+
provider: "foundation",
|
|
244
|
+
model: "foundation-model",
|
|
245
|
+
system: "system",
|
|
246
|
+
messages: [{ role: "user", content: input.command.text }],
|
|
247
|
+
tools: [],
|
|
248
|
+
},
|
|
249
|
+
} as never,
|
|
250
|
+
{
|
|
251
|
+
type: "model/reconciliation-required",
|
|
252
|
+
turn,
|
|
253
|
+
step: 1,
|
|
254
|
+
requestId: `request-${runId}`,
|
|
255
|
+
reason,
|
|
256
|
+
} as never,
|
|
257
|
+
);
|
|
258
|
+
throw new BotTurnReconciliationRequiredError(reason, appended);
|
|
259
|
+
}
|
|
260
|
+
if (outcome.interrupted !== undefined && uncertain) {
|
|
261
|
+
const reason = `Model response outcome is uncertain after cancellation: ${outcome.interrupted}`;
|
|
262
|
+
await persist({
|
|
263
|
+
type: "model/reconciliation-required",
|
|
264
|
+
turn,
|
|
265
|
+
step: 1,
|
|
266
|
+
requestId: `request-${runId}`,
|
|
267
|
+
reason,
|
|
268
|
+
} as never);
|
|
269
|
+
throw new BotTurnReconciliationRequiredError(reason, appended);
|
|
270
|
+
}
|
|
196
271
|
if (outcome.interrupted !== undefined) {
|
|
197
272
|
await persist({
|
|
198
273
|
type: "turn/end",
|
|
@@ -576,3 +651,179 @@ describe("eviction between the two Turns", () => {
|
|
|
576
651
|
expect(restarted.observed).toHaveLength(1);
|
|
577
652
|
});
|
|
578
653
|
});
|
|
654
|
+
|
|
655
|
+
describe("a durable log left inside a Turn", () => {
|
|
656
|
+
test("is repaired at admission instead of refusing every later Turn", async () => {
|
|
657
|
+
const storage = new MemoryStorage();
|
|
658
|
+
// Exactly what a Turn that threw between `turn/start` and `turn/end`
|
|
659
|
+
// leaves behind: an open Turn, and no run to close it.
|
|
660
|
+
storage.values.set("latest-events", [
|
|
661
|
+
{
|
|
662
|
+
type: "session/created",
|
|
663
|
+
createdAt: "2026-09-03T00:00:00.000Z",
|
|
664
|
+
seq: 0,
|
|
665
|
+
timestamp: "2026-09-03T00:00:00.000Z",
|
|
666
|
+
},
|
|
667
|
+
{
|
|
668
|
+
type: "turn/start",
|
|
669
|
+
turn: 1,
|
|
670
|
+
seq: 1,
|
|
671
|
+
timestamp: "2026-09-03T00:00:01.000Z",
|
|
672
|
+
},
|
|
673
|
+
]);
|
|
674
|
+
const probe = createAuthority(storage);
|
|
675
|
+
|
|
676
|
+
const run = probe.authority.run(command("run-1", "hello"));
|
|
677
|
+
await probe.handle("run-1").started;
|
|
678
|
+
probe.handle("run-1").finish();
|
|
679
|
+
await run;
|
|
680
|
+
|
|
681
|
+
const events = storage.values.get("latest-events") as Array<{
|
|
682
|
+
type: string;
|
|
683
|
+
turn?: number;
|
|
684
|
+
}>;
|
|
685
|
+
// The orphaned Turn is closed, so the new one starts.
|
|
686
|
+
expect(events[2]).toMatchObject({
|
|
687
|
+
type: "turn/end",
|
|
688
|
+
turn: 1,
|
|
689
|
+
outcome: "interrupted",
|
|
690
|
+
});
|
|
691
|
+
expect(storedRun(storage, "run-1").status).toBe("completed");
|
|
692
|
+
});
|
|
693
|
+
});
|
|
694
|
+
|
|
695
|
+
describe("a failing recovery of an older Turn", () => {
|
|
696
|
+
test("does not swallow the message the User just sent", async () => {
|
|
697
|
+
const storage = new MemoryStorage();
|
|
698
|
+
const probe = createAuthority(storage);
|
|
699
|
+
const first = probe.authority.run(command("run-1", "first"));
|
|
700
|
+
await probe.handle("run-1").started;
|
|
701
|
+
// Evicted mid-Turn: run-1 stays active and durable, and the object that
|
|
702
|
+
// comes back recovers it — badly.
|
|
703
|
+
const restarted = createAuthority(storage, {
|
|
704
|
+
failRecovery: (runId) => runId === "run-1",
|
|
705
|
+
});
|
|
706
|
+
const second = restarted.authority
|
|
707
|
+
.run(
|
|
708
|
+
command("run-2", "second", {
|
|
709
|
+
lane: "user",
|
|
710
|
+
supersedes: { runId: "run-1" },
|
|
711
|
+
}),
|
|
712
|
+
)
|
|
713
|
+
.catch(() => undefined);
|
|
714
|
+
await admitted();
|
|
715
|
+
|
|
716
|
+
// The new message is durable regardless of what happened to the old Turn.
|
|
717
|
+
// Before this, the recovery's own error threw out of `run()` before
|
|
718
|
+
// admission was ever attempted and the message was simply gone.
|
|
719
|
+
expect(storedRun(storage, "run-2").runId).toBe("run-2");
|
|
720
|
+
expect(storedRun(storage, "run-2").input).toBe("second");
|
|
721
|
+
probe.handle("run-1").finish();
|
|
722
|
+
await first.catch(() => undefined);
|
|
723
|
+
restarted.handle("run-2").finish();
|
|
724
|
+
await second;
|
|
725
|
+
});
|
|
726
|
+
});
|
|
727
|
+
|
|
728
|
+
describe("the run admission fence index", () => {
|
|
729
|
+
test("ages the oldest entry out rather than refusing the operation", async () => {
|
|
730
|
+
const storage = new MemoryStorage();
|
|
731
|
+
const probe = createAuthority(storage);
|
|
732
|
+
for (let index = 0; index < 300; index += 1) {
|
|
733
|
+
await probe.authority.fenceRunAdmission(identity, `fence-${index}`);
|
|
734
|
+
}
|
|
735
|
+
const fences = storage.values.get("run-admission-fences") as string[];
|
|
736
|
+
expect(fences.length).toBeLessThanOrEqual(256);
|
|
737
|
+
// The newest fence is the one that still matters; the oldest aged out.
|
|
738
|
+
expect(fences.at(-1)).toBe("fence-299");
|
|
739
|
+
expect(fences).not.toContain("fence-0");
|
|
740
|
+
});
|
|
741
|
+
});
|
|
742
|
+
|
|
743
|
+
describe("a Turn queued behind a parked run", () => {
|
|
744
|
+
test("is refused rather than answered with an empty completion", async () => {
|
|
745
|
+
const storage = new MemoryStorage();
|
|
746
|
+
// The first Turn has not dispatched when the second arrives, so it is left
|
|
747
|
+
// to finish and the second queues behind it. It then parks on a provider
|
|
748
|
+
// outcome only a User can retrieve.
|
|
749
|
+
const probe = createAuthority(storage, {
|
|
750
|
+
dispatch: () => false,
|
|
751
|
+
parkOnRelease: (runId) => runId === "run-1",
|
|
752
|
+
});
|
|
753
|
+
|
|
754
|
+
const first = probe.authority.run(command("run-1", "first"));
|
|
755
|
+
await probe.handle("run-1").started;
|
|
756
|
+
const second = probe.authority.run(
|
|
757
|
+
command("run-2", "second", {
|
|
758
|
+
lane: "user",
|
|
759
|
+
supersedes: { runId: "run-1" },
|
|
760
|
+
}),
|
|
761
|
+
);
|
|
762
|
+
await admitted();
|
|
763
|
+
probe.handle("run-1").finish();
|
|
764
|
+
await first.catch(() => undefined);
|
|
765
|
+
|
|
766
|
+
await expect(second).rejects.toThrow(
|
|
767
|
+
/is queued: the active run requires reconciliation/,
|
|
768
|
+
);
|
|
769
|
+
// And it is still owed a Turn: durable, queued, and started by the
|
|
770
|
+
// reconciliation's own settlement or by the recovery alarm.
|
|
771
|
+
const queued = storedRun(storage, "run-2");
|
|
772
|
+
expect(queued.status).toBe("running");
|
|
773
|
+
expect(queued.phase).toBe("queued");
|
|
774
|
+
expect(storage.values.get("pending-run")).toBe("run-2");
|
|
775
|
+
});
|
|
776
|
+
});
|
|
777
|
+
|
|
778
|
+
describe("an interrupt while the model is streaming", () => {
|
|
779
|
+
test("a superseded Turn settles superseded rather than parking the Bot", async () => {
|
|
780
|
+
const storage = new MemoryStorage();
|
|
781
|
+
const probe = createAuthority(storage, { uncertain: () => true });
|
|
782
|
+
|
|
783
|
+
const first = probe.authority.run(command("run-1", "first"));
|
|
784
|
+
await probe.handle("run-1").started;
|
|
785
|
+
const second = probe.authority.run(
|
|
786
|
+
command("run-2", "second", {
|
|
787
|
+
lane: "user",
|
|
788
|
+
supersedes: { runId: "run-1" },
|
|
789
|
+
}),
|
|
790
|
+
);
|
|
791
|
+
await first.catch(() => undefined);
|
|
792
|
+
probe.handle("run-2").finish();
|
|
793
|
+
const result = await second;
|
|
794
|
+
|
|
795
|
+
// The provider outcome of a Turn nobody is waiting for is worthless: the
|
|
796
|
+
// intent the User expressed is what settles it.
|
|
797
|
+
const superseded = storedRun(storage, "run-1");
|
|
798
|
+
expect(superseded.status).toBe("superseded");
|
|
799
|
+
expect(superseded.supersededBy).toBe("run-2");
|
|
800
|
+
expect(storage.values.get("active-run")).toBeUndefined();
|
|
801
|
+
expect(result.text).toBe("done: second");
|
|
802
|
+
});
|
|
803
|
+
|
|
804
|
+
test("a stopped Turn settles cancelled and the next message is admitted", async () => {
|
|
805
|
+
const storage = new MemoryStorage();
|
|
806
|
+
const probe = createAuthority(storage, { uncertain: () => true });
|
|
807
|
+
|
|
808
|
+
const first = probe.authority.run(command("run-1", "first"));
|
|
809
|
+
await probe.handle("run-1").started;
|
|
810
|
+
// What an authenticated Stop writes before it signals the Agent.
|
|
811
|
+
const stopped = storedRun(storage, "run-1");
|
|
812
|
+
storage.values.set("run:run-1", {
|
|
813
|
+
...stopped,
|
|
814
|
+
stopRequestedAt: "2026-09-03T00:00:05.000Z",
|
|
815
|
+
});
|
|
816
|
+
probe.handle("run-1").interrupt("stopped by an authenticated Stop command");
|
|
817
|
+
await first.catch(() => undefined);
|
|
818
|
+
|
|
819
|
+
expect(storedRun(storage, "run-1").status).toBe("cancelled");
|
|
820
|
+
expect(storage.values.get("active-run")).toBeUndefined();
|
|
821
|
+
|
|
822
|
+
// And the Bot takes the next message straight away, with no
|
|
823
|
+
// reconciliation standing between the User and their Bot.
|
|
824
|
+
const next = probe.authority.run(command("run-2", "second"));
|
|
825
|
+
await probe.handle("run-2").started;
|
|
826
|
+
probe.handle("run-2").finish();
|
|
827
|
+
expect((await next).text).toBe("done: second");
|
|
828
|
+
});
|
|
829
|
+
});
|