@frockbot/kernel-do 0.3.4 → 0.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@frockbot/kernel-do",
3
- "version": "0.3.4",
3
+ "version": "0.3.5",
4
4
  "private": false,
5
5
  "type": "module",
6
6
  "exports": {
@@ -12,8 +12,8 @@
12
12
  "typecheck": "tsc --noEmit -p tsconfig.json"
13
13
  },
14
14
  "dependencies": {
15
- "@frockbot/kernel-composition": "0.3.4",
16
- "@frockbot/kernel-contracts": "0.3.4",
15
+ "@frockbot/kernel-composition": "0.3.5",
16
+ "@frockbot/kernel-contracts": "0.3.5",
17
17
  "cordis": "4.0.0-rc.8"
18
18
  },
19
19
  "devDependencies": {
package/src/authority.ts CHANGED
@@ -34,11 +34,13 @@ import {
34
34
  eventsForFailedRun,
35
35
  latestModelRequestJournalState,
36
36
  planBotRunRecovery,
37
+ repairOrphanedOpenTurnV1,
37
38
  unresolvedModelRequestFailure,
38
39
  } from "./run-recovery.js";
39
40
  import {
40
41
  BotTurnReconciliationRequiredError,
41
42
  BotTurnRecoveryRequiredError,
43
+ BotTurnRefusedError,
42
44
  } from "./turn-errors.js";
43
45
  import {
44
46
  ACTIVE_RUN_KEY,
@@ -150,6 +152,22 @@ export const SUPERSEDED_TURN_REASON_V1 = "superseded by a new user message";
150
152
  /** How many times a queued Turn retries the object before giving up. */
151
153
  const MAX_QUEUED_RUN_START_ATTEMPTS = 8;
152
154
 
155
+ /**
156
+ * True when this object has already durably decided to throw the Turn away.
157
+ *
158
+ * Reconciliation exists to retrieve an external outcome the Turn still needs.
159
+ * A Turn a Stop or a supersede has already discarded needs nothing: its
160
+ * provider outcome cannot change what it settles as, and parking it would keep
161
+ * the active-run marker — and so refuse every later message — over an answer
162
+ * nobody is waiting for. The intent the User expressed wins, and the run
163
+ * settles `cancelled` or `superseded` with everything it had already said.
164
+ */
165
+ function runWasDiscardedV1(
166
+ run: { stopRequestedAt?: string; supersededAt?: string } | undefined,
167
+ ): boolean {
168
+ return Boolean(run?.stopRequestedAt || run?.supersededAt);
169
+ }
170
+
153
171
  export interface BotDurableAuthorityOptions<Snapshot> {
154
172
  state: DurableObjectState;
155
173
  codec: StoredRunCodecV1<Snapshot>;
@@ -193,7 +211,14 @@ export class BotDurableAuthority<Snapshot> {
193
211
 
194
212
  async run(command: OwnedBotTurnCommand): Promise<BotTurnCompletion> {
195
213
  await this.assertMatchingRunCommand(command);
196
- await this.recoverActiveRun();
214
+ // Recovering whatever this object was left holding must never decide the
215
+ // fate of a new command. `recoverActiveRun` executes the *previous* Turn
216
+ // inline and rethrows, so a recovery that failed — an uncertain effect, a
217
+ // mount failure, a provider that was down — threw before the new message
218
+ // was ever admitted, and the person's message was simply lost. The old
219
+ // Turn is durable either way and the alarm retries it; admission now
220
+ // refuses or supersedes on its own terms.
221
+ await this.recoverActiveRun().catch(() => undefined);
197
222
  const replay = await this.settledRunResult(command);
198
223
  if (replay) return replay;
199
224
  const admission = await this.acceptRun(command);
@@ -240,8 +265,10 @@ export class BotDurableAuthority<Snapshot> {
240
265
  const promoted = await this.promoteQueuedRun(command.runId);
241
266
  if (promoted === "blocked") {
242
267
  // Another Turn holds the object. Recovery drives it to its own
243
- // durable terminal or resumable state, and this one tries again.
244
- await this.recoverActiveRun();
268
+ // durable terminal or resumable state, and this one tries again
269
+ // including when that recovery fails, which is the other Turn's
270
+ // problem and not this one's.
271
+ await this.recoverActiveRun().catch(() => undefined);
245
272
  // Unless what holds the object is an uncertain effect. That is
246
273
  // settled by an explicit reconciliation the User asks for, on their
247
274
  // own clock, and retrying against it would only burn this caller's
@@ -249,11 +276,14 @@ export class BotDurableAuthority<Snapshot> {
249
276
  // run is durable: it stays queued, and the reconciliation's own
250
277
  // settlement — or the recovery alarm — starts it.
251
278
  if (await this.activeRunAwaitsReconciliation()) {
252
- return {
253
- runId: command.runId,
254
- text: "",
255
- events: [],
256
- } satisfies BotTurnCompletion;
279
+ // A Turn that has not run is not a completed Turn. Answering with
280
+ // an empty completion made the browser render the person's new
281
+ // message as answered with silence; the durable queue entry stays,
282
+ // and the refusal says why nothing has happened yet.
283
+ throw new BotTurnRefusedError(
284
+ "reconciliation-required",
285
+ `run "${command.runId}" is queued: the active run requires reconciliation before another Turn can be admitted`,
286
+ );
257
287
  }
258
288
  continue;
259
289
  }
@@ -369,12 +399,22 @@ export class BotDurableAuthority<Snapshot> {
369
399
  (await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
370
400
  ).map(decodeSessionEvent);
371
401
  const settings = run.configurationSnapshot;
372
- await transaction.put(key, {
373
- ...run,
374
- status: "running",
375
- phase: "executing",
376
- failure: undefined,
377
- } satisfies StoredRunV1<Snapshot>);
402
+ // The failure is *removed*, not set to `undefined`: a running run that
403
+ // carries a `failure` key is a shape the run record does not allow, and
404
+ // writing one turned "resolve this Turn" into a record nothing could
405
+ // read afterwards. `require` checks it here, where the write is, rather
406
+ // than leaving the projector to fail on every later read.
407
+ const { failure: _failure, ...resumed } = run;
408
+ await transaction.put(
409
+ key,
410
+ structuredClone(
411
+ this.codec.require({
412
+ ...resumed,
413
+ status: "running",
414
+ phase: "executing",
415
+ } satisfies StoredRunV1<Snapshot>),
416
+ ),
417
+ );
378
418
  await this.refreshRecoveryAlarm(transaction);
379
419
  return { run, latest, settings };
380
420
  });
@@ -480,8 +520,9 @@ export class BotDurableAuthority<Snapshot> {
480
520
  }
481
521
  const modelState = latestModelRequestJournalState(events);
482
522
  if (
483
- error instanceof BotTurnReconciliationRequiredError ||
484
- modelState.status === "unresolved"
523
+ (error instanceof BotTurnReconciliationRequiredError ||
524
+ modelState.status === "unresolved") &&
525
+ !runWasDiscardedV1(durableRun)
485
526
  ) {
486
527
  await this.requireRunReconciliation(
487
528
  command.runId,
@@ -612,8 +653,9 @@ export class BotDurableAuthority<Snapshot> {
612
653
  }
613
654
  const modelState = latestModelRequestJournalState(events);
614
655
  if (
615
- error instanceof BotTurnReconciliationRequiredError ||
616
- modelState.status === "unresolved"
656
+ (error instanceof BotTurnReconciliationRequiredError ||
657
+ modelState.status === "unresolved") &&
658
+ !runWasDiscardedV1(durableRun)
617
659
  ) {
618
660
  await this.requireRunReconciliation(
619
661
  run.runId,
@@ -659,7 +701,8 @@ export class BotDurableAuthority<Snapshot> {
659
701
  );
660
702
  if (!run) return undefined;
661
703
  if (run.commandFingerprint !== botTurnCommandFingerprintV1(command)) {
662
- throw new Error(
704
+ throw new BotTurnRefusedError(
705
+ "duplicate",
663
706
  `Turn idempotency key "${runId}" was reused for a different command`,
664
707
  );
665
708
  }
@@ -674,7 +717,8 @@ export class BotDurableAuthority<Snapshot> {
674
717
  };
675
718
  }
676
719
  if (run.status !== "completed") {
677
- throw new Error(
720
+ throw new BotTurnRefusedError(
721
+ "duplicate",
678
722
  `run "${runId}" already exists with status ${run.status}`,
679
723
  );
680
724
  }
@@ -702,7 +746,8 @@ export class BotDurableAuthority<Snapshot> {
702
746
  run &&
703
747
  run.commandFingerprint !== botTurnCommandFingerprintV1(command)
704
748
  ) {
705
- throw new Error(
749
+ throw new BotTurnRefusedError(
750
+ "duplicate",
706
751
  `Turn idempotency key "${command.runId}" was reused for a different command`,
707
752
  );
708
753
  }
@@ -788,7 +833,16 @@ export class BotDurableAuthority<Snapshot> {
788
833
  this.ctx.storage.get<BotIdentity>(IDENTITY_KEY),
789
834
  ]);
790
835
  const run = this.codec.optional(storedRun);
791
- if (run?.status === "reconciliation-required" && identity) return;
836
+ if (run?.status === "reconciliation-required" && identity) {
837
+ // A parked run is not this alarm's to settle — only an explicit
838
+ // reconciliation settles it — but returning without rescheduling
839
+ // dropped the object's *other* deadlines with it: a Routine due while
840
+ // a Bot sat parked never fired, and nothing set the alarm again.
841
+ await this.ctx.storage.transaction((transaction) =>
842
+ this.refreshRecoveryAlarm(transaction),
843
+ );
844
+ return;
845
+ }
792
846
  }
793
847
  await this.recoverActiveRun();
794
848
  }
@@ -855,17 +909,15 @@ export class BotDurableAuthority<Snapshot> {
855
909
  const storedFences = storedRunAdmissionFences(
856
910
  await transaction.get<unknown>(RUN_ADMISSION_FENCE_INDEX_KEY),
857
911
  );
858
- if (
859
- !storedFences.includes(runId) &&
860
- storedFences.length >= MAX_RUN_ADMISSION_FENCES
861
- ) {
862
- throw new Error("Run admission fence capacity reached");
863
- }
912
+ // A bounded FIFO, not a cliff. Nothing ever evicted an entry, so a Bot
913
+ // that had refused 256 sends over its life answered every later fence
914
+ // with a 500 and left the client retrying "Turn admission lookup
915
+ // failed" forever. A run id old enough to age out here can no longer
916
+ // be admitted by any live caller.
917
+ const kept = storedFences.filter((fenced) => fenced !== runId);
918
+ while (kept.length >= MAX_RUN_ADMISSION_FENCES) kept.shift();
864
919
  await transaction.put({
865
- [RUN_ADMISSION_FENCE_INDEX_KEY]: [
866
- ...storedFences.filter((fenced) => fenced !== runId),
867
- runId,
868
- ],
920
+ [RUN_ADMISSION_FENCE_INDEX_KEY]: [...kept, runId],
869
921
  [IDENTITY_KEY]: durableIdentity ?? identity,
870
922
  });
871
923
  await transaction.delete(`${RUN_ADMISSION_FENCE_PREFIX}${runId}`);
@@ -932,7 +984,10 @@ export class BotDurableAuthority<Snapshot> {
932
984
  fences.includes(command.runId) ||
933
985
  (await this.ctx.storage.get(fenceKey))
934
986
  ) {
935
- throw new Error(`run "${command.runId}" admission was fenced`);
987
+ throw new BotTurnRefusedError(
988
+ "fenced",
989
+ `run "${command.runId}" admission was fenced`,
990
+ );
936
991
  }
937
992
  const settings = await this.hooks.resolveAdmissionSnapshot(command);
938
993
  // Materialized before the transaction; the pin itself is read inside it.
@@ -944,20 +999,30 @@ export class BotDurableAuthority<Snapshot> {
944
999
  if (
945
1000
  existing.commandFingerprint !== botTurnCommandFingerprintV1(command)
946
1001
  ) {
947
- throw new Error(
1002
+ throw new BotTurnRefusedError(
1003
+ "duplicate",
948
1004
  `Turn idempotency key "${command.runId}" was reused for a different command`,
949
1005
  );
950
1006
  }
951
1007
  if (existing.status === "completed") {
952
- throw new Error(`run "${command.runId}" already completed`);
1008
+ throw new BotTurnRefusedError(
1009
+ "duplicate",
1010
+ `run "${command.runId}" already completed`,
1011
+ );
953
1012
  }
954
- throw new Error(`run "${command.runId}" already exists`);
1013
+ throw new BotTurnRefusedError(
1014
+ "duplicate",
1015
+ `run "${command.runId}" already exists`,
1016
+ );
955
1017
  }
956
1018
  const fences = storedRunAdmissionFences(
957
1019
  await transaction.get<unknown>(RUN_ADMISSION_FENCE_INDEX_KEY),
958
1020
  );
959
1021
  if (fences.includes(command.runId) || (await transaction.get(fenceKey))) {
960
- throw new Error(`run "${command.runId}" admission was fenced`);
1022
+ throw new BotTurnRefusedError(
1023
+ "fenced",
1024
+ `run "${command.runId}" admission was fenced`,
1025
+ );
961
1026
  }
962
1027
  const identity = await transaction.get<BotIdentity>(IDENTITY_KEY);
963
1028
  if (
@@ -970,9 +1035,25 @@ export class BotDurableAuthority<Snapshot> {
970
1035
  const supersede = activeRunId
971
1036
  ? await this.planSupersede(transaction, command, activeRunId)
972
1037
  : undefined;
973
- const latestEvents = (
1038
+ const storedEvents = (
974
1039
  (await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
975
1040
  ).map(decodeSessionEvent);
1041
+ // A Turn that died between `turn/start` and `turn/end` — an event the
1042
+ // encoder refused, a durable write that failed — left the log open, and
1043
+ // every later Turn failed validation with "turn N started while turn
1044
+ // N-1 is open". Nothing owned that repair, because the run that would
1045
+ // have closed it is already terminal, so admission does: with nothing
1046
+ // executing, an open Turn is one nobody is going to finish.
1047
+ const repairs = activeRunId
1048
+ ? []
1049
+ : repairOrphanedOpenTurnV1(command.sessionId, storedEvents);
1050
+ const latestEvents = [...storedEvents, ...repairs];
1051
+ if (repairs.length > 0) {
1052
+ await transaction.put(
1053
+ LATEST_EVENTS_KEY,
1054
+ structuredClone(latestEvents.map(decodeSessionEvent)),
1055
+ );
1056
+ }
976
1057
  const admittedSettings = await this.hooks.admittedSnapshot(
977
1058
  transaction,
978
1059
  settings,
@@ -1060,21 +1141,23 @@ export class BotDurableAuthority<Snapshot> {
1060
1141
  // no run when the person pressed send — supersedes exactly as a named one
1061
1142
  // does; only an absent field is "no intent", and that is still refused.
1062
1143
  if (lane !== "user" || !command.supersedes) {
1063
- throw new Error("bot already has an active run");
1144
+ throw new BotTurnRefusedError("busy", "bot already has an active run");
1064
1145
  }
1065
1146
  const active = this.codec.optional(
1066
1147
  await transaction.get<unknown>(`${RUN_PREFIX}${activeRunId}`),
1067
1148
  );
1068
- if (!active) throw new Error("bot already has an active run");
1149
+ if (!active)
1150
+ throw new BotTurnRefusedError("busy", "bot already has an active run");
1069
1151
  if (active.status === "reconciliation-required") {
1070
1152
  // An uncertain external effect is never abandoned to admit something
1071
1153
  // else: the outcome has to be retrieved before this object runs again.
1072
- throw new Error(
1154
+ throw new BotTurnRefusedError(
1155
+ "reconciliation-required",
1073
1156
  `run "${activeRunId}" requires reconciliation before another Turn can be admitted`,
1074
1157
  );
1075
1158
  }
1076
1159
  if (active.status !== "running") {
1077
- throw new Error("bot already has an active run");
1160
+ throw new BotTurnRefusedError("busy", "bot already has an active run");
1078
1161
  }
1079
1162
  const pendingRunId = await transaction.get<string>(PENDING_RUN_KEY);
1080
1163
  // A Turn that has not dispatched a model request has no durable work to
@@ -1400,6 +1483,22 @@ export class BotDurableAuthority<Snapshot> {
1400
1483
  await this.refreshRecoveryAlarm(transaction);
1401
1484
  return { kind: "resume" as const, run, latest, settings };
1402
1485
  }
1486
+ if (runWasDiscardedV1(run)) {
1487
+ // Recovery of a Turn Stop or supersede already discarded settles it on
1488
+ // that intent rather than parking it: nothing is owed the answer.
1489
+ await failStoredRun(
1490
+ this.codec,
1491
+ transaction,
1492
+ this.terminalKeys(run.runId),
1493
+ run.runId,
1494
+ latest.slice(0, run.previousEventCount),
1495
+ [...run.events, ...plan.repairs],
1496
+ "Execution outcome requires reconciliation before it can resume",
1497
+ this.supersededPackageRecords(),
1498
+ );
1499
+ await this.refreshRecoveryAlarm(transaction);
1500
+ return undefined;
1501
+ }
1403
1502
  await transaction.put({
1404
1503
  [key]: {
1405
1504
  ...run,
@@ -182,6 +182,46 @@ export function planBotRunRecovery<Snapshot>(
182
182
  return { kind: "reconcile", repairs: session.reconcileForResume() };
183
183
  }
184
184
 
185
+ /** True when the durable log ends inside a Turn nothing is going to finish. */
186
+ export function hasOrphanedOpenTurnV1(
187
+ events: readonly SessionEvent[],
188
+ ): boolean {
189
+ let openTurn: number | undefined;
190
+ for (const event of events) {
191
+ if (event.type === "turn/start") openTurn = event.turn;
192
+ if (event.type === "turn/end" && event.turn === openTurn) {
193
+ openTurn = undefined;
194
+ }
195
+ }
196
+ return openTurn !== undefined;
197
+ }
198
+
199
+ /**
200
+ * Closes a Turn the log was left inside, so the next one can start.
201
+ *
202
+ * A Turn that threw between `turn/start` and `turn/end` — an event the
203
+ * encoder refused, a durable write that failed — leaves an open turn in the
204
+ * durable log, and the next Turn on that Bot fails validation with "turn N
205
+ * started while turn N-1 is open". Forever: nothing owned the repair, because
206
+ * the run that would have written the `turn/end` is already terminal. This is
207
+ * that repair, applied when no run is executing, so an interrupted Turn is
208
+ * recorded as interrupted rather than wedging the Bot.
209
+ *
210
+ * A log too malformed to reconcile is left exactly as it is: repairing it
211
+ * blindly would invent history.
212
+ */
213
+ export function repairOrphanedOpenTurnV1(
214
+ sessionId: string,
215
+ latest: readonly SessionEvent[],
216
+ ): SessionEvent[] {
217
+ if (!hasOrphanedOpenTurnV1(latest)) return [];
218
+ try {
219
+ return new Session(sessionId, () => {}, latest).reconcileInterrupted();
220
+ } catch {
221
+ return [];
222
+ }
223
+ }
224
+
185
225
  export function eventsForFailedRun(
186
226
  durableRun: { events: SessionEvent[] } | undefined,
187
227
  error: unknown,
@@ -25,6 +25,50 @@ export class BotTurnReconciliationRequiredError extends Error {
25
25
  }
26
26
  }
27
27
 
28
+ /** Why the Bot declined to admit a Turn. */
29
+ export type BotTurnRefusalCodeV1 =
30
+ "busy" | "reconciliation-required" | "fenced" | "duplicate";
31
+
32
+ const BOT_TURN_REFUSAL_PREFIX_V1 = "BotTurnRefusedError:";
33
+
34
+ /**
35
+ * An admission the Bot declined, as a typed error rather than a sentence.
36
+ *
37
+ * A refusal crosses a Durable Object RPC boundary, which preserves an error's
38
+ * `name` and `message` and drops everything else — so the code rides on the
39
+ * name, the way the gateway already relies on `name` for `BotNotFoundError`.
40
+ * Classifying these by matching prose against `error.message` meant any
41
+ * reword silently turned an ordinary 409 into a 500, and two real messages
42
+ * already fell through.
43
+ */
44
+ export class BotTurnRefusedError extends Error {
45
+ constructor(
46
+ readonly code: BotTurnRefusalCodeV1,
47
+ message: string,
48
+ ) {
49
+ super(message);
50
+ this.name = `${BOT_TURN_REFUSAL_PREFIX_V1}${code}`;
51
+ }
52
+ }
53
+
54
+ /** The refusal an error carries, or `undefined` when it is not one. */
55
+ export function botTurnRefusalCodeV1(
56
+ error: unknown,
57
+ ): BotTurnRefusalCodeV1 | undefined {
58
+ const name =
59
+ typeof error === "object" && error !== null && "name" in error
60
+ ? String((error as { name: unknown }).name)
61
+ : "";
62
+ if (!name.startsWith(BOT_TURN_REFUSAL_PREFIX_V1)) return undefined;
63
+ const code = name.slice(BOT_TURN_REFUSAL_PREFIX_V1.length);
64
+ return code === "busy" ||
65
+ code === "reconciliation-required" ||
66
+ code === "fenced" ||
67
+ code === "duplicate"
68
+ ? code
69
+ : undefined;
70
+ }
71
+
28
72
  export class BotTurnRecoveryRequiredError extends Error {
29
73
  constructor(readonly events: SessionEvent[]) {
30
74
  super("Bot turn has a durable outcome settlement pending");
@@ -12,6 +12,10 @@ import {
12
12
  type OwnedBotTurnCommand,
13
13
  } from "./authority.ts";
14
14
  import { MemoryStorage } from "./memory-storage.fixture.ts";
15
+ import {
16
+ BotTurnReconciliationRequiredError,
17
+ BotTurnRecoveryRequiredError,
18
+ } from "./turn-errors.ts";
15
19
  import {
16
20
  createStoredRunCodecV1,
17
21
  storedRunLaneV1,
@@ -108,7 +112,23 @@ function deferred<T>(): Deferred<T> {
108
112
  */
109
113
  function createAuthority(
110
114
  storage: MemoryStorage,
111
- options: { dispatch?(runId: string): boolean } = {},
115
+ options: {
116
+ dispatch?(runId: string): boolean;
117
+ /**
118
+ * Ends the named Turn the way the Agent loop ends one whose model stream
119
+ * was aborted mid-flight: a journaled `model/request` with no durable
120
+ * provider outcome, and a reconciliation demand.
121
+ */
122
+ uncertain?(runId: string): boolean;
123
+ /**
124
+ * Parks the named Turn when it is released: the provider call it had
125
+ * dispatched by then has no durable outcome, and nothing but an explicit
126
+ * reconciliation can settle it.
127
+ */
128
+ parkOnRelease?(runId: string): boolean;
129
+ /** Fails the recovery of an evicted Turn, leaving it active and owed. */
130
+ failRecovery?(runId: string): boolean;
131
+ } = {},
112
132
  ): Probe {
113
133
  const observed: BotTurnExecutionInput<undefined>[] = [];
114
134
  const interrupts: { runId: string; reason: string }[] = [];
@@ -138,6 +158,9 @@ function createAuthority(
138
158
  executeTurn: async (input) => {
139
159
  observed.push(input);
140
160
  const runId = input.command.runId;
161
+ if (input.resume && options.failRecovery?.(runId)) {
162
+ throw new BotTurnRecoveryRequiredError([]);
163
+ }
141
164
  const turn = observed.length;
142
165
  const handle = handleFor(runId);
143
166
  let seq = input.previousEvents.length;
@@ -166,7 +189,22 @@ function createAuthority(
166
189
  text: input.command.text,
167
190
  } as never,
168
191
  );
169
- if (options.dispatch?.(runId) ?? true) {
192
+ const uncertain = options.uncertain?.(runId) ?? false;
193
+ if (uncertain) {
194
+ await persist({
195
+ type: "model/request",
196
+ turn,
197
+ step: 1,
198
+ request: {
199
+ requestId: `request-${runId}`,
200
+ provider: "foundation",
201
+ model: "foundation-model",
202
+ system: "system",
203
+ messages: [{ role: "user", content: input.command.text }],
204
+ tools: [],
205
+ },
206
+ } as never);
207
+ } else if (options.dispatch?.(runId) ?? true) {
170
208
  await persist(
171
209
  {
172
210
  type: "model/request",
@@ -193,6 +231,43 @@ function createAuthority(
193
231
  }
194
232
  handle.started.resolve();
195
233
  const outcome = await handle.settled.promise;
234
+ if (outcome.interrupted === undefined && options.parkOnRelease?.(runId)) {
235
+ const reason = `Model request "request-${runId}" has no durable provider outcome`;
236
+ await persist(
237
+ {
238
+ type: "model/request",
239
+ turn,
240
+ step: 1,
241
+ request: {
242
+ requestId: `request-${runId}`,
243
+ provider: "foundation",
244
+ model: "foundation-model",
245
+ system: "system",
246
+ messages: [{ role: "user", content: input.command.text }],
247
+ tools: [],
248
+ },
249
+ } as never,
250
+ {
251
+ type: "model/reconciliation-required",
252
+ turn,
253
+ step: 1,
254
+ requestId: `request-${runId}`,
255
+ reason,
256
+ } as never,
257
+ );
258
+ throw new BotTurnReconciliationRequiredError(reason, appended);
259
+ }
260
+ if (outcome.interrupted !== undefined && uncertain) {
261
+ const reason = `Model response outcome is uncertain after cancellation: ${outcome.interrupted}`;
262
+ await persist({
263
+ type: "model/reconciliation-required",
264
+ turn,
265
+ step: 1,
266
+ requestId: `request-${runId}`,
267
+ reason,
268
+ } as never);
269
+ throw new BotTurnReconciliationRequiredError(reason, appended);
270
+ }
196
271
  if (outcome.interrupted !== undefined) {
197
272
  await persist({
198
273
  type: "turn/end",
@@ -576,3 +651,179 @@ describe("eviction between the two Turns", () => {
576
651
  expect(restarted.observed).toHaveLength(1);
577
652
  });
578
653
  });
654
+
655
+ describe("a durable log left inside a Turn", () => {
656
+ test("is repaired at admission instead of refusing every later Turn", async () => {
657
+ const storage = new MemoryStorage();
658
+ // Exactly what a Turn that threw between `turn/start` and `turn/end`
659
+ // leaves behind: an open Turn, and no run to close it.
660
+ storage.values.set("latest-events", [
661
+ {
662
+ type: "session/created",
663
+ createdAt: "2026-09-03T00:00:00.000Z",
664
+ seq: 0,
665
+ timestamp: "2026-09-03T00:00:00.000Z",
666
+ },
667
+ {
668
+ type: "turn/start",
669
+ turn: 1,
670
+ seq: 1,
671
+ timestamp: "2026-09-03T00:00:01.000Z",
672
+ },
673
+ ]);
674
+ const probe = createAuthority(storage);
675
+
676
+ const run = probe.authority.run(command("run-1", "hello"));
677
+ await probe.handle("run-1").started;
678
+ probe.handle("run-1").finish();
679
+ await run;
680
+
681
+ const events = storage.values.get("latest-events") as Array<{
682
+ type: string;
683
+ turn?: number;
684
+ }>;
685
+ // The orphaned Turn is closed, so the new one starts.
686
+ expect(events[2]).toMatchObject({
687
+ type: "turn/end",
688
+ turn: 1,
689
+ outcome: "interrupted",
690
+ });
691
+ expect(storedRun(storage, "run-1").status).toBe("completed");
692
+ });
693
+ });
694
+
695
+ describe("a failing recovery of an older Turn", () => {
696
+ test("does not swallow the message the User just sent", async () => {
697
+ const storage = new MemoryStorage();
698
+ const probe = createAuthority(storage);
699
+ const first = probe.authority.run(command("run-1", "first"));
700
+ await probe.handle("run-1").started;
701
+ // Evicted mid-Turn: run-1 stays active and durable, and the object that
702
+ // comes back recovers it — badly.
703
+ const restarted = createAuthority(storage, {
704
+ failRecovery: (runId) => runId === "run-1",
705
+ });
706
+ const second = restarted.authority
707
+ .run(
708
+ command("run-2", "second", {
709
+ lane: "user",
710
+ supersedes: { runId: "run-1" },
711
+ }),
712
+ )
713
+ .catch(() => undefined);
714
+ await admitted();
715
+
716
+ // The new message is durable regardless of what happened to the old Turn.
717
+ // Before this, the recovery's own error threw out of `run()` before
718
+ // admission was ever attempted and the message was simply gone.
719
+ expect(storedRun(storage, "run-2").runId).toBe("run-2");
720
+ expect(storedRun(storage, "run-2").input).toBe("second");
721
+ probe.handle("run-1").finish();
722
+ await first.catch(() => undefined);
723
+ restarted.handle("run-2").finish();
724
+ await second;
725
+ });
726
+ });
727
+
728
+ describe("the run admission fence index", () => {
729
+ test("ages the oldest entry out rather than refusing the operation", async () => {
730
+ const storage = new MemoryStorage();
731
+ const probe = createAuthority(storage);
732
+ for (let index = 0; index < 300; index += 1) {
733
+ await probe.authority.fenceRunAdmission(identity, `fence-${index}`);
734
+ }
735
+ const fences = storage.values.get("run-admission-fences") as string[];
736
+ expect(fences.length).toBeLessThanOrEqual(256);
737
+ // The newest fence is the one that still matters; the oldest aged out.
738
+ expect(fences.at(-1)).toBe("fence-299");
739
+ expect(fences).not.toContain("fence-0");
740
+ });
741
+ });
742
+
743
+ describe("a Turn queued behind a parked run", () => {
744
+ test("is refused rather than answered with an empty completion", async () => {
745
+ const storage = new MemoryStorage();
746
+ // The first Turn has not dispatched when the second arrives, so it is left
747
+ // to finish and the second queues behind it. It then parks on a provider
748
+ // outcome only a User can retrieve.
749
+ const probe = createAuthority(storage, {
750
+ dispatch: () => false,
751
+ parkOnRelease: (runId) => runId === "run-1",
752
+ });
753
+
754
+ const first = probe.authority.run(command("run-1", "first"));
755
+ await probe.handle("run-1").started;
756
+ const second = probe.authority.run(
757
+ command("run-2", "second", {
758
+ lane: "user",
759
+ supersedes: { runId: "run-1" },
760
+ }),
761
+ );
762
+ await admitted();
763
+ probe.handle("run-1").finish();
764
+ await first.catch(() => undefined);
765
+
766
+ await expect(second).rejects.toThrow(
767
+ /is queued: the active run requires reconciliation/,
768
+ );
769
+ // And it is still owed a Turn: durable, queued, and started by the
770
+ // reconciliation's own settlement or by the recovery alarm.
771
+ const queued = storedRun(storage, "run-2");
772
+ expect(queued.status).toBe("running");
773
+ expect(queued.phase).toBe("queued");
774
+ expect(storage.values.get("pending-run")).toBe("run-2");
775
+ });
776
+ });
777
+
778
+ describe("an interrupt while the model is streaming", () => {
779
+ test("a superseded Turn settles superseded rather than parking the Bot", async () => {
780
+ const storage = new MemoryStorage();
781
+ const probe = createAuthority(storage, { uncertain: () => true });
782
+
783
+ const first = probe.authority.run(command("run-1", "first"));
784
+ await probe.handle("run-1").started;
785
+ const second = probe.authority.run(
786
+ command("run-2", "second", {
787
+ lane: "user",
788
+ supersedes: { runId: "run-1" },
789
+ }),
790
+ );
791
+ await first.catch(() => undefined);
792
+ probe.handle("run-2").finish();
793
+ const result = await second;
794
+
795
+ // The provider outcome of a Turn nobody is waiting for is worthless: the
796
+ // intent the User expressed is what settles it.
797
+ const superseded = storedRun(storage, "run-1");
798
+ expect(superseded.status).toBe("superseded");
799
+ expect(superseded.supersededBy).toBe("run-2");
800
+ expect(storage.values.get("active-run")).toBeUndefined();
801
+ expect(result.text).toBe("done: second");
802
+ });
803
+
804
+ test("a stopped Turn settles cancelled and the next message is admitted", async () => {
805
+ const storage = new MemoryStorage();
806
+ const probe = createAuthority(storage, { uncertain: () => true });
807
+
808
+ const first = probe.authority.run(command("run-1", "first"));
809
+ await probe.handle("run-1").started;
810
+ // What an authenticated Stop writes before it signals the Agent.
811
+ const stopped = storedRun(storage, "run-1");
812
+ storage.values.set("run:run-1", {
813
+ ...stopped,
814
+ stopRequestedAt: "2026-09-03T00:00:05.000Z",
815
+ });
816
+ probe.handle("run-1").interrupt("stopped by an authenticated Stop command");
817
+ await first.catch(() => undefined);
818
+
819
+ expect(storedRun(storage, "run-1").status).toBe("cancelled");
820
+ expect(storage.values.get("active-run")).toBeUndefined();
821
+
822
+ // And the Bot takes the next message straight away, with no
823
+ // reconciliation standing between the User and their Bot.
824
+ const next = probe.authority.run(command("run-2", "second"));
825
+ await probe.handle("run-2").started;
826
+ probe.handle("run-2").finish();
827
+ expect((await next).text).toBe("done: second");
828
+ });
829
+ });