@frockbot/kernel-do 0.3.5 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/authority.ts +42 -3
- package/src/run-records.ts +9 -1
- package/src/run-recovery.test.ts +120 -12
- package/src/run-recovery.ts +46 -2
- package/src/run-terminal-open-turn.test.ts +203 -0
- package/src/run-terminal.ts +51 -8
- package/src/turn-admission.test.ts +38 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@frockbot/kernel-do",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.6",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
"typecheck": "tsc --noEmit -p tsconfig.json"
|
|
13
13
|
},
|
|
14
14
|
"dependencies": {
|
|
15
|
-
"@frockbot/kernel-composition": "0.3.
|
|
16
|
-
"@frockbot/kernel-contracts": "0.3.
|
|
15
|
+
"@frockbot/kernel-composition": "0.3.6",
|
|
16
|
+
"@frockbot/kernel-contracts": "0.3.6",
|
|
17
17
|
"cordis": "4.0.0-rc.8"
|
|
18
18
|
},
|
|
19
19
|
"devDependencies": {
|
package/src/authority.ts
CHANGED
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
eventsForFailedRun,
|
|
35
35
|
latestModelRequestJournalState,
|
|
36
36
|
planBotRunRecovery,
|
|
37
|
+
type ProviderReconcilesV1,
|
|
37
38
|
repairOrphanedOpenTurnV1,
|
|
38
39
|
unresolvedModelRequestFailure,
|
|
39
40
|
} from "./run-recovery.js";
|
|
@@ -144,6 +145,16 @@ export interface BotDurableAuthorityHooks<Snapshot> {
|
|
|
144
145
|
run: StoredRunV1<Snapshot>;
|
|
145
146
|
read<T>(key: string): Promise<T | undefined>;
|
|
146
147
|
}): Promise<Record<string, unknown>>;
|
|
148
|
+
/**
|
|
149
|
+
* Whether the named provider can be asked what happened to a model request
|
|
150
|
+
* it never answered (ADR 0028).
|
|
151
|
+
*
|
|
152
|
+
* Synchronous and pure, because it is consulted inside the recovery
|
|
153
|
+
* transaction: it answers from what the deployment knows about a provider
|
|
154
|
+
* Package, never by reaching one. Absent means every provider reconciles,
|
|
155
|
+
* which is the behaviour that predates the ADR.
|
|
156
|
+
*/
|
|
157
|
+
providerReconciles?: ProviderReconcilesV1;
|
|
147
158
|
}
|
|
148
159
|
|
|
149
160
|
/** What a `turn/end` records when a later user message took a Turn's place. */
|
|
@@ -1044,7 +1055,24 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1044
1055
|
// N-1 is open". Nothing owned that repair, because the run that would
|
|
1045
1056
|
// have closed it is already terminal, so admission does: with nothing
|
|
1046
1057
|
// executing, an open Turn is one nobody is going to finish.
|
|
1047
|
-
|
|
1058
|
+
//
|
|
1059
|
+
// The pointer alone is not the test. A Bot can hold an `active-run` id
|
|
1060
|
+
// whose record is already terminal — a settlement that landed while the
|
|
1061
|
+
// pointer clear did not, a supersede whose Turn ended between the two
|
|
1062
|
+
// writes — and gating the repair on the pointer left exactly those Bots
|
|
1063
|
+
// wedged. What matters is whether anything is still entitled to write
|
|
1064
|
+
// that Turn's end: a `running` record is, and so is a
|
|
1065
|
+
// `reconciliation-required` one, whose Turn is held open on purpose
|
|
1066
|
+
// until its outcome is retrieved. Nothing else is.
|
|
1067
|
+
const activeRun = activeRunId
|
|
1068
|
+
? this.codec.optional(
|
|
1069
|
+
await transaction.get<unknown>(`${RUN_PREFIX}${activeRunId}`),
|
|
1070
|
+
)
|
|
1071
|
+
: undefined;
|
|
1072
|
+
const stillOwned =
|
|
1073
|
+
activeRun?.status === "running" ||
|
|
1074
|
+
activeRun?.status === "reconciliation-required";
|
|
1075
|
+
const repairs = stillOwned
|
|
1048
1076
|
? []
|
|
1049
1077
|
: repairOrphanedOpenTurnV1(command.sessionId, storedEvents);
|
|
1050
1078
|
const latestEvents = [...storedEvents, ...repairs];
|
|
@@ -1418,7 +1446,12 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1418
1446
|
const latest = (
|
|
1419
1447
|
(await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
|
|
1420
1448
|
).map(decodeSessionEvent);
|
|
1421
|
-
const plan = planBotRunRecovery(
|
|
1449
|
+
const plan = planBotRunRecovery(
|
|
1450
|
+
run,
|
|
1451
|
+
latest,
|
|
1452
|
+
this.codec,
|
|
1453
|
+
this.hooks.providerReconciles ?? (() => true),
|
|
1454
|
+
);
|
|
1422
1455
|
if (plan.kind === "complete") {
|
|
1423
1456
|
const result = {
|
|
1424
1457
|
runId: run.runId,
|
|
@@ -1443,13 +1476,19 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1443
1476
|
return undefined;
|
|
1444
1477
|
}
|
|
1445
1478
|
if (plan.kind === "fail") {
|
|
1479
|
+
// The repairs matter when the failure is ADR 0028's: they close the
|
|
1480
|
+
// tool occurrences the restart left open, so the settled run's journal
|
|
1481
|
+
// is a complete account rather than one that stops mid-sentence twice.
|
|
1482
|
+
const events = plan.repairs
|
|
1483
|
+
? [...run.events, ...plan.repairs]
|
|
1484
|
+
: run.events;
|
|
1446
1485
|
await failStoredRun(
|
|
1447
1486
|
this.codec,
|
|
1448
1487
|
transaction,
|
|
1449
1488
|
this.terminalKeys(run.runId),
|
|
1450
1489
|
run.runId,
|
|
1451
1490
|
latest.slice(0, run.previousEventCount),
|
|
1452
|
-
|
|
1491
|
+
events,
|
|
1453
1492
|
plan.failure,
|
|
1454
1493
|
this.supersededPackageRecords(),
|
|
1455
1494
|
);
|
package/src/run-records.ts
CHANGED
|
@@ -758,7 +758,15 @@ export function botTurnCommandFingerprintV1(
|
|
|
758
758
|
...(lane === defaultRunLaneV1(turnType) ? {} : { lane }),
|
|
759
759
|
...(command.subagentRole ? { subagentRole: command.subagentRole } : {}),
|
|
760
760
|
...(command.origin ? { origin: command.origin } : {}),
|
|
761
|
-
|
|
761
|
+
// The *intent* is part of the command's identity, exactly as ADR 0024
|
|
762
|
+
// requires: a replay of a command that carried no supersede can never
|
|
763
|
+
// become one that interrupts a second Turn. The provenance is not. A
|
|
764
|
+
// client retrying the same send — same commandId, same text — names
|
|
765
|
+
// whichever run it happened to have observed by then, and that is a fact
|
|
766
|
+
// about its polling, not about what the person asked for. Hashing it
|
|
767
|
+
// turned an ordinary retry into "this idempotency key was reused for a
|
|
768
|
+
// different command" and refused the send.
|
|
769
|
+
...(command.supersedes ? { supersedes: true } : {}),
|
|
762
770
|
...(skills.length > 0 ? { skills: skills.map(formatSkillRefV1) } : {}),
|
|
763
771
|
...(command.directTool ? { directTool: command.directTool } : {}),
|
|
764
772
|
})}`;
|
package/src/run-recovery.test.ts
CHANGED
|
@@ -2,8 +2,11 @@ import { describe, expect, test } from "bun:test";
|
|
|
2
2
|
import type { SessionEvent } from "@frockbot/kernel-contracts";
|
|
3
3
|
import {
|
|
4
4
|
latestModelRequestJournalState,
|
|
5
|
+
planBotRunRecovery,
|
|
6
|
+
UNRECONCILABLE_RUN_FAILURE_V1,
|
|
5
7
|
unresolvedModelRequestFailure,
|
|
6
8
|
} from "./run-recovery.js";
|
|
9
|
+
import { createStoredRunCodecV1, type StoredRunV1 } from "./run-records.js";
|
|
7
10
|
|
|
8
11
|
// Distributive, so each member of the union keeps its own fields: a bare
|
|
9
12
|
// `Omit` over the union collapses to the keys they all share.
|
|
@@ -76,18 +79,6 @@ describe("unresolvedModelRequestFailure", () => {
|
|
|
76
79
|
);
|
|
77
80
|
});
|
|
78
81
|
|
|
79
|
-
test("reads the last reason when the Turn was retried", () => {
|
|
80
|
-
expect(
|
|
81
|
-
unresolved(
|
|
82
|
-
request,
|
|
83
|
-
reconciliationRequired("request-1", "first attempt"),
|
|
84
|
-
reconciliationRequired("request-1", "second attempt"),
|
|
85
|
-
),
|
|
86
|
-
).toBe(
|
|
87
|
-
'Model request "request-1" has no durable provider outcome: second attempt',
|
|
88
|
-
);
|
|
89
|
-
});
|
|
90
|
-
|
|
91
82
|
test("ignores a reason journaled against another request", () => {
|
|
92
83
|
expect(
|
|
93
84
|
unresolved(
|
|
@@ -104,3 +95,120 @@ describe("unresolvedModelRequestFailure", () => {
|
|
|
104
95
|
);
|
|
105
96
|
});
|
|
106
97
|
});
|
|
98
|
+
|
|
99
|
+
// ADR 0028. A restart mid-Turn used to park every in-flight run on a
|
|
100
|
+
// reconciliation nobody could perform: the providers this deployment actually
|
|
101
|
+
// uses expose no response retrieval, so the banner's Resolve action had one
|
|
102
|
+
// possible outcome and the Bot stayed wedged until somebody clicked it.
|
|
103
|
+
describe("a restart with no retrievable provider outcome", () => {
|
|
104
|
+
const codec = createStoredRunCodecV1<null>({
|
|
105
|
+
decodeRunId: (value) => String(value),
|
|
106
|
+
decodeConfigurationSnapshot: () => null,
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
function runWith(events: SessionEvent[]): StoredRunV1<null> {
|
|
110
|
+
return {
|
|
111
|
+
runId: "run-1",
|
|
112
|
+
commandFingerprint: "fingerprint-1",
|
|
113
|
+
sessionId: "user-1:bot-1",
|
|
114
|
+
acceptedAt: new Date(Date.UTC(2026, 8, 3)).toISOString(),
|
|
115
|
+
input: "hello",
|
|
116
|
+
events,
|
|
117
|
+
effectAdmissions: [],
|
|
118
|
+
status: "running",
|
|
119
|
+
phase: "executing",
|
|
120
|
+
compositionGenerationId: "generation-1",
|
|
121
|
+
configurationSnapshot: null,
|
|
122
|
+
previousEventCount: 0,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** `Session` requires a zero-based contiguous log; this file's own stamper is one-based. */
|
|
127
|
+
function durableJournal(...events: UnstampedEvent[]): SessionEvent[] {
|
|
128
|
+
return events.map(
|
|
129
|
+
(event, index) =>
|
|
130
|
+
({
|
|
131
|
+
...event,
|
|
132
|
+
seq: index,
|
|
133
|
+
timestamp: new Date(Date.UTC(2026, 8, 3, 0, 0, index)).toISOString(),
|
|
134
|
+
}) as SessionEvent,
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const openTurn: UnstampedEvent[] = [
|
|
139
|
+
{
|
|
140
|
+
type: "session/created",
|
|
141
|
+
createdAt: new Date(Date.UTC(2026, 8, 3)).toISOString(),
|
|
142
|
+
},
|
|
143
|
+
{ type: "input/queued", messageId: "message-1", text: "hello" },
|
|
144
|
+
{ type: "turn/start", turn: 1 },
|
|
145
|
+
{ type: "step/start", turn: 1, step: 1 },
|
|
146
|
+
{
|
|
147
|
+
type: "user/message",
|
|
148
|
+
turn: 1,
|
|
149
|
+
step: 1,
|
|
150
|
+
messageId: "message-1",
|
|
151
|
+
text: "hello",
|
|
152
|
+
},
|
|
153
|
+
];
|
|
154
|
+
|
|
155
|
+
test("settles the run as failed rather than parking it", () => {
|
|
156
|
+
const events = durableJournal(...openTurn, request);
|
|
157
|
+
const plan = planBotRunRecovery(
|
|
158
|
+
runWith(events),
|
|
159
|
+
events,
|
|
160
|
+
codec,
|
|
161
|
+
(provider) => provider === "foundation",
|
|
162
|
+
);
|
|
163
|
+
|
|
164
|
+
expect(plan.kind).toBe("fail");
|
|
165
|
+
expect(plan.kind === "fail" ? plan.failure : "").toBe(
|
|
166
|
+
UNRECONCILABLE_RUN_FAILURE_V1,
|
|
167
|
+
);
|
|
168
|
+
// Whatever repairs the resume would have written travel with the
|
|
169
|
+
// settlement, so an unresolved tool occurrence is closed rather than left
|
|
170
|
+
// open in a record nothing will ever revisit. This journal needs none.
|
|
171
|
+
expect(plan.kind === "fail" ? plan.repairs : undefined).toEqual([]);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
test("keeps parking a run whose provider can be asked", () => {
|
|
175
|
+
const events = durableJournal(...openTurn, request);
|
|
176
|
+
const plan = planBotRunRecovery(runWith(events), events, codec, () => true);
|
|
177
|
+
|
|
178
|
+
expect(plan.kind).toBe("reconcile");
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
test("parks by default, so a host that names no policy is unaffected", () => {
|
|
182
|
+
const events = durableJournal(...openTurn, request);
|
|
183
|
+
|
|
184
|
+
expect(planBotRunRecovery(runWith(events), events, codec).kind).toBe(
|
|
185
|
+
"reconcile",
|
|
186
|
+
);
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
test("preserves the words the Turn had already streamed", () => {
|
|
190
|
+
const events = durableJournal(...openTurn, request, {
|
|
191
|
+
type: "assistant/chunk",
|
|
192
|
+
turn: 1,
|
|
193
|
+
step: 1,
|
|
194
|
+
requestId: "request-1",
|
|
195
|
+
text: "Half a thought",
|
|
196
|
+
});
|
|
197
|
+
const plan = planBotRunRecovery(
|
|
198
|
+
runWith(events),
|
|
199
|
+
events,
|
|
200
|
+
codec,
|
|
201
|
+
() => false,
|
|
202
|
+
);
|
|
203
|
+
|
|
204
|
+
expect(plan.kind).toBe("fail");
|
|
205
|
+
// Nothing in the plan discards the journal: the settled record carries the
|
|
206
|
+
// run's own events, and the projection reads the partial answer back out.
|
|
207
|
+
expect(
|
|
208
|
+
events.some(
|
|
209
|
+
(event) =>
|
|
210
|
+
event.type === "assistant/chunk" && event.text === "Half a thought",
|
|
211
|
+
),
|
|
212
|
+
).toBe(true);
|
|
213
|
+
});
|
|
214
|
+
});
|
package/src/run-recovery.ts
CHANGED
|
@@ -11,11 +11,42 @@ import type { StoredRunCodecV1, StoredRunV1 } from "./run-records.js";
|
|
|
11
11
|
|
|
12
12
|
export type BotRunRecoveryPlan =
|
|
13
13
|
| { kind: "complete"; responseText: string }
|
|
14
|
-
| { kind: "fail"; failure: string }
|
|
14
|
+
| { kind: "fail"; failure: string; repairs?: SessionEvent[] }
|
|
15
15
|
| { kind: "restart"; previous: SessionEvent[] }
|
|
16
16
|
| { kind: "resume" }
|
|
17
17
|
| { kind: "reconcile"; repairs: SessionEvent[] };
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* What a Turn says when a restart caught it mid-answer and nobody can be asked
|
|
21
|
+
* how it ended (ADR 0028).
|
|
22
|
+
*
|
|
23
|
+
* It is written for the person watching, not for an operator: they saw the Bot
|
|
24
|
+
* start talking and then stop, and the only useful thing to tell them is that
|
|
25
|
+
* it will not be finishing that sentence and sending again is safe.
|
|
26
|
+
*/
|
|
27
|
+
export const UNRECONCILABLE_RUN_FAILURE_V1 =
|
|
28
|
+
"This Turn stopped partway — the service restarted while the model was answering, and there is no way to find out how that request ended. Try sending it again.";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Whether the provider a run was talking to can be asked what happened to a
|
|
32
|
+
* request it never answered.
|
|
33
|
+
*
|
|
34
|
+
* Given the provider id off the run's own durable `model/request`, so the
|
|
35
|
+
* answer is the same on every recovery of the same run, with no dependency on
|
|
36
|
+
* what happens to be mounted or resident.
|
|
37
|
+
*/
|
|
38
|
+
export type ProviderReconcilesV1 = (providerId: string) => boolean;
|
|
39
|
+
|
|
40
|
+
/** The provider the run's most recent durable model request was addressed to. */
|
|
41
|
+
export function latestModelRequestProviderV1(
|
|
42
|
+
events: readonly SessionEvent[],
|
|
43
|
+
): string | undefined {
|
|
44
|
+
const request = events.findLast((event) => event.type === "model/request");
|
|
45
|
+
return request?.type === "model/request"
|
|
46
|
+
? request.request.provider
|
|
47
|
+
: undefined;
|
|
48
|
+
}
|
|
49
|
+
|
|
19
50
|
export type ModelRequestJournalState =
|
|
20
51
|
| { status: "none" }
|
|
21
52
|
| {
|
|
@@ -90,6 +121,7 @@ export function planBotRunRecovery<Snapshot>(
|
|
|
90
121
|
run: StoredRunV1<Snapshot>,
|
|
91
122
|
latest: readonly SessionEvent[],
|
|
92
123
|
codec: StoredRunCodecV1<Snapshot>,
|
|
124
|
+
providerReconciles: ProviderReconcilesV1 = () => true,
|
|
93
125
|
): BotRunRecoveryPlan {
|
|
94
126
|
codec.require(run);
|
|
95
127
|
let toolJournal: ReturnType<typeof validateToolOccurrenceJournal>;
|
|
@@ -179,7 +211,19 @@ export function planBotRunRecovery<Snapshot>(
|
|
|
179
211
|
};
|
|
180
212
|
}
|
|
181
213
|
const session = new Session(run.sessionId, () => {}, latest);
|
|
182
|
-
|
|
214
|
+
const repairs = session.reconcileForResume();
|
|
215
|
+
// ADR 0028. A Turn whose model outcome is unknown is parked only when
|
|
216
|
+
// somebody can actually be asked. When the provider offers no retrieval,
|
|
217
|
+
// parking is not caution — it is a dead end: nothing will ever arrive to
|
|
218
|
+
// resolve it, the Bot stays wedged behind it, and the person is handed a
|
|
219
|
+
// Resolve button whose only possible answer is "give up". So the run is
|
|
220
|
+
// settled `failed` here, with its repairs and every streamed word it had
|
|
221
|
+
// already sent kept in the journal.
|
|
222
|
+
const provider = latestModelRequestProviderV1(run.events);
|
|
223
|
+
if (provider !== undefined && !providerReconciles(provider)) {
|
|
224
|
+
return { kind: "fail", failure: UNRECONCILABLE_RUN_FAILURE_V1, repairs };
|
|
225
|
+
}
|
|
226
|
+
return { kind: "reconcile", repairs };
|
|
183
227
|
}
|
|
184
228
|
|
|
185
229
|
/** True when the durable log ends inside a Turn nothing is going to finish. */
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
// The exact sequence that wedged a Bot forever.
|
|
2
|
+
//
|
|
3
|
+
// A Turn interrupted mid-answer unwinds without writing a `turn/end`: its model
|
|
4
|
+
// request has no durable outcome and a `turn/end` would claim to know how it
|
|
5
|
+
// ended. Right while the run might resume; wrong once it will not. The
|
|
6
|
+
// settlement committed those events as they stood, so the durable session log
|
|
7
|
+
// ended inside an open turn, and every later message on that Bot failed
|
|
8
|
+
// validation with `turn 2 started while turn 1 is open` — printed verbatim into
|
|
9
|
+
// the person's next bubble, forever.
|
|
10
|
+
import { describe, expect, test } from "bun:test";
|
|
11
|
+
import {
|
|
12
|
+
Session,
|
|
13
|
+
type SessionEvent,
|
|
14
|
+
validateToolOccurrenceJournal,
|
|
15
|
+
} from "@frockbot/kernel-contracts";
|
|
16
|
+
import { MemoryStorage } from "./memory-storage.fixture.ts";
|
|
17
|
+
import { createStoredRunCodecV1, type StoredRunV1 } from "./run-records.ts";
|
|
18
|
+
import {
|
|
19
|
+
cancelStoredRun,
|
|
20
|
+
failStoredRun,
|
|
21
|
+
supersedeStoredRun,
|
|
22
|
+
} from "./run-terminal.ts";
|
|
23
|
+
|
|
24
|
+
const codec = createStoredRunCodecV1<null>({
|
|
25
|
+
decodeRunId: (value) => String(value),
|
|
26
|
+
decodeConfigurationSnapshot: () => null,
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
const SESSION_ID = "user-1:primary";
|
|
30
|
+
|
|
31
|
+
const KEYS = {
|
|
32
|
+
run: "run:run-1",
|
|
33
|
+
activeRun: "active-run",
|
|
34
|
+
latestEvents: "latest-events",
|
|
35
|
+
notificationPrefix: "notification:",
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* A Turn stopped after its model request went uncertain: `turn/start`,
|
|
40
|
+
* `step/start`, a `user/message`, a `model/request`, and a
|
|
41
|
+
* `model/reconciliation-required` — and then nothing. This is what the Agent
|
|
42
|
+
* loop leaves behind when it unwinds on an abort.
|
|
43
|
+
*/
|
|
44
|
+
function interruptedJournal(): SessionEvent[] {
|
|
45
|
+
const events: SessionEvent[] = [];
|
|
46
|
+
const session = new Session(SESSION_ID, (envelope) => {
|
|
47
|
+
events.push(envelope.event);
|
|
48
|
+
});
|
|
49
|
+
session.appendBatch([
|
|
50
|
+
{ type: "turn/start", turn: 1 },
|
|
51
|
+
{ type: "step/start", turn: 1, step: 1 },
|
|
52
|
+
{
|
|
53
|
+
type: "user/message",
|
|
54
|
+
turn: 1,
|
|
55
|
+
step: 1,
|
|
56
|
+
messageId: "message-1",
|
|
57
|
+
text: "hello",
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
type: "model/request",
|
|
61
|
+
turn: 1,
|
|
62
|
+
step: 1,
|
|
63
|
+
request: {
|
|
64
|
+
requestId: "request-1",
|
|
65
|
+
provider: "flock-ai",
|
|
66
|
+
model: "@flock/auto",
|
|
67
|
+
system: "",
|
|
68
|
+
messages: [],
|
|
69
|
+
tools: [],
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
type: "model/reconciliation-required",
|
|
74
|
+
turn: 1,
|
|
75
|
+
step: 1,
|
|
76
|
+
requestId: "request-1",
|
|
77
|
+
reason: "Model response outcome is uncertain after cancellation",
|
|
78
|
+
},
|
|
79
|
+
]);
|
|
80
|
+
return events;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function storedRun(
|
|
84
|
+
events: SessionEvent[],
|
|
85
|
+
intent: Partial<StoredRunV1<null>>,
|
|
86
|
+
): StoredRunV1<null> {
|
|
87
|
+
return {
|
|
88
|
+
runId: "run-1",
|
|
89
|
+
commandFingerprint: "fingerprint-1",
|
|
90
|
+
sessionId: SESSION_ID,
|
|
91
|
+
acceptedAt: "2026-09-03T00:00:00.000Z",
|
|
92
|
+
input: "hello",
|
|
93
|
+
events,
|
|
94
|
+
effectAdmissions: [],
|
|
95
|
+
status: "running",
|
|
96
|
+
phase: "executing",
|
|
97
|
+
compositionGenerationId: "generation-1",
|
|
98
|
+
configurationSnapshot: null,
|
|
99
|
+
previousEventCount: 0,
|
|
100
|
+
...intent,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function settled(
|
|
105
|
+
intent: Partial<StoredRunV1<null>>,
|
|
106
|
+
settle: (storage: MemoryStorage, events: SessionEvent[]) => Promise<unknown>,
|
|
107
|
+
): Promise<{ storage: MemoryStorage; latest: SessionEvent[] }> {
|
|
108
|
+
const storage = new MemoryStorage();
|
|
109
|
+
const events = interruptedJournal();
|
|
110
|
+
await storage.put({
|
|
111
|
+
[KEYS.activeRun]: "run-1",
|
|
112
|
+
[KEYS.run]: storedRun(events, intent),
|
|
113
|
+
[KEYS.latestEvents]: events,
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
await settle(storage, events);
|
|
117
|
+
|
|
118
|
+
return {
|
|
119
|
+
storage,
|
|
120
|
+
latest: storage.values.get(KEYS.latestEvents) as SessionEvent[],
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** What the next Turn does: start turn 2 on the log the settlement left. */
|
|
125
|
+
function admitNextTurn(latest: SessionEvent[]): void {
|
|
126
|
+
const session = new Session(SESSION_ID, () => {}, latest);
|
|
127
|
+
session.append({ type: "turn/start", turn: 2 });
|
|
128
|
+
validateToolOccurrenceJournal(session.events);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
describe("settling a Turn interrupted mid-answer", () => {
|
|
132
|
+
test("a superseded run leaves a log the next Turn can start on", async () => {
|
|
133
|
+
const { latest, storage } = await settled(
|
|
134
|
+
{ supersededAt: "2026-09-03T00:01:00.000Z", supersededBy: "run-2" },
|
|
135
|
+
(store, events) =>
|
|
136
|
+
supersedeStoredRun(codec, store, KEYS, "run-1", [], events),
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
// Before the fix this threw "turn 2 started while turn 1 is open", and
|
|
140
|
+
// every later message on this Bot answered 500 with that sentence.
|
|
141
|
+
expect(() => admitNextTurn(latest)).not.toThrow();
|
|
142
|
+
expect(latest.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
|
|
143
|
+
// The settled record carries the same closed account, not a different one.
|
|
144
|
+
const record = storage.values.get(KEYS.run) as StoredRunV1<null>;
|
|
145
|
+
expect(record.status).toBe("superseded");
|
|
146
|
+
expect(record.events.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("a stopped run leaves a log the next Turn can start on", async () => {
|
|
150
|
+
const { latest } = await settled(
|
|
151
|
+
{ stopRequestedAt: "2026-09-03T00:01:00.000Z" },
|
|
152
|
+
(store, events) =>
|
|
153
|
+
cancelStoredRun(codec, store, KEYS, "run-1", [], events),
|
|
154
|
+
);
|
|
155
|
+
|
|
156
|
+
expect(() => admitNextTurn(latest)).not.toThrow();
|
|
157
|
+
expect(latest.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test("a failed run leaves a log the next Turn can start on", async () => {
|
|
161
|
+
const { latest } = await settled({}, (store, events) =>
|
|
162
|
+
failStoredRun(
|
|
163
|
+
codec,
|
|
164
|
+
store,
|
|
165
|
+
KEYS,
|
|
166
|
+
"run-1",
|
|
167
|
+
[],
|
|
168
|
+
events,
|
|
169
|
+
"the service restarted",
|
|
170
|
+
),
|
|
171
|
+
);
|
|
172
|
+
|
|
173
|
+
expect(() => admitNextTurn(latest)).not.toThrow();
|
|
174
|
+
expect(latest.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
test("a Turn that closed itself is not closed twice", async () => {
|
|
178
|
+
const storage = new MemoryStorage();
|
|
179
|
+
const events: SessionEvent[] = [];
|
|
180
|
+
const session = new Session(SESSION_ID, (envelope) => {
|
|
181
|
+
events.push(envelope.event);
|
|
182
|
+
});
|
|
183
|
+
session.appendBatch([
|
|
184
|
+
{ type: "turn/start", turn: 1 },
|
|
185
|
+
{ type: "step/start", turn: 1, step: 1 },
|
|
186
|
+
{ type: "step/end", turn: 1, step: 1, outcome: "cancelled" },
|
|
187
|
+
{ type: "turn/end", turn: 1, outcome: "cancelled" },
|
|
188
|
+
]);
|
|
189
|
+
await storage.put({
|
|
190
|
+
[KEYS.activeRun]: "run-1",
|
|
191
|
+
[KEYS.run]: storedRun(events, {
|
|
192
|
+
stopRequestedAt: "2026-09-03T00:01:00.000Z",
|
|
193
|
+
}),
|
|
194
|
+
[KEYS.latestEvents]: events,
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
await cancelStoredRun(codec, storage, KEYS, "run-1", [], events);
|
|
198
|
+
|
|
199
|
+
const latest = storage.values.get(KEYS.latestEvents) as SessionEvent[];
|
|
200
|
+
expect(latest.filter((event) => event.type === "turn/end")).toHaveLength(1);
|
|
201
|
+
expect(() => admitNextTurn(latest)).not.toThrow();
|
|
202
|
+
});
|
|
203
|
+
});
|
package/src/run-terminal.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
decodeSessionEvent,
|
|
3
|
+
Session,
|
|
3
4
|
type SessionEvent,
|
|
4
5
|
} from "@frockbot/kernel-contracts";
|
|
5
6
|
import type {
|
|
@@ -8,6 +9,47 @@ import type {
|
|
|
8
9
|
StoredRunV1,
|
|
9
10
|
} from "./run-records.js";
|
|
10
11
|
|
|
12
|
+
/**
|
|
13
|
+
* The events a terminal settlement commits, with any Turn they were left
|
|
14
|
+
* inside closed.
|
|
15
|
+
*
|
|
16
|
+
* A Turn interrupted mid-answer unwinds without writing a `turn/end`: the
|
|
17
|
+
* outcome of its model request is unknown, and a `turn/end` would claim to know
|
|
18
|
+
* how it ended. That is right while the run might still resume — and wrong the
|
|
19
|
+
* moment it will not. Settling one and committing its events as they stand left
|
|
20
|
+
* an open turn in the durable session log, so `turn N started while turn N-1 is
|
|
21
|
+
* open` refused every later message on that Bot, forever, and printed itself
|
|
22
|
+
* verbatim into the person's next bubble.
|
|
23
|
+
*
|
|
24
|
+
* So closing the turn happens exactly here: at the one point where the run is
|
|
25
|
+
* certainly not resuming. `reconcileInterrupted` writes the same repair the
|
|
26
|
+
* recovery path already writes — every unresolved tool occurrence closed as
|
|
27
|
+
* `interrupted`, then `step/end` and `turn/end` — so the settled log is a
|
|
28
|
+
* complete account and the next Turn starts on a closed one.
|
|
29
|
+
*
|
|
30
|
+
* A log that is already closed produces no repairs, and one too malformed to
|
|
31
|
+
* reconcile is left exactly as it is: repairing that blindly would invent
|
|
32
|
+
* history.
|
|
33
|
+
*/
|
|
34
|
+
function settledEventsV1(
|
|
35
|
+
sessionId: string,
|
|
36
|
+
previous: readonly SessionEvent[],
|
|
37
|
+
events: readonly SessionEvent[],
|
|
38
|
+
): { events: SessionEvent[]; latestEvents: SessionEvent[] } {
|
|
39
|
+
const decoded = events.map(decodeSessionEvent);
|
|
40
|
+
const latest = [...previous, ...decoded].map(decodeSessionEvent);
|
|
41
|
+
let repairs: SessionEvent[] = [];
|
|
42
|
+
try {
|
|
43
|
+
repairs = new Session(sessionId, () => {}, latest).reconcileInterrupted();
|
|
44
|
+
} catch {
|
|
45
|
+
repairs = [];
|
|
46
|
+
}
|
|
47
|
+
return {
|
|
48
|
+
events: [...decoded, ...repairs],
|
|
49
|
+
latestEvents: [...latest, ...repairs],
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
11
53
|
export interface RunTerminalStorage {
|
|
12
54
|
get<T>(key: string): Promise<T | undefined>;
|
|
13
55
|
put(entries: Record<string, unknown>): Promise<void>;
|
|
@@ -83,7 +125,8 @@ export async function supersedeStoredRun<Snapshot>(
|
|
|
83
125
|
if (!run.supersededAt) {
|
|
84
126
|
throw new Error(`run "${runId}" has no durable supersede intent`);
|
|
85
127
|
}
|
|
86
|
-
const
|
|
128
|
+
const settledEvents = settledEventsV1(run.sessionId, previous, events);
|
|
129
|
+
const decodedEvents = settledEvents.events;
|
|
87
130
|
const { responseText: _text, failure: _failure, ...settled } = run;
|
|
88
131
|
// A run superseded while still queued never started, never appended an
|
|
89
132
|
// event, and never spoke: it settles as a record on its own and leaves both
|
|
@@ -105,9 +148,7 @@ export async function supersedeStoredRun<Snapshot>(
|
|
|
105
148
|
...(queued
|
|
106
149
|
? {}
|
|
107
150
|
: {
|
|
108
|
-
[keys.latestEvents]: structuredClone(
|
|
109
|
-
[...previous, ...decodedEvents].map(decodeSessionEvent),
|
|
110
|
-
),
|
|
151
|
+
[keys.latestEvents]: structuredClone(settledEvents.latestEvents),
|
|
111
152
|
}),
|
|
112
153
|
};
|
|
113
154
|
if (packageRecords && !queued) {
|
|
@@ -226,8 +267,9 @@ export async function cancelStoredRun<Snapshot>(
|
|
|
226
267
|
if (!run.stopRequestedAt) {
|
|
227
268
|
throw new Error(`run "${runId}" has no durable stop intent`);
|
|
228
269
|
}
|
|
229
|
-
const
|
|
230
|
-
const
|
|
270
|
+
const settledEvents = settledEventsV1(run.sessionId, previous, events);
|
|
271
|
+
const decodedEvents = settledEvents.events;
|
|
272
|
+
const latestEvents = settledEvents.latestEvents;
|
|
231
273
|
const { responseText: _text, failure: _failure, ...settled } = run;
|
|
232
274
|
const cancelled = codec.require({
|
|
233
275
|
...settled,
|
|
@@ -278,8 +320,9 @@ export async function failStoredRun<Snapshot>(
|
|
|
278
320
|
supersededRecords,
|
|
279
321
|
);
|
|
280
322
|
}
|
|
281
|
-
const
|
|
282
|
-
const
|
|
323
|
+
const settledEvents = settledEventsV1(run.sessionId, previous, events);
|
|
324
|
+
const decodedEvents = settledEvents.events;
|
|
325
|
+
const latestEvents = settledEvents.latestEvents;
|
|
283
326
|
const failed = codec.require({
|
|
284
327
|
...run,
|
|
285
328
|
events: decodedEvents,
|
|
@@ -448,3 +448,41 @@ describe("an admitted Turn re-mounts on its recorded turn type", () => {
|
|
|
448
448
|
});
|
|
449
449
|
});
|
|
450
450
|
});
|
|
451
|
+
|
|
452
|
+
// robustness F18. The composer sends `supersedes` on every send and names
|
|
453
|
+
// whichever run it happened to have observed. A retry of the same send names a
|
|
454
|
+
// different one — or none — and used to be refused as a reused idempotency key.
|
|
455
|
+
describe("a retried send is idempotent whatever run it names", () => {
|
|
456
|
+
const command = {
|
|
457
|
+
userId: "user-1",
|
|
458
|
+
botId: "primary",
|
|
459
|
+
runId: "run-1",
|
|
460
|
+
sessionId: "user-1:primary",
|
|
461
|
+
acceptedAt: "2026-08-31T01:00:00.000Z",
|
|
462
|
+
text: "hello",
|
|
463
|
+
lane: "user" as const,
|
|
464
|
+
};
|
|
465
|
+
|
|
466
|
+
test("the observed run id is not part of the command's identity", () => {
|
|
467
|
+
const first = botTurnCommandFingerprintV1({ ...command, supersedes: {} });
|
|
468
|
+
|
|
469
|
+
expect(
|
|
470
|
+
botTurnCommandFingerprintV1({
|
|
471
|
+
...command,
|
|
472
|
+
supersedes: { runId: "run-0" },
|
|
473
|
+
}),
|
|
474
|
+
).toBe(first);
|
|
475
|
+
expect(
|
|
476
|
+
botTurnCommandFingerprintV1({
|
|
477
|
+
...command,
|
|
478
|
+
supersedes: { runId: "run-99" },
|
|
479
|
+
}),
|
|
480
|
+
).toBe(first);
|
|
481
|
+
});
|
|
482
|
+
|
|
483
|
+
test("but the intent itself still is, so a replay cannot gain one", () => {
|
|
484
|
+
expect(
|
|
485
|
+
botTurnCommandFingerprintV1({ ...command, supersedes: {} }),
|
|
486
|
+
).not.toBe(botTurnCommandFingerprintV1(command));
|
|
487
|
+
});
|
|
488
|
+
});
|