@frockbot/kernel-do 0.3.5 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/authority.ts +234 -10
- package/src/conversations.test.ts +159 -0
- package/src/conversations.ts +126 -0
- package/src/index.ts +1 -0
- package/src/run-records.ts +9 -1
- package/src/run-recovery.test.ts +120 -12
- package/src/run-recovery.ts +116 -2
- package/src/run-terminal-open-turn.test.ts +203 -0
- package/src/run-terminal.ts +58 -8
- package/src/storage-keys.ts +15 -0
- package/src/turn-admission.test.ts +38 -0
- package/src/turn-supersede.test.ts +85 -1
package/src/run-recovery.test.ts
CHANGED
|
@@ -2,8 +2,11 @@ import { describe, expect, test } from "bun:test";
|
|
|
2
2
|
import type { SessionEvent } from "@frockbot/kernel-contracts";
|
|
3
3
|
import {
|
|
4
4
|
latestModelRequestJournalState,
|
|
5
|
+
planBotRunRecovery,
|
|
6
|
+
UNRECONCILABLE_RUN_FAILURE_V1,
|
|
5
7
|
unresolvedModelRequestFailure,
|
|
6
8
|
} from "./run-recovery.js";
|
|
9
|
+
import { createStoredRunCodecV1, type StoredRunV1 } from "./run-records.js";
|
|
7
10
|
|
|
8
11
|
// Distributive, so each member of the union keeps its own fields: a bare
|
|
9
12
|
// `Omit` over the union collapses to the keys they all share.
|
|
@@ -76,18 +79,6 @@ describe("unresolvedModelRequestFailure", () => {
|
|
|
76
79
|
);
|
|
77
80
|
});
|
|
78
81
|
|
|
79
|
-
test("reads the last reason when the Turn was retried", () => {
|
|
80
|
-
expect(
|
|
81
|
-
unresolved(
|
|
82
|
-
request,
|
|
83
|
-
reconciliationRequired("request-1", "first attempt"),
|
|
84
|
-
reconciliationRequired("request-1", "second attempt"),
|
|
85
|
-
),
|
|
86
|
-
).toBe(
|
|
87
|
-
'Model request "request-1" has no durable provider outcome: second attempt',
|
|
88
|
-
);
|
|
89
|
-
});
|
|
90
|
-
|
|
91
82
|
test("ignores a reason journaled against another request", () => {
|
|
92
83
|
expect(
|
|
93
84
|
unresolved(
|
|
@@ -104,3 +95,120 @@ describe("unresolvedModelRequestFailure", () => {
|
|
|
104
95
|
);
|
|
105
96
|
});
|
|
106
97
|
});
|
|
98
|
+
|
|
99
|
+
// ADR 0028. A restart mid-Turn used to park every in-flight run on a
|
|
100
|
+
// reconciliation nobody could perform: the providers this deployment actually
|
|
101
|
+
// uses expose no response retrieval, so the banner's Resolve action had one
|
|
102
|
+
// possible outcome and the Bot stayed wedged until somebody clicked it.
|
|
103
|
+
describe("a restart with no retrievable provider outcome", () => {
|
|
104
|
+
const codec = createStoredRunCodecV1<null>({
|
|
105
|
+
decodeRunId: (value) => String(value),
|
|
106
|
+
decodeConfigurationSnapshot: () => null,
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
function runWith(events: SessionEvent[]): StoredRunV1<null> {
|
|
110
|
+
return {
|
|
111
|
+
runId: "run-1",
|
|
112
|
+
commandFingerprint: "fingerprint-1",
|
|
113
|
+
sessionId: "user-1:bot-1",
|
|
114
|
+
acceptedAt: new Date(Date.UTC(2026, 8, 3)).toISOString(),
|
|
115
|
+
input: "hello",
|
|
116
|
+
events,
|
|
117
|
+
effectAdmissions: [],
|
|
118
|
+
status: "running",
|
|
119
|
+
phase: "executing",
|
|
120
|
+
compositionGenerationId: "generation-1",
|
|
121
|
+
configurationSnapshot: null,
|
|
122
|
+
previousEventCount: 0,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** `Session` requires a zero-based contiguous log; this file's own stamper is one-based. */
|
|
127
|
+
function durableJournal(...events: UnstampedEvent[]): SessionEvent[] {
|
|
128
|
+
return events.map(
|
|
129
|
+
(event, index) =>
|
|
130
|
+
({
|
|
131
|
+
...event,
|
|
132
|
+
seq: index,
|
|
133
|
+
timestamp: new Date(Date.UTC(2026, 8, 3, 0, 0, index)).toISOString(),
|
|
134
|
+
}) as SessionEvent,
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const openTurn: UnstampedEvent[] = [
|
|
139
|
+
{
|
|
140
|
+
type: "session/created",
|
|
141
|
+
createdAt: new Date(Date.UTC(2026, 8, 3)).toISOString(),
|
|
142
|
+
},
|
|
143
|
+
{ type: "input/queued", messageId: "message-1", text: "hello" },
|
|
144
|
+
{ type: "turn/start", turn: 1 },
|
|
145
|
+
{ type: "step/start", turn: 1, step: 1 },
|
|
146
|
+
{
|
|
147
|
+
type: "user/message",
|
|
148
|
+
turn: 1,
|
|
149
|
+
step: 1,
|
|
150
|
+
messageId: "message-1",
|
|
151
|
+
text: "hello",
|
|
152
|
+
},
|
|
153
|
+
];
|
|
154
|
+
|
|
155
|
+
test("settles the run as failed rather than parking it", () => {
|
|
156
|
+
const events = durableJournal(...openTurn, request);
|
|
157
|
+
const plan = planBotRunRecovery(
|
|
158
|
+
runWith(events),
|
|
159
|
+
events,
|
|
160
|
+
codec,
|
|
161
|
+
(provider) => provider === "foundation",
|
|
162
|
+
);
|
|
163
|
+
|
|
164
|
+
expect(plan.kind).toBe("fail");
|
|
165
|
+
expect(plan.kind === "fail" ? plan.failure : "").toBe(
|
|
166
|
+
UNRECONCILABLE_RUN_FAILURE_V1,
|
|
167
|
+
);
|
|
168
|
+
// Whatever repairs the resume would have written travel with the
|
|
169
|
+
// settlement, so an unresolved tool occurrence is closed rather than left
|
|
170
|
+
// open in a record nothing will ever revisit. This journal needs none.
|
|
171
|
+
expect(plan.kind === "fail" ? plan.repairs : undefined).toEqual([]);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
test("keeps parking a run whose provider can be asked", () => {
|
|
175
|
+
const events = durableJournal(...openTurn, request);
|
|
176
|
+
const plan = planBotRunRecovery(runWith(events), events, codec, () => true);
|
|
177
|
+
|
|
178
|
+
expect(plan.kind).toBe("reconcile");
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
test("parks by default, so a host that names no policy is unaffected", () => {
|
|
182
|
+
const events = durableJournal(...openTurn, request);
|
|
183
|
+
|
|
184
|
+
expect(planBotRunRecovery(runWith(events), events, codec).kind).toBe(
|
|
185
|
+
"reconcile",
|
|
186
|
+
);
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
test("preserves the words the Turn had already streamed", () => {
|
|
190
|
+
const events = durableJournal(...openTurn, request, {
|
|
191
|
+
type: "assistant/chunk",
|
|
192
|
+
turn: 1,
|
|
193
|
+
step: 1,
|
|
194
|
+
requestId: "request-1",
|
|
195
|
+
text: "Half a thought",
|
|
196
|
+
});
|
|
197
|
+
const plan = planBotRunRecovery(
|
|
198
|
+
runWith(events),
|
|
199
|
+
events,
|
|
200
|
+
codec,
|
|
201
|
+
() => false,
|
|
202
|
+
);
|
|
203
|
+
|
|
204
|
+
expect(plan.kind).toBe("fail");
|
|
205
|
+
// Nothing in the plan discards the journal: the settled record carries the
|
|
206
|
+
// run's own events, and the projection reads the partial answer back out.
|
|
207
|
+
expect(
|
|
208
|
+
events.some(
|
|
209
|
+
(event) =>
|
|
210
|
+
event.type === "assistant/chunk" && event.text === "Half a thought",
|
|
211
|
+
),
|
|
212
|
+
).toBe(true);
|
|
213
|
+
});
|
|
214
|
+
});
|
package/src/run-recovery.ts
CHANGED
|
@@ -11,11 +11,42 @@ import type { StoredRunCodecV1, StoredRunV1 } from "./run-records.js";
|
|
|
11
11
|
|
|
12
12
|
export type BotRunRecoveryPlan =
|
|
13
13
|
| { kind: "complete"; responseText: string }
|
|
14
|
-
| { kind: "fail"; failure: string }
|
|
14
|
+
| { kind: "fail"; failure: string; repairs?: SessionEvent[] }
|
|
15
15
|
| { kind: "restart"; previous: SessionEvent[] }
|
|
16
16
|
| { kind: "resume" }
|
|
17
17
|
| { kind: "reconcile"; repairs: SessionEvent[] };
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* What a Turn says when a restart caught it mid-answer and nobody can be asked
|
|
21
|
+
* how it ended (ADR 0028).
|
|
22
|
+
*
|
|
23
|
+
* It is written for the person watching, not for an operator: they saw the Bot
|
|
24
|
+
* start talking and then stop, and the only useful thing to tell them is that
|
|
25
|
+
* it will not be finishing that sentence and sending again is safe.
|
|
26
|
+
*/
|
|
27
|
+
export const UNRECONCILABLE_RUN_FAILURE_V1 =
|
|
28
|
+
"This Turn stopped partway — the service restarted while the model was answering, and there is no way to find out how that request ended. Try sending it again.";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Whether the provider a run was talking to can be asked what happened to a
|
|
32
|
+
* request it never answered.
|
|
33
|
+
*
|
|
34
|
+
* Given the provider id off the run's own durable `model/request`, so the
|
|
35
|
+
* answer is the same on every recovery of the same run, with no dependency on
|
|
36
|
+
* what happens to be mounted or resident.
|
|
37
|
+
*/
|
|
38
|
+
export type ProviderReconcilesV1 = (providerId: string) => boolean;
|
|
39
|
+
|
|
40
|
+
/** The provider the run's most recent durable model request was addressed to. */
|
|
41
|
+
export function latestModelRequestProviderV1(
|
|
42
|
+
events: readonly SessionEvent[],
|
|
43
|
+
): string | undefined {
|
|
44
|
+
const request = events.findLast((event) => event.type === "model/request");
|
|
45
|
+
return request?.type === "model/request"
|
|
46
|
+
? request.request.provider
|
|
47
|
+
: undefined;
|
|
48
|
+
}
|
|
49
|
+
|
|
19
50
|
export type ModelRequestJournalState =
|
|
20
51
|
| { status: "none" }
|
|
21
52
|
| {
|
|
@@ -90,6 +121,7 @@ export function planBotRunRecovery<Snapshot>(
|
|
|
90
121
|
run: StoredRunV1<Snapshot>,
|
|
91
122
|
latest: readonly SessionEvent[],
|
|
92
123
|
codec: StoredRunCodecV1<Snapshot>,
|
|
124
|
+
providerReconciles: ProviderReconcilesV1 = () => true,
|
|
93
125
|
): BotRunRecoveryPlan {
|
|
94
126
|
codec.require(run);
|
|
95
127
|
let toolJournal: ReturnType<typeof validateToolOccurrenceJournal>;
|
|
@@ -179,7 +211,19 @@ export function planBotRunRecovery<Snapshot>(
|
|
|
179
211
|
};
|
|
180
212
|
}
|
|
181
213
|
const session = new Session(run.sessionId, () => {}, latest);
|
|
182
|
-
|
|
214
|
+
const repairs = session.reconcileForResume();
|
|
215
|
+
// ADR 0028. A Turn whose model outcome is unknown is parked only when
|
|
216
|
+
// somebody can actually be asked. When the provider offers no retrieval,
|
|
217
|
+
// parking is not caution — it is a dead end: nothing will ever arrive to
|
|
218
|
+
// resolve it, the Bot stays wedged behind it, and the person is handed a
|
|
219
|
+
// Resolve button whose only possible answer is "give up". So the run is
|
|
220
|
+
// settled `failed` here, with its repairs and every streamed word it had
|
|
221
|
+
// already sent kept in the journal.
|
|
222
|
+
const provider = latestModelRequestProviderV1(run.events);
|
|
223
|
+
if (provider !== undefined && !providerReconciles(provider)) {
|
|
224
|
+
return { kind: "fail", failure: UNRECONCILABLE_RUN_FAILURE_V1, repairs };
|
|
225
|
+
}
|
|
226
|
+
return { kind: "reconcile", repairs };
|
|
183
227
|
}
|
|
184
228
|
|
|
185
229
|
/** True when the durable log ends inside a Turn nothing is going to finish. */
|
|
@@ -222,6 +266,76 @@ export function repairOrphanedOpenTurnV1(
|
|
|
222
266
|
}
|
|
223
267
|
}
|
|
224
268
|
|
|
269
|
+
/**
|
|
270
|
+
* True when *any* Turn in the log was never closed — including one buried
|
|
271
|
+
* behind later Turns that are themselves well formed.
|
|
272
|
+
*
|
|
273
|
+
* `hasOrphanedOpenTurnV1` only sees a log that *ends* inside a Turn, and that
|
|
274
|
+
* is the shape a wedged Bot stops having after its very first retry. The Agent
|
|
275
|
+
* loop journals `turn/start` durably and only then assembles the request, so
|
|
276
|
+
* the Turn that discovers the invariant is broken has already written its own
|
|
277
|
+
* `turn/start`, and its `finally` writes a matching `turn/end` carrying the
|
|
278
|
+
* validation message. The log that comes out of that ends closed — with the
|
|
279
|
+
* abandoned Turn still open several events back — so the trailing-open test
|
|
280
|
+
* says there is nothing to repair, and every later message fails the same way.
|
|
281
|
+
*/
|
|
282
|
+
export function hasUnclosedTurnV1(events: readonly SessionEvent[]): boolean {
|
|
283
|
+
let openTurn: number | undefined;
|
|
284
|
+
for (const event of events) {
|
|
285
|
+
if (event.type === "turn/start") {
|
|
286
|
+
if (openTurn !== undefined) return true;
|
|
287
|
+
openTurn = event.turn;
|
|
288
|
+
}
|
|
289
|
+
if (event.type === "turn/end" && event.turn === openTurn) {
|
|
290
|
+
openTurn = undefined;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
return openTurn !== undefined;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* The whole durable log with every abandoned Turn closed, or `undefined` when
|
|
298
|
+
* there is nothing to repair or the log cannot be repaired without inventing
|
|
299
|
+
* history.
|
|
300
|
+
*
|
|
301
|
+
* A Turn left open in the middle of the log cannot be closed by appending:
|
|
302
|
+
* `turn/end` for it would land after the Turns that followed, and the log
|
|
303
|
+
* would still read as "turn N started while turn N-1 is open". So the repair
|
|
304
|
+
* *rewrites* the log, inserting the closing events at the point the Turn was
|
|
305
|
+
* abandoned and resequencing what follows. The inserted events are the ones
|
|
306
|
+
* the interrupted-run repair already writes — every unresolved tool occurrence
|
|
307
|
+
* closed as `interrupted`, then `step/end`, then `turn/end` with outcome
|
|
308
|
+
* `interrupted` — so a Turn nobody finished reads as one nobody finished.
|
|
309
|
+
*
|
|
310
|
+
* Only called where nothing is entitled to write those ends: at admission and
|
|
311
|
+
* promotion with no run executing, and at settlement, where the run that owned
|
|
312
|
+
* the Turn has just stopped.
|
|
313
|
+
*/
|
|
314
|
+
export function repairedSessionLogV1(
|
|
315
|
+
sessionId: string,
|
|
316
|
+
latest: readonly SessionEvent[],
|
|
317
|
+
): SessionEvent[] | undefined {
|
|
318
|
+
if (!hasUnclosedTurnV1(latest)) return undefined;
|
|
319
|
+
let repaired: SessionEvent[] = [];
|
|
320
|
+
const closeOpenTurn = (): boolean => {
|
|
321
|
+
if (repaired.length === 0 || !hasOrphanedOpenTurnV1(repaired)) return true;
|
|
322
|
+
try {
|
|
323
|
+
const session = new Session(sessionId, () => {}, repaired);
|
|
324
|
+
session.reconcileInterrupted();
|
|
325
|
+
repaired = [...session.events];
|
|
326
|
+
} catch {
|
|
327
|
+
return false;
|
|
328
|
+
}
|
|
329
|
+
return !hasOrphanedOpenTurnV1(repaired);
|
|
330
|
+
};
|
|
331
|
+
for (const event of latest) {
|
|
332
|
+
if (event.type === "turn/start" && !closeOpenTurn()) return undefined;
|
|
333
|
+
repaired.push({ ...event, seq: repaired.length });
|
|
334
|
+
}
|
|
335
|
+
if (!closeOpenTurn()) return undefined;
|
|
336
|
+
return repaired;
|
|
337
|
+
}
|
|
338
|
+
|
|
225
339
|
export function eventsForFailedRun(
|
|
226
340
|
durableRun: { events: SessionEvent[] } | undefined,
|
|
227
341
|
error: unknown,
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
// The exact sequence that wedged a Bot forever.
|
|
2
|
+
//
|
|
3
|
+
// A Turn interrupted mid-answer unwinds without writing a `turn/end`: its model
|
|
4
|
+
// request has no durable outcome and a `turn/end` would claim to know how it
|
|
5
|
+
// ended. Right while the run might resume; wrong once it will not. The
|
|
6
|
+
// settlement committed those events as they stood, so the durable session log
|
|
7
|
+
// ended inside an open turn, and every later message on that Bot failed
|
|
8
|
+
// validation with `turn 2 started while turn 1 is open` — printed verbatim into
|
|
9
|
+
// the person's next bubble, forever.
|
|
10
|
+
import { describe, expect, test } from "bun:test";
|
|
11
|
+
import {
|
|
12
|
+
Session,
|
|
13
|
+
type SessionEvent,
|
|
14
|
+
validateToolOccurrenceJournal,
|
|
15
|
+
} from "@frockbot/kernel-contracts";
|
|
16
|
+
import { MemoryStorage } from "./memory-storage.fixture.ts";
|
|
17
|
+
import { createStoredRunCodecV1, type StoredRunV1 } from "./run-records.ts";
|
|
18
|
+
import {
|
|
19
|
+
cancelStoredRun,
|
|
20
|
+
failStoredRun,
|
|
21
|
+
supersedeStoredRun,
|
|
22
|
+
} from "./run-terminal.ts";
|
|
23
|
+
|
|
24
|
+
const codec = createStoredRunCodecV1<null>({
|
|
25
|
+
decodeRunId: (value) => String(value),
|
|
26
|
+
decodeConfigurationSnapshot: () => null,
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
const SESSION_ID = "user-1:primary";
|
|
30
|
+
|
|
31
|
+
const KEYS = {
|
|
32
|
+
run: "run:run-1",
|
|
33
|
+
activeRun: "active-run",
|
|
34
|
+
latestEvents: "latest-events",
|
|
35
|
+
notificationPrefix: "notification:",
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* A Turn stopped after its model request went uncertain: `turn/start`,
|
|
40
|
+
* `step/start`, a `user/message`, a `model/request`, and a
|
|
41
|
+
* `model/reconciliation-required` — and then nothing. This is what the Agent
|
|
42
|
+
* loop leaves behind when it unwinds on an abort.
|
|
43
|
+
*/
|
|
44
|
+
function interruptedJournal(): SessionEvent[] {
|
|
45
|
+
const events: SessionEvent[] = [];
|
|
46
|
+
const session = new Session(SESSION_ID, (envelope) => {
|
|
47
|
+
events.push(envelope.event);
|
|
48
|
+
});
|
|
49
|
+
session.appendBatch([
|
|
50
|
+
{ type: "turn/start", turn: 1 },
|
|
51
|
+
{ type: "step/start", turn: 1, step: 1 },
|
|
52
|
+
{
|
|
53
|
+
type: "user/message",
|
|
54
|
+
turn: 1,
|
|
55
|
+
step: 1,
|
|
56
|
+
messageId: "message-1",
|
|
57
|
+
text: "hello",
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
type: "model/request",
|
|
61
|
+
turn: 1,
|
|
62
|
+
step: 1,
|
|
63
|
+
request: {
|
|
64
|
+
requestId: "request-1",
|
|
65
|
+
provider: "flock-ai",
|
|
66
|
+
model: "@flock/auto",
|
|
67
|
+
system: "",
|
|
68
|
+
messages: [],
|
|
69
|
+
tools: [],
|
|
70
|
+
},
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
type: "model/reconciliation-required",
|
|
74
|
+
turn: 1,
|
|
75
|
+
step: 1,
|
|
76
|
+
requestId: "request-1",
|
|
77
|
+
reason: "Model response outcome is uncertain after cancellation",
|
|
78
|
+
},
|
|
79
|
+
]);
|
|
80
|
+
return events;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function storedRun(
|
|
84
|
+
events: SessionEvent[],
|
|
85
|
+
intent: Partial<StoredRunV1<null>>,
|
|
86
|
+
): StoredRunV1<null> {
|
|
87
|
+
return {
|
|
88
|
+
runId: "run-1",
|
|
89
|
+
commandFingerprint: "fingerprint-1",
|
|
90
|
+
sessionId: SESSION_ID,
|
|
91
|
+
acceptedAt: "2026-09-03T00:00:00.000Z",
|
|
92
|
+
input: "hello",
|
|
93
|
+
events,
|
|
94
|
+
effectAdmissions: [],
|
|
95
|
+
status: "running",
|
|
96
|
+
phase: "executing",
|
|
97
|
+
compositionGenerationId: "generation-1",
|
|
98
|
+
configurationSnapshot: null,
|
|
99
|
+
previousEventCount: 0,
|
|
100
|
+
...intent,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function settled(
|
|
105
|
+
intent: Partial<StoredRunV1<null>>,
|
|
106
|
+
settle: (storage: MemoryStorage, events: SessionEvent[]) => Promise<unknown>,
|
|
107
|
+
): Promise<{ storage: MemoryStorage; latest: SessionEvent[] }> {
|
|
108
|
+
const storage = new MemoryStorage();
|
|
109
|
+
const events = interruptedJournal();
|
|
110
|
+
await storage.put({
|
|
111
|
+
[KEYS.activeRun]: "run-1",
|
|
112
|
+
[KEYS.run]: storedRun(events, intent),
|
|
113
|
+
[KEYS.latestEvents]: events,
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
await settle(storage, events);
|
|
117
|
+
|
|
118
|
+
return {
|
|
119
|
+
storage,
|
|
120
|
+
latest: storage.values.get(KEYS.latestEvents) as SessionEvent[],
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** What the next Turn does: start turn 2 on the log the settlement left. */
|
|
125
|
+
function admitNextTurn(latest: SessionEvent[]): void {
|
|
126
|
+
const session = new Session(SESSION_ID, () => {}, latest);
|
|
127
|
+
session.append({ type: "turn/start", turn: 2 });
|
|
128
|
+
validateToolOccurrenceJournal(session.events);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
describe("settling a Turn interrupted mid-answer", () => {
|
|
132
|
+
test("a superseded run leaves a log the next Turn can start on", async () => {
|
|
133
|
+
const { latest, storage } = await settled(
|
|
134
|
+
{ supersededAt: "2026-09-03T00:01:00.000Z", supersededBy: "run-2" },
|
|
135
|
+
(store, events) =>
|
|
136
|
+
supersedeStoredRun(codec, store, KEYS, "run-1", [], events),
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
// Before the fix this threw "turn 2 started while turn 1 is open", and
|
|
140
|
+
// every later message on this Bot answered 500 with that sentence.
|
|
141
|
+
expect(() => admitNextTurn(latest)).not.toThrow();
|
|
142
|
+
expect(latest.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
|
|
143
|
+
// The settled record carries the same closed account, not a different one.
|
|
144
|
+
const record = storage.values.get(KEYS.run) as StoredRunV1<null>;
|
|
145
|
+
expect(record.status).toBe("superseded");
|
|
146
|
+
expect(record.events.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("a stopped run leaves a log the next Turn can start on", async () => {
|
|
150
|
+
const { latest } = await settled(
|
|
151
|
+
{ stopRequestedAt: "2026-09-03T00:01:00.000Z" },
|
|
152
|
+
(store, events) =>
|
|
153
|
+
cancelStoredRun(codec, store, KEYS, "run-1", [], events),
|
|
154
|
+
);
|
|
155
|
+
|
|
156
|
+
expect(() => admitNextTurn(latest)).not.toThrow();
|
|
157
|
+
expect(latest.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test("a failed run leaves a log the next Turn can start on", async () => {
|
|
161
|
+
const { latest } = await settled({}, (store, events) =>
|
|
162
|
+
failStoredRun(
|
|
163
|
+
codec,
|
|
164
|
+
store,
|
|
165
|
+
KEYS,
|
|
166
|
+
"run-1",
|
|
167
|
+
[],
|
|
168
|
+
events,
|
|
169
|
+
"the service restarted",
|
|
170
|
+
),
|
|
171
|
+
);
|
|
172
|
+
|
|
173
|
+
expect(() => admitNextTurn(latest)).not.toThrow();
|
|
174
|
+
expect(latest.at(-1)).toMatchObject({ type: "turn/end", turn: 1 });
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
test("a Turn that closed itself is not closed twice", async () => {
|
|
178
|
+
const storage = new MemoryStorage();
|
|
179
|
+
const events: SessionEvent[] = [];
|
|
180
|
+
const session = new Session(SESSION_ID, (envelope) => {
|
|
181
|
+
events.push(envelope.event);
|
|
182
|
+
});
|
|
183
|
+
session.appendBatch([
|
|
184
|
+
{ type: "turn/start", turn: 1 },
|
|
185
|
+
{ type: "step/start", turn: 1, step: 1 },
|
|
186
|
+
{ type: "step/end", turn: 1, step: 1, outcome: "cancelled" },
|
|
187
|
+
{ type: "turn/end", turn: 1, outcome: "cancelled" },
|
|
188
|
+
]);
|
|
189
|
+
await storage.put({
|
|
190
|
+
[KEYS.activeRun]: "run-1",
|
|
191
|
+
[KEYS.run]: storedRun(events, {
|
|
192
|
+
stopRequestedAt: "2026-09-03T00:01:00.000Z",
|
|
193
|
+
}),
|
|
194
|
+
[KEYS.latestEvents]: events,
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
await cancelStoredRun(codec, storage, KEYS, "run-1", [], events);
|
|
198
|
+
|
|
199
|
+
const latest = storage.values.get(KEYS.latestEvents) as SessionEvent[];
|
|
200
|
+
expect(latest.filter((event) => event.type === "turn/end")).toHaveLength(1);
|
|
201
|
+
expect(() => admitNextTurn(latest)).not.toThrow();
|
|
202
|
+
});
|
|
203
|
+
});
|
package/src/run-terminal.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
decodeSessionEvent,
|
|
3
|
+
Session,
|
|
3
4
|
type SessionEvent,
|
|
4
5
|
} from "@frockbot/kernel-contracts";
|
|
5
6
|
import type {
|
|
@@ -7,6 +8,54 @@ import type {
|
|
|
7
8
|
StoredRunCodecV1,
|
|
8
9
|
StoredRunV1,
|
|
9
10
|
} from "./run-records.js";
|
|
11
|
+
import { repairedSessionLogV1 } from "./run-recovery.js";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The events a terminal settlement commits, with any Turn they were left
|
|
15
|
+
* inside closed.
|
|
16
|
+
*
|
|
17
|
+
* A Turn interrupted mid-answer unwinds without writing a `turn/end`: the
|
|
18
|
+
* outcome of its model request is unknown, and a `turn/end` would claim to know
|
|
19
|
+
* how it ended. That is right while the run might still resume — and wrong the
|
|
20
|
+
* moment it will not. Settling one and committing its events as they stand left
|
|
21
|
+
* an open turn in the durable session log, so `turn N started while turn N-1 is
|
|
22
|
+
* open` refused every later message on that Bot, forever, and printed itself
|
|
23
|
+
* verbatim into the person's next bubble.
|
|
24
|
+
*
|
|
25
|
+
* So closing the turn happens exactly here: at the one point where the run is
|
|
26
|
+
* certainly not resuming. `reconcileInterrupted` writes the same repair the
|
|
27
|
+
* recovery path already writes — every unresolved tool occurrence closed as
|
|
28
|
+
* `interrupted`, then `step/end` and `turn/end` — so the settled log is a
|
|
29
|
+
* complete account and the next Turn starts on a closed one.
|
|
30
|
+
*
|
|
31
|
+
* A log that is already closed produces no repairs, and one too malformed to
|
|
32
|
+
* reconcile is left exactly as it is: repairing that blindly would invent
|
|
33
|
+
* history.
|
|
34
|
+
*/
|
|
35
|
+
function settledEventsV1(
|
|
36
|
+
sessionId: string,
|
|
37
|
+
previous: readonly SessionEvent[],
|
|
38
|
+
events: readonly SessionEvent[],
|
|
39
|
+
): { events: SessionEvent[]; latestEvents: SessionEvent[] } {
|
|
40
|
+
const decoded = events.map(decodeSessionEvent);
|
|
41
|
+
const latest = [...previous, ...decoded].map(decodeSessionEvent);
|
|
42
|
+
let repairs: SessionEvent[] = [];
|
|
43
|
+
try {
|
|
44
|
+
repairs = new Session(sessionId, () => {}, latest).reconcileInterrupted();
|
|
45
|
+
} catch {
|
|
46
|
+
repairs = [];
|
|
47
|
+
}
|
|
48
|
+
const settled = [...latest, ...repairs];
|
|
49
|
+
// A Turn abandoned earlier in the log cannot be closed by appending, and a
|
|
50
|
+
// settlement that only appends leaves it open forever. The run's own events
|
|
51
|
+
// are committed as they stand — that record is this run's account, not the
|
|
52
|
+
// conversation's — while the forward log is repaired in place so the next
|
|
53
|
+
// Turn starts on a log that reads as a complete history.
|
|
54
|
+
return {
|
|
55
|
+
events: [...decoded, ...repairs],
|
|
56
|
+
latestEvents: repairedSessionLogV1(sessionId, settled) ?? settled,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
10
59
|
|
|
11
60
|
export interface RunTerminalStorage {
|
|
12
61
|
get<T>(key: string): Promise<T | undefined>;
|
|
@@ -83,7 +132,8 @@ export async function supersedeStoredRun<Snapshot>(
|
|
|
83
132
|
if (!run.supersededAt) {
|
|
84
133
|
throw new Error(`run "${runId}" has no durable supersede intent`);
|
|
85
134
|
}
|
|
86
|
-
const
|
|
135
|
+
const settledEvents = settledEventsV1(run.sessionId, previous, events);
|
|
136
|
+
const decodedEvents = settledEvents.events;
|
|
87
137
|
const { responseText: _text, failure: _failure, ...settled } = run;
|
|
88
138
|
// A run superseded while still queued never started, never appended an
|
|
89
139
|
// event, and never spoke: it settles as a record on its own and leaves both
|
|
@@ -105,9 +155,7 @@ export async function supersedeStoredRun<Snapshot>(
|
|
|
105
155
|
...(queued
|
|
106
156
|
? {}
|
|
107
157
|
: {
|
|
108
|
-
[keys.latestEvents]: structuredClone(
|
|
109
|
-
[...previous, ...decodedEvents].map(decodeSessionEvent),
|
|
110
|
-
),
|
|
158
|
+
[keys.latestEvents]: structuredClone(settledEvents.latestEvents),
|
|
111
159
|
}),
|
|
112
160
|
};
|
|
113
161
|
if (packageRecords && !queued) {
|
|
@@ -226,8 +274,9 @@ export async function cancelStoredRun<Snapshot>(
|
|
|
226
274
|
if (!run.stopRequestedAt) {
|
|
227
275
|
throw new Error(`run "${runId}" has no durable stop intent`);
|
|
228
276
|
}
|
|
229
|
-
const
|
|
230
|
-
const
|
|
277
|
+
const settledEvents = settledEventsV1(run.sessionId, previous, events);
|
|
278
|
+
const decodedEvents = settledEvents.events;
|
|
279
|
+
const latestEvents = settledEvents.latestEvents;
|
|
231
280
|
const { responseText: _text, failure: _failure, ...settled } = run;
|
|
232
281
|
const cancelled = codec.require({
|
|
233
282
|
...settled,
|
|
@@ -278,8 +327,9 @@ export async function failStoredRun<Snapshot>(
|
|
|
278
327
|
supersededRecords,
|
|
279
328
|
);
|
|
280
329
|
}
|
|
281
|
-
const
|
|
282
|
-
const
|
|
330
|
+
const settledEvents = settledEventsV1(run.sessionId, previous, events);
|
|
331
|
+
const decodedEvents = settledEvents.events;
|
|
332
|
+
const latestEvents = settledEvents.latestEvents;
|
|
283
333
|
const failed = codec.require({
|
|
284
334
|
...run,
|
|
285
335
|
events: decodedEvents,
|
package/src/storage-keys.ts
CHANGED
|
@@ -135,3 +135,18 @@ export function storedRunAdmissionFences(input: unknown): string[] {
|
|
|
135
135
|
export function workspaceSyncEffectKey(effectId: string): string {
|
|
136
136
|
return `${WORKSPACE_SYNC_EFFECT_PREFIX}${effectId}`;
|
|
137
137
|
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The conversation the Bot's chat Session is currently on.
|
|
141
|
+
*
|
|
142
|
+
* One Bot has one conversational Session at a time. Starting a new
|
|
143
|
+
* conversation ends the current one and begins the next: the durable event log
|
|
144
|
+
* the next Turn derives its request from is empty again, while every Turn the
|
|
145
|
+
* earlier conversations recorded stays durable and readable under its own
|
|
146
|
+
* Session id.
|
|
147
|
+
*/
|
|
148
|
+
export const CONVERSATION_KEY = "conversation";
|
|
149
|
+
/** The conversations this Bot has already ended, newest last. */
|
|
150
|
+
export const CONVERSATION_INDEX_KEY = "conversation-index";
|
|
151
|
+
/** How many ended conversations stay listable. Older ones drop off the list. */
|
|
152
|
+
export const MAX_LISTED_CONVERSATIONS = 64;
|
|
@@ -448,3 +448,41 @@ describe("an admitted Turn re-mounts on its recorded turn type", () => {
|
|
|
448
448
|
});
|
|
449
449
|
});
|
|
450
450
|
});
|
|
451
|
+
|
|
452
|
+
// robustness F18. The composer sends `supersedes` on every send and names
|
|
453
|
+
// whichever run it happened to have observed. A retry of the same send names a
|
|
454
|
+
// different one — or none — and used to be refused as a reused idempotency key.
|
|
455
|
+
describe("a retried send is idempotent whatever run it names", () => {
|
|
456
|
+
const command = {
|
|
457
|
+
userId: "user-1",
|
|
458
|
+
botId: "primary",
|
|
459
|
+
runId: "run-1",
|
|
460
|
+
sessionId: "user-1:primary",
|
|
461
|
+
acceptedAt: "2026-08-31T01:00:00.000Z",
|
|
462
|
+
text: "hello",
|
|
463
|
+
lane: "user" as const,
|
|
464
|
+
};
|
|
465
|
+
|
|
466
|
+
test("the observed run id is not part of the command's identity", () => {
|
|
467
|
+
const first = botTurnCommandFingerprintV1({ ...command, supersedes: {} });
|
|
468
|
+
|
|
469
|
+
expect(
|
|
470
|
+
botTurnCommandFingerprintV1({
|
|
471
|
+
...command,
|
|
472
|
+
supersedes: { runId: "run-0" },
|
|
473
|
+
}),
|
|
474
|
+
).toBe(first);
|
|
475
|
+
expect(
|
|
476
|
+
botTurnCommandFingerprintV1({
|
|
477
|
+
...command,
|
|
478
|
+
supersedes: { runId: "run-99" },
|
|
479
|
+
}),
|
|
480
|
+
).toBe(first);
|
|
481
|
+
});
|
|
482
|
+
|
|
483
|
+
test("but the intent itself still is, so a replay cannot gain one", () => {
|
|
484
|
+
expect(
|
|
485
|
+
botTurnCommandFingerprintV1({ ...command, supersedes: {} }),
|
|
486
|
+
).not.toBe(botTurnCommandFingerprintV1(command));
|
|
487
|
+
});
|
|
488
|
+
});
|