@frockbot/kernel-do 0.3.6 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/authority.ts +323 -39
- package/src/conversations.test.ts +159 -0
- package/src/conversations.ts +126 -0
- package/src/index.ts +1 -0
- package/src/run-failure-bounds.test.ts +46 -0
- package/src/run-records.ts +30 -0
- package/src/run-recovery.ts +70 -0
- package/src/run-terminal.ts +8 -1
- package/src/storage-keys.ts +15 -0
- package/src/turn-supersede.test.ts +207 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@frockbot/kernel-do",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.8",
|
|
4
4
|
"private": false,
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -12,8 +12,8 @@
|
|
|
12
12
|
"typecheck": "tsc --noEmit -p tsconfig.json"
|
|
13
13
|
},
|
|
14
14
|
"dependencies": {
|
|
15
|
-
"@frockbot/kernel-composition": "0.3.
|
|
16
|
-
"@frockbot/kernel-contracts": "0.3.
|
|
15
|
+
"@frockbot/kernel-composition": "0.3.8",
|
|
16
|
+
"@frockbot/kernel-contracts": "0.3.8",
|
|
17
17
|
"cordis": "4.0.0-rc.8"
|
|
18
18
|
},
|
|
19
19
|
"devDependencies": {
|
package/src/authority.ts
CHANGED
|
@@ -12,6 +12,7 @@ import type { CompositionGenerationV1 } from "@frockbot/kernel-composition/gener
|
|
|
12
12
|
import { DurableCompositionStore } from "./composition-store.js";
|
|
13
13
|
import { DurableCompositionFailureLog } from "./composition-failures.js";
|
|
14
14
|
import {
|
|
15
|
+
boundedRunFailureV1,
|
|
15
16
|
botTurnCommandFingerprintV1,
|
|
16
17
|
defaultRunLaneV1,
|
|
17
18
|
storedRunAdmissionV1,
|
|
@@ -35,7 +36,7 @@ import {
|
|
|
35
36
|
latestModelRequestJournalState,
|
|
36
37
|
planBotRunRecovery,
|
|
37
38
|
type ProviderReconcilesV1,
|
|
38
|
-
|
|
39
|
+
repairedSessionLogV1,
|
|
39
40
|
unresolvedModelRequestFailure,
|
|
40
41
|
} from "./run-recovery.js";
|
|
41
42
|
import {
|
|
@@ -43,8 +44,20 @@ import {
|
|
|
43
44
|
BotTurnRecoveryRequiredError,
|
|
44
45
|
BotTurnRefusedError,
|
|
45
46
|
} from "./turn-errors.js";
|
|
47
|
+
import {
|
|
48
|
+
botConversationBaseSessionIdV1,
|
|
49
|
+
conversationSessionIdV1,
|
|
50
|
+
decodeConversationRecordV1,
|
|
51
|
+
decodeStoredConversationV1,
|
|
52
|
+
firstConversationV1,
|
|
53
|
+
type ConversationRecordV1,
|
|
54
|
+
type StoredConversationV1,
|
|
55
|
+
} from "./conversations.js";
|
|
46
56
|
import {
|
|
47
57
|
ACTIVE_RUN_KEY,
|
|
58
|
+
CONVERSATION_INDEX_KEY,
|
|
59
|
+
CONVERSATION_KEY,
|
|
60
|
+
MAX_LISTED_CONVERSATIONS,
|
|
48
61
|
PENDING_RUN_KEY,
|
|
49
62
|
IDENTITY_KEY,
|
|
50
63
|
LATEST_EVENTS_KEY,
|
|
@@ -163,6 +176,16 @@ export const SUPERSEDED_TURN_REASON_V1 = "superseded by a new user message";
|
|
|
163
176
|
/** How many times a queued Turn retries the object before giving up. */
|
|
164
177
|
const MAX_QUEUED_RUN_START_ATTEMPTS = 8;
|
|
165
178
|
|
|
179
|
+
/**
|
|
180
|
+
* The failure a discarded Turn is settled with when recovery finds it.
|
|
181
|
+
*
|
|
182
|
+
* It is never read by anybody: `failStoredRun` routes a run carrying a Stop or
|
|
183
|
+
* supersede intent to `cancelStoredRun`/`supersedeStoredRun`, and both drop the
|
|
184
|
+
* failure — the User's own intent is the outcome, not an error.
|
|
185
|
+
*/
|
|
186
|
+
const DISCARDED_RUN_RECOVERY_FAILURE_V1 =
|
|
187
|
+
"Turn was discarded before recovery could resume it";
|
|
188
|
+
|
|
166
189
|
/**
|
|
167
190
|
* True when this object has already durably decided to throw the Turn away.
|
|
168
191
|
*
|
|
@@ -220,7 +243,8 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
220
243
|
});
|
|
221
244
|
}
|
|
222
245
|
|
|
223
|
-
async run(
|
|
246
|
+
async run(input: OwnedBotTurnCommand): Promise<BotTurnCompletion> {
|
|
247
|
+
const command = await this.conversationScopedCommand(input);
|
|
224
248
|
await this.assertMatchingRunCommand(command);
|
|
225
249
|
// Recovering whatever this object was left holding must never decide the
|
|
226
250
|
// fate of a new command. `recoverActiveRun` executes the *previous* Turn
|
|
@@ -368,9 +392,15 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
368
392
|
if (await transaction.get<string>(ACTIVE_RUN_KEY)) {
|
|
369
393
|
return "blocked" as const;
|
|
370
394
|
}
|
|
371
|
-
const
|
|
395
|
+
const storedEvents = (
|
|
372
396
|
(await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
|
|
373
397
|
).map(decodeSessionEvent);
|
|
398
|
+
// A queued Turn was admitted while another was executing, so admission
|
|
399
|
+
// could not repair the log: something was still entitled to close that
|
|
400
|
+
// Turn. Here the active-run marker is gone and nothing is, so the same
|
|
401
|
+
// repair applies before this Turn starts on it.
|
|
402
|
+
const repaired = repairedSessionLogV1(run.sessionId, storedEvents);
|
|
403
|
+
const latestEvents = repaired ?? storedEvents;
|
|
374
404
|
const promoted = this.codec.require({
|
|
375
405
|
...run,
|
|
376
406
|
phase: "admitted",
|
|
@@ -379,6 +409,13 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
379
409
|
await transaction.put({
|
|
380
410
|
[key]: structuredClone(promoted),
|
|
381
411
|
[ACTIVE_RUN_KEY]: runId,
|
|
412
|
+
...(repaired
|
|
413
|
+
? {
|
|
414
|
+
[LATEST_EVENTS_KEY]: structuredClone(
|
|
415
|
+
repaired.map(decodeSessionEvent),
|
|
416
|
+
),
|
|
417
|
+
}
|
|
418
|
+
: {}),
|
|
382
419
|
});
|
|
383
420
|
await transaction.delete(PENDING_RUN_KEY);
|
|
384
421
|
await this.refreshRecoveryAlarm(transaction);
|
|
@@ -451,10 +488,43 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
451
488
|
`Reconciliation was explicitly abandoned: ${failure}`,
|
|
452
489
|
);
|
|
453
490
|
}
|
|
491
|
+
// "Try again" that ends in a settled run is a *successful* abandon, not a
|
|
492
|
+
// failed request. Rethrowing here made the button answer 409 and left the
|
|
493
|
+
// browser reading a run it thought had not moved — and the read that
|
|
494
|
+
// followed 500'd on the half-repaired record. The run is durable and
|
|
495
|
+
// terminal by this point, and its own record says why it ended, so the
|
|
496
|
+
// caller is handed that record and reads the reason from the transcript.
|
|
497
|
+
const settled = await this.settledReconciliationResult(runId);
|
|
498
|
+
if (settled) return settled;
|
|
454
499
|
throw error;
|
|
455
500
|
}
|
|
456
501
|
}
|
|
457
502
|
|
|
503
|
+
/**
|
|
504
|
+
* The completion an abandoned reconciliation reports once the run it was
|
|
505
|
+
* resolving has reached a terminal state — whatever that state turned out to
|
|
506
|
+
* be. Anything still open is not this method's to answer for.
|
|
507
|
+
*/
|
|
508
|
+
private async settledReconciliationResult(
|
|
509
|
+
runId: string,
|
|
510
|
+
): Promise<BotTurnCompletion | undefined> {
|
|
511
|
+
const run = this.codec.optional(
|
|
512
|
+
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
513
|
+
);
|
|
514
|
+
if (
|
|
515
|
+
run?.status !== "failed" &&
|
|
516
|
+
run?.status !== "cancelled" &&
|
|
517
|
+
run?.status !== "superseded"
|
|
518
|
+
) {
|
|
519
|
+
return undefined;
|
|
520
|
+
}
|
|
521
|
+
return {
|
|
522
|
+
runId,
|
|
523
|
+
text: run.responseText ?? "",
|
|
524
|
+
events: structuredClone(run.events),
|
|
525
|
+
};
|
|
526
|
+
}
|
|
527
|
+
|
|
458
528
|
private withNotification(
|
|
459
529
|
snapshot: Snapshot,
|
|
460
530
|
result: BotTurnCompletion,
|
|
@@ -546,7 +616,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
546
616
|
throw new Error(message);
|
|
547
617
|
}
|
|
548
618
|
await this.failRun(command.runId, previous, events, message);
|
|
549
|
-
const settled = await this.
|
|
619
|
+
const settled = await this.discardedRunResult(command.runId);
|
|
550
620
|
if (settled) return settled;
|
|
551
621
|
throw new Error(message);
|
|
552
622
|
} finally {
|
|
@@ -557,17 +627,25 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
557
627
|
}
|
|
558
628
|
|
|
559
629
|
/**
|
|
560
|
-
* The completion a Turn
|
|
561
|
-
* failure: the Turn settled durably,
|
|
562
|
-
* and its caller reads the rest of
|
|
630
|
+
* The completion a discarded Turn reports — one the User stopped, or one a
|
|
631
|
+
* later message replaced. Neither is a failure: the Turn settled durably,
|
|
632
|
+
* keeping everything it had already sent, and its caller reads the rest of
|
|
633
|
+
* the conversation from durable state.
|
|
634
|
+
*
|
|
635
|
+
* Stop used to be missing from here, so the long-lived `POST /turns` the
|
|
636
|
+
* composer was still holding open answered 500 the instant Stop was pressed:
|
|
637
|
+
* the UI said "You stopped this." and the console said the send had failed.
|
|
638
|
+
* A Turn the person stopped on purpose is the most ordinary outcome there is.
|
|
563
639
|
*/
|
|
564
|
-
private async
|
|
640
|
+
private async discardedRunResult(
|
|
565
641
|
runId: string,
|
|
566
642
|
): Promise<BotTurnCompletion | undefined> {
|
|
567
643
|
const run = this.codec.optional(
|
|
568
644
|
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
569
645
|
);
|
|
570
|
-
if (run?.status !== "superseded")
|
|
646
|
+
if (run?.status !== "superseded" && run?.status !== "cancelled") {
|
|
647
|
+
return undefined;
|
|
648
|
+
}
|
|
571
649
|
return { runId, text: "", events: structuredClone(run.events) };
|
|
572
650
|
}
|
|
573
651
|
|
|
@@ -582,7 +660,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
582
660
|
const run = this.codec.optional(
|
|
583
661
|
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
584
662
|
);
|
|
585
|
-
if (run?.status === "superseded") {
|
|
663
|
+
if (run?.status === "superseded" || run?.status === "cancelled") {
|
|
586
664
|
return { runId, text: "", events: structuredClone(run.events) };
|
|
587
665
|
}
|
|
588
666
|
if (run?.status !== "completed") return undefined;
|
|
@@ -679,7 +757,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
679
757
|
throw new Error(message);
|
|
680
758
|
}
|
|
681
759
|
await this.failRun(run.runId, previous, events, message);
|
|
682
|
-
const settled = await this.
|
|
760
|
+
const settled = await this.discardedRunResult(run.runId);
|
|
683
761
|
if (settled) return settled;
|
|
684
762
|
throw new Error(message);
|
|
685
763
|
} finally {
|
|
@@ -717,10 +795,11 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
717
795
|
`Turn idempotency key "${runId}" was reused for a different command`,
|
|
718
796
|
);
|
|
719
797
|
}
|
|
720
|
-
// A Turn another user message took the place of
|
|
721
|
-
// not a failure: it settled durably, said
|
|
722
|
-
// and the caller reads the rest from durable
|
|
723
|
-
|
|
798
|
+
// A Turn the User stopped, or one another user message took the place of,
|
|
799
|
+
// is an ordinary outcome and not a failure: it settled durably, said
|
|
800
|
+
// whatever it had already said, and the caller reads the rest from durable
|
|
801
|
+
// state. A retry of either replays that settlement rather than refusing.
|
|
802
|
+
if (run.status === "superseded" || run.status === "cancelled") {
|
|
724
803
|
return {
|
|
725
804
|
runId,
|
|
726
805
|
text: "",
|
|
@@ -855,7 +934,30 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
855
934
|
return;
|
|
856
935
|
}
|
|
857
936
|
}
|
|
858
|
-
|
|
937
|
+
// An alarm has no caller. A rejection here is an uncaught exception in the
|
|
938
|
+
// object, and in the dev Worker it took the whole process down: a Stop left
|
|
939
|
+
// a run whose model outcome was uncertain, recovery re-entered it,
|
|
940
|
+
// `executeAdmittedRun` rethrew after recording the failure durably, and
|
|
941
|
+
// wrangler exited mid-run for every agent sharing the stack.
|
|
942
|
+
//
|
|
943
|
+
// Nothing about that throw is actionable here. Recovery has already written
|
|
944
|
+
// whatever it decided to durable storage before it rethrew, so the only
|
|
945
|
+
// thing left to do is record the reason and make sure the object still has
|
|
946
|
+
// a deadline — the re-arm is deliberately in a `finally`, because a failed
|
|
947
|
+
// recovery is exactly the case where the *next* firing matters most.
|
|
948
|
+
try {
|
|
949
|
+
await this.recoverActiveRun();
|
|
950
|
+
} catch (error) {
|
|
951
|
+
console.error(
|
|
952
|
+
`Bot run recovery alarm failed: ${
|
|
953
|
+
error instanceof Error ? error.message : String(error)
|
|
954
|
+
}`,
|
|
955
|
+
);
|
|
956
|
+
} finally {
|
|
957
|
+
await this.ctx.storage
|
|
958
|
+
.transaction((transaction) => this.refreshRecoveryAlarm(transaction))
|
|
959
|
+
.catch(() => undefined);
|
|
960
|
+
}
|
|
859
961
|
}
|
|
860
962
|
|
|
861
963
|
/** Active run id, for Package projections of durable run state. */
|
|
@@ -863,6 +965,158 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
863
965
|
return this.ctx.storage.get<string>(ACTIVE_RUN_KEY);
|
|
864
966
|
}
|
|
865
967
|
|
|
968
|
+
/** The conversation this Bot's chat Session is on. */
|
|
969
|
+
async readConversation(): Promise<StoredConversationV1> {
|
|
970
|
+
return (
|
|
971
|
+
decodeStoredConversationV1(
|
|
972
|
+
await this.ctx.storage.get<unknown>(CONVERSATION_KEY),
|
|
973
|
+
) ?? firstConversationV1(new Date().toISOString())
|
|
974
|
+
);
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
/**
|
|
978
|
+
* The conversations this Bot has had, newest first, the current one included.
|
|
979
|
+
*
|
|
980
|
+
* Ended conversations are listed from a bounded index rather than
|
|
981
|
+
* reconstructed from the run log: the run index is paged and a conversation
|
|
982
|
+
* with no surviving runs is still a conversation the User had.
|
|
983
|
+
*/
|
|
984
|
+
async listConversations(
|
|
985
|
+
identity?: BotIdentity,
|
|
986
|
+
): Promise<ConversationRecordV1[]> {
|
|
987
|
+
const known =
|
|
988
|
+
identity ?? (await this.ctx.storage.get<BotIdentity>(IDENTITY_KEY));
|
|
989
|
+
if (!known) return [];
|
|
990
|
+
const base = botConversationBaseSessionIdV1(known);
|
|
991
|
+
const current = await this.readConversation();
|
|
992
|
+
const ended = (
|
|
993
|
+
(await this.ctx.storage.get<unknown[]>(CONVERSATION_INDEX_KEY)) ?? []
|
|
994
|
+
).flatMap((entry) => {
|
|
995
|
+
try {
|
|
996
|
+
return [decodeConversationRecordV1(entry)];
|
|
997
|
+
} catch {
|
|
998
|
+
// One unreadable record is skipped, never a list that throws: a
|
|
999
|
+
// conversation you cannot name must not hide the ones you can.
|
|
1000
|
+
return [];
|
|
1001
|
+
}
|
|
1002
|
+
});
|
|
1003
|
+
return [
|
|
1004
|
+
{
|
|
1005
|
+
schemaVersion: 1 as const,
|
|
1006
|
+
sessionId: conversationSessionIdV1(base, current.ordinal),
|
|
1007
|
+
ordinal: current.ordinal,
|
|
1008
|
+
startedAt: current.startedAt,
|
|
1009
|
+
},
|
|
1010
|
+
...ended,
|
|
1011
|
+
].sort((left, right) => right.ordinal - left.ordinal);
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
/**
|
|
1015
|
+
* The Session id this Bot's chat Turns are recording right now, or
|
|
1016
|
+
* `undefined` before the object has admitted anything and learned its
|
|
1017
|
+
* identity. A reader that has to say which Turns are "this conversation"
|
|
1018
|
+
* asks here rather than reconstructing the id.
|
|
1019
|
+
*/
|
|
1020
|
+
async readConversationSessionId(): Promise<string | undefined> {
|
|
1021
|
+
const identity = await this.ctx.storage.get<BotIdentity>(IDENTITY_KEY);
|
|
1022
|
+
if (!identity) return undefined;
|
|
1023
|
+
const conversation = await this.readConversation();
|
|
1024
|
+
return conversationSessionIdV1(
|
|
1025
|
+
botConversationBaseSessionIdV1(identity),
|
|
1026
|
+
conversation.ordinal,
|
|
1027
|
+
);
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
/**
|
|
1031
|
+
* Ends the current conversation and starts the next one.
|
|
1032
|
+
*
|
|
1033
|
+
* The durable event log the next Turn derives its request from is emptied,
|
|
1034
|
+
* so history stops growing without bound; the runs of the conversation just
|
|
1035
|
+
* ended keep their events and their Session id and stay readable. Refused
|
|
1036
|
+
* while a Turn is admitted: the log a running Turn is appending to is not
|
|
1037
|
+
* something a click may pull out from under it.
|
|
1038
|
+
*/
|
|
1039
|
+
async startConversation(
|
|
1040
|
+
identity: BotIdentity,
|
|
1041
|
+
): Promise<ConversationRecordV1> {
|
|
1042
|
+
await this.assertIdentity(identity);
|
|
1043
|
+
await this.recoverActiveRun();
|
|
1044
|
+
const base = botConversationBaseSessionIdV1(identity);
|
|
1045
|
+
return this.ctx.storage.transaction(async (transaction) => {
|
|
1046
|
+
const active = await transaction.get<string>(ACTIVE_RUN_KEY);
|
|
1047
|
+
const pending = await transaction.get<string>(PENDING_RUN_KEY);
|
|
1048
|
+
if (active || pending) {
|
|
1049
|
+
throw new Error(
|
|
1050
|
+
"This Bot is still working on a Turn. Wait for it to finish, then start a new conversation.",
|
|
1051
|
+
);
|
|
1052
|
+
}
|
|
1053
|
+
const current =
|
|
1054
|
+
decodeStoredConversationV1(
|
|
1055
|
+
await transaction.get<unknown>(CONVERSATION_KEY),
|
|
1056
|
+
) ?? firstConversationV1(new Date().toISOString());
|
|
1057
|
+
const endedAt = new Date().toISOString();
|
|
1058
|
+
const ended = (
|
|
1059
|
+
(await transaction.get<unknown[]>(CONVERSATION_INDEX_KEY)) ?? []
|
|
1060
|
+
).flatMap((entry) => {
|
|
1061
|
+
try {
|
|
1062
|
+
return [decodeConversationRecordV1(entry)];
|
|
1063
|
+
} catch {
|
|
1064
|
+
return [];
|
|
1065
|
+
}
|
|
1066
|
+
});
|
|
1067
|
+
const next: StoredConversationV1 = {
|
|
1068
|
+
schemaVersion: 1,
|
|
1069
|
+
ordinal: current.ordinal + 1,
|
|
1070
|
+
startedAt: endedAt,
|
|
1071
|
+
};
|
|
1072
|
+
await transaction.put({
|
|
1073
|
+
[CONVERSATION_KEY]: next,
|
|
1074
|
+
[CONVERSATION_INDEX_KEY]: [
|
|
1075
|
+
{
|
|
1076
|
+
schemaVersion: 1 as const,
|
|
1077
|
+
sessionId: conversationSessionIdV1(base, current.ordinal),
|
|
1078
|
+
ordinal: current.ordinal,
|
|
1079
|
+
startedAt: current.startedAt,
|
|
1080
|
+
endedAt,
|
|
1081
|
+
},
|
|
1082
|
+
...ended,
|
|
1083
|
+
]
|
|
1084
|
+
.sort((left, right) => right.ordinal - left.ordinal)
|
|
1085
|
+
.slice(0, MAX_LISTED_CONVERSATIONS),
|
|
1086
|
+
// The next Turn derives its messages from an empty log. Nothing is
|
|
1087
|
+
// deleted: `run:<id>` still holds every event of every Turn.
|
|
1088
|
+
[LATEST_EVENTS_KEY]: [],
|
|
1089
|
+
});
|
|
1090
|
+
return {
|
|
1091
|
+
schemaVersion: 1 as const,
|
|
1092
|
+
sessionId: conversationSessionIdV1(base, next.ordinal),
|
|
1093
|
+
ordinal: next.ordinal,
|
|
1094
|
+
startedAt: next.startedAt,
|
|
1095
|
+
};
|
|
1096
|
+
});
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
/**
|
|
1100
|
+
* The command as this object's durable conversation state addresses it.
|
|
1101
|
+
*
|
|
1102
|
+
* A client names the Bot's conversational Session by its base id and knows
|
|
1103
|
+
* nothing about conversations; which conversation that is, is durable state
|
|
1104
|
+
* here. Every other Session id — a Routine's `routine:<id>`, a subagent's —
|
|
1105
|
+
* is left exactly as its producer wrote it.
|
|
1106
|
+
*/
|
|
1107
|
+
private async conversationScopedCommand(
|
|
1108
|
+
command: OwnedBotTurnCommand,
|
|
1109
|
+
): Promise<OwnedBotTurnCommand> {
|
|
1110
|
+
const base = botConversationBaseSessionIdV1(command);
|
|
1111
|
+
if (command.sessionId !== base) return command;
|
|
1112
|
+
const conversation = await this.readConversation();
|
|
1113
|
+
if (conversation.ordinal <= 1) return command;
|
|
1114
|
+
return {
|
|
1115
|
+
...command,
|
|
1116
|
+
sessionId: conversationSessionIdV1(base, conversation.ordinal),
|
|
1117
|
+
};
|
|
1118
|
+
}
|
|
1119
|
+
|
|
866
1120
|
/** Durable run record, unchecked against its lookup key. */
|
|
867
1121
|
async readStoredRun(
|
|
868
1122
|
runId: string,
|
|
@@ -1072,11 +1326,18 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1072
1326
|
const stillOwned =
|
|
1073
1327
|
activeRun?.status === "running" ||
|
|
1074
1328
|
activeRun?.status === "reconciliation-required";
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1329
|
+
//
|
|
1330
|
+
// The repair rewrites the whole log rather than appending to it: by the
|
|
1331
|
+
// time anyone notices, the abandoned Turn is usually no longer the last
|
|
1332
|
+
// thing in the log. Each refused message journals its own `turn/start`
|
|
1333
|
+
// before it assembles the request that discovers the breakage, and its
|
|
1334
|
+
// `finally` writes the matching `turn/end`, so the log ends closed with
|
|
1335
|
+
// the abandoned Turn still open behind it. Appending cannot close that.
|
|
1336
|
+
const repaired = stillOwned
|
|
1337
|
+
? undefined
|
|
1338
|
+
: repairedSessionLogV1(command.sessionId, storedEvents);
|
|
1339
|
+
const latestEvents = repaired ?? storedEvents;
|
|
1340
|
+
if (repaired) {
|
|
1080
1341
|
await transaction.put(
|
|
1081
1342
|
LATEST_EVENTS_KEY,
|
|
1082
1343
|
structuredClone(latestEvents.map(decodeSessionEvent)),
|
|
@@ -1333,6 +1594,20 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1333
1594
|
});
|
|
1334
1595
|
}
|
|
1335
1596
|
|
|
1597
|
+
/**
|
|
1598
|
+
* Settles a run `failed` on a reason the authority composed from an error.
|
|
1599
|
+
*
|
|
1600
|
+
* The reason is bounded on the way in because nothing upstream bounds an
|
|
1601
|
+
* error's `message`: a provider that echoes the request back produced one far
|
|
1602
|
+
* past what the record allows, the settlement wrote it anyway, and every
|
|
1603
|
+
* later read of that run threw — so a Turn that failed once went on to 500
|
|
1604
|
+
* the transcript endpoint for ever. A reason a person reads loses nothing by
|
|
1605
|
+
* being cut; a transcript nobody can read loses everything.
|
|
1606
|
+
*
|
|
1607
|
+
* Recovery's own `failStoredRun` is deliberately not routed through here: a
|
|
1608
|
+
* failure derived from a malformed durable history is the one case where
|
|
1609
|
+
* refusing to settle, and keeping the work active, is the right answer.
|
|
1610
|
+
*/
|
|
1336
1611
|
private async failRun(
|
|
1337
1612
|
runId: string,
|
|
1338
1613
|
previous: SessionEvent[],
|
|
@@ -1347,13 +1622,14 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1347
1622
|
runId,
|
|
1348
1623
|
previous,
|
|
1349
1624
|
events,
|
|
1350
|
-
failure,
|
|
1625
|
+
boundedRunFailureV1(failure),
|
|
1351
1626
|
this.supersededPackageRecords(),
|
|
1352
1627
|
);
|
|
1353
1628
|
await this.refreshRecoveryAlarm(transaction);
|
|
1354
1629
|
});
|
|
1355
1630
|
}
|
|
1356
1631
|
|
|
1632
|
+
/** Parks a run on a reason the authority composed, bounded as `failRun`'s is. */
|
|
1357
1633
|
private async requireRunReconciliation(
|
|
1358
1634
|
runId: string,
|
|
1359
1635
|
previous: SessionEvent[],
|
|
@@ -1368,7 +1644,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1368
1644
|
runId,
|
|
1369
1645
|
previous,
|
|
1370
1646
|
events,
|
|
1371
|
-
failure,
|
|
1647
|
+
boundedRunFailureV1(failure),
|
|
1372
1648
|
);
|
|
1373
1649
|
await this.refreshRecoveryAlarm(transaction);
|
|
1374
1650
|
});
|
|
@@ -1446,6 +1722,30 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1446
1722
|
const latest = (
|
|
1447
1723
|
(await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
|
|
1448
1724
|
).map(decodeSessionEvent);
|
|
1725
|
+
// A Turn the User stopped, or one a later message replaced, is terminal
|
|
1726
|
+
// in intent before recovery ever looks at it. There is nothing to
|
|
1727
|
+
// recover: no answer is owed, and the provider outcome cannot change what
|
|
1728
|
+
// it settles as. Re-entering it is how the Worker died — the run resumed,
|
|
1729
|
+
// reached "Model response outcome is uncertain after cancellation", and
|
|
1730
|
+
// the alarm had nothing to hand the rejection to.
|
|
1731
|
+
//
|
|
1732
|
+
// `failStoredRun` routes a discarded run to `cancelStoredRun` or
|
|
1733
|
+
// `supersedeStoredRun` on the intent that is already durable, and closes
|
|
1734
|
+
// the open turn on the way, so the settled log is a complete account.
|
|
1735
|
+
if (runWasDiscardedV1(run)) {
|
|
1736
|
+
await failStoredRun(
|
|
1737
|
+
this.codec,
|
|
1738
|
+
transaction,
|
|
1739
|
+
this.terminalKeys(run.runId),
|
|
1740
|
+
run.runId,
|
|
1741
|
+
latest.slice(0, run.previousEventCount),
|
|
1742
|
+
run.events,
|
|
1743
|
+
DISCARDED_RUN_RECOVERY_FAILURE_V1,
|
|
1744
|
+
this.supersededPackageRecords(),
|
|
1745
|
+
);
|
|
1746
|
+
await this.refreshRecoveryAlarm(transaction);
|
|
1747
|
+
return undefined;
|
|
1748
|
+
}
|
|
1449
1749
|
const plan = planBotRunRecovery(
|
|
1450
1750
|
run,
|
|
1451
1751
|
latest,
|
|
@@ -1522,22 +1822,6 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1522
1822
|
await this.refreshRecoveryAlarm(transaction);
|
|
1523
1823
|
return { kind: "resume" as const, run, latest, settings };
|
|
1524
1824
|
}
|
|
1525
|
-
if (runWasDiscardedV1(run)) {
|
|
1526
|
-
// Recovery of a Turn Stop or supersede already discarded settles it on
|
|
1527
|
-
// that intent rather than parking it: nothing is owed the answer.
|
|
1528
|
-
await failStoredRun(
|
|
1529
|
-
this.codec,
|
|
1530
|
-
transaction,
|
|
1531
|
-
this.terminalKeys(run.runId),
|
|
1532
|
-
run.runId,
|
|
1533
|
-
latest.slice(0, run.previousEventCount),
|
|
1534
|
-
[...run.events, ...plan.repairs],
|
|
1535
|
-
"Execution outcome requires reconciliation before it can resume",
|
|
1536
|
-
this.supersededPackageRecords(),
|
|
1537
|
-
);
|
|
1538
|
-
await this.refreshRecoveryAlarm(transaction);
|
|
1539
|
-
return undefined;
|
|
1540
|
-
}
|
|
1541
1825
|
await transaction.put({
|
|
1542
1826
|
[key]: {
|
|
1543
1827
|
...run,
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import {
|
|
3
|
+
bootstrapGeneration,
|
|
4
|
+
type CompositionGenerationV1,
|
|
5
|
+
} from "@frockbot/kernel-composition/generation";
|
|
6
|
+
import type { SessionEvent } from "@frockbot/kernel-contracts";
|
|
7
|
+
import {
|
|
8
|
+
BotDurableAuthority,
|
|
9
|
+
type BotDurableAuthorityHooks,
|
|
10
|
+
} from "./authority.ts";
|
|
11
|
+
import {
|
|
12
|
+
conversationSessionIdV1,
|
|
13
|
+
isConversationSessionIdV1,
|
|
14
|
+
} from "./conversations.ts";
|
|
15
|
+
import { MemoryStorage } from "./memory-storage.fixture.ts";
|
|
16
|
+
import { createStoredRunCodecV1 } from "./run-records.ts";
|
|
17
|
+
|
|
18
|
+
const codec = createStoredRunCodecV1<undefined>({
|
|
19
|
+
decodeRunId: (value) => value as string,
|
|
20
|
+
decodeConfigurationSnapshot: () => undefined,
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
const IDENTITY = { userId: "user-1", botId: "primary" };
|
|
24
|
+
|
|
25
|
+
function bootstrap(): Promise<CompositionGenerationV1> {
|
|
26
|
+
return bootstrapGeneration(
|
|
27
|
+
[
|
|
28
|
+
{
|
|
29
|
+
packageId: "shell",
|
|
30
|
+
specifier: "@frockbot/plugin-shell",
|
|
31
|
+
version: "0.0.1",
|
|
32
|
+
manifest: { id: "shell", version: "0.0.1" },
|
|
33
|
+
},
|
|
34
|
+
],
|
|
35
|
+
{ createdAt: "2026-08-31T00:00:00.000Z" },
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function createAuthority(storage: MemoryStorage) {
|
|
40
|
+
const sessions: string[] = [];
|
|
41
|
+
const hooks: BotDurableAuthorityHooks<undefined> = {
|
|
42
|
+
resolveAdmissionSnapshot: () => Promise.resolve(undefined),
|
|
43
|
+
bootstrapComposition: () => bootstrap(),
|
|
44
|
+
admittedSnapshot: () => Promise.resolve(undefined),
|
|
45
|
+
executeTurn: async (input) => {
|
|
46
|
+
sessions.push(input.command.sessionId);
|
|
47
|
+
const events: SessionEvent[] = [
|
|
48
|
+
{
|
|
49
|
+
type: "turn/admission",
|
|
50
|
+
seq: input.previousEvents.length,
|
|
51
|
+
timestamp: "2026-08-31T01:00:01.000Z",
|
|
52
|
+
turn: input.previousEvents.length + 1,
|
|
53
|
+
turnType: "chat",
|
|
54
|
+
},
|
|
55
|
+
];
|
|
56
|
+
await input.persistSessionEvents(input.command.sessionId, events);
|
|
57
|
+
return { runId: input.command.runId, text: "ok", events };
|
|
58
|
+
},
|
|
59
|
+
notification: () => undefined,
|
|
60
|
+
scheduledDeadlines: () => Promise.resolve([]),
|
|
61
|
+
scheduledWorkInFlight: () => false,
|
|
62
|
+
deferScheduledWork: () => Promise.resolve(),
|
|
63
|
+
settleScheduledWork: () => Promise.resolve(),
|
|
64
|
+
};
|
|
65
|
+
return {
|
|
66
|
+
authority: new BotDurableAuthority<undefined>({
|
|
67
|
+
state: { storage } as unknown as DurableObjectState,
|
|
68
|
+
codec,
|
|
69
|
+
hooks,
|
|
70
|
+
}),
|
|
71
|
+
sessions,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function command(runId: string) {
|
|
76
|
+
return {
|
|
77
|
+
...IDENTITY,
|
|
78
|
+
runId,
|
|
79
|
+
sessionId: "user-1:primary",
|
|
80
|
+
acceptedAt: `2026-08-31T01:00:0${runId.slice(-1)}.000Z`,
|
|
81
|
+
text: `message ${runId}`,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
describe("a Bot's Session id names the conversation it is on", () => {
|
|
86
|
+
test("the first conversation is the bare Session id", () => {
|
|
87
|
+
expect(conversationSessionIdV1("user-1:primary", 1)).toBe("user-1:primary");
|
|
88
|
+
expect(conversationSessionIdV1("user-1:primary", 3)).toBe(
|
|
89
|
+
"user-1:primary#3",
|
|
90
|
+
);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("only a Bot's own conversations match its base id", () => {
|
|
94
|
+
const base = "user-1:primary";
|
|
95
|
+
expect(isConversationSessionIdV1(base, base)).toBe(true);
|
|
96
|
+
expect(isConversationSessionIdV1(base, `${base}#2`)).toBe(true);
|
|
97
|
+
expect(isConversationSessionIdV1(base, "routine:morning")).toBe(false);
|
|
98
|
+
expect(isConversationSessionIdV1(base, `${base}#0`)).toBe(false);
|
|
99
|
+
expect(isConversationSessionIdV1(base, `${base}#x`)).toBe(false);
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
describe("starting a new conversation", () => {
|
|
104
|
+
test("empties the log the next Turn derives from and keeps the old Turns", async () => {
|
|
105
|
+
const storage = new MemoryStorage();
|
|
106
|
+
const probe = createAuthority(storage);
|
|
107
|
+
|
|
108
|
+
await probe.authority.run(command("run-1"));
|
|
109
|
+
expect((storage.values.get("latest-events") as SessionEvent[]).length).toBe(
|
|
110
|
+
1,
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
const started = await probe.authority.startConversation(IDENTITY);
|
|
114
|
+
expect(started.ordinal).toBe(2);
|
|
115
|
+
expect(started.sessionId).toBe("user-1:primary#2");
|
|
116
|
+
// The unbounded log is the bug: the next Turn starts from nothing.
|
|
117
|
+
expect(storage.values.get("latest-events")).toEqual([]);
|
|
118
|
+
// The conversation just ended is still on disk, Turn for Turn.
|
|
119
|
+
expect(
|
|
120
|
+
(storage.values.get("run:run-1") as { events: SessionEvent[] }).events
|
|
121
|
+
.length,
|
|
122
|
+
).toBe(1);
|
|
123
|
+
|
|
124
|
+
await probe.authority.run(command("run-2"));
|
|
125
|
+
// The new Turn ran in the new Session, and saw none of the old history.
|
|
126
|
+
expect(probe.sessions).toEqual(["user-1:primary", "user-1:primary#2"]);
|
|
127
|
+
expect(
|
|
128
|
+
(storage.values.get("run:run-2") as { previousEventCount: number })
|
|
129
|
+
.previousEventCount,
|
|
130
|
+
).toBe(0);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("lists the conversations the Bot has had, newest first", async () => {
|
|
134
|
+
const storage = new MemoryStorage();
|
|
135
|
+
const probe = createAuthority(storage);
|
|
136
|
+
|
|
137
|
+
await probe.authority.run(command("run-1"));
|
|
138
|
+
await probe.authority.startConversation(IDENTITY);
|
|
139
|
+
await probe.authority.run(command("run-2"));
|
|
140
|
+
|
|
141
|
+
const conversations = await probe.authority.listConversations(IDENTITY);
|
|
142
|
+
expect(conversations.map((entry) => entry.sessionId)).toEqual([
|
|
143
|
+
"user-1:primary#2",
|
|
144
|
+
"user-1:primary",
|
|
145
|
+
]);
|
|
146
|
+
expect(conversations[0]?.endedAt).toBeUndefined();
|
|
147
|
+
expect(conversations[1]?.endedAt).toBeString();
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
test("is refused while a Turn is still admitted", async () => {
|
|
151
|
+
const storage = new MemoryStorage();
|
|
152
|
+
const probe = createAuthority(storage);
|
|
153
|
+
storage.values.set("active-run", "run-9");
|
|
154
|
+
|
|
155
|
+
await expect(probe.authority.startConversation(IDENTITY)).rejects.toThrow(
|
|
156
|
+
/still working on a Turn/,
|
|
157
|
+
);
|
|
158
|
+
});
|
|
159
|
+
});
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conversations: how one Bot has more than one chat Session over its life.
|
|
3
|
+
*
|
|
4
|
+
* A Bot's conversational Session id was `<userId>:<botId>` forever, so its
|
|
5
|
+
* durable event log only ever grew and there was no way to put a conversation
|
|
6
|
+
* down and start another. A conversation numbers that Session: the first is
|
|
7
|
+
* the bare id, so nothing already stored changes, and each one after it is the
|
|
8
|
+
* same id with `#<ordinal>` appended.
|
|
9
|
+
*
|
|
10
|
+
* Starting a new conversation is a durable boundary, not a deletion. The event
|
|
11
|
+
* log the next Turn derives its model request from is empty again; every Turn
|
|
12
|
+
* of every earlier conversation stays in the run index under the Session id it
|
|
13
|
+
* recorded, so the earlier conversation is still readable.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
/** The conversation a Bot's chat Session is currently on. */
|
|
17
|
+
export interface StoredConversationV1 {
|
|
18
|
+
schemaVersion: 1;
|
|
19
|
+
/** 1 is the Session every Bot starts on and the one already on disk. */
|
|
20
|
+
ordinal: number;
|
|
21
|
+
startedAt: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** One conversation a Bot has had, current or ended. */
|
|
25
|
+
export interface ConversationRecordV1 {
|
|
26
|
+
schemaVersion: 1;
|
|
27
|
+
/** The Session id its Turns recorded. */
|
|
28
|
+
sessionId: string;
|
|
29
|
+
ordinal: number;
|
|
30
|
+
startedAt: string;
|
|
31
|
+
/** Absent while this is the conversation the Bot is on. */
|
|
32
|
+
endedAt?: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const MAX_CONVERSATION_ORDINAL = 1_000_000;
|
|
36
|
+
|
|
37
|
+
/** The conversation a Bot with nothing stored is on. */
|
|
38
|
+
export function firstConversationV1(startedAt: string): StoredConversationV1 {
|
|
39
|
+
return { schemaVersion: 1, ordinal: 1, startedAt };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function decodeStoredConversationV1(
|
|
43
|
+
input: unknown,
|
|
44
|
+
): StoredConversationV1 | undefined {
|
|
45
|
+
if (input === undefined || input === null) return undefined;
|
|
46
|
+
if (typeof input !== "object") {
|
|
47
|
+
throw new Error("stored conversation is invalid");
|
|
48
|
+
}
|
|
49
|
+
const value = input as Record<string, unknown>;
|
|
50
|
+
if (value.schemaVersion !== 1) {
|
|
51
|
+
throw new Error("stored conversation.schemaVersion is invalid");
|
|
52
|
+
}
|
|
53
|
+
if (
|
|
54
|
+
typeof value.ordinal !== "number" ||
|
|
55
|
+
!Number.isSafeInteger(value.ordinal) ||
|
|
56
|
+
value.ordinal < 1 ||
|
|
57
|
+
value.ordinal > MAX_CONVERSATION_ORDINAL
|
|
58
|
+
) {
|
|
59
|
+
throw new Error("stored conversation.ordinal is invalid");
|
|
60
|
+
}
|
|
61
|
+
if (typeof value.startedAt !== "string" || value.startedAt.length === 0) {
|
|
62
|
+
throw new Error("stored conversation.startedAt is invalid");
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
schemaVersion: 1,
|
|
66
|
+
ordinal: value.ordinal,
|
|
67
|
+
startedAt: value.startedAt,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function decodeConversationRecordV1(
|
|
72
|
+
input: unknown,
|
|
73
|
+
): ConversationRecordV1 {
|
|
74
|
+
if (typeof input !== "object" || input === null) {
|
|
75
|
+
throw new Error("conversation record is invalid");
|
|
76
|
+
}
|
|
77
|
+
const value = input as Record<string, unknown>;
|
|
78
|
+
const stored = decodeStoredConversationV1({
|
|
79
|
+
schemaVersion: value.schemaVersion,
|
|
80
|
+
ordinal: value.ordinal,
|
|
81
|
+
startedAt: value.startedAt,
|
|
82
|
+
})!;
|
|
83
|
+
if (typeof value.sessionId !== "string" || value.sessionId.length === 0) {
|
|
84
|
+
throw new Error("conversation record.sessionId is invalid");
|
|
85
|
+
}
|
|
86
|
+
if (value.endedAt !== undefined && typeof value.endedAt !== "string") {
|
|
87
|
+
throw new Error("conversation record.endedAt is invalid");
|
|
88
|
+
}
|
|
89
|
+
return {
|
|
90
|
+
...stored,
|
|
91
|
+
sessionId: value.sessionId,
|
|
92
|
+
...(value.endedAt ? { endedAt: value.endedAt as string } : {}),
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* The Session id a Bot's chat Turns record while it is on this conversation.
|
|
98
|
+
*
|
|
99
|
+
* The first conversation is the bare id on purpose: every Session already
|
|
100
|
+
* stored is conversation 1, so nothing has to be migrated for it to be one.
|
|
101
|
+
*/
|
|
102
|
+
export function conversationSessionIdV1(base: string, ordinal: number): string {
|
|
103
|
+
return ordinal <= 1 ? base : `${base}#${ordinal}`;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** The Session id a Bot's conversational Turns are addressed to. */
|
|
107
|
+
export function botConversationBaseSessionIdV1(identity: {
|
|
108
|
+
userId: string;
|
|
109
|
+
botId: string;
|
|
110
|
+
}): string {
|
|
111
|
+
return `${identity.userId}:${identity.botId}`;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* True when this Session id names a conversation of that Bot — the bare base
|
|
116
|
+
* id or the base id with an ordinal. A Routine firing's `routine:<id>` and a
|
|
117
|
+
* subagent's Session are deliberately not conversations and never match.
|
|
118
|
+
*/
|
|
119
|
+
export function isConversationSessionIdV1(
|
|
120
|
+
base: string,
|
|
121
|
+
sessionId: string,
|
|
122
|
+
): boolean {
|
|
123
|
+
if (sessionId === base) return true;
|
|
124
|
+
if (!sessionId.startsWith(`${base}#`)) return false;
|
|
125
|
+
return /^[1-9][0-9]{0,6}$/.test(sessionId.slice(base.length + 1));
|
|
126
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// A failure the record could not hold, and the transcript it took with it.
|
|
2
|
+
//
|
|
3
|
+
// A run's `failure` is whatever an error's `message` happened to be, and
|
|
4
|
+
// nothing upstream bounds it. A provider that echoed the request back produced
|
|
5
|
+
// one past the record's own limit, the settlement wrote it anyway, and every
|
|
6
|
+
// later read of that run threw `has invalid failure` — so a Turn that failed
|
|
7
|
+
// once went on to 500 the transcript endpoint for ever, and "Try again" could
|
|
8
|
+
// not get the person out of it either.
|
|
9
|
+
import { describe, expect, test } from "bun:test";
|
|
10
|
+
import {
|
|
11
|
+
boundedRunFailureV1,
|
|
12
|
+
MAX_RUN_FAILURE_BYTES_V1,
|
|
13
|
+
} from "./run-records.ts";
|
|
14
|
+
|
|
15
|
+
const bytes = (value: string) => new TextEncoder().encode(value).byteLength;
|
|
16
|
+
|
|
17
|
+
describe("the failure a settlement may durably write", () => {
|
|
18
|
+
test("leaves a reason that already fits exactly as it was written", () => {
|
|
19
|
+
const reason = "Reconciliation was explicitly abandoned: response lost";
|
|
20
|
+
expect(boundedRunFailureV1(reason)).toBe(reason);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
test("cuts one the record could not hold, and says it was cut", () => {
|
|
24
|
+
const bounded = boundedRunFailureV1(
|
|
25
|
+
"x".repeat(MAX_RUN_FAILURE_BYTES_V1 * 2),
|
|
26
|
+
);
|
|
27
|
+
expect(bytes(bounded)).toBeLessThanOrEqual(MAX_RUN_FAILURE_BYTES_V1);
|
|
28
|
+
expect(bounded.endsWith("…")).toBe(true);
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test("keeps the opening words, which are the ones a person reads", () => {
|
|
32
|
+
const bounded = boundedRunFailureV1(
|
|
33
|
+
`Model request failed: ${"detail ".repeat(MAX_RUN_FAILURE_BYTES_V1)}`,
|
|
34
|
+
);
|
|
35
|
+
expect(bounded.startsWith("Model request failed: detail")).toBe(true);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
test("never cuts inside a character", () => {
|
|
39
|
+
// Multi-byte throughout: a byte-wise slice would leave a lone surrogate
|
|
40
|
+
// and the decoder would refuse the record for a different reason.
|
|
41
|
+
const bounded = boundedRunFailureV1("🐑".repeat(MAX_RUN_FAILURE_BYTES_V1));
|
|
42
|
+
expect(bytes(bounded)).toBeLessThanOrEqual(MAX_RUN_FAILURE_BYTES_V1);
|
|
43
|
+
expect(bounded).not.toContain("�");
|
|
44
|
+
expect([...bounded].every((character) => character.length <= 2)).toBe(true);
|
|
45
|
+
});
|
|
46
|
+
});
|
package/src/run-records.ts
CHANGED
|
@@ -277,6 +277,36 @@ function boundedString(
|
|
|
277
277
|
);
|
|
278
278
|
}
|
|
279
279
|
|
|
280
|
+
/** The largest failure a run record may carry, in UTF-8 bytes. */
|
|
281
|
+
export const MAX_RUN_FAILURE_BYTES_V1 = 8_000;
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* The failure a settlement may durably write, cut to what the record allows.
|
|
285
|
+
*
|
|
286
|
+
* A failure string is whatever an error's `message` happened to be, and nothing
|
|
287
|
+
* upstream bounds it — a provider that echoes a request back, or a message
|
|
288
|
+
* built by concatenating one, easily runs past the limit. Writing it anyway
|
|
289
|
+
* produced a record the codec refused on every later read, so a Turn that
|
|
290
|
+
* failed once went on to 500 the transcript endpoint for ever. The message is
|
|
291
|
+
* for a person to read: cutting it costs nothing the record does not already
|
|
292
|
+
* hold, and losing the whole transcript costs everything.
|
|
293
|
+
*/
|
|
294
|
+
export function boundedRunFailureV1(failure: string): string {
|
|
295
|
+
if (UTF8_ENCODER.encode(failure).byteLength <= MAX_RUN_FAILURE_BYTES_V1) {
|
|
296
|
+
return failure;
|
|
297
|
+
}
|
|
298
|
+
const ellipsis = "…";
|
|
299
|
+
const budget =
|
|
300
|
+
MAX_RUN_FAILURE_BYTES_V1 - UTF8_ENCODER.encode(ellipsis).byteLength;
|
|
301
|
+
let kept = failure;
|
|
302
|
+
// Cutting by characters and re-measuring keeps the result valid UTF-8; a
|
|
303
|
+
// byte-wise slice can land inside a multi-byte sequence.
|
|
304
|
+
while (UTF8_ENCODER.encode(kept).byteLength > budget) {
|
|
305
|
+
kept = kept.slice(0, Math.max(0, Math.floor(kept.length * 0.9) - 1));
|
|
306
|
+
}
|
|
307
|
+
return `${kept}${ellipsis}`;
|
|
308
|
+
}
|
|
309
|
+
|
|
280
310
|
function decodeDirectToolCommandV1(value: unknown): DirectToolCommandV1 {
|
|
281
311
|
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
282
312
|
throw new Error("stored run has invalid direct tool command");
|
package/src/run-recovery.ts
CHANGED
|
@@ -266,6 +266,76 @@ export function repairOrphanedOpenTurnV1(
|
|
|
266
266
|
}
|
|
267
267
|
}
|
|
268
268
|
|
|
269
|
+
/**
|
|
270
|
+
* True when *any* Turn in the log was never closed — including one buried
|
|
271
|
+
* behind later Turns that are themselves well formed.
|
|
272
|
+
*
|
|
273
|
+
* `hasOrphanedOpenTurnV1` only sees a log that *ends* inside a Turn, and that
|
|
274
|
+
* is the shape a wedged Bot stops having after its very first retry. The Agent
|
|
275
|
+
* loop journals `turn/start` durably and only then assembles the request, so
|
|
276
|
+
* the Turn that discovers the invariant is broken has already written its own
|
|
277
|
+
* `turn/start`, and its `finally` writes a matching `turn/end` carrying the
|
|
278
|
+
* validation message. The log that comes out of that ends closed — with the
|
|
279
|
+
* abandoned Turn still open several events back — so the trailing-open test
|
|
280
|
+
* says there is nothing to repair, and every later message fails the same way.
|
|
281
|
+
*/
|
|
282
|
+
export function hasUnclosedTurnV1(events: readonly SessionEvent[]): boolean {
|
|
283
|
+
let openTurn: number | undefined;
|
|
284
|
+
for (const event of events) {
|
|
285
|
+
if (event.type === "turn/start") {
|
|
286
|
+
if (openTurn !== undefined) return true;
|
|
287
|
+
openTurn = event.turn;
|
|
288
|
+
}
|
|
289
|
+
if (event.type === "turn/end" && event.turn === openTurn) {
|
|
290
|
+
openTurn = undefined;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
return openTurn !== undefined;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* The whole durable log with every abandoned Turn closed, or `undefined` when
|
|
298
|
+
* there is nothing to repair or the log cannot be repaired without inventing
|
|
299
|
+
* history.
|
|
300
|
+
*
|
|
301
|
+
* A Turn left open in the middle of the log cannot be closed by appending:
|
|
302
|
+
* `turn/end` for it would land after the Turns that followed, and the log
|
|
303
|
+
* would still read as "turn N started while turn N-1 is open". So the repair
|
|
304
|
+
* *rewrites* the log, inserting the closing events at the point the Turn was
|
|
305
|
+
* abandoned and resequencing what follows. The inserted events are the ones
|
|
306
|
+
* the interrupted-run repair already writes — every unresolved tool occurrence
|
|
307
|
+
* closed as `interrupted`, then `step/end`, then `turn/end` with outcome
|
|
308
|
+
* `interrupted` — so a Turn nobody finished reads as one nobody finished.
|
|
309
|
+
*
|
|
310
|
+
* Only called where nothing is entitled to write those ends: at admission and
|
|
311
|
+
* promotion with no run executing, and at settlement, where the run that owned
|
|
312
|
+
* the Turn has just stopped.
|
|
313
|
+
*/
|
|
314
|
+
export function repairedSessionLogV1(
|
|
315
|
+
sessionId: string,
|
|
316
|
+
latest: readonly SessionEvent[],
|
|
317
|
+
): SessionEvent[] | undefined {
|
|
318
|
+
if (!hasUnclosedTurnV1(latest)) return undefined;
|
|
319
|
+
let repaired: SessionEvent[] = [];
|
|
320
|
+
const closeOpenTurn = (): boolean => {
|
|
321
|
+
if (repaired.length === 0 || !hasOrphanedOpenTurnV1(repaired)) return true;
|
|
322
|
+
try {
|
|
323
|
+
const session = new Session(sessionId, () => {}, repaired);
|
|
324
|
+
session.reconcileInterrupted();
|
|
325
|
+
repaired = [...session.events];
|
|
326
|
+
} catch {
|
|
327
|
+
return false;
|
|
328
|
+
}
|
|
329
|
+
return !hasOrphanedOpenTurnV1(repaired);
|
|
330
|
+
};
|
|
331
|
+
for (const event of latest) {
|
|
332
|
+
if (event.type === "turn/start" && !closeOpenTurn()) return undefined;
|
|
333
|
+
repaired.push({ ...event, seq: repaired.length });
|
|
334
|
+
}
|
|
335
|
+
if (!closeOpenTurn()) return undefined;
|
|
336
|
+
return repaired;
|
|
337
|
+
}
|
|
338
|
+
|
|
269
339
|
export function eventsForFailedRun(
|
|
270
340
|
durableRun: { events: SessionEvent[] } | undefined,
|
|
271
341
|
error: unknown,
|
package/src/run-terminal.ts
CHANGED
|
@@ -8,6 +8,7 @@ import type {
|
|
|
8
8
|
StoredRunCodecV1,
|
|
9
9
|
StoredRunV1,
|
|
10
10
|
} from "./run-records.js";
|
|
11
|
+
import { repairedSessionLogV1 } from "./run-recovery.js";
|
|
11
12
|
|
|
12
13
|
/**
|
|
13
14
|
* The events a terminal settlement commits, with any Turn they were left
|
|
@@ -44,9 +45,15 @@ function settledEventsV1(
|
|
|
44
45
|
} catch {
|
|
45
46
|
repairs = [];
|
|
46
47
|
}
|
|
48
|
+
const settled = [...latest, ...repairs];
|
|
49
|
+
// A Turn abandoned earlier in the log cannot be closed by appending, and a
|
|
50
|
+
// settlement that only appends leaves it open forever. The run's own events
|
|
51
|
+
// are committed as they stand — that record is this run's account, not the
|
|
52
|
+
// conversation's — while the forward log is repaired in place so the next
|
|
53
|
+
// Turn starts on a log that reads as a complete history.
|
|
47
54
|
return {
|
|
48
55
|
events: [...decoded, ...repairs],
|
|
49
|
-
latestEvents:
|
|
56
|
+
latestEvents: repairedSessionLogV1(sessionId, settled) ?? settled,
|
|
50
57
|
};
|
|
51
58
|
}
|
|
52
59
|
|
package/src/storage-keys.ts
CHANGED
|
@@ -135,3 +135,18 @@ export function storedRunAdmissionFences(input: unknown): string[] {
|
|
|
135
135
|
export function workspaceSyncEffectKey(effectId: string): string {
|
|
136
136
|
return `${WORKSPACE_SYNC_EFFECT_PREFIX}${effectId}`;
|
|
137
137
|
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The conversation the Bot's chat Session is currently on.
|
|
141
|
+
*
|
|
142
|
+
* One Bot has one conversational Session at a time. Starting a new
|
|
143
|
+
* conversation ends the current one and begins the next: the durable event log
|
|
144
|
+
* the next Turn derives its request from is empty again, while every Turn the
|
|
145
|
+
* earlier conversations recorded stays durable and readable under its own
|
|
146
|
+
* Session id.
|
|
147
|
+
*/
|
|
148
|
+
export const CONVERSATION_KEY = "conversation";
|
|
149
|
+
/** The conversations this Bot has already ended, newest last. */
|
|
150
|
+
export const CONVERSATION_INDEX_KEY = "conversation-index";
|
|
151
|
+
/** How many ended conversations stay listable. Older ones drop off the list. */
|
|
152
|
+
export const MAX_LISTED_CONVERSATIONS = 64;
|
|
@@ -3,7 +3,10 @@ import {
|
|
|
3
3
|
bootstrapGeneration,
|
|
4
4
|
type CompositionGenerationV1,
|
|
5
5
|
} from "@frockbot/kernel-composition/generation";
|
|
6
|
-
import
|
|
6
|
+
import {
|
|
7
|
+
type SessionEvent,
|
|
8
|
+
validateToolOccurrenceJournal,
|
|
9
|
+
} from "@frockbot/kernel-contracts";
|
|
7
10
|
import {
|
|
8
11
|
BotDurableAuthority,
|
|
9
12
|
SUPERSEDED_TURN_REASON_V1,
|
|
@@ -690,6 +693,87 @@ describe("a durable log left inside a Turn", () => {
|
|
|
690
693
|
});
|
|
691
694
|
expect(storedRun(storage, "run-1").status).toBe("completed");
|
|
692
695
|
});
|
|
696
|
+
|
|
697
|
+
test("is repaired even once refused Turns have been logged behind it", async () => {
|
|
698
|
+
const storage = new MemoryStorage();
|
|
699
|
+
// What production actually holds on a Bot wedged before the repair
|
|
700
|
+
// existed. The Agent loop journals `turn/start` durably and only then
|
|
701
|
+
// assembles the request that discovers turn 1 is still open, so every
|
|
702
|
+
// refused message left a *complete* Turn of its own behind the abandoned
|
|
703
|
+
// one — and the log stopped ending inside a Turn. The trailing-open test
|
|
704
|
+
// then said there was nothing to repair, so admission repaired nothing and
|
|
705
|
+
// the next message failed exactly the same way, forever.
|
|
706
|
+
storage.values.set("latest-events", [
|
|
707
|
+
{
|
|
708
|
+
type: "session/created",
|
|
709
|
+
createdAt: "2026-09-03T00:00:00.000Z",
|
|
710
|
+
seq: 0,
|
|
711
|
+
timestamp: "2026-09-03T00:00:00.000Z",
|
|
712
|
+
},
|
|
713
|
+
{
|
|
714
|
+
type: "turn/start",
|
|
715
|
+
turn: 1,
|
|
716
|
+
seq: 1,
|
|
717
|
+
timestamp: "2026-09-03T00:00:01.000Z",
|
|
718
|
+
},
|
|
719
|
+
{
|
|
720
|
+
type: "turn/start",
|
|
721
|
+
turn: 2,
|
|
722
|
+
seq: 2,
|
|
723
|
+
timestamp: "2026-09-03T00:00:02.000Z",
|
|
724
|
+
},
|
|
725
|
+
{
|
|
726
|
+
type: "turn/end",
|
|
727
|
+
turn: 2,
|
|
728
|
+
outcome: "model-error",
|
|
729
|
+
reason: "turn 2 started while turn 1 is open",
|
|
730
|
+
seq: 3,
|
|
731
|
+
timestamp: "2026-09-03T00:00:03.000Z",
|
|
732
|
+
},
|
|
733
|
+
]);
|
|
734
|
+
const probe = createAuthority(storage);
|
|
735
|
+
|
|
736
|
+
const run = probe.authority.run(command("run-1", "hello"));
|
|
737
|
+
await probe.handle("run-1").started;
|
|
738
|
+
probe.handle("run-1").finish();
|
|
739
|
+
await run;
|
|
740
|
+
|
|
741
|
+
const events = storage.values.get("latest-events") as SessionEvent[];
|
|
742
|
+
// Turn 1 is closed where it was abandoned, not after the Turns that
|
|
743
|
+
// followed it, and the log is resequenced around the insertion.
|
|
744
|
+
expect(
|
|
745
|
+
events
|
|
746
|
+
.slice(0, 5)
|
|
747
|
+
.map((event) =>
|
|
748
|
+
event.type === "turn/start" || event.type === "turn/end"
|
|
749
|
+
? `${event.type}:${event.turn}`
|
|
750
|
+
: event.type,
|
|
751
|
+
),
|
|
752
|
+
).toEqual([
|
|
753
|
+
"session/created",
|
|
754
|
+
"turn/start:1",
|
|
755
|
+
"turn/end:1",
|
|
756
|
+
"turn/start:2",
|
|
757
|
+
"turn/end:2",
|
|
758
|
+
]);
|
|
759
|
+
expect(events[2]).toMatchObject({
|
|
760
|
+
type: "turn/end",
|
|
761
|
+
turn: 1,
|
|
762
|
+
outcome: "interrupted",
|
|
763
|
+
});
|
|
764
|
+
expect(events.map((event) => event.seq)).toEqual(
|
|
765
|
+
events.map((_event, index) => index),
|
|
766
|
+
);
|
|
767
|
+
// The repaired history is one the invariant accepts, which is what the
|
|
768
|
+
// refused Turns were failing on. (The stub Agent below numbers its own
|
|
769
|
+
// Turn rather than reading `nextTurn`, so only the repaired prefix is the
|
|
770
|
+
// subject here.)
|
|
771
|
+
expect(() =>
|
|
772
|
+
validateToolOccurrenceJournal(events.slice(0, 5)),
|
|
773
|
+
).not.toThrow();
|
|
774
|
+
expect(storedRun(storage, "run-1").status).toBe("completed");
|
|
775
|
+
expect(storedRun(storage, "run-1").previousEventCount).toBe(5);
|
|
776
|
+
});
|
|
693
777
|
});
|
|
694
778
|
|
|
695
779
|
describe("a failing recovery of an older Turn", () => {
|
|
@@ -827,3 +911,125 @@ describe("an interrupt while the model is streaming", () => {
|
|
|
827
911
|
expect((await next).text).toBe("done: second");
|
|
828
912
|
});
|
|
829
913
|
});
|
|
914
|
+
|
|
915
|
+
describe("a discarded Turn never crashes the object", () => {
|
|
916
|
+
test("the long-lived caller is answered with the cancelled run, not a throw", async () => {
|
|
917
|
+
const storage = new MemoryStorage();
|
|
918
|
+
const probe = createAuthority(storage, { uncertain: () => true });
|
|
919
|
+
|
|
920
|
+
const first = probe.authority.run(command("run-1", "first"));
|
|
921
|
+
await probe.handle("run-1").started;
|
|
922
|
+
const stopped = storedRun(storage, "run-1");
|
|
923
|
+
storage.values.set("run:run-1", {
|
|
924
|
+
...stopped,
|
|
925
|
+
stopRequestedAt: "2026-09-03T00:00:05.000Z",
|
|
926
|
+
});
|
|
927
|
+
probe.handle("run-1").interrupt("agent cancelled by user");
|
|
928
|
+
|
|
929
|
+
// The composer is still holding this request open when Stop is pressed. It
|
|
930
|
+
// used to be answered with a 500 and a red console error while the UI
|
|
931
|
+
// beside it said "You stopped this." A Turn the person stopped on purpose
|
|
932
|
+
// is an ordinary outcome and settles as one.
|
|
933
|
+
const settled = await first;
|
|
934
|
+
expect(settled.runId).toBe("run-1");
|
|
935
|
+
expect(settled.text).toBe("");
|
|
936
|
+
expect(storedRun(storage, "run-1").status).toBe("cancelled");
|
|
937
|
+
expect(storage.values.get("active-run")).toBeUndefined();
|
|
938
|
+
});
|
|
939
|
+
|
|
940
|
+
test("recovery settles a stopped Turn instead of re-entering it", async () => {
|
|
941
|
+
const storage = new MemoryStorage();
|
|
942
|
+
const probe = createAuthority(storage, { uncertain: () => true });
|
|
943
|
+
|
|
944
|
+
const first = probe.authority.run(command("run-1", "first"));
|
|
945
|
+
await probe.handle("run-1").started;
|
|
946
|
+
const running = storedRun(storage, "run-1");
|
|
947
|
+
storage.values.set("run:run-1", {
|
|
948
|
+
...running,
|
|
949
|
+
stopRequestedAt: "2026-09-03T00:00:05.000Z",
|
|
950
|
+
});
|
|
951
|
+
// The object is evicted with the Stop durable and the Turn still active:
|
|
952
|
+
// exactly the state the recovery alarm wakes up to.
|
|
953
|
+
first.catch(() => undefined);
|
|
954
|
+
|
|
955
|
+
const evicted = createAuthority(storage, { uncertain: () => true });
|
|
956
|
+
await expect(evicted.authority.alarm()).resolves.toBeUndefined();
|
|
957
|
+
|
|
958
|
+
// Re-entering it is what took the dev Worker down: the run resumed, reached
|
|
959
|
+
// "Model response outcome is uncertain after cancellation", and the alarm
|
|
960
|
+
// had nobody to hand the rejection to. There is nothing to recover — the
|
|
961
|
+
// User already said to throw it away.
|
|
962
|
+
expect(evicted.observed).toEqual([]);
|
|
963
|
+
expect(storedRun(storage, "run-1").status).toBe("cancelled");
|
|
964
|
+
expect(storage.values.get("active-run")).toBeUndefined();
|
|
965
|
+
});
|
|
966
|
+
|
|
967
|
+
test("recovery settles a superseded Turn instead of re-entering it", async () => {
|
|
968
|
+
const storage = new MemoryStorage();
|
|
969
|
+
const probe = createAuthority(storage, { uncertain: () => true });
|
|
970
|
+
|
|
971
|
+
const first = probe.authority.run(command("run-1", "first"));
|
|
972
|
+
await probe.handle("run-1").started;
|
|
973
|
+
const running = storedRun(storage, "run-1");
|
|
974
|
+
storage.values.set("run:run-1", {
|
|
975
|
+
...running,
|
|
976
|
+
supersededAt: "2026-09-03T00:00:05.000Z",
|
|
977
|
+
supersededBy: "run-2",
|
|
978
|
+
});
|
|
979
|
+
first.catch(() => undefined);
|
|
980
|
+
|
|
981
|
+
const evicted = createAuthority(storage, { uncertain: () => true });
|
|
982
|
+
await expect(evicted.authority.alarm()).resolves.toBeUndefined();
|
|
983
|
+
|
|
984
|
+
expect(evicted.observed).toEqual([]);
|
|
985
|
+
expect(storedRun(storage, "run-1").status).toBe("superseded");
|
|
986
|
+
expect(storage.values.get("active-run")).toBeUndefined();
|
|
987
|
+
});
|
|
988
|
+
|
|
989
|
+
test("an alarm records a recovery failure instead of rejecting", async () => {
|
|
990
|
+
const storage = new MemoryStorage();
|
|
991
|
+
const probe = createAuthority(storage, { failRecovery: () => true });
|
|
992
|
+
|
|
993
|
+
const first = probe.authority.run(command("run-1", "first"));
|
|
994
|
+
await probe.handle("run-1").started;
|
|
995
|
+
first.catch(() => undefined);
|
|
996
|
+
|
|
997
|
+
// A recovery that cannot run the Turn is a durable fact, not a fault of the
|
|
998
|
+
// alarm: an alarm has no caller, so anything it lets escape is an uncaught
|
|
999
|
+
// exception in the object — one of the ways the dev Worker died.
|
|
1000
|
+
const evicted = createAuthority(storage, { failRecovery: () => true });
|
|
1001
|
+
await expect(evicted.authority.alarm()).resolves.toBeUndefined();
|
|
1002
|
+
// And the object still has a deadline, so the next firing tries again.
|
|
1003
|
+
expect(storage.alarmAt).toBeGreaterThan(0);
|
|
1004
|
+
});
|
|
1005
|
+
});
|
|
1006
|
+
|
|
1007
|
+
describe("Try again on a parked Turn", () => {
|
|
1008
|
+
test("answers with the run it settled rather than throwing", async () => {
|
|
1009
|
+
const storage = new MemoryStorage();
|
|
1010
|
+
// The Turn parks on a provider outcome only a User can retrieve, which is
|
|
1011
|
+
// the state the Resolve Turn button exists for.
|
|
1012
|
+
const probe = createAuthority(storage, {
|
|
1013
|
+
dispatch: () => false,
|
|
1014
|
+
parkOnRelease: () => true,
|
|
1015
|
+
});
|
|
1016
|
+
|
|
1017
|
+
const first = probe.authority.run(command("run-1", "first"));
|
|
1018
|
+
await probe.handle("run-1").started;
|
|
1019
|
+
probe.handle("run-1").finish();
|
|
1020
|
+
await first.catch(() => undefined);
|
|
1021
|
+
expect(storedRun(storage, "run-1").status).toBe("reconciliation-required");
|
|
1022
|
+
|
|
1023
|
+
// "Try again": the retry fails again, the run is abandoned, and that is a
|
|
1024
|
+
// successful abandon — not a failed request. Rethrowing here made the
|
|
1025
|
+
// button answer 409, and the transcript read the browser makes straight
|
|
1026
|
+
// afterwards 500 on the half-repaired record.
|
|
1027
|
+
const abandoned = await probe.authority.reconcileRun(identity, "run-1");
|
|
1028
|
+
expect(abandoned.runId).toBe("run-1");
|
|
1029
|
+
|
|
1030
|
+
const settled = storedRun(storage, "run-1");
|
|
1031
|
+
expect(settled.status).toBe("failed");
|
|
1032
|
+
expect(settled.failure).toContain("explicitly abandoned");
|
|
1033
|
+
expect(storage.values.get("active-run")).toBeUndefined();
|
|
1034
|
+
});
|
|
1035
|
+
});
|