@frockbot/kernel-do 0.3.15 → 0.3.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/applets.test.ts +6 -1
- package/src/applets.ts +3 -1
- package/src/authority.ts +319 -159
- package/src/conversations.test.ts +7 -9
- package/src/index.ts +1 -0
- package/src/model-timeout-settlement.test.ts +8 -8
- package/src/run-liveness.test.ts +2 -1
- package/src/run-records.ts +142 -6
- package/src/run-recovery.ts +2 -1
- package/src/run-terminal-open-turn.test.ts +9 -4
- package/src/run-terminal.ts +61 -44
- package/src/session-event-log.test.ts +267 -0
- package/src/session-event-log.ts +731 -0
- package/src/session-log-size.test.ts +250 -0
- package/src/storage-keys.ts +12 -0
- package/src/turn-admission.test.ts +54 -9
- package/src/turn-supersede.test.ts +161 -8
package/src/authority.ts
CHANGED
|
@@ -16,6 +16,9 @@ import {
|
|
|
16
16
|
botTurnCommandFingerprintV1,
|
|
17
17
|
defaultRunLaneV1,
|
|
18
18
|
storedRunAdmissionV1,
|
|
19
|
+
storedRunLaneV1,
|
|
20
|
+
storedRunEventFieldsV2,
|
|
21
|
+
storedRunRecordV2,
|
|
19
22
|
storedRunSubagentRoleV1,
|
|
20
23
|
storedRunTurnTypeV1,
|
|
21
24
|
type BotNotificationIntent,
|
|
@@ -41,6 +44,10 @@ import {
|
|
|
41
44
|
unresolvedModelRequestFailure,
|
|
42
45
|
} from "./run-recovery.js";
|
|
43
46
|
import { runLivenessV1, STALE_RUNNING_RUN_FAILURE_V1 } from "./run-liveness.js";
|
|
47
|
+
import {
|
|
48
|
+
SessionEventLog,
|
|
49
|
+
type SessionEventLogStorage,
|
|
50
|
+
} from "./session-event-log.js";
|
|
44
51
|
import {
|
|
45
52
|
BotTurnReconciliationRequiredError,
|
|
46
53
|
BotTurnRecoveryRequiredError,
|
|
@@ -61,6 +68,8 @@ import {
|
|
|
61
68
|
CONVERSATION_INDEX_KEY,
|
|
62
69
|
CONVERSATION_KEY,
|
|
63
70
|
MAX_LISTED_CONVERSATIONS,
|
|
71
|
+
MAX_PENDING_AGENT_RUNS_V1,
|
|
72
|
+
PENDING_AGENT_RUN_PREFIX,
|
|
64
73
|
PENDING_RUN_KEY,
|
|
65
74
|
IDENTITY_KEY,
|
|
66
75
|
LATEST_EVENTS_KEY,
|
|
@@ -71,6 +80,7 @@ import {
|
|
|
71
80
|
RUN_ADMISSION_FENCE_PREFIX,
|
|
72
81
|
RUN_INDEX_PREFIX,
|
|
73
82
|
RUN_PREFIX,
|
|
83
|
+
pendingAgentRunKey,
|
|
74
84
|
runIndexKey,
|
|
75
85
|
storedRunAdmissionFences,
|
|
76
86
|
} from "./storage-keys.js";
|
|
@@ -384,20 +394,35 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
384
394
|
return this.ctx.storage.transaction(async (transaction) => {
|
|
385
395
|
const pendingRunId = await transaction.get<string>(PENDING_RUN_KEY);
|
|
386
396
|
const run = this.codec.optional(await transaction.get<unknown>(key));
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
)
|
|
397
|
+
const lane = run ? storedRunLaneV1(run) : undefined;
|
|
398
|
+
const firstPendingAgent = await transaction.list<string>({
|
|
399
|
+
prefix: PENDING_AGENT_RUN_PREFIX,
|
|
400
|
+
limit: 1,
|
|
401
|
+
});
|
|
402
|
+
const firstPendingAgentEntry = firstPendingAgent.entries().next()
|
|
403
|
+
.value as [string, string] | undefined;
|
|
404
|
+
if (!run || run.status !== "running" || run.phase !== "queued") {
|
|
405
|
+
return "not-queued" as const;
|
|
406
|
+
}
|
|
407
|
+
if (lane === "agent") {
|
|
408
|
+
// A User Turn always has first claim on an idle Bot, and agent Turns
|
|
409
|
+
// retain FIFO order behind it. The run is still queued in either case;
|
|
410
|
+
// reporting `not-queued` here would strand its blocking caller even
|
|
411
|
+
// though the durable queue entry remains.
|
|
412
|
+
if (pendingRunId !== undefined) return "blocked" as const;
|
|
413
|
+
if (firstPendingAgentEntry?.[1] !== runId) {
|
|
414
|
+
return firstPendingAgentEntry
|
|
415
|
+
? ("blocked" as const)
|
|
416
|
+
: ("not-queued" as const);
|
|
417
|
+
}
|
|
418
|
+
} else if (pendingRunId !== runId) {
|
|
393
419
|
return "not-queued" as const;
|
|
394
420
|
}
|
|
395
421
|
if (await transaction.get<string>(ACTIVE_RUN_KEY)) {
|
|
396
422
|
return "blocked" as const;
|
|
397
423
|
}
|
|
398
|
-
const
|
|
399
|
-
|
|
400
|
-
).map(decodeSessionEvent);
|
|
424
|
+
const eventLog = new SessionEventLog(transaction);
|
|
425
|
+
const storedEvents = await eventLog.migrate(run.sessionId);
|
|
401
426
|
// A queued Turn was admitted while another was executing, so admission
|
|
402
427
|
// could not repair the log: something was still entitled to close that
|
|
403
428
|
// Turn. Here the active-run marker is gone and nothing is, so the same
|
|
@@ -408,19 +433,18 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
408
433
|
...run,
|
|
409
434
|
phase: "admitted",
|
|
410
435
|
previousEventCount: latestEvents.length,
|
|
436
|
+
...storedRunEventFieldsV2(latestEvents.length, []),
|
|
411
437
|
} satisfies StoredRunV1<Snapshot>);
|
|
438
|
+
if (repaired) await eventLog.rewrite(run.sessionId, latestEvents);
|
|
412
439
|
await transaction.put({
|
|
413
|
-
[key]: structuredClone(promoted),
|
|
440
|
+
[key]: structuredClone(storedRunRecordV2(promoted)),
|
|
414
441
|
[ACTIVE_RUN_KEY]: runId,
|
|
415
|
-
...(repaired
|
|
416
|
-
? {
|
|
417
|
-
[LATEST_EVENTS_KEY]: structuredClone(
|
|
418
|
-
repaired.map(decodeSessionEvent),
|
|
419
|
-
),
|
|
420
|
-
}
|
|
421
|
-
: {}),
|
|
422
442
|
});
|
|
423
|
-
|
|
443
|
+
if (lane === "agent" && firstPendingAgentEntry) {
|
|
444
|
+
await transaction.delete(firstPendingAgentEntry[0]);
|
|
445
|
+
} else {
|
|
446
|
+
await transaction.delete(PENDING_RUN_KEY);
|
|
447
|
+
}
|
|
424
448
|
await this.refreshRecoveryAlarm(transaction);
|
|
425
449
|
return {
|
|
426
450
|
previous: latestEvents,
|
|
@@ -437,7 +461,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
437
461
|
await this.assertIdentity(identity);
|
|
438
462
|
const key = `${RUN_PREFIX}${runId}`;
|
|
439
463
|
const recovery = await this.ctx.storage.transaction(async (transaction) => {
|
|
440
|
-
const run = this.
|
|
464
|
+
const run = await this.readRunFrom(transaction, runId);
|
|
441
465
|
const activeRunId = await transaction.get<string>(ACTIVE_RUN_KEY);
|
|
442
466
|
if (
|
|
443
467
|
!run ||
|
|
@@ -446,9 +470,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
446
470
|
) {
|
|
447
471
|
throw new Error(`run "${runId}" does not require reconciliation`);
|
|
448
472
|
}
|
|
449
|
-
const latest = (
|
|
450
|
-
(await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
|
|
451
|
-
).map(decodeSessionEvent);
|
|
473
|
+
const latest = await new SessionEventLog(transaction).read(run.sessionId);
|
|
452
474
|
const settings = run.configurationSnapshot;
|
|
453
475
|
// The failure is *removed*, not set to `undefined`: a running run that
|
|
454
476
|
// carries a `failure` key is a shape the run record does not allow, and
|
|
@@ -456,15 +478,14 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
456
478
|
// read afterwards. `require` checks it here, where the write is, rather
|
|
457
479
|
// than leaving the projector to fail on every later read.
|
|
458
480
|
const { failure: _failure, ...resumed } = run;
|
|
481
|
+
const resumedRun = this.codec.require({
|
|
482
|
+
...resumed,
|
|
483
|
+
status: "running",
|
|
484
|
+
phase: "executing",
|
|
485
|
+
} satisfies StoredRunV1<Snapshot>);
|
|
459
486
|
await transaction.put(
|
|
460
487
|
key,
|
|
461
|
-
structuredClone(
|
|
462
|
-
this.codec.require({
|
|
463
|
-
...resumed,
|
|
464
|
-
status: "running",
|
|
465
|
-
phase: "executing",
|
|
466
|
-
} satisfies StoredRunV1<Snapshot>),
|
|
467
|
-
),
|
|
488
|
+
structuredClone(storedRunRecordV2(resumedRun)),
|
|
468
489
|
);
|
|
469
490
|
await this.refreshRecoveryAlarm(transaction);
|
|
470
491
|
return { run, latest, settings };
|
|
@@ -477,9 +498,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
477
498
|
recovery.settings,
|
|
478
499
|
);
|
|
479
500
|
} catch (error) {
|
|
480
|
-
const current = this.
|
|
481
|
-
await this.ctx.storage.get<unknown>(key),
|
|
482
|
-
);
|
|
501
|
+
const current = await this.readRun(runId);
|
|
483
502
|
if (current?.status === "reconciliation-required") {
|
|
484
503
|
const previous = recovery.latest.slice(0, current.previousEventCount);
|
|
485
504
|
const failure =
|
|
@@ -511,9 +530,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
511
530
|
private async settledTerminalRunResult(
|
|
512
531
|
runId: string,
|
|
513
532
|
): Promise<BotTurnCompletion | undefined> {
|
|
514
|
-
const run = this.
|
|
515
|
-
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
516
|
-
);
|
|
533
|
+
const run = await this.readRun(runId);
|
|
517
534
|
if (
|
|
518
535
|
run?.status !== "failed" &&
|
|
519
536
|
run?.status !== "cancelled" &&
|
|
@@ -572,10 +589,13 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
572
589
|
if (!run || run.status !== "running") {
|
|
573
590
|
throw new Error(`run "${command.runId}" is not resumable`);
|
|
574
591
|
}
|
|
575
|
-
await transaction.put(
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
592
|
+
await transaction.put(
|
|
593
|
+
key,
|
|
594
|
+
storedRunRecordV2({
|
|
595
|
+
...run,
|
|
596
|
+
phase: "executing",
|
|
597
|
+
} satisfies StoredRunV1<Snapshot>),
|
|
598
|
+
);
|
|
579
599
|
await this.refreshRecoveryAlarm(transaction);
|
|
580
600
|
});
|
|
581
601
|
const result = await this.hooks.executeTurn({
|
|
@@ -592,9 +612,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
592
612
|
await this.completeRun(command.runId, previous, completed, settings);
|
|
593
613
|
return completed;
|
|
594
614
|
} catch (error) {
|
|
595
|
-
const durableRun = this.
|
|
596
|
-
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${command.runId}`),
|
|
597
|
-
);
|
|
615
|
+
const durableRun = await this.readRun(command.runId);
|
|
598
616
|
const events = eventsForFailedRun(durableRun, error);
|
|
599
617
|
const message =
|
|
600
618
|
error instanceof Error ? error.message : "Bot turn failed";
|
|
@@ -652,9 +670,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
652
670
|
private async discardedRunResult(
|
|
653
671
|
runId: string,
|
|
654
672
|
): Promise<BotTurnCompletion | undefined> {
|
|
655
|
-
const run = this.
|
|
656
|
-
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
657
|
-
);
|
|
673
|
+
const run = await this.readRun(runId);
|
|
658
674
|
if (run?.status !== "superseded" && run?.status !== "cancelled") {
|
|
659
675
|
return undefined;
|
|
660
676
|
}
|
|
@@ -669,9 +685,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
669
685
|
private async terminalRunResult(
|
|
670
686
|
runId: string,
|
|
671
687
|
): Promise<BotTurnCompletion | undefined> {
|
|
672
|
-
const run = this.
|
|
673
|
-
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
674
|
-
);
|
|
688
|
+
const run = await this.readRun(runId);
|
|
675
689
|
if (run?.status === "superseded" || run?.status === "cancelled") {
|
|
676
690
|
return { runId, text: "", events: structuredClone(run.events) };
|
|
677
691
|
}
|
|
@@ -714,6 +728,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
714
728
|
// Recovery re-mounts on the recorded turn type, so the resumed Turn
|
|
715
729
|
// sees the same trimmed catalog the evicted one did.
|
|
716
730
|
turnType: storedRunTurnTypeV1(run),
|
|
731
|
+
lane: storedRunLaneV1(run),
|
|
717
732
|
...(storedRunSubagentRoleV1(run)
|
|
718
733
|
? { subagentRole: storedRunSubagentRoleV1(run) }
|
|
719
734
|
: {}),
|
|
@@ -730,9 +745,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
730
745
|
? { admittedRequest: modelState.request.request }
|
|
731
746
|
: {}),
|
|
732
747
|
});
|
|
733
|
-
const durableRun = this.
|
|
734
|
-
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${run.runId}`),
|
|
735
|
-
);
|
|
748
|
+
const durableRun = await this.readRun(run.runId);
|
|
736
749
|
if (!durableRun) throw new Error(`run "${run.runId}" was not accepted`);
|
|
737
750
|
const fullResult = {
|
|
738
751
|
...result,
|
|
@@ -742,9 +755,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
742
755
|
await this.completeRun(run.runId, previous, completed, settings);
|
|
743
756
|
return completed;
|
|
744
757
|
} catch (error) {
|
|
745
|
-
const durableRun = this.
|
|
746
|
-
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${run.runId}`),
|
|
747
|
-
);
|
|
758
|
+
const durableRun = await this.readRun(run.runId);
|
|
748
759
|
const events = durableRun?.events ?? run.events;
|
|
749
760
|
const message =
|
|
750
761
|
error instanceof Error ? error.message : "Bot turn failed";
|
|
@@ -780,16 +791,17 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
780
791
|
|
|
781
792
|
private async deferRunRecovery(runId: string): Promise<void> {
|
|
782
793
|
await this.ctx.storage.transaction(async (transaction) => {
|
|
783
|
-
const run = this.
|
|
784
|
-
await transaction.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
785
|
-
);
|
|
794
|
+
const run = await this.readRunFrom(transaction, runId);
|
|
786
795
|
if (!run || run.status !== "running") {
|
|
787
796
|
throw new Error(`run "${runId}" is not resumable`);
|
|
788
797
|
}
|
|
789
|
-
await transaction.put(
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
798
|
+
await transaction.put(
|
|
799
|
+
`${RUN_PREFIX}${runId}`,
|
|
800
|
+
storedRunRecordV2({
|
|
801
|
+
...run,
|
|
802
|
+
phase: "executing",
|
|
803
|
+
} satisfies StoredRunV1<Snapshot>),
|
|
804
|
+
);
|
|
793
805
|
await this.refreshRecoveryAlarm(transaction);
|
|
794
806
|
});
|
|
795
807
|
}
|
|
@@ -798,9 +810,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
798
810
|
command: OwnedBotTurnCommand,
|
|
799
811
|
): Promise<BotTurnCompletion | undefined> {
|
|
800
812
|
const { runId } = command;
|
|
801
|
-
const run = this.
|
|
802
|
-
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
803
|
-
);
|
|
813
|
+
const run = await this.readRun(runId);
|
|
804
814
|
if (!run) return undefined;
|
|
805
815
|
if (run.commandFingerprint !== botTurnCommandFingerprintV1(command)) {
|
|
806
816
|
throw new BotTurnRefusedError(
|
|
@@ -900,10 +910,13 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
900
910
|
throw new Error(`run "${runId}" is not resumable`);
|
|
901
911
|
}
|
|
902
912
|
if (run.compositionGenerationId === compositionGenerationId) return;
|
|
903
|
-
await transaction.put(
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
913
|
+
await transaction.put(
|
|
914
|
+
key,
|
|
915
|
+
storedRunRecordV2({
|
|
916
|
+
...run,
|
|
917
|
+
compositionGenerationId,
|
|
918
|
+
} satisfies StoredRunV1<Snapshot>),
|
|
919
|
+
);
|
|
907
920
|
});
|
|
908
921
|
}
|
|
909
922
|
|
|
@@ -928,6 +941,26 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
928
941
|
});
|
|
929
942
|
return;
|
|
930
943
|
}
|
|
944
|
+
const [activeBeforeAlarm, pendingUser, pendingAgents] = await Promise.all([
|
|
945
|
+
this.ctx.storage.get<string>(ACTIVE_RUN_KEY),
|
|
946
|
+
this.ctx.storage.get<string>(PENDING_RUN_KEY),
|
|
947
|
+
this.ctx.storage.list<string>({
|
|
948
|
+
prefix: PENDING_AGENT_RUN_PREFIX,
|
|
949
|
+
limit: 1,
|
|
950
|
+
}),
|
|
951
|
+
]);
|
|
952
|
+
// An admitted Turn is work already owed. It runs before a due Routine;
|
|
953
|
+
// otherwise a busy schedule can starve a Bot-to-Bot question indefinitely.
|
|
954
|
+
if (!activeBeforeAlarm && (pendingUser || pendingAgents.size > 0)) {
|
|
955
|
+
try {
|
|
956
|
+
await this.recoverQueuedRun();
|
|
957
|
+
} finally {
|
|
958
|
+
await this.ctx.storage.transaction((transaction) =>
|
|
959
|
+
this.refreshRecoveryAlarm(transaction),
|
|
960
|
+
);
|
|
961
|
+
}
|
|
962
|
+
return;
|
|
963
|
+
}
|
|
931
964
|
await this.hooks.settleScheduledWork();
|
|
932
965
|
const activeRunId = await this.ctx.storage.get<string>(ACTIVE_RUN_KEY);
|
|
933
966
|
if (activeRunId) {
|
|
@@ -1058,7 +1091,11 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1058
1091
|
return this.ctx.storage.transaction(async (transaction) => {
|
|
1059
1092
|
const active = await transaction.get<string>(ACTIVE_RUN_KEY);
|
|
1060
1093
|
const pending = await transaction.get<string>(PENDING_RUN_KEY);
|
|
1061
|
-
|
|
1094
|
+
const pendingAgent = await transaction.list<string>({
|
|
1095
|
+
prefix: PENDING_AGENT_RUN_PREFIX,
|
|
1096
|
+
limit: 1,
|
|
1097
|
+
});
|
|
1098
|
+
if (active || pending || pendingAgent.size > 0) {
|
|
1062
1099
|
// A typed refusal, not a bare Error: the Durable Object boundary turns
|
|
1063
1100
|
// this one case into a 409 value rather than letting it escape the
|
|
1064
1101
|
// object's entry frame as an uncaught exception.
|
|
@@ -1083,12 +1120,17 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1083
1120
|
ordinal: current.ordinal + 1,
|
|
1084
1121
|
startedAt: endedAt,
|
|
1085
1122
|
};
|
|
1123
|
+
const currentSessionId = conversationSessionIdV1(base, current.ordinal);
|
|
1124
|
+
const nextSessionId = conversationSessionIdV1(base, next.ordinal);
|
|
1125
|
+
const eventLog = new SessionEventLog(transaction);
|
|
1126
|
+
await eventLog.migrate(currentSessionId);
|
|
1127
|
+
await eventLog.clearCurrent(nextSessionId);
|
|
1086
1128
|
await transaction.put({
|
|
1087
1129
|
[CONVERSATION_KEY]: next,
|
|
1088
1130
|
[CONVERSATION_INDEX_KEY]: [
|
|
1089
1131
|
{
|
|
1090
1132
|
schemaVersion: 1 as const,
|
|
1091
|
-
sessionId:
|
|
1133
|
+
sessionId: currentSessionId,
|
|
1092
1134
|
ordinal: current.ordinal,
|
|
1093
1135
|
startedAt: current.startedAt,
|
|
1094
1136
|
endedAt,
|
|
@@ -1097,13 +1139,10 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1097
1139
|
]
|
|
1098
1140
|
.sort((left, right) => right.ordinal - left.ordinal)
|
|
1099
1141
|
.slice(0, MAX_LISTED_CONVERSATIONS),
|
|
1100
|
-
// The next Turn derives its messages from an empty log. Nothing is
|
|
1101
|
-
// deleted: `run:<id>` still holds every event of every Turn.
|
|
1102
|
-
[LATEST_EVENTS_KEY]: [],
|
|
1103
1142
|
});
|
|
1104
1143
|
return {
|
|
1105
1144
|
schemaVersion: 1 as const,
|
|
1106
|
-
sessionId:
|
|
1145
|
+
sessionId: nextSessionId,
|
|
1107
1146
|
ordinal: next.ordinal,
|
|
1108
1147
|
startedAt: next.startedAt,
|
|
1109
1148
|
};
|
|
@@ -1135,9 +1174,64 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1135
1174
|
async readStoredRun(
|
|
1136
1175
|
runId: string,
|
|
1137
1176
|
): Promise<StoredRunV1<Snapshot> | undefined> {
|
|
1138
|
-
return this.
|
|
1177
|
+
return this.readRunFrom(this.ctx.storage, runId);
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1180
|
+
private async readRunFrom(
|
|
1181
|
+
storage: SessionEventLogStorage,
|
|
1182
|
+
runId: string,
|
|
1183
|
+
): Promise<StoredRunV1<Snapshot> | undefined> {
|
|
1184
|
+
const run = this.codec.optional(
|
|
1185
|
+
await storage.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
1186
|
+
);
|
|
1187
|
+
if (!run?.eventRange) return run;
|
|
1188
|
+
const events = await new SessionEventLog(storage).readRange(
|
|
1189
|
+
run.sessionId,
|
|
1190
|
+
run.eventRange.startSeq,
|
|
1191
|
+
run.eventRange.endSeq,
|
|
1192
|
+
);
|
|
1193
|
+
if (events.length !== run.eventRange.endSeq - run.eventRange.startSeq) {
|
|
1194
|
+
throw new Error(`run "${run.runId}" has an incomplete event range`);
|
|
1195
|
+
}
|
|
1196
|
+
return this.codec.require({ ...run, events });
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
/** Exact Session history, reconstructed through the paged durable log. */
|
|
1200
|
+
async readSessionEvents(sessionId: string): Promise<SessionEvent[]> {
|
|
1201
|
+
return new SessionEventLog(this.ctx.storage).read(sessionId);
|
|
1202
|
+
}
|
|
1203
|
+
|
|
1204
|
+
/**
|
|
1205
|
+
* The bounded durable event projections for a run. This is the inspection
|
|
1206
|
+
* path: recovery and client transcript projection use `readStoredRun` and
|
|
1207
|
+
* therefore receive exact events, while a debug snapshot never hydrates a
|
|
1208
|
+
* multi-megabyte prompt merely to cut it again.
|
|
1209
|
+
*/
|
|
1210
|
+
async readRunEventProjections(runId: string): Promise<
|
|
1211
|
+
| {
|
|
1212
|
+
run: StoredRunV1<Snapshot>;
|
|
1213
|
+
events: unknown[];
|
|
1214
|
+
eventCount: number;
|
|
1215
|
+
}
|
|
1216
|
+
| undefined
|
|
1217
|
+
> {
|
|
1218
|
+
const run = this.codec.optional(
|
|
1139
1219
|
await this.ctx.storage.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
1140
1220
|
);
|
|
1221
|
+
if (!run) return undefined;
|
|
1222
|
+
if (!run.eventRange) {
|
|
1223
|
+
return { run, events: run.events, eventCount: run.events.length };
|
|
1224
|
+
}
|
|
1225
|
+
const events = await new SessionEventLog(this.ctx.storage).readProjections(
|
|
1226
|
+
run.sessionId,
|
|
1227
|
+
run.eventRange.startSeq,
|
|
1228
|
+
run.eventRange.endSeq,
|
|
1229
|
+
);
|
|
1230
|
+
const eventCount = run.eventRange.endSeq - run.eventRange.startSeq;
|
|
1231
|
+
if (events.length !== eventCount) {
|
|
1232
|
+
throw new Error(`run "${run.runId}" has an incomplete event range`);
|
|
1233
|
+
}
|
|
1234
|
+
return { run, events, eventCount };
|
|
1141
1235
|
}
|
|
1142
1236
|
|
|
1143
1237
|
/** Durable run record, checked against the key it was looked up by. */
|
|
@@ -1183,9 +1277,9 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1183
1277
|
if (runId === this.executingRunId) return true;
|
|
1184
1278
|
const run = await this.readRun(runId);
|
|
1185
1279
|
if (!run || run.status !== "running") return false;
|
|
1186
|
-
const sessionEvents = (
|
|
1187
|
-
|
|
1188
|
-
)
|
|
1280
|
+
const sessionEvents = await new SessionEventLog(this.ctx.storage).read(
|
|
1281
|
+
run.sessionId,
|
|
1282
|
+
);
|
|
1189
1283
|
if (runLivenessV1({ run, sessionEvents }).working) return true;
|
|
1190
1284
|
await this.settleStaleRun(runId);
|
|
1191
1285
|
return false;
|
|
@@ -1202,13 +1296,9 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1202
1296
|
private async settleStaleRun(runId: string): Promise<void> {
|
|
1203
1297
|
await this.ctx.storage.transaction(async (transaction) => {
|
|
1204
1298
|
if (runId === this.executingRunId) return;
|
|
1205
|
-
const run = this.
|
|
1206
|
-
await transaction.get<unknown>(`${RUN_PREFIX}${runId}`),
|
|
1207
|
-
);
|
|
1299
|
+
const run = await this.readRunFrom(transaction, runId);
|
|
1208
1300
|
if (!run || run.runId !== runId || run.status !== "running") return;
|
|
1209
|
-
const latest = (
|
|
1210
|
-
(await transaction.get<SessionEvent[]>(LATEST_EVENTS_KEY)) ?? []
|
|
1211
|
-
).map(decodeSessionEvent);
|
|
1301
|
+
const latest = await new SessionEventLog(transaction).read(run.sessionId);
|
|
1212
1302
|
if (runLivenessV1({ run, sessionEvents: latest }).working) return;
|
|
1213
1303
|
await failStoredRun(
|
|
1214
1304
|
this.codec,
|
|
@@ -1283,9 +1373,13 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1283
1373
|
async refreshRecoveryAlarm(
|
|
1284
1374
|
transaction: DurableObjectTransaction,
|
|
1285
1375
|
): Promise<void> {
|
|
1286
|
-
const [activeRunId, scheduled] = await Promise.all([
|
|
1376
|
+
const [activeRunId, scheduled, pendingAgents] = await Promise.all([
|
|
1287
1377
|
transaction.get<string>(ACTIVE_RUN_KEY),
|
|
1288
1378
|
this.hooks.scheduledDeadlines(transaction),
|
|
1379
|
+
transaction.list<string>({
|
|
1380
|
+
prefix: PENDING_AGENT_RUN_PREFIX,
|
|
1381
|
+
limit: 1,
|
|
1382
|
+
}),
|
|
1289
1383
|
]);
|
|
1290
1384
|
const activeRun = activeRunId
|
|
1291
1385
|
? this.codec.optional(
|
|
@@ -1300,7 +1394,8 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1300
1394
|
deadlines.push(Date.now() + RECOVERY_ALARM_DELAY_MS);
|
|
1301
1395
|
} else if (
|
|
1302
1396
|
!activeRunId &&
|
|
1303
|
-
(await transaction.get<string>(PENDING_RUN_KEY))
|
|
1397
|
+
((await transaction.get<string>(PENDING_RUN_KEY)) ||
|
|
1398
|
+
pendingAgents.size > 0)
|
|
1304
1399
|
) {
|
|
1305
1400
|
// A Turn admitted and waiting is work this object owes, so it keeps the
|
|
1306
1401
|
// recovery alarm even with nothing running.
|
|
@@ -1386,12 +1481,64 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1386
1481
|
throw new Error("Bot authority does not match its durable identity");
|
|
1387
1482
|
}
|
|
1388
1483
|
const activeRunId = await transaction.get<string>(ACTIVE_RUN_KEY);
|
|
1389
|
-
const
|
|
1390
|
-
|
|
1484
|
+
const pendingUserRunId = await transaction.get<string>(PENDING_RUN_KEY);
|
|
1485
|
+
const lane = command.lane ?? defaultRunLaneV1(command.turnType ?? "chat");
|
|
1486
|
+
const activeRun = activeRunId
|
|
1487
|
+
? this.codec.optional(
|
|
1488
|
+
await transaction.get<unknown>(`${RUN_PREFIX}${activeRunId}`),
|
|
1489
|
+
)
|
|
1391
1490
|
: undefined;
|
|
1392
|
-
const
|
|
1393
|
-
|
|
1394
|
-
)
|
|
1491
|
+
const pendingAgents = await transaction.list<string>({
|
|
1492
|
+
prefix: PENDING_AGENT_RUN_PREFIX,
|
|
1493
|
+
});
|
|
1494
|
+
const hasPendingAgent = pendingAgents.size > 0;
|
|
1495
|
+
let supersede: ((supersededBy: string) => Promise<boolean>) | undefined;
|
|
1496
|
+
if (activeRunId) {
|
|
1497
|
+
if (
|
|
1498
|
+
lane === "agent" &&
|
|
1499
|
+
activeRun?.status === "reconciliation-required"
|
|
1500
|
+
) {
|
|
1501
|
+
throw new BotTurnRefusedError(
|
|
1502
|
+
"reconciliation-required",
|
|
1503
|
+
"bot cannot admit agent work while its active run requires reconciliation",
|
|
1504
|
+
);
|
|
1505
|
+
}
|
|
1506
|
+
if (lane === "user") {
|
|
1507
|
+
supersede = await this.planSupersede(
|
|
1508
|
+
transaction,
|
|
1509
|
+
command,
|
|
1510
|
+
activeRunId,
|
|
1511
|
+
);
|
|
1512
|
+
} else if (lane === "background") {
|
|
1513
|
+
throw new BotTurnRefusedError(
|
|
1514
|
+
"busy",
|
|
1515
|
+
"bot already has an active run",
|
|
1516
|
+
);
|
|
1517
|
+
}
|
|
1518
|
+
} else if (
|
|
1519
|
+
lane === "background" &&
|
|
1520
|
+
(pendingUserRunId || hasPendingAgent)
|
|
1521
|
+
) {
|
|
1522
|
+
throw new BotTurnRefusedError(
|
|
1523
|
+
"busy",
|
|
1524
|
+
"bot has queued conversational work",
|
|
1525
|
+
);
|
|
1526
|
+
}
|
|
1527
|
+
const queued =
|
|
1528
|
+
Boolean(activeRunId) ||
|
|
1529
|
+
(lane === "agent" && (Boolean(pendingUserRunId) || hasPendingAgent));
|
|
1530
|
+
if (
|
|
1531
|
+
lane === "agent" &&
|
|
1532
|
+
queued &&
|
|
1533
|
+
pendingAgents.size >= MAX_PENDING_AGENT_RUNS_V1
|
|
1534
|
+
) {
|
|
1535
|
+
throw new BotTurnRefusedError(
|
|
1536
|
+
"busy",
|
|
1537
|
+
`bot agent queue is full (${MAX_PENDING_AGENT_RUNS_V1} Turns)`,
|
|
1538
|
+
);
|
|
1539
|
+
}
|
|
1540
|
+
const eventLog = new SessionEventLog(transaction);
|
|
1541
|
+
const storedEvents = await eventLog.migrate(command.sessionId);
|
|
1395
1542
|
// A Turn that died between `turn/start` and `turn/end` — an event the
|
|
1396
1543
|
// encoder refused, a durable write that failed — left the log open, and
|
|
1397
1544
|
// every later Turn failed validation with "turn N started while turn
|
|
@@ -1407,11 +1554,6 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1407
1554
|
// that Turn's end: a `running` record is, and so is a
|
|
1408
1555
|
// `reconciliation-required` one, whose Turn is held open on purpose
|
|
1409
1556
|
// until its outcome is retrieved. Nothing else is.
|
|
1410
|
-
const activeRun = activeRunId
|
|
1411
|
-
? this.codec.optional(
|
|
1412
|
-
await transaction.get<unknown>(`${RUN_PREFIX}${activeRunId}`),
|
|
1413
|
-
)
|
|
1414
|
-
: undefined;
|
|
1415
1557
|
const stillOwned =
|
|
1416
1558
|
activeRun?.status === "running" ||
|
|
1417
1559
|
activeRun?.status === "reconciliation-required";
|
|
@@ -1426,12 +1568,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1426
1568
|
? undefined
|
|
1427
1569
|
: repairedSessionLogV1(command.sessionId, storedEvents);
|
|
1428
1570
|
const latestEvents = repaired ?? storedEvents;
|
|
1429
|
-
if (repaired)
|
|
1430
|
-
await transaction.put(
|
|
1431
|
-
LATEST_EVENTS_KEY,
|
|
1432
|
-
structuredClone(latestEvents.map(decodeSessionEvent)),
|
|
1433
|
-
);
|
|
1434
|
-
}
|
|
1571
|
+
if (repaired) await eventLog.rewrite(command.sessionId, latestEvents);
|
|
1435
1572
|
const admittedSettings = await this.hooks.admittedSnapshot(
|
|
1436
1573
|
transaction,
|
|
1437
1574
|
settings,
|
|
@@ -1449,7 +1586,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1449
1586
|
// A queued Turn is admitted — durable, ordered, and owed a terminal
|
|
1450
1587
|
// state — but has not started. Its `previousEventCount` is recomputed
|
|
1451
1588
|
// when it is promoted, because the Turn ahead of it is still writing.
|
|
1452
|
-
phase:
|
|
1589
|
+
phase: queued ? "queued" : "admitted",
|
|
1453
1590
|
compositionGenerationId: pin.generationId,
|
|
1454
1591
|
configurationSnapshot: structuredClone(admittedSettings),
|
|
1455
1592
|
previousEventCount: latestEvents.length,
|
|
@@ -1464,10 +1601,15 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1464
1601
|
: {}),
|
|
1465
1602
|
} satisfies StoredRunV1<Snapshot>);
|
|
1466
1603
|
await transaction.put({
|
|
1467
|
-
[key]: admittedRun,
|
|
1604
|
+
[key]: storedRunRecordV2(admittedRun),
|
|
1468
1605
|
[runIndexKey(command.acceptedAt, command.runId)]: command.runId,
|
|
1469
|
-
...(
|
|
1470
|
-
?
|
|
1606
|
+
...(queued
|
|
1607
|
+
? lane === "agent"
|
|
1608
|
+
? {
|
|
1609
|
+
[pendingAgentRunKey(command.acceptedAt, command.runId)]:
|
|
1610
|
+
command.runId,
|
|
1611
|
+
}
|
|
1612
|
+
: { [PENDING_RUN_KEY]: command.runId }
|
|
1471
1613
|
: { [ACTIVE_RUN_KEY]: command.runId }),
|
|
1472
1614
|
[IDENTITY_KEY]: identity ?? {
|
|
1473
1615
|
userId: command.userId,
|
|
@@ -1476,13 +1618,15 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1476
1618
|
});
|
|
1477
1619
|
const interrupted = supersede ? await supersede(command.runId) : false;
|
|
1478
1620
|
await this.refreshRecoveryAlarm(transaction);
|
|
1479
|
-
if (
|
|
1621
|
+
if (queued) {
|
|
1480
1622
|
return {
|
|
1481
1623
|
kind: "queued" as const,
|
|
1482
1624
|
// Only a Turn whose supersede intent was actually recorded is
|
|
1483
1625
|
// interrupted. One that had not dispatched a model request is left
|
|
1484
1626
|
// to finish, and the new message simply waits behind it.
|
|
1485
|
-
...(interrupted
|
|
1627
|
+
...(interrupted && activeRunId
|
|
1628
|
+
? { interrupt: { runId: activeRunId } }
|
|
1629
|
+
: {}),
|
|
1486
1630
|
};
|
|
1487
1631
|
}
|
|
1488
1632
|
return {
|
|
@@ -1521,9 +1665,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1521
1665
|
if (lane !== "user" || !command.supersedes) {
|
|
1522
1666
|
throw new BotTurnRefusedError("busy", "bot already has an active run");
|
|
1523
1667
|
}
|
|
1524
|
-
const active = this.
|
|
1525
|
-
await transaction.get<unknown>(`${RUN_PREFIX}${activeRunId}`),
|
|
1526
|
-
);
|
|
1668
|
+
const active = await this.readRunFrom(transaction, activeRunId);
|
|
1527
1669
|
if (!active)
|
|
1528
1670
|
throw new BotTurnRefusedError("busy", "bot already has an active run");
|
|
1529
1671
|
if (active.status === "reconciliation-required") {
|
|
@@ -1551,15 +1693,14 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1551
1693
|
}
|
|
1552
1694
|
if (!dispatched) return false;
|
|
1553
1695
|
if (active.supersededAt) return true;
|
|
1696
|
+
const superseded = this.codec.require({
|
|
1697
|
+
...active,
|
|
1698
|
+
supersededAt: new Date().toISOString(),
|
|
1699
|
+
supersededBy,
|
|
1700
|
+
} satisfies StoredRunV1<Snapshot>);
|
|
1554
1701
|
await transaction.put(
|
|
1555
1702
|
`${RUN_PREFIX}${activeRunId}`,
|
|
1556
|
-
structuredClone(
|
|
1557
|
-
this.codec.require({
|
|
1558
|
-
...active,
|
|
1559
|
-
supersededAt: new Date().toISOString(),
|
|
1560
|
-
supersededBy,
|
|
1561
|
-
} satisfies StoredRunV1<Snapshot>),
|
|
1562
|
-
),
|
|
1703
|
+
structuredClone(storedRunRecordV2(superseded)),
|
|
1563
1704
|
);
|
|
1564
1705
|
return true;
|
|
1565
1706
|
};
|
|
@@ -1580,18 +1721,14 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1580
1721
|
return;
|
|
1581
1722
|
}
|
|
1582
1723
|
const { responseText: _text, failure: _failure, ...settled } = queued;
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
supersededBy,
|
|
1592
|
-
} satisfies StoredRunV1<Snapshot>),
|
|
1593
|
-
),
|
|
1594
|
-
);
|
|
1724
|
+
const superseded = this.codec.require({
|
|
1725
|
+
...settled,
|
|
1726
|
+
status: "superseded",
|
|
1727
|
+
phase: "admitted",
|
|
1728
|
+
supersededAt: new Date().toISOString(),
|
|
1729
|
+
supersededBy,
|
|
1730
|
+
} satisfies StoredRunV1<Snapshot>);
|
|
1731
|
+
await transaction.put(key, structuredClone(storedRunRecordV2(superseded)));
|
|
1595
1732
|
}
|
|
1596
1733
|
|
|
1597
1734
|
private async persistRunEvents(
|
|
@@ -1604,11 +1741,10 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1604
1741
|
if (durableEvents.length === 0) return;
|
|
1605
1742
|
const key = `${RUN_PREFIX}${runId}`;
|
|
1606
1743
|
await this.ctx.storage.transaction(async (transaction) => {
|
|
1607
|
-
const run = this.
|
|
1744
|
+
const run = await this.readRunFrom(transaction, runId);
|
|
1608
1745
|
if (!run) throw new Error(`run "${runId}" was not accepted`);
|
|
1609
|
-
const
|
|
1610
|
-
|
|
1611
|
-
).map(decodeSessionEvent);
|
|
1746
|
+
const eventLog = new SessionEventLog(transaction);
|
|
1747
|
+
const latest = await eventLog.read(run.sessionId);
|
|
1612
1748
|
for (const [index, event] of durableEvents.entries()) {
|
|
1613
1749
|
if (event.seq !== latest.length + index) {
|
|
1614
1750
|
throw new Error(
|
|
@@ -1618,12 +1754,13 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1618
1754
|
}
|
|
1619
1755
|
const next = this.codec.require({
|
|
1620
1756
|
...run,
|
|
1621
|
-
|
|
1757
|
+
...storedRunEventFieldsV2(run.previousEventCount, [
|
|
1758
|
+
...run.events,
|
|
1759
|
+
...durableEvents,
|
|
1760
|
+
]),
|
|
1622
1761
|
} satisfies StoredRunV1<Snapshot>);
|
|
1623
|
-
await
|
|
1624
|
-
|
|
1625
|
-
[LATEST_EVENTS_KEY]: structuredClone([...latest, ...durableEvents]),
|
|
1626
|
-
});
|
|
1762
|
+
await eventLog.append(run.sessionId, durableEvents);
|
|
1763
|
+
await transaction.put(key, structuredClone(storedRunRecordV2(next)));
|
|
1627
1764
|
});
|
|
1628
1765
|
}
|
|
1629
1766
|
|
|
@@ -1792,7 +1929,16 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1792
1929
|
* do the promoting itself.
|
|
1793
1930
|
*/
|
|
1794
1931
|
private async recoverQueuedRun(): Promise<void> {
|
|
1795
|
-
const
|
|
1932
|
+
const pendingUserRunId =
|
|
1933
|
+
await this.ctx.storage.get<string>(PENDING_RUN_KEY);
|
|
1934
|
+
const pendingAgents = pendingUserRunId
|
|
1935
|
+
? new Map<string, string>()
|
|
1936
|
+
: await this.ctx.storage.list<string>({
|
|
1937
|
+
prefix: PENDING_AGENT_RUN_PREFIX,
|
|
1938
|
+
limit: 1,
|
|
1939
|
+
});
|
|
1940
|
+
const pendingRunId =
|
|
1941
|
+
pendingUserRunId ?? pendingAgents.values().next().value;
|
|
1796
1942
|
if (!pendingRunId || this.queuedWaiters.has(pendingRunId)) return;
|
|
1797
1943
|
if (pendingRunId === this.executingRunId) return;
|
|
1798
1944
|
const durableIdentity =
|
|
@@ -1823,6 +1969,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1823
1969
|
acceptedAt: run.acceptedAt,
|
|
1824
1970
|
text: run.input,
|
|
1825
1971
|
turnType: storedRunTurnTypeV1(run),
|
|
1972
|
+
lane: storedRunLaneV1(run),
|
|
1826
1973
|
...(storedRunSubagentRoleV1(run)
|
|
1827
1974
|
? { subagentRole: storedRunSubagentRoleV1(run) }
|
|
1828
1975
|
: {}),
|
|
@@ -1841,7 +1988,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1841
1988
|
const recovery = await this.ctx.storage.transaction(async (transaction) => {
|
|
1842
1989
|
const current = await transaction.get<string>(ACTIVE_RUN_KEY);
|
|
1843
1990
|
if (!current || current === this.executingRunId) return undefined;
|
|
1844
|
-
const run = this.
|
|
1991
|
+
const run = await this.readRunFrom(transaction, activeRunId);
|
|
1845
1992
|
if (run?.status === "reconciliation-required") {
|
|
1846
1993
|
await this.refreshRecoveryAlarm(transaction);
|
|
1847
1994
|
return undefined;
|
|
@@ -1850,9 +1997,8 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1850
1997
|
await this.refreshRecoveryAlarm(transaction);
|
|
1851
1998
|
return undefined;
|
|
1852
1999
|
}
|
|
1853
|
-
const
|
|
1854
|
-
|
|
1855
|
-
).map(decodeSessionEvent);
|
|
2000
|
+
const eventLog = new SessionEventLog(transaction);
|
|
2001
|
+
const latest = await eventLog.read(run.sessionId);
|
|
1856
2002
|
// A Turn the User stopped, or one a later message replaced, is terminal
|
|
1857
2003
|
// in intent before recovery ever looks at it. There is nothing to
|
|
1858
2004
|
// recover: no answer is owed, and the provider outcome cannot change what
|
|
@@ -1928,14 +2074,20 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1928
2074
|
}
|
|
1929
2075
|
if (plan.kind === "restart") {
|
|
1930
2076
|
const settings = run.configurationSnapshot;
|
|
1931
|
-
await
|
|
1932
|
-
|
|
2077
|
+
await eventLog.rewrite(run.sessionId, plan.previous);
|
|
2078
|
+
await transaction.put(
|
|
2079
|
+
key,
|
|
2080
|
+
storedRunRecordV2({
|
|
1933
2081
|
...run,
|
|
1934
2082
|
events: [],
|
|
2083
|
+
eventRange: {
|
|
2084
|
+
startSeq: plan.previous.length,
|
|
2085
|
+
endSeq: plan.previous.length,
|
|
2086
|
+
},
|
|
2087
|
+
previousEventCount: plan.previous.length,
|
|
1935
2088
|
phase: "admitted",
|
|
1936
|
-
} satisfies StoredRunV1<Snapshot
|
|
1937
|
-
|
|
1938
|
-
});
|
|
2089
|
+
} satisfies StoredRunV1<Snapshot>),
|
|
2090
|
+
);
|
|
1939
2091
|
await this.refreshRecoveryAlarm(transaction);
|
|
1940
2092
|
return {
|
|
1941
2093
|
kind: "restart" as const,
|
|
@@ -1946,24 +2098,31 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1946
2098
|
}
|
|
1947
2099
|
if (plan.kind === "resume") {
|
|
1948
2100
|
const settings = run.configurationSnapshot;
|
|
1949
|
-
await transaction.put(
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
2101
|
+
await transaction.put(
|
|
2102
|
+
key,
|
|
2103
|
+
storedRunRecordV2({
|
|
2104
|
+
...run,
|
|
2105
|
+
phase: "executing",
|
|
2106
|
+
} satisfies StoredRunV1<Snapshot>),
|
|
2107
|
+
);
|
|
1953
2108
|
await this.refreshRecoveryAlarm(transaction);
|
|
1954
2109
|
return { kind: "resume" as const, run, latest, settings };
|
|
1955
2110
|
}
|
|
1956
|
-
await
|
|
1957
|
-
|
|
2111
|
+
await eventLog.append(run.sessionId, plan.repairs);
|
|
2112
|
+
await transaction.put(
|
|
2113
|
+
key,
|
|
2114
|
+
storedRunRecordV2({
|
|
1958
2115
|
...run,
|
|
1959
|
-
|
|
2116
|
+
...storedRunEventFieldsV2(run.previousEventCount, [
|
|
2117
|
+
...run.events,
|
|
2118
|
+
...plan.repairs,
|
|
2119
|
+
]),
|
|
1960
2120
|
status: "reconciliation-required",
|
|
1961
2121
|
phase: "reconciliation-required",
|
|
1962
2122
|
failure:
|
|
1963
2123
|
"Execution outcome requires reconciliation before it can resume",
|
|
1964
|
-
} satisfies StoredRunV1<Snapshot
|
|
1965
|
-
|
|
1966
|
-
});
|
|
2124
|
+
} satisfies StoredRunV1<Snapshot>),
|
|
2125
|
+
);
|
|
1967
2126
|
await this.refreshRecoveryAlarm(transaction);
|
|
1968
2127
|
return undefined;
|
|
1969
2128
|
});
|
|
@@ -1987,6 +2146,7 @@ export class BotDurableAuthority<Snapshot> {
|
|
|
1987
2146
|
acceptedAt: recovery.run.acceptedAt,
|
|
1988
2147
|
text: recovery.run.input,
|
|
1989
2148
|
turnType: storedRunTurnTypeV1(recovery.run),
|
|
2149
|
+
lane: storedRunLaneV1(recovery.run),
|
|
1990
2150
|
...(storedRunSubagentRoleV1(recovery.run)
|
|
1991
2151
|
? { subagentRole: storedRunSubagentRoleV1(recovery.run) }
|
|
1992
2152
|
: {}),
|