@modelprofile.com/flexharness 18.0.0 → 18.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -28,6 +28,7 @@ import type {
28
28
  IFlexPendingReversionRelease,
29
29
  IFlexResolvedScope,
30
30
  IFlexResultLimits,
31
+ IFlexScopeLoadMetrics,
31
32
  IFlexScopeSnapshot,
32
33
  IFlexSession,
33
34
  IFlexSessionImportLimits,
@@ -43,6 +44,8 @@ import type {
43
44
  TFlexMessagePart,
44
45
  TFlexProjectionSnapshot,
45
46
  TFlexReversionPolicy,
47
+ TFlexScopeLoadObserver,
48
+ TFlexScopeLoadOutcome,
46
49
  } from './interfaces.js';
47
50
  import { InMemoryFlexHarnessStores } from './classes.stores.js';
48
51
  import { FlexCodeSandbox } from './classes.codesandbox.js';
@@ -50,9 +53,12 @@ import {
50
53
  assertJsonSerializable,
51
54
  canonicalJson,
52
55
  cloneSerializable,
56
+ cloneSerializableSliced,
53
57
  deepFreeze,
54
58
  resolveJsonLimits,
55
59
  } from './utils.json.js';
60
+ import { FlexTimeSlicer } from './classes.timeslicer.js';
61
+ import { liveScopeSessions, normalizeScopeSnapshot } from './utils.scopesnapshot.js';
56
62
  import { hydrateAgentMessages } from './utils.prompt.js';
57
63
  import { createProjectionChange, freezeProjectionEntries } from './utils.projectionchange.js';
58
64
  import {
@@ -95,6 +101,8 @@ import type {
95
101
  IActiveRun,
96
102
  IScopeAdmissionState,
97
103
  IFlexMessageCursor,
104
+ IProjectionContent,
105
+ IScopeLoadPass,
98
106
  TEventDetails,
99
107
  } from './interfaces.flexharness.js';
100
108
  import {
@@ -131,20 +139,6 @@ const defaultMaxSubagentCallsPerRun = 32;
131
139
  const maximumMaxSubagentCallsPerRun = 128;
132
140
  const repairCancellationMessage = 'The process stopped before this run completed.';
133
141
 
134
- function deriveLegacySessionGenerationId(
135
- storageKey: string,
136
- sessionId: string,
137
- createdAt: string,
138
- ): string {
139
- const digest = plugins.crypto.createHash('sha256').update(JSON.stringify([
140
- 'flexharness-legacy-session-generation-v1',
141
- storageKey,
142
- sessionId,
143
- createdAt,
144
- ])).digest('hex');
145
- return `legacy_${digest}`;
146
- }
147
-
148
142
  /**
149
143
  * The engine behind {@link FlexHarness}: its options, scope state, events and error projection, and
150
144
  * the services that implement its API.
@@ -172,6 +166,7 @@ export class FlexHarnessCore<TScope> {
172
166
  public readonly turnReversionProvider: IFlexHarnessOptions<TScope>['turnReversionProvider'];
173
167
  public readonly reversionPolicy: TFlexReversionPolicy;
174
168
  private readonly externalErrorProjector?: TFlexExternalErrorProjector;
169
+ private readonly scopeLoadObserver?: TFlexScopeLoadObserver;
175
170
  public readonly subagents: ReadonlyMap<string, Readonly<IFlexSubagentDefinition>>;
176
171
  public readonly builtInTools: INormalizedBuiltInTools;
177
172
  public readonly cacheWarmOptions: INormalizedCacheWarmOptions;
@@ -180,6 +175,8 @@ export class FlexHarnessCore<TScope> {
180
175
  public readonly maxSubagentDepth: number;
181
176
  public readonly maxSubagentCallsPerRun: number;
182
177
  public readonly stateLoads = new Map<string, Promise<IStorageState>>();
178
+ /** The state each settled successful load loaded. */
179
+ private readonly loadedStates = new WeakMap<Promise<IStorageState>, IStorageState>();
183
180
  public readonly scopeAdmissions = new Map<string, IScopeAdmissionState>();
184
181
  public readonly listeners = new Set<TFlexHarnessEventListener>();
185
182
  public readonly trustedInternalErrors = new WeakSet<object>();
@@ -248,6 +245,10 @@ export class FlexHarnessCore<TScope> {
248
245
  );
249
246
  }
250
247
  this.externalErrorProjector = options.externalErrorProjector;
248
+ if (options.scopeLoadObserver !== undefined && typeof options.scopeLoadObserver !== 'function') {
249
+ throw new FlexHarnessValidationError('scopeLoadObserver must be a function.');
250
+ }
251
+ this.scopeLoadObserver = options.scopeLoadObserver;
251
252
  this.subagents = normalizeSubagents(options.subagents);
252
253
  this.builtInTools = normalizeBuiltInTools(options.builtInTools);
253
254
  this.cacheWarmOptions = normalizeCacheWarmOptions(options.cacheWarm);
@@ -358,6 +359,15 @@ export class FlexHarnessCore<TScope> {
358
359
 
359
360
  public canonicalOutcomes(events: readonly plugins.TAgentEvent[]): Map<string, TCanonicalOutcome> {
360
361
  const outcomes = new Map<string, TCanonicalOutcome>();
362
+ for (const event of events) this.rememberCanonicalOutcomes(outcomes, event);
363
+ return outcomes;
364
+ }
365
+
366
+ /** Adds the canonical outcomes `event` records to `outcomes`. */
367
+ private rememberCanonicalOutcomes(
368
+ outcomes: Map<string, TCanonicalOutcome>,
369
+ event: plugins.TAgentEvent,
370
+ ): void {
361
371
  const remember = (generationId: string, outcome: TCanonicalOutcome) => {
362
372
  const existing = outcomes.get(generationId);
363
373
  if (existing && existing !== outcome) {
@@ -365,15 +375,12 @@ export class FlexHarnessCore<TScope> {
365
375
  }
366
376
  outcomes.set(generationId, outcome);
367
377
  };
368
- for (const event of events) {
369
- if (event.type === 'generation-outcome') remember(event.generationId, event.outcome);
370
- if (event.type === 'context-compaction') {
371
- for (const transaction of event.archivedTransactions ?? []) {
372
- remember(transaction.generationId, transaction.outcome);
373
- }
378
+ if (event.type === 'generation-outcome') remember(event.generationId, event.outcome);
379
+ if (event.type === 'context-compaction') {
380
+ for (const transaction of event.archivedTransactions ?? []) {
381
+ remember(transaction.generationId, transaction.outcome);
374
382
  }
375
383
  }
376
- return outcomes;
377
384
  }
378
385
 
379
386
  public async durableCanonicalOutcomes(
@@ -383,40 +390,56 @@ export class FlexHarnessCore<TScope> {
383
390
  return this.canonicalOutcomes(snapshot?.events ?? []);
384
391
  }
385
392
 
393
+ /**
394
+ * Loads a scope's state for every request of the scope, and reports what the load took to the
395
+ * scope load observer. Its long steps are cut into slices of time, so the event loop keeps
396
+ * running beside it; disposal and the scope's retirement stop it at the next slice (the abort of
397
+ * `compactorLifecycleController`), a request that stops waiting for it does not.
398
+ */
386
399
  private async loadState(
387
400
  storageKey: string,
388
401
  scopeId: string,
389
402
  scope: TScope,
390
403
  compactorLifecycleController: AbortController,
391
404
  ): Promise<IStorageState> {
392
- const snapshot = cloneSerializable(await this.stores.scopes.load(storageKey) ?? {
393
- schemaVersion: 1,
394
- revision: 0,
395
- sessions: [],
396
- tombstones: [],
397
- } satisfies IFlexScopeSnapshot);
398
- let scopeChanged = false;
399
- const repairedGenerationSequence = snapshot.revision + 1;
400
- for (const session of snapshot.sessions) {
401
- if (session.sessionGenerationId !== undefined) continue;
402
- session.sessionGenerationId = deriveLegacySessionGenerationId(
403
- storageKey,
404
- session.sessionId,
405
- session.createdAt,
406
- );
407
- session.sessionGenerationSequence = repairedGenerationSequence;
408
- scopeChanged = true;
409
- }
410
- for (const tombstone of snapshot.tombstones) {
411
- if (tombstone.sessionGenerationId !== undefined) continue;
412
- tombstone.sessionGenerationId = deriveLegacySessionGenerationId(
413
- storageKey,
414
- tombstone.sessionId,
415
- tombstone.deletedAt,
416
- );
417
- tombstone.sessionGenerationSequence = repairedGenerationSequence;
418
- scopeChanged = true;
405
+ const startedAt = performance.now();
406
+ const pass: IScopeLoadPass = {
407
+ slicer: new FlexTimeSlicer(compactorLifecycleController.signal),
408
+ metrics: { scopeId, storageKey, sessions: 0, storeReadMs: 0, agentSessionMs: 0, repairMs: 0 },
409
+ };
410
+ let outcome: TFlexScopeLoadOutcome = 'failed';
411
+ try {
412
+ const state = await this.loadStateSliced(storageKey, scopeId, scope, compactorLifecycleController, pass);
413
+ outcome = 'loaded';
414
+ return state;
415
+ } catch (error) {
416
+ if (compactorLifecycleController.signal.aborted) outcome = 'aborted';
417
+ throw error;
418
+ } finally {
419
+ this.observeScopeLoad({ ...pass.metrics, totalMs: performance.now() - startedAt, outcome });
419
420
  }
421
+ }
422
+
423
+ /** Reports a scope load to the scope load observer, which can neither delay nor fail it. */
424
+ private observeScopeLoad(metrics: IFlexScopeLoadMetrics): void {
425
+ const observer = this.scopeLoadObserver;
426
+ if (!observer) return;
427
+ void (async () => observer(metrics))().catch(() => undefined);
428
+ }
429
+
430
+ private async loadStateSliced(
431
+ storageKey: string,
432
+ scopeId: string,
433
+ scope: TScope,
434
+ compactorLifecycleController: AbortController,
435
+ pass: IScopeLoadPass,
436
+ ): Promise<IStorageState> {
437
+ const readStartedAt = performance.now();
438
+ const stored = await this.stores.scopes.load(storageKey);
439
+ pass.metrics.storeReadMs += performance.now() - readStartedAt;
440
+ compactorLifecycleController.signal.throwIfAborted();
441
+ const { snapshot, changed } = normalizeScopeSnapshot(storageKey, stored);
442
+ let scopeChanged = changed;
420
443
  const state: IStorageState = {
421
444
  storageKey,
422
445
  compactorLifecycleController,
@@ -444,22 +467,40 @@ export class FlexHarnessCore<TScope> {
444
467
  detachedCleanups: new Set(),
445
468
  };
446
469
  const loadedSessions: IStoredSessionState[] = [];
470
+ // Until every live session is loaded, `state` holds only part of the scope's sessions.
471
+ let sessionsLoaded = false;
447
472
  try {
448
- for (const metadata of snapshot.sessions) {
449
- if (state.tombstones.has(metadata.sessionId)) continue;
450
- const stored = await this.loadSessionRuntime(state, metadata, scopeId, scope);
451
- state.sessions.set(metadata.sessionId, stored);
452
- loadedSessions.push(stored);
453
- scopeChanged = (await this.repairLoadedSession(
454
- state.storageKey,
455
- stored,
473
+ for (const metadata of liveScopeSessions(snapshot)) {
474
+ compactorLifecycleController.signal.throwIfAborted();
475
+ const stored = await this.loadSessionRuntime(
456
476
  state,
477
+ metadata,
457
478
  scopeId,
458
479
  scope,
459
- )) || scopeChanged;
480
+ undefined,
481
+ false,
482
+ pass,
483
+ );
484
+ state.sessions.set(metadata.sessionId, stored);
485
+ loadedSessions.push(stored);
486
+ pass.metrics.sessions = loadedSessions.length;
487
+ const repairStartedAt = performance.now();
488
+ try {
489
+ scopeChanged = (await this.repairLoadedSession(
490
+ state.storageKey,
491
+ stored,
492
+ state,
493
+ scopeId,
494
+ scope,
495
+ pass.slicer,
496
+ )) || scopeChanged;
497
+ } finally {
498
+ pass.metrics.repairMs += performance.now() - repairStartedAt;
499
+ }
460
500
  await this.reversions.drainReversionReleases(state, stored, scopeId, scope, false);
461
501
  if (stored.projectionRevision < 0) throw new Error('Invalid projection revision.');
462
502
  }
503
+ sessionsLoaded = true;
463
504
  for (const stored of state.sessions.values()) {
464
505
  let ancestorId = stored.session.parentSessionId;
465
506
  const visited = new Set<string>([stored.session.sessionId]);
@@ -514,15 +555,19 @@ export class FlexHarnessCore<TScope> {
514
555
  }
515
556
  return state;
516
557
  } catch (error) {
517
- for (const rootSessionId of new Set([...state.tombstones.values()].map((tombstone) =>
518
- tombstone.rootSessionId ?? tombstone.sessionId))) {
519
- const key = JSON.stringify([storageKey, rootSessionId]);
520
- this.lifecycle.orphanedTombstoneOwners.set(key, {
521
- state,
522
- rootSessionId,
523
- scopeId,
524
- scope,
525
- });
558
+ // A tombstone cleanup saves the scope from `state`, so it may finish only where `state` holds
559
+ // every live session. Before that no tombstone was touched, and the next load cleans them up.
560
+ if (sessionsLoaded) {
561
+ for (const rootSessionId of new Set([...state.tombstones.values()].map((tombstone) =>
562
+ tombstone.rootSessionId ?? tombstone.sessionId))) {
563
+ const key = JSON.stringify([storageKey, rootSessionId]);
564
+ this.lifecycle.orphanedTombstoneOwners.set(key, {
565
+ state,
566
+ rootSessionId,
567
+ scopeId,
568
+ scope,
569
+ });
570
+ }
526
571
  }
527
572
  const closeResults = await Promise.allSettled(
528
573
  loadedSessions.map((stored) => this.lifecycle.closeStoredSession(stored)),
@@ -590,6 +635,7 @@ export class FlexHarnessCore<TScope> {
590
635
  scope: TScope,
591
636
  compactorLifecycleController = new AbortController(),
592
637
  newSession = false,
638
+ pass?: IScopeLoadPass,
593
639
  ): Promise<IStoredSessionState> {
594
640
  const sessionId = metadata.sessionId;
595
641
  const compactorContext = this.compaction.createCompactorContext(
@@ -598,12 +644,14 @@ export class FlexHarnessCore<TScope> {
598
644
  state.storageKey,
599
645
  sessionId,
600
646
  );
647
+ const readStartedAt = performance.now();
601
648
  const [projectionResult, permissionResult, eventStoreResult, jobStoreResult] = await Promise.allSettled([
602
649
  this.stores.projections.load(state.storageKey, sessionId),
603
650
  this.stores.permissions.load(state.storageKey, sessionId),
604
651
  this.stores.agentEvents.getStore(state.storageKey, sessionId),
605
652
  this.stores.jobs.getStore(state.storageKey, sessionId),
606
653
  ]);
654
+ if (pass) pass.metrics.storeReadMs += performance.now() - readStartedAt;
607
655
  const acquisitionErrors = [projectionResult, permissionResult, eventStoreResult, jobStoreResult]
608
656
  .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
609
657
  .map((result) => result.reason);
@@ -676,11 +724,20 @@ export class FlexHarnessCore<TScope> {
676
724
  throw new FlexHarnessValidationError('Execution context provider returned an invalid handle.');
677
725
  }
678
726
  }
727
+ // A scope load copies and freezes the messages in slices of time. The baseline shares the
728
+ // frozen copies, as it is only ever replaced, never edited.
729
+ const messages = pass
730
+ ? await cloneSerializableSliced(projection?.messages ?? [], pass.slicer)
731
+ : cloneSerializable(projection?.messages ?? []);
732
+ for (const message of messages) {
733
+ deepFreeze(message);
734
+ if (pass?.slicer.due) await pass.slicer.pause();
735
+ }
679
736
  const stored: IStoredSessionState = {
680
737
  storageKey: state.storageKey,
681
738
  compactorContext,
682
739
  session: cloneSerializable(metadata),
683
- messages: cloneSerializable(projection?.messages ?? []),
740
+ messages,
684
741
  stagedTerminals: cloneSerializable(projection?.stagedTerminals ?? []),
685
742
  reversionSegments,
686
743
  revertCursor: currentProjection?.revertCursor ?? legacyProjection?.revertCursor ?? 0,
@@ -694,7 +751,12 @@ export class FlexHarnessCore<TScope> {
694
751
  projectionSchemaVersion: projection?.schemaVersion ?? 3,
695
752
  projectionReconciliationRequired: false,
696
753
  ...(projection
697
- ? { projectionBaseline: cloneSerializable(projection) }
754
+ ? {
755
+ projectionBaseline: deepFreeze({
756
+ ...cloneSerializable({ ...projection, messages: [] }),
757
+ messages: messages.slice(),
758
+ } as TFlexProjectionSnapshot),
759
+ }
698
760
  : {}),
699
761
  projectionRevision: projection?.revision ?? 0,
700
762
  projectionQueue: Promise.resolve(),
@@ -867,10 +929,15 @@ export class FlexHarnessCore<TScope> {
867
929
  onRetry: (event) => this.callbacks.onRetry(state, sessionId, event),
868
930
  onUsage: (event) => this.callbacks.onModelCallUsage(state, stored, event),
869
931
  };
870
- stored.agentSession = await this.compaction.withCompactorContext(
871
- compactorContext,
872
- () => plugins.AgentSession.create(agentSessionOptions),
873
- );
932
+ const agentSessionStartedAt = performance.now();
933
+ try {
934
+ stored.agentSession = await this.compaction.withCompactorContext(
935
+ compactorContext,
936
+ () => plugins.AgentSession.create(agentSessionOptions),
937
+ );
938
+ } finally {
939
+ if (pass) pass.metrics.agentSessionMs += performance.now() - agentSessionStartedAt;
940
+ }
874
941
  return stored;
875
942
  } catch (error) {
876
943
  const cleanupErrors: unknown[] = [];
@@ -905,26 +972,25 @@ export class FlexHarnessCore<TScope> {
905
972
  }
906
973
  }
907
974
 
975
+ /**
976
+ * Repairs what a stopped process left in a loaded session: every run gets its terminal, the
977
+ * reversion state its runs' outcomes, the session its latest turn's state. Linear in the
978
+ * session's size, in slices of `slicer`'s time; it saves the projection only if its content
979
+ * changed.
980
+ */
908
981
  private async repairLoadedSession(
909
982
  storageKey: string,
910
983
  stored: IStoredSessionState,
911
- state?: IStorageState,
912
- scopeId?: string,
913
- scope?: TScope,
984
+ state: IStorageState,
985
+ scopeId: string,
986
+ scope: TScope,
987
+ slicer: FlexTimeSlicer,
914
988
  ): Promise<boolean> {
915
989
  if (stored.session.origin !== undefined) {
916
990
  this.assertImportedSessionState(stored);
917
991
  return false;
918
992
  }
919
- const beforeProjection = JSON.stringify({
920
- messages: stored.messages,
921
- stagedTerminals: stored.stagedTerminals,
922
- reversionSegments: stored.reversionSegments,
923
- revertCursor: stored.revertCursor,
924
- excludedRunIds: stored.excludedRunIds,
925
- pendingReversion: stored.pendingReversion,
926
- pendingReversionReleases: stored.pendingReversionReleases,
927
- });
993
+ const before = this.projectionContent(stored);
928
994
  const beforeSession = JSON.stringify(stored.session);
929
995
  if (
930
996
  !this.turnReversionProvider
@@ -957,75 +1023,28 @@ export class FlexHarnessCore<TScope> {
957
1023
  );
958
1024
  }
959
1025
  if (stored.pendingReversion?.kind === 'capture' && stored.pendingReversionReleases.length > 0) {
960
- if (!state || scopeId === undefined || scope === undefined) {
961
- throw new FlexHarnessValidationError('Pending capture recovery is missing its scope context.');
962
- }
963
1026
  await this.reversions.drainReversionReleases(state, stored, scopeId, scope, false);
964
1027
  this.assertStateAcceptingWork(state);
965
1028
  }
1029
+ // The outcomes and model context of the session's runs, before a pending reversion resumes.
966
1030
  const agentEvents = stored.agentSession.getEvents();
967
- const outcomes = this.canonicalOutcomes(agentEvents);
1031
+ const outcomes = new Map<string, TCanonicalOutcome>();
1032
+ for (const event of agentEvents) {
1033
+ this.rememberCanonicalOutcomes(outcomes, event);
1034
+ if (slicer.due) await slicer.pause();
1035
+ }
968
1036
  const firstHiddenCandidate = this.reversions.completedReversionCandidates(stored)[stored.revertCursor];
969
1037
  const reversionVisibilityBoundary = firstHiddenCandidate
970
1038
  ? stored.reversionSegments.findIndex((segment) => segment.runId === firstHiddenCandidate.runId)
971
1039
  : stored.reversionSegments.length;
972
- const stages = new Map(stored.stagedTerminals.map((terminal) => [terminal.runId, terminal]));
973
- const runIds = new Set([
974
- ...stored.messages.map((message) => message.runId),
975
- ...stored.stagedTerminals.map((terminal) => terminal.runId),
976
- ]);
977
- // The session's state is the latest real turn's: a cache-warm turn's terminal never sets it.
978
- let latestTerminal: IFlexTerminalProjection | undefined;
979
- for (const runId of runIds) {
980
- const outcome = outcomes.get(runId);
981
- const stage = stages.get(runId);
982
- if (outcome === 'accepted' && stage?.status === 'completed') {
983
- // A cache-warm exchange leaves the model context with its outcome.
984
- this.runs.replaceVisibleRun(stored, this.runs.withKeptInContext(stage, !stage.assistantMessage.cacheWarm));
985
- if (!stage.assistantMessage.cacheWarm) latestTerminal = stage;
986
- } else if (outcome !== 'accepted') {
987
- const visible = stored.messages.filter((message) => message.runId === runId);
988
- // A run that failed or was stopped keeps its terminal and its error: the staged one, or the
989
- // one it published. Only a run that never reached its terminal is repaired as interrupted.
990
- const ended = stage && stage.status !== 'completed'
991
- ? stage
992
- : stage ? undefined : this.publishedTerminal(runId, visible);
993
- if (ended) {
994
- if (ended === stage) {
995
- this.runs.replaceVisibleRun(stored, this.runs.withKeptInContext(
996
- stage,
997
- plugins.isGenerationInModelContext(agentEvents, runId),
998
- ));
999
- }
1000
- if (!ended.assistantMessage.cacheWarm) latestTerminal = ended;
1001
- } else if (visible.length > 0 || stage) {
1002
- const status = outcome === 'rejected' ? 'failed' : 'cancelled';
1003
- const terminal = this.runs.withKeptInContext(
1004
- this.repairTerminal(stored.session.sessionId, runId, visible, stage, status),
1005
- plugins.isGenerationInModelContext(agentEvents, runId),
1006
- );
1007
- this.runs.replaceVisibleRun(stored, terminal);
1008
- if (!terminal.assistantMessage.cacheWarm) latestTerminal = terminal;
1009
- }
1010
- } else {
1011
- const visibleSuccess = stored.messages.some((message) =>
1012
- message.runId === runId && message.status === 'completed');
1013
- if (visibleSuccess) {
1014
- const terminal = this.repairTerminal(
1015
- stored.session.sessionId,
1016
- runId,
1017
- stored.messages.filter((message) => message.runId === runId),
1018
- undefined,
1019
- 'completed',
1020
- );
1021
- if (!terminal.assistantMessage.cacheWarm) latestTerminal = terminal;
1022
- }
1023
- }
1024
- }
1040
+ const latestTerminal = await this.repairRunTerminals(stored, agentEvents, outcomes, slicer);
1025
1041
  stored.stagedTerminals = stored.stagedTerminals.filter((terminal) => !outcomes.has(terminal.runId));
1026
1042
  if (stored.stagedTerminals.length > 0) stored.stagedTerminals = [];
1027
- const activeEvents = stored.agentSession.getEvents();
1028
- const activeEventIds = new Set(activeEvents.map((event) => event.id));
1043
+ const activeEventIds = new Set<string>();
1044
+ for (const event of stored.agentSession.getEvents()) {
1045
+ activeEventIds.add(event.id);
1046
+ if (slicer.due) await slicer.pause();
1047
+ }
1029
1048
  stored.reversionSegments = stored.reversionSegments.map((segment) => {
1030
1049
  const contextAvailable = segment.contextAvailable
1031
1050
  && (segment.eventIds.length === 0 || segment.eventIds.some((id) => activeEventIds.has(id)));
@@ -1047,12 +1066,9 @@ export class FlexHarnessCore<TScope> {
1047
1066
  this.reversions.commitArchivedHiddenBranchState(stored);
1048
1067
  this.reversions.pruneReversionState(stored, false, true);
1049
1068
  if (stored.pendingReversion?.kind === 'apply') {
1050
- if (!state || scopeId === undefined || scope === undefined) {
1051
- throw new FlexHarnessValidationError('Pending reversion recovery is missing its scope context.');
1052
- }
1053
1069
  await this.reversions.resumePendingApply(state, stored, scopeId, scope);
1054
1070
  } else if (stored.pendingReversion?.kind === 'capture') {
1055
- if (!this.turnReversionProvider || !state || scopeId === undefined || scope === undefined) {
1071
+ if (!this.turnReversionProvider) {
1056
1072
  throw new FlexHarnessValidationError('Pending capture recovery requires its reversion provider.');
1057
1073
  }
1058
1074
  await this.reversions.recoverPendingCapture(state, stored, scopeId, scope, outcomes);
@@ -1084,17 +1100,12 @@ export class FlexHarnessCore<TScope> {
1084
1100
  };
1085
1101
  stored.session.updatedAt = timestamp;
1086
1102
  }
1087
- const projectionChanged = beforeProjection !== JSON.stringify({
1088
- messages: stored.messages,
1089
- stagedTerminals: stored.stagedTerminals,
1090
- reversionSegments: stored.reversionSegments,
1091
- revertCursor: stored.revertCursor,
1092
- excludedRunIds: stored.excludedRunIds,
1093
- pendingReversion: stored.pendingReversion,
1094
- pendingReversionReleases: stored.pendingReversionReleases,
1095
- });
1096
- if (projectionChanged) {
1103
+ if (await this.projectionContentChanged(before, stored, slicer)) {
1097
1104
  const expected = stored.projectionRevision;
1105
+ for (const message of stored.messages) {
1106
+ deepFreeze(message);
1107
+ if (slicer.due) await slicer.pause();
1108
+ }
1098
1109
  freezeProjectionEntries(stored);
1099
1110
  const change = this.createProjectionChange(stored, expected + 1);
1100
1111
  await this.stores.projections.save(
@@ -1112,6 +1123,215 @@ export class FlexHarnessCore<TScope> {
1112
1123
  return beforeSession !== JSON.stringify(stored.session);
1113
1124
  }
1114
1125
 
1126
+ /**
1127
+ * Gives every run of a loaded session its terminal, in one pass over its messages, and returns
1128
+ * the terminal of its latest real turn. A run with an accepted completed stage takes the stage;
1129
+ * a run that failed or was stopped keeps the terminal it staged or published; a run that never
1130
+ * reached its terminal is repaired as failed or cancelled. Each terminal goes where
1131
+ * `runs.replaceVisibleRun()` would put it.
1132
+ */
1133
+ private async repairRunTerminals(
1134
+ stored: IStoredSessionState,
1135
+ agentEvents: readonly plugins.TAgentEvent[],
1136
+ outcomes: ReadonlyMap<string, TCanonicalOutcome>,
1137
+ slicer: FlexTimeSlicer,
1138
+ ): Promise<IFlexTerminalProjection | undefined> {
1139
+ const runMessages = new Map<string, IFlexMessage[]>();
1140
+ const lastRunMessage = new Map<string, IFlexMessage>();
1141
+ for (const message of stored.messages) {
1142
+ let visible = runMessages.get(message.runId);
1143
+ if (!visible) {
1144
+ visible = [];
1145
+ runMessages.set(message.runId, visible);
1146
+ }
1147
+ visible.push(message);
1148
+ lastRunMessage.set(message.runId, message);
1149
+ if (slicer.due) await slicer.pause();
1150
+ }
1151
+ const stages = new Map(stored.stagedTerminals.map((terminal) => [terminal.runId, terminal]));
1152
+ const runIds = new Set([
1153
+ ...runMessages.keys(),
1154
+ ...stored.stagedTerminals.map((terminal) => terminal.runId),
1155
+ ]);
1156
+ let modelContextRunIds: Set<string> | undefined;
1157
+ const inModelContext = (runId: string) =>
1158
+ (modelContextRunIds ??= plugins.getGenerationsInModelContext(agentEvents)).has(runId);
1159
+ const terminals = new Map<string, IFlexTerminalProjection>();
1160
+ // The session's state is the latest real turn's: a cache-warm turn's terminal never sets it. The
1161
+ // terminal of an accepted run without a completed stage only names that state, so it is built
1162
+ // for the latest such run alone.
1163
+ let latestTerminal: IFlexTerminalProjection | (() => IFlexTerminalProjection) | undefined;
1164
+ for (const runId of runIds) {
1165
+ const outcome = outcomes.get(runId);
1166
+ const stage = stages.get(runId);
1167
+ const visible = runMessages.get(runId) ?? [];
1168
+ if (outcome === 'accepted' && stage?.status === 'completed') {
1169
+ // A cache-warm exchange leaves the model context with its outcome.
1170
+ terminals.set(runId, this.runs.withKeptInContext(stage, !stage.assistantMessage.cacheWarm));
1171
+ if (!stage.assistantMessage.cacheWarm) latestTerminal = stage;
1172
+ } else if (outcome !== 'accepted') {
1173
+ // A run that failed or was stopped keeps its terminal and its error: the staged one, or the
1174
+ // one it published. Only a run that never reached its terminal is repaired as interrupted.
1175
+ const ended = stage && stage.status !== 'completed'
1176
+ ? stage
1177
+ : stage ? undefined : this.publishedTerminal(runId, visible);
1178
+ if (ended) {
1179
+ if (ended === stage) {
1180
+ terminals.set(runId, this.runs.withKeptInContext(stage, inModelContext(runId)));
1181
+ }
1182
+ if (!ended.assistantMessage.cacheWarm) latestTerminal = ended;
1183
+ } else if (visible.length > 0 || stage) {
1184
+ const status = outcome === 'rejected' ? 'failed' : 'cancelled';
1185
+ const terminal = this.runs.withKeptInContext(
1186
+ this.repairTerminal(stored.session.sessionId, runId, visible, stage, status),
1187
+ inModelContext(runId),
1188
+ );
1189
+ terminals.set(runId, terminal);
1190
+ if (!terminal.assistantMessage.cacheWarm) latestTerminal = terminal;
1191
+ }
1192
+ } else if (visible.some((message) => message.status === 'completed')) {
1193
+ let assistant: IFlexMessage | undefined;
1194
+ for (const message of visible) if (message.role === 'assistant') assistant = message;
1195
+ if (!assistant?.cacheWarm) {
1196
+ latestTerminal = () =>
1197
+ this.repairTerminal(stored.session.sessionId, runId, visible, undefined, 'completed');
1198
+ }
1199
+ }
1200
+ if (slicer.due) await slicer.pause();
1201
+ }
1202
+ if (terminals.size > 0) {
1203
+ stored.messages = await this.placeRunTerminals(
1204
+ stored.messages,
1205
+ terminals,
1206
+ runMessages,
1207
+ lastRunMessage,
1208
+ inModelContext,
1209
+ slicer,
1210
+ );
1211
+ }
1212
+ return typeof latestTerminal === 'function' ? latestTerminal() : latestTerminal;
1213
+ }
1214
+
1215
+ /**
1216
+ * The messages with each run's terminal in place, built in one pass: what
1217
+ * `runs.replaceVisibleRun()` gives for each terminal in turn. A run's projected terminal messages
1218
+ * are replaced, its other user messages take the prompt's `keptInContext`, and a terminal
1219
+ * message not projected yet goes before the run's first message (its prompt) or after its last
1220
+ * (its answer); for a run without messages both go to the end.
1221
+ */
1222
+ private async placeRunTerminals(
1223
+ messages: readonly IFlexMessage[],
1224
+ terminals: ReadonlyMap<string, IFlexTerminalProjection>,
1225
+ runMessages: ReadonlyMap<string, readonly IFlexMessage[]>,
1226
+ lastRunMessage: ReadonlyMap<string, IFlexMessage>,
1227
+ inModelContext: (runId: string) => boolean,
1228
+ slicer: FlexTimeSlicer,
1229
+ ): Promise<IFlexMessage[]> {
1230
+ interface IPlacement {
1231
+ terminal: IFlexTerminalProjection;
1232
+ replacements: Map<string, IFlexMessage>;
1233
+ keptInContext: boolean;
1234
+ placeUser: boolean;
1235
+ placeAssistant: boolean;
1236
+ }
1237
+ const placements = new Map<string, IPlacement>();
1238
+ for (const [runId, terminal] of terminals) {
1239
+ const replacements = new Map([
1240
+ ...(terminal.userMessage ? [[terminal.userMessage.messageId, terminal.userMessage] as const] : []),
1241
+ [terminal.assistantMessage.messageId, terminal.assistantMessage],
1242
+ ]);
1243
+ const projectedIds = new Set((runMessages.get(runId) ?? []).map((message) => message.messageId));
1244
+ placements.set(runId, {
1245
+ terminal,
1246
+ replacements,
1247
+ keptInContext: terminal.userMessage?.keptInContext
1248
+ ?? (terminal.status === 'completed' || inModelContext(runId)),
1249
+ placeUser: terminal.userMessage !== undefined && !projectedIds.has(terminal.userMessage.messageId),
1250
+ placeAssistant: !projectedIds.has(terminal.assistantMessage.messageId),
1251
+ });
1252
+ }
1253
+ const placed: IFlexMessage[] = [];
1254
+ const startedRuns = new Set<string>();
1255
+ for (const message of messages) {
1256
+ const placement = placements.get(message.runId);
1257
+ if (!placement) {
1258
+ placed.push(message);
1259
+ } else {
1260
+ if (!startedRuns.has(message.runId)) {
1261
+ startedRuns.add(message.runId);
1262
+ if (placement.placeUser) placed.push(cloneSerializable(placement.terminal.userMessage!));
1263
+ }
1264
+ const replacement = placement.replacements.get(message.messageId);
1265
+ if (replacement) {
1266
+ placed.push(cloneSerializable(replacement));
1267
+ } else {
1268
+ // The steers the run took in share its prompt's outcome.
1269
+ placed.push(message.role === 'user' && message.keptInContext !== placement.keptInContext
1270
+ ? { ...cloneSerializable(message), keptInContext: placement.keptInContext }
1271
+ : message);
1272
+ }
1273
+ if (placement.placeAssistant && lastRunMessage.get(message.runId) === message) {
1274
+ placed.push(cloneSerializable(placement.terminal.assistantMessage));
1275
+ }
1276
+ }
1277
+ if (slicer.due) await slicer.pause();
1278
+ }
1279
+ for (const [runId, placement] of placements) {
1280
+ if (startedRuns.has(runId)) continue;
1281
+ if (placement.placeUser) placed.push(cloneSerializable(placement.terminal.userMessage!));
1282
+ if (placement.placeAssistant) placed.push(cloneSerializable(placement.terminal.assistantMessage));
1283
+ }
1284
+ return placed;
1285
+ }
1286
+
1287
+ /**
1288
+ * The projection content a repair may change, as it is before the repair. Messages and reversion
1289
+ * segments are frozen and replaced, never edited, so the lists keep their entries; the other
1290
+ * fields are small and kept encoded.
1291
+ */
1292
+ private projectionContent(stored: IStoredSessionState): IProjectionContent {
1293
+ return {
1294
+ messages: stored.messages.slice(),
1295
+ reversionSegments: stored.reversionSegments.slice(),
1296
+ stagedTerminals: JSON.stringify(stored.stagedTerminals),
1297
+ revertCursor: stored.revertCursor,
1298
+ excludedRunIds: JSON.stringify(stored.excludedRunIds),
1299
+ pendingReversion: JSON.stringify(stored.pendingReversion),
1300
+ pendingReversionReleases: JSON.stringify(stored.pendingReversionReleases),
1301
+ };
1302
+ }
1303
+
1304
+ /**
1305
+ * Whether the session's projection content differs from `before` as JSON. A list entry that is
1306
+ * still the same object is unchanged, so only an entry replaced since is encoded, one at a time.
1307
+ */
1308
+ private async projectionContentChanged(
1309
+ before: IProjectionContent,
1310
+ stored: IStoredSessionState,
1311
+ slicer: FlexTimeSlicer,
1312
+ ): Promise<boolean> {
1313
+ if (
1314
+ stored.revertCursor !== before.revertCursor
1315
+ || JSON.stringify(stored.stagedTerminals) !== before.stagedTerminals
1316
+ || JSON.stringify(stored.excludedRunIds) !== before.excludedRunIds
1317
+ || JSON.stringify(stored.pendingReversion) !== before.pendingReversion
1318
+ || JSON.stringify(stored.pendingReversionReleases) !== before.pendingReversionReleases
1319
+ ) return true;
1320
+ const listChanged = async (previous: readonly object[], current: readonly object[]) => {
1321
+ if (previous.length !== current.length) return true;
1322
+ for (let index = 0; index < current.length; index++) {
1323
+ if (
1324
+ previous[index] !== current[index]
1325
+ && JSON.stringify(previous[index]) !== JSON.stringify(current[index])
1326
+ ) return true;
1327
+ if (slicer.due) await slicer.pause();
1328
+ }
1329
+ return false;
1330
+ };
1331
+ return await listChanged(before.messages, stored.messages)
1332
+ || await listChanged(before.reversionSegments, stored.reversionSegments);
1333
+ }
1334
+
1115
1335
  /**
1116
1336
  * The failed or cancelled terminal a run published, read from its projected messages. A run
1117
1337
  * still running has its prompt or its last assistant message streaming; it has none.
@@ -1396,9 +1616,8 @@ export class FlexHarnessCore<TScope> {
1396
1616
  const beforeRevision = stored.projectionRevision;
1397
1617
  const beforeSchemaVersion = stored.projectionSchemaVersion;
1398
1618
  const beforeReconciliationRequired = stored.projectionReconciliationRequired;
1399
- const beforeBaseline = stored.projectionBaseline === undefined
1400
- ? undefined
1401
- : cloneSerializable(stored.projectionBaseline);
1619
+ // The baseline is deep-frozen and only ever replaced, never edited, so it needs no copy.
1620
+ const beforeBaseline = stored.projectionBaseline;
1402
1621
  const beforeMetrics = cloneSerializable(stored.metrics);
1403
1622
  // List entries are frozen and replaced rather than edited, so copies of the lists suffice.
1404
1623
  const beforeMessages = stored.messages.slice();
@@ -1645,7 +1864,7 @@ export class FlexHarnessCore<TScope> {
1645
1864
  delete stored.projectionBaseline;
1646
1865
  this.rememberSavedProjectionLists(stored);
1647
1866
  } else {
1648
- stored.projectionBaseline = cloneSerializable(projection);
1867
+ stored.projectionBaseline = deepFreeze(cloneSerializable(projection));
1649
1868
  delete stored.savedProjectionLists;
1650
1869
  }
1651
1870
  stored.projectionSchemaVersion = projection.schemaVersion;
@@ -1713,28 +1932,13 @@ export class FlexHarnessCore<TScope> {
1713
1932
  public async resolveState(
1714
1933
  scopeId: string,
1715
1934
  ): Promise<{ scope: IFlexResolvedScope<TScope>; state: IStorageState }> {
1716
- this.assertOpen();
1717
- validateIdentifier(scopeId, 'scopeId');
1718
- const admission = this.getScopeAdmission(scopeId);
1719
- if (admission.retiring) throw this.createScopeRetirementError();
1720
- const generation = admission.generation;
1721
- admission.inFlightResolvers++;
1722
- try {
1723
- const scope = await this.scopeResolver.resolveScope(scopeId);
1724
- this.assertOpen();
1725
- if (admission.retiring || admission.generation !== generation) throw this.createScopeRetirementError();
1726
- validateIdentifier(scope.storageKey, 'resolved storageKey');
1727
- if (this.lifecycle.storageDrains.has(scope.storageKey)) throw this.createScopeRetirementError();
1935
+ return this.withScopeAdmission(scopeId, async (scope, admitted) => {
1728
1936
  let stateLoad = this.stateLoads.get(scope.storageKey);
1729
1937
  if (!stateLoad) {
1730
1938
  const orphanErrors = await this.lifecycle.closeOrphanedResources(scope.storageKey);
1731
1939
  if (orphanErrors.length > 0) throw combineErrors(orphanErrors);
1732
1940
  this.assertOpen();
1733
- if (
1734
- admission.retiring
1735
- || admission.generation !== generation
1736
- || this.lifecycle.storageDrains.has(scope.storageKey)
1737
- ) throw this.createScopeRetirementError();
1941
+ if (!admitted()) throw this.createScopeRetirementError();
1738
1942
  stateLoad = this.stateLoads.get(scope.storageKey);
1739
1943
  if (!stateLoad) {
1740
1944
  const compactorLifecycleController = new AbortController();
@@ -1742,15 +1946,18 @@ export class FlexHarnessCore<TScope> {
1742
1946
  scope.storageKey,
1743
1947
  compactorLifecycleController,
1744
1948
  );
1745
- stateLoad = this.loadState(
1949
+ const load = this.loadState(
1746
1950
  scope.storageKey,
1747
1951
  scopeId,
1748
1952
  scope.scope,
1749
1953
  compactorLifecycleController,
1750
1954
  );
1751
- this.stateLoads.set(scope.storageKey, stateLoad);
1752
- void stateLoad.catch(() => {
1753
- if (this.stateLoads.get(scope.storageKey) === stateLoad) {
1955
+ stateLoad = load;
1956
+ this.stateLoads.set(scope.storageKey, load);
1957
+ void load.then((state) => {
1958
+ this.loadedStates.set(load, state);
1959
+ }, () => {
1960
+ if (this.stateLoads.get(scope.storageKey) === load) {
1754
1961
  this.stateLoads.delete(scope.storageKey);
1755
1962
  if (
1756
1963
  this.lifecycle.storageCompactorLifecycleControllers.get(scope.storageKey)
@@ -1760,38 +1967,75 @@ export class FlexHarnessCore<TScope> {
1760
1967
  });
1761
1968
  }
1762
1969
  }
1763
- const state = await stateLoad;
1970
+ return { scope, state: await this.admittedState(scopeId, scope, stateLoad, admitted) };
1971
+ });
1972
+ }
1973
+
1974
+ /** Whether `stateLoad` has settled with its loaded state. */
1975
+ public isStateLoaded(stateLoad: Promise<IStorageState>): boolean {
1976
+ return this.loadedStates.has(stateLoad);
1977
+ }
1978
+
1979
+ /**
1980
+ * Runs `operation` for the resolved scope `scopeId` while the scope is admitted: not retiring,
1981
+ * not retired since, not draining. `admitted()` tells whether that still holds.
1982
+ */
1983
+ public async withScopeAdmission<TValue>(
1984
+ scopeId: string,
1985
+ operation: (scope: IFlexResolvedScope<TScope>, admitted: () => boolean) => Promise<TValue>,
1986
+ ): Promise<TValue> {
1987
+ this.assertOpen();
1988
+ validateIdentifier(scopeId, 'scopeId');
1989
+ const admission = this.getScopeAdmission(scopeId);
1990
+ if (admission.retiring) throw this.createScopeRetirementError();
1991
+ const generation = admission.generation;
1992
+ admission.inFlightResolvers++;
1993
+ try {
1994
+ const scope = await this.scopeResolver.resolveScope(scopeId);
1764
1995
  this.assertOpen();
1765
- if (state.lifecycle === 'fenced') {
1766
- const drain = this.lifecycle.storageDrains.get(scope.storageKey) ?? this.lifecycle.drainStorage(
1767
- scope.storageKey,
1768
- stateLoad,
1769
- this.trustInternalError(new FlexHarnessAbortError('The session namespace was fenced.')),
1770
- { scopeId, scope: scope.scope },
1771
- );
1772
- if (this.commands.ambientSlashCommandOwner({ storageKey: scope.storageKey })) {
1773
- throw this.createScopeRetirementError();
1774
- }
1775
- try {
1776
- await drain;
1777
- } catch {
1778
- // Fenced cleanup ownership remains available to explicit retirement or disposal.
1779
- }
1780
- throw this.createScopeRetirementError();
1781
- }
1782
- if (
1783
- admission.retiring
1784
- || admission.generation !== generation
1785
- || this.lifecycle.storageDrains.has(scope.storageKey)
1786
- || state.lifecycle !== 'active'
1787
- ) throw this.createScopeRetirementError();
1788
- return { scope, state };
1996
+ if (admission.retiring || admission.generation !== generation) throw this.createScopeRetirementError();
1997
+ validateIdentifier(scope.storageKey, 'resolved storageKey');
1998
+ if (this.lifecycle.storageDrains.has(scope.storageKey)) throw this.createScopeRetirementError();
1999
+ return await operation(scope, () =>
2000
+ !admission.retiring
2001
+ && admission.generation === generation
2002
+ && !this.lifecycle.storageDrains.has(scope.storageKey));
1789
2003
  } finally {
1790
2004
  admission.inFlightResolvers--;
1791
2005
  this.pruneScopeAdmission(scopeId, admission);
1792
2006
  }
1793
2007
  }
1794
2008
 
2009
+ /** The state `stateLoad` loads, once it is active and the scope still admitted. */
2010
+ public async admittedState(
2011
+ scopeId: string,
2012
+ scope: IFlexResolvedScope<TScope>,
2013
+ stateLoad: Promise<IStorageState>,
2014
+ admitted: () => boolean,
2015
+ ): Promise<IStorageState> {
2016
+ const state = await stateLoad;
2017
+ this.assertOpen();
2018
+ if (state.lifecycle === 'fenced') {
2019
+ const drain = this.lifecycle.storageDrains.get(scope.storageKey) ?? this.lifecycle.drainStorage(
2020
+ scope.storageKey,
2021
+ stateLoad,
2022
+ this.trustInternalError(new FlexHarnessAbortError('The session namespace was fenced.')),
2023
+ { scopeId, scope: scope.scope },
2024
+ );
2025
+ if (this.commands.ambientSlashCommandOwner({ storageKey: scope.storageKey })) {
2026
+ throw this.createScopeRetirementError();
2027
+ }
2028
+ try {
2029
+ await drain;
2030
+ } catch {
2031
+ // Fenced cleanup ownership remains available to explicit retirement or disposal.
2032
+ }
2033
+ throw this.createScopeRetirementError();
2034
+ }
2035
+ if (!admitted() || state.lifecycle !== 'active') throw this.createScopeRetirementError();
2036
+ return state;
2037
+ }
2038
+
1795
2039
  public getScopeAdmission(scopeId: string): IScopeAdmissionState {
1796
2040
  let admission = this.scopeAdmissions.get(scopeId);
1797
2041
  if (!admission) {