@modelprofile.com/flexharness 18.0.0 → 18.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/dist_ts/00_commitinfo_data.js +1 -1
  2. package/dist_ts/classes.flexharness.cachewarm.js +3 -3
  3. package/dist_ts/classes.flexharness.callbacks.d.ts +37 -0
  4. package/dist_ts/classes.flexharness.callbacks.js +129 -2
  5. package/dist_ts/classes.flexharness.compaction.js +2 -2
  6. package/dist_ts/classes.flexharness.configurations.js +4 -4
  7. package/dist_ts/classes.flexharness.core.d.ts +96 -5
  8. package/dist_ts/classes.flexharness.core.js +848 -301
  9. package/dist_ts/classes.flexharness.d.ts +10 -1
  10. package/dist_ts/classes.flexharness.deletion.d.ts +13 -0
  11. package/dist_ts/classes.flexharness.deletion.js +122 -40
  12. package/dist_ts/classes.flexharness.imports.js +26 -21
  13. package/dist_ts/classes.flexharness.js +16 -1
  14. package/dist_ts/classes.flexharness.lifecycle.js +9 -3
  15. package/dist_ts/classes.flexharness.projects.js +2 -2
  16. package/dist_ts/classes.flexharness.prompts.js +14 -8
  17. package/dist_ts/classes.flexharness.reversions.js +3 -3
  18. package/dist_ts/classes.flexharness.runs.js +18 -2
  19. package/dist_ts/classes.flexharness.sessions.d.ts +8 -1
  20. package/dist_ts/classes.flexharness.sessions.js +40 -14
  21. package/dist_ts/classes.flexharness.slashcommands.js +3 -3
  22. package/dist_ts/classes.flexharness.subagents.js +37 -16
  23. package/dist_ts/classes.flexharness.tooldispatch.js +3 -3
  24. package/dist_ts/classes.scoperecovery.d.ts +36 -0
  25. package/dist_ts/classes.scoperecovery.js +228 -0
  26. package/dist_ts/classes.stores.d.ts +14 -1
  27. package/dist_ts/classes.stores.js +150 -17
  28. package/dist_ts/classes.timeslicer.d.ts +21 -0
  29. package/dist_ts/classes.timeslicer.js +31 -0
  30. package/dist_ts/index.d.ts +1 -0
  31. package/dist_ts/index.js +2 -1
  32. package/dist_ts/interfaces.d.ts +120 -0
  33. package/dist_ts/interfaces.flexharness.d.ts +56 -1
  34. package/dist_ts/interfaces.js +1 -1
  35. package/dist_ts/plugins.d.ts +2 -2
  36. package/dist_ts/plugins.js +3 -3
  37. package/dist_ts/utils.harnessoptions.d.ts +2 -0
  38. package/dist_ts/utils.harnessoptions.js +16 -11
  39. package/dist_ts/utils.json.d.ts +6 -0
  40. package/dist_ts/utils.json.js +75 -20
  41. package/dist_ts/utils.scopesnapshot.d.ts +14 -0
  42. package/dist_ts/utils.scopesnapshot.js +58 -0
  43. package/dist_ts_stores_nosqldb/classes.nosqldbstores.d.ts +3 -2
  44. package/dist_ts_stores_nosqldb/classes.nosqldbstores.js +33 -2
  45. package/dist_ts_stores_nosqldb/classes.recordstore.d.ts +10 -1
  46. package/dist_ts_stores_nosqldb/classes.recordstore.js +75 -7
  47. package/package.json +3 -3
  48. package/readme.md +46 -3
  49. package/ts/00_commitinfo_data.ts +1 -1
  50. package/ts/classes.flexharness.cachewarm.ts +2 -2
  51. package/ts/classes.flexharness.callbacks.ts +142 -1
  52. package/ts/classes.flexharness.compaction.ts +1 -1
  53. package/ts/classes.flexharness.configurations.ts +3 -3
  54. package/ts/classes.flexharness.core.ts +902 -328
  55. package/ts/classes.flexharness.deletion.ts +161 -52
  56. package/ts/classes.flexharness.imports.ts +26 -21
  57. package/ts/classes.flexharness.lifecycle.ts +8 -2
  58. package/ts/classes.flexharness.projects.ts +1 -1
  59. package/ts/classes.flexharness.prompts.ts +11 -7
  60. package/ts/classes.flexharness.reversions.ts +2 -2
  61. package/ts/classes.flexharness.runs.ts +16 -1
  62. package/ts/classes.flexharness.sessions.ts +43 -13
  63. package/ts/classes.flexharness.slashcommands.ts +2 -2
  64. package/ts/classes.flexharness.subagents.ts +40 -15
  65. package/ts/classes.flexharness.tooldispatch.ts +2 -2
  66. package/ts/classes.flexharness.ts +22 -0
  67. package/ts/classes.scoperecovery.ts +290 -0
  68. package/ts/classes.stores.ts +150 -16
  69. package/ts/classes.timeslicer.ts +32 -0
  70. package/ts/index.ts +2 -0
  71. package/ts/interfaces.flexharness.ts +60 -0
  72. package/ts/interfaces.ts +129 -0
  73. package/ts/plugins.ts +2 -0
  74. package/ts/readme.md +46 -3
  75. package/ts/utils.harnessoptions.ts +18 -12
  76. package/ts/utils.json.ts +70 -18
  77. package/ts/utils.scopesnapshot.ts +75 -0
  78. package/ts_stores_nosqldb/classes.nosqldbstores.ts +52 -2
  79. package/ts_stores_nosqldb/classes.recordstore.ts +78 -8
  80. package/ts_stores_nosqldb/readme.md +10 -0
@@ -14,6 +14,7 @@ import {
14
14
  } from './errors.js';
15
15
  import type {
16
16
  IFlexAgentSessionPolicy,
17
+ IFlexCacheWarmClock,
17
18
  IFlexCallbackLimits,
18
19
  IFlexExecutionContextHandle,
19
20
  IFlexHarnessOptions,
@@ -28,6 +29,7 @@ import type {
28
29
  IFlexPendingReversionRelease,
29
30
  IFlexResolvedScope,
30
31
  IFlexResultLimits,
32
+ IFlexScopeLoadMetrics,
31
33
  IFlexScopeSnapshot,
32
34
  IFlexSession,
33
35
  IFlexSessionImportLimits,
@@ -43,6 +45,8 @@ import type {
43
45
  TFlexMessagePart,
44
46
  TFlexProjectionSnapshot,
45
47
  TFlexReversionPolicy,
48
+ TFlexScopeLoadObserver,
49
+ TFlexScopeLoadOutcome,
46
50
  } from './interfaces.js';
47
51
  import { InMemoryFlexHarnessStores } from './classes.stores.js';
48
52
  import { FlexCodeSandbox } from './classes.codesandbox.js';
@@ -50,9 +54,12 @@ import {
50
54
  assertJsonSerializable,
51
55
  canonicalJson,
52
56
  cloneSerializable,
57
+ cloneSerializableSliced,
53
58
  deepFreeze,
54
59
  resolveJsonLimits,
55
60
  } from './utils.json.js';
61
+ import { FlexTimeSlicer } from './classes.timeslicer.js';
62
+ import { liveScopeSessions, normalizeScopeSnapshot } from './utils.scopesnapshot.js';
56
63
  import { hydrateAgentMessages } from './utils.prompt.js';
57
64
  import { createProjectionChange, freezeProjectionEntries } from './utils.projectionchange.js';
58
65
  import {
@@ -60,6 +67,7 @@ import {
60
67
  normalizeAttachmentResolver,
61
68
  normalizeBuiltInTools,
62
69
  normalizeCacheWarmOptions,
70
+ normalizeClock,
63
71
  normalizeSlashCommands,
64
72
  normalizeToolCatalogOptions,
65
73
  normalizeSubagents,
@@ -95,7 +103,10 @@ import type {
95
103
  IActiveRun,
96
104
  IScopeAdmissionState,
97
105
  IFlexMessageCursor,
106
+ IProjectionContent,
107
+ IScopeLoadPass,
98
108
  TEventDetails,
109
+ TProjectionSaved,
99
110
  } from './interfaces.flexharness.js';
100
111
  import {
101
112
  scopeRetirementMessage,
@@ -131,20 +142,6 @@ const defaultMaxSubagentCallsPerRun = 32;
131
142
  const maximumMaxSubagentCallsPerRun = 128;
132
143
  const repairCancellationMessage = 'The process stopped before this run completed.';
133
144
 
134
- function deriveLegacySessionGenerationId(
135
- storageKey: string,
136
- sessionId: string,
137
- createdAt: string,
138
- ): string {
139
- const digest = plugins.crypto.createHash('sha256').update(JSON.stringify([
140
- 'flexharness-legacy-session-generation-v1',
141
- storageKey,
142
- sessionId,
143
- createdAt,
144
- ])).digest('hex');
145
- return `legacy_${digest}`;
146
- }
147
-
148
145
  /**
149
146
  * The engine behind {@link FlexHarness}: its options, scope state, events and error projection, and
150
147
  * the services that implement its API.
@@ -161,6 +158,8 @@ export class FlexHarnessCore<TScope> {
161
158
  /** The URL protocols, colon included, of the attachment references prompts may carry. */
162
159
  public readonly attachmentReferenceProtocols: ReadonlySet<string>;
163
160
  public readonly stores: IFlexHarnessStores;
161
+ public readonly sessionLoading: 'eager' | 'session-local';
162
+ private readonly legacyQualifications = new Set<string>();
164
163
  public readonly agentSessionPolicy: IFlexAgentSessionPolicy<TScope>;
165
164
  public readonly toolOutputLimits: Required<NonNullable<IFlexHarnessOptions<TScope>['toolOutputLimits']>>;
166
165
  public readonly callbackLimits: Required<IFlexCallbackLimits>;
@@ -172,14 +171,19 @@ export class FlexHarnessCore<TScope> {
172
171
  public readonly turnReversionProvider: IFlexHarnessOptions<TScope>['turnReversionProvider'];
173
172
  public readonly reversionPolicy: TFlexReversionPolicy;
174
173
  private readonly externalErrorProjector?: TFlexExternalErrorProjector;
174
+ private readonly scopeLoadObserver?: TFlexScopeLoadObserver;
175
175
  public readonly subagents: ReadonlyMap<string, Readonly<IFlexSubagentDefinition>>;
176
176
  public readonly builtInTools: INormalizedBuiltInTools;
177
177
  public readonly cacheWarmOptions: INormalizedCacheWarmOptions;
178
+ /** The clock of running turns' projection checkpoints: their spacing and save timings. */
179
+ public readonly checkpointClock: IFlexCacheWarmClock;
178
180
  public readonly toolCatalogOptions: Readonly<IFlexToolCatalogOptions>;
179
181
  public readonly slashCommands: ReadonlyMap<string, TRegisteredSlashCommand<TScope>>;
180
182
  public readonly maxSubagentDepth: number;
181
183
  public readonly maxSubagentCallsPerRun: number;
182
184
  public readonly stateLoads = new Map<string, Promise<IStorageState>>();
185
+ /** The state each settled successful load loaded. */
186
+ private readonly loadedStates = new WeakMap<Promise<IStorageState>, IStorageState>();
183
187
  public readonly scopeAdmissions = new Map<string, IScopeAdmissionState>();
184
188
  public readonly listeners = new Set<TFlexHarnessEventListener>();
185
189
  public readonly trustedInternalErrors = new WeakSet<object>();
@@ -220,6 +224,13 @@ export class FlexHarnessCore<TScope> {
220
224
  this.attachmentReferenceProtocols = normalizeAttachmentResolver(options.attachmentResolver);
221
225
  this.attachmentResolver = options.attachmentResolver;
222
226
  this.stores = requireHarnessStores(options.stores ?? new InMemoryFlexHarnessStores());
227
+ this.sessionLoading = options.sessionLoading ?? 'eager';
228
+ if (!['eager', 'session-local'].includes(this.sessionLoading)) {
229
+ throw new FlexHarnessValidationError('sessionLoading must be "eager" or "session-local".');
230
+ }
231
+ if (this.sessionLoading === 'session-local' && !this.stores.scopes.sessionLocal) {
232
+ throw new FlexHarnessValidationError('session-local loading requires scopes.sessionLocal capability.');
233
+ }
223
234
  this.agentSessionPolicy = normalizeAgentSessionPolicy(options.agentSessionPolicy);
224
235
  this.toolOutputLimits = resolveJsonLimits(options.toolOutputLimits);
225
236
  this.callbackLimits = resolveCallbackLimits(options.callbackLimits);
@@ -248,9 +259,14 @@ export class FlexHarnessCore<TScope> {
248
259
  );
249
260
  }
250
261
  this.externalErrorProjector = options.externalErrorProjector;
262
+ if (options.scopeLoadObserver !== undefined && typeof options.scopeLoadObserver !== 'function') {
263
+ throw new FlexHarnessValidationError('scopeLoadObserver must be a function.');
264
+ }
265
+ this.scopeLoadObserver = options.scopeLoadObserver;
251
266
  this.subagents = normalizeSubagents(options.subagents);
252
267
  this.builtInTools = normalizeBuiltInTools(options.builtInTools);
253
268
  this.cacheWarmOptions = normalizeCacheWarmOptions(options.cacheWarm);
269
+ this.checkpointClock = normalizeClock(options.projectionCheckpointClock, 'projectionCheckpointClock');
254
270
  const toolCatalogOptions = normalizeToolCatalogOptions(options.toolCatalog);
255
271
  this.toolCatalogOptions = Object.freeze({
256
272
  declarations: toolCatalogOptions.declarations,
@@ -358,6 +374,15 @@ export class FlexHarnessCore<TScope> {
358
374
 
359
375
  public canonicalOutcomes(events: readonly plugins.TAgentEvent[]): Map<string, TCanonicalOutcome> {
360
376
  const outcomes = new Map<string, TCanonicalOutcome>();
377
+ for (const event of events) this.rememberCanonicalOutcomes(outcomes, event);
378
+ return outcomes;
379
+ }
380
+
381
+ /** Adds the canonical outcomes `event` records to `outcomes`. */
382
+ private rememberCanonicalOutcomes(
383
+ outcomes: Map<string, TCanonicalOutcome>,
384
+ event: plugins.TAgentEvent,
385
+ ): void {
361
386
  const remember = (generationId: string, outcome: TCanonicalOutcome) => {
362
387
  const existing = outcomes.get(generationId);
363
388
  if (existing && existing !== outcome) {
@@ -365,15 +390,12 @@ export class FlexHarnessCore<TScope> {
365
390
  }
366
391
  outcomes.set(generationId, outcome);
367
392
  };
368
- for (const event of events) {
369
- if (event.type === 'generation-outcome') remember(event.generationId, event.outcome);
370
- if (event.type === 'context-compaction') {
371
- for (const transaction of event.archivedTransactions ?? []) {
372
- remember(transaction.generationId, transaction.outcome);
373
- }
393
+ if (event.type === 'generation-outcome') remember(event.generationId, event.outcome);
394
+ if (event.type === 'context-compaction') {
395
+ for (const transaction of event.archivedTransactions ?? []) {
396
+ remember(transaction.generationId, transaction.outcome);
374
397
  }
375
398
  }
376
- return outcomes;
377
399
  }
378
400
 
379
401
  public async durableCanonicalOutcomes(
@@ -383,46 +405,78 @@ export class FlexHarnessCore<TScope> {
383
405
  return this.canonicalOutcomes(snapshot?.events ?? []);
384
406
  }
385
407
 
408
+ /**
409
+ * Loads a scope's state for every request of the scope, and reports what the load took to the
410
+ * scope load observer. Its long steps are cut into slices of time, so the event loop keeps
411
+ * running beside it; disposal and the scope's retirement stop it at the next slice (the abort of
412
+ * `compactorLifecycleController`), a request that stops waiting for it does not.
413
+ */
386
414
  private async loadState(
387
415
  storageKey: string,
388
416
  scopeId: string,
389
417
  scope: TScope,
390
418
  compactorLifecycleController: AbortController,
391
419
  ): Promise<IStorageState> {
392
- const snapshot = cloneSerializable(await this.stores.scopes.load(storageKey) ?? {
393
- schemaVersion: 1,
394
- revision: 0,
395
- sessions: [],
396
- tombstones: [],
397
- } satisfies IFlexScopeSnapshot);
398
- let scopeChanged = false;
399
- const repairedGenerationSequence = snapshot.revision + 1;
400
- for (const session of snapshot.sessions) {
401
- if (session.sessionGenerationId !== undefined) continue;
402
- session.sessionGenerationId = deriveLegacySessionGenerationId(
403
- storageKey,
404
- session.sessionId,
405
- session.createdAt,
406
- );
407
- session.sessionGenerationSequence = repairedGenerationSequence;
408
- scopeChanged = true;
420
+ const startedAt = performance.now();
421
+ const pass: IScopeLoadPass = {
422
+ slicer: new FlexTimeSlicer(compactorLifecycleController.signal),
423
+ metrics: { scopeId, storageKey, sessions: 0, storeReadMs: 0, agentSessionMs: 0, repairMs: 0 },
424
+ };
425
+ let outcome: TFlexScopeLoadOutcome = 'failed';
426
+ try {
427
+ const state = await this.loadStateSliced(storageKey, scopeId, scope, compactorLifecycleController, pass);
428
+ outcome = 'loaded';
429
+ return state;
430
+ } catch (error) {
431
+ if (compactorLifecycleController.signal.aborted) outcome = 'aborted';
432
+ this.stores.scopes.sessionLocal?.releaseOwnership(storageKey);
433
+ throw error;
434
+ } finally {
435
+ this.observeScopeLoad({ ...pass.metrics, totalMs: performance.now() - startedAt, outcome });
409
436
  }
410
- for (const tombstone of snapshot.tombstones) {
411
- if (tombstone.sessionGenerationId !== undefined) continue;
412
- tombstone.sessionGenerationId = deriveLegacySessionGenerationId(
413
- storageKey,
414
- tombstone.sessionId,
415
- tombstone.deletedAt,
437
+ }
438
+
439
+ /** Reports a scope load to the scope load observer, which can neither delay nor fail it. */
440
+ private observeScopeLoad(metrics: IFlexScopeLoadMetrics): void {
441
+ const observer = this.scopeLoadObserver;
442
+ if (!observer) return;
443
+ void (async () => observer(metrics))().catch(() => undefined);
444
+ }
445
+
446
+ private async loadStateSliced(
447
+ storageKey: string,
448
+ scopeId: string,
449
+ scope: TScope,
450
+ compactorLifecycleController: AbortController,
451
+ pass: IScopeLoadPass,
452
+ ): Promise<IStorageState> {
453
+ const readStartedAt = performance.now();
454
+ if (this.sessionLoading === 'eager') this.stores.scopes.sessionLocal?.releaseOwnership(storageKey);
455
+ const local = this.sessionLoading === 'session-local'
456
+ ? await this.stores.scopes.sessionLocal!.loadCatalogue(storageKey)
457
+ : undefined;
458
+ const qualifyingLegacy = local !== undefined && !local.qualified && this.legacyQualifications.has(storageKey);
459
+ if (local && !local.qualified && !qualifyingLegacy) {
460
+ throw new FlexHarnessValidationError(
461
+ 'Session-local catalogue requires explicit prepareScope({ qualifyLegacy: true }) under exclusive namespace ownership.',
416
462
  );
417
- tombstone.sessionGenerationSequence = repairedGenerationSequence;
418
- scopeChanged = true;
419
463
  }
464
+ const stored = local ? local.snapshot : await this.stores.scopes.load(storageKey);
465
+ pass.metrics.storeReadMs += performance.now() - readStartedAt;
466
+ compactorLifecycleController.signal.throwIfAborted();
467
+ const { snapshot, changed } = normalizeScopeSnapshot(storageKey, stored);
468
+ let scopeChanged = changed || (qualifyingLegacy && stored === undefined);
420
469
  const state: IStorageState = {
421
470
  storageKey,
422
471
  compactorLifecycleController,
423
472
  scopeIdHint: scopeId,
424
473
  scopeContext: { scopeId, scope },
425
474
  revision: snapshot.revision,
475
+ catalogue: new Map(liveScopeSessions(snapshot).map((session) => [session.sessionId, session])),
476
+ sessionLocal: local !== undefined,
477
+ qualifyingLegacy,
478
+ recoverySessionIds: new Set(local?.recoverySessionIds ?? []),
479
+ sessionLoads: new Map(),
426
480
  sessions: new Map(),
427
481
  retainedSessionCleanups: new Map(),
428
482
  sessionDeletions: new Map(),
@@ -443,39 +497,65 @@ export class FlexHarnessCore<TScope> {
443
497
  fenceAdditionalErrors: [],
444
498
  detachedCleanups: new Set(),
445
499
  };
500
+ for (const sessionId of state.recoverySessionIds) {
501
+ if (!state.catalogue.has(sessionId) && !state.tombstones.has(sessionId)) {
502
+ throw new FlexHarnessValidationError(`Recovery obligation "${sessionId}" has no canonical session or tombstone owner.`);
503
+ }
504
+ }
446
505
  const loadedSessions: IStoredSessionState[] = [];
506
+ // Eager qualification still joins every runtime; the independent catalogue is already complete.
507
+ let sessionsLoaded = false;
447
508
  try {
448
- for (const metadata of snapshot.sessions) {
449
- if (state.tombstones.has(metadata.sessionId)) continue;
450
- const stored = await this.loadSessionRuntime(state, metadata, scopeId, scope);
451
- state.sessions.set(metadata.sessionId, stored);
452
- loadedSessions.push(stored);
453
- scopeChanged = (await this.repairLoadedSession(
454
- state.storageKey,
455
- stored,
509
+ // An authoritative recovery obligation gates the namespace; unrelated sessions stay cold.
510
+ for (const metadata of liveScopeSessions(snapshot).filter((session) =>
511
+ !state.sessionLocal || qualifyingLegacy || state.recoverySessionIds.has(session.sessionId))) {
512
+ compactorLifecycleController.signal.throwIfAborted();
513
+ const stored = await this.loadSessionRuntime(
456
514
  state,
515
+ metadata,
457
516
  scopeId,
458
517
  scope,
459
- )) || scopeChanged;
460
- await this.reversions.drainReversionReleases(state, stored, scopeId, scope, false);
518
+ undefined,
519
+ false,
520
+ pass,
521
+ );
522
+ state.sessions.set(metadata.sessionId, stored);
523
+ loadedSessions.push(stored);
524
+ pass.metrics.sessions = loadedSessions.length;
525
+ const repairStartedAt = performance.now();
526
+ try {
527
+ scopeChanged = (await this.repairLoadedSession(
528
+ state.storageKey,
529
+ stored,
530
+ state,
531
+ scopeId,
532
+ scope,
533
+ pass.slicer,
534
+ )) || scopeChanged;
535
+ } finally {
536
+ pass.metrics.repairMs += performance.now() - repairStartedAt;
537
+ }
538
+ state.catalogue.set(metadata.sessionId, stored.session);
539
+ await this.reversions.drainReversionReleases(state, stored, scopeId, scope, state.sessionLocal);
461
540
  if (stored.projectionRevision < 0) throw new Error('Invalid projection revision.');
462
541
  }
463
- for (const stored of state.sessions.values()) {
464
- let ancestorId = stored.session.parentSessionId;
465
- const visited = new Set<string>([stored.session.sessionId]);
542
+ sessionsLoaded = true;
543
+ for (const metadata of state.catalogue.values()) {
544
+ let ancestorId = metadata.parentSessionId;
545
+ const visited = new Set<string>([metadata.sessionId]);
466
546
  while (ancestorId) {
467
547
  if (visited.has(ancestorId)) {
468
548
  throw new FlexHarnessValidationError(
469
- `Live session "${stored.session.sessionId}" has a cyclic ancestor chain.`,
549
+ `Live session "${metadata.sessionId}" has a cyclic ancestor chain.`,
470
550
  );
471
551
  }
472
552
  visited.add(ancestorId);
473
553
  if (state.tombstones.has(ancestorId)) {
474
554
  throw new FlexHarnessValidationError(
475
- `Live session "${stored.session.sessionId}" descends from tombstoned ancestor "${ancestorId}".`,
555
+ `Live session "${metadata.sessionId}" descends from tombstoned ancestor "${ancestorId}".`,
476
556
  );
477
557
  }
478
- ancestorId = state.sessions.get(ancestorId)?.session.parentSessionId;
558
+ ancestorId = state.catalogue.get(ancestorId)?.parentSessionId;
479
559
  }
480
560
  }
481
561
  const tombstoneRoots = this.deletion.orderTombstoneRootsChildFirst(
@@ -497,6 +577,12 @@ export class FlexHarnessCore<TScope> {
497
577
  );
498
578
  await this.deletion.cleanupSessionDomains(state, tombstone);
499
579
  }
580
+ for (const tombstone of group) {
581
+ if (state.sessionLocal && !state.qualifyingLegacy) {
582
+ await this.stores.scopes.sessionLocal!.setRecoveryRequired(storageKey, tombstone.sessionId, false);
583
+ }
584
+ state.recoverySessionIds.delete(tombstone.sessionId);
585
+ }
500
586
  for (const tombstone of group) state.tombstones.delete(tombstone.sessionId);
501
587
  scopeChanged = true;
502
588
  } catch (error) {
@@ -509,20 +595,35 @@ export class FlexHarnessCore<TScope> {
509
595
  }
510
596
  if (scopeChanged) {
511
597
  const next = this.createScopeSnapshot(state, state.revision + 1);
512
- await this.stores.scopes.save(storageKey, next, state.revision);
598
+ await this.saveScopeSnapshot(state, next, state.revision);
513
599
  state.revision = next.revision;
514
600
  }
601
+ if (state.sessionLocal) {
602
+ if (qualifyingLegacy) {
603
+ await this.stores.scopes.sessionLocal!.qualifyCatalogue(storageKey,
604
+ this.createScopeSnapshot(state, state.revision), [...state.recoverySessionIds]);
605
+ state.qualifyingLegacy = false;
606
+ }
607
+ for (const loaded of loadedSessions) {
608
+ loaded.recovering = false;
609
+ await this.updateRecoveryObligation(state, loaded);
610
+ }
611
+ }
515
612
  return state;
516
613
  } catch (error) {
517
- for (const rootSessionId of new Set([...state.tombstones.values()].map((tombstone) =>
518
- tombstone.rootSessionId ?? tombstone.sessionId))) {
519
- const key = JSON.stringify([storageKey, rootSessionId]);
520
- this.lifecycle.orphanedTombstoneOwners.set(key, {
521
- state,
522
- rootSessionId,
523
- scopeId,
524
- scope,
525
- });
614
+ // No tombstone was touched until the selected recovery/qualification sessions all loaded.
615
+ // Retain cleanup owners only after that boundary; the catalogue stays complete throughout.
616
+ if (sessionsLoaded) {
617
+ for (const rootSessionId of new Set([...state.tombstones.values()].map((tombstone) =>
618
+ tombstone.rootSessionId ?? tombstone.sessionId))) {
619
+ const key = JSON.stringify([storageKey, rootSessionId]);
620
+ this.lifecycle.orphanedTombstoneOwners.set(key, {
621
+ state,
622
+ rootSessionId,
623
+ scopeId,
624
+ scope,
625
+ });
626
+ }
526
627
  }
527
628
  const closeResults = await Promise.allSettled(
528
629
  loadedSessions.map((stored) => this.lifecycle.closeStoredSession(stored)),
@@ -541,6 +642,208 @@ export class FlexHarnessCore<TScope> {
541
642
  }
542
643
  }
543
644
 
645
+ /** Enumerates metadata/proof only; explicit prepareScope owns startup recovery. */
646
+ public async getScopeCatalogue(scopeId: string): Promise<import('./interfaces.js').IFlexScopeCatalogue> {
647
+ return this.withScopeAdmission(scopeId, async (scope, admitted) => {
648
+ const load = this.stateLoads.get(scope.storageKey);
649
+ if (load && this.isStateLoaded(load)) {
650
+ const state = await this.admittedState(scopeId, scope, load, admitted);
651
+ await state.scopeQueue;
652
+ this.assertStateAcceptingWork(state);
653
+ return publicSnapshot({
654
+ sessions: [...state.catalogue.values()]
655
+ .filter((session) => !state.initializingSessions.has(session.sessionId))
656
+ .sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
657
+ recoverySessionIds: [...state.recoverySessionIds].sort(),
658
+ });
659
+ }
660
+ if (this.sessionLoading !== 'session-local') {
661
+ await this.resolveState(scopeId);
662
+ return this.getScopeCatalogue(scopeId);
663
+ }
664
+ try {
665
+ const catalogue = await this.stores.scopes.sessionLocal!.loadCatalogue(scope.storageKey);
666
+ this.assertOpen();
667
+ if (!admitted()) throw this.createScopeRetirementError();
668
+ if (!catalogue.qualified) {
669
+ throw new FlexHarnessValidationError('Session-local catalogue requires explicit legacy qualification.');
670
+ }
671
+ const { snapshot } = normalizeScopeSnapshot(scope.storageKey, catalogue.snapshot);
672
+ return publicSnapshot({ sessions: liveScopeSessions(snapshot).sort((left, right) =>
673
+ left.sessionId.localeCompare(right.sessionId)), recoverySessionIds: catalogue.recoverySessionIds.slice().sort() });
674
+ } finally {
675
+ // A metadata-only reader owns no runtime lifecycle on which to retain writer activation.
676
+ if (!this.stateLoads.has(scope.storageKey)) this.stores.scopes.sessionLocal!.releaseOwnership(scope.storageKey);
677
+ }
678
+ });
679
+ }
680
+
681
+ public async prepareScope(
682
+ scopeId: string,
683
+ options: import('./interfaces.js').IFlexPrepareScopeOptions = {},
684
+ ): Promise<import('./interfaces.js').IFlexScopeCatalogue> {
685
+ if (!options || typeof options !== 'object' || Array.isArray(options)
686
+ || Object.keys(options).some((key) => key !== 'qualifyLegacy')
687
+ || (options.qualifyLegacy !== undefined && typeof options.qualifyLegacy !== 'boolean')) {
688
+ throw new FlexHarnessValidationError('prepareScope options must contain only qualifyLegacy (boolean).');
689
+ }
690
+ return this.withScopeAdmission(scopeId, async (scope) => {
691
+ if (options.qualifyLegacy) this.legacyQualifications.add(scope.storageKey);
692
+ try {
693
+ await this.resolveState(scopeId);
694
+ return await this.getScopeCatalogue(scopeId);
695
+ } finally {
696
+ if (options.qualifyLegacy) this.legacyQualifications.delete(scope.storageKey);
697
+ }
698
+ });
699
+ }
700
+
701
+ public async hydrateSession(
702
+ state: IStorageState,
703
+ sessionId: string,
704
+ scopeId: string,
705
+ scope: TScope,
706
+ ): Promise<IStoredSessionState> {
707
+ const pending = state.sessionLoads.get(sessionId);
708
+ if (pending) return pending;
709
+ const existing = state.sessions.get(sessionId);
710
+ if (existing) return existing;
711
+ let load!: Promise<IStoredSessionState>;
712
+ load = (async () => {
713
+ await this.sessionReadBarrier(state, sessionId);
714
+ this.assertStateAcceptingWork(state);
715
+ const metadata = state.catalogue.get(sessionId);
716
+ if (!metadata || state.tombstones.has(sessionId)) throw new FlexHarnessNotFoundError('Session', sessionId);
717
+ const pass: IScopeLoadPass = {
718
+ slicer: new FlexTimeSlicer(state.compactorLifecycleController.signal),
719
+ metrics: { scopeId, storageKey: state.storageKey, sessions: 1, storeReadMs: 0, agentSessionMs: 0, repairMs: 0 },
720
+ };
721
+ const stored = await this.loadSessionRuntime(state, metadata, scopeId, scope, undefined, false, pass);
722
+ try {
723
+ state.compactorLifecycleController.signal.throwIfAborted();
724
+ this.assertStateAcceptingWork(state);
725
+ const current = state.catalogue.get(sessionId);
726
+ if (!current || state.tombstones.has(sessionId)
727
+ || current.sessionGenerationId !== metadata.sessionGenerationId
728
+ || current.sessionGenerationSequence !== metadata.sessionGenerationSequence) {
729
+ throw new FlexHarnessAbortError('The session was deleted while it was being hydrated.');
730
+ }
731
+ // Runtime ownership is visible to drain before repair can acquire more resources.
732
+ state.sessions.set(sessionId, stored);
733
+ await this.updateRecoveryObligation(state, stored, true);
734
+ const changed = await this.repairLoadedSession(state.storageKey, stored, state, scopeId, scope, pass.slicer);
735
+ await this.reversions.drainReversionReleases(state, stored, scopeId, scope, state.sessionLocal);
736
+ if (changed) await this.mutateSessionScope(state, sessionId, () => undefined);
737
+ else state.catalogue.set(sessionId, stored.session);
738
+ stored.recovering = false;
739
+ await this.updateRecoveryObligation(state, stored);
740
+ this.assertStateAcceptingWork(state);
741
+ return stored;
742
+ } catch (error) {
743
+ if (state.sessions.get(sessionId) === stored) state.sessions.delete(sessionId);
744
+ try {
745
+ await this.lifecycle.closeStoredSession(stored);
746
+ } catch (closeError) {
747
+ this.lifecycle.orphanedStoredSessions.add(stored);
748
+ throw combineErrors([error, closeError]);
749
+ }
750
+ throw error;
751
+ }
752
+ })().finally(() => {
753
+ if (state.sessionLoads.get(sessionId) === load) state.sessionLoads.delete(sessionId);
754
+ });
755
+ state.sessionLoads.set(sessionId, load);
756
+ return load;
757
+ }
758
+
759
+ /** Retries ownerless shared-workspace obligations, never an active run's normal capture. */
760
+ private async recoverSharedState(state: IStorageState, scopeId: string, scope: TScope): Promise<void> {
761
+ if (!state.sessionLocal) return;
762
+ if (state.sharedRecovery) return state.sharedRecovery;
763
+ const ids = [...state.recoverySessionIds].filter((id) =>
764
+ !state.activeRuns.has(id) && !state.initializingSessions.has(id)
765
+ && !(state.sessions.get(id)?.outstandingPromptsById.size));
766
+ const roots = [...new Set([...state.tombstones.values()].map((entry) => entry.rootSessionId ?? entry.sessionId))]
767
+ .filter((id) => !state.sessionDeletions.has(id) && !state.tombstoneCleanups.has(id)
768
+ && !state.initializingSessions.has(id));
769
+ if (ids.length === 0 && roots.length === 0) return;
770
+ let recovery!: Promise<void>;
771
+ recovery = (async () => {
772
+ for (const id of ids) {
773
+ if (state.tombstones.has(id)) continue;
774
+ if (!state.catalogue.has(id)) {
775
+ throw new FlexHarnessValidationError(`Recovery obligation "${id}" has no canonical session or tombstone owner.`);
776
+ }
777
+ if (state.sessionLoads.has(id)) { await state.sessionLoads.get(id); continue; }
778
+ const stored = state.sessions.get(id);
779
+ if (!stored) { await this.hydrateSession(state, id, scopeId, scope); continue; }
780
+ stored.recovering = true;
781
+ await this.repairLoadedSession(state.storageKey, stored, state, scopeId, scope,
782
+ new FlexTimeSlicer(state.compactorLifecycleController.signal));
783
+ await this.reversions.drainReversionReleases(state, stored, scopeId, scope, true);
784
+ // A preceding failed recovery may already have repaired metadata only in memory.
785
+ await this.mutateSessionScope(state, id, () => undefined);
786
+ stored.recovering = false;
787
+ await this.updateRecoveryObligation(state, stored);
788
+ }
789
+ for (const root of roots) await this.deletion.finishTombstoneCleanup(state, root, scopeId, { scopeId, scope });
790
+ })().finally(() => { if (state.sharedRecovery === recovery) delete state.sharedRecovery; });
791
+ state.sharedRecovery = recovery;
792
+ return recovery;
793
+ }
794
+
795
+ public async hydrateAllSessions(state: IStorageState, scopeId: string, scope: TScope): Promise<void> {
796
+ for (const sessionId of state.catalogue.keys()) await this.hydrateSession(state, sessionId, scopeId, scope);
797
+ }
798
+
799
+ private async saveScopeSnapshot(state: IStorageState, snapshot: IFlexScopeSnapshot, expected: number): Promise<void> {
800
+ if (state.sessionLocal && !state.qualifyingLegacy) {
801
+ await this.stores.scopes.sessionLocal!.saveCatalogue(state.storageKey, snapshot, expected);
802
+ } else await this.stores.scopes.save(state.storageKey, snapshot, expected);
803
+ }
804
+
805
+ /** Registers before the projection's pending intent can reach any workspace provider. */
806
+ public async updateRecoveryObligation(state: IStorageState, stored: IStoredSessionState, registerOnly = false): Promise<void> {
807
+ if (!state.sessionLocal) return;
808
+ const sessionId = stored.session.sessionId;
809
+ const active = state.activeRuns.get(sessionId);
810
+ const required = stored.pendingReversion !== undefined || stored.pendingReversionReleases.length > 0
811
+ || stored.stagedTerminals.length > 0 || stored.metrics.pendingGenerationId !== undefined
812
+ || stored.metrics.pendingCompactionId !== undefined
813
+ || (stored.recovering === true && state.recoverySessionIds.has(sessionId))
814
+ || (this.turnReversionProvider !== undefined && active !== undefined && !active.finishedEventEmitted);
815
+ if (registerOnly && !required) return;
816
+ if (state.qualifyingLegacy) {
817
+ if (required) state.recoverySessionIds.add(sessionId);
818
+ else state.recoverySessionIds.delete(sessionId);
819
+ return;
820
+ }
821
+ // Re-registering is idempotent. The durable sidecar is authoritative, not this cache.
822
+ try {
823
+ await this.stores.scopes.sessionLocal!.setRecoveryRequired(state.storageKey, sessionId, required);
824
+ if (required) state.recoverySessionIds.add(sessionId);
825
+ else state.recoverySessionIds.delete(sessionId);
826
+ } catch (error) {
827
+ state.lifecycle = 'fenced';
828
+ throw error;
829
+ }
830
+ }
831
+
832
+ /**
833
+ * Brings a session's recovery obligation up to date after an outcome is already recorded: a
834
+ * finished prompt or run, a saved projection. Its failure never changes that outcome. The
835
+ * durable obligation stays as it was, which the next load repairs, and the namespace is fenced
836
+ * through its drain instead of throwing into the caller.
837
+ */
838
+ public async settleRecoveryObligation(state: IStorageState, stored: IStoredSessionState): Promise<void> {
839
+ if (!state.sessionLocal || state.lifecycle !== 'active') return;
840
+ try {
841
+ await this.updateRecoveryObligation(state, stored);
842
+ } catch (error) {
843
+ this.lifecycle.deferNamespaceDrain(state, error);
844
+ }
845
+ }
846
+
544
847
  public createUninitializedStoredSession(
545
848
  session: IFlexSession,
546
849
  storageKey: string,
@@ -590,6 +893,7 @@ export class FlexHarnessCore<TScope> {
590
893
  scope: TScope,
591
894
  compactorLifecycleController = new AbortController(),
592
895
  newSession = false,
896
+ pass?: IScopeLoadPass,
593
897
  ): Promise<IStoredSessionState> {
594
898
  const sessionId = metadata.sessionId;
595
899
  const compactorContext = this.compaction.createCompactorContext(
@@ -598,12 +902,14 @@ export class FlexHarnessCore<TScope> {
598
902
  state.storageKey,
599
903
  sessionId,
600
904
  );
905
+ const readStartedAt = performance.now();
601
906
  const [projectionResult, permissionResult, eventStoreResult, jobStoreResult] = await Promise.allSettled([
602
907
  this.stores.projections.load(state.storageKey, sessionId),
603
908
  this.stores.permissions.load(state.storageKey, sessionId),
604
909
  this.stores.agentEvents.getStore(state.storageKey, sessionId),
605
910
  this.stores.jobs.getStore(state.storageKey, sessionId),
606
911
  ]);
912
+ if (pass) pass.metrics.storeReadMs += performance.now() - readStartedAt;
607
913
  const acquisitionErrors = [projectionResult, permissionResult, eventStoreResult, jobStoreResult]
608
914
  .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
609
915
  .map((result) => result.reason);
@@ -676,11 +982,21 @@ export class FlexHarnessCore<TScope> {
676
982
  throw new FlexHarnessValidationError('Execution context provider returned an invalid handle.');
677
983
  }
678
984
  }
985
+ // A scope load copies and freezes the messages in slices of time. The baseline shares the
986
+ // frozen copies, as it is only ever replaced, never edited.
987
+ const messages = pass
988
+ ? await cloneSerializableSliced(projection?.messages ?? [], pass.slicer)
989
+ : cloneSerializable(projection?.messages ?? []);
990
+ for (const message of messages) {
991
+ deepFreeze(message);
992
+ if (pass?.slicer.due) await pass.slicer.pause();
993
+ }
679
994
  const stored: IStoredSessionState = {
680
995
  storageKey: state.storageKey,
996
+ recovering: !newSession,
681
997
  compactorContext,
682
998
  session: cloneSerializable(metadata),
683
- messages: cloneSerializable(projection?.messages ?? []),
999
+ messages,
684
1000
  stagedTerminals: cloneSerializable(projection?.stagedTerminals ?? []),
685
1001
  reversionSegments,
686
1002
  revertCursor: currentProjection?.revertCursor ?? legacyProjection?.revertCursor ?? 0,
@@ -694,7 +1010,12 @@ export class FlexHarnessCore<TScope> {
694
1010
  projectionSchemaVersion: projection?.schemaVersion ?? 3,
695
1011
  projectionReconciliationRequired: false,
696
1012
  ...(projection
697
- ? { projectionBaseline: cloneSerializable(projection) }
1013
+ ? {
1014
+ projectionBaseline: deepFreeze({
1015
+ ...cloneSerializable({ ...projection, messages: [] }),
1016
+ messages: messages.slice(),
1017
+ } as TFlexProjectionSnapshot),
1018
+ }
698
1019
  : {}),
699
1020
  projectionRevision: projection?.revision ?? 0,
700
1021
  projectionQueue: Promise.resolve(),
@@ -867,10 +1188,15 @@ export class FlexHarnessCore<TScope> {
867
1188
  onRetry: (event) => this.callbacks.onRetry(state, sessionId, event),
868
1189
  onUsage: (event) => this.callbacks.onModelCallUsage(state, stored, event),
869
1190
  };
870
- stored.agentSession = await this.compaction.withCompactorContext(
871
- compactorContext,
872
- () => plugins.AgentSession.create(agentSessionOptions),
873
- );
1191
+ const agentSessionStartedAt = performance.now();
1192
+ try {
1193
+ stored.agentSession = await this.compaction.withCompactorContext(
1194
+ compactorContext,
1195
+ () => plugins.AgentSession.create(agentSessionOptions),
1196
+ );
1197
+ } finally {
1198
+ if (pass) pass.metrics.agentSessionMs += performance.now() - agentSessionStartedAt;
1199
+ }
874
1200
  return stored;
875
1201
  } catch (error) {
876
1202
  const cleanupErrors: unknown[] = [];
@@ -905,26 +1231,25 @@ export class FlexHarnessCore<TScope> {
905
1231
  }
906
1232
  }
907
1233
 
1234
+ /**
1235
+ * Repairs what a stopped process left in a loaded session: every run gets its terminal, the
1236
+ * reversion state its runs' outcomes, the session its latest turn's state. Linear in the
1237
+ * session's size, in slices of `slicer`'s time; it saves the projection only if its content
1238
+ * changed.
1239
+ */
908
1240
  private async repairLoadedSession(
909
1241
  storageKey: string,
910
1242
  stored: IStoredSessionState,
911
- state?: IStorageState,
912
- scopeId?: string,
913
- scope?: TScope,
1243
+ state: IStorageState,
1244
+ scopeId: string,
1245
+ scope: TScope,
1246
+ slicer: FlexTimeSlicer,
914
1247
  ): Promise<boolean> {
915
1248
  if (stored.session.origin !== undefined) {
916
1249
  this.assertImportedSessionState(stored);
917
1250
  return false;
918
1251
  }
919
- const beforeProjection = JSON.stringify({
920
- messages: stored.messages,
921
- stagedTerminals: stored.stagedTerminals,
922
- reversionSegments: stored.reversionSegments,
923
- revertCursor: stored.revertCursor,
924
- excludedRunIds: stored.excludedRunIds,
925
- pendingReversion: stored.pendingReversion,
926
- pendingReversionReleases: stored.pendingReversionReleases,
927
- });
1252
+ const before = this.projectionContent(stored);
928
1253
  const beforeSession = JSON.stringify(stored.session);
929
1254
  if (
930
1255
  !this.turnReversionProvider
@@ -957,75 +1282,28 @@ export class FlexHarnessCore<TScope> {
957
1282
  );
958
1283
  }
959
1284
  if (stored.pendingReversion?.kind === 'capture' && stored.pendingReversionReleases.length > 0) {
960
- if (!state || scopeId === undefined || scope === undefined) {
961
- throw new FlexHarnessValidationError('Pending capture recovery is missing its scope context.');
962
- }
963
- await this.reversions.drainReversionReleases(state, stored, scopeId, scope, false);
1285
+ await this.reversions.drainReversionReleases(state, stored, scopeId, scope, state.sessionLocal);
964
1286
  this.assertStateAcceptingWork(state);
965
1287
  }
1288
+ // The outcomes and model context of the session's runs, before a pending reversion resumes.
966
1289
  const agentEvents = stored.agentSession.getEvents();
967
- const outcomes = this.canonicalOutcomes(agentEvents);
1290
+ const outcomes = new Map<string, TCanonicalOutcome>();
1291
+ for (const event of agentEvents) {
1292
+ this.rememberCanonicalOutcomes(outcomes, event);
1293
+ if (slicer.due) await slicer.pause();
1294
+ }
968
1295
  const firstHiddenCandidate = this.reversions.completedReversionCandidates(stored)[stored.revertCursor];
969
1296
  const reversionVisibilityBoundary = firstHiddenCandidate
970
1297
  ? stored.reversionSegments.findIndex((segment) => segment.runId === firstHiddenCandidate.runId)
971
1298
  : stored.reversionSegments.length;
972
- const stages = new Map(stored.stagedTerminals.map((terminal) => [terminal.runId, terminal]));
973
- const runIds = new Set([
974
- ...stored.messages.map((message) => message.runId),
975
- ...stored.stagedTerminals.map((terminal) => terminal.runId),
976
- ]);
977
- // The session's state is the latest real turn's: a cache-warm turn's terminal never sets it.
978
- let latestTerminal: IFlexTerminalProjection | undefined;
979
- for (const runId of runIds) {
980
- const outcome = outcomes.get(runId);
981
- const stage = stages.get(runId);
982
- if (outcome === 'accepted' && stage?.status === 'completed') {
983
- // A cache-warm exchange leaves the model context with its outcome.
984
- this.runs.replaceVisibleRun(stored, this.runs.withKeptInContext(stage, !stage.assistantMessage.cacheWarm));
985
- if (!stage.assistantMessage.cacheWarm) latestTerminal = stage;
986
- } else if (outcome !== 'accepted') {
987
- const visible = stored.messages.filter((message) => message.runId === runId);
988
- // A run that failed or was stopped keeps its terminal and its error: the staged one, or the
989
- // one it published. Only a run that never reached its terminal is repaired as interrupted.
990
- const ended = stage && stage.status !== 'completed'
991
- ? stage
992
- : stage ? undefined : this.publishedTerminal(runId, visible);
993
- if (ended) {
994
- if (ended === stage) {
995
- this.runs.replaceVisibleRun(stored, this.runs.withKeptInContext(
996
- stage,
997
- plugins.isGenerationInModelContext(agentEvents, runId),
998
- ));
999
- }
1000
- if (!ended.assistantMessage.cacheWarm) latestTerminal = ended;
1001
- } else if (visible.length > 0 || stage) {
1002
- const status = outcome === 'rejected' ? 'failed' : 'cancelled';
1003
- const terminal = this.runs.withKeptInContext(
1004
- this.repairTerminal(stored.session.sessionId, runId, visible, stage, status),
1005
- plugins.isGenerationInModelContext(agentEvents, runId),
1006
- );
1007
- this.runs.replaceVisibleRun(stored, terminal);
1008
- if (!terminal.assistantMessage.cacheWarm) latestTerminal = terminal;
1009
- }
1010
- } else {
1011
- const visibleSuccess = stored.messages.some((message) =>
1012
- message.runId === runId && message.status === 'completed');
1013
- if (visibleSuccess) {
1014
- const terminal = this.repairTerminal(
1015
- stored.session.sessionId,
1016
- runId,
1017
- stored.messages.filter((message) => message.runId === runId),
1018
- undefined,
1019
- 'completed',
1020
- );
1021
- if (!terminal.assistantMessage.cacheWarm) latestTerminal = terminal;
1022
- }
1023
- }
1024
- }
1299
+ const latestTerminal = await this.repairRunTerminals(stored, agentEvents, outcomes, slicer);
1025
1300
  stored.stagedTerminals = stored.stagedTerminals.filter((terminal) => !outcomes.has(terminal.runId));
1026
1301
  if (stored.stagedTerminals.length > 0) stored.stagedTerminals = [];
1027
- const activeEvents = stored.agentSession.getEvents();
1028
- const activeEventIds = new Set(activeEvents.map((event) => event.id));
1302
+ const activeEventIds = new Set<string>();
1303
+ for (const event of stored.agentSession.getEvents()) {
1304
+ activeEventIds.add(event.id);
1305
+ if (slicer.due) await slicer.pause();
1306
+ }
1029
1307
  stored.reversionSegments = stored.reversionSegments.map((segment) => {
1030
1308
  const contextAvailable = segment.contextAvailable
1031
1309
  && (segment.eventIds.length === 0 || segment.eventIds.some((id) => activeEventIds.has(id)));
@@ -1047,12 +1325,9 @@ export class FlexHarnessCore<TScope> {
1047
1325
  this.reversions.commitArchivedHiddenBranchState(stored);
1048
1326
  this.reversions.pruneReversionState(stored, false, true);
1049
1327
  if (stored.pendingReversion?.kind === 'apply') {
1050
- if (!state || scopeId === undefined || scope === undefined) {
1051
- throw new FlexHarnessValidationError('Pending reversion recovery is missing its scope context.');
1052
- }
1053
1328
  await this.reversions.resumePendingApply(state, stored, scopeId, scope);
1054
1329
  } else if (stored.pendingReversion?.kind === 'capture') {
1055
- if (!this.turnReversionProvider || !state || scopeId === undefined || scope === undefined) {
1330
+ if (!this.turnReversionProvider) {
1056
1331
  throw new FlexHarnessValidationError('Pending capture recovery requires its reversion provider.');
1057
1332
  }
1058
1333
  await this.reversions.recoverPendingCapture(state, stored, scopeId, scope, outcomes);
@@ -1084,17 +1359,12 @@ export class FlexHarnessCore<TScope> {
1084
1359
  };
1085
1360
  stored.session.updatedAt = timestamp;
1086
1361
  }
1087
- const projectionChanged = beforeProjection !== JSON.stringify({
1088
- messages: stored.messages,
1089
- stagedTerminals: stored.stagedTerminals,
1090
- reversionSegments: stored.reversionSegments,
1091
- revertCursor: stored.revertCursor,
1092
- excludedRunIds: stored.excludedRunIds,
1093
- pendingReversion: stored.pendingReversion,
1094
- pendingReversionReleases: stored.pendingReversionReleases,
1095
- });
1096
- if (projectionChanged) {
1362
+ if (await this.projectionContentChanged(before, stored, slicer)) {
1097
1363
  const expected = stored.projectionRevision;
1364
+ for (const message of stored.messages) {
1365
+ deepFreeze(message);
1366
+ if (slicer.due) await slicer.pause();
1367
+ }
1098
1368
  freezeProjectionEntries(stored);
1099
1369
  const change = this.createProjectionChange(stored, expected + 1);
1100
1370
  await this.stores.projections.save(
@@ -1112,6 +1382,215 @@ export class FlexHarnessCore<TScope> {
1112
1382
  return beforeSession !== JSON.stringify(stored.session);
1113
1383
  }
1114
1384
 
1385
+ /**
1386
+ * Gives every run of a loaded session its terminal, in one pass over its messages, and returns
1387
+ * the terminal of its latest real turn. A run with an accepted completed stage takes the stage;
1388
+ * a run that failed or was stopped keeps the terminal it staged or published; a run that never
1389
+ * reached its terminal is repaired as failed or cancelled. Each terminal goes where
1390
+ * `runs.replaceVisibleRun()` would put it.
1391
+ */
1392
+ private async repairRunTerminals(
1393
+ stored: IStoredSessionState,
1394
+ agentEvents: readonly plugins.TAgentEvent[],
1395
+ outcomes: ReadonlyMap<string, TCanonicalOutcome>,
1396
+ slicer: FlexTimeSlicer,
1397
+ ): Promise<IFlexTerminalProjection | undefined> {
1398
+ const runMessages = new Map<string, IFlexMessage[]>();
1399
+ const lastRunMessage = new Map<string, IFlexMessage>();
1400
+ for (const message of stored.messages) {
1401
+ let visible = runMessages.get(message.runId);
1402
+ if (!visible) {
1403
+ visible = [];
1404
+ runMessages.set(message.runId, visible);
1405
+ }
1406
+ visible.push(message);
1407
+ lastRunMessage.set(message.runId, message);
1408
+ if (slicer.due) await slicer.pause();
1409
+ }
1410
+ const stages = new Map(stored.stagedTerminals.map((terminal) => [terminal.runId, terminal]));
1411
+ const runIds = new Set([
1412
+ ...runMessages.keys(),
1413
+ ...stored.stagedTerminals.map((terminal) => terminal.runId),
1414
+ ]);
1415
+ let modelContextRunIds: Set<string> | undefined;
1416
+ const inModelContext = (runId: string) =>
1417
+ (modelContextRunIds ??= plugins.getGenerationsInModelContext(agentEvents)).has(runId);
1418
+ const terminals = new Map<string, IFlexTerminalProjection>();
1419
+ // The session's state is the latest real turn's: a cache-warm turn's terminal never sets it. The
1420
+ // terminal of an accepted run without a completed stage only names that state, so it is built
1421
+ // for the latest such run alone.
1422
+ let latestTerminal: IFlexTerminalProjection | (() => IFlexTerminalProjection) | undefined;
1423
+ for (const runId of runIds) {
1424
+ const outcome = outcomes.get(runId);
1425
+ const stage = stages.get(runId);
1426
+ const visible = runMessages.get(runId) ?? [];
1427
+ if (outcome === 'accepted' && stage?.status === 'completed') {
1428
+ // A cache-warm exchange leaves the model context with its outcome.
1429
+ terminals.set(runId, this.runs.withKeptInContext(stage, !stage.assistantMessage.cacheWarm));
1430
+ if (!stage.assistantMessage.cacheWarm) latestTerminal = stage;
1431
+ } else if (outcome !== 'accepted') {
1432
+ // A run that failed or was stopped keeps its terminal and its error: the staged one, or the
1433
+ // one it published. Only a run that never reached its terminal is repaired as interrupted.
1434
+ const ended = stage && stage.status !== 'completed'
1435
+ ? stage
1436
+ : stage ? undefined : this.publishedTerminal(runId, visible);
1437
+ if (ended) {
1438
+ if (ended === stage) {
1439
+ terminals.set(runId, this.runs.withKeptInContext(stage, inModelContext(runId)));
1440
+ }
1441
+ if (!ended.assistantMessage.cacheWarm) latestTerminal = ended;
1442
+ } else if (visible.length > 0 || stage) {
1443
+ const status = outcome === 'rejected' ? 'failed' : 'cancelled';
1444
+ const terminal = this.runs.withKeptInContext(
1445
+ this.repairTerminal(stored.session.sessionId, runId, visible, stage, status),
1446
+ inModelContext(runId),
1447
+ );
1448
+ terminals.set(runId, terminal);
1449
+ if (!terminal.assistantMessage.cacheWarm) latestTerminal = terminal;
1450
+ }
1451
+ } else if (visible.some((message) => message.status === 'completed')) {
1452
+ let assistant: IFlexMessage | undefined;
1453
+ for (const message of visible) if (message.role === 'assistant') assistant = message;
1454
+ if (!assistant?.cacheWarm) {
1455
+ latestTerminal = () =>
1456
+ this.repairTerminal(stored.session.sessionId, runId, visible, undefined, 'completed');
1457
+ }
1458
+ }
1459
+ if (slicer.due) await slicer.pause();
1460
+ }
1461
+ if (terminals.size > 0) {
1462
+ stored.messages = await this.placeRunTerminals(
1463
+ stored.messages,
1464
+ terminals,
1465
+ runMessages,
1466
+ lastRunMessage,
1467
+ inModelContext,
1468
+ slicer,
1469
+ );
1470
+ }
1471
+ return typeof latestTerminal === 'function' ? latestTerminal() : latestTerminal;
1472
+ }
1473
+
1474
+ /**
1475
+ * The messages with each run's terminal in place, built in one pass: what
1476
+ * `runs.replaceVisibleRun()` gives for each terminal in turn. A run's projected terminal messages
1477
+ * are replaced, its other user messages take the prompt's `keptInContext`, and a terminal
1478
+ * message not projected yet goes before the run's first message (its prompt) or after its last
1479
+ * (its answer); for a run without messages both go to the end.
1480
+ */
1481
+ private async placeRunTerminals(
1482
+ messages: readonly IFlexMessage[],
1483
+ terminals: ReadonlyMap<string, IFlexTerminalProjection>,
1484
+ runMessages: ReadonlyMap<string, readonly IFlexMessage[]>,
1485
+ lastRunMessage: ReadonlyMap<string, IFlexMessage>,
1486
+ inModelContext: (runId: string) => boolean,
1487
+ slicer: FlexTimeSlicer,
1488
+ ): Promise<IFlexMessage[]> {
1489
+ interface IPlacement {
1490
+ terminal: IFlexTerminalProjection;
1491
+ replacements: Map<string, IFlexMessage>;
1492
+ keptInContext: boolean;
1493
+ placeUser: boolean;
1494
+ placeAssistant: boolean;
1495
+ }
1496
+ const placements = new Map<string, IPlacement>();
1497
+ for (const [runId, terminal] of terminals) {
1498
+ const replacements = new Map([
1499
+ ...(terminal.userMessage ? [[terminal.userMessage.messageId, terminal.userMessage] as const] : []),
1500
+ [terminal.assistantMessage.messageId, terminal.assistantMessage],
1501
+ ]);
1502
+ const projectedIds = new Set((runMessages.get(runId) ?? []).map((message) => message.messageId));
1503
+ placements.set(runId, {
1504
+ terminal,
1505
+ replacements,
1506
+ keptInContext: terminal.userMessage?.keptInContext
1507
+ ?? (terminal.status === 'completed' || inModelContext(runId)),
1508
+ placeUser: terminal.userMessage !== undefined && !projectedIds.has(terminal.userMessage.messageId),
1509
+ placeAssistant: !projectedIds.has(terminal.assistantMessage.messageId),
1510
+ });
1511
+ }
1512
+ const placed: IFlexMessage[] = [];
1513
+ const startedRuns = new Set<string>();
1514
+ for (const message of messages) {
1515
+ const placement = placements.get(message.runId);
1516
+ if (!placement) {
1517
+ placed.push(message);
1518
+ } else {
1519
+ if (!startedRuns.has(message.runId)) {
1520
+ startedRuns.add(message.runId);
1521
+ if (placement.placeUser) placed.push(cloneSerializable(placement.terminal.userMessage!));
1522
+ }
1523
+ const replacement = placement.replacements.get(message.messageId);
1524
+ if (replacement) {
1525
+ placed.push(cloneSerializable(replacement));
1526
+ } else {
1527
+ // The steers the run took in share its prompt's outcome.
1528
+ placed.push(message.role === 'user' && message.keptInContext !== placement.keptInContext
1529
+ ? { ...cloneSerializable(message), keptInContext: placement.keptInContext }
1530
+ : message);
1531
+ }
1532
+ if (placement.placeAssistant && lastRunMessage.get(message.runId) === message) {
1533
+ placed.push(cloneSerializable(placement.terminal.assistantMessage));
1534
+ }
1535
+ }
1536
+ if (slicer.due) await slicer.pause();
1537
+ }
1538
+ for (const [runId, placement] of placements) {
1539
+ if (startedRuns.has(runId)) continue;
1540
+ if (placement.placeUser) placed.push(cloneSerializable(placement.terminal.userMessage!));
1541
+ if (placement.placeAssistant) placed.push(cloneSerializable(placement.terminal.assistantMessage));
1542
+ }
1543
+ return placed;
1544
+ }
1545
+
1546
+ /**
1547
+ * The projection content a repair may change, as it is before the repair. Messages and reversion
1548
+ * segments are frozen and replaced, never edited, so the lists keep their entries; the other
1549
+ * fields are small and kept encoded.
1550
+ */
1551
+ private projectionContent(stored: IStoredSessionState): IProjectionContent {
1552
+ return {
1553
+ messages: stored.messages.slice(),
1554
+ reversionSegments: stored.reversionSegments.slice(),
1555
+ stagedTerminals: JSON.stringify(stored.stagedTerminals),
1556
+ revertCursor: stored.revertCursor,
1557
+ excludedRunIds: JSON.stringify(stored.excludedRunIds),
1558
+ pendingReversion: JSON.stringify(stored.pendingReversion),
1559
+ pendingReversionReleases: JSON.stringify(stored.pendingReversionReleases),
1560
+ };
1561
+ }
1562
+
1563
+ /**
1564
+ * Whether the session's projection content differs from `before` as JSON. A list entry that is
1565
+ * still the same object is unchanged, so only an entry replaced since is encoded, one at a time.
1566
+ */
1567
+ private async projectionContentChanged(
1568
+ before: IProjectionContent,
1569
+ stored: IStoredSessionState,
1570
+ slicer: FlexTimeSlicer,
1571
+ ): Promise<boolean> {
1572
+ if (
1573
+ stored.revertCursor !== before.revertCursor
1574
+ || JSON.stringify(stored.stagedTerminals) !== before.stagedTerminals
1575
+ || JSON.stringify(stored.excludedRunIds) !== before.excludedRunIds
1576
+ || JSON.stringify(stored.pendingReversion) !== before.pendingReversion
1577
+ || JSON.stringify(stored.pendingReversionReleases) !== before.pendingReversionReleases
1578
+ ) return true;
1579
+ const listChanged = async (previous: readonly object[], current: readonly object[]) => {
1580
+ if (previous.length !== current.length) return true;
1581
+ for (let index = 0; index < current.length; index++) {
1582
+ if (
1583
+ previous[index] !== current[index]
1584
+ && JSON.stringify(previous[index]) !== JSON.stringify(current[index])
1585
+ ) return true;
1586
+ if (slicer.due) await slicer.pause();
1587
+ }
1588
+ return false;
1589
+ };
1590
+ return await listChanged(before.messages, stored.messages)
1591
+ || await listChanged(before.reversionSegments, stored.reversionSegments);
1592
+ }
1593
+
1115
1594
  /**
1116
1595
  * The failed or cancelled terminal a run published, read from its projected messages. A run
1117
1596
  * still running has its prompt or its last assistant message streaming; it has none.
@@ -1273,6 +1752,7 @@ export class FlexHarnessCore<TScope> {
1273
1752
  onlySessionId?: string,
1274
1753
  ): Promise<TValue> {
1275
1754
  const beforeRevision = state.revision;
1755
+ const beforeCatalogue = new Map([...state.catalogue].map(([id, metadata]) => [id, cloneSerializable(metadata)]));
1276
1756
  const beforeSessions = new Map([...state.sessions].map(([id, stored]) => [id, {
1277
1757
  stored,
1278
1758
  metadata: cloneSerializable(stored.session),
@@ -1284,18 +1764,20 @@ export class FlexHarnessCore<TScope> {
1284
1764
  const beforeSnapshot = (): IFlexScopeSnapshot => ({
1285
1765
  schemaVersion: 1,
1286
1766
  revision: beforeRevision,
1287
- sessions: [...beforeSessions.values()]
1288
- .map((entry) => entry.metadata)
1767
+ sessions: [...beforeCatalogue.values()]
1289
1768
  .sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
1290
1769
  tombstones: [...beforeTombstones.values()]
1291
1770
  .sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
1292
1771
  });
1293
1772
  const restore = () => {
1294
1773
  state.revision = beforeRevision;
1774
+ state.catalogue.clear();
1775
+ for (const [id, metadata] of beforeCatalogue) state.catalogue.set(id, metadata);
1295
1776
  state.sessions.clear();
1296
1777
  for (const [id, entry] of beforeSessions) {
1297
1778
  entry.stored.session = entry.metadata;
1298
1779
  state.sessions.set(id, entry.stored);
1780
+ state.catalogue.set(id, entry.metadata);
1299
1781
  }
1300
1782
  state.tombstones.clear();
1301
1783
  for (const [id, value] of beforeTombstones) state.tombstones.set(id, value);
@@ -1307,8 +1789,12 @@ export class FlexHarnessCore<TScope> {
1307
1789
  let result: TValue;
1308
1790
  try {
1309
1791
  result = mutation();
1792
+ for (const id of beforeSessions.keys()) {
1793
+ if (!state.sessions.has(id)) state.catalogue.delete(id);
1794
+ }
1795
+ for (const [id, stored] of state.sessions) state.catalogue.set(id, stored.session);
1310
1796
  if (onlySessionId !== undefined) {
1311
- this.assertOnlySessionChanged(state, onlySessionId, beforeSessions, beforeTombstones, beforeRetainedCleanups);
1797
+ this.assertOnlySessionChanged(state, onlySessionId, beforeSessions, beforeTombstones, beforeRetainedCleanups, beforeCatalogue);
1312
1798
  }
1313
1799
  } catch (error) {
1314
1800
  restore();
@@ -1316,7 +1802,7 @@ export class FlexHarnessCore<TScope> {
1316
1802
  }
1317
1803
  const snapshot = this.createScopeSnapshot(state, beforeRevision + 1);
1318
1804
  try {
1319
- await this.stores.scopes.save(state.storageKey, snapshot, beforeRevision);
1805
+ await this.saveScopeSnapshot(state, snapshot, beforeRevision);
1320
1806
  state.revision = snapshot.revision;
1321
1807
  return result;
1322
1808
  } catch (error) {
@@ -1362,9 +1848,11 @@ export class FlexHarnessCore<TScope> {
1362
1848
  beforeSessions: ReadonlyMap<string, { stored: IStoredSessionState; metadata: IFlexSession }>,
1363
1849
  beforeTombstones: ReadonlyMap<string, IFlexSessionTombstone>,
1364
1850
  beforeRetainedCleanups: ReadonlyMap<string, unknown>,
1851
+ beforeCatalogue: ReadonlyMap<string, IFlexSession>,
1365
1852
  ): void {
1366
1853
  const changed = (id: string) => {
1367
1854
  if (id === sessionId) return false;
1855
+ if (canonicalJson(beforeCatalogue.get(id)) !== canonicalJson(state.catalogue.get(id))) return true;
1368
1856
  const before = beforeSessions.get(id);
1369
1857
  const stored = state.sessions.get(id);
1370
1858
  if (before?.stored !== stored) return true;
@@ -1373,6 +1861,7 @@ export class FlexHarnessCore<TScope> {
1373
1861
  return beforeRetainedCleanups.get(id) !== state.retainedSessionCleanups.get(id);
1374
1862
  };
1375
1863
  const ids = new Set([
1864
+ ...beforeCatalogue.keys(), ...state.catalogue.keys(),
1376
1865
  ...beforeSessions.keys(), ...state.sessions.keys(),
1377
1866
  ...beforeTombstones.keys(), ...state.tombstones.keys(),
1378
1867
  ...beforeRetainedCleanups.keys(), ...state.retainedSessionCleanups.keys(),
@@ -1386,141 +1875,185 @@ export class FlexHarnessCore<TScope> {
1386
1875
  }
1387
1876
  }
1388
1877
 
1878
+ /**
1879
+ * Changes `stored`'s projection in its FIFO and saves it. `onSaved` gets the checkpoint clock's
1880
+ * times the save started and completed, right after it completed.
1881
+ */
1389
1882
  public mutateProjection<TValue>(
1390
1883
  state: IStorageState,
1391
1884
  stored: IStoredSessionState,
1392
1885
  mutation: () => TValue,
1393
1886
  _allowDuringDrain = false,
1887
+ onSaved?: TProjectionSaved,
1394
1888
  ): Promise<TValue> {
1395
- const operation = stored.projectionQueue.then(async () => {
1396
- const beforeRevision = stored.projectionRevision;
1397
- const beforeSchemaVersion = stored.projectionSchemaVersion;
1398
- const beforeReconciliationRequired = stored.projectionReconciliationRequired;
1399
- const beforeBaseline = stored.projectionBaseline === undefined
1400
- ? undefined
1401
- : cloneSerializable(stored.projectionBaseline);
1402
- const beforeMetrics = cloneSerializable(stored.metrics);
1403
- // List entries are frozen and replaced rather than edited, so copies of the lists suffice.
1404
- const beforeMessages = stored.messages.slice();
1405
- const beforeStages = cloneSerializable(stored.stagedTerminals);
1406
- const beforeSegments = stored.reversionSegments.slice();
1407
- const beforeCursor = stored.revertCursor;
1408
- const beforeExcludedRunIds = cloneSerializable(stored.excludedRunIds);
1409
- const beforePendingReversion = stored.pendingReversion === undefined
1410
- ? undefined
1411
- : cloneSerializable(stored.pendingReversion);
1412
- const beforePendingReversionReleases = cloneSerializable(stored.pendingReversionReleases);
1413
- // The copies above stay untouched until a restore, so the projection as it was before the
1414
- // mutation is built from them only when a failed save has to be reconciled.
1415
- const beforeSnapshot = (): TFlexProjectionSnapshot => beforeBaseline ?? {
1416
- schemaVersion: 3,
1417
- revision: beforeRevision,
1418
- metrics: beforeMetrics,
1419
- messages: beforeMessages,
1420
- stagedTerminals: beforeStages,
1421
- reversionSegments: beforeSegments,
1422
- revertCursor: beforeCursor,
1423
- excludedRunIds: beforeExcludedRunIds,
1424
- ...(beforePendingReversion === undefined ? {} : { pendingReversion: beforePendingReversion }),
1425
- pendingReversionReleases: beforePendingReversionReleases,
1426
- };
1427
- const restore = () => {
1428
- stored.projectionSchemaVersion = beforeSchemaVersion;
1429
- stored.projectionReconciliationRequired = beforeReconciliationRequired;
1430
- if (beforeBaseline === undefined) delete stored.projectionBaseline;
1431
- else stored.projectionBaseline = beforeBaseline;
1432
- stored.projectionRevision = beforeRevision;
1433
- stored.metrics = beforeMetrics;
1434
- stored.messages = beforeMessages;
1435
- stored.stagedTerminals = beforeStages;
1436
- stored.reversionSegments = beforeSegments;
1437
- stored.revertCursor = beforeCursor;
1438
- stored.excludedRunIds = beforeExcludedRunIds;
1439
- if (beforePendingReversion === undefined) delete stored.pendingReversion;
1440
- else stored.pendingReversion = beforePendingReversion;
1441
- stored.pendingReversionReleases = beforePendingReversionReleases;
1442
- };
1443
- let result: TValue;
1444
- try {
1445
- result = mutation();
1446
- } catch (error) {
1889
+ return this.enqueueProjectionOperation(stored, () => this.saveProjectionMutation(state, stored, mutation, onSaved));
1890
+ }
1891
+
1892
+ /**
1893
+ * A running turn's projection checkpoint, at its turn in the projection FIFO: `prepare` decides
1894
+ * there what to save and returns the mutation, or nothing, which saves nothing. Resolves whether
1895
+ * it saved.
1896
+ */
1897
+ public checkpointProjection(
1898
+ state: IStorageState,
1899
+ stored: IStoredSessionState,
1900
+ prepare: () => (() => void) | undefined,
1901
+ onSaved: TProjectionSaved,
1902
+ ): Promise<boolean> {
1903
+ return this.enqueueProjectionOperation(stored, async () => {
1904
+ const mutation = prepare();
1905
+ if (!mutation) return false;
1906
+ await this.saveProjectionMutation(state, stored, mutation, onSaved);
1907
+ return true;
1908
+ });
1909
+ }
1910
+
1911
+ private enqueueProjectionOperation<TValue>(
1912
+ stored: IStoredSessionState,
1913
+ operation: () => Promise<TValue>,
1914
+ ): Promise<TValue> {
1915
+ const queued = stored.projectionQueue.then(operation);
1916
+ stored.projectionQueue = queued.then(() => undefined, () => undefined);
1917
+ return queued;
1918
+ }
1919
+
1920
+ private async saveProjectionMutation<TValue>(
1921
+ state: IStorageState,
1922
+ stored: IStoredSessionState,
1923
+ mutation: () => TValue,
1924
+ onSaved: TProjectionSaved | undefined,
1925
+ ): Promise<TValue> {
1926
+ const beforeRevision = stored.projectionRevision;
1927
+ const beforeSchemaVersion = stored.projectionSchemaVersion;
1928
+ const beforeReconciliationRequired = stored.projectionReconciliationRequired;
1929
+ // The baseline is deep-frozen and only ever replaced, never edited, so it needs no copy.
1930
+ const beforeBaseline = stored.projectionBaseline;
1931
+ const beforeMetrics = cloneSerializable(stored.metrics);
1932
+ // List entries are frozen and replaced rather than edited, so copies of the lists suffice.
1933
+ const beforeMessages = stored.messages.slice();
1934
+ const beforeStages = cloneSerializable(stored.stagedTerminals);
1935
+ const beforeSegments = stored.reversionSegments.slice();
1936
+ const beforeCursor = stored.revertCursor;
1937
+ const beforeExcludedRunIds = cloneSerializable(stored.excludedRunIds);
1938
+ const beforePendingReversion = stored.pendingReversion === undefined
1939
+ ? undefined
1940
+ : cloneSerializable(stored.pendingReversion);
1941
+ const beforePendingReversionReleases = cloneSerializable(stored.pendingReversionReleases);
1942
+ // The copies above stay untouched until a restore, so the projection as it was before the
1943
+ // mutation is built from them only when a failed save has to be reconciled.
1944
+ const beforeSnapshot = (): TFlexProjectionSnapshot => beforeBaseline ?? {
1945
+ schemaVersion: 3,
1946
+ revision: beforeRevision,
1947
+ metrics: beforeMetrics,
1948
+ messages: beforeMessages,
1949
+ stagedTerminals: beforeStages,
1950
+ reversionSegments: beforeSegments,
1951
+ revertCursor: beforeCursor,
1952
+ excludedRunIds: beforeExcludedRunIds,
1953
+ ...(beforePendingReversion === undefined ? {} : { pendingReversion: beforePendingReversion }),
1954
+ pendingReversionReleases: beforePendingReversionReleases,
1955
+ };
1956
+ const restore = () => {
1957
+ stored.projectionSchemaVersion = beforeSchemaVersion;
1958
+ stored.projectionReconciliationRequired = beforeReconciliationRequired;
1959
+ if (beforeBaseline === undefined) delete stored.projectionBaseline;
1960
+ else stored.projectionBaseline = beforeBaseline;
1961
+ stored.projectionRevision = beforeRevision;
1962
+ stored.metrics = beforeMetrics;
1963
+ stored.messages = beforeMessages;
1964
+ stored.stagedTerminals = beforeStages;
1965
+ stored.reversionSegments = beforeSegments;
1966
+ stored.revertCursor = beforeCursor;
1967
+ stored.excludedRunIds = beforeExcludedRunIds;
1968
+ if (beforePendingReversion === undefined) delete stored.pendingReversion;
1969
+ else stored.pendingReversion = beforePendingReversion;
1970
+ stored.pendingReversionReleases = beforePendingReversionReleases;
1971
+ };
1972
+ let result: TValue;
1973
+ try {
1974
+ result = mutation();
1975
+ } catch (error) {
1976
+ restore();
1977
+ throw error;
1978
+ }
1979
+ freezeProjectionEntries(stored);
1980
+ const revision = beforeRevision + 1;
1981
+ try {
1982
+ if (state.sessionLocal) await this.updateRecoveryObligation(state, stored, true);
1983
+ const saveStartedAt = onSaved ? this.checkpointClock.now() : 0;
1984
+ await this.stores.projections.save(
1985
+ state.storageKey,
1986
+ stored.session.sessionId,
1987
+ this.createProjectionChange(stored, revision),
1988
+ beforeRevision,
1989
+ );
1990
+ const saveCompletedAt = onSaved ? this.checkpointClock.now() : 0;
1991
+ stored.projectionSchemaVersion = 3;
1992
+ delete stored.projectionBaseline;
1993
+ this.rememberSavedProjectionLists(stored);
1994
+ stored.projectionReconciliationRequired = false;
1995
+ stored.projectionRevision = revision;
1996
+ onSaved?.(saveStartedAt, saveCompletedAt);
1997
+ } catch (error) {
1998
+ if (error instanceof FlexHarnessStoreConflictError) {
1447
1999
  restore();
1448
2000
  throw error;
1449
2001
  }
1450
- freezeProjectionEntries(stored);
1451
- const revision = beforeRevision + 1;
2002
+ if (error instanceof FlexHarnessStoreCommitUncertainError) {
2003
+ stored.projectionSchemaVersion = 3;
2004
+ delete stored.savedProjectionLists;
2005
+ stored.projectionReconciliationRequired = true;
2006
+ stored.projectionRevision = revision;
2007
+ state.lifecycle = 'fenced';
2008
+ throw error;
2009
+ }
2010
+ const snapshot = this.createProjectionSnapshot(stored, revision);
2011
+ let current: TFlexProjectionSnapshot | undefined;
2012
+ let reconciliationError: unknown;
1452
2013
  try {
1453
- await this.stores.projections.save(
2014
+ current = await this.stores.projections.load(
1454
2015
  state.storageKey,
1455
2016
  stored.session.sessionId,
1456
- this.createProjectionChange(stored, revision),
1457
- beforeRevision,
1458
2017
  );
2018
+ } catch (loadError) {
2019
+ reconciliationError = loadError;
2020
+ }
2021
+ if (current && JSON.stringify(current) === JSON.stringify(snapshot)) {
1459
2022
  stored.projectionSchemaVersion = 3;
1460
2023
  delete stored.projectionBaseline;
1461
2024
  this.rememberSavedProjectionLists(stored);
1462
2025
  stored.projectionReconciliationRequired = false;
1463
2026
  stored.projectionRevision = revision;
1464
- return result;
1465
- } catch (error) {
1466
- if (error instanceof FlexHarnessStoreConflictError) {
1467
- restore();
1468
- throw error;
1469
- }
1470
- if (error instanceof FlexHarnessStoreCommitUncertainError) {
1471
- stored.projectionSchemaVersion = 3;
1472
- delete stored.savedProjectionLists;
1473
- stored.projectionReconciliationRequired = true;
1474
- stored.projectionRevision = revision;
1475
- state.lifecycle = 'fenced';
1476
- throw error;
1477
- }
1478
- const snapshot = this.createProjectionSnapshot(stored, revision);
1479
- let current: TFlexProjectionSnapshot | undefined;
1480
- let reconciliationError: unknown;
1481
- try {
1482
- current = await this.stores.projections.load(
1483
- state.storageKey,
1484
- stored.session.sessionId,
1485
- );
1486
- } catch (loadError) {
1487
- reconciliationError = loadError;
1488
- }
1489
- if (current && JSON.stringify(current) === JSON.stringify(snapshot)) {
1490
- stored.projectionSchemaVersion = 3;
1491
- delete stored.projectionBaseline;
1492
- this.rememberSavedProjectionLists(stored);
1493
- stored.projectionReconciliationRequired = false;
1494
- stored.projectionRevision = revision;
1495
- throw error;
1496
- }
1497
- if (
1498
- (current === undefined && beforeRevision === 0)
1499
- || (current !== undefined && JSON.stringify(current) === JSON.stringify(beforeSnapshot()))
1500
- ) {
1501
- restore();
1502
- throw error;
1503
- }
1504
- stored.projectionSchemaVersion = 3;
1505
- delete stored.savedProjectionLists;
1506
- stored.projectionReconciliationRequired = true;
1507
- stored.projectionRevision = revision;
1508
- state.lifecycle = 'fenced';
1509
- throw reconciliationError === undefined
1510
- ? error
1511
- : combineErrors([error, reconciliationError]);
2027
+ throw error;
1512
2028
  }
1513
- });
1514
- stored.projectionQueue = operation.then(() => undefined, () => undefined);
1515
- return operation;
2029
+ if (
2030
+ (current === undefined && beforeRevision === 0)
2031
+ || (current !== undefined && JSON.stringify(current) === JSON.stringify(beforeSnapshot()))
2032
+ ) {
2033
+ restore();
2034
+ throw error;
2035
+ }
2036
+ stored.projectionSchemaVersion = 3;
2037
+ delete stored.savedProjectionLists;
2038
+ stored.projectionReconciliationRequired = true;
2039
+ stored.projectionRevision = revision;
2040
+ state.lifecycle = 'fenced';
2041
+ throw reconciliationError === undefined
2042
+ ? error
2043
+ : combineErrors([error, reconciliationError]);
2044
+ }
2045
+ // The projection is saved: what follows is a separate durable step, never classified as the
2046
+ // save's failure.
2047
+ if (state.sessionLocal) await this.settleRecoveryObligation(state, stored);
2048
+ return result;
1516
2049
  }
1517
2050
 
1518
2051
  private createScopeSnapshot(state: IStorageState, revision: number): IFlexScopeSnapshot {
1519
2052
  return {
1520
2053
  schemaVersion: 1,
1521
2054
  revision,
1522
- sessions: [...state.sessions.values()]
1523
- .map((stored) => cloneSerializable(stored.session))
2055
+ sessions: [...state.catalogue.values()]
2056
+ .map((metadata) => cloneSerializable(state.sessions.get(metadata.sessionId)?.session ?? metadata))
1524
2057
  .sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
1525
2058
  tombstones: [...state.tombstones.values()]
1526
2059
  .map((value) => cloneSerializable(value))
@@ -1645,7 +2178,7 @@ export class FlexHarnessCore<TScope> {
1645
2178
  delete stored.projectionBaseline;
1646
2179
  this.rememberSavedProjectionLists(stored);
1647
2180
  } else {
1648
- stored.projectionBaseline = cloneSerializable(projection);
2181
+ stored.projectionBaseline = deepFreeze(cloneSerializable(projection));
1649
2182
  delete stored.savedProjectionLists;
1650
2183
  }
1651
2184
  stored.projectionSchemaVersion = projection.schemaVersion;
@@ -1701,6 +2234,7 @@ export class FlexHarnessCore<TScope> {
1701
2234
 
1702
2235
  private sessionMetadata(state: IStorageState, sessionId: string): IFlexSession | undefined {
1703
2236
  return state.sessions.get(sessionId)?.session
2237
+ ?? state.catalogue.get(sessionId)
1704
2238
  ?? state.retainedSessionCleanups.get(sessionId)?.stored.session;
1705
2239
  }
1706
2240
 
@@ -1712,29 +2246,15 @@ export class FlexHarnessCore<TScope> {
1712
2246
 
1713
2247
  public async resolveState(
1714
2248
  scopeId: string,
2249
+ sessionId?: string,
1715
2250
  ): Promise<{ scope: IFlexResolvedScope<TScope>; state: IStorageState }> {
1716
- this.assertOpen();
1717
- validateIdentifier(scopeId, 'scopeId');
1718
- const admission = this.getScopeAdmission(scopeId);
1719
- if (admission.retiring) throw this.createScopeRetirementError();
1720
- const generation = admission.generation;
1721
- admission.inFlightResolvers++;
1722
- try {
1723
- const scope = await this.scopeResolver.resolveScope(scopeId);
1724
- this.assertOpen();
1725
- if (admission.retiring || admission.generation !== generation) throw this.createScopeRetirementError();
1726
- validateIdentifier(scope.storageKey, 'resolved storageKey');
1727
- if (this.lifecycle.storageDrains.has(scope.storageKey)) throw this.createScopeRetirementError();
2251
+ return this.withScopeAdmission(scopeId, async (scope, admitted) => {
1728
2252
  let stateLoad = this.stateLoads.get(scope.storageKey);
1729
2253
  if (!stateLoad) {
1730
2254
  const orphanErrors = await this.lifecycle.closeOrphanedResources(scope.storageKey);
1731
2255
  if (orphanErrors.length > 0) throw combineErrors(orphanErrors);
1732
2256
  this.assertOpen();
1733
- if (
1734
- admission.retiring
1735
- || admission.generation !== generation
1736
- || this.lifecycle.storageDrains.has(scope.storageKey)
1737
- ) throw this.createScopeRetirementError();
2257
+ if (!admitted()) throw this.createScopeRetirementError();
1738
2258
  stateLoad = this.stateLoads.get(scope.storageKey);
1739
2259
  if (!stateLoad) {
1740
2260
  const compactorLifecycleController = new AbortController();
@@ -1742,15 +2262,18 @@ export class FlexHarnessCore<TScope> {
1742
2262
  scope.storageKey,
1743
2263
  compactorLifecycleController,
1744
2264
  );
1745
- stateLoad = this.loadState(
2265
+ const load = this.loadState(
1746
2266
  scope.storageKey,
1747
2267
  scopeId,
1748
2268
  scope.scope,
1749
2269
  compactorLifecycleController,
1750
2270
  );
1751
- this.stateLoads.set(scope.storageKey, stateLoad);
1752
- void stateLoad.catch(() => {
1753
- if (this.stateLoads.get(scope.storageKey) === stateLoad) {
2271
+ stateLoad = load;
2272
+ this.stateLoads.set(scope.storageKey, load);
2273
+ void load.then((state) => {
2274
+ this.loadedStates.set(load, state);
2275
+ }, () => {
2276
+ if (this.stateLoads.get(scope.storageKey) === load) {
1754
2277
  this.stateLoads.delete(scope.storageKey);
1755
2278
  if (
1756
2279
  this.lifecycle.storageCompactorLifecycleControllers.get(scope.storageKey)
@@ -1760,38 +2283,83 @@ export class FlexHarnessCore<TScope> {
1760
2283
  });
1761
2284
  }
1762
2285
  }
1763
- const state = await stateLoad;
1764
- this.assertOpen();
1765
- if (state.lifecycle === 'fenced') {
1766
- const drain = this.lifecycle.storageDrains.get(scope.storageKey) ?? this.lifecycle.drainStorage(
1767
- scope.storageKey,
1768
- stateLoad,
1769
- this.trustInternalError(new FlexHarnessAbortError('The session namespace was fenced.')),
1770
- { scopeId, scope: scope.scope },
1771
- );
1772
- if (this.commands.ambientSlashCommandOwner({ storageKey: scope.storageKey })) {
1773
- throw this.createScopeRetirementError();
1774
- }
1775
- try {
1776
- await drain;
1777
- } catch {
1778
- // Fenced cleanup ownership remains available to explicit retirement or disposal.
1779
- }
1780
- throw this.createScopeRetirementError();
2286
+ const state = await this.admittedState(scopeId, scope, stateLoad, admitted);
2287
+ if (state.sessionLocal) await this.recoverSharedState(state, scopeId, scope.scope);
2288
+ if (sessionId !== undefined && state.catalogue.has(sessionId)) {
2289
+ await this.hydrateSession(state, sessionId, scopeId, scope.scope);
2290
+ this.assertOpen();
2291
+ if (!admitted()) throw this.createScopeRetirementError();
2292
+ this.assertStateAcceptingWork(state);
1781
2293
  }
1782
- if (
1783
- admission.retiring
1784
- || admission.generation !== generation
1785
- || this.lifecycle.storageDrains.has(scope.storageKey)
1786
- || state.lifecycle !== 'active'
1787
- ) throw this.createScopeRetirementError();
1788
2294
  return { scope, state };
2295
+ });
2296
+ }
2297
+
2298
+ /** Whether `stateLoad` has settled with its loaded state. */
2299
+ public isStateLoaded(stateLoad: Promise<IStorageState>): boolean {
2300
+ return this.loadedStates.has(stateLoad);
2301
+ }
2302
+
2303
+ /**
2304
+ * Runs `operation` for the resolved scope `scopeId` while the scope is admitted: not retiring,
2305
+ * not retired since, not draining. `admitted()` tells whether that still holds.
2306
+ */
2307
+ public async withScopeAdmission<TValue>(
2308
+ scopeId: string,
2309
+ operation: (scope: IFlexResolvedScope<TScope>, admitted: () => boolean) => Promise<TValue>,
2310
+ ): Promise<TValue> {
2311
+ this.assertOpen();
2312
+ validateIdentifier(scopeId, 'scopeId');
2313
+ const admission = this.getScopeAdmission(scopeId);
2314
+ if (admission.retiring) throw this.createScopeRetirementError();
2315
+ const generation = admission.generation;
2316
+ admission.inFlightResolvers++;
2317
+ try {
2318
+ const scope = await this.scopeResolver.resolveScope(scopeId);
2319
+ this.assertOpen();
2320
+ if (admission.retiring || admission.generation !== generation) throw this.createScopeRetirementError();
2321
+ validateIdentifier(scope.storageKey, 'resolved storageKey');
2322
+ if (this.lifecycle.storageDrains.has(scope.storageKey)) throw this.createScopeRetirementError();
2323
+ return await operation(scope, () =>
2324
+ !admission.retiring
2325
+ && admission.generation === generation
2326
+ && !this.lifecycle.storageDrains.has(scope.storageKey));
1789
2327
  } finally {
1790
2328
  admission.inFlightResolvers--;
1791
2329
  this.pruneScopeAdmission(scopeId, admission);
1792
2330
  }
1793
2331
  }
1794
2332
 
2333
+ /** The state `stateLoad` loads, once it is active and the scope still admitted. */
2334
+ public async admittedState(
2335
+ scopeId: string,
2336
+ scope: IFlexResolvedScope<TScope>,
2337
+ stateLoad: Promise<IStorageState>,
2338
+ admitted: () => boolean,
2339
+ ): Promise<IStorageState> {
2340
+ const state = await stateLoad;
2341
+ this.assertOpen();
2342
+ if (state.lifecycle === 'fenced') {
2343
+ const drain = this.lifecycle.storageDrains.get(scope.storageKey) ?? this.lifecycle.drainStorage(
2344
+ scope.storageKey,
2345
+ stateLoad,
2346
+ this.trustInternalError(new FlexHarnessAbortError('The session namespace was fenced.')),
2347
+ { scopeId, scope: scope.scope },
2348
+ );
2349
+ if (this.commands.ambientSlashCommandOwner({ storageKey: scope.storageKey })) {
2350
+ throw this.createScopeRetirementError();
2351
+ }
2352
+ try {
2353
+ await drain;
2354
+ } catch {
2355
+ // Fenced cleanup ownership remains available to explicit retirement or disposal.
2356
+ }
2357
+ throw this.createScopeRetirementError();
2358
+ }
2359
+ if (!admitted() || state.lifecycle !== 'active') throw this.createScopeRetirementError();
2360
+ return state;
2361
+ }
2362
+
1795
2363
  public getScopeAdmission(scopeId: string): IScopeAdmissionState {
1796
2364
  let admission = this.scopeAdmissions.get(scopeId);
1797
2365
  if (!admission) {
@@ -1956,16 +2524,22 @@ export class FlexHarnessCore<TScope> {
1956
2524
  * one id each; a position is searched again only after it moved (a steer, a restored or
1957
2525
  * reloaded projection).
1958
2526
  */
1959
- private partEventCoordinates(
1960
- run: IActiveRun,
1961
- partId: string,
1962
- ): { messageIndex: number; partIndex: number } {
2527
+ /** The run's remembered positions, its assistant message's checked; -1 when it is not projected. */
2528
+ public assistantMessagePositions(run: IActiveRun): NonNullable<IActiveRun['partEventPositions']> {
1963
2529
  const positions = run.partEventPositions ??= { messageIndex: -1, partIndexes: new Map() };
1964
2530
  if (run.stored.messages[positions.messageIndex]?.messageId !== run.assistantMessageId) {
1965
2531
  positions.messageIndex = run.stored.messages.findIndex(
1966
2532
  (message) => message.messageId === run.assistantMessageId,
1967
2533
  );
1968
2534
  }
2535
+ return positions;
2536
+ }
2537
+
2538
+ private partEventCoordinates(
2539
+ run: IActiveRun,
2540
+ partId: string,
2541
+ ): { messageIndex: number; partIndex: number } {
2542
+ const positions = this.assistantMessagePositions(run);
1969
2543
  let partIndex = positions.partIndexes.get(partId) ?? -1;
1970
2544
  if (run.callbackParts[partIndex]?.partId !== partId) {
1971
2545
  partIndex = run.callbackParts.findIndex((entry) => entry.partId === partId);