@modelprofile.com/flexharness 18.8.0 → 18.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist_ts/00_commitinfo_data.js +1 -1
  2. package/dist_ts/classes.flexharness.core.d.ts +19 -0
  3. package/dist_ts/classes.flexharness.core.js +358 -67
  4. package/dist_ts/classes.flexharness.deletion.d.ts +1 -1
  5. package/dist_ts/classes.flexharness.deletion.js +147 -141
  6. package/dist_ts/classes.flexharness.imports.js +3 -3
  7. package/dist_ts/classes.flexharness.lifecycle.js +4 -1
  8. package/dist_ts/classes.flexharness.permissions.js +5 -5
  9. package/dist_ts/classes.flexharness.prompts.js +5 -6
  10. package/dist_ts/classes.flexharness.results.js +3 -3
  11. package/dist_ts/classes.flexharness.reversions.js +320 -316
  12. package/dist_ts/classes.flexharness.sessions.js +19 -3
  13. package/dist_ts/classes.flexharness.subagents.js +2 -2
  14. package/dist_ts/classes.recoverygate.d.ts +30 -0
  15. package/dist_ts/classes.recoverygate.js +105 -0
  16. package/dist_ts/classes.scoperecovery.d.ts +9 -1
  17. package/dist_ts/classes.scoperecovery.js +94 -26
  18. package/dist_ts/interfaces.d.ts +22 -0
  19. package/dist_ts/interfaces.flexharness.d.ts +5 -0
  20. package/dist_ts/interfaces.js +1 -1
  21. package/dist_ts/plugins.d.ts +1 -1
  22. package/dist_ts/plugins.js +1 -1
  23. package/dist_ts_stores_nosqldb/classes.nosqldbstores.js +5 -1
  24. package/package.json +3 -3
  25. package/readme.md +5 -3
  26. package/ts/00_commitinfo_data.ts +1 -1
  27. package/ts/classes.flexharness.core.ts +337 -34
  28. package/ts/classes.flexharness.deletion.ts +24 -15
  29. package/ts/classes.flexharness.imports.ts +3 -2
  30. package/ts/classes.flexharness.lifecycle.ts +3 -0
  31. package/ts/classes.flexharness.permissions.ts +8 -4
  32. package/ts/classes.flexharness.prompts.ts +10 -8
  33. package/ts/classes.flexharness.results.ts +3 -2
  34. package/ts/classes.flexharness.reversions.ts +13 -6
  35. package/ts/classes.flexharness.sessions.ts +18 -2
  36. package/ts/classes.flexharness.subagents.ts +5 -4
  37. package/ts/classes.recoverygate.ts +111 -0
  38. package/ts/classes.scoperecovery.ts +102 -25
  39. package/ts/interfaces.flexharness.ts +5 -0
  40. package/ts/interfaces.ts +23 -0
  41. package/ts/plugins.ts +1 -0
  42. package/ts/readme.md +5 -3
  43. package/ts_stores_nosqldb/classes.nosqldbstores.ts +5 -0
  44. package/ts_stores_nosqldb/readme.md +1 -3
@@ -9,6 +9,7 @@ import {
9
9
  FlexHarnessSessionBusyError,
10
10
  FlexHarnessStoreConflictError,
11
11
  FlexHarnessStoreCommitUncertainError,
12
+ FlexHarnessStoreFormatError,
12
13
  FlexHarnessUnresolvedToolExecutionsError,
13
14
  FlexHarnessValidationError,
14
15
  modelLimitErrorInfo,
@@ -51,6 +52,8 @@ import type {
51
52
  } from './interfaces.js';
52
53
  import { InMemoryFlexHarnessStores } from './classes.stores.js';
53
54
  import { FlexCodeSandbox } from './classes.codesandbox.js';
55
+ import { FlexRecoveryProducerGate } from './classes.recoverygate.js';
56
+ import { flexSessionDigest } from './classes.scoperecovery.js';
54
57
  import {
55
58
  assertJsonSerializable,
56
59
  canonicalJson,
@@ -160,6 +163,9 @@ export class FlexHarnessCore<TScope> {
160
163
  public readonly attachmentReferenceProtocols: ReadonlySet<string>;
161
164
  public readonly stores: IFlexHarnessStores;
162
165
  public readonly sessionLoading: 'eager' | 'session-local';
166
+ public readonly cleanRecoveryConfigured: boolean;
167
+ private readonly cleanRecoveryAllStorageKeys: boolean;
168
+ private readonly cleanRecoveryStorageKeys: ReadonlySet<string>;
163
169
  private readonly legacyQualifications = new Set<string>();
164
170
  public readonly agentSessionPolicy: IFlexAgentSessionPolicy<TScope>;
165
171
  public readonly toolOutputLimits: Required<NonNullable<IFlexHarnessOptions<TScope>['toolOutputLimits']>>;
@@ -232,6 +238,34 @@ export class FlexHarnessCore<TScope> {
232
238
  if (this.sessionLoading === 'session-local' && !this.stores.scopes.sessionLocal) {
233
239
  throw new FlexHarnessValidationError('session-local loading requires scopes.sessionLocal capability.');
234
240
  }
241
+ const frontier = options.recoveryFrontier ?? 'compatibility';
242
+ this.cleanRecoveryConfigured = frontier !== 'compatibility';
243
+ if (frontier === 'compatibility') {
244
+ this.cleanRecoveryAllStorageKeys = false;
245
+ this.cleanRecoveryStorageKeys = new Set();
246
+ } else {
247
+ if (!frontier || typeof frontier !== 'object' || Array.isArray(frontier)
248
+ || frontier.mode !== 'clean-read'
249
+ || Object.keys(frontier).some((key) => key !== 'mode' && key !== 'storageKeys')) {
250
+ throw new FlexHarnessValidationError('recoveryFrontier must be "compatibility" or a clean-read configuration.');
251
+ }
252
+ const local = this.stores.scopes.sessionLocal;
253
+ if (this.sessionLoading !== 'session-local' || !local?.readCleanSessionProof
254
+ || !local.publishSessionClean || !this.stores.projections.loadHead
255
+ || !this.stores.agentEvents.loadHead) {
256
+ throw new FlexHarnessValidationError('clean-read recovery requires session-local loading, schema-2 publication and bounded projection and Agent-event heads.');
257
+ }
258
+ if (frontier.storageKeys !== undefined
259
+ && (!Array.isArray(frontier.storageKeys) || frontier.storageKeys.length === 0)) {
260
+ throw new FlexHarnessValidationError('recoveryFrontier.storageKeys must be a non-empty array.');
261
+ }
262
+ for (const key of frontier.storageKeys ?? []) validateIdentifier(key, 'recoveryFrontier storageKey');
263
+ if (frontier.storageKeys && new Set(frontier.storageKeys).size !== frontier.storageKeys.length) {
264
+ throw new FlexHarnessValidationError('recoveryFrontier.storageKeys must be unique.');
265
+ }
266
+ this.cleanRecoveryAllStorageKeys = frontier.storageKeys === undefined;
267
+ this.cleanRecoveryStorageKeys = new Set(frontier.storageKeys ?? []);
268
+ }
235
269
  this.agentSessionPolicy = normalizeAgentSessionPolicy(options.agentSessionPolicy);
236
270
  this.toolOutputLimits = resolveJsonLimits(options.toolOutputLimits);
237
271
  this.callbackLimits = resolveCallbackLimits(options.callbackLimits);
@@ -459,6 +493,9 @@ export class FlexHarnessCore<TScope> {
459
493
  const local = this.sessionLoading === 'session-local'
460
494
  ? await this.stores.scopes.sessionLocal!.loadCatalogue(storageKey)
461
495
  : undefined;
496
+ if (local && this.cleanRecoveryEnabled(storageKey) && local.recoveryRevision === undefined) {
497
+ throw new FlexHarnessValidationError('clean-read recovery requires an exact sidecar revision.');
498
+ }
462
499
  const qualifyingLegacy = local !== undefined && !local.qualified && this.legacyQualifications.has(storageKey);
463
500
  if (local && !local.qualified && !qualifyingLegacy) {
464
501
  throw new FlexHarnessValidationError(
@@ -479,7 +516,13 @@ export class FlexHarnessCore<TScope> {
479
516
  catalogue: new Map(liveScopeSessions(snapshot).map((session) => [session.sessionId, session])),
480
517
  sessionLocal: local !== undefined,
481
518
  qualifyingLegacy,
482
- recoverySessionIds: new Set(local?.recoverySessionIds ?? []),
519
+ recoverySessionIds: new Set(local && local.recoverySchemaVersion === 2
520
+ && !this.cleanRecoveryEnabled(storageKey)
521
+ ? [...new Set([...local.recoverySessionIds,
522
+ ...(local.snapshot?.sessions.map((session) => session.sessionId) ?? [])])]
523
+ : local?.recoverySessionIds ?? []),
524
+ recoveryMarks: new Map(),
525
+ recoveryProducerGates: new Map(),
483
526
  sessionLoads: new Map(),
484
527
  sessions: new Map(),
485
528
  retainedSessionCleanups: new Map(),
@@ -514,6 +557,10 @@ export class FlexHarnessCore<TScope> {
514
557
  for (const metadata of liveScopeSessions(snapshot).filter((session) =>
515
558
  !state.sessionLocal || qualifyingLegacy || state.recoverySessionIds.has(session.sessionId))) {
516
559
  compactorLifecycleController.signal.throwIfAborted();
560
+ if (local?.recoverySchemaVersion === 2 && !this.cleanRecoveryEnabled(storageKey)) {
561
+ // A compatibility open always invalidates inherited clean proof before full repair.
562
+ await this.markRecoveryRequired(state, metadata.sessionId);
563
+ }
517
564
  const stored = await this.loadSessionRuntime(
518
565
  state,
519
566
  metadata,
@@ -586,6 +633,7 @@ export class FlexHarnessCore<TScope> {
586
633
  await this.stores.scopes.sessionLocal!.setRecoveryRequired(storageKey, tombstone.sessionId, false);
587
634
  }
588
635
  state.recoverySessionIds.delete(tombstone.sessionId);
636
+ state.recoveryMarks.delete(tombstone.sessionId);
589
637
  }
590
638
  for (const tombstone of group) state.tombstones.delete(tombstone.sessionId);
591
639
  scopeChanged = true;
@@ -611,6 +659,7 @@ export class FlexHarnessCore<TScope> {
611
659
  for (const loaded of loadedSessions) {
612
660
  loaded.recovering = false;
613
661
  await this.updateRecoveryObligation(state, loaded);
662
+ await this.settleQuiescentSessionRecovery(state, loaded.session.sessionId);
614
663
  }
615
664
  }
616
665
  return state;
@@ -741,6 +790,7 @@ export class FlexHarnessCore<TScope> {
741
790
  else state.catalogue.set(sessionId, stored.session);
742
791
  stored.recovering = false;
743
792
  await this.updateRecoveryObligation(state, stored);
793
+ await this.settleQuiescentSessionRecovery(state, sessionId);
744
794
  this.assertStateAcceptingWork(state);
745
795
  return stored;
746
796
  } catch (error) {
@@ -781,6 +831,8 @@ export class FlexHarnessCore<TScope> {
781
831
  if (state.sessionLoads.has(id)) { await state.sessionLoads.get(id); continue; }
782
832
  const stored = state.sessions.get(id);
783
833
  if (!stored) { await this.hydrateSession(state, id, scopeId, scope); continue; }
834
+ await this.settleQuiescentSessionRecovery(state, id);
835
+ if (!state.recoverySessionIds.has(id)) continue;
784
836
  stored.recovering = true;
785
837
  await this.repairLoadedSession(state.storageKey, stored, state, scopeId, scope,
786
838
  new FlexTimeSlicer(state.compactorLifecycleController.signal));
@@ -789,6 +841,7 @@ export class FlexHarnessCore<TScope> {
789
841
  await this.mutateSessionScope(state, id, () => undefined);
790
842
  stored.recovering = false;
791
843
  await this.updateRecoveryObligation(state, stored);
844
+ await this.settleQuiescentSessionRecovery(state, id);
792
845
  }
793
846
  for (const root of roots) await this.deletion.finishTombstoneCleanup(state, root, scopeId, { scopeId, scope });
794
847
  })().finally(() => { if (state.sharedRecovery === recovery) delete state.sharedRecovery; });
@@ -806,6 +859,203 @@ export class FlexHarnessCore<TScope> {
806
859
  } else await this.stores.scopes.save(state.storageKey, snapshot, expected);
807
860
  }
808
861
 
862
+ /** Durable session fence before any recovery-bearing write or external effect. */
863
+ public markRecoveryRequired(state: IStorageState, sessionId: string): Promise<void> {
864
+ if (!state.sessionLocal) return Promise.resolve();
865
+ if (state.qualifyingLegacy) {
866
+ state.recoverySessionIds.add(sessionId);
867
+ return Promise.resolve();
868
+ }
869
+ if (state.lifecycle === 'fenced') {
870
+ return Promise.reject(new FlexHarnessAbortError('The session recovery mark is fenced.'));
871
+ }
872
+ const existing = state.recoveryMarks.get(sessionId);
873
+ if (existing) return existing;
874
+ const mark = Promise.resolve()
875
+ .then(() => this.stores.scopes.sessionLocal!.setRecoveryRequired(state.storageKey, sessionId, true))
876
+ .then(() => { state.recoverySessionIds.add(sessionId); })
877
+ .catch((error: unknown) => {
878
+ state.recoveryMarks.delete(sessionId);
879
+ state.lifecycle = 'fenced';
880
+ throw error;
881
+ });
882
+ state.recoveryMarks.set(sessionId, mark);
883
+ return mark;
884
+ }
885
+
886
+ private recoveryProducerGate(state: IStorageState, sessionId: string): FlexRecoveryProducerGate {
887
+ let gate = state.recoveryProducerGates.get(sessionId);
888
+ if (!gate) {
889
+ gate = new FlexRecoveryProducerGate(() => {
890
+ void this.settleQuiescentSessionRecovery(state, sessionId)
891
+ .catch((error: unknown) => this.lifecycle.deferNamespaceDrain(state, error));
892
+ });
893
+ state.recoveryProducerGates.set(sessionId, gate);
894
+ }
895
+ return gate;
896
+ }
897
+
898
+ private cleanRecoveryEnabled(storageKey: string): boolean {
899
+ return this.cleanRecoveryAllStorageKeys || this.cleanRecoveryStorageKeys.has(storageKey);
900
+ }
901
+
902
+ /** Holds producer ownership from before the mark until the dependent operation settles. */
903
+ public withRecoveryProducer<TValue>(
904
+ state: IStorageState,
905
+ sessionId: string,
906
+ operation: () => Promise<TValue>,
907
+ ): Promise<TValue> {
908
+ if (!state.sessionLocal) return operation();
909
+ return this.recoveryProducerGate(state, sessionId).producer(async () => {
910
+ await this.markRecoveryRequired(state, sessionId);
911
+ return operation();
912
+ });
913
+ }
914
+
915
+ /** Holds all affected sessions of one scope CAS until its save and reconciliation settle. */
916
+ public withRecoveryProducerSections<TValue>(
917
+ state: IStorageState,
918
+ sessionIds: Iterable<string>,
919
+ operation: () => Promise<TValue>,
920
+ ): Promise<TValue> {
921
+ if (!state.sessionLocal) return operation();
922
+ const ids = [...new Set(sessionIds)].sort();
923
+ const enter = (index: number): Promise<TValue> => index === ids.length
924
+ ? operation()
925
+ : this.recoveryProducerGate(state, ids[index]).producer(() => enter(index + 1));
926
+ return enter(0);
927
+ }
928
+
929
+ /** Reserves one session's clean-publication boundary without draining other sessions. */
930
+ public withRecoveryPublication<TValue>(
931
+ state: IStorageState,
932
+ sessionId: string,
933
+ operation: () => Promise<TValue>,
934
+ ): Promise<TValue> {
935
+ return this.recoveryProducerGate(state, sessionId).publish(async () => {
936
+ try {
937
+ return await operation();
938
+ } finally {
939
+ // A publication may replace the dirty bit with clean proof. The next producer must
940
+ // durably mark again even if its previous mark had been coalesced in this runtime.
941
+ state.recoveryMarks.delete(sessionId);
942
+ }
943
+ });
944
+ }
945
+
946
+ /** Reads a cold session through one exclusive, authenticated recovery frontier. */
947
+ public readGuardedCleanSessionMetadata(state: IStorageState, sessionId: string): Promise<IFlexSession | undefined> {
948
+ const local = this.stores.scopes.sessionLocal;
949
+ if (!state.sessionLocal || !this.cleanRecoveryEnabled(state.storageKey) || !local?.readCleanSessionProof
950
+ || !this.stores.projections.loadHead || !this.stores.agentEvents.loadHead) {
951
+ return Promise.resolve(undefined);
952
+ }
953
+ return this.withRecoveryPublication(state, sessionId, async () => {
954
+ const current = state.catalogue.get(sessionId);
955
+ if (!current || state.tombstones.has(sessionId) || state.initializingSessions.has(sessionId)) return undefined;
956
+ const proof = await local.readCleanSessionProof!(state.storageKey, sessionId);
957
+ if (!proof) {
958
+ if (!state.recoverySessionIds.has(sessionId)) {
959
+ const catalogue = await local.loadCatalogue(state.storageKey);
960
+ if (!catalogue.qualified) throw new FlexHarnessStoreFormatError('Session recovery proof became unqualified.');
961
+ await this.markRecoveryRequired(state, sessionId);
962
+ }
963
+ return undefined;
964
+ }
965
+ if (flexSessionDigest(proof.session) !== flexSessionDigest(current)) {
966
+ await this.markRecoveryRequired(state, sessionId);
967
+ return undefined;
968
+ }
969
+ const [projection, agentEvents] = await Promise.all([
970
+ this.stores.projections.loadHead!(state.storageKey, sessionId),
971
+ this.stores.agentEvents.loadHead!(state.storageKey, sessionId),
972
+ ]);
973
+ const matches = (actual: { incarnationId?: string; revision: number; contentRoot: string } | undefined,
974
+ expected: { incarnationId: string; revision: number; contentRoot: string }) =>
975
+ actual?.incarnationId === expected.incarnationId
976
+ && actual.revision === expected.revision
977
+ && actual.contentRoot === expected.contentRoot;
978
+ if (!matches(projection, proof.clean.projection)
979
+ || !matches(agentEvents, proof.clean.agentEvents)) {
980
+ // A valid but stale clean proof is replaced by dirty before repair can touch an effect.
981
+ await this.markRecoveryRequired(state, sessionId);
982
+ return undefined;
983
+ }
984
+ return proof.session;
985
+ });
986
+ }
987
+
988
+ /** Settle a fully repaired session after its producers and pending effects have drained. */
989
+ public async settleQuiescentSessionRecovery(state: IStorageState, sessionId: string): Promise<void> {
990
+ const local = this.stores.scopes.sessionLocal;
991
+ if (!state.sessionLocal || state.qualifyingLegacy || state.lifecycle !== 'active'
992
+ || !local || !state.recoverySessionIds.has(sessionId)) return;
993
+ const cleanRead = this.cleanRecoveryEnabled(state.storageKey);
994
+ if (cleanRead && (!local.publishSessionClean || !this.stores.projections.loadHead
995
+ || !this.stores.agentEvents.loadHead)) return;
996
+ const gate = this.recoveryProducerGate(state, sessionId);
997
+ if (!gate.canPublish) return;
998
+ try {
999
+ await gate.publish(async () => {
1000
+ const stored = state.sessions.get(sessionId);
1001
+ if (!stored || state.lifecycle !== 'active' || state.qualifyingLegacy || stored.recovering
1002
+ || !state.recoverySessionIds.has(sessionId)
1003
+ || stored.projectionReconciliationRequired || stored.metricsFailure !== undefined
1004
+ || state.initializingSessions.has(sessionId)
1005
+ || state.tombstones.has(sessionId) || state.activeRuns.has(sessionId)
1006
+ || stored.outstandingPromptsById.size > 0 || stored.pendingReversion !== undefined
1007
+ || stored.pendingReversionReleases.length > 0 || stored.stagedTerminals.length > 0
1008
+ || stored.metrics.pendingGenerationId !== undefined
1009
+ || stored.metrics.pendingCompactionId !== undefined
1010
+ || [...state.pendingPermissions.values()].some((permission) =>
1011
+ !permission.settled && permission.request.sessionId === sessionId)) return;
1012
+ const catalogue = await local.loadCatalogue(state.storageKey);
1013
+ const session = catalogue.snapshot?.sessions.find((entry) => entry.sessionId === sessionId);
1014
+ if (!catalogue.qualified || !session
1015
+ || flexSessionDigest(session) !== flexSessionDigest(stored.session)) return;
1016
+ if (!cleanRead) {
1017
+ // Compatibility has recovered this runtime. Schema 2 still requires full recovery
1018
+ // in the next runtime, since this mode does not publish authenticated clean proof.
1019
+ if (catalogue.recoverySchemaVersion !== 2) {
1020
+ await local.setRecoveryRequired(state.storageKey, sessionId, false);
1021
+ }
1022
+ state.recoverySessionIds.delete(sessionId);
1023
+ state.recoveryMarks.delete(sessionId);
1024
+ return;
1025
+ }
1026
+ if (catalogue.recoveryRevision === undefined) return;
1027
+ const [projection, agentEvents] = await Promise.all([
1028
+ this.stores.projections.loadHead!(state.storageKey, sessionId),
1029
+ this.stores.agentEvents.loadHead!(state.storageKey, sessionId),
1030
+ ]);
1031
+ if (!projection?.incarnationId || !agentEvents?.incarnationId) return;
1032
+ await local.publishSessionClean!(state.storageKey, catalogue.recoveryRevision, {
1033
+ sessionId,
1034
+ state: 'clean',
1035
+ ...requireSessionGeneration(session),
1036
+ sessionDigest: flexSessionDigest(session),
1037
+ projection: { incarnationId: projection.incarnationId, revision: projection.revision,
1038
+ contentRoot: projection.contentRoot },
1039
+ agentEvents: { incarnationId: agentEvents.incarnationId, revision: agentEvents.revision,
1040
+ contentRoot: agentEvents.contentRoot },
1041
+ });
1042
+ state.recoverySessionIds.delete(sessionId);
1043
+ state.recoveryMarks.delete(sessionId);
1044
+ });
1045
+ } catch (error) {
1046
+ // Another session can advance the namespace sidecar between our read and exact CAS.
1047
+ // This session remains durably dirty and another settled read can publish it.
1048
+ if (error instanceof FlexHarnessStoreConflictError) return;
1049
+ throw error;
1050
+ }
1051
+ }
1052
+
1053
+ /** A generation may keep executing tools after its admission method has returned. */
1054
+ public admitRecoveryRun(state: IStorageState, sessionId: string, completion: Promise<unknown>): void {
1055
+ if (!state.sessionLocal) return;
1056
+ this.recoveryProducerGate(state, sessionId).admitRun(completion);
1057
+ }
1058
+
809
1059
  /** Registers before the projection's pending intent can reach any workspace provider. */
810
1060
  public async updateRecoveryObligation(state: IStorageState, stored: IStoredSessionState, registerOnly = false): Promise<void> {
811
1061
  if (!state.sessionLocal) return;
@@ -816,21 +1066,15 @@ export class FlexHarnessCore<TScope> {
816
1066
  || stored.metrics.pendingCompactionId !== undefined
817
1067
  || (stored.recovering === true && state.recoverySessionIds.has(sessionId))
818
1068
  || (this.turnReversionProvider !== undefined && active !== undefined && !active.finishedEventEmitted);
819
- if (registerOnly && !required) return;
820
1069
  if (state.qualifyingLegacy) {
821
1070
  if (required) state.recoverySessionIds.add(sessionId);
822
- else state.recoverySessionIds.delete(sessionId);
1071
+ else if (!registerOnly) state.recoverySessionIds.delete(sessionId);
823
1072
  return;
824
1073
  }
825
- // Re-registering is idempotent. The durable sidecar is authoritative, not this cache.
826
- try {
827
- await this.stores.scopes.sessionLocal!.setRecoveryRequired(state.storageKey, sessionId, required);
828
- if (required) state.recoverySessionIds.add(sessionId);
829
- else state.recoverySessionIds.delete(sessionId);
830
- } catch (error) {
831
- state.lifecycle = 'fenced';
832
- throw error;
833
- }
1074
+ // The quiescent settlement path clears marks after the event, job, permission and metadata
1075
+ // writers have settled.
1076
+ if (!required) return;
1077
+ await this.markRecoveryRequired(state, sessionId);
834
1078
  }
835
1079
 
836
1080
  /**
@@ -843,6 +1087,7 @@ export class FlexHarnessCore<TScope> {
843
1087
  if (!state.sessionLocal || state.lifecycle !== 'active') return;
844
1088
  try {
845
1089
  await this.updateRecoveryObligation(state, stored);
1090
+ await this.settleQuiescentSessionRecovery(state, stored.session.sessionId);
846
1091
  } catch (error) {
847
1092
  this.lifecycle.deferNamespaceDrain(state, error);
848
1093
  }
@@ -963,8 +1208,24 @@ export class FlexHarnessCore<TScope> {
963
1208
  protocolVersion: 1,
964
1209
  }));
965
1210
  const permission = permissionResult.value;
966
- const eventStore = eventStoreResult.value;
967
- const jobStore = jobStoreResult.value;
1211
+ const rawEventStore = eventStoreResult.value;
1212
+ const eventStore: plugins.IAgentEventStore = {
1213
+ load: (id) => rawEventStore.load(id),
1214
+ ...(rawEventStore.loadHead ? { loadHead: (id: string) => rawEventStore.loadHead!(id) } : {}),
1215
+ save: (id, change, expectedRevision) => this.withRecoveryProducer(state, sessionId,
1216
+ () => rawEventStore.save(id, change, expectedRevision)),
1217
+ ...(rawEventStore.archive ? { archive: (archive: plugins.IAgentEventArchive) =>
1218
+ this.withRecoveryProducer(state, sessionId, () => rawEventStore.archive!(archive)) } : {}),
1219
+ ...(rawEventStore.loadArchive ? {
1220
+ loadArchive: (id: string, archiveId: string) => rawEventStore.loadArchive!(id, archiveId),
1221
+ } : {}),
1222
+ };
1223
+ const rawJobStore = jobStoreResult.value;
1224
+ const jobStore: plugins.IToolJobStore = {
1225
+ load: (abortSignal) => rawJobStore.load(abortSignal),
1226
+ save: (jobs, expectedRevision, abortSignal) => this.withRecoveryProducer(state, sessionId,
1227
+ () => rawJobStore.save(jobs, expectedRevision, abortSignal)),
1228
+ };
968
1229
  let executionContextHandle: IFlexExecutionContextHandle | undefined;
969
1230
  try {
970
1231
  if (this.executionContextProvider) {
@@ -1065,19 +1326,30 @@ export class FlexHarnessCore<TScope> {
1065
1326
  } = this.agentSessionPolicy;
1066
1327
  const executionContext = executionContextHandle?.context;
1067
1328
  const jobContext = executionContext?.jobs;
1068
- const contextualExecutionContext = executionContext && jobContext?.subscribe
1329
+ const guardedJobs: NonNullable<plugins.IToolExecutionContext['jobs']> | undefined = jobContext
1069
1330
  ? {
1070
- ...executionContext,
1071
- jobs: {
1072
- start: jobContext.start.bind(jobContext),
1073
- get: jobContext.get.bind(jobContext),
1074
- list: jobContext.list.bind(jobContext),
1075
- ...(jobContext.abort ? { abort: jobContext.abort.bind(jobContext) } : {}),
1331
+ start: (request, invocationOptions) => this.withRecoveryProducer(state, sessionId,
1332
+ async () => {
1333
+ const handle = await jobContext.start(request, invocationOptions);
1334
+ if (state.sessionLocal) {
1335
+ this.recoveryProducerGate(state, sessionId).admitBackgroundJob(handle.completion);
1336
+ }
1337
+ return handle;
1338
+ }),
1339
+ get: jobContext.get.bind(jobContext),
1340
+ list: jobContext.list.bind(jobContext),
1341
+ ...(jobContext.abort ? { abort: (executionId: string) => this.withRecoveryProducer(
1342
+ state, sessionId, () => jobContext.abort!(executionId)) } : {}),
1343
+ ...(jobContext.subscribe ? {
1076
1344
  subscribe: (listener: Parameters<NonNullable<typeof jobContext.subscribe>>[0]) =>
1077
1345
  jobContext.subscribe!((event) =>
1078
- this.compaction.withCompactorContext(compactorContext, () => listener(event))),
1079
- },
1346
+ this.withRecoveryProducer(state, sessionId, () => Promise.resolve(
1347
+ this.compaction.withCompactorContext(compactorContext, () => listener(event))))),
1348
+ } : {}),
1080
1349
  }
1350
+ : undefined;
1351
+ const contextualExecutionContext = executionContext && guardedJobs
1352
+ ? { ...executionContext, jobs: guardedJobs }
1081
1353
  : executionContext;
1082
1354
  const agentSessionOptions: plugins.IAgentSessionOptions & {
1083
1355
  transactionOutcomeErrorProjector: (error: unknown) => string;
@@ -1371,12 +1643,8 @@ export class FlexHarnessCore<TScope> {
1371
1643
  }
1372
1644
  freezeProjectionEntries(stored);
1373
1645
  const change = this.createProjectionChange(stored, expected + 1);
1374
- await this.stores.projections.save(
1375
- storageKey,
1376
- stored.session.sessionId,
1377
- change,
1378
- expected,
1379
- );
1646
+ await this.withRecoveryProducer(state, stored.session.sessionId, () =>
1647
+ this.stores.projections.save(storageKey, stored.session.sessionId, change, expected));
1380
1648
  stored.projectionSchemaVersion = 3;
1381
1649
  delete stored.projectionBaseline;
1382
1650
  this.rememberSavedProjectionLists(stored);
@@ -1805,9 +2073,41 @@ export class FlexHarnessCore<TScope> {
1805
2073
  throw error;
1806
2074
  }
1807
2075
  const snapshot = this.createScopeSnapshot(state, beforeRevision + 1);
2076
+ const affectedOwners = new Set([
2077
+ ...beforeCatalogue.keys(), ...beforeTombstones.keys(),
2078
+ ...state.catalogue.keys(), ...state.tombstones.keys(),
2079
+ ]);
2080
+ for (const id of [...affectedOwners]) {
2081
+ if (canonicalJson(beforeCatalogue.get(id)) === canonicalJson(state.catalogue.get(id))
2082
+ && canonicalJson(beforeTombstones.get(id)) === canonicalJson(state.tombstones.get(id))) {
2083
+ affectedOwners.delete(id);
2084
+ }
2085
+ }
2086
+ return this.withRecoveryProducerSections(state, affectedOwners, async () => {
1808
2087
  try {
2088
+ // An existing owner must be dirty before its catalogue metadata or tombstone changes.
2089
+ // A new owner is published by this CAS first: marking it before publication could leave
2090
+ // an orphan sidecar ID if the CAS never commits.
2091
+ const changedOwners = new Set([...beforeCatalogue.keys(), ...beforeTombstones.keys()]);
2092
+ for (const id of changedOwners) {
2093
+ const wasLive = beforeCatalogue.has(id);
2094
+ const isLive = state.catalogue.has(id);
2095
+ const isTombstoned = state.tombstones.has(id);
2096
+ if (!wasLive && !isLive && !isTombstoned) continue; // completed tombstone removal
2097
+ if (canonicalJson(beforeCatalogue.get(id)) !== canonicalJson(state.catalogue.get(id))
2098
+ || canonicalJson(beforeTombstones.get(id)) !== canonicalJson(state.tombstones.get(id))) {
2099
+ await this.markRecoveryRequired(state, id);
2100
+ }
2101
+ }
1809
2102
  await this.saveScopeSnapshot(state, snapshot, beforeRevision);
1810
2103
  state.revision = snapshot.revision;
2104
+ // The scope CAS creates the owner. Only now can its durable dirty bit be published without
2105
+ // making an orphan obligation if that CAS was refused.
2106
+ for (const id of new Set([...state.catalogue.keys(), ...state.tombstones.keys()])) {
2107
+ if (!beforeCatalogue.has(id) && !beforeTombstones.has(id)) {
2108
+ await this.markRecoveryRequired(state, id);
2109
+ }
2110
+ }
1811
2111
  return result;
1812
2112
  } catch (error) {
1813
2113
  if (error instanceof FlexHarnessStoreConflictError) {
@@ -1826,13 +2126,13 @@ export class FlexHarnessCore<TScope> {
1826
2126
  } catch (loadError) {
1827
2127
  reconciliationError = loadError;
1828
2128
  }
1829
- if (current && JSON.stringify(current) === JSON.stringify(snapshot)) {
2129
+ if (current && canonicalJson(current) === canonicalJson(snapshot)) {
1830
2130
  state.revision = snapshot.revision;
1831
2131
  throw error;
1832
2132
  }
1833
2133
  if (
1834
2134
  (current === undefined && beforeRevision === 0)
1835
- || (current !== undefined && JSON.stringify(current) === JSON.stringify(beforeSnapshot()))
2135
+ || (current !== undefined && canonicalJson(current) === canonicalJson(beforeSnapshot()))
1836
2136
  ) {
1837
2137
  restore();
1838
2138
  throw error;
@@ -1843,6 +2143,7 @@ export class FlexHarnessCore<TScope> {
1843
2143
  ? error
1844
2144
  : combineErrors([error, reconciliationError]);
1845
2145
  }
2146
+ });
1846
2147
  }
1847
2148
 
1848
2149
  /** Holds a mutation declared for one session to that session: no other session, tombstone or cleanup changed. */
@@ -1890,7 +2191,8 @@ export class FlexHarnessCore<TScope> {
1890
2191
  _allowDuringDrain = false,
1891
2192
  onSaved?: TProjectionSaved,
1892
2193
  ): Promise<TValue> {
1893
- return this.enqueueProjectionOperation(stored, () => this.saveProjectionMutation(state, stored, mutation, onSaved));
2194
+ return this.enqueueProjectionOperation(stored, () => this.withRecoveryProducer(
2195
+ state, stored.session.sessionId, () => this.saveProjectionMutation(state, stored, mutation, onSaved)));
1894
2196
  }
1895
2197
 
1896
2198
  /**
@@ -1907,7 +2209,8 @@ export class FlexHarnessCore<TScope> {
1907
2209
  return this.enqueueProjectionOperation(stored, async () => {
1908
2210
  const mutation = prepare();
1909
2211
  if (!mutation) return false;
1910
- await this.saveProjectionMutation(state, stored, mutation, onSaved);
2212
+ await this.withRecoveryProducer(state, stored.session.sessionId,
2213
+ () => this.saveProjectionMutation(state, stored, mutation, onSaved));
1911
2214
  return true;
1912
2215
  });
1913
2216
  }
@@ -1983,7 +2286,7 @@ export class FlexHarnessCore<TScope> {
1983
2286
  freezeProjectionEntries(stored);
1984
2287
  const revision = beforeRevision + 1;
1985
2288
  try {
1986
- if (state.sessionLocal) await this.updateRecoveryObligation(state, stored, true);
2289
+ await this.markRecoveryRequired(state, stored.session.sessionId);
1987
2290
  const saveStartedAt = onSaved ? this.checkpointClock.now() : 0;
1988
2291
  await this.stores.projections.save(
1989
2292
  state.storageKey,