@modelprofile.com/flexharness 18.8.0 → 18.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.flexharness.core.d.ts +19 -0
- package/dist_ts/classes.flexharness.core.js +358 -67
- package/dist_ts/classes.flexharness.deletion.d.ts +1 -1
- package/dist_ts/classes.flexharness.deletion.js +147 -141
- package/dist_ts/classes.flexharness.imports.js +3 -3
- package/dist_ts/classes.flexharness.lifecycle.js +4 -1
- package/dist_ts/classes.flexharness.permissions.js +5 -5
- package/dist_ts/classes.flexharness.prompts.js +5 -6
- package/dist_ts/classes.flexharness.results.js +3 -3
- package/dist_ts/classes.flexharness.reversions.js +320 -316
- package/dist_ts/classes.flexharness.sessions.js +19 -3
- package/dist_ts/classes.flexharness.subagents.js +2 -2
- package/dist_ts/classes.recoverygate.d.ts +30 -0
- package/dist_ts/classes.recoverygate.js +105 -0
- package/dist_ts/classes.scoperecovery.d.ts +9 -1
- package/dist_ts/classes.scoperecovery.js +94 -26
- package/dist_ts/interfaces.d.ts +22 -0
- package/dist_ts/interfaces.flexharness.d.ts +5 -0
- package/dist_ts/interfaces.js +1 -1
- package/dist_ts/plugins.d.ts +1 -1
- package/dist_ts/plugins.js +1 -1
- package/dist_ts_stores_nosqldb/classes.nosqldbstores.js +5 -1
- package/package.json +3 -3
- package/readme.md +5 -3
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.flexharness.core.ts +337 -34
- package/ts/classes.flexharness.deletion.ts +24 -15
- package/ts/classes.flexharness.imports.ts +3 -2
- package/ts/classes.flexharness.lifecycle.ts +3 -0
- package/ts/classes.flexharness.permissions.ts +8 -4
- package/ts/classes.flexharness.prompts.ts +10 -8
- package/ts/classes.flexharness.results.ts +3 -2
- package/ts/classes.flexharness.reversions.ts +13 -6
- package/ts/classes.flexharness.sessions.ts +18 -2
- package/ts/classes.flexharness.subagents.ts +5 -4
- package/ts/classes.recoverygate.ts +111 -0
- package/ts/classes.scoperecovery.ts +102 -25
- package/ts/interfaces.flexharness.ts +5 -0
- package/ts/interfaces.ts +23 -0
- package/ts/plugins.ts +1 -0
- package/ts/readme.md +5 -3
- package/ts_stores_nosqldb/classes.nosqldbstores.ts +5 -0
- package/ts_stores_nosqldb/readme.md +1 -3
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
FlexHarnessSessionBusyError,
|
|
10
10
|
FlexHarnessStoreConflictError,
|
|
11
11
|
FlexHarnessStoreCommitUncertainError,
|
|
12
|
+
FlexHarnessStoreFormatError,
|
|
12
13
|
FlexHarnessUnresolvedToolExecutionsError,
|
|
13
14
|
FlexHarnessValidationError,
|
|
14
15
|
modelLimitErrorInfo,
|
|
@@ -51,6 +52,8 @@ import type {
|
|
|
51
52
|
} from './interfaces.js';
|
|
52
53
|
import { InMemoryFlexHarnessStores } from './classes.stores.js';
|
|
53
54
|
import { FlexCodeSandbox } from './classes.codesandbox.js';
|
|
55
|
+
import { FlexRecoveryProducerGate } from './classes.recoverygate.js';
|
|
56
|
+
import { flexSessionDigest } from './classes.scoperecovery.js';
|
|
54
57
|
import {
|
|
55
58
|
assertJsonSerializable,
|
|
56
59
|
canonicalJson,
|
|
@@ -160,6 +163,9 @@ export class FlexHarnessCore<TScope> {
|
|
|
160
163
|
public readonly attachmentReferenceProtocols: ReadonlySet<string>;
|
|
161
164
|
public readonly stores: IFlexHarnessStores;
|
|
162
165
|
public readonly sessionLoading: 'eager' | 'session-local';
|
|
166
|
+
public readonly cleanRecoveryConfigured: boolean;
|
|
167
|
+
private readonly cleanRecoveryAllStorageKeys: boolean;
|
|
168
|
+
private readonly cleanRecoveryStorageKeys: ReadonlySet<string>;
|
|
163
169
|
private readonly legacyQualifications = new Set<string>();
|
|
164
170
|
public readonly agentSessionPolicy: IFlexAgentSessionPolicy<TScope>;
|
|
165
171
|
public readonly toolOutputLimits: Required<NonNullable<IFlexHarnessOptions<TScope>['toolOutputLimits']>>;
|
|
@@ -232,6 +238,34 @@ export class FlexHarnessCore<TScope> {
|
|
|
232
238
|
if (this.sessionLoading === 'session-local' && !this.stores.scopes.sessionLocal) {
|
|
233
239
|
throw new FlexHarnessValidationError('session-local loading requires scopes.sessionLocal capability.');
|
|
234
240
|
}
|
|
241
|
+
const frontier = options.recoveryFrontier ?? 'compatibility';
|
|
242
|
+
this.cleanRecoveryConfigured = frontier !== 'compatibility';
|
|
243
|
+
if (frontier === 'compatibility') {
|
|
244
|
+
this.cleanRecoveryAllStorageKeys = false;
|
|
245
|
+
this.cleanRecoveryStorageKeys = new Set();
|
|
246
|
+
} else {
|
|
247
|
+
if (!frontier || typeof frontier !== 'object' || Array.isArray(frontier)
|
|
248
|
+
|| frontier.mode !== 'clean-read'
|
|
249
|
+
|| Object.keys(frontier).some((key) => key !== 'mode' && key !== 'storageKeys')) {
|
|
250
|
+
throw new FlexHarnessValidationError('recoveryFrontier must be "compatibility" or a clean-read configuration.');
|
|
251
|
+
}
|
|
252
|
+
const local = this.stores.scopes.sessionLocal;
|
|
253
|
+
if (this.sessionLoading !== 'session-local' || !local?.readCleanSessionProof
|
|
254
|
+
|| !local.publishSessionClean || !this.stores.projections.loadHead
|
|
255
|
+
|| !this.stores.agentEvents.loadHead) {
|
|
256
|
+
throw new FlexHarnessValidationError('clean-read recovery requires session-local loading, schema-2 publication and bounded projection and Agent-event heads.');
|
|
257
|
+
}
|
|
258
|
+
if (frontier.storageKeys !== undefined
|
|
259
|
+
&& (!Array.isArray(frontier.storageKeys) || frontier.storageKeys.length === 0)) {
|
|
260
|
+
throw new FlexHarnessValidationError('recoveryFrontier.storageKeys must be a non-empty array.');
|
|
261
|
+
}
|
|
262
|
+
for (const key of frontier.storageKeys ?? []) validateIdentifier(key, 'recoveryFrontier storageKey');
|
|
263
|
+
if (frontier.storageKeys && new Set(frontier.storageKeys).size !== frontier.storageKeys.length) {
|
|
264
|
+
throw new FlexHarnessValidationError('recoveryFrontier.storageKeys must be unique.');
|
|
265
|
+
}
|
|
266
|
+
this.cleanRecoveryAllStorageKeys = frontier.storageKeys === undefined;
|
|
267
|
+
this.cleanRecoveryStorageKeys = new Set(frontier.storageKeys ?? []);
|
|
268
|
+
}
|
|
235
269
|
this.agentSessionPolicy = normalizeAgentSessionPolicy(options.agentSessionPolicy);
|
|
236
270
|
this.toolOutputLimits = resolveJsonLimits(options.toolOutputLimits);
|
|
237
271
|
this.callbackLimits = resolveCallbackLimits(options.callbackLimits);
|
|
@@ -459,6 +493,9 @@ export class FlexHarnessCore<TScope> {
|
|
|
459
493
|
const local = this.sessionLoading === 'session-local'
|
|
460
494
|
? await this.stores.scopes.sessionLocal!.loadCatalogue(storageKey)
|
|
461
495
|
: undefined;
|
|
496
|
+
if (local && this.cleanRecoveryEnabled(storageKey) && local.recoveryRevision === undefined) {
|
|
497
|
+
throw new FlexHarnessValidationError('clean-read recovery requires an exact sidecar revision.');
|
|
498
|
+
}
|
|
462
499
|
const qualifyingLegacy = local !== undefined && !local.qualified && this.legacyQualifications.has(storageKey);
|
|
463
500
|
if (local && !local.qualified && !qualifyingLegacy) {
|
|
464
501
|
throw new FlexHarnessValidationError(
|
|
@@ -479,7 +516,13 @@ export class FlexHarnessCore<TScope> {
|
|
|
479
516
|
catalogue: new Map(liveScopeSessions(snapshot).map((session) => [session.sessionId, session])),
|
|
480
517
|
sessionLocal: local !== undefined,
|
|
481
518
|
qualifyingLegacy,
|
|
482
|
-
recoverySessionIds: new Set(local
|
|
519
|
+
recoverySessionIds: new Set(local && local.recoverySchemaVersion === 2
|
|
520
|
+
&& !this.cleanRecoveryEnabled(storageKey)
|
|
521
|
+
? [...new Set([...local.recoverySessionIds,
|
|
522
|
+
...(local.snapshot?.sessions.map((session) => session.sessionId) ?? [])])]
|
|
523
|
+
: local?.recoverySessionIds ?? []),
|
|
524
|
+
recoveryMarks: new Map(),
|
|
525
|
+
recoveryProducerGates: new Map(),
|
|
483
526
|
sessionLoads: new Map(),
|
|
484
527
|
sessions: new Map(),
|
|
485
528
|
retainedSessionCleanups: new Map(),
|
|
@@ -514,6 +557,10 @@ export class FlexHarnessCore<TScope> {
|
|
|
514
557
|
for (const metadata of liveScopeSessions(snapshot).filter((session) =>
|
|
515
558
|
!state.sessionLocal || qualifyingLegacy || state.recoverySessionIds.has(session.sessionId))) {
|
|
516
559
|
compactorLifecycleController.signal.throwIfAborted();
|
|
560
|
+
if (local?.recoverySchemaVersion === 2 && !this.cleanRecoveryEnabled(storageKey)) {
|
|
561
|
+
// A compatibility open always invalidates inherited clean proof before full repair.
|
|
562
|
+
await this.markRecoveryRequired(state, metadata.sessionId);
|
|
563
|
+
}
|
|
517
564
|
const stored = await this.loadSessionRuntime(
|
|
518
565
|
state,
|
|
519
566
|
metadata,
|
|
@@ -586,6 +633,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
586
633
|
await this.stores.scopes.sessionLocal!.setRecoveryRequired(storageKey, tombstone.sessionId, false);
|
|
587
634
|
}
|
|
588
635
|
state.recoverySessionIds.delete(tombstone.sessionId);
|
|
636
|
+
state.recoveryMarks.delete(tombstone.sessionId);
|
|
589
637
|
}
|
|
590
638
|
for (const tombstone of group) state.tombstones.delete(tombstone.sessionId);
|
|
591
639
|
scopeChanged = true;
|
|
@@ -611,6 +659,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
611
659
|
for (const loaded of loadedSessions) {
|
|
612
660
|
loaded.recovering = false;
|
|
613
661
|
await this.updateRecoveryObligation(state, loaded);
|
|
662
|
+
await this.settleQuiescentSessionRecovery(state, loaded.session.sessionId);
|
|
614
663
|
}
|
|
615
664
|
}
|
|
616
665
|
return state;
|
|
@@ -741,6 +790,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
741
790
|
else state.catalogue.set(sessionId, stored.session);
|
|
742
791
|
stored.recovering = false;
|
|
743
792
|
await this.updateRecoveryObligation(state, stored);
|
|
793
|
+
await this.settleQuiescentSessionRecovery(state, sessionId);
|
|
744
794
|
this.assertStateAcceptingWork(state);
|
|
745
795
|
return stored;
|
|
746
796
|
} catch (error) {
|
|
@@ -781,6 +831,8 @@ export class FlexHarnessCore<TScope> {
|
|
|
781
831
|
if (state.sessionLoads.has(id)) { await state.sessionLoads.get(id); continue; }
|
|
782
832
|
const stored = state.sessions.get(id);
|
|
783
833
|
if (!stored) { await this.hydrateSession(state, id, scopeId, scope); continue; }
|
|
834
|
+
await this.settleQuiescentSessionRecovery(state, id);
|
|
835
|
+
if (!state.recoverySessionIds.has(id)) continue;
|
|
784
836
|
stored.recovering = true;
|
|
785
837
|
await this.repairLoadedSession(state.storageKey, stored, state, scopeId, scope,
|
|
786
838
|
new FlexTimeSlicer(state.compactorLifecycleController.signal));
|
|
@@ -789,6 +841,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
789
841
|
await this.mutateSessionScope(state, id, () => undefined);
|
|
790
842
|
stored.recovering = false;
|
|
791
843
|
await this.updateRecoveryObligation(state, stored);
|
|
844
|
+
await this.settleQuiescentSessionRecovery(state, id);
|
|
792
845
|
}
|
|
793
846
|
for (const root of roots) await this.deletion.finishTombstoneCleanup(state, root, scopeId, { scopeId, scope });
|
|
794
847
|
})().finally(() => { if (state.sharedRecovery === recovery) delete state.sharedRecovery; });
|
|
@@ -806,6 +859,203 @@ export class FlexHarnessCore<TScope> {
|
|
|
806
859
|
} else await this.stores.scopes.save(state.storageKey, snapshot, expected);
|
|
807
860
|
}
|
|
808
861
|
|
|
862
|
+
/** Durable session fence before any recovery-bearing write or external effect. */
|
|
863
|
+
public markRecoveryRequired(state: IStorageState, sessionId: string): Promise<void> {
|
|
864
|
+
if (!state.sessionLocal) return Promise.resolve();
|
|
865
|
+
if (state.qualifyingLegacy) {
|
|
866
|
+
state.recoverySessionIds.add(sessionId);
|
|
867
|
+
return Promise.resolve();
|
|
868
|
+
}
|
|
869
|
+
if (state.lifecycle === 'fenced') {
|
|
870
|
+
return Promise.reject(new FlexHarnessAbortError('The session recovery mark is fenced.'));
|
|
871
|
+
}
|
|
872
|
+
const existing = state.recoveryMarks.get(sessionId);
|
|
873
|
+
if (existing) return existing;
|
|
874
|
+
const mark = Promise.resolve()
|
|
875
|
+
.then(() => this.stores.scopes.sessionLocal!.setRecoveryRequired(state.storageKey, sessionId, true))
|
|
876
|
+
.then(() => { state.recoverySessionIds.add(sessionId); })
|
|
877
|
+
.catch((error: unknown) => {
|
|
878
|
+
state.recoveryMarks.delete(sessionId);
|
|
879
|
+
state.lifecycle = 'fenced';
|
|
880
|
+
throw error;
|
|
881
|
+
});
|
|
882
|
+
state.recoveryMarks.set(sessionId, mark);
|
|
883
|
+
return mark;
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
private recoveryProducerGate(state: IStorageState, sessionId: string): FlexRecoveryProducerGate {
|
|
887
|
+
let gate = state.recoveryProducerGates.get(sessionId);
|
|
888
|
+
if (!gate) {
|
|
889
|
+
gate = new FlexRecoveryProducerGate(() => {
|
|
890
|
+
void this.settleQuiescentSessionRecovery(state, sessionId)
|
|
891
|
+
.catch((error: unknown) => this.lifecycle.deferNamespaceDrain(state, error));
|
|
892
|
+
});
|
|
893
|
+
state.recoveryProducerGates.set(sessionId, gate);
|
|
894
|
+
}
|
|
895
|
+
return gate;
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
private cleanRecoveryEnabled(storageKey: string): boolean {
|
|
899
|
+
return this.cleanRecoveryAllStorageKeys || this.cleanRecoveryStorageKeys.has(storageKey);
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
/** Holds producer ownership from before the mark until the dependent operation settles. */
|
|
903
|
+
public withRecoveryProducer<TValue>(
|
|
904
|
+
state: IStorageState,
|
|
905
|
+
sessionId: string,
|
|
906
|
+
operation: () => Promise<TValue>,
|
|
907
|
+
): Promise<TValue> {
|
|
908
|
+
if (!state.sessionLocal) return operation();
|
|
909
|
+
return this.recoveryProducerGate(state, sessionId).producer(async () => {
|
|
910
|
+
await this.markRecoveryRequired(state, sessionId);
|
|
911
|
+
return operation();
|
|
912
|
+
});
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
/** Holds all affected sessions of one scope CAS until its save and reconciliation settle. */
|
|
916
|
+
public withRecoveryProducerSections<TValue>(
|
|
917
|
+
state: IStorageState,
|
|
918
|
+
sessionIds: Iterable<string>,
|
|
919
|
+
operation: () => Promise<TValue>,
|
|
920
|
+
): Promise<TValue> {
|
|
921
|
+
if (!state.sessionLocal) return operation();
|
|
922
|
+
const ids = [...new Set(sessionIds)].sort();
|
|
923
|
+
const enter = (index: number): Promise<TValue> => index === ids.length
|
|
924
|
+
? operation()
|
|
925
|
+
: this.recoveryProducerGate(state, ids[index]).producer(() => enter(index + 1));
|
|
926
|
+
return enter(0);
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
/** Reserves one session's clean-publication boundary without draining other sessions. */
|
|
930
|
+
public withRecoveryPublication<TValue>(
|
|
931
|
+
state: IStorageState,
|
|
932
|
+
sessionId: string,
|
|
933
|
+
operation: () => Promise<TValue>,
|
|
934
|
+
): Promise<TValue> {
|
|
935
|
+
return this.recoveryProducerGate(state, sessionId).publish(async () => {
|
|
936
|
+
try {
|
|
937
|
+
return await operation();
|
|
938
|
+
} finally {
|
|
939
|
+
// A publication may replace the dirty bit with clean proof. The next producer must
|
|
940
|
+
// durably mark again even if its previous mark had been coalesced in this runtime.
|
|
941
|
+
state.recoveryMarks.delete(sessionId);
|
|
942
|
+
}
|
|
943
|
+
});
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
/** Reads a cold session through one exclusive, authenticated recovery frontier. */
|
|
947
|
+
public readGuardedCleanSessionMetadata(state: IStorageState, sessionId: string): Promise<IFlexSession | undefined> {
|
|
948
|
+
const local = this.stores.scopes.sessionLocal;
|
|
949
|
+
if (!state.sessionLocal || !this.cleanRecoveryEnabled(state.storageKey) || !local?.readCleanSessionProof
|
|
950
|
+
|| !this.stores.projections.loadHead || !this.stores.agentEvents.loadHead) {
|
|
951
|
+
return Promise.resolve(undefined);
|
|
952
|
+
}
|
|
953
|
+
return this.withRecoveryPublication(state, sessionId, async () => {
|
|
954
|
+
const current = state.catalogue.get(sessionId);
|
|
955
|
+
if (!current || state.tombstones.has(sessionId) || state.initializingSessions.has(sessionId)) return undefined;
|
|
956
|
+
const proof = await local.readCleanSessionProof!(state.storageKey, sessionId);
|
|
957
|
+
if (!proof) {
|
|
958
|
+
if (!state.recoverySessionIds.has(sessionId)) {
|
|
959
|
+
const catalogue = await local.loadCatalogue(state.storageKey);
|
|
960
|
+
if (!catalogue.qualified) throw new FlexHarnessStoreFormatError('Session recovery proof became unqualified.');
|
|
961
|
+
await this.markRecoveryRequired(state, sessionId);
|
|
962
|
+
}
|
|
963
|
+
return undefined;
|
|
964
|
+
}
|
|
965
|
+
if (flexSessionDigest(proof.session) !== flexSessionDigest(current)) {
|
|
966
|
+
await this.markRecoveryRequired(state, sessionId);
|
|
967
|
+
return undefined;
|
|
968
|
+
}
|
|
969
|
+
const [projection, agentEvents] = await Promise.all([
|
|
970
|
+
this.stores.projections.loadHead!(state.storageKey, sessionId),
|
|
971
|
+
this.stores.agentEvents.loadHead!(state.storageKey, sessionId),
|
|
972
|
+
]);
|
|
973
|
+
const matches = (actual: { incarnationId?: string; revision: number; contentRoot: string } | undefined,
|
|
974
|
+
expected: { incarnationId: string; revision: number; contentRoot: string }) =>
|
|
975
|
+
actual?.incarnationId === expected.incarnationId
|
|
976
|
+
&& actual.revision === expected.revision
|
|
977
|
+
&& actual.contentRoot === expected.contentRoot;
|
|
978
|
+
if (!matches(projection, proof.clean.projection)
|
|
979
|
+
|| !matches(agentEvents, proof.clean.agentEvents)) {
|
|
980
|
+
// A valid but stale clean proof is replaced by dirty before repair can touch an effect.
|
|
981
|
+
await this.markRecoveryRequired(state, sessionId);
|
|
982
|
+
return undefined;
|
|
983
|
+
}
|
|
984
|
+
return proof.session;
|
|
985
|
+
});
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
/** Settle a fully repaired session after its producers and pending effects have drained. */
|
|
989
|
+
public async settleQuiescentSessionRecovery(state: IStorageState, sessionId: string): Promise<void> {
|
|
990
|
+
const local = this.stores.scopes.sessionLocal;
|
|
991
|
+
if (!state.sessionLocal || state.qualifyingLegacy || state.lifecycle !== 'active'
|
|
992
|
+
|| !local || !state.recoverySessionIds.has(sessionId)) return;
|
|
993
|
+
const cleanRead = this.cleanRecoveryEnabled(state.storageKey);
|
|
994
|
+
if (cleanRead && (!local.publishSessionClean || !this.stores.projections.loadHead
|
|
995
|
+
|| !this.stores.agentEvents.loadHead)) return;
|
|
996
|
+
const gate = this.recoveryProducerGate(state, sessionId);
|
|
997
|
+
if (!gate.canPublish) return;
|
|
998
|
+
try {
|
|
999
|
+
await gate.publish(async () => {
|
|
1000
|
+
const stored = state.sessions.get(sessionId);
|
|
1001
|
+
if (!stored || state.lifecycle !== 'active' || state.qualifyingLegacy || stored.recovering
|
|
1002
|
+
|| !state.recoverySessionIds.has(sessionId)
|
|
1003
|
+
|| stored.projectionReconciliationRequired || stored.metricsFailure !== undefined
|
|
1004
|
+
|| state.initializingSessions.has(sessionId)
|
|
1005
|
+
|| state.tombstones.has(sessionId) || state.activeRuns.has(sessionId)
|
|
1006
|
+
|| stored.outstandingPromptsById.size > 0 || stored.pendingReversion !== undefined
|
|
1007
|
+
|| stored.pendingReversionReleases.length > 0 || stored.stagedTerminals.length > 0
|
|
1008
|
+
|| stored.metrics.pendingGenerationId !== undefined
|
|
1009
|
+
|| stored.metrics.pendingCompactionId !== undefined
|
|
1010
|
+
|| [...state.pendingPermissions.values()].some((permission) =>
|
|
1011
|
+
!permission.settled && permission.request.sessionId === sessionId)) return;
|
|
1012
|
+
const catalogue = await local.loadCatalogue(state.storageKey);
|
|
1013
|
+
const session = catalogue.snapshot?.sessions.find((entry) => entry.sessionId === sessionId);
|
|
1014
|
+
if (!catalogue.qualified || !session
|
|
1015
|
+
|| flexSessionDigest(session) !== flexSessionDigest(stored.session)) return;
|
|
1016
|
+
if (!cleanRead) {
|
|
1017
|
+
// Compatibility has recovered this runtime. Schema 2 still requires full recovery
|
|
1018
|
+
// in the next runtime, since this mode does not publish authenticated clean proof.
|
|
1019
|
+
if (catalogue.recoverySchemaVersion !== 2) {
|
|
1020
|
+
await local.setRecoveryRequired(state.storageKey, sessionId, false);
|
|
1021
|
+
}
|
|
1022
|
+
state.recoverySessionIds.delete(sessionId);
|
|
1023
|
+
state.recoveryMarks.delete(sessionId);
|
|
1024
|
+
return;
|
|
1025
|
+
}
|
|
1026
|
+
if (catalogue.recoveryRevision === undefined) return;
|
|
1027
|
+
const [projection, agentEvents] = await Promise.all([
|
|
1028
|
+
this.stores.projections.loadHead!(state.storageKey, sessionId),
|
|
1029
|
+
this.stores.agentEvents.loadHead!(state.storageKey, sessionId),
|
|
1030
|
+
]);
|
|
1031
|
+
if (!projection?.incarnationId || !agentEvents?.incarnationId) return;
|
|
1032
|
+
await local.publishSessionClean!(state.storageKey, catalogue.recoveryRevision, {
|
|
1033
|
+
sessionId,
|
|
1034
|
+
state: 'clean',
|
|
1035
|
+
...requireSessionGeneration(session),
|
|
1036
|
+
sessionDigest: flexSessionDigest(session),
|
|
1037
|
+
projection: { incarnationId: projection.incarnationId, revision: projection.revision,
|
|
1038
|
+
contentRoot: projection.contentRoot },
|
|
1039
|
+
agentEvents: { incarnationId: agentEvents.incarnationId, revision: agentEvents.revision,
|
|
1040
|
+
contentRoot: agentEvents.contentRoot },
|
|
1041
|
+
});
|
|
1042
|
+
state.recoverySessionIds.delete(sessionId);
|
|
1043
|
+
state.recoveryMarks.delete(sessionId);
|
|
1044
|
+
});
|
|
1045
|
+
} catch (error) {
|
|
1046
|
+
// Another session can advance the namespace sidecar between our read and exact CAS.
|
|
1047
|
+
// This session remains durably dirty and another settled read can publish it.
|
|
1048
|
+
if (error instanceof FlexHarnessStoreConflictError) return;
|
|
1049
|
+
throw error;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
/** A generation may keep executing tools after its admission method has returned. */
|
|
1054
|
+
public admitRecoveryRun(state: IStorageState, sessionId: string, completion: Promise<unknown>): void {
|
|
1055
|
+
if (!state.sessionLocal) return;
|
|
1056
|
+
this.recoveryProducerGate(state, sessionId).admitRun(completion);
|
|
1057
|
+
}
|
|
1058
|
+
|
|
809
1059
|
/** Registers before the projection's pending intent can reach any workspace provider. */
|
|
810
1060
|
public async updateRecoveryObligation(state: IStorageState, stored: IStoredSessionState, registerOnly = false): Promise<void> {
|
|
811
1061
|
if (!state.sessionLocal) return;
|
|
@@ -816,21 +1066,15 @@ export class FlexHarnessCore<TScope> {
|
|
|
816
1066
|
|| stored.metrics.pendingCompactionId !== undefined
|
|
817
1067
|
|| (stored.recovering === true && state.recoverySessionIds.has(sessionId))
|
|
818
1068
|
|| (this.turnReversionProvider !== undefined && active !== undefined && !active.finishedEventEmitted);
|
|
819
|
-
if (registerOnly && !required) return;
|
|
820
1069
|
if (state.qualifyingLegacy) {
|
|
821
1070
|
if (required) state.recoverySessionIds.add(sessionId);
|
|
822
|
-
else state.recoverySessionIds.delete(sessionId);
|
|
1071
|
+
else if (!registerOnly) state.recoverySessionIds.delete(sessionId);
|
|
823
1072
|
return;
|
|
824
1073
|
}
|
|
825
|
-
//
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
else state.recoverySessionIds.delete(sessionId);
|
|
830
|
-
} catch (error) {
|
|
831
|
-
state.lifecycle = 'fenced';
|
|
832
|
-
throw error;
|
|
833
|
-
}
|
|
1074
|
+
// The quiescent settlement path clears marks after the event, job, permission and metadata
|
|
1075
|
+
// writers have settled.
|
|
1076
|
+
if (!required) return;
|
|
1077
|
+
await this.markRecoveryRequired(state, sessionId);
|
|
834
1078
|
}
|
|
835
1079
|
|
|
836
1080
|
/**
|
|
@@ -843,6 +1087,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
843
1087
|
if (!state.sessionLocal || state.lifecycle !== 'active') return;
|
|
844
1088
|
try {
|
|
845
1089
|
await this.updateRecoveryObligation(state, stored);
|
|
1090
|
+
await this.settleQuiescentSessionRecovery(state, stored.session.sessionId);
|
|
846
1091
|
} catch (error) {
|
|
847
1092
|
this.lifecycle.deferNamespaceDrain(state, error);
|
|
848
1093
|
}
|
|
@@ -963,8 +1208,24 @@ export class FlexHarnessCore<TScope> {
|
|
|
963
1208
|
protocolVersion: 1,
|
|
964
1209
|
}));
|
|
965
1210
|
const permission = permissionResult.value;
|
|
966
|
-
const
|
|
967
|
-
const
|
|
1211
|
+
const rawEventStore = eventStoreResult.value;
|
|
1212
|
+
const eventStore: plugins.IAgentEventStore = {
|
|
1213
|
+
load: (id) => rawEventStore.load(id),
|
|
1214
|
+
...(rawEventStore.loadHead ? { loadHead: (id: string) => rawEventStore.loadHead!(id) } : {}),
|
|
1215
|
+
save: (id, change, expectedRevision) => this.withRecoveryProducer(state, sessionId,
|
|
1216
|
+
() => rawEventStore.save(id, change, expectedRevision)),
|
|
1217
|
+
...(rawEventStore.archive ? { archive: (archive: plugins.IAgentEventArchive) =>
|
|
1218
|
+
this.withRecoveryProducer(state, sessionId, () => rawEventStore.archive!(archive)) } : {}),
|
|
1219
|
+
...(rawEventStore.loadArchive ? {
|
|
1220
|
+
loadArchive: (id: string, archiveId: string) => rawEventStore.loadArchive!(id, archiveId),
|
|
1221
|
+
} : {}),
|
|
1222
|
+
};
|
|
1223
|
+
const rawJobStore = jobStoreResult.value;
|
|
1224
|
+
const jobStore: plugins.IToolJobStore = {
|
|
1225
|
+
load: (abortSignal) => rawJobStore.load(abortSignal),
|
|
1226
|
+
save: (jobs, expectedRevision, abortSignal) => this.withRecoveryProducer(state, sessionId,
|
|
1227
|
+
() => rawJobStore.save(jobs, expectedRevision, abortSignal)),
|
|
1228
|
+
};
|
|
968
1229
|
let executionContextHandle: IFlexExecutionContextHandle | undefined;
|
|
969
1230
|
try {
|
|
970
1231
|
if (this.executionContextProvider) {
|
|
@@ -1065,19 +1326,30 @@ export class FlexHarnessCore<TScope> {
|
|
|
1065
1326
|
} = this.agentSessionPolicy;
|
|
1066
1327
|
const executionContext = executionContextHandle?.context;
|
|
1067
1328
|
const jobContext = executionContext?.jobs;
|
|
1068
|
-
const
|
|
1329
|
+
const guardedJobs: NonNullable<plugins.IToolExecutionContext['jobs']> | undefined = jobContext
|
|
1069
1330
|
? {
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1331
|
+
start: (request, invocationOptions) => this.withRecoveryProducer(state, sessionId,
|
|
1332
|
+
async () => {
|
|
1333
|
+
const handle = await jobContext.start(request, invocationOptions);
|
|
1334
|
+
if (state.sessionLocal) {
|
|
1335
|
+
this.recoveryProducerGate(state, sessionId).admitBackgroundJob(handle.completion);
|
|
1336
|
+
}
|
|
1337
|
+
return handle;
|
|
1338
|
+
}),
|
|
1339
|
+
get: jobContext.get.bind(jobContext),
|
|
1340
|
+
list: jobContext.list.bind(jobContext),
|
|
1341
|
+
...(jobContext.abort ? { abort: (executionId: string) => this.withRecoveryProducer(
|
|
1342
|
+
state, sessionId, () => jobContext.abort!(executionId)) } : {}),
|
|
1343
|
+
...(jobContext.subscribe ? {
|
|
1076
1344
|
subscribe: (listener: Parameters<NonNullable<typeof jobContext.subscribe>>[0]) =>
|
|
1077
1345
|
jobContext.subscribe!((event) =>
|
|
1078
|
-
this.
|
|
1079
|
-
|
|
1346
|
+
this.withRecoveryProducer(state, sessionId, () => Promise.resolve(
|
|
1347
|
+
this.compaction.withCompactorContext(compactorContext, () => listener(event))))),
|
|
1348
|
+
} : {}),
|
|
1080
1349
|
}
|
|
1350
|
+
: undefined;
|
|
1351
|
+
const contextualExecutionContext = executionContext && guardedJobs
|
|
1352
|
+
? { ...executionContext, jobs: guardedJobs }
|
|
1081
1353
|
: executionContext;
|
|
1082
1354
|
const agentSessionOptions: plugins.IAgentSessionOptions & {
|
|
1083
1355
|
transactionOutcomeErrorProjector: (error: unknown) => string;
|
|
@@ -1371,12 +1643,8 @@ export class FlexHarnessCore<TScope> {
|
|
|
1371
1643
|
}
|
|
1372
1644
|
freezeProjectionEntries(stored);
|
|
1373
1645
|
const change = this.createProjectionChange(stored, expected + 1);
|
|
1374
|
-
await this.
|
|
1375
|
-
storageKey,
|
|
1376
|
-
stored.session.sessionId,
|
|
1377
|
-
change,
|
|
1378
|
-
expected,
|
|
1379
|
-
);
|
|
1646
|
+
await this.withRecoveryProducer(state, stored.session.sessionId, () =>
|
|
1647
|
+
this.stores.projections.save(storageKey, stored.session.sessionId, change, expected));
|
|
1380
1648
|
stored.projectionSchemaVersion = 3;
|
|
1381
1649
|
delete stored.projectionBaseline;
|
|
1382
1650
|
this.rememberSavedProjectionLists(stored);
|
|
@@ -1805,9 +2073,41 @@ export class FlexHarnessCore<TScope> {
|
|
|
1805
2073
|
throw error;
|
|
1806
2074
|
}
|
|
1807
2075
|
const snapshot = this.createScopeSnapshot(state, beforeRevision + 1);
|
|
2076
|
+
const affectedOwners = new Set([
|
|
2077
|
+
...beforeCatalogue.keys(), ...beforeTombstones.keys(),
|
|
2078
|
+
...state.catalogue.keys(), ...state.tombstones.keys(),
|
|
2079
|
+
]);
|
|
2080
|
+
for (const id of [...affectedOwners]) {
|
|
2081
|
+
if (canonicalJson(beforeCatalogue.get(id)) === canonicalJson(state.catalogue.get(id))
|
|
2082
|
+
&& canonicalJson(beforeTombstones.get(id)) === canonicalJson(state.tombstones.get(id))) {
|
|
2083
|
+
affectedOwners.delete(id);
|
|
2084
|
+
}
|
|
2085
|
+
}
|
|
2086
|
+
return this.withRecoveryProducerSections(state, affectedOwners, async () => {
|
|
1808
2087
|
try {
|
|
2088
|
+
// An existing owner must be dirty before its catalogue metadata or tombstone changes.
|
|
2089
|
+
// A new owner is published by this CAS first: marking it before publication could leave
|
|
2090
|
+
// an orphan sidecar ID if the CAS never commits.
|
|
2091
|
+
const changedOwners = new Set([...beforeCatalogue.keys(), ...beforeTombstones.keys()]);
|
|
2092
|
+
for (const id of changedOwners) {
|
|
2093
|
+
const wasLive = beforeCatalogue.has(id);
|
|
2094
|
+
const isLive = state.catalogue.has(id);
|
|
2095
|
+
const isTombstoned = state.tombstones.has(id);
|
|
2096
|
+
if (!wasLive && !isLive && !isTombstoned) continue; // completed tombstone removal
|
|
2097
|
+
if (canonicalJson(beforeCatalogue.get(id)) !== canonicalJson(state.catalogue.get(id))
|
|
2098
|
+
|| canonicalJson(beforeTombstones.get(id)) !== canonicalJson(state.tombstones.get(id))) {
|
|
2099
|
+
await this.markRecoveryRequired(state, id);
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
1809
2102
|
await this.saveScopeSnapshot(state, snapshot, beforeRevision);
|
|
1810
2103
|
state.revision = snapshot.revision;
|
|
2104
|
+
// The scope CAS creates the owner. Only now can its durable dirty bit be published without
|
|
2105
|
+
// making an orphan obligation if that CAS was refused.
|
|
2106
|
+
for (const id of new Set([...state.catalogue.keys(), ...state.tombstones.keys()])) {
|
|
2107
|
+
if (!beforeCatalogue.has(id) && !beforeTombstones.has(id)) {
|
|
2108
|
+
await this.markRecoveryRequired(state, id);
|
|
2109
|
+
}
|
|
2110
|
+
}
|
|
1811
2111
|
return result;
|
|
1812
2112
|
} catch (error) {
|
|
1813
2113
|
if (error instanceof FlexHarnessStoreConflictError) {
|
|
@@ -1826,13 +2126,13 @@ export class FlexHarnessCore<TScope> {
|
|
|
1826
2126
|
} catch (loadError) {
|
|
1827
2127
|
reconciliationError = loadError;
|
|
1828
2128
|
}
|
|
1829
|
-
if (current &&
|
|
2129
|
+
if (current && canonicalJson(current) === canonicalJson(snapshot)) {
|
|
1830
2130
|
state.revision = snapshot.revision;
|
|
1831
2131
|
throw error;
|
|
1832
2132
|
}
|
|
1833
2133
|
if (
|
|
1834
2134
|
(current === undefined && beforeRevision === 0)
|
|
1835
|
-
|| (current !== undefined &&
|
|
2135
|
+
|| (current !== undefined && canonicalJson(current) === canonicalJson(beforeSnapshot()))
|
|
1836
2136
|
) {
|
|
1837
2137
|
restore();
|
|
1838
2138
|
throw error;
|
|
@@ -1843,6 +2143,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1843
2143
|
? error
|
|
1844
2144
|
: combineErrors([error, reconciliationError]);
|
|
1845
2145
|
}
|
|
2146
|
+
});
|
|
1846
2147
|
}
|
|
1847
2148
|
|
|
1848
2149
|
/** Holds a mutation declared for one session to that session: no other session, tombstone or cleanup changed. */
|
|
@@ -1890,7 +2191,8 @@ export class FlexHarnessCore<TScope> {
|
|
|
1890
2191
|
_allowDuringDrain = false,
|
|
1891
2192
|
onSaved?: TProjectionSaved,
|
|
1892
2193
|
): Promise<TValue> {
|
|
1893
|
-
return this.enqueueProjectionOperation(stored, () => this.
|
|
2194
|
+
return this.enqueueProjectionOperation(stored, () => this.withRecoveryProducer(
|
|
2195
|
+
state, stored.session.sessionId, () => this.saveProjectionMutation(state, stored, mutation, onSaved)));
|
|
1894
2196
|
}
|
|
1895
2197
|
|
|
1896
2198
|
/**
|
|
@@ -1907,7 +2209,8 @@ export class FlexHarnessCore<TScope> {
|
|
|
1907
2209
|
return this.enqueueProjectionOperation(stored, async () => {
|
|
1908
2210
|
const mutation = prepare();
|
|
1909
2211
|
if (!mutation) return false;
|
|
1910
|
-
await this.
|
|
2212
|
+
await this.withRecoveryProducer(state, stored.session.sessionId,
|
|
2213
|
+
() => this.saveProjectionMutation(state, stored, mutation, onSaved));
|
|
1911
2214
|
return true;
|
|
1912
2215
|
});
|
|
1913
2216
|
}
|
|
@@ -1983,7 +2286,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1983
2286
|
freezeProjectionEntries(stored);
|
|
1984
2287
|
const revision = beforeRevision + 1;
|
|
1985
2288
|
try {
|
|
1986
|
-
|
|
2289
|
+
await this.markRecoveryRequired(state, stored.session.sessionId);
|
|
1987
2290
|
const saveStartedAt = onSaved ? this.checkpointClock.now() : 0;
|
|
1988
2291
|
await this.stores.projections.save(
|
|
1989
2292
|
state.storageKey,
|