@modelprofile.com/flexharness 18.1.0 → 18.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.flexharness.cachewarm.js +3 -3
- package/dist_ts/classes.flexharness.callbacks.d.ts +37 -0
- package/dist_ts/classes.flexharness.callbacks.js +129 -2
- package/dist_ts/classes.flexharness.compaction.js +2 -2
- package/dist_ts/classes.flexharness.configurations.js +4 -4
- package/dist_ts/classes.flexharness.core.d.ts +39 -4
- package/dist_ts/classes.flexharness.core.js +467 -134
- package/dist_ts/classes.flexharness.d.ts +4 -0
- package/dist_ts/classes.flexharness.deletion.d.ts +13 -0
- package/dist_ts/classes.flexharness.deletion.js +132 -48
- package/dist_ts/classes.flexharness.imports.js +26 -21
- package/dist_ts/classes.flexharness.js +9 -1
- package/dist_ts/classes.flexharness.lifecycle.js +9 -3
- package/dist_ts/classes.flexharness.projects.js +2 -2
- package/dist_ts/classes.flexharness.prompts.js +14 -8
- package/dist_ts/classes.flexharness.reversions.js +3 -3
- package/dist_ts/classes.flexharness.runs.js +18 -2
- package/dist_ts/classes.flexharness.sessions.d.ts +1 -1
- package/dist_ts/classes.flexharness.sessions.js +18 -18
- package/dist_ts/classes.flexharness.slashcommands.js +3 -3
- package/dist_ts/classes.flexharness.subagents.js +37 -16
- package/dist_ts/classes.flexharness.tooldispatch.js +3 -3
- package/dist_ts/classes.scoperecovery.d.ts +37 -0
- package/dist_ts/classes.scoperecovery.js +229 -0
- package/dist_ts/classes.stores.d.ts +14 -1
- package/dist_ts/classes.stores.js +150 -17
- package/dist_ts/index.d.ts +1 -0
- package/dist_ts/index.js +2 -1
- package/dist_ts/interfaces.d.ts +113 -2
- package/dist_ts/interfaces.flexharness.d.ts +41 -5
- package/dist_ts/interfaces.js +1 -1
- package/dist_ts/utils.harnessoptions.d.ts +2 -0
- package/dist_ts/utils.harnessoptions.js +16 -11
- package/dist_ts_stores_nosqldb/classes.nosqldbstores.d.ts +3 -2
- package/dist_ts_stores_nosqldb/classes.nosqldbstores.js +33 -2
- package/dist_ts_stores_nosqldb/classes.recordstore.d.ts +10 -1
- package/dist_ts_stores_nosqldb/classes.recordstore.js +75 -7
- package/package.json +3 -3
- package/readme.md +43 -5
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.flexharness.cachewarm.ts +2 -2
- package/ts/classes.flexharness.callbacks.ts +142 -1
- package/ts/classes.flexharness.compaction.ts +1 -1
- package/ts/classes.flexharness.configurations.ts +3 -3
- package/ts/classes.flexharness.core.ts +463 -133
- package/ts/classes.flexharness.deletion.ts +178 -62
- package/ts/classes.flexharness.imports.ts +26 -21
- package/ts/classes.flexharness.lifecycle.ts +8 -2
- package/ts/classes.flexharness.projects.ts +1 -1
- package/ts/classes.flexharness.prompts.ts +11 -7
- package/ts/classes.flexharness.reversions.ts +2 -2
- package/ts/classes.flexharness.runs.ts +16 -1
- package/ts/classes.flexharness.sessions.ts +17 -17
- package/ts/classes.flexharness.slashcommands.ts +2 -2
- package/ts/classes.flexharness.subagents.ts +40 -15
- package/ts/classes.flexharness.tooldispatch.ts +2 -2
- package/ts/classes.flexharness.ts +13 -0
- package/ts/classes.scoperecovery.ts +291 -0
- package/ts/classes.stores.ts +150 -16
- package/ts/index.ts +2 -0
- package/ts/interfaces.flexharness.ts +44 -4
- package/ts/interfaces.ts +124 -2
- package/ts/readme.md +43 -5
- package/ts/utils.harnessoptions.ts +18 -12
- package/ts_stores_nosqldb/classes.nosqldbstores.ts +52 -2
- package/ts_stores_nosqldb/classes.recordstore.ts +78 -8
- package/ts_stores_nosqldb/readme.md +10 -0
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
} from './errors.js';
|
|
15
15
|
import type {
|
|
16
16
|
IFlexAgentSessionPolicy,
|
|
17
|
+
IFlexCacheWarmClock,
|
|
17
18
|
IFlexCallbackLimits,
|
|
18
19
|
IFlexExecutionContextHandle,
|
|
19
20
|
IFlexHarnessOptions,
|
|
@@ -66,6 +67,7 @@ import {
|
|
|
66
67
|
normalizeAttachmentResolver,
|
|
67
68
|
normalizeBuiltInTools,
|
|
68
69
|
normalizeCacheWarmOptions,
|
|
70
|
+
normalizeClock,
|
|
69
71
|
normalizeSlashCommands,
|
|
70
72
|
normalizeToolCatalogOptions,
|
|
71
73
|
normalizeSubagents,
|
|
@@ -104,6 +106,7 @@ import type {
|
|
|
104
106
|
IProjectionContent,
|
|
105
107
|
IScopeLoadPass,
|
|
106
108
|
TEventDetails,
|
|
109
|
+
TProjectionSaved,
|
|
107
110
|
} from './interfaces.flexharness.js';
|
|
108
111
|
import {
|
|
109
112
|
scopeRetirementMessage,
|
|
@@ -155,6 +158,8 @@ export class FlexHarnessCore<TScope> {
|
|
|
155
158
|
/** The URL protocols, colon included, of the attachment references prompts may carry. */
|
|
156
159
|
public readonly attachmentReferenceProtocols: ReadonlySet<string>;
|
|
157
160
|
public readonly stores: IFlexHarnessStores;
|
|
161
|
+
public readonly sessionLoading: 'eager' | 'session-local';
|
|
162
|
+
private readonly legacyQualifications = new Set<string>();
|
|
158
163
|
public readonly agentSessionPolicy: IFlexAgentSessionPolicy<TScope>;
|
|
159
164
|
public readonly toolOutputLimits: Required<NonNullable<IFlexHarnessOptions<TScope>['toolOutputLimits']>>;
|
|
160
165
|
public readonly callbackLimits: Required<IFlexCallbackLimits>;
|
|
@@ -170,6 +175,8 @@ export class FlexHarnessCore<TScope> {
|
|
|
170
175
|
public readonly subagents: ReadonlyMap<string, Readonly<IFlexSubagentDefinition>>;
|
|
171
176
|
public readonly builtInTools: INormalizedBuiltInTools;
|
|
172
177
|
public readonly cacheWarmOptions: INormalizedCacheWarmOptions;
|
|
178
|
+
/** The clock of running turns' projection checkpoints: their spacing and save timings. */
|
|
179
|
+
public readonly checkpointClock: IFlexCacheWarmClock;
|
|
173
180
|
public readonly toolCatalogOptions: Readonly<IFlexToolCatalogOptions>;
|
|
174
181
|
public readonly slashCommands: ReadonlyMap<string, TRegisteredSlashCommand<TScope>>;
|
|
175
182
|
public readonly maxSubagentDepth: number;
|
|
@@ -217,6 +224,13 @@ export class FlexHarnessCore<TScope> {
|
|
|
217
224
|
this.attachmentReferenceProtocols = normalizeAttachmentResolver(options.attachmentResolver);
|
|
218
225
|
this.attachmentResolver = options.attachmentResolver;
|
|
219
226
|
this.stores = requireHarnessStores(options.stores ?? new InMemoryFlexHarnessStores());
|
|
227
|
+
this.sessionLoading = options.sessionLoading ?? 'eager';
|
|
228
|
+
if (!['eager', 'session-local'].includes(this.sessionLoading)) {
|
|
229
|
+
throw new FlexHarnessValidationError('sessionLoading must be "eager" or "session-local".');
|
|
230
|
+
}
|
|
231
|
+
if (this.sessionLoading === 'session-local' && !this.stores.scopes.sessionLocal) {
|
|
232
|
+
throw new FlexHarnessValidationError('session-local loading requires scopes.sessionLocal capability.');
|
|
233
|
+
}
|
|
220
234
|
this.agentSessionPolicy = normalizeAgentSessionPolicy(options.agentSessionPolicy);
|
|
221
235
|
this.toolOutputLimits = resolveJsonLimits(options.toolOutputLimits);
|
|
222
236
|
this.callbackLimits = resolveCallbackLimits(options.callbackLimits);
|
|
@@ -252,6 +266,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
252
266
|
this.subagents = normalizeSubagents(options.subagents);
|
|
253
267
|
this.builtInTools = normalizeBuiltInTools(options.builtInTools);
|
|
254
268
|
this.cacheWarmOptions = normalizeCacheWarmOptions(options.cacheWarm);
|
|
269
|
+
this.checkpointClock = normalizeClock(options.projectionCheckpointClock, 'projectionCheckpointClock');
|
|
255
270
|
const toolCatalogOptions = normalizeToolCatalogOptions(options.toolCatalog);
|
|
256
271
|
this.toolCatalogOptions = Object.freeze({
|
|
257
272
|
declarations: toolCatalogOptions.declarations,
|
|
@@ -414,6 +429,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
414
429
|
return state;
|
|
415
430
|
} catch (error) {
|
|
416
431
|
if (compactorLifecycleController.signal.aborted) outcome = 'aborted';
|
|
432
|
+
this.stores.scopes.sessionLocal?.releaseOwnership(storageKey);
|
|
417
433
|
throw error;
|
|
418
434
|
} finally {
|
|
419
435
|
this.observeScopeLoad({ ...pass.metrics, totalMs: performance.now() - startedAt, outcome });
|
|
@@ -435,17 +451,32 @@ export class FlexHarnessCore<TScope> {
|
|
|
435
451
|
pass: IScopeLoadPass,
|
|
436
452
|
): Promise<IStorageState> {
|
|
437
453
|
const readStartedAt = performance.now();
|
|
438
|
-
|
|
454
|
+
if (this.sessionLoading === 'eager') this.stores.scopes.sessionLocal?.releaseOwnership(storageKey);
|
|
455
|
+
const local = this.sessionLoading === 'session-local'
|
|
456
|
+
? await this.stores.scopes.sessionLocal!.loadCatalogue(storageKey)
|
|
457
|
+
: undefined;
|
|
458
|
+
const qualifyingLegacy = local !== undefined && !local.qualified && this.legacyQualifications.has(storageKey);
|
|
459
|
+
if (local && !local.qualified && !qualifyingLegacy) {
|
|
460
|
+
throw new FlexHarnessValidationError(
|
|
461
|
+
'Session-local catalogue requires explicit prepareScope({ qualifyLegacy: true }) under exclusive namespace ownership.',
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
const stored = local ? local.snapshot : await this.stores.scopes.load(storageKey);
|
|
439
465
|
pass.metrics.storeReadMs += performance.now() - readStartedAt;
|
|
440
466
|
compactorLifecycleController.signal.throwIfAborted();
|
|
441
467
|
const { snapshot, changed } = normalizeScopeSnapshot(storageKey, stored);
|
|
442
|
-
let scopeChanged = changed;
|
|
468
|
+
let scopeChanged = changed || (qualifyingLegacy && stored === undefined);
|
|
443
469
|
const state: IStorageState = {
|
|
444
470
|
storageKey,
|
|
445
471
|
compactorLifecycleController,
|
|
446
472
|
scopeIdHint: scopeId,
|
|
447
473
|
scopeContext: { scopeId, scope },
|
|
448
474
|
revision: snapshot.revision,
|
|
475
|
+
catalogue: new Map(liveScopeSessions(snapshot).map((session) => [session.sessionId, session])),
|
|
476
|
+
sessionLocal: local !== undefined,
|
|
477
|
+
qualifyingLegacy,
|
|
478
|
+
recoverySessionIds: new Set(local?.recoverySessionIds ?? []),
|
|
479
|
+
sessionLoads: new Map(),
|
|
449
480
|
sessions: new Map(),
|
|
450
481
|
retainedSessionCleanups: new Map(),
|
|
451
482
|
sessionDeletions: new Map(),
|
|
@@ -466,11 +497,18 @@ export class FlexHarnessCore<TScope> {
|
|
|
466
497
|
fenceAdditionalErrors: [],
|
|
467
498
|
detachedCleanups: new Set(),
|
|
468
499
|
};
|
|
500
|
+
for (const sessionId of state.recoverySessionIds) {
|
|
501
|
+
if (!state.catalogue.has(sessionId) && !state.tombstones.has(sessionId)) {
|
|
502
|
+
throw new FlexHarnessValidationError(`Recovery obligation "${sessionId}" has no canonical session or tombstone owner.`);
|
|
503
|
+
}
|
|
504
|
+
}
|
|
469
505
|
const loadedSessions: IStoredSessionState[] = [];
|
|
470
|
-
//
|
|
506
|
+
// Eager qualification still joins every runtime; the independent catalogue is already complete.
|
|
471
507
|
let sessionsLoaded = false;
|
|
472
508
|
try {
|
|
473
|
-
|
|
509
|
+
// An authoritative recovery obligation gates the namespace; unrelated sessions stay cold.
|
|
510
|
+
for (const metadata of liveScopeSessions(snapshot).filter((session) =>
|
|
511
|
+
!state.sessionLocal || qualifyingLegacy || state.recoverySessionIds.has(session.sessionId))) {
|
|
474
512
|
compactorLifecycleController.signal.throwIfAborted();
|
|
475
513
|
const stored = await this.loadSessionRuntime(
|
|
476
514
|
state,
|
|
@@ -497,26 +535,27 @@ export class FlexHarnessCore<TScope> {
|
|
|
497
535
|
} finally {
|
|
498
536
|
pass.metrics.repairMs += performance.now() - repairStartedAt;
|
|
499
537
|
}
|
|
500
|
-
|
|
538
|
+
state.catalogue.set(metadata.sessionId, stored.session);
|
|
539
|
+
await this.reversions.drainReversionReleases(state, stored, scopeId, scope, state.sessionLocal);
|
|
501
540
|
if (stored.projectionRevision < 0) throw new Error('Invalid projection revision.');
|
|
502
541
|
}
|
|
503
542
|
sessionsLoaded = true;
|
|
504
|
-
for (const
|
|
505
|
-
let ancestorId =
|
|
506
|
-
const visited = new Set<string>([
|
|
543
|
+
for (const metadata of state.catalogue.values()) {
|
|
544
|
+
let ancestorId = metadata.parentSessionId;
|
|
545
|
+
const visited = new Set<string>([metadata.sessionId]);
|
|
507
546
|
while (ancestorId) {
|
|
508
547
|
if (visited.has(ancestorId)) {
|
|
509
548
|
throw new FlexHarnessValidationError(
|
|
510
|
-
`Live session "${
|
|
549
|
+
`Live session "${metadata.sessionId}" has a cyclic ancestor chain.`,
|
|
511
550
|
);
|
|
512
551
|
}
|
|
513
552
|
visited.add(ancestorId);
|
|
514
553
|
if (state.tombstones.has(ancestorId)) {
|
|
515
554
|
throw new FlexHarnessValidationError(
|
|
516
|
-
`Live session "${
|
|
555
|
+
`Live session "${metadata.sessionId}" descends from tombstoned ancestor "${ancestorId}".`,
|
|
517
556
|
);
|
|
518
557
|
}
|
|
519
|
-
ancestorId = state.
|
|
558
|
+
ancestorId = state.catalogue.get(ancestorId)?.parentSessionId;
|
|
520
559
|
}
|
|
521
560
|
}
|
|
522
561
|
const tombstoneRoots = this.deletion.orderTombstoneRootsChildFirst(
|
|
@@ -538,6 +577,12 @@ export class FlexHarnessCore<TScope> {
|
|
|
538
577
|
);
|
|
539
578
|
await this.deletion.cleanupSessionDomains(state, tombstone);
|
|
540
579
|
}
|
|
580
|
+
for (const tombstone of group) {
|
|
581
|
+
if (state.sessionLocal && !state.qualifyingLegacy) {
|
|
582
|
+
await this.stores.scopes.sessionLocal!.setRecoveryRequired(storageKey, tombstone.sessionId, false);
|
|
583
|
+
}
|
|
584
|
+
state.recoverySessionIds.delete(tombstone.sessionId);
|
|
585
|
+
}
|
|
541
586
|
for (const tombstone of group) state.tombstones.delete(tombstone.sessionId);
|
|
542
587
|
scopeChanged = true;
|
|
543
588
|
} catch (error) {
|
|
@@ -550,13 +595,24 @@ export class FlexHarnessCore<TScope> {
|
|
|
550
595
|
}
|
|
551
596
|
if (scopeChanged) {
|
|
552
597
|
const next = this.createScopeSnapshot(state, state.revision + 1);
|
|
553
|
-
await this.
|
|
598
|
+
await this.saveScopeSnapshot(state, next, state.revision);
|
|
554
599
|
state.revision = next.revision;
|
|
555
600
|
}
|
|
601
|
+
if (state.sessionLocal) {
|
|
602
|
+
if (qualifyingLegacy) {
|
|
603
|
+
await this.stores.scopes.sessionLocal!.qualifyCatalogue(storageKey,
|
|
604
|
+
this.createScopeSnapshot(state, state.revision), [...state.recoverySessionIds]);
|
|
605
|
+
state.qualifyingLegacy = false;
|
|
606
|
+
}
|
|
607
|
+
for (const loaded of loadedSessions) {
|
|
608
|
+
loaded.recovering = false;
|
|
609
|
+
await this.updateRecoveryObligation(state, loaded);
|
|
610
|
+
}
|
|
611
|
+
}
|
|
556
612
|
return state;
|
|
557
613
|
} catch (error) {
|
|
558
|
-
//
|
|
559
|
-
//
|
|
614
|
+
// No tombstone was touched until the selected recovery/qualification sessions all loaded.
|
|
615
|
+
// Retain cleanup owners only after that boundary; the catalogue stays complete throughout.
|
|
560
616
|
if (sessionsLoaded) {
|
|
561
617
|
for (const rootSessionId of new Set([...state.tombstones.values()].map((tombstone) =>
|
|
562
618
|
tombstone.rootSessionId ?? tombstone.sessionId))) {
|
|
@@ -586,6 +642,208 @@ export class FlexHarnessCore<TScope> {
|
|
|
586
642
|
}
|
|
587
643
|
}
|
|
588
644
|
|
|
645
|
+
/** Enumerates metadata/proof only; explicit prepareScope owns startup recovery. */
|
|
646
|
+
public async getScopeCatalogue(scopeId: string): Promise<import('./interfaces.js').IFlexScopeCatalogue> {
|
|
647
|
+
return this.withScopeAdmission(scopeId, async (scope, admitted) => {
|
|
648
|
+
const load = this.stateLoads.get(scope.storageKey);
|
|
649
|
+
if (load && this.isStateLoaded(load)) {
|
|
650
|
+
const state = await this.admittedState(scopeId, scope, load, admitted);
|
|
651
|
+
await state.scopeQueue;
|
|
652
|
+
this.assertStateAcceptingWork(state);
|
|
653
|
+
return publicSnapshot({
|
|
654
|
+
sessions: [...state.catalogue.values()]
|
|
655
|
+
.filter((session) => !state.initializingSessions.has(session.sessionId))
|
|
656
|
+
.sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
|
|
657
|
+
recoverySessionIds: [...state.recoverySessionIds].sort(),
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
if (this.sessionLoading !== 'session-local') {
|
|
661
|
+
await this.resolveState(scopeId);
|
|
662
|
+
return this.getScopeCatalogue(scopeId);
|
|
663
|
+
}
|
|
664
|
+
try {
|
|
665
|
+
const catalogue = await this.stores.scopes.sessionLocal!.loadCatalogue(scope.storageKey);
|
|
666
|
+
this.assertOpen();
|
|
667
|
+
if (!admitted()) throw this.createScopeRetirementError();
|
|
668
|
+
if (!catalogue.qualified) {
|
|
669
|
+
throw new FlexHarnessValidationError('Session-local catalogue requires explicit legacy qualification.');
|
|
670
|
+
}
|
|
671
|
+
const { snapshot } = normalizeScopeSnapshot(scope.storageKey, catalogue.snapshot);
|
|
672
|
+
return publicSnapshot({ sessions: liveScopeSessions(snapshot).sort((left, right) =>
|
|
673
|
+
left.sessionId.localeCompare(right.sessionId)), recoverySessionIds: catalogue.recoverySessionIds.slice().sort() });
|
|
674
|
+
} finally {
|
|
675
|
+
// A metadata-only reader owns no runtime lifecycle on which to retain writer activation.
|
|
676
|
+
if (!this.stateLoads.has(scope.storageKey)) this.stores.scopes.sessionLocal!.releaseOwnership(scope.storageKey);
|
|
677
|
+
}
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
public async prepareScope(
|
|
682
|
+
scopeId: string,
|
|
683
|
+
options: import('./interfaces.js').IFlexPrepareScopeOptions = {},
|
|
684
|
+
): Promise<import('./interfaces.js').IFlexScopeCatalogue> {
|
|
685
|
+
if (!options || typeof options !== 'object' || Array.isArray(options)
|
|
686
|
+
|| Object.keys(options).some((key) => key !== 'qualifyLegacy')
|
|
687
|
+
|| (options.qualifyLegacy !== undefined && typeof options.qualifyLegacy !== 'boolean')) {
|
|
688
|
+
throw new FlexHarnessValidationError('prepareScope options must contain only qualifyLegacy (boolean).');
|
|
689
|
+
}
|
|
690
|
+
return this.withScopeAdmission(scopeId, async (scope) => {
|
|
691
|
+
if (options.qualifyLegacy) this.legacyQualifications.add(scope.storageKey);
|
|
692
|
+
try {
|
|
693
|
+
await this.resolveState(scopeId);
|
|
694
|
+
return await this.getScopeCatalogue(scopeId);
|
|
695
|
+
} finally {
|
|
696
|
+
if (options.qualifyLegacy) this.legacyQualifications.delete(scope.storageKey);
|
|
697
|
+
}
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
public async hydrateSession(
|
|
702
|
+
state: IStorageState,
|
|
703
|
+
sessionId: string,
|
|
704
|
+
scopeId: string,
|
|
705
|
+
scope: TScope,
|
|
706
|
+
): Promise<IStoredSessionState> {
|
|
707
|
+
const pending = state.sessionLoads.get(sessionId);
|
|
708
|
+
if (pending) return pending;
|
|
709
|
+
const existing = state.sessions.get(sessionId);
|
|
710
|
+
if (existing) return existing;
|
|
711
|
+
let load!: Promise<IStoredSessionState>;
|
|
712
|
+
load = (async () => {
|
|
713
|
+
await this.sessionReadBarrier(state, sessionId);
|
|
714
|
+
this.assertStateAcceptingWork(state);
|
|
715
|
+
const metadata = state.catalogue.get(sessionId);
|
|
716
|
+
if (!metadata || state.tombstones.has(sessionId)) throw new FlexHarnessNotFoundError('Session', sessionId);
|
|
717
|
+
const pass: IScopeLoadPass = {
|
|
718
|
+
slicer: new FlexTimeSlicer(state.compactorLifecycleController.signal),
|
|
719
|
+
metrics: { scopeId, storageKey: state.storageKey, sessions: 1, storeReadMs: 0, agentSessionMs: 0, repairMs: 0 },
|
|
720
|
+
};
|
|
721
|
+
const stored = await this.loadSessionRuntime(state, metadata, scopeId, scope, undefined, false, pass);
|
|
722
|
+
try {
|
|
723
|
+
state.compactorLifecycleController.signal.throwIfAborted();
|
|
724
|
+
this.assertStateAcceptingWork(state);
|
|
725
|
+
const current = state.catalogue.get(sessionId);
|
|
726
|
+
if (!current || state.tombstones.has(sessionId)
|
|
727
|
+
|| current.sessionGenerationId !== metadata.sessionGenerationId
|
|
728
|
+
|| current.sessionGenerationSequence !== metadata.sessionGenerationSequence) {
|
|
729
|
+
throw new FlexHarnessAbortError('The session was deleted while it was being hydrated.');
|
|
730
|
+
}
|
|
731
|
+
// Runtime ownership is visible to drain before repair can acquire more resources.
|
|
732
|
+
state.sessions.set(sessionId, stored);
|
|
733
|
+
await this.updateRecoveryObligation(state, stored, true);
|
|
734
|
+
const changed = await this.repairLoadedSession(state.storageKey, stored, state, scopeId, scope, pass.slicer);
|
|
735
|
+
await this.reversions.drainReversionReleases(state, stored, scopeId, scope, state.sessionLocal);
|
|
736
|
+
if (changed) await this.mutateSessionScope(state, sessionId, () => undefined);
|
|
737
|
+
else state.catalogue.set(sessionId, stored.session);
|
|
738
|
+
stored.recovering = false;
|
|
739
|
+
await this.updateRecoveryObligation(state, stored);
|
|
740
|
+
this.assertStateAcceptingWork(state);
|
|
741
|
+
return stored;
|
|
742
|
+
} catch (error) {
|
|
743
|
+
if (state.sessions.get(sessionId) === stored) state.sessions.delete(sessionId);
|
|
744
|
+
try {
|
|
745
|
+
await this.lifecycle.closeStoredSession(stored);
|
|
746
|
+
} catch (closeError) {
|
|
747
|
+
this.lifecycle.orphanedStoredSessions.add(stored);
|
|
748
|
+
throw combineErrors([error, closeError]);
|
|
749
|
+
}
|
|
750
|
+
throw error;
|
|
751
|
+
}
|
|
752
|
+
})().finally(() => {
|
|
753
|
+
if (state.sessionLoads.get(sessionId) === load) state.sessionLoads.delete(sessionId);
|
|
754
|
+
});
|
|
755
|
+
state.sessionLoads.set(sessionId, load);
|
|
756
|
+
return load;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
/** Retries ownerless shared-workspace obligations, never an active run's normal capture. */
|
|
760
|
+
private async recoverSharedState(state: IStorageState, scopeId: string, scope: TScope): Promise<void> {
|
|
761
|
+
if (!state.sessionLocal) return;
|
|
762
|
+
if (state.sharedRecovery) return state.sharedRecovery;
|
|
763
|
+
const ids = [...state.recoverySessionIds].filter((id) =>
|
|
764
|
+
!state.activeRuns.has(id) && !state.initializingSessions.has(id)
|
|
765
|
+
&& !(state.sessions.get(id)?.outstandingPromptsById.size));
|
|
766
|
+
const roots = [...new Set([...state.tombstones.values()].map((entry) => entry.rootSessionId ?? entry.sessionId))]
|
|
767
|
+
.filter((id) => !state.sessionDeletions.has(id) && !state.tombstoneCleanups.has(id)
|
|
768
|
+
&& !state.initializingSessions.has(id));
|
|
769
|
+
if (ids.length === 0 && roots.length === 0) return;
|
|
770
|
+
let recovery!: Promise<void>;
|
|
771
|
+
recovery = (async () => {
|
|
772
|
+
for (const id of ids) {
|
|
773
|
+
if (state.tombstones.has(id)) continue;
|
|
774
|
+
if (!state.catalogue.has(id)) {
|
|
775
|
+
throw new FlexHarnessValidationError(`Recovery obligation "${id}" has no canonical session or tombstone owner.`);
|
|
776
|
+
}
|
|
777
|
+
if (state.sessionLoads.has(id)) { await state.sessionLoads.get(id); continue; }
|
|
778
|
+
const stored = state.sessions.get(id);
|
|
779
|
+
if (!stored) { await this.hydrateSession(state, id, scopeId, scope); continue; }
|
|
780
|
+
stored.recovering = true;
|
|
781
|
+
await this.repairLoadedSession(state.storageKey, stored, state, scopeId, scope,
|
|
782
|
+
new FlexTimeSlicer(state.compactorLifecycleController.signal));
|
|
783
|
+
await this.reversions.drainReversionReleases(state, stored, scopeId, scope, true);
|
|
784
|
+
// A preceding failed recovery may already have repaired metadata only in memory.
|
|
785
|
+
await this.mutateSessionScope(state, id, () => undefined);
|
|
786
|
+
stored.recovering = false;
|
|
787
|
+
await this.updateRecoveryObligation(state, stored);
|
|
788
|
+
}
|
|
789
|
+
for (const root of roots) await this.deletion.finishTombstoneCleanup(state, root, scopeId, { scopeId, scope });
|
|
790
|
+
})().finally(() => { if (state.sharedRecovery === recovery) delete state.sharedRecovery; });
|
|
791
|
+
state.sharedRecovery = recovery;
|
|
792
|
+
return recovery;
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
public async hydrateAllSessions(state: IStorageState, scopeId: string, scope: TScope): Promise<void> {
|
|
796
|
+
for (const sessionId of state.catalogue.keys()) await this.hydrateSession(state, sessionId, scopeId, scope);
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
private async saveScopeSnapshot(state: IStorageState, snapshot: IFlexScopeSnapshot, expected: number): Promise<void> {
|
|
800
|
+
if (state.sessionLocal && !state.qualifyingLegacy) {
|
|
801
|
+
await this.stores.scopes.sessionLocal!.saveCatalogue(state.storageKey, snapshot, expected);
|
|
802
|
+
} else await this.stores.scopes.save(state.storageKey, snapshot, expected);
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/** Registers before the projection's pending intent can reach any workspace provider. */
|
|
806
|
+
public async updateRecoveryObligation(state: IStorageState, stored: IStoredSessionState, registerOnly = false): Promise<void> {
|
|
807
|
+
if (!state.sessionLocal) return;
|
|
808
|
+
const sessionId = stored.session.sessionId;
|
|
809
|
+
const active = state.activeRuns.get(sessionId);
|
|
810
|
+
const required = stored.pendingReversion !== undefined || stored.pendingReversionReleases.length > 0
|
|
811
|
+
|| stored.stagedTerminals.length > 0 || stored.metrics.pendingGenerationId !== undefined
|
|
812
|
+
|| stored.metrics.pendingCompactionId !== undefined
|
|
813
|
+
|| (stored.recovering === true && state.recoverySessionIds.has(sessionId))
|
|
814
|
+
|| (this.turnReversionProvider !== undefined && active !== undefined && !active.finishedEventEmitted);
|
|
815
|
+
if (registerOnly && !required) return;
|
|
816
|
+
if (state.qualifyingLegacy) {
|
|
817
|
+
if (required) state.recoverySessionIds.add(sessionId);
|
|
818
|
+
else state.recoverySessionIds.delete(sessionId);
|
|
819
|
+
return;
|
|
820
|
+
}
|
|
821
|
+
// Re-registering is idempotent. The durable sidecar is authoritative, not this cache.
|
|
822
|
+
try {
|
|
823
|
+
await this.stores.scopes.sessionLocal!.setRecoveryRequired(state.storageKey, sessionId, required);
|
|
824
|
+
if (required) state.recoverySessionIds.add(sessionId);
|
|
825
|
+
else state.recoverySessionIds.delete(sessionId);
|
|
826
|
+
} catch (error) {
|
|
827
|
+
state.lifecycle = 'fenced';
|
|
828
|
+
throw error;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* Brings a session's recovery obligation up to date after an outcome is already recorded: a
|
|
834
|
+
* finished prompt or run, a saved projection. Its failure never changes that outcome. The
|
|
835
|
+
* durable obligation stays as it was, which the next load repairs, and the namespace is fenced
|
|
836
|
+
* through its drain instead of throwing into the caller.
|
|
837
|
+
*/
|
|
838
|
+
public async settleRecoveryObligation(state: IStorageState, stored: IStoredSessionState): Promise<void> {
|
|
839
|
+
if (!state.sessionLocal || state.lifecycle !== 'active') return;
|
|
840
|
+
try {
|
|
841
|
+
await this.updateRecoveryObligation(state, stored);
|
|
842
|
+
} catch (error) {
|
|
843
|
+
this.lifecycle.deferNamespaceDrain(state, error);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
|
|
589
847
|
public createUninitializedStoredSession(
|
|
590
848
|
session: IFlexSession,
|
|
591
849
|
storageKey: string,
|
|
@@ -735,6 +993,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
735
993
|
}
|
|
736
994
|
const stored: IStoredSessionState = {
|
|
737
995
|
storageKey: state.storageKey,
|
|
996
|
+
recovering: !newSession,
|
|
738
997
|
compactorContext,
|
|
739
998
|
session: cloneSerializable(metadata),
|
|
740
999
|
messages,
|
|
@@ -1023,7 +1282,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1023
1282
|
);
|
|
1024
1283
|
}
|
|
1025
1284
|
if (stored.pendingReversion?.kind === 'capture' && stored.pendingReversionReleases.length > 0) {
|
|
1026
|
-
await this.reversions.drainReversionReleases(state, stored, scopeId, scope,
|
|
1285
|
+
await this.reversions.drainReversionReleases(state, stored, scopeId, scope, state.sessionLocal);
|
|
1027
1286
|
this.assertStateAcceptingWork(state);
|
|
1028
1287
|
}
|
|
1029
1288
|
// The outcomes and model context of the session's runs, before a pending reversion resumes.
|
|
@@ -1493,6 +1752,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1493
1752
|
onlySessionId?: string,
|
|
1494
1753
|
): Promise<TValue> {
|
|
1495
1754
|
const beforeRevision = state.revision;
|
|
1755
|
+
const beforeCatalogue = new Map([...state.catalogue].map(([id, metadata]) => [id, cloneSerializable(metadata)]));
|
|
1496
1756
|
const beforeSessions = new Map([...state.sessions].map(([id, stored]) => [id, {
|
|
1497
1757
|
stored,
|
|
1498
1758
|
metadata: cloneSerializable(stored.session),
|
|
@@ -1504,18 +1764,20 @@ export class FlexHarnessCore<TScope> {
|
|
|
1504
1764
|
const beforeSnapshot = (): IFlexScopeSnapshot => ({
|
|
1505
1765
|
schemaVersion: 1,
|
|
1506
1766
|
revision: beforeRevision,
|
|
1507
|
-
sessions: [...
|
|
1508
|
-
.map((entry) => entry.metadata)
|
|
1767
|
+
sessions: [...beforeCatalogue.values()]
|
|
1509
1768
|
.sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
|
|
1510
1769
|
tombstones: [...beforeTombstones.values()]
|
|
1511
1770
|
.sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
|
|
1512
1771
|
});
|
|
1513
1772
|
const restore = () => {
|
|
1514
1773
|
state.revision = beforeRevision;
|
|
1774
|
+
state.catalogue.clear();
|
|
1775
|
+
for (const [id, metadata] of beforeCatalogue) state.catalogue.set(id, metadata);
|
|
1515
1776
|
state.sessions.clear();
|
|
1516
1777
|
for (const [id, entry] of beforeSessions) {
|
|
1517
1778
|
entry.stored.session = entry.metadata;
|
|
1518
1779
|
state.sessions.set(id, entry.stored);
|
|
1780
|
+
state.catalogue.set(id, entry.metadata);
|
|
1519
1781
|
}
|
|
1520
1782
|
state.tombstones.clear();
|
|
1521
1783
|
for (const [id, value] of beforeTombstones) state.tombstones.set(id, value);
|
|
@@ -1527,8 +1789,12 @@ export class FlexHarnessCore<TScope> {
|
|
|
1527
1789
|
let result: TValue;
|
|
1528
1790
|
try {
|
|
1529
1791
|
result = mutation();
|
|
1792
|
+
for (const id of beforeSessions.keys()) {
|
|
1793
|
+
if (!state.sessions.has(id)) state.catalogue.delete(id);
|
|
1794
|
+
}
|
|
1795
|
+
for (const [id, stored] of state.sessions) state.catalogue.set(id, stored.session);
|
|
1530
1796
|
if (onlySessionId !== undefined) {
|
|
1531
|
-
this.assertOnlySessionChanged(state, onlySessionId, beforeSessions, beforeTombstones, beforeRetainedCleanups);
|
|
1797
|
+
this.assertOnlySessionChanged(state, onlySessionId, beforeSessions, beforeTombstones, beforeRetainedCleanups, beforeCatalogue);
|
|
1532
1798
|
}
|
|
1533
1799
|
} catch (error) {
|
|
1534
1800
|
restore();
|
|
@@ -1536,7 +1802,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1536
1802
|
}
|
|
1537
1803
|
const snapshot = this.createScopeSnapshot(state, beforeRevision + 1);
|
|
1538
1804
|
try {
|
|
1539
|
-
await this.
|
|
1805
|
+
await this.saveScopeSnapshot(state, snapshot, beforeRevision);
|
|
1540
1806
|
state.revision = snapshot.revision;
|
|
1541
1807
|
return result;
|
|
1542
1808
|
} catch (error) {
|
|
@@ -1582,9 +1848,11 @@ export class FlexHarnessCore<TScope> {
|
|
|
1582
1848
|
beforeSessions: ReadonlyMap<string, { stored: IStoredSessionState; metadata: IFlexSession }>,
|
|
1583
1849
|
beforeTombstones: ReadonlyMap<string, IFlexSessionTombstone>,
|
|
1584
1850
|
beforeRetainedCleanups: ReadonlyMap<string, unknown>,
|
|
1851
|
+
beforeCatalogue: ReadonlyMap<string, IFlexSession>,
|
|
1585
1852
|
): void {
|
|
1586
1853
|
const changed = (id: string) => {
|
|
1587
1854
|
if (id === sessionId) return false;
|
|
1855
|
+
if (canonicalJson(beforeCatalogue.get(id)) !== canonicalJson(state.catalogue.get(id))) return true;
|
|
1588
1856
|
const before = beforeSessions.get(id);
|
|
1589
1857
|
const stored = state.sessions.get(id);
|
|
1590
1858
|
if (before?.stored !== stored) return true;
|
|
@@ -1593,6 +1861,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1593
1861
|
return beforeRetainedCleanups.get(id) !== state.retainedSessionCleanups.get(id);
|
|
1594
1862
|
};
|
|
1595
1863
|
const ids = new Set([
|
|
1864
|
+
...beforeCatalogue.keys(), ...state.catalogue.keys(),
|
|
1596
1865
|
...beforeSessions.keys(), ...state.sessions.keys(),
|
|
1597
1866
|
...beforeTombstones.keys(), ...state.tombstones.keys(),
|
|
1598
1867
|
...beforeRetainedCleanups.keys(), ...state.retainedSessionCleanups.keys(),
|
|
@@ -1606,140 +1875,185 @@ export class FlexHarnessCore<TScope> {
|
|
|
1606
1875
|
}
|
|
1607
1876
|
}
|
|
1608
1877
|
|
|
1878
|
+
/**
|
|
1879
|
+
* Changes `stored`'s projection in its FIFO and saves it. `onSaved` gets the checkpoint clock's
|
|
1880
|
+
* times the save started and completed, right after it completed.
|
|
1881
|
+
*/
|
|
1609
1882
|
public mutateProjection<TValue>(
|
|
1610
1883
|
state: IStorageState,
|
|
1611
1884
|
stored: IStoredSessionState,
|
|
1612
1885
|
mutation: () => TValue,
|
|
1613
1886
|
_allowDuringDrain = false,
|
|
1887
|
+
onSaved?: TProjectionSaved,
|
|
1614
1888
|
): Promise<TValue> {
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1889
|
+
return this.enqueueProjectionOperation(stored, () => this.saveProjectionMutation(state, stored, mutation, onSaved));
|
|
1890
|
+
}
|
|
1891
|
+
|
|
1892
|
+
/**
|
|
1893
|
+
* A running turn's projection checkpoint, at its turn in the projection FIFO: `prepare` decides
|
|
1894
|
+
* there what to save and returns the mutation, or nothing, which saves nothing. Resolves whether
|
|
1895
|
+
* it saved.
|
|
1896
|
+
*/
|
|
1897
|
+
public checkpointProjection(
|
|
1898
|
+
state: IStorageState,
|
|
1899
|
+
stored: IStoredSessionState,
|
|
1900
|
+
prepare: () => (() => void) | undefined,
|
|
1901
|
+
onSaved: TProjectionSaved,
|
|
1902
|
+
): Promise<boolean> {
|
|
1903
|
+
return this.enqueueProjectionOperation(stored, async () => {
|
|
1904
|
+
const mutation = prepare();
|
|
1905
|
+
if (!mutation) return false;
|
|
1906
|
+
await this.saveProjectionMutation(state, stored, mutation, onSaved);
|
|
1907
|
+
return true;
|
|
1908
|
+
});
|
|
1909
|
+
}
|
|
1910
|
+
|
|
1911
|
+
private enqueueProjectionOperation<TValue>(
|
|
1912
|
+
stored: IStoredSessionState,
|
|
1913
|
+
operation: () => Promise<TValue>,
|
|
1914
|
+
): Promise<TValue> {
|
|
1915
|
+
const queued = stored.projectionQueue.then(operation);
|
|
1916
|
+
stored.projectionQueue = queued.then(() => undefined, () => undefined);
|
|
1917
|
+
return queued;
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
private async saveProjectionMutation<TValue>(
|
|
1921
|
+
state: IStorageState,
|
|
1922
|
+
stored: IStoredSessionState,
|
|
1923
|
+
mutation: () => TValue,
|
|
1924
|
+
onSaved: TProjectionSaved | undefined,
|
|
1925
|
+
): Promise<TValue> {
|
|
1926
|
+
const beforeRevision = stored.projectionRevision;
|
|
1927
|
+
const beforeSchemaVersion = stored.projectionSchemaVersion;
|
|
1928
|
+
const beforeReconciliationRequired = stored.projectionReconciliationRequired;
|
|
1929
|
+
// The baseline is deep-frozen and only ever replaced, never edited, so it needs no copy.
|
|
1930
|
+
const beforeBaseline = stored.projectionBaseline;
|
|
1931
|
+
const beforeMetrics = cloneSerializable(stored.metrics);
|
|
1932
|
+
// List entries are frozen and replaced rather than edited, so copies of the lists suffice.
|
|
1933
|
+
const beforeMessages = stored.messages.slice();
|
|
1934
|
+
const beforeStages = cloneSerializable(stored.stagedTerminals);
|
|
1935
|
+
const beforeSegments = stored.reversionSegments.slice();
|
|
1936
|
+
const beforeCursor = stored.revertCursor;
|
|
1937
|
+
const beforeExcludedRunIds = cloneSerializable(stored.excludedRunIds);
|
|
1938
|
+
const beforePendingReversion = stored.pendingReversion === undefined
|
|
1939
|
+
? undefined
|
|
1940
|
+
: cloneSerializable(stored.pendingReversion);
|
|
1941
|
+
const beforePendingReversionReleases = cloneSerializable(stored.pendingReversionReleases);
|
|
1942
|
+
// The copies above stay untouched until a restore, so the projection as it was before the
|
|
1943
|
+
// mutation is built from them only when a failed save has to be reconciled.
|
|
1944
|
+
const beforeSnapshot = (): TFlexProjectionSnapshot => beforeBaseline ?? {
|
|
1945
|
+
schemaVersion: 3,
|
|
1946
|
+
revision: beforeRevision,
|
|
1947
|
+
metrics: beforeMetrics,
|
|
1948
|
+
messages: beforeMessages,
|
|
1949
|
+
stagedTerminals: beforeStages,
|
|
1950
|
+
reversionSegments: beforeSegments,
|
|
1951
|
+
revertCursor: beforeCursor,
|
|
1952
|
+
excludedRunIds: beforeExcludedRunIds,
|
|
1953
|
+
...(beforePendingReversion === undefined ? {} : { pendingReversion: beforePendingReversion }),
|
|
1954
|
+
pendingReversionReleases: beforePendingReversionReleases,
|
|
1955
|
+
};
|
|
1956
|
+
const restore = () => {
|
|
1957
|
+
stored.projectionSchemaVersion = beforeSchemaVersion;
|
|
1958
|
+
stored.projectionReconciliationRequired = beforeReconciliationRequired;
|
|
1959
|
+
if (beforeBaseline === undefined) delete stored.projectionBaseline;
|
|
1960
|
+
else stored.projectionBaseline = beforeBaseline;
|
|
1961
|
+
stored.projectionRevision = beforeRevision;
|
|
1962
|
+
stored.metrics = beforeMetrics;
|
|
1963
|
+
stored.messages = beforeMessages;
|
|
1964
|
+
stored.stagedTerminals = beforeStages;
|
|
1965
|
+
stored.reversionSegments = beforeSegments;
|
|
1966
|
+
stored.revertCursor = beforeCursor;
|
|
1967
|
+
stored.excludedRunIds = beforeExcludedRunIds;
|
|
1968
|
+
if (beforePendingReversion === undefined) delete stored.pendingReversion;
|
|
1969
|
+
else stored.pendingReversion = beforePendingReversion;
|
|
1970
|
+
stored.pendingReversionReleases = beforePendingReversionReleases;
|
|
1971
|
+
};
|
|
1972
|
+
let result: TValue;
|
|
1973
|
+
try {
|
|
1974
|
+
result = mutation();
|
|
1975
|
+
} catch (error) {
|
|
1976
|
+
restore();
|
|
1977
|
+
throw error;
|
|
1978
|
+
}
|
|
1979
|
+
freezeProjectionEntries(stored);
|
|
1980
|
+
const revision = beforeRevision + 1;
|
|
1981
|
+
try {
|
|
1982
|
+
if (state.sessionLocal) await this.updateRecoveryObligation(state, stored, true);
|
|
1983
|
+
const saveStartedAt = onSaved ? this.checkpointClock.now() : 0;
|
|
1984
|
+
await this.stores.projections.save(
|
|
1985
|
+
state.storageKey,
|
|
1986
|
+
stored.session.sessionId,
|
|
1987
|
+
this.createProjectionChange(stored, revision),
|
|
1988
|
+
beforeRevision,
|
|
1989
|
+
);
|
|
1990
|
+
const saveCompletedAt = onSaved ? this.checkpointClock.now() : 0;
|
|
1991
|
+
stored.projectionSchemaVersion = 3;
|
|
1992
|
+
delete stored.projectionBaseline;
|
|
1993
|
+
this.rememberSavedProjectionLists(stored);
|
|
1994
|
+
stored.projectionReconciliationRequired = false;
|
|
1995
|
+
stored.projectionRevision = revision;
|
|
1996
|
+
onSaved?.(saveStartedAt, saveCompletedAt);
|
|
1997
|
+
} catch (error) {
|
|
1998
|
+
if (error instanceof FlexHarnessStoreConflictError) {
|
|
1666
1999
|
restore();
|
|
1667
2000
|
throw error;
|
|
1668
2001
|
}
|
|
1669
|
-
|
|
1670
|
-
|
|
2002
|
+
if (error instanceof FlexHarnessStoreCommitUncertainError) {
|
|
2003
|
+
stored.projectionSchemaVersion = 3;
|
|
2004
|
+
delete stored.savedProjectionLists;
|
|
2005
|
+
stored.projectionReconciliationRequired = true;
|
|
2006
|
+
stored.projectionRevision = revision;
|
|
2007
|
+
state.lifecycle = 'fenced';
|
|
2008
|
+
throw error;
|
|
2009
|
+
}
|
|
2010
|
+
const snapshot = this.createProjectionSnapshot(stored, revision);
|
|
2011
|
+
let current: TFlexProjectionSnapshot | undefined;
|
|
2012
|
+
let reconciliationError: unknown;
|
|
1671
2013
|
try {
|
|
1672
|
-
await this.stores.projections.
|
|
2014
|
+
current = await this.stores.projections.load(
|
|
1673
2015
|
state.storageKey,
|
|
1674
2016
|
stored.session.sessionId,
|
|
1675
|
-
this.createProjectionChange(stored, revision),
|
|
1676
|
-
beforeRevision,
|
|
1677
2017
|
);
|
|
2018
|
+
} catch (loadError) {
|
|
2019
|
+
reconciliationError = loadError;
|
|
2020
|
+
}
|
|
2021
|
+
if (current && JSON.stringify(current) === JSON.stringify(snapshot)) {
|
|
1678
2022
|
stored.projectionSchemaVersion = 3;
|
|
1679
2023
|
delete stored.projectionBaseline;
|
|
1680
2024
|
this.rememberSavedProjectionLists(stored);
|
|
1681
2025
|
stored.projectionReconciliationRequired = false;
|
|
1682
2026
|
stored.projectionRevision = revision;
|
|
1683
|
-
|
|
1684
|
-
} catch (error) {
|
|
1685
|
-
if (error instanceof FlexHarnessStoreConflictError) {
|
|
1686
|
-
restore();
|
|
1687
|
-
throw error;
|
|
1688
|
-
}
|
|
1689
|
-
if (error instanceof FlexHarnessStoreCommitUncertainError) {
|
|
1690
|
-
stored.projectionSchemaVersion = 3;
|
|
1691
|
-
delete stored.savedProjectionLists;
|
|
1692
|
-
stored.projectionReconciliationRequired = true;
|
|
1693
|
-
stored.projectionRevision = revision;
|
|
1694
|
-
state.lifecycle = 'fenced';
|
|
1695
|
-
throw error;
|
|
1696
|
-
}
|
|
1697
|
-
const snapshot = this.createProjectionSnapshot(stored, revision);
|
|
1698
|
-
let current: TFlexProjectionSnapshot | undefined;
|
|
1699
|
-
let reconciliationError: unknown;
|
|
1700
|
-
try {
|
|
1701
|
-
current = await this.stores.projections.load(
|
|
1702
|
-
state.storageKey,
|
|
1703
|
-
stored.session.sessionId,
|
|
1704
|
-
);
|
|
1705
|
-
} catch (loadError) {
|
|
1706
|
-
reconciliationError = loadError;
|
|
1707
|
-
}
|
|
1708
|
-
if (current && JSON.stringify(current) === JSON.stringify(snapshot)) {
|
|
1709
|
-
stored.projectionSchemaVersion = 3;
|
|
1710
|
-
delete stored.projectionBaseline;
|
|
1711
|
-
this.rememberSavedProjectionLists(stored);
|
|
1712
|
-
stored.projectionReconciliationRequired = false;
|
|
1713
|
-
stored.projectionRevision = revision;
|
|
1714
|
-
throw error;
|
|
1715
|
-
}
|
|
1716
|
-
if (
|
|
1717
|
-
(current === undefined && beforeRevision === 0)
|
|
1718
|
-
|| (current !== undefined && JSON.stringify(current) === JSON.stringify(beforeSnapshot()))
|
|
1719
|
-
) {
|
|
1720
|
-
restore();
|
|
1721
|
-
throw error;
|
|
1722
|
-
}
|
|
1723
|
-
stored.projectionSchemaVersion = 3;
|
|
1724
|
-
delete stored.savedProjectionLists;
|
|
1725
|
-
stored.projectionReconciliationRequired = true;
|
|
1726
|
-
stored.projectionRevision = revision;
|
|
1727
|
-
state.lifecycle = 'fenced';
|
|
1728
|
-
throw reconciliationError === undefined
|
|
1729
|
-
? error
|
|
1730
|
-
: combineErrors([error, reconciliationError]);
|
|
2027
|
+
throw error;
|
|
1731
2028
|
}
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
2029
|
+
if (
|
|
2030
|
+
(current === undefined && beforeRevision === 0)
|
|
2031
|
+
|| (current !== undefined && JSON.stringify(current) === JSON.stringify(beforeSnapshot()))
|
|
2032
|
+
) {
|
|
2033
|
+
restore();
|
|
2034
|
+
throw error;
|
|
2035
|
+
}
|
|
2036
|
+
stored.projectionSchemaVersion = 3;
|
|
2037
|
+
delete stored.savedProjectionLists;
|
|
2038
|
+
stored.projectionReconciliationRequired = true;
|
|
2039
|
+
stored.projectionRevision = revision;
|
|
2040
|
+
state.lifecycle = 'fenced';
|
|
2041
|
+
throw reconciliationError === undefined
|
|
2042
|
+
? error
|
|
2043
|
+
: combineErrors([error, reconciliationError]);
|
|
2044
|
+
}
|
|
2045
|
+
// The projection is saved: what follows is a separate durable step, never classified as the
|
|
2046
|
+
// save's failure.
|
|
2047
|
+
if (state.sessionLocal) await this.settleRecoveryObligation(state, stored);
|
|
2048
|
+
return result;
|
|
1735
2049
|
}
|
|
1736
2050
|
|
|
1737
2051
|
private createScopeSnapshot(state: IStorageState, revision: number): IFlexScopeSnapshot {
|
|
1738
2052
|
return {
|
|
1739
2053
|
schemaVersion: 1,
|
|
1740
2054
|
revision,
|
|
1741
|
-
sessions: [...state.
|
|
1742
|
-
.map((
|
|
2055
|
+
sessions: [...state.catalogue.values()]
|
|
2056
|
+
.map((metadata) => cloneSerializable(state.sessions.get(metadata.sessionId)?.session ?? metadata))
|
|
1743
2057
|
.sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
|
|
1744
2058
|
tombstones: [...state.tombstones.values()]
|
|
1745
2059
|
.map((value) => cloneSerializable(value))
|
|
@@ -1920,6 +2234,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1920
2234
|
|
|
1921
2235
|
private sessionMetadata(state: IStorageState, sessionId: string): IFlexSession | undefined {
|
|
1922
2236
|
return state.sessions.get(sessionId)?.session
|
|
2237
|
+
?? state.catalogue.get(sessionId)
|
|
1923
2238
|
?? state.retainedSessionCleanups.get(sessionId)?.stored.session;
|
|
1924
2239
|
}
|
|
1925
2240
|
|
|
@@ -1931,6 +2246,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1931
2246
|
|
|
1932
2247
|
public async resolveState(
|
|
1933
2248
|
scopeId: string,
|
|
2249
|
+
sessionId?: string,
|
|
1934
2250
|
): Promise<{ scope: IFlexResolvedScope<TScope>; state: IStorageState }> {
|
|
1935
2251
|
return this.withScopeAdmission(scopeId, async (scope, admitted) => {
|
|
1936
2252
|
let stateLoad = this.stateLoads.get(scope.storageKey);
|
|
@@ -1967,7 +2283,15 @@ export class FlexHarnessCore<TScope> {
|
|
|
1967
2283
|
});
|
|
1968
2284
|
}
|
|
1969
2285
|
}
|
|
1970
|
-
|
|
2286
|
+
const state = await this.admittedState(scopeId, scope, stateLoad, admitted);
|
|
2287
|
+
if (state.sessionLocal) await this.recoverSharedState(state, scopeId, scope.scope);
|
|
2288
|
+
if (sessionId !== undefined && state.catalogue.has(sessionId)) {
|
|
2289
|
+
await this.hydrateSession(state, sessionId, scopeId, scope.scope);
|
|
2290
|
+
this.assertOpen();
|
|
2291
|
+
if (!admitted()) throw this.createScopeRetirementError();
|
|
2292
|
+
this.assertStateAcceptingWork(state);
|
|
2293
|
+
}
|
|
2294
|
+
return { scope, state };
|
|
1971
2295
|
});
|
|
1972
2296
|
}
|
|
1973
2297
|
|
|
@@ -2200,16 +2524,22 @@ export class FlexHarnessCore<TScope> {
|
|
|
2200
2524
|
* one id each; a position is searched again only after it moved (a steer, a restored or
|
|
2201
2525
|
* reloaded projection).
|
|
2202
2526
|
*/
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
partId: string,
|
|
2206
|
-
): { messageIndex: number; partIndex: number } {
|
|
2527
|
+
/** The run's remembered positions, its assistant message's checked; -1 when it is not projected. */
|
|
2528
|
+
public assistantMessagePositions(run: IActiveRun): NonNullable<IActiveRun['partEventPositions']> {
|
|
2207
2529
|
const positions = run.partEventPositions ??= { messageIndex: -1, partIndexes: new Map() };
|
|
2208
2530
|
if (run.stored.messages[positions.messageIndex]?.messageId !== run.assistantMessageId) {
|
|
2209
2531
|
positions.messageIndex = run.stored.messages.findIndex(
|
|
2210
2532
|
(message) => message.messageId === run.assistantMessageId,
|
|
2211
2533
|
);
|
|
2212
2534
|
}
|
|
2535
|
+
return positions;
|
|
2536
|
+
}
|
|
2537
|
+
|
|
2538
|
+
private partEventCoordinates(
|
|
2539
|
+
run: IActiveRun,
|
|
2540
|
+
partId: string,
|
|
2541
|
+
): { messageIndex: number; partIndex: number } {
|
|
2542
|
+
const positions = this.assistantMessagePositions(run);
|
|
2213
2543
|
let partIndex = positions.partIndexes.get(partId) ?? -1;
|
|
2214
2544
|
if (run.callbackParts[partIndex]?.partId !== partId) {
|
|
2215
2545
|
partIndex = run.callbackParts.findIndex((entry) => entry.partId === partId);
|