@modelprofile.com/flexharness 18.0.0 → 18.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.flexharness.cachewarm.js +3 -3
- package/dist_ts/classes.flexharness.callbacks.d.ts +37 -0
- package/dist_ts/classes.flexharness.callbacks.js +129 -2
- package/dist_ts/classes.flexharness.compaction.js +2 -2
- package/dist_ts/classes.flexharness.configurations.js +4 -4
- package/dist_ts/classes.flexharness.core.d.ts +96 -5
- package/dist_ts/classes.flexharness.core.js +848 -301
- package/dist_ts/classes.flexharness.d.ts +10 -1
- package/dist_ts/classes.flexharness.deletion.d.ts +13 -0
- package/dist_ts/classes.flexharness.deletion.js +122 -40
- package/dist_ts/classes.flexharness.imports.js +26 -21
- package/dist_ts/classes.flexharness.js +16 -1
- package/dist_ts/classes.flexharness.lifecycle.js +9 -3
- package/dist_ts/classes.flexharness.projects.js +2 -2
- package/dist_ts/classes.flexharness.prompts.js +14 -8
- package/dist_ts/classes.flexharness.reversions.js +3 -3
- package/dist_ts/classes.flexharness.runs.js +18 -2
- package/dist_ts/classes.flexharness.sessions.d.ts +8 -1
- package/dist_ts/classes.flexharness.sessions.js +40 -14
- package/dist_ts/classes.flexharness.slashcommands.js +3 -3
- package/dist_ts/classes.flexharness.subagents.js +37 -16
- package/dist_ts/classes.flexharness.tooldispatch.js +3 -3
- package/dist_ts/classes.scoperecovery.d.ts +36 -0
- package/dist_ts/classes.scoperecovery.js +228 -0
- package/dist_ts/classes.stores.d.ts +14 -1
- package/dist_ts/classes.stores.js +150 -17
- package/dist_ts/classes.timeslicer.d.ts +21 -0
- package/dist_ts/classes.timeslicer.js +31 -0
- package/dist_ts/index.d.ts +1 -0
- package/dist_ts/index.js +2 -1
- package/dist_ts/interfaces.d.ts +120 -0
- package/dist_ts/interfaces.flexharness.d.ts +56 -1
- package/dist_ts/interfaces.js +1 -1
- package/dist_ts/plugins.d.ts +2 -2
- package/dist_ts/plugins.js +3 -3
- package/dist_ts/utils.harnessoptions.d.ts +2 -0
- package/dist_ts/utils.harnessoptions.js +16 -11
- package/dist_ts/utils.json.d.ts +6 -0
- package/dist_ts/utils.json.js +75 -20
- package/dist_ts/utils.scopesnapshot.d.ts +14 -0
- package/dist_ts/utils.scopesnapshot.js +58 -0
- package/dist_ts_stores_nosqldb/classes.nosqldbstores.d.ts +3 -2
- package/dist_ts_stores_nosqldb/classes.nosqldbstores.js +33 -2
- package/dist_ts_stores_nosqldb/classes.recordstore.d.ts +10 -1
- package/dist_ts_stores_nosqldb/classes.recordstore.js +75 -7
- package/package.json +3 -3
- package/readme.md +46 -3
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.flexharness.cachewarm.ts +2 -2
- package/ts/classes.flexharness.callbacks.ts +142 -1
- package/ts/classes.flexharness.compaction.ts +1 -1
- package/ts/classes.flexharness.configurations.ts +3 -3
- package/ts/classes.flexharness.core.ts +902 -328
- package/ts/classes.flexharness.deletion.ts +161 -52
- package/ts/classes.flexharness.imports.ts +26 -21
- package/ts/classes.flexharness.lifecycle.ts +8 -2
- package/ts/classes.flexharness.projects.ts +1 -1
- package/ts/classes.flexharness.prompts.ts +11 -7
- package/ts/classes.flexharness.reversions.ts +2 -2
- package/ts/classes.flexharness.runs.ts +16 -1
- package/ts/classes.flexharness.sessions.ts +43 -13
- package/ts/classes.flexharness.slashcommands.ts +2 -2
- package/ts/classes.flexharness.subagents.ts +40 -15
- package/ts/classes.flexharness.tooldispatch.ts +2 -2
- package/ts/classes.flexharness.ts +22 -0
- package/ts/classes.scoperecovery.ts +290 -0
- package/ts/classes.stores.ts +150 -16
- package/ts/classes.timeslicer.ts +32 -0
- package/ts/index.ts +2 -0
- package/ts/interfaces.flexharness.ts +60 -0
- package/ts/interfaces.ts +129 -0
- package/ts/plugins.ts +2 -0
- package/ts/readme.md +46 -3
- package/ts/utils.harnessoptions.ts +18 -12
- package/ts/utils.json.ts +70 -18
- package/ts/utils.scopesnapshot.ts +75 -0
- package/ts_stores_nosqldb/classes.nosqldbstores.ts +52 -2
- package/ts_stores_nosqldb/classes.recordstore.ts +78 -8
- package/ts_stores_nosqldb/readme.md +10 -0
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
} from './errors.js';
|
|
15
15
|
import type {
|
|
16
16
|
IFlexAgentSessionPolicy,
|
|
17
|
+
IFlexCacheWarmClock,
|
|
17
18
|
IFlexCallbackLimits,
|
|
18
19
|
IFlexExecutionContextHandle,
|
|
19
20
|
IFlexHarnessOptions,
|
|
@@ -28,6 +29,7 @@ import type {
|
|
|
28
29
|
IFlexPendingReversionRelease,
|
|
29
30
|
IFlexResolvedScope,
|
|
30
31
|
IFlexResultLimits,
|
|
32
|
+
IFlexScopeLoadMetrics,
|
|
31
33
|
IFlexScopeSnapshot,
|
|
32
34
|
IFlexSession,
|
|
33
35
|
IFlexSessionImportLimits,
|
|
@@ -43,6 +45,8 @@ import type {
|
|
|
43
45
|
TFlexMessagePart,
|
|
44
46
|
TFlexProjectionSnapshot,
|
|
45
47
|
TFlexReversionPolicy,
|
|
48
|
+
TFlexScopeLoadObserver,
|
|
49
|
+
TFlexScopeLoadOutcome,
|
|
46
50
|
} from './interfaces.js';
|
|
47
51
|
import { InMemoryFlexHarnessStores } from './classes.stores.js';
|
|
48
52
|
import { FlexCodeSandbox } from './classes.codesandbox.js';
|
|
@@ -50,9 +54,12 @@ import {
|
|
|
50
54
|
assertJsonSerializable,
|
|
51
55
|
canonicalJson,
|
|
52
56
|
cloneSerializable,
|
|
57
|
+
cloneSerializableSliced,
|
|
53
58
|
deepFreeze,
|
|
54
59
|
resolveJsonLimits,
|
|
55
60
|
} from './utils.json.js';
|
|
61
|
+
import { FlexTimeSlicer } from './classes.timeslicer.js';
|
|
62
|
+
import { liveScopeSessions, normalizeScopeSnapshot } from './utils.scopesnapshot.js';
|
|
56
63
|
import { hydrateAgentMessages } from './utils.prompt.js';
|
|
57
64
|
import { createProjectionChange, freezeProjectionEntries } from './utils.projectionchange.js';
|
|
58
65
|
import {
|
|
@@ -60,6 +67,7 @@ import {
|
|
|
60
67
|
normalizeAttachmentResolver,
|
|
61
68
|
normalizeBuiltInTools,
|
|
62
69
|
normalizeCacheWarmOptions,
|
|
70
|
+
normalizeClock,
|
|
63
71
|
normalizeSlashCommands,
|
|
64
72
|
normalizeToolCatalogOptions,
|
|
65
73
|
normalizeSubagents,
|
|
@@ -95,7 +103,10 @@ import type {
|
|
|
95
103
|
IActiveRun,
|
|
96
104
|
IScopeAdmissionState,
|
|
97
105
|
IFlexMessageCursor,
|
|
106
|
+
IProjectionContent,
|
|
107
|
+
IScopeLoadPass,
|
|
98
108
|
TEventDetails,
|
|
109
|
+
TProjectionSaved,
|
|
99
110
|
} from './interfaces.flexharness.js';
|
|
100
111
|
import {
|
|
101
112
|
scopeRetirementMessage,
|
|
@@ -131,20 +142,6 @@ const defaultMaxSubagentCallsPerRun = 32;
|
|
|
131
142
|
const maximumMaxSubagentCallsPerRun = 128;
|
|
132
143
|
const repairCancellationMessage = 'The process stopped before this run completed.';
|
|
133
144
|
|
|
134
|
-
function deriveLegacySessionGenerationId(
|
|
135
|
-
storageKey: string,
|
|
136
|
-
sessionId: string,
|
|
137
|
-
createdAt: string,
|
|
138
|
-
): string {
|
|
139
|
-
const digest = plugins.crypto.createHash('sha256').update(JSON.stringify([
|
|
140
|
-
'flexharness-legacy-session-generation-v1',
|
|
141
|
-
storageKey,
|
|
142
|
-
sessionId,
|
|
143
|
-
createdAt,
|
|
144
|
-
])).digest('hex');
|
|
145
|
-
return `legacy_${digest}`;
|
|
146
|
-
}
|
|
147
|
-
|
|
148
145
|
/**
|
|
149
146
|
* The engine behind {@link FlexHarness}: its options, scope state, events and error projection, and
|
|
150
147
|
* the services that implement its API.
|
|
@@ -161,6 +158,8 @@ export class FlexHarnessCore<TScope> {
|
|
|
161
158
|
/** The URL protocols, colon included, of the attachment references prompts may carry. */
|
|
162
159
|
public readonly attachmentReferenceProtocols: ReadonlySet<string>;
|
|
163
160
|
public readonly stores: IFlexHarnessStores;
|
|
161
|
+
public readonly sessionLoading: 'eager' | 'session-local';
|
|
162
|
+
private readonly legacyQualifications = new Set<string>();
|
|
164
163
|
public readonly agentSessionPolicy: IFlexAgentSessionPolicy<TScope>;
|
|
165
164
|
public readonly toolOutputLimits: Required<NonNullable<IFlexHarnessOptions<TScope>['toolOutputLimits']>>;
|
|
166
165
|
public readonly callbackLimits: Required<IFlexCallbackLimits>;
|
|
@@ -172,14 +171,19 @@ export class FlexHarnessCore<TScope> {
|
|
|
172
171
|
public readonly turnReversionProvider: IFlexHarnessOptions<TScope>['turnReversionProvider'];
|
|
173
172
|
public readonly reversionPolicy: TFlexReversionPolicy;
|
|
174
173
|
private readonly externalErrorProjector?: TFlexExternalErrorProjector;
|
|
174
|
+
private readonly scopeLoadObserver?: TFlexScopeLoadObserver;
|
|
175
175
|
public readonly subagents: ReadonlyMap<string, Readonly<IFlexSubagentDefinition>>;
|
|
176
176
|
public readonly builtInTools: INormalizedBuiltInTools;
|
|
177
177
|
public readonly cacheWarmOptions: INormalizedCacheWarmOptions;
|
|
178
|
+
/** The clock of running turns' projection checkpoints: their spacing and save timings. */
|
|
179
|
+
public readonly checkpointClock: IFlexCacheWarmClock;
|
|
178
180
|
public readonly toolCatalogOptions: Readonly<IFlexToolCatalogOptions>;
|
|
179
181
|
public readonly slashCommands: ReadonlyMap<string, TRegisteredSlashCommand<TScope>>;
|
|
180
182
|
public readonly maxSubagentDepth: number;
|
|
181
183
|
public readonly maxSubagentCallsPerRun: number;
|
|
182
184
|
public readonly stateLoads = new Map<string, Promise<IStorageState>>();
|
|
185
|
+
/** The state each settled successful load loaded. */
|
|
186
|
+
private readonly loadedStates = new WeakMap<Promise<IStorageState>, IStorageState>();
|
|
183
187
|
public readonly scopeAdmissions = new Map<string, IScopeAdmissionState>();
|
|
184
188
|
public readonly listeners = new Set<TFlexHarnessEventListener>();
|
|
185
189
|
public readonly trustedInternalErrors = new WeakSet<object>();
|
|
@@ -220,6 +224,13 @@ export class FlexHarnessCore<TScope> {
|
|
|
220
224
|
this.attachmentReferenceProtocols = normalizeAttachmentResolver(options.attachmentResolver);
|
|
221
225
|
this.attachmentResolver = options.attachmentResolver;
|
|
222
226
|
this.stores = requireHarnessStores(options.stores ?? new InMemoryFlexHarnessStores());
|
|
227
|
+
this.sessionLoading = options.sessionLoading ?? 'eager';
|
|
228
|
+
if (!['eager', 'session-local'].includes(this.sessionLoading)) {
|
|
229
|
+
throw new FlexHarnessValidationError('sessionLoading must be "eager" or "session-local".');
|
|
230
|
+
}
|
|
231
|
+
if (this.sessionLoading === 'session-local' && !this.stores.scopes.sessionLocal) {
|
|
232
|
+
throw new FlexHarnessValidationError('session-local loading requires scopes.sessionLocal capability.');
|
|
233
|
+
}
|
|
223
234
|
this.agentSessionPolicy = normalizeAgentSessionPolicy(options.agentSessionPolicy);
|
|
224
235
|
this.toolOutputLimits = resolveJsonLimits(options.toolOutputLimits);
|
|
225
236
|
this.callbackLimits = resolveCallbackLimits(options.callbackLimits);
|
|
@@ -248,9 +259,14 @@ export class FlexHarnessCore<TScope> {
|
|
|
248
259
|
);
|
|
249
260
|
}
|
|
250
261
|
this.externalErrorProjector = options.externalErrorProjector;
|
|
262
|
+
if (options.scopeLoadObserver !== undefined && typeof options.scopeLoadObserver !== 'function') {
|
|
263
|
+
throw new FlexHarnessValidationError('scopeLoadObserver must be a function.');
|
|
264
|
+
}
|
|
265
|
+
this.scopeLoadObserver = options.scopeLoadObserver;
|
|
251
266
|
this.subagents = normalizeSubagents(options.subagents);
|
|
252
267
|
this.builtInTools = normalizeBuiltInTools(options.builtInTools);
|
|
253
268
|
this.cacheWarmOptions = normalizeCacheWarmOptions(options.cacheWarm);
|
|
269
|
+
this.checkpointClock = normalizeClock(options.projectionCheckpointClock, 'projectionCheckpointClock');
|
|
254
270
|
const toolCatalogOptions = normalizeToolCatalogOptions(options.toolCatalog);
|
|
255
271
|
this.toolCatalogOptions = Object.freeze({
|
|
256
272
|
declarations: toolCatalogOptions.declarations,
|
|
@@ -358,6 +374,15 @@ export class FlexHarnessCore<TScope> {
|
|
|
358
374
|
|
|
359
375
|
public canonicalOutcomes(events: readonly plugins.TAgentEvent[]): Map<string, TCanonicalOutcome> {
|
|
360
376
|
const outcomes = new Map<string, TCanonicalOutcome>();
|
|
377
|
+
for (const event of events) this.rememberCanonicalOutcomes(outcomes, event);
|
|
378
|
+
return outcomes;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/** Adds the canonical outcomes `event` records to `outcomes`. */
|
|
382
|
+
private rememberCanonicalOutcomes(
|
|
383
|
+
outcomes: Map<string, TCanonicalOutcome>,
|
|
384
|
+
event: plugins.TAgentEvent,
|
|
385
|
+
): void {
|
|
361
386
|
const remember = (generationId: string, outcome: TCanonicalOutcome) => {
|
|
362
387
|
const existing = outcomes.get(generationId);
|
|
363
388
|
if (existing && existing !== outcome) {
|
|
@@ -365,15 +390,12 @@ export class FlexHarnessCore<TScope> {
|
|
|
365
390
|
}
|
|
366
391
|
outcomes.set(generationId, outcome);
|
|
367
392
|
};
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
remember(transaction.generationId, transaction.outcome);
|
|
373
|
-
}
|
|
393
|
+
if (event.type === 'generation-outcome') remember(event.generationId, event.outcome);
|
|
394
|
+
if (event.type === 'context-compaction') {
|
|
395
|
+
for (const transaction of event.archivedTransactions ?? []) {
|
|
396
|
+
remember(transaction.generationId, transaction.outcome);
|
|
374
397
|
}
|
|
375
398
|
}
|
|
376
|
-
return outcomes;
|
|
377
399
|
}
|
|
378
400
|
|
|
379
401
|
public async durableCanonicalOutcomes(
|
|
@@ -383,46 +405,78 @@ export class FlexHarnessCore<TScope> {
|
|
|
383
405
|
return this.canonicalOutcomes(snapshot?.events ?? []);
|
|
384
406
|
}
|
|
385
407
|
|
|
408
|
+
/**
|
|
409
|
+
* Loads a scope's state for every request of the scope, and reports what the load took to the
|
|
410
|
+
* scope load observer. Its long steps are cut into slices of time, so the event loop keeps
|
|
411
|
+
* running beside it; disposal and the scope's retirement stop it at the next slice (the abort of
|
|
412
|
+
* `compactorLifecycleController`), a request that stops waiting for it does not.
|
|
413
|
+
*/
|
|
386
414
|
private async loadState(
|
|
387
415
|
storageKey: string,
|
|
388
416
|
scopeId: string,
|
|
389
417
|
scope: TScope,
|
|
390
418
|
compactorLifecycleController: AbortController,
|
|
391
419
|
): Promise<IStorageState> {
|
|
392
|
-
const
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
sessions:
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
scopeChanged = true;
|
|
420
|
+
const startedAt = performance.now();
|
|
421
|
+
const pass: IScopeLoadPass = {
|
|
422
|
+
slicer: new FlexTimeSlicer(compactorLifecycleController.signal),
|
|
423
|
+
metrics: { scopeId, storageKey, sessions: 0, storeReadMs: 0, agentSessionMs: 0, repairMs: 0 },
|
|
424
|
+
};
|
|
425
|
+
let outcome: TFlexScopeLoadOutcome = 'failed';
|
|
426
|
+
try {
|
|
427
|
+
const state = await this.loadStateSliced(storageKey, scopeId, scope, compactorLifecycleController, pass);
|
|
428
|
+
outcome = 'loaded';
|
|
429
|
+
return state;
|
|
430
|
+
} catch (error) {
|
|
431
|
+
if (compactorLifecycleController.signal.aborted) outcome = 'aborted';
|
|
432
|
+
this.stores.scopes.sessionLocal?.releaseOwnership(storageKey);
|
|
433
|
+
throw error;
|
|
434
|
+
} finally {
|
|
435
|
+
this.observeScopeLoad({ ...pass.metrics, totalMs: performance.now() - startedAt, outcome });
|
|
409
436
|
}
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/** Reports a scope load to the scope load observer, which can neither delay nor fail it. */
|
|
440
|
+
private observeScopeLoad(metrics: IFlexScopeLoadMetrics): void {
|
|
441
|
+
const observer = this.scopeLoadObserver;
|
|
442
|
+
if (!observer) return;
|
|
443
|
+
void (async () => observer(metrics))().catch(() => undefined);
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
private async loadStateSliced(
|
|
447
|
+
storageKey: string,
|
|
448
|
+
scopeId: string,
|
|
449
|
+
scope: TScope,
|
|
450
|
+
compactorLifecycleController: AbortController,
|
|
451
|
+
pass: IScopeLoadPass,
|
|
452
|
+
): Promise<IStorageState> {
|
|
453
|
+
const readStartedAt = performance.now();
|
|
454
|
+
if (this.sessionLoading === 'eager') this.stores.scopes.sessionLocal?.releaseOwnership(storageKey);
|
|
455
|
+
const local = this.sessionLoading === 'session-local'
|
|
456
|
+
? await this.stores.scopes.sessionLocal!.loadCatalogue(storageKey)
|
|
457
|
+
: undefined;
|
|
458
|
+
const qualifyingLegacy = local !== undefined && !local.qualified && this.legacyQualifications.has(storageKey);
|
|
459
|
+
if (local && !local.qualified && !qualifyingLegacy) {
|
|
460
|
+
throw new FlexHarnessValidationError(
|
|
461
|
+
'Session-local catalogue requires explicit prepareScope({ qualifyLegacy: true }) under exclusive namespace ownership.',
|
|
416
462
|
);
|
|
417
|
-
tombstone.sessionGenerationSequence = repairedGenerationSequence;
|
|
418
|
-
scopeChanged = true;
|
|
419
463
|
}
|
|
464
|
+
const stored = local ? local.snapshot : await this.stores.scopes.load(storageKey);
|
|
465
|
+
pass.metrics.storeReadMs += performance.now() - readStartedAt;
|
|
466
|
+
compactorLifecycleController.signal.throwIfAborted();
|
|
467
|
+
const { snapshot, changed } = normalizeScopeSnapshot(storageKey, stored);
|
|
468
|
+
let scopeChanged = changed || (qualifyingLegacy && stored === undefined);
|
|
420
469
|
const state: IStorageState = {
|
|
421
470
|
storageKey,
|
|
422
471
|
compactorLifecycleController,
|
|
423
472
|
scopeIdHint: scopeId,
|
|
424
473
|
scopeContext: { scopeId, scope },
|
|
425
474
|
revision: snapshot.revision,
|
|
475
|
+
catalogue: new Map(liveScopeSessions(snapshot).map((session) => [session.sessionId, session])),
|
|
476
|
+
sessionLocal: local !== undefined,
|
|
477
|
+
qualifyingLegacy,
|
|
478
|
+
recoverySessionIds: new Set(local?.recoverySessionIds ?? []),
|
|
479
|
+
sessionLoads: new Map(),
|
|
426
480
|
sessions: new Map(),
|
|
427
481
|
retainedSessionCleanups: new Map(),
|
|
428
482
|
sessionDeletions: new Map(),
|
|
@@ -443,39 +497,65 @@ export class FlexHarnessCore<TScope> {
|
|
|
443
497
|
fenceAdditionalErrors: [],
|
|
444
498
|
detachedCleanups: new Set(),
|
|
445
499
|
};
|
|
500
|
+
for (const sessionId of state.recoverySessionIds) {
|
|
501
|
+
if (!state.catalogue.has(sessionId) && !state.tombstones.has(sessionId)) {
|
|
502
|
+
throw new FlexHarnessValidationError(`Recovery obligation "${sessionId}" has no canonical session or tombstone owner.`);
|
|
503
|
+
}
|
|
504
|
+
}
|
|
446
505
|
const loadedSessions: IStoredSessionState[] = [];
|
|
506
|
+
// Eager qualification still joins every runtime; the independent catalogue is already complete.
|
|
507
|
+
let sessionsLoaded = false;
|
|
447
508
|
try {
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
scopeChanged = (await this.repairLoadedSession(
|
|
454
|
-
state.storageKey,
|
|
455
|
-
stored,
|
|
509
|
+
// An authoritative recovery obligation gates the namespace; unrelated sessions stay cold.
|
|
510
|
+
for (const metadata of liveScopeSessions(snapshot).filter((session) =>
|
|
511
|
+
!state.sessionLocal || qualifyingLegacy || state.recoverySessionIds.has(session.sessionId))) {
|
|
512
|
+
compactorLifecycleController.signal.throwIfAborted();
|
|
513
|
+
const stored = await this.loadSessionRuntime(
|
|
456
514
|
state,
|
|
515
|
+
metadata,
|
|
457
516
|
scopeId,
|
|
458
517
|
scope,
|
|
459
|
-
|
|
460
|
-
|
|
518
|
+
undefined,
|
|
519
|
+
false,
|
|
520
|
+
pass,
|
|
521
|
+
);
|
|
522
|
+
state.sessions.set(metadata.sessionId, stored);
|
|
523
|
+
loadedSessions.push(stored);
|
|
524
|
+
pass.metrics.sessions = loadedSessions.length;
|
|
525
|
+
const repairStartedAt = performance.now();
|
|
526
|
+
try {
|
|
527
|
+
scopeChanged = (await this.repairLoadedSession(
|
|
528
|
+
state.storageKey,
|
|
529
|
+
stored,
|
|
530
|
+
state,
|
|
531
|
+
scopeId,
|
|
532
|
+
scope,
|
|
533
|
+
pass.slicer,
|
|
534
|
+
)) || scopeChanged;
|
|
535
|
+
} finally {
|
|
536
|
+
pass.metrics.repairMs += performance.now() - repairStartedAt;
|
|
537
|
+
}
|
|
538
|
+
state.catalogue.set(metadata.sessionId, stored.session);
|
|
539
|
+
await this.reversions.drainReversionReleases(state, stored, scopeId, scope, state.sessionLocal);
|
|
461
540
|
if (stored.projectionRevision < 0) throw new Error('Invalid projection revision.');
|
|
462
541
|
}
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
542
|
+
sessionsLoaded = true;
|
|
543
|
+
for (const metadata of state.catalogue.values()) {
|
|
544
|
+
let ancestorId = metadata.parentSessionId;
|
|
545
|
+
const visited = new Set<string>([metadata.sessionId]);
|
|
466
546
|
while (ancestorId) {
|
|
467
547
|
if (visited.has(ancestorId)) {
|
|
468
548
|
throw new FlexHarnessValidationError(
|
|
469
|
-
`Live session "${
|
|
549
|
+
`Live session "${metadata.sessionId}" has a cyclic ancestor chain.`,
|
|
470
550
|
);
|
|
471
551
|
}
|
|
472
552
|
visited.add(ancestorId);
|
|
473
553
|
if (state.tombstones.has(ancestorId)) {
|
|
474
554
|
throw new FlexHarnessValidationError(
|
|
475
|
-
`Live session "${
|
|
555
|
+
`Live session "${metadata.sessionId}" descends from tombstoned ancestor "${ancestorId}".`,
|
|
476
556
|
);
|
|
477
557
|
}
|
|
478
|
-
ancestorId = state.
|
|
558
|
+
ancestorId = state.catalogue.get(ancestorId)?.parentSessionId;
|
|
479
559
|
}
|
|
480
560
|
}
|
|
481
561
|
const tombstoneRoots = this.deletion.orderTombstoneRootsChildFirst(
|
|
@@ -497,6 +577,12 @@ export class FlexHarnessCore<TScope> {
|
|
|
497
577
|
);
|
|
498
578
|
await this.deletion.cleanupSessionDomains(state, tombstone);
|
|
499
579
|
}
|
|
580
|
+
for (const tombstone of group) {
|
|
581
|
+
if (state.sessionLocal && !state.qualifyingLegacy) {
|
|
582
|
+
await this.stores.scopes.sessionLocal!.setRecoveryRequired(storageKey, tombstone.sessionId, false);
|
|
583
|
+
}
|
|
584
|
+
state.recoverySessionIds.delete(tombstone.sessionId);
|
|
585
|
+
}
|
|
500
586
|
for (const tombstone of group) state.tombstones.delete(tombstone.sessionId);
|
|
501
587
|
scopeChanged = true;
|
|
502
588
|
} catch (error) {
|
|
@@ -509,20 +595,35 @@ export class FlexHarnessCore<TScope> {
|
|
|
509
595
|
}
|
|
510
596
|
if (scopeChanged) {
|
|
511
597
|
const next = this.createScopeSnapshot(state, state.revision + 1);
|
|
512
|
-
await this.
|
|
598
|
+
await this.saveScopeSnapshot(state, next, state.revision);
|
|
513
599
|
state.revision = next.revision;
|
|
514
600
|
}
|
|
601
|
+
if (state.sessionLocal) {
|
|
602
|
+
if (qualifyingLegacy) {
|
|
603
|
+
await this.stores.scopes.sessionLocal!.qualifyCatalogue(storageKey,
|
|
604
|
+
this.createScopeSnapshot(state, state.revision), [...state.recoverySessionIds]);
|
|
605
|
+
state.qualifyingLegacy = false;
|
|
606
|
+
}
|
|
607
|
+
for (const loaded of loadedSessions) {
|
|
608
|
+
loaded.recovering = false;
|
|
609
|
+
await this.updateRecoveryObligation(state, loaded);
|
|
610
|
+
}
|
|
611
|
+
}
|
|
515
612
|
return state;
|
|
516
613
|
} catch (error) {
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
rootSessionId
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
614
|
+
// No tombstone was touched until the selected recovery/qualification sessions all loaded.
|
|
615
|
+
// Retain cleanup owners only after that boundary; the catalogue stays complete throughout.
|
|
616
|
+
if (sessionsLoaded) {
|
|
617
|
+
for (const rootSessionId of new Set([...state.tombstones.values()].map((tombstone) =>
|
|
618
|
+
tombstone.rootSessionId ?? tombstone.sessionId))) {
|
|
619
|
+
const key = JSON.stringify([storageKey, rootSessionId]);
|
|
620
|
+
this.lifecycle.orphanedTombstoneOwners.set(key, {
|
|
621
|
+
state,
|
|
622
|
+
rootSessionId,
|
|
623
|
+
scopeId,
|
|
624
|
+
scope,
|
|
625
|
+
});
|
|
626
|
+
}
|
|
526
627
|
}
|
|
527
628
|
const closeResults = await Promise.allSettled(
|
|
528
629
|
loadedSessions.map((stored) => this.lifecycle.closeStoredSession(stored)),
|
|
@@ -541,6 +642,208 @@ export class FlexHarnessCore<TScope> {
|
|
|
541
642
|
}
|
|
542
643
|
}
|
|
543
644
|
|
|
645
|
+
/** Enumerates metadata/proof only; explicit prepareScope owns startup recovery. */
|
|
646
|
+
public async getScopeCatalogue(scopeId: string): Promise<import('./interfaces.js').IFlexScopeCatalogue> {
|
|
647
|
+
return this.withScopeAdmission(scopeId, async (scope, admitted) => {
|
|
648
|
+
const load = this.stateLoads.get(scope.storageKey);
|
|
649
|
+
if (load && this.isStateLoaded(load)) {
|
|
650
|
+
const state = await this.admittedState(scopeId, scope, load, admitted);
|
|
651
|
+
await state.scopeQueue;
|
|
652
|
+
this.assertStateAcceptingWork(state);
|
|
653
|
+
return publicSnapshot({
|
|
654
|
+
sessions: [...state.catalogue.values()]
|
|
655
|
+
.filter((session) => !state.initializingSessions.has(session.sessionId))
|
|
656
|
+
.sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
|
|
657
|
+
recoverySessionIds: [...state.recoverySessionIds].sort(),
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
if (this.sessionLoading !== 'session-local') {
|
|
661
|
+
await this.resolveState(scopeId);
|
|
662
|
+
return this.getScopeCatalogue(scopeId);
|
|
663
|
+
}
|
|
664
|
+
try {
|
|
665
|
+
const catalogue = await this.stores.scopes.sessionLocal!.loadCatalogue(scope.storageKey);
|
|
666
|
+
this.assertOpen();
|
|
667
|
+
if (!admitted()) throw this.createScopeRetirementError();
|
|
668
|
+
if (!catalogue.qualified) {
|
|
669
|
+
throw new FlexHarnessValidationError('Session-local catalogue requires explicit legacy qualification.');
|
|
670
|
+
}
|
|
671
|
+
const { snapshot } = normalizeScopeSnapshot(scope.storageKey, catalogue.snapshot);
|
|
672
|
+
return publicSnapshot({ sessions: liveScopeSessions(snapshot).sort((left, right) =>
|
|
673
|
+
left.sessionId.localeCompare(right.sessionId)), recoverySessionIds: catalogue.recoverySessionIds.slice().sort() });
|
|
674
|
+
} finally {
|
|
675
|
+
// A metadata-only reader owns no runtime lifecycle on which to retain writer activation.
|
|
676
|
+
if (!this.stateLoads.has(scope.storageKey)) this.stores.scopes.sessionLocal!.releaseOwnership(scope.storageKey);
|
|
677
|
+
}
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
public async prepareScope(
|
|
682
|
+
scopeId: string,
|
|
683
|
+
options: import('./interfaces.js').IFlexPrepareScopeOptions = {},
|
|
684
|
+
): Promise<import('./interfaces.js').IFlexScopeCatalogue> {
|
|
685
|
+
if (!options || typeof options !== 'object' || Array.isArray(options)
|
|
686
|
+
|| Object.keys(options).some((key) => key !== 'qualifyLegacy')
|
|
687
|
+
|| (options.qualifyLegacy !== undefined && typeof options.qualifyLegacy !== 'boolean')) {
|
|
688
|
+
throw new FlexHarnessValidationError('prepareScope options must contain only qualifyLegacy (boolean).');
|
|
689
|
+
}
|
|
690
|
+
return this.withScopeAdmission(scopeId, async (scope) => {
|
|
691
|
+
if (options.qualifyLegacy) this.legacyQualifications.add(scope.storageKey);
|
|
692
|
+
try {
|
|
693
|
+
await this.resolveState(scopeId);
|
|
694
|
+
return await this.getScopeCatalogue(scopeId);
|
|
695
|
+
} finally {
|
|
696
|
+
if (options.qualifyLegacy) this.legacyQualifications.delete(scope.storageKey);
|
|
697
|
+
}
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
public async hydrateSession(
|
|
702
|
+
state: IStorageState,
|
|
703
|
+
sessionId: string,
|
|
704
|
+
scopeId: string,
|
|
705
|
+
scope: TScope,
|
|
706
|
+
): Promise<IStoredSessionState> {
|
|
707
|
+
const pending = state.sessionLoads.get(sessionId);
|
|
708
|
+
if (pending) return pending;
|
|
709
|
+
const existing = state.sessions.get(sessionId);
|
|
710
|
+
if (existing) return existing;
|
|
711
|
+
let load!: Promise<IStoredSessionState>;
|
|
712
|
+
load = (async () => {
|
|
713
|
+
await this.sessionReadBarrier(state, sessionId);
|
|
714
|
+
this.assertStateAcceptingWork(state);
|
|
715
|
+
const metadata = state.catalogue.get(sessionId);
|
|
716
|
+
if (!metadata || state.tombstones.has(sessionId)) throw new FlexHarnessNotFoundError('Session', sessionId);
|
|
717
|
+
const pass: IScopeLoadPass = {
|
|
718
|
+
slicer: new FlexTimeSlicer(state.compactorLifecycleController.signal),
|
|
719
|
+
metrics: { scopeId, storageKey: state.storageKey, sessions: 1, storeReadMs: 0, agentSessionMs: 0, repairMs: 0 },
|
|
720
|
+
};
|
|
721
|
+
const stored = await this.loadSessionRuntime(state, metadata, scopeId, scope, undefined, false, pass);
|
|
722
|
+
try {
|
|
723
|
+
state.compactorLifecycleController.signal.throwIfAborted();
|
|
724
|
+
this.assertStateAcceptingWork(state);
|
|
725
|
+
const current = state.catalogue.get(sessionId);
|
|
726
|
+
if (!current || state.tombstones.has(sessionId)
|
|
727
|
+
|| current.sessionGenerationId !== metadata.sessionGenerationId
|
|
728
|
+
|| current.sessionGenerationSequence !== metadata.sessionGenerationSequence) {
|
|
729
|
+
throw new FlexHarnessAbortError('The session was deleted while it was being hydrated.');
|
|
730
|
+
}
|
|
731
|
+
// Runtime ownership is visible to drain before repair can acquire more resources.
|
|
732
|
+
state.sessions.set(sessionId, stored);
|
|
733
|
+
await this.updateRecoveryObligation(state, stored, true);
|
|
734
|
+
const changed = await this.repairLoadedSession(state.storageKey, stored, state, scopeId, scope, pass.slicer);
|
|
735
|
+
await this.reversions.drainReversionReleases(state, stored, scopeId, scope, state.sessionLocal);
|
|
736
|
+
if (changed) await this.mutateSessionScope(state, sessionId, () => undefined);
|
|
737
|
+
else state.catalogue.set(sessionId, stored.session);
|
|
738
|
+
stored.recovering = false;
|
|
739
|
+
await this.updateRecoveryObligation(state, stored);
|
|
740
|
+
this.assertStateAcceptingWork(state);
|
|
741
|
+
return stored;
|
|
742
|
+
} catch (error) {
|
|
743
|
+
if (state.sessions.get(sessionId) === stored) state.sessions.delete(sessionId);
|
|
744
|
+
try {
|
|
745
|
+
await this.lifecycle.closeStoredSession(stored);
|
|
746
|
+
} catch (closeError) {
|
|
747
|
+
this.lifecycle.orphanedStoredSessions.add(stored);
|
|
748
|
+
throw combineErrors([error, closeError]);
|
|
749
|
+
}
|
|
750
|
+
throw error;
|
|
751
|
+
}
|
|
752
|
+
})().finally(() => {
|
|
753
|
+
if (state.sessionLoads.get(sessionId) === load) state.sessionLoads.delete(sessionId);
|
|
754
|
+
});
|
|
755
|
+
state.sessionLoads.set(sessionId, load);
|
|
756
|
+
return load;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
/** Retries ownerless shared-workspace obligations, never an active run's normal capture. */
|
|
760
|
+
private async recoverSharedState(state: IStorageState, scopeId: string, scope: TScope): Promise<void> {
|
|
761
|
+
if (!state.sessionLocal) return;
|
|
762
|
+
if (state.sharedRecovery) return state.sharedRecovery;
|
|
763
|
+
const ids = [...state.recoverySessionIds].filter((id) =>
|
|
764
|
+
!state.activeRuns.has(id) && !state.initializingSessions.has(id)
|
|
765
|
+
&& !(state.sessions.get(id)?.outstandingPromptsById.size));
|
|
766
|
+
const roots = [...new Set([...state.tombstones.values()].map((entry) => entry.rootSessionId ?? entry.sessionId))]
|
|
767
|
+
.filter((id) => !state.sessionDeletions.has(id) && !state.tombstoneCleanups.has(id)
|
|
768
|
+
&& !state.initializingSessions.has(id));
|
|
769
|
+
if (ids.length === 0 && roots.length === 0) return;
|
|
770
|
+
let recovery!: Promise<void>;
|
|
771
|
+
recovery = (async () => {
|
|
772
|
+
for (const id of ids) {
|
|
773
|
+
if (state.tombstones.has(id)) continue;
|
|
774
|
+
if (!state.catalogue.has(id)) {
|
|
775
|
+
throw new FlexHarnessValidationError(`Recovery obligation "${id}" has no canonical session or tombstone owner.`);
|
|
776
|
+
}
|
|
777
|
+
if (state.sessionLoads.has(id)) { await state.sessionLoads.get(id); continue; }
|
|
778
|
+
const stored = state.sessions.get(id);
|
|
779
|
+
if (!stored) { await this.hydrateSession(state, id, scopeId, scope); continue; }
|
|
780
|
+
stored.recovering = true;
|
|
781
|
+
await this.repairLoadedSession(state.storageKey, stored, state, scopeId, scope,
|
|
782
|
+
new FlexTimeSlicer(state.compactorLifecycleController.signal));
|
|
783
|
+
await this.reversions.drainReversionReleases(state, stored, scopeId, scope, true);
|
|
784
|
+
// A preceding failed recovery may already have repaired metadata only in memory.
|
|
785
|
+
await this.mutateSessionScope(state, id, () => undefined);
|
|
786
|
+
stored.recovering = false;
|
|
787
|
+
await this.updateRecoveryObligation(state, stored);
|
|
788
|
+
}
|
|
789
|
+
for (const root of roots) await this.deletion.finishTombstoneCleanup(state, root, scopeId, { scopeId, scope });
|
|
790
|
+
})().finally(() => { if (state.sharedRecovery === recovery) delete state.sharedRecovery; });
|
|
791
|
+
state.sharedRecovery = recovery;
|
|
792
|
+
return recovery;
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
public async hydrateAllSessions(state: IStorageState, scopeId: string, scope: TScope): Promise<void> {
|
|
796
|
+
for (const sessionId of state.catalogue.keys()) await this.hydrateSession(state, sessionId, scopeId, scope);
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
private async saveScopeSnapshot(state: IStorageState, snapshot: IFlexScopeSnapshot, expected: number): Promise<void> {
|
|
800
|
+
if (state.sessionLocal && !state.qualifyingLegacy) {
|
|
801
|
+
await this.stores.scopes.sessionLocal!.saveCatalogue(state.storageKey, snapshot, expected);
|
|
802
|
+
} else await this.stores.scopes.save(state.storageKey, snapshot, expected);
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/** Registers before the projection's pending intent can reach any workspace provider. */
|
|
806
|
+
public async updateRecoveryObligation(state: IStorageState, stored: IStoredSessionState, registerOnly = false): Promise<void> {
|
|
807
|
+
if (!state.sessionLocal) return;
|
|
808
|
+
const sessionId = stored.session.sessionId;
|
|
809
|
+
const active = state.activeRuns.get(sessionId);
|
|
810
|
+
const required = stored.pendingReversion !== undefined || stored.pendingReversionReleases.length > 0
|
|
811
|
+
|| stored.stagedTerminals.length > 0 || stored.metrics.pendingGenerationId !== undefined
|
|
812
|
+
|| stored.metrics.pendingCompactionId !== undefined
|
|
813
|
+
|| (stored.recovering === true && state.recoverySessionIds.has(sessionId))
|
|
814
|
+
|| (this.turnReversionProvider !== undefined && active !== undefined && !active.finishedEventEmitted);
|
|
815
|
+
if (registerOnly && !required) return;
|
|
816
|
+
if (state.qualifyingLegacy) {
|
|
817
|
+
if (required) state.recoverySessionIds.add(sessionId);
|
|
818
|
+
else state.recoverySessionIds.delete(sessionId);
|
|
819
|
+
return;
|
|
820
|
+
}
|
|
821
|
+
// Re-registering is idempotent. The durable sidecar is authoritative, not this cache.
|
|
822
|
+
try {
|
|
823
|
+
await this.stores.scopes.sessionLocal!.setRecoveryRequired(state.storageKey, sessionId, required);
|
|
824
|
+
if (required) state.recoverySessionIds.add(sessionId);
|
|
825
|
+
else state.recoverySessionIds.delete(sessionId);
|
|
826
|
+
} catch (error) {
|
|
827
|
+
state.lifecycle = 'fenced';
|
|
828
|
+
throw error;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/**
|
|
833
|
+
* Brings a session's recovery obligation up to date after an outcome is already recorded: a
|
|
834
|
+
* finished prompt or run, a saved projection. Its failure never changes that outcome. The
|
|
835
|
+
* durable obligation stays as it was, which the next load repairs, and the namespace is fenced
|
|
836
|
+
* through its drain instead of throwing into the caller.
|
|
837
|
+
*/
|
|
838
|
+
public async settleRecoveryObligation(state: IStorageState, stored: IStoredSessionState): Promise<void> {
|
|
839
|
+
if (!state.sessionLocal || state.lifecycle !== 'active') return;
|
|
840
|
+
try {
|
|
841
|
+
await this.updateRecoveryObligation(state, stored);
|
|
842
|
+
} catch (error) {
|
|
843
|
+
this.lifecycle.deferNamespaceDrain(state, error);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
|
|
544
847
|
public createUninitializedStoredSession(
|
|
545
848
|
session: IFlexSession,
|
|
546
849
|
storageKey: string,
|
|
@@ -590,6 +893,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
590
893
|
scope: TScope,
|
|
591
894
|
compactorLifecycleController = new AbortController(),
|
|
592
895
|
newSession = false,
|
|
896
|
+
pass?: IScopeLoadPass,
|
|
593
897
|
): Promise<IStoredSessionState> {
|
|
594
898
|
const sessionId = metadata.sessionId;
|
|
595
899
|
const compactorContext = this.compaction.createCompactorContext(
|
|
@@ -598,12 +902,14 @@ export class FlexHarnessCore<TScope> {
|
|
|
598
902
|
state.storageKey,
|
|
599
903
|
sessionId,
|
|
600
904
|
);
|
|
905
|
+
const readStartedAt = performance.now();
|
|
601
906
|
const [projectionResult, permissionResult, eventStoreResult, jobStoreResult] = await Promise.allSettled([
|
|
602
907
|
this.stores.projections.load(state.storageKey, sessionId),
|
|
603
908
|
this.stores.permissions.load(state.storageKey, sessionId),
|
|
604
909
|
this.stores.agentEvents.getStore(state.storageKey, sessionId),
|
|
605
910
|
this.stores.jobs.getStore(state.storageKey, sessionId),
|
|
606
911
|
]);
|
|
912
|
+
if (pass) pass.metrics.storeReadMs += performance.now() - readStartedAt;
|
|
607
913
|
const acquisitionErrors = [projectionResult, permissionResult, eventStoreResult, jobStoreResult]
|
|
608
914
|
.filter((result): result is PromiseRejectedResult => result.status === 'rejected')
|
|
609
915
|
.map((result) => result.reason);
|
|
@@ -676,11 +982,21 @@ export class FlexHarnessCore<TScope> {
|
|
|
676
982
|
throw new FlexHarnessValidationError('Execution context provider returned an invalid handle.');
|
|
677
983
|
}
|
|
678
984
|
}
|
|
985
|
+
// A scope load copies and freezes the messages in slices of time. The baseline shares the
|
|
986
|
+
// frozen copies, as it is only ever replaced, never edited.
|
|
987
|
+
const messages = pass
|
|
988
|
+
? await cloneSerializableSliced(projection?.messages ?? [], pass.slicer)
|
|
989
|
+
: cloneSerializable(projection?.messages ?? []);
|
|
990
|
+
for (const message of messages) {
|
|
991
|
+
deepFreeze(message);
|
|
992
|
+
if (pass?.slicer.due) await pass.slicer.pause();
|
|
993
|
+
}
|
|
679
994
|
const stored: IStoredSessionState = {
|
|
680
995
|
storageKey: state.storageKey,
|
|
996
|
+
recovering: !newSession,
|
|
681
997
|
compactorContext,
|
|
682
998
|
session: cloneSerializable(metadata),
|
|
683
|
-
messages
|
|
999
|
+
messages,
|
|
684
1000
|
stagedTerminals: cloneSerializable(projection?.stagedTerminals ?? []),
|
|
685
1001
|
reversionSegments,
|
|
686
1002
|
revertCursor: currentProjection?.revertCursor ?? legacyProjection?.revertCursor ?? 0,
|
|
@@ -694,7 +1010,12 @@ export class FlexHarnessCore<TScope> {
|
|
|
694
1010
|
projectionSchemaVersion: projection?.schemaVersion ?? 3,
|
|
695
1011
|
projectionReconciliationRequired: false,
|
|
696
1012
|
...(projection
|
|
697
|
-
? {
|
|
1013
|
+
? {
|
|
1014
|
+
projectionBaseline: deepFreeze({
|
|
1015
|
+
...cloneSerializable({ ...projection, messages: [] }),
|
|
1016
|
+
messages: messages.slice(),
|
|
1017
|
+
} as TFlexProjectionSnapshot),
|
|
1018
|
+
}
|
|
698
1019
|
: {}),
|
|
699
1020
|
projectionRevision: projection?.revision ?? 0,
|
|
700
1021
|
projectionQueue: Promise.resolve(),
|
|
@@ -867,10 +1188,15 @@ export class FlexHarnessCore<TScope> {
|
|
|
867
1188
|
onRetry: (event) => this.callbacks.onRetry(state, sessionId, event),
|
|
868
1189
|
onUsage: (event) => this.callbacks.onModelCallUsage(state, stored, event),
|
|
869
1190
|
};
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
1191
|
+
const agentSessionStartedAt = performance.now();
|
|
1192
|
+
try {
|
|
1193
|
+
stored.agentSession = await this.compaction.withCompactorContext(
|
|
1194
|
+
compactorContext,
|
|
1195
|
+
() => plugins.AgentSession.create(agentSessionOptions),
|
|
1196
|
+
);
|
|
1197
|
+
} finally {
|
|
1198
|
+
if (pass) pass.metrics.agentSessionMs += performance.now() - agentSessionStartedAt;
|
|
1199
|
+
}
|
|
874
1200
|
return stored;
|
|
875
1201
|
} catch (error) {
|
|
876
1202
|
const cleanupErrors: unknown[] = [];
|
|
@@ -905,26 +1231,25 @@ export class FlexHarnessCore<TScope> {
|
|
|
905
1231
|
}
|
|
906
1232
|
}
|
|
907
1233
|
|
|
1234
|
+
/**
|
|
1235
|
+
* Repairs what a stopped process left in a loaded session: every run gets its terminal, the
|
|
1236
|
+
* reversion state its runs' outcomes, the session its latest turn's state. Linear in the
|
|
1237
|
+
* session's size, in slices of `slicer`'s time; it saves the projection only if its content
|
|
1238
|
+
* changed.
|
|
1239
|
+
*/
|
|
908
1240
|
private async repairLoadedSession(
|
|
909
1241
|
storageKey: string,
|
|
910
1242
|
stored: IStoredSessionState,
|
|
911
|
-
state
|
|
912
|
-
scopeId
|
|
913
|
-
scope
|
|
1243
|
+
state: IStorageState,
|
|
1244
|
+
scopeId: string,
|
|
1245
|
+
scope: TScope,
|
|
1246
|
+
slicer: FlexTimeSlicer,
|
|
914
1247
|
): Promise<boolean> {
|
|
915
1248
|
if (stored.session.origin !== undefined) {
|
|
916
1249
|
this.assertImportedSessionState(stored);
|
|
917
1250
|
return false;
|
|
918
1251
|
}
|
|
919
|
-
const
|
|
920
|
-
messages: stored.messages,
|
|
921
|
-
stagedTerminals: stored.stagedTerminals,
|
|
922
|
-
reversionSegments: stored.reversionSegments,
|
|
923
|
-
revertCursor: stored.revertCursor,
|
|
924
|
-
excludedRunIds: stored.excludedRunIds,
|
|
925
|
-
pendingReversion: stored.pendingReversion,
|
|
926
|
-
pendingReversionReleases: stored.pendingReversionReleases,
|
|
927
|
-
});
|
|
1252
|
+
const before = this.projectionContent(stored);
|
|
928
1253
|
const beforeSession = JSON.stringify(stored.session);
|
|
929
1254
|
if (
|
|
930
1255
|
!this.turnReversionProvider
|
|
@@ -957,75 +1282,28 @@ export class FlexHarnessCore<TScope> {
|
|
|
957
1282
|
);
|
|
958
1283
|
}
|
|
959
1284
|
if (stored.pendingReversion?.kind === 'capture' && stored.pendingReversionReleases.length > 0) {
|
|
960
|
-
|
|
961
|
-
throw new FlexHarnessValidationError('Pending capture recovery is missing its scope context.');
|
|
962
|
-
}
|
|
963
|
-
await this.reversions.drainReversionReleases(state, stored, scopeId, scope, false);
|
|
1285
|
+
await this.reversions.drainReversionReleases(state, stored, scopeId, scope, state.sessionLocal);
|
|
964
1286
|
this.assertStateAcceptingWork(state);
|
|
965
1287
|
}
|
|
1288
|
+
// The outcomes and model context of the session's runs, before a pending reversion resumes.
|
|
966
1289
|
const agentEvents = stored.agentSession.getEvents();
|
|
967
|
-
const outcomes =
|
|
1290
|
+
const outcomes = new Map<string, TCanonicalOutcome>();
|
|
1291
|
+
for (const event of agentEvents) {
|
|
1292
|
+
this.rememberCanonicalOutcomes(outcomes, event);
|
|
1293
|
+
if (slicer.due) await slicer.pause();
|
|
1294
|
+
}
|
|
968
1295
|
const firstHiddenCandidate = this.reversions.completedReversionCandidates(stored)[stored.revertCursor];
|
|
969
1296
|
const reversionVisibilityBoundary = firstHiddenCandidate
|
|
970
1297
|
? stored.reversionSegments.findIndex((segment) => segment.runId === firstHiddenCandidate.runId)
|
|
971
1298
|
: stored.reversionSegments.length;
|
|
972
|
-
const
|
|
973
|
-
const runIds = new Set([
|
|
974
|
-
...stored.messages.map((message) => message.runId),
|
|
975
|
-
...stored.stagedTerminals.map((terminal) => terminal.runId),
|
|
976
|
-
]);
|
|
977
|
-
// The session's state is the latest real turn's: a cache-warm turn's terminal never sets it.
|
|
978
|
-
let latestTerminal: IFlexTerminalProjection | undefined;
|
|
979
|
-
for (const runId of runIds) {
|
|
980
|
-
const outcome = outcomes.get(runId);
|
|
981
|
-
const stage = stages.get(runId);
|
|
982
|
-
if (outcome === 'accepted' && stage?.status === 'completed') {
|
|
983
|
-
// A cache-warm exchange leaves the model context with its outcome.
|
|
984
|
-
this.runs.replaceVisibleRun(stored, this.runs.withKeptInContext(stage, !stage.assistantMessage.cacheWarm));
|
|
985
|
-
if (!stage.assistantMessage.cacheWarm) latestTerminal = stage;
|
|
986
|
-
} else if (outcome !== 'accepted') {
|
|
987
|
-
const visible = stored.messages.filter((message) => message.runId === runId);
|
|
988
|
-
// A run that failed or was stopped keeps its terminal and its error: the staged one, or the
|
|
989
|
-
// one it published. Only a run that never reached its terminal is repaired as interrupted.
|
|
990
|
-
const ended = stage && stage.status !== 'completed'
|
|
991
|
-
? stage
|
|
992
|
-
: stage ? undefined : this.publishedTerminal(runId, visible);
|
|
993
|
-
if (ended) {
|
|
994
|
-
if (ended === stage) {
|
|
995
|
-
this.runs.replaceVisibleRun(stored, this.runs.withKeptInContext(
|
|
996
|
-
stage,
|
|
997
|
-
plugins.isGenerationInModelContext(agentEvents, runId),
|
|
998
|
-
));
|
|
999
|
-
}
|
|
1000
|
-
if (!ended.assistantMessage.cacheWarm) latestTerminal = ended;
|
|
1001
|
-
} else if (visible.length > 0 || stage) {
|
|
1002
|
-
const status = outcome === 'rejected' ? 'failed' : 'cancelled';
|
|
1003
|
-
const terminal = this.runs.withKeptInContext(
|
|
1004
|
-
this.repairTerminal(stored.session.sessionId, runId, visible, stage, status),
|
|
1005
|
-
plugins.isGenerationInModelContext(agentEvents, runId),
|
|
1006
|
-
);
|
|
1007
|
-
this.runs.replaceVisibleRun(stored, terminal);
|
|
1008
|
-
if (!terminal.assistantMessage.cacheWarm) latestTerminal = terminal;
|
|
1009
|
-
}
|
|
1010
|
-
} else {
|
|
1011
|
-
const visibleSuccess = stored.messages.some((message) =>
|
|
1012
|
-
message.runId === runId && message.status === 'completed');
|
|
1013
|
-
if (visibleSuccess) {
|
|
1014
|
-
const terminal = this.repairTerminal(
|
|
1015
|
-
stored.session.sessionId,
|
|
1016
|
-
runId,
|
|
1017
|
-
stored.messages.filter((message) => message.runId === runId),
|
|
1018
|
-
undefined,
|
|
1019
|
-
'completed',
|
|
1020
|
-
);
|
|
1021
|
-
if (!terminal.assistantMessage.cacheWarm) latestTerminal = terminal;
|
|
1022
|
-
}
|
|
1023
|
-
}
|
|
1024
|
-
}
|
|
1299
|
+
const latestTerminal = await this.repairRunTerminals(stored, agentEvents, outcomes, slicer);
|
|
1025
1300
|
stored.stagedTerminals = stored.stagedTerminals.filter((terminal) => !outcomes.has(terminal.runId));
|
|
1026
1301
|
if (stored.stagedTerminals.length > 0) stored.stagedTerminals = [];
|
|
1027
|
-
const
|
|
1028
|
-
const
|
|
1302
|
+
const activeEventIds = new Set<string>();
|
|
1303
|
+
for (const event of stored.agentSession.getEvents()) {
|
|
1304
|
+
activeEventIds.add(event.id);
|
|
1305
|
+
if (slicer.due) await slicer.pause();
|
|
1306
|
+
}
|
|
1029
1307
|
stored.reversionSegments = stored.reversionSegments.map((segment) => {
|
|
1030
1308
|
const contextAvailable = segment.contextAvailable
|
|
1031
1309
|
&& (segment.eventIds.length === 0 || segment.eventIds.some((id) => activeEventIds.has(id)));
|
|
@@ -1047,12 +1325,9 @@ export class FlexHarnessCore<TScope> {
|
|
|
1047
1325
|
this.reversions.commitArchivedHiddenBranchState(stored);
|
|
1048
1326
|
this.reversions.pruneReversionState(stored, false, true);
|
|
1049
1327
|
if (stored.pendingReversion?.kind === 'apply') {
|
|
1050
|
-
if (!state || scopeId === undefined || scope === undefined) {
|
|
1051
|
-
throw new FlexHarnessValidationError('Pending reversion recovery is missing its scope context.');
|
|
1052
|
-
}
|
|
1053
1328
|
await this.reversions.resumePendingApply(state, stored, scopeId, scope);
|
|
1054
1329
|
} else if (stored.pendingReversion?.kind === 'capture') {
|
|
1055
|
-
if (!this.turnReversionProvider
|
|
1330
|
+
if (!this.turnReversionProvider) {
|
|
1056
1331
|
throw new FlexHarnessValidationError('Pending capture recovery requires its reversion provider.');
|
|
1057
1332
|
}
|
|
1058
1333
|
await this.reversions.recoverPendingCapture(state, stored, scopeId, scope, outcomes);
|
|
@@ -1084,17 +1359,12 @@ export class FlexHarnessCore<TScope> {
|
|
|
1084
1359
|
};
|
|
1085
1360
|
stored.session.updatedAt = timestamp;
|
|
1086
1361
|
}
|
|
1087
|
-
|
|
1088
|
-
messages: stored.messages,
|
|
1089
|
-
stagedTerminals: stored.stagedTerminals,
|
|
1090
|
-
reversionSegments: stored.reversionSegments,
|
|
1091
|
-
revertCursor: stored.revertCursor,
|
|
1092
|
-
excludedRunIds: stored.excludedRunIds,
|
|
1093
|
-
pendingReversion: stored.pendingReversion,
|
|
1094
|
-
pendingReversionReleases: stored.pendingReversionReleases,
|
|
1095
|
-
});
|
|
1096
|
-
if (projectionChanged) {
|
|
1362
|
+
if (await this.projectionContentChanged(before, stored, slicer)) {
|
|
1097
1363
|
const expected = stored.projectionRevision;
|
|
1364
|
+
for (const message of stored.messages) {
|
|
1365
|
+
deepFreeze(message);
|
|
1366
|
+
if (slicer.due) await slicer.pause();
|
|
1367
|
+
}
|
|
1098
1368
|
freezeProjectionEntries(stored);
|
|
1099
1369
|
const change = this.createProjectionChange(stored, expected + 1);
|
|
1100
1370
|
await this.stores.projections.save(
|
|
@@ -1112,6 +1382,215 @@ export class FlexHarnessCore<TScope> {
|
|
|
1112
1382
|
return beforeSession !== JSON.stringify(stored.session);
|
|
1113
1383
|
}
|
|
1114
1384
|
|
|
1385
|
+
/**
|
|
1386
|
+
* Gives every run of a loaded session its terminal, in one pass over its messages, and returns
|
|
1387
|
+
* the terminal of its latest real turn. A run with an accepted completed stage takes the stage;
|
|
1388
|
+
* a run that failed or was stopped keeps the terminal it staged or published; a run that never
|
|
1389
|
+
* reached its terminal is repaired as failed or cancelled. Each terminal goes where
|
|
1390
|
+
* `runs.replaceVisibleRun()` would put it.
|
|
1391
|
+
*/
|
|
1392
|
+
private async repairRunTerminals(
|
|
1393
|
+
stored: IStoredSessionState,
|
|
1394
|
+
agentEvents: readonly plugins.TAgentEvent[],
|
|
1395
|
+
outcomes: ReadonlyMap<string, TCanonicalOutcome>,
|
|
1396
|
+
slicer: FlexTimeSlicer,
|
|
1397
|
+
): Promise<IFlexTerminalProjection | undefined> {
|
|
1398
|
+
const runMessages = new Map<string, IFlexMessage[]>();
|
|
1399
|
+
const lastRunMessage = new Map<string, IFlexMessage>();
|
|
1400
|
+
for (const message of stored.messages) {
|
|
1401
|
+
let visible = runMessages.get(message.runId);
|
|
1402
|
+
if (!visible) {
|
|
1403
|
+
visible = [];
|
|
1404
|
+
runMessages.set(message.runId, visible);
|
|
1405
|
+
}
|
|
1406
|
+
visible.push(message);
|
|
1407
|
+
lastRunMessage.set(message.runId, message);
|
|
1408
|
+
if (slicer.due) await slicer.pause();
|
|
1409
|
+
}
|
|
1410
|
+
const stages = new Map(stored.stagedTerminals.map((terminal) => [terminal.runId, terminal]));
|
|
1411
|
+
const runIds = new Set([
|
|
1412
|
+
...runMessages.keys(),
|
|
1413
|
+
...stored.stagedTerminals.map((terminal) => terminal.runId),
|
|
1414
|
+
]);
|
|
1415
|
+
let modelContextRunIds: Set<string> | undefined;
|
|
1416
|
+
const inModelContext = (runId: string) =>
|
|
1417
|
+
(modelContextRunIds ??= plugins.getGenerationsInModelContext(agentEvents)).has(runId);
|
|
1418
|
+
const terminals = new Map<string, IFlexTerminalProjection>();
|
|
1419
|
+
// The session's state is the latest real turn's: a cache-warm turn's terminal never sets it. The
|
|
1420
|
+
// terminal of an accepted run without a completed stage only names that state, so it is built
|
|
1421
|
+
// for the latest such run alone.
|
|
1422
|
+
let latestTerminal: IFlexTerminalProjection | (() => IFlexTerminalProjection) | undefined;
|
|
1423
|
+
for (const runId of runIds) {
|
|
1424
|
+
const outcome = outcomes.get(runId);
|
|
1425
|
+
const stage = stages.get(runId);
|
|
1426
|
+
const visible = runMessages.get(runId) ?? [];
|
|
1427
|
+
if (outcome === 'accepted' && stage?.status === 'completed') {
|
|
1428
|
+
// A cache-warm exchange leaves the model context with its outcome.
|
|
1429
|
+
terminals.set(runId, this.runs.withKeptInContext(stage, !stage.assistantMessage.cacheWarm));
|
|
1430
|
+
if (!stage.assistantMessage.cacheWarm) latestTerminal = stage;
|
|
1431
|
+
} else if (outcome !== 'accepted') {
|
|
1432
|
+
// A run that failed or was stopped keeps its terminal and its error: the staged one, or the
|
|
1433
|
+
// one it published. Only a run that never reached its terminal is repaired as interrupted.
|
|
1434
|
+
const ended = stage && stage.status !== 'completed'
|
|
1435
|
+
? stage
|
|
1436
|
+
: stage ? undefined : this.publishedTerminal(runId, visible);
|
|
1437
|
+
if (ended) {
|
|
1438
|
+
if (ended === stage) {
|
|
1439
|
+
terminals.set(runId, this.runs.withKeptInContext(stage, inModelContext(runId)));
|
|
1440
|
+
}
|
|
1441
|
+
if (!ended.assistantMessage.cacheWarm) latestTerminal = ended;
|
|
1442
|
+
} else if (visible.length > 0 || stage) {
|
|
1443
|
+
const status = outcome === 'rejected' ? 'failed' : 'cancelled';
|
|
1444
|
+
const terminal = this.runs.withKeptInContext(
|
|
1445
|
+
this.repairTerminal(stored.session.sessionId, runId, visible, stage, status),
|
|
1446
|
+
inModelContext(runId),
|
|
1447
|
+
);
|
|
1448
|
+
terminals.set(runId, terminal);
|
|
1449
|
+
if (!terminal.assistantMessage.cacheWarm) latestTerminal = terminal;
|
|
1450
|
+
}
|
|
1451
|
+
} else if (visible.some((message) => message.status === 'completed')) {
|
|
1452
|
+
let assistant: IFlexMessage | undefined;
|
|
1453
|
+
for (const message of visible) if (message.role === 'assistant') assistant = message;
|
|
1454
|
+
if (!assistant?.cacheWarm) {
|
|
1455
|
+
latestTerminal = () =>
|
|
1456
|
+
this.repairTerminal(stored.session.sessionId, runId, visible, undefined, 'completed');
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1459
|
+
if (slicer.due) await slicer.pause();
|
|
1460
|
+
}
|
|
1461
|
+
if (terminals.size > 0) {
|
|
1462
|
+
stored.messages = await this.placeRunTerminals(
|
|
1463
|
+
stored.messages,
|
|
1464
|
+
terminals,
|
|
1465
|
+
runMessages,
|
|
1466
|
+
lastRunMessage,
|
|
1467
|
+
inModelContext,
|
|
1468
|
+
slicer,
|
|
1469
|
+
);
|
|
1470
|
+
}
|
|
1471
|
+
return typeof latestTerminal === 'function' ? latestTerminal() : latestTerminal;
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
/**
|
|
1475
|
+
* The messages with each run's terminal in place, built in one pass: what
|
|
1476
|
+
* `runs.replaceVisibleRun()` gives for each terminal in turn. A run's projected terminal messages
|
|
1477
|
+
* are replaced, its other user messages take the prompt's `keptInContext`, and a terminal
|
|
1478
|
+
* message not projected yet goes before the run's first message (its prompt) or after its last
|
|
1479
|
+
* (its answer); for a run without messages both go to the end.
|
|
1480
|
+
*/
|
|
1481
|
+
private async placeRunTerminals(
|
|
1482
|
+
messages: readonly IFlexMessage[],
|
|
1483
|
+
terminals: ReadonlyMap<string, IFlexTerminalProjection>,
|
|
1484
|
+
runMessages: ReadonlyMap<string, readonly IFlexMessage[]>,
|
|
1485
|
+
lastRunMessage: ReadonlyMap<string, IFlexMessage>,
|
|
1486
|
+
inModelContext: (runId: string) => boolean,
|
|
1487
|
+
slicer: FlexTimeSlicer,
|
|
1488
|
+
): Promise<IFlexMessage[]> {
|
|
1489
|
+
interface IPlacement {
|
|
1490
|
+
terminal: IFlexTerminalProjection;
|
|
1491
|
+
replacements: Map<string, IFlexMessage>;
|
|
1492
|
+
keptInContext: boolean;
|
|
1493
|
+
placeUser: boolean;
|
|
1494
|
+
placeAssistant: boolean;
|
|
1495
|
+
}
|
|
1496
|
+
const placements = new Map<string, IPlacement>();
|
|
1497
|
+
for (const [runId, terminal] of terminals) {
|
|
1498
|
+
const replacements = new Map([
|
|
1499
|
+
...(terminal.userMessage ? [[terminal.userMessage.messageId, terminal.userMessage] as const] : []),
|
|
1500
|
+
[terminal.assistantMessage.messageId, terminal.assistantMessage],
|
|
1501
|
+
]);
|
|
1502
|
+
const projectedIds = new Set((runMessages.get(runId) ?? []).map((message) => message.messageId));
|
|
1503
|
+
placements.set(runId, {
|
|
1504
|
+
terminal,
|
|
1505
|
+
replacements,
|
|
1506
|
+
keptInContext: terminal.userMessage?.keptInContext
|
|
1507
|
+
?? (terminal.status === 'completed' || inModelContext(runId)),
|
|
1508
|
+
placeUser: terminal.userMessage !== undefined && !projectedIds.has(terminal.userMessage.messageId),
|
|
1509
|
+
placeAssistant: !projectedIds.has(terminal.assistantMessage.messageId),
|
|
1510
|
+
});
|
|
1511
|
+
}
|
|
1512
|
+
const placed: IFlexMessage[] = [];
|
|
1513
|
+
const startedRuns = new Set<string>();
|
|
1514
|
+
for (const message of messages) {
|
|
1515
|
+
const placement = placements.get(message.runId);
|
|
1516
|
+
if (!placement) {
|
|
1517
|
+
placed.push(message);
|
|
1518
|
+
} else {
|
|
1519
|
+
if (!startedRuns.has(message.runId)) {
|
|
1520
|
+
startedRuns.add(message.runId);
|
|
1521
|
+
if (placement.placeUser) placed.push(cloneSerializable(placement.terminal.userMessage!));
|
|
1522
|
+
}
|
|
1523
|
+
const replacement = placement.replacements.get(message.messageId);
|
|
1524
|
+
if (replacement) {
|
|
1525
|
+
placed.push(cloneSerializable(replacement));
|
|
1526
|
+
} else {
|
|
1527
|
+
// The steers the run took in share its prompt's outcome.
|
|
1528
|
+
placed.push(message.role === 'user' && message.keptInContext !== placement.keptInContext
|
|
1529
|
+
? { ...cloneSerializable(message), keptInContext: placement.keptInContext }
|
|
1530
|
+
: message);
|
|
1531
|
+
}
|
|
1532
|
+
if (placement.placeAssistant && lastRunMessage.get(message.runId) === message) {
|
|
1533
|
+
placed.push(cloneSerializable(placement.terminal.assistantMessage));
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
if (slicer.due) await slicer.pause();
|
|
1537
|
+
}
|
|
1538
|
+
for (const [runId, placement] of placements) {
|
|
1539
|
+
if (startedRuns.has(runId)) continue;
|
|
1540
|
+
if (placement.placeUser) placed.push(cloneSerializable(placement.terminal.userMessage!));
|
|
1541
|
+
if (placement.placeAssistant) placed.push(cloneSerializable(placement.terminal.assistantMessage));
|
|
1542
|
+
}
|
|
1543
|
+
return placed;
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
/**
|
|
1547
|
+
* The projection content a repair may change, as it is before the repair. Messages and reversion
|
|
1548
|
+
* segments are frozen and replaced, never edited, so the lists keep their entries; the other
|
|
1549
|
+
* fields are small and kept encoded.
|
|
1550
|
+
*/
|
|
1551
|
+
private projectionContent(stored: IStoredSessionState): IProjectionContent {
|
|
1552
|
+
return {
|
|
1553
|
+
messages: stored.messages.slice(),
|
|
1554
|
+
reversionSegments: stored.reversionSegments.slice(),
|
|
1555
|
+
stagedTerminals: JSON.stringify(stored.stagedTerminals),
|
|
1556
|
+
revertCursor: stored.revertCursor,
|
|
1557
|
+
excludedRunIds: JSON.stringify(stored.excludedRunIds),
|
|
1558
|
+
pendingReversion: JSON.stringify(stored.pendingReversion),
|
|
1559
|
+
pendingReversionReleases: JSON.stringify(stored.pendingReversionReleases),
|
|
1560
|
+
};
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
/**
|
|
1564
|
+
* Whether the session's projection content differs from `before` as JSON. A list entry that is
|
|
1565
|
+
* still the same object is unchanged, so only an entry replaced since is encoded, one at a time.
|
|
1566
|
+
*/
|
|
1567
|
+
private async projectionContentChanged(
|
|
1568
|
+
before: IProjectionContent,
|
|
1569
|
+
stored: IStoredSessionState,
|
|
1570
|
+
slicer: FlexTimeSlicer,
|
|
1571
|
+
): Promise<boolean> {
|
|
1572
|
+
if (
|
|
1573
|
+
stored.revertCursor !== before.revertCursor
|
|
1574
|
+
|| JSON.stringify(stored.stagedTerminals) !== before.stagedTerminals
|
|
1575
|
+
|| JSON.stringify(stored.excludedRunIds) !== before.excludedRunIds
|
|
1576
|
+
|| JSON.stringify(stored.pendingReversion) !== before.pendingReversion
|
|
1577
|
+
|| JSON.stringify(stored.pendingReversionReleases) !== before.pendingReversionReleases
|
|
1578
|
+
) return true;
|
|
1579
|
+
const listChanged = async (previous: readonly object[], current: readonly object[]) => {
|
|
1580
|
+
if (previous.length !== current.length) return true;
|
|
1581
|
+
for (let index = 0; index < current.length; index++) {
|
|
1582
|
+
if (
|
|
1583
|
+
previous[index] !== current[index]
|
|
1584
|
+
&& JSON.stringify(previous[index]) !== JSON.stringify(current[index])
|
|
1585
|
+
) return true;
|
|
1586
|
+
if (slicer.due) await slicer.pause();
|
|
1587
|
+
}
|
|
1588
|
+
return false;
|
|
1589
|
+
};
|
|
1590
|
+
return await listChanged(before.messages, stored.messages)
|
|
1591
|
+
|| await listChanged(before.reversionSegments, stored.reversionSegments);
|
|
1592
|
+
}
|
|
1593
|
+
|
|
1115
1594
|
/**
|
|
1116
1595
|
* The failed or cancelled terminal a run published, read from its projected messages. A run
|
|
1117
1596
|
* still running has its prompt or its last assistant message streaming; it has none.
|
|
@@ -1273,6 +1752,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1273
1752
|
onlySessionId?: string,
|
|
1274
1753
|
): Promise<TValue> {
|
|
1275
1754
|
const beforeRevision = state.revision;
|
|
1755
|
+
const beforeCatalogue = new Map([...state.catalogue].map(([id, metadata]) => [id, cloneSerializable(metadata)]));
|
|
1276
1756
|
const beforeSessions = new Map([...state.sessions].map(([id, stored]) => [id, {
|
|
1277
1757
|
stored,
|
|
1278
1758
|
metadata: cloneSerializable(stored.session),
|
|
@@ -1284,18 +1764,20 @@ export class FlexHarnessCore<TScope> {
|
|
|
1284
1764
|
const beforeSnapshot = (): IFlexScopeSnapshot => ({
|
|
1285
1765
|
schemaVersion: 1,
|
|
1286
1766
|
revision: beforeRevision,
|
|
1287
|
-
sessions: [...
|
|
1288
|
-
.map((entry) => entry.metadata)
|
|
1767
|
+
sessions: [...beforeCatalogue.values()]
|
|
1289
1768
|
.sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
|
|
1290
1769
|
tombstones: [...beforeTombstones.values()]
|
|
1291
1770
|
.sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
|
|
1292
1771
|
});
|
|
1293
1772
|
const restore = () => {
|
|
1294
1773
|
state.revision = beforeRevision;
|
|
1774
|
+
state.catalogue.clear();
|
|
1775
|
+
for (const [id, metadata] of beforeCatalogue) state.catalogue.set(id, metadata);
|
|
1295
1776
|
state.sessions.clear();
|
|
1296
1777
|
for (const [id, entry] of beforeSessions) {
|
|
1297
1778
|
entry.stored.session = entry.metadata;
|
|
1298
1779
|
state.sessions.set(id, entry.stored);
|
|
1780
|
+
state.catalogue.set(id, entry.metadata);
|
|
1299
1781
|
}
|
|
1300
1782
|
state.tombstones.clear();
|
|
1301
1783
|
for (const [id, value] of beforeTombstones) state.tombstones.set(id, value);
|
|
@@ -1307,8 +1789,12 @@ export class FlexHarnessCore<TScope> {
|
|
|
1307
1789
|
let result: TValue;
|
|
1308
1790
|
try {
|
|
1309
1791
|
result = mutation();
|
|
1792
|
+
for (const id of beforeSessions.keys()) {
|
|
1793
|
+
if (!state.sessions.has(id)) state.catalogue.delete(id);
|
|
1794
|
+
}
|
|
1795
|
+
for (const [id, stored] of state.sessions) state.catalogue.set(id, stored.session);
|
|
1310
1796
|
if (onlySessionId !== undefined) {
|
|
1311
|
-
this.assertOnlySessionChanged(state, onlySessionId, beforeSessions, beforeTombstones, beforeRetainedCleanups);
|
|
1797
|
+
this.assertOnlySessionChanged(state, onlySessionId, beforeSessions, beforeTombstones, beforeRetainedCleanups, beforeCatalogue);
|
|
1312
1798
|
}
|
|
1313
1799
|
} catch (error) {
|
|
1314
1800
|
restore();
|
|
@@ -1316,7 +1802,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1316
1802
|
}
|
|
1317
1803
|
const snapshot = this.createScopeSnapshot(state, beforeRevision + 1);
|
|
1318
1804
|
try {
|
|
1319
|
-
await this.
|
|
1805
|
+
await this.saveScopeSnapshot(state, snapshot, beforeRevision);
|
|
1320
1806
|
state.revision = snapshot.revision;
|
|
1321
1807
|
return result;
|
|
1322
1808
|
} catch (error) {
|
|
@@ -1362,9 +1848,11 @@ export class FlexHarnessCore<TScope> {
|
|
|
1362
1848
|
beforeSessions: ReadonlyMap<string, { stored: IStoredSessionState; metadata: IFlexSession }>,
|
|
1363
1849
|
beforeTombstones: ReadonlyMap<string, IFlexSessionTombstone>,
|
|
1364
1850
|
beforeRetainedCleanups: ReadonlyMap<string, unknown>,
|
|
1851
|
+
beforeCatalogue: ReadonlyMap<string, IFlexSession>,
|
|
1365
1852
|
): void {
|
|
1366
1853
|
const changed = (id: string) => {
|
|
1367
1854
|
if (id === sessionId) return false;
|
|
1855
|
+
if (canonicalJson(beforeCatalogue.get(id)) !== canonicalJson(state.catalogue.get(id))) return true;
|
|
1368
1856
|
const before = beforeSessions.get(id);
|
|
1369
1857
|
const stored = state.sessions.get(id);
|
|
1370
1858
|
if (before?.stored !== stored) return true;
|
|
@@ -1373,6 +1861,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1373
1861
|
return beforeRetainedCleanups.get(id) !== state.retainedSessionCleanups.get(id);
|
|
1374
1862
|
};
|
|
1375
1863
|
const ids = new Set([
|
|
1864
|
+
...beforeCatalogue.keys(), ...state.catalogue.keys(),
|
|
1376
1865
|
...beforeSessions.keys(), ...state.sessions.keys(),
|
|
1377
1866
|
...beforeTombstones.keys(), ...state.tombstones.keys(),
|
|
1378
1867
|
...beforeRetainedCleanups.keys(), ...state.retainedSessionCleanups.keys(),
|
|
@@ -1386,141 +1875,185 @@ export class FlexHarnessCore<TScope> {
|
|
|
1386
1875
|
}
|
|
1387
1876
|
}
|
|
1388
1877
|
|
|
1878
|
+
/**
|
|
1879
|
+
* Changes `stored`'s projection in its FIFO and saves it. `onSaved` gets the checkpoint clock's
|
|
1880
|
+
* times the save started and completed, right after it completed.
|
|
1881
|
+
*/
|
|
1389
1882
|
public mutateProjection<TValue>(
|
|
1390
1883
|
state: IStorageState,
|
|
1391
1884
|
stored: IStoredSessionState,
|
|
1392
1885
|
mutation: () => TValue,
|
|
1393
1886
|
_allowDuringDrain = false,
|
|
1887
|
+
onSaved?: TProjectionSaved,
|
|
1394
1888
|
): Promise<TValue> {
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1889
|
+
return this.enqueueProjectionOperation(stored, () => this.saveProjectionMutation(state, stored, mutation, onSaved));
|
|
1890
|
+
}
|
|
1891
|
+
|
|
1892
|
+
/**
|
|
1893
|
+
* A running turn's projection checkpoint, at its turn in the projection FIFO: `prepare` decides
|
|
1894
|
+
* there what to save and returns the mutation, or nothing, which saves nothing. Resolves whether
|
|
1895
|
+
* it saved.
|
|
1896
|
+
*/
|
|
1897
|
+
public checkpointProjection(
|
|
1898
|
+
state: IStorageState,
|
|
1899
|
+
stored: IStoredSessionState,
|
|
1900
|
+
prepare: () => (() => void) | undefined,
|
|
1901
|
+
onSaved: TProjectionSaved,
|
|
1902
|
+
): Promise<boolean> {
|
|
1903
|
+
return this.enqueueProjectionOperation(stored, async () => {
|
|
1904
|
+
const mutation = prepare();
|
|
1905
|
+
if (!mutation) return false;
|
|
1906
|
+
await this.saveProjectionMutation(state, stored, mutation, onSaved);
|
|
1907
|
+
return true;
|
|
1908
|
+
});
|
|
1909
|
+
}
|
|
1910
|
+
|
|
1911
|
+
private enqueueProjectionOperation<TValue>(
|
|
1912
|
+
stored: IStoredSessionState,
|
|
1913
|
+
operation: () => Promise<TValue>,
|
|
1914
|
+
): Promise<TValue> {
|
|
1915
|
+
const queued = stored.projectionQueue.then(operation);
|
|
1916
|
+
stored.projectionQueue = queued.then(() => undefined, () => undefined);
|
|
1917
|
+
return queued;
|
|
1918
|
+
}
|
|
1919
|
+
|
|
1920
|
+
private async saveProjectionMutation<TValue>(
|
|
1921
|
+
state: IStorageState,
|
|
1922
|
+
stored: IStoredSessionState,
|
|
1923
|
+
mutation: () => TValue,
|
|
1924
|
+
onSaved: TProjectionSaved | undefined,
|
|
1925
|
+
): Promise<TValue> {
|
|
1926
|
+
const beforeRevision = stored.projectionRevision;
|
|
1927
|
+
const beforeSchemaVersion = stored.projectionSchemaVersion;
|
|
1928
|
+
const beforeReconciliationRequired = stored.projectionReconciliationRequired;
|
|
1929
|
+
// The baseline is deep-frozen and only ever replaced, never edited, so it needs no copy.
|
|
1930
|
+
const beforeBaseline = stored.projectionBaseline;
|
|
1931
|
+
const beforeMetrics = cloneSerializable(stored.metrics);
|
|
1932
|
+
// List entries are frozen and replaced rather than edited, so copies of the lists suffice.
|
|
1933
|
+
const beforeMessages = stored.messages.slice();
|
|
1934
|
+
const beforeStages = cloneSerializable(stored.stagedTerminals);
|
|
1935
|
+
const beforeSegments = stored.reversionSegments.slice();
|
|
1936
|
+
const beforeCursor = stored.revertCursor;
|
|
1937
|
+
const beforeExcludedRunIds = cloneSerializable(stored.excludedRunIds);
|
|
1938
|
+
const beforePendingReversion = stored.pendingReversion === undefined
|
|
1939
|
+
? undefined
|
|
1940
|
+
: cloneSerializable(stored.pendingReversion);
|
|
1941
|
+
const beforePendingReversionReleases = cloneSerializable(stored.pendingReversionReleases);
|
|
1942
|
+
// The copies above stay untouched until a restore, so the projection as it was before the
|
|
1943
|
+
// mutation is built from them only when a failed save has to be reconciled.
|
|
1944
|
+
const beforeSnapshot = (): TFlexProjectionSnapshot => beforeBaseline ?? {
|
|
1945
|
+
schemaVersion: 3,
|
|
1946
|
+
revision: beforeRevision,
|
|
1947
|
+
metrics: beforeMetrics,
|
|
1948
|
+
messages: beforeMessages,
|
|
1949
|
+
stagedTerminals: beforeStages,
|
|
1950
|
+
reversionSegments: beforeSegments,
|
|
1951
|
+
revertCursor: beforeCursor,
|
|
1952
|
+
excludedRunIds: beforeExcludedRunIds,
|
|
1953
|
+
...(beforePendingReversion === undefined ? {} : { pendingReversion: beforePendingReversion }),
|
|
1954
|
+
pendingReversionReleases: beforePendingReversionReleases,
|
|
1955
|
+
};
|
|
1956
|
+
const restore = () => {
|
|
1957
|
+
stored.projectionSchemaVersion = beforeSchemaVersion;
|
|
1958
|
+
stored.projectionReconciliationRequired = beforeReconciliationRequired;
|
|
1959
|
+
if (beforeBaseline === undefined) delete stored.projectionBaseline;
|
|
1960
|
+
else stored.projectionBaseline = beforeBaseline;
|
|
1961
|
+
stored.projectionRevision = beforeRevision;
|
|
1962
|
+
stored.metrics = beforeMetrics;
|
|
1963
|
+
stored.messages = beforeMessages;
|
|
1964
|
+
stored.stagedTerminals = beforeStages;
|
|
1965
|
+
stored.reversionSegments = beforeSegments;
|
|
1966
|
+
stored.revertCursor = beforeCursor;
|
|
1967
|
+
stored.excludedRunIds = beforeExcludedRunIds;
|
|
1968
|
+
if (beforePendingReversion === undefined) delete stored.pendingReversion;
|
|
1969
|
+
else stored.pendingReversion = beforePendingReversion;
|
|
1970
|
+
stored.pendingReversionReleases = beforePendingReversionReleases;
|
|
1971
|
+
};
|
|
1972
|
+
let result: TValue;
|
|
1973
|
+
try {
|
|
1974
|
+
result = mutation();
|
|
1975
|
+
} catch (error) {
|
|
1976
|
+
restore();
|
|
1977
|
+
throw error;
|
|
1978
|
+
}
|
|
1979
|
+
freezeProjectionEntries(stored);
|
|
1980
|
+
const revision = beforeRevision + 1;
|
|
1981
|
+
try {
|
|
1982
|
+
if (state.sessionLocal) await this.updateRecoveryObligation(state, stored, true);
|
|
1983
|
+
const saveStartedAt = onSaved ? this.checkpointClock.now() : 0;
|
|
1984
|
+
await this.stores.projections.save(
|
|
1985
|
+
state.storageKey,
|
|
1986
|
+
stored.session.sessionId,
|
|
1987
|
+
this.createProjectionChange(stored, revision),
|
|
1988
|
+
beforeRevision,
|
|
1989
|
+
);
|
|
1990
|
+
const saveCompletedAt = onSaved ? this.checkpointClock.now() : 0;
|
|
1991
|
+
stored.projectionSchemaVersion = 3;
|
|
1992
|
+
delete stored.projectionBaseline;
|
|
1993
|
+
this.rememberSavedProjectionLists(stored);
|
|
1994
|
+
stored.projectionReconciliationRequired = false;
|
|
1995
|
+
stored.projectionRevision = revision;
|
|
1996
|
+
onSaved?.(saveStartedAt, saveCompletedAt);
|
|
1997
|
+
} catch (error) {
|
|
1998
|
+
if (error instanceof FlexHarnessStoreConflictError) {
|
|
1447
1999
|
restore();
|
|
1448
2000
|
throw error;
|
|
1449
2001
|
}
|
|
1450
|
-
|
|
1451
|
-
|
|
2002
|
+
if (error instanceof FlexHarnessStoreCommitUncertainError) {
|
|
2003
|
+
stored.projectionSchemaVersion = 3;
|
|
2004
|
+
delete stored.savedProjectionLists;
|
|
2005
|
+
stored.projectionReconciliationRequired = true;
|
|
2006
|
+
stored.projectionRevision = revision;
|
|
2007
|
+
state.lifecycle = 'fenced';
|
|
2008
|
+
throw error;
|
|
2009
|
+
}
|
|
2010
|
+
const snapshot = this.createProjectionSnapshot(stored, revision);
|
|
2011
|
+
let current: TFlexProjectionSnapshot | undefined;
|
|
2012
|
+
let reconciliationError: unknown;
|
|
1452
2013
|
try {
|
|
1453
|
-
await this.stores.projections.
|
|
2014
|
+
current = await this.stores.projections.load(
|
|
1454
2015
|
state.storageKey,
|
|
1455
2016
|
stored.session.sessionId,
|
|
1456
|
-
this.createProjectionChange(stored, revision),
|
|
1457
|
-
beforeRevision,
|
|
1458
2017
|
);
|
|
2018
|
+
} catch (loadError) {
|
|
2019
|
+
reconciliationError = loadError;
|
|
2020
|
+
}
|
|
2021
|
+
if (current && JSON.stringify(current) === JSON.stringify(snapshot)) {
|
|
1459
2022
|
stored.projectionSchemaVersion = 3;
|
|
1460
2023
|
delete stored.projectionBaseline;
|
|
1461
2024
|
this.rememberSavedProjectionLists(stored);
|
|
1462
2025
|
stored.projectionReconciliationRequired = false;
|
|
1463
2026
|
stored.projectionRevision = revision;
|
|
1464
|
-
|
|
1465
|
-
} catch (error) {
|
|
1466
|
-
if (error instanceof FlexHarnessStoreConflictError) {
|
|
1467
|
-
restore();
|
|
1468
|
-
throw error;
|
|
1469
|
-
}
|
|
1470
|
-
if (error instanceof FlexHarnessStoreCommitUncertainError) {
|
|
1471
|
-
stored.projectionSchemaVersion = 3;
|
|
1472
|
-
delete stored.savedProjectionLists;
|
|
1473
|
-
stored.projectionReconciliationRequired = true;
|
|
1474
|
-
stored.projectionRevision = revision;
|
|
1475
|
-
state.lifecycle = 'fenced';
|
|
1476
|
-
throw error;
|
|
1477
|
-
}
|
|
1478
|
-
const snapshot = this.createProjectionSnapshot(stored, revision);
|
|
1479
|
-
let current: TFlexProjectionSnapshot | undefined;
|
|
1480
|
-
let reconciliationError: unknown;
|
|
1481
|
-
try {
|
|
1482
|
-
current = await this.stores.projections.load(
|
|
1483
|
-
state.storageKey,
|
|
1484
|
-
stored.session.sessionId,
|
|
1485
|
-
);
|
|
1486
|
-
} catch (loadError) {
|
|
1487
|
-
reconciliationError = loadError;
|
|
1488
|
-
}
|
|
1489
|
-
if (current && JSON.stringify(current) === JSON.stringify(snapshot)) {
|
|
1490
|
-
stored.projectionSchemaVersion = 3;
|
|
1491
|
-
delete stored.projectionBaseline;
|
|
1492
|
-
this.rememberSavedProjectionLists(stored);
|
|
1493
|
-
stored.projectionReconciliationRequired = false;
|
|
1494
|
-
stored.projectionRevision = revision;
|
|
1495
|
-
throw error;
|
|
1496
|
-
}
|
|
1497
|
-
if (
|
|
1498
|
-
(current === undefined && beforeRevision === 0)
|
|
1499
|
-
|| (current !== undefined && JSON.stringify(current) === JSON.stringify(beforeSnapshot()))
|
|
1500
|
-
) {
|
|
1501
|
-
restore();
|
|
1502
|
-
throw error;
|
|
1503
|
-
}
|
|
1504
|
-
stored.projectionSchemaVersion = 3;
|
|
1505
|
-
delete stored.savedProjectionLists;
|
|
1506
|
-
stored.projectionReconciliationRequired = true;
|
|
1507
|
-
stored.projectionRevision = revision;
|
|
1508
|
-
state.lifecycle = 'fenced';
|
|
1509
|
-
throw reconciliationError === undefined
|
|
1510
|
-
? error
|
|
1511
|
-
: combineErrors([error, reconciliationError]);
|
|
2027
|
+
throw error;
|
|
1512
2028
|
}
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
2029
|
+
if (
|
|
2030
|
+
(current === undefined && beforeRevision === 0)
|
|
2031
|
+
|| (current !== undefined && JSON.stringify(current) === JSON.stringify(beforeSnapshot()))
|
|
2032
|
+
) {
|
|
2033
|
+
restore();
|
|
2034
|
+
throw error;
|
|
2035
|
+
}
|
|
2036
|
+
stored.projectionSchemaVersion = 3;
|
|
2037
|
+
delete stored.savedProjectionLists;
|
|
2038
|
+
stored.projectionReconciliationRequired = true;
|
|
2039
|
+
stored.projectionRevision = revision;
|
|
2040
|
+
state.lifecycle = 'fenced';
|
|
2041
|
+
throw reconciliationError === undefined
|
|
2042
|
+
? error
|
|
2043
|
+
: combineErrors([error, reconciliationError]);
|
|
2044
|
+
}
|
|
2045
|
+
// The projection is saved: what follows is a separate durable step, never classified as the
|
|
2046
|
+
// save's failure.
|
|
2047
|
+
if (state.sessionLocal) await this.settleRecoveryObligation(state, stored);
|
|
2048
|
+
return result;
|
|
1516
2049
|
}
|
|
1517
2050
|
|
|
1518
2051
|
private createScopeSnapshot(state: IStorageState, revision: number): IFlexScopeSnapshot {
|
|
1519
2052
|
return {
|
|
1520
2053
|
schemaVersion: 1,
|
|
1521
2054
|
revision,
|
|
1522
|
-
sessions: [...state.
|
|
1523
|
-
.map((
|
|
2055
|
+
sessions: [...state.catalogue.values()]
|
|
2056
|
+
.map((metadata) => cloneSerializable(state.sessions.get(metadata.sessionId)?.session ?? metadata))
|
|
1524
2057
|
.sort((left, right) => left.sessionId.localeCompare(right.sessionId)),
|
|
1525
2058
|
tombstones: [...state.tombstones.values()]
|
|
1526
2059
|
.map((value) => cloneSerializable(value))
|
|
@@ -1645,7 +2178,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1645
2178
|
delete stored.projectionBaseline;
|
|
1646
2179
|
this.rememberSavedProjectionLists(stored);
|
|
1647
2180
|
} else {
|
|
1648
|
-
stored.projectionBaseline = cloneSerializable(projection);
|
|
2181
|
+
stored.projectionBaseline = deepFreeze(cloneSerializable(projection));
|
|
1649
2182
|
delete stored.savedProjectionLists;
|
|
1650
2183
|
}
|
|
1651
2184
|
stored.projectionSchemaVersion = projection.schemaVersion;
|
|
@@ -1701,6 +2234,7 @@ export class FlexHarnessCore<TScope> {
|
|
|
1701
2234
|
|
|
1702
2235
|
private sessionMetadata(state: IStorageState, sessionId: string): IFlexSession | undefined {
|
|
1703
2236
|
return state.sessions.get(sessionId)?.session
|
|
2237
|
+
?? state.catalogue.get(sessionId)
|
|
1704
2238
|
?? state.retainedSessionCleanups.get(sessionId)?.stored.session;
|
|
1705
2239
|
}
|
|
1706
2240
|
|
|
@@ -1712,29 +2246,15 @@ export class FlexHarnessCore<TScope> {
|
|
|
1712
2246
|
|
|
1713
2247
|
public async resolveState(
|
|
1714
2248
|
scopeId: string,
|
|
2249
|
+
sessionId?: string,
|
|
1715
2250
|
): Promise<{ scope: IFlexResolvedScope<TScope>; state: IStorageState }> {
|
|
1716
|
-
this.
|
|
1717
|
-
validateIdentifier(scopeId, 'scopeId');
|
|
1718
|
-
const admission = this.getScopeAdmission(scopeId);
|
|
1719
|
-
if (admission.retiring) throw this.createScopeRetirementError();
|
|
1720
|
-
const generation = admission.generation;
|
|
1721
|
-
admission.inFlightResolvers++;
|
|
1722
|
-
try {
|
|
1723
|
-
const scope = await this.scopeResolver.resolveScope(scopeId);
|
|
1724
|
-
this.assertOpen();
|
|
1725
|
-
if (admission.retiring || admission.generation !== generation) throw this.createScopeRetirementError();
|
|
1726
|
-
validateIdentifier(scope.storageKey, 'resolved storageKey');
|
|
1727
|
-
if (this.lifecycle.storageDrains.has(scope.storageKey)) throw this.createScopeRetirementError();
|
|
2251
|
+
return this.withScopeAdmission(scopeId, async (scope, admitted) => {
|
|
1728
2252
|
let stateLoad = this.stateLoads.get(scope.storageKey);
|
|
1729
2253
|
if (!stateLoad) {
|
|
1730
2254
|
const orphanErrors = await this.lifecycle.closeOrphanedResources(scope.storageKey);
|
|
1731
2255
|
if (orphanErrors.length > 0) throw combineErrors(orphanErrors);
|
|
1732
2256
|
this.assertOpen();
|
|
1733
|
-
if (
|
|
1734
|
-
admission.retiring
|
|
1735
|
-
|| admission.generation !== generation
|
|
1736
|
-
|| this.lifecycle.storageDrains.has(scope.storageKey)
|
|
1737
|
-
) throw this.createScopeRetirementError();
|
|
2257
|
+
if (!admitted()) throw this.createScopeRetirementError();
|
|
1738
2258
|
stateLoad = this.stateLoads.get(scope.storageKey);
|
|
1739
2259
|
if (!stateLoad) {
|
|
1740
2260
|
const compactorLifecycleController = new AbortController();
|
|
@@ -1742,15 +2262,18 @@ export class FlexHarnessCore<TScope> {
|
|
|
1742
2262
|
scope.storageKey,
|
|
1743
2263
|
compactorLifecycleController,
|
|
1744
2264
|
);
|
|
1745
|
-
|
|
2265
|
+
const load = this.loadState(
|
|
1746
2266
|
scope.storageKey,
|
|
1747
2267
|
scopeId,
|
|
1748
2268
|
scope.scope,
|
|
1749
2269
|
compactorLifecycleController,
|
|
1750
2270
|
);
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
2271
|
+
stateLoad = load;
|
|
2272
|
+
this.stateLoads.set(scope.storageKey, load);
|
|
2273
|
+
void load.then((state) => {
|
|
2274
|
+
this.loadedStates.set(load, state);
|
|
2275
|
+
}, () => {
|
|
2276
|
+
if (this.stateLoads.get(scope.storageKey) === load) {
|
|
1754
2277
|
this.stateLoads.delete(scope.storageKey);
|
|
1755
2278
|
if (
|
|
1756
2279
|
this.lifecycle.storageCompactorLifecycleControllers.get(scope.storageKey)
|
|
@@ -1760,38 +2283,83 @@ export class FlexHarnessCore<TScope> {
|
|
|
1760
2283
|
});
|
|
1761
2284
|
}
|
|
1762
2285
|
}
|
|
1763
|
-
const state = await stateLoad;
|
|
1764
|
-
this.
|
|
1765
|
-
if (state.
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
{ scopeId, scope: scope.scope },
|
|
1771
|
-
);
|
|
1772
|
-
if (this.commands.ambientSlashCommandOwner({ storageKey: scope.storageKey })) {
|
|
1773
|
-
throw this.createScopeRetirementError();
|
|
1774
|
-
}
|
|
1775
|
-
try {
|
|
1776
|
-
await drain;
|
|
1777
|
-
} catch {
|
|
1778
|
-
// Fenced cleanup ownership remains available to explicit retirement or disposal.
|
|
1779
|
-
}
|
|
1780
|
-
throw this.createScopeRetirementError();
|
|
2286
|
+
const state = await this.admittedState(scopeId, scope, stateLoad, admitted);
|
|
2287
|
+
if (state.sessionLocal) await this.recoverSharedState(state, scopeId, scope.scope);
|
|
2288
|
+
if (sessionId !== undefined && state.catalogue.has(sessionId)) {
|
|
2289
|
+
await this.hydrateSession(state, sessionId, scopeId, scope.scope);
|
|
2290
|
+
this.assertOpen();
|
|
2291
|
+
if (!admitted()) throw this.createScopeRetirementError();
|
|
2292
|
+
this.assertStateAcceptingWork(state);
|
|
1781
2293
|
}
|
|
1782
|
-
if (
|
|
1783
|
-
admission.retiring
|
|
1784
|
-
|| admission.generation !== generation
|
|
1785
|
-
|| this.lifecycle.storageDrains.has(scope.storageKey)
|
|
1786
|
-
|| state.lifecycle !== 'active'
|
|
1787
|
-
) throw this.createScopeRetirementError();
|
|
1788
2294
|
return { scope, state };
|
|
2295
|
+
});
|
|
2296
|
+
}
|
|
2297
|
+
|
|
2298
|
+
/** Whether `stateLoad` has settled with its loaded state. */
|
|
2299
|
+
public isStateLoaded(stateLoad: Promise<IStorageState>): boolean {
|
|
2300
|
+
return this.loadedStates.has(stateLoad);
|
|
2301
|
+
}
|
|
2302
|
+
|
|
2303
|
+
/**
|
|
2304
|
+
* Runs `operation` for the resolved scope `scopeId` while the scope is admitted: not retiring,
|
|
2305
|
+
* not retired since, not draining. `admitted()` tells whether that still holds.
|
|
2306
|
+
*/
|
|
2307
|
+
public async withScopeAdmission<TValue>(
|
|
2308
|
+
scopeId: string,
|
|
2309
|
+
operation: (scope: IFlexResolvedScope<TScope>, admitted: () => boolean) => Promise<TValue>,
|
|
2310
|
+
): Promise<TValue> {
|
|
2311
|
+
this.assertOpen();
|
|
2312
|
+
validateIdentifier(scopeId, 'scopeId');
|
|
2313
|
+
const admission = this.getScopeAdmission(scopeId);
|
|
2314
|
+
if (admission.retiring) throw this.createScopeRetirementError();
|
|
2315
|
+
const generation = admission.generation;
|
|
2316
|
+
admission.inFlightResolvers++;
|
|
2317
|
+
try {
|
|
2318
|
+
const scope = await this.scopeResolver.resolveScope(scopeId);
|
|
2319
|
+
this.assertOpen();
|
|
2320
|
+
if (admission.retiring || admission.generation !== generation) throw this.createScopeRetirementError();
|
|
2321
|
+
validateIdentifier(scope.storageKey, 'resolved storageKey');
|
|
2322
|
+
if (this.lifecycle.storageDrains.has(scope.storageKey)) throw this.createScopeRetirementError();
|
|
2323
|
+
return await operation(scope, () =>
|
|
2324
|
+
!admission.retiring
|
|
2325
|
+
&& admission.generation === generation
|
|
2326
|
+
&& !this.lifecycle.storageDrains.has(scope.storageKey));
|
|
1789
2327
|
} finally {
|
|
1790
2328
|
admission.inFlightResolvers--;
|
|
1791
2329
|
this.pruneScopeAdmission(scopeId, admission);
|
|
1792
2330
|
}
|
|
1793
2331
|
}
|
|
1794
2332
|
|
|
2333
|
+
/** The state `stateLoad` loads, once it is active and the scope still admitted. */
|
|
2334
|
+
public async admittedState(
|
|
2335
|
+
scopeId: string,
|
|
2336
|
+
scope: IFlexResolvedScope<TScope>,
|
|
2337
|
+
stateLoad: Promise<IStorageState>,
|
|
2338
|
+
admitted: () => boolean,
|
|
2339
|
+
): Promise<IStorageState> {
|
|
2340
|
+
const state = await stateLoad;
|
|
2341
|
+
this.assertOpen();
|
|
2342
|
+
if (state.lifecycle === 'fenced') {
|
|
2343
|
+
const drain = this.lifecycle.storageDrains.get(scope.storageKey) ?? this.lifecycle.drainStorage(
|
|
2344
|
+
scope.storageKey,
|
|
2345
|
+
stateLoad,
|
|
2346
|
+
this.trustInternalError(new FlexHarnessAbortError('The session namespace was fenced.')),
|
|
2347
|
+
{ scopeId, scope: scope.scope },
|
|
2348
|
+
);
|
|
2349
|
+
if (this.commands.ambientSlashCommandOwner({ storageKey: scope.storageKey })) {
|
|
2350
|
+
throw this.createScopeRetirementError();
|
|
2351
|
+
}
|
|
2352
|
+
try {
|
|
2353
|
+
await drain;
|
|
2354
|
+
} catch {
|
|
2355
|
+
// Fenced cleanup ownership remains available to explicit retirement or disposal.
|
|
2356
|
+
}
|
|
2357
|
+
throw this.createScopeRetirementError();
|
|
2358
|
+
}
|
|
2359
|
+
if (!admitted() || state.lifecycle !== 'active') throw this.createScopeRetirementError();
|
|
2360
|
+
return state;
|
|
2361
|
+
}
|
|
2362
|
+
|
|
1795
2363
|
public getScopeAdmission(scopeId: string): IScopeAdmissionState {
|
|
1796
2364
|
let admission = this.scopeAdmissions.get(scopeId);
|
|
1797
2365
|
if (!admission) {
|
|
@@ -1956,16 +2524,22 @@ export class FlexHarnessCore<TScope> {
|
|
|
1956
2524
|
* one id each; a position is searched again only after it moved (a steer, a restored or
|
|
1957
2525
|
* reloaded projection).
|
|
1958
2526
|
*/
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
partId: string,
|
|
1962
|
-
): { messageIndex: number; partIndex: number } {
|
|
2527
|
+
/** The run's remembered positions, its assistant message's checked; -1 when it is not projected. */
|
|
2528
|
+
public assistantMessagePositions(run: IActiveRun): NonNullable<IActiveRun['partEventPositions']> {
|
|
1963
2529
|
const positions = run.partEventPositions ??= { messageIndex: -1, partIndexes: new Map() };
|
|
1964
2530
|
if (run.stored.messages[positions.messageIndex]?.messageId !== run.assistantMessageId) {
|
|
1965
2531
|
positions.messageIndex = run.stored.messages.findIndex(
|
|
1966
2532
|
(message) => message.messageId === run.assistantMessageId,
|
|
1967
2533
|
);
|
|
1968
2534
|
}
|
|
2535
|
+
return positions;
|
|
2536
|
+
}
|
|
2537
|
+
|
|
2538
|
+
private partEventCoordinates(
|
|
2539
|
+
run: IActiveRun,
|
|
2540
|
+
partId: string,
|
|
2541
|
+
): { messageIndex: number; partIndex: number } {
|
|
2542
|
+
const positions = this.assistantMessagePositions(run);
|
|
1969
2543
|
let partIndex = positions.partIndexes.get(partId) ?? -1;
|
|
1970
2544
|
if (run.callbackParts[partIndex]?.partId !== partId) {
|
|
1971
2545
|
partIndex = run.callbackParts.findIndex((entry) => entry.partId === partId);
|