@modelprofile.com/flexharness 18.8.0 → 18.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.flexharness.core.d.ts +19 -0
- package/dist_ts/classes.flexharness.core.js +358 -67
- package/dist_ts/classes.flexharness.deletion.d.ts +1 -1
- package/dist_ts/classes.flexharness.deletion.js +147 -141
- package/dist_ts/classes.flexharness.imports.js +3 -3
- package/dist_ts/classes.flexharness.lifecycle.js +4 -1
- package/dist_ts/classes.flexharness.permissions.js +5 -5
- package/dist_ts/classes.flexharness.prompts.js +5 -6
- package/dist_ts/classes.flexharness.results.js +3 -3
- package/dist_ts/classes.flexharness.reversions.js +320 -316
- package/dist_ts/classes.flexharness.sessions.js +19 -3
- package/dist_ts/classes.flexharness.subagents.js +2 -2
- package/dist_ts/classes.recoverygate.d.ts +30 -0
- package/dist_ts/classes.recoverygate.js +105 -0
- package/dist_ts/classes.scoperecovery.d.ts +9 -1
- package/dist_ts/classes.scoperecovery.js +94 -26
- package/dist_ts/interfaces.d.ts +22 -0
- package/dist_ts/interfaces.flexharness.d.ts +5 -0
- package/dist_ts/interfaces.js +1 -1
- package/dist_ts/plugins.d.ts +1 -1
- package/dist_ts/plugins.js +1 -1
- package/dist_ts_stores_nosqldb/classes.nosqldbstores.js +5 -1
- package/package.json +3 -3
- package/readme.md +5 -3
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.flexharness.core.ts +337 -34
- package/ts/classes.flexharness.deletion.ts +24 -15
- package/ts/classes.flexharness.imports.ts +3 -2
- package/ts/classes.flexharness.lifecycle.ts +3 -0
- package/ts/classes.flexharness.permissions.ts +8 -4
- package/ts/classes.flexharness.prompts.ts +10 -8
- package/ts/classes.flexharness.results.ts +3 -2
- package/ts/classes.flexharness.reversions.ts +13 -6
- package/ts/classes.flexharness.sessions.ts +18 -2
- package/ts/classes.flexharness.subagents.ts +5 -4
- package/ts/classes.recoverygate.ts +111 -0
- package/ts/classes.scoperecovery.ts +102 -25
- package/ts/interfaces.flexharness.ts +5 -0
- package/ts/interfaces.ts +23 -0
- package/ts/plugins.ts +1 -0
- package/ts/readme.md +5 -3
- package/ts_stores_nosqldb/classes.nosqldbstores.ts +5 -0
- package/ts_stores_nosqldb/readme.md +1 -3
|
@@ -89,7 +89,7 @@ export class FlexHarnessSessionDeletion<TScope> {
|
|
|
89
89
|
],
|
|
90
90
|
validated.authorizedBySessionId,
|
|
91
91
|
);
|
|
92
|
-
const reservation = await this.reserveFencedSessionDeletion(state, scopeId, scope.scope, () =>
|
|
92
|
+
const reservation = await this.reserveFencedSessionDeletion(state, scopeId, scope.scope, validated.root.sessionId, () =>
|
|
93
93
|
this.reserveLiveSessionDeletion(
|
|
94
94
|
state,
|
|
95
95
|
validated.root.sessionId,
|
|
@@ -146,7 +146,7 @@ export class FlexHarnessSessionDeletion<TScope> {
|
|
|
146
146
|
if (!session) {
|
|
147
147
|
return { deletion: this.startSessionDeletion(state, scopeId, scope.scope, sessionId) };
|
|
148
148
|
}
|
|
149
|
-
const reservation = await this.reserveFencedSessionDeletion(state, scopeId, scope.scope, () =>
|
|
149
|
+
const reservation = await this.reserveFencedSessionDeletion(state, scopeId, scope.scope, sessionId, () =>
|
|
150
150
|
this.reserveLiveSessionDeletion(
|
|
151
151
|
state,
|
|
152
152
|
sessionId,
|
|
@@ -306,7 +306,7 @@ export class FlexHarnessSessionDeletion<TScope> {
|
|
|
306
306
|
return;
|
|
307
307
|
}
|
|
308
308
|
const reservation = await this.core.withScopeMutationOwnership(state, () =>
|
|
309
|
-
this.reserveFencedSessionDeletion(state, scopeId, scope, () => {
|
|
309
|
+
this.reserveFencedSessionDeletion(state, scopeId, scope, sessionId, () => {
|
|
310
310
|
const session = state.catalogue.get(sessionId);
|
|
311
311
|
if (!session) throw new FlexHarnessNotFoundError('Session', sessionId);
|
|
312
312
|
const subtree = this.collectSessionSubtree(state, sessionId);
|
|
@@ -331,12 +331,19 @@ export class FlexHarnessSessionDeletion<TScope> {
|
|
|
331
331
|
* the reservation itself, since promotion refuses a session the scope no longer holds. A save
|
|
332
332
|
* that fails rolls the reservation back: the sessions stay, their runs cancelled.
|
|
333
333
|
*/
|
|
334
|
-
private reserveFencedSessionDeletion(
|
|
334
|
+
private async reserveFencedSessionDeletion(
|
|
335
335
|
state: IStorageState,
|
|
336
336
|
scopeId: string,
|
|
337
337
|
scope: TScope,
|
|
338
|
+
sessionId: string,
|
|
338
339
|
reserve: () => IReservedSessionDeletion,
|
|
339
340
|
): Promise<IReservedSessionDeletion> {
|
|
341
|
+
const subtree = this.collectSessionSubtree(state, sessionId);
|
|
342
|
+
return this.core.withRecoveryProducerSections(state, subtree.map((session) => session.sessionId), async () => {
|
|
343
|
+
if (state.sessionLocal) {
|
|
344
|
+
await Promise.all(subtree.map((session) =>
|
|
345
|
+
this.core.markRecoveryRequired(state, session.sessionId)));
|
|
346
|
+
}
|
|
340
347
|
let reserved: IReservedSessionDeletion | undefined;
|
|
341
348
|
// `mutateScopeOwned` applies the mutation before its first await, so the fence precedes the save.
|
|
342
349
|
const saved = this.core.mutateScopeOwned(state, () => {
|
|
@@ -345,6 +352,7 @@ export class FlexHarnessSessionDeletion<TScope> {
|
|
|
345
352
|
});
|
|
346
353
|
if (reserved) this.fenceDeletedSessions(state, scopeId, scope, reserved);
|
|
347
354
|
return saved;
|
|
355
|
+
});
|
|
348
356
|
}
|
|
349
357
|
|
|
350
358
|
/** Stops what runs for the reserved sessions; `finishReservedSessionDeletion` awaits it. */
|
|
@@ -424,10 +432,8 @@ export class FlexHarnessSessionDeletion<TScope> {
|
|
|
424
432
|
}
|
|
425
433
|
}
|
|
426
434
|
try {
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
...await this.core.lifecycle.closeOrphanedResources(state.storageKey),
|
|
430
|
-
];
|
|
435
|
+
if (slashCommandErrors.length > 0) throw combineErrors(slashCommandErrors);
|
|
436
|
+
const orphanErrors = await this.core.lifecycle.closeOrphanedResources(state.storageKey);
|
|
431
437
|
if (orphanErrors.length > 0) throw combineErrors(orphanErrors);
|
|
432
438
|
for (const descendantRoot of descendantRoots) {
|
|
433
439
|
await this.finishTombstoneCleanup(
|
|
@@ -453,6 +459,7 @@ export class FlexHarnessSessionDeletion<TScope> {
|
|
|
453
459
|
state: IStorageState,
|
|
454
460
|
tombstone: IFlexSessionTombstone,
|
|
455
461
|
): Promise<void> {
|
|
462
|
+
return this.core.withRecoveryProducer(state, tombstone.sessionId, async () => {
|
|
456
463
|
const storageKey = state.storageKey;
|
|
457
464
|
const sessionId = tombstone.sessionId;
|
|
458
465
|
const projectManagementCleanup = async (): Promise<void> => {
|
|
@@ -483,6 +490,7 @@ export class FlexHarnessSessionDeletion<TScope> {
|
|
|
483
490
|
.filter((result): result is PromiseRejectedResult => result.status === 'rejected')
|
|
484
491
|
.map((result) => result.reason);
|
|
485
492
|
if (errors.length > 0) throw combineErrors(errors);
|
|
493
|
+
});
|
|
486
494
|
}
|
|
487
495
|
|
|
488
496
|
private collectSessionSubtree(
|
|
@@ -588,6 +596,8 @@ export class FlexHarnessSessionDeletion<TScope> {
|
|
|
588
596
|
}
|
|
589
597
|
const group = this.tombstoneGroup(state, rootSessionId);
|
|
590
598
|
if (group.length === 0) return;
|
|
599
|
+
return this.core.withRecoveryProducerSections(state, group.map((entry) => entry.sessionId), async () => {
|
|
600
|
+
await Promise.all(group.map((entry) => this.core.markRecoveryRequired(state, entry.sessionId)));
|
|
591
601
|
const exactInvocation = invocation ?? {
|
|
592
602
|
scopeId: state.scopeContext.scopeId,
|
|
593
603
|
scope: state.scopeContext.scope as TScope,
|
|
@@ -688,7 +698,6 @@ export class FlexHarnessSessionDeletion<TScope> {
|
|
|
688
698
|
'Tombstone cleanup requires its exact scope context before deleting projection data.',
|
|
689
699
|
);
|
|
690
700
|
}
|
|
691
|
-
if (!retained) await this.setDeletionRecoveryRequired(state, sessionId, true);
|
|
692
701
|
await this.core.reversions.releaseDeletedSessionReversions(
|
|
693
702
|
state,
|
|
694
703
|
state.storageKey,
|
|
@@ -704,30 +713,30 @@ export class FlexHarnessSessionDeletion<TScope> {
|
|
|
704
713
|
// Domains and reversion references are durably resolved. Clear the redundant sidecar bit
|
|
705
714
|
// while the durable tombstone still prevents reuse and remains a recovery owner on a crash.
|
|
706
715
|
// Removing the tombstone first would leave an unresolvable orphan bit if its clear failed.
|
|
707
|
-
for (const tombstone of group) await this.
|
|
716
|
+
for (const tombstone of group) await this.clearDeletionRecoveryRequired(state, tombstone.sessionId);
|
|
708
717
|
await this.core.mutateScope(state, () => {
|
|
709
718
|
for (const tombstone of group) {
|
|
710
719
|
state.tombstones.delete(tombstone.sessionId);
|
|
711
720
|
state.retainedSessionCleanups.delete(tombstone.sessionId);
|
|
712
721
|
}
|
|
713
722
|
}, true);
|
|
723
|
+
});
|
|
714
724
|
}
|
|
715
725
|
|
|
716
|
-
private async
|
|
726
|
+
private async clearDeletionRecoveryRequired(
|
|
717
727
|
state: IStorageState,
|
|
718
728
|
sessionId: string,
|
|
719
|
-
required: boolean,
|
|
720
729
|
): Promise<void> {
|
|
721
730
|
if (!state.sessionLocal) return;
|
|
722
731
|
if (!state.qualifyingLegacy) {
|
|
723
732
|
try {
|
|
724
|
-
await this.core.stores.scopes.sessionLocal!.setRecoveryRequired(state.storageKey, sessionId,
|
|
733
|
+
await this.core.stores.scopes.sessionLocal!.setRecoveryRequired(state.storageKey, sessionId, false);
|
|
725
734
|
} catch (error) {
|
|
726
735
|
state.lifecycle = 'fenced';
|
|
727
736
|
throw error;
|
|
728
737
|
}
|
|
729
738
|
}
|
|
730
|
-
|
|
731
|
-
|
|
739
|
+
state.recoveryMarks.delete(sessionId);
|
|
740
|
+
state.recoverySessionIds.delete(sessionId);
|
|
732
741
|
}
|
|
733
742
|
}
|
|
@@ -236,7 +236,8 @@ export class FlexHarnessSessionImports<TScope> {
|
|
|
236
236
|
});
|
|
237
237
|
});
|
|
238
238
|
reserved = true;
|
|
239
|
-
await this.core.
|
|
239
|
+
await this.core.withRecoveryProducer(state, sessionId, () =>
|
|
240
|
+
this.core.stores.projections.save(state.storageKey, sessionId, createWholeProjectionChange({
|
|
240
241
|
schemaVersion: 3,
|
|
241
242
|
revision: 1,
|
|
242
243
|
metrics: createSessionMetricsState(false),
|
|
@@ -246,7 +247,7 @@ export class FlexHarnessSessionImports<TScope> {
|
|
|
246
247
|
revertCursor: 0,
|
|
247
248
|
excludedRunIds: [],
|
|
248
249
|
pendingReversionReleases: [],
|
|
249
|
-
}), 0);
|
|
250
|
+
}), 0));
|
|
250
251
|
const runtime = await this.core.loadSessionRuntime(state, metadata, scopeId, scope);
|
|
251
252
|
loaded = runtime;
|
|
252
253
|
await this.core.mutateScope(state, () => {
|
|
@@ -722,6 +722,9 @@ export class FlexHarnessLifecycle<TScope> {
|
|
|
722
722
|
this.core.appendUnexpectedErrors(errors, error);
|
|
723
723
|
}
|
|
724
724
|
}
|
|
725
|
+
// Retirement has closed publication admission. Join the sidecar write before releasing
|
|
726
|
+
// the storage key so a new runtime cannot race a late recovery update.
|
|
727
|
+
await Promise.all([...state.recoveryProducerGates.values()].map((gate) => gate.waitForPublication()));
|
|
725
728
|
errors.push(...state.fenceAdditionalErrors.splice(0));
|
|
726
729
|
if (errors.length > 0) throw combineErrors(errors);
|
|
727
730
|
state.lifecycle = 'retired';
|
|
@@ -381,6 +381,7 @@ export class FlexHarnessPermissions<TScope> {
|
|
|
381
381
|
const initialError = this.core.projectExternalError(pending.run, error, 'persistence');
|
|
382
382
|
try {
|
|
383
383
|
await this.reconcileRememberedPermissionRemoval(
|
|
384
|
+
state,
|
|
384
385
|
pending.run.stored,
|
|
385
386
|
state.storageKey,
|
|
386
387
|
rememberKey,
|
|
@@ -393,6 +394,7 @@ export class FlexHarnessPermissions<TScope> {
|
|
|
393
394
|
}
|
|
394
395
|
|
|
395
396
|
private async reconcileRememberedPermissionRemoval(
|
|
397
|
+
state: IStorageState,
|
|
396
398
|
stored: IStoredSessionState,
|
|
397
399
|
storageKey: string,
|
|
398
400
|
rememberKey: string,
|
|
@@ -414,12 +416,13 @@ export class FlexHarnessPermissions<TScope> {
|
|
|
414
416
|
(entry) => entry !== rememberKey,
|
|
415
417
|
),
|
|
416
418
|
};
|
|
417
|
-
await this.core.
|
|
419
|
+
await this.core.withRecoveryProducer(state, stored.session.sessionId, () =>
|
|
420
|
+
this.core.stores.permissions.save(
|
|
418
421
|
storageKey,
|
|
419
422
|
stored.session.sessionId,
|
|
420
423
|
snapshot,
|
|
421
424
|
current.revision,
|
|
422
|
-
);
|
|
425
|
+
));
|
|
423
426
|
stored.rememberedPermissionKeys = new Set(snapshot.rememberedPermissionKeys);
|
|
424
427
|
stored.permissionRevision = snapshot.revision;
|
|
425
428
|
return;
|
|
@@ -489,12 +492,13 @@ export class FlexHarnessPermissions<TScope> {
|
|
|
489
492
|
rememberedPermissionKeys: [...stored.rememberedPermissionKeys].sort(),
|
|
490
493
|
};
|
|
491
494
|
try {
|
|
492
|
-
await this.core.
|
|
495
|
+
await this.core.withRecoveryProducer(state, stored.session.sessionId, () =>
|
|
496
|
+
this.core.stores.permissions.save(
|
|
493
497
|
state.storageKey,
|
|
494
498
|
stored.session.sessionId,
|
|
495
499
|
snapshot,
|
|
496
500
|
beforeRevision,
|
|
497
|
-
);
|
|
501
|
+
));
|
|
498
502
|
stored.permissionRevision = snapshot.revision;
|
|
499
503
|
return result;
|
|
500
504
|
} catch (error) {
|
|
@@ -668,7 +668,8 @@ export class FlexHarnessPromptQueue<TScope> {
|
|
|
668
668
|
const run = this.core.runs.createActiveRun(queued);
|
|
669
669
|
state.activeRuns.set(queued.sessionId, run);
|
|
670
670
|
try {
|
|
671
|
-
await this.
|
|
671
|
+
await this.core.withRecoveryProducer(state, queued.sessionId,
|
|
672
|
+
() => this.promoteQueuedPrompt(queued, run));
|
|
672
673
|
return;
|
|
673
674
|
} catch (error) {
|
|
674
675
|
if (state.activeRuns.get(queued.sessionId) === run) state.activeRuns.delete(queued.sessionId);
|
|
@@ -751,13 +752,13 @@ export class FlexHarnessPromptQueue<TScope> {
|
|
|
751
752
|
this.assertPromptPromotion(queued, run);
|
|
752
753
|
// A cache-warm exchange is model-visible only to its own model calls.
|
|
753
754
|
const beginOptions = { generationId: run.runId, ...(run.cacheWarm ? { ephemeral: true as const } : {}) };
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
755
|
+
run.transaction = await this.core.withRecoveryProducer(run.state, run.sessionId, () =>
|
|
756
|
+
this.core.compaction.withRunCompactorContext(
|
|
757
|
+
run,
|
|
758
|
+
() => generationPrompt === undefined
|
|
759
|
+
? run.stored.agentSession.beginContinuation(beginOptions)
|
|
760
|
+
: run.stored.agentSession.beginGeneration(generationPrompt, beginOptions),
|
|
761
|
+
));
|
|
761
762
|
this.assertPromptPromotion(queued, run);
|
|
762
763
|
const reservation = this.core.runs.createReservation(run, prompt, queued.promptId);
|
|
763
764
|
closeReservationPublication = this.core.openPublication(run.state, run.sessionId);
|
|
@@ -928,6 +929,7 @@ export class FlexHarnessPromptQueue<TScope> {
|
|
|
928
929
|
error,
|
|
929
930
|
),
|
|
930
931
|
).finally(() => this.schedulePromptQueueDrain(run.state, run.stored));
|
|
932
|
+
this.core.admitRecoveryRun(run.state, run.sessionId, execution);
|
|
931
933
|
void execution.catch(() => undefined);
|
|
932
934
|
}
|
|
933
935
|
|
|
@@ -49,14 +49,15 @@ export class FlexHarnessResults<TScope> {
|
|
|
49
49
|
// A budget too small for even the handle keeps the truncated output.
|
|
50
50
|
if (jsonBytes(result) > maxBytes) return normalizeJsonValue(output, this.core.toolOutputLimits);
|
|
51
51
|
const { storageKey } = run.state;
|
|
52
|
-
await queueOperation(this.resultQueues, this.resultQueueKey(storageKey, run.sessionId),
|
|
52
|
+
await queueOperation(this.resultQueues, this.resultQueueKey(storageKey, run.sessionId), () =>
|
|
53
|
+
this.core.withRecoveryProducer(run.state, run.sessionId, async () => {
|
|
53
54
|
await results.save(storageKey, run.sessionId, handle, stored);
|
|
54
55
|
// The session keeps its latest results; older ones are deleted.
|
|
55
56
|
const handles = await results.list(storageKey, run.sessionId);
|
|
56
57
|
for (const old of handles.slice(0, Math.max(0, handles.length - this.core.resultLimits.maxResultsPerSession))) {
|
|
57
58
|
await results.delete(storageKey, run.sessionId, old);
|
|
58
59
|
}
|
|
59
|
-
});
|
|
60
|
+
}));
|
|
60
61
|
return result as unknown as TJsonValue;
|
|
61
62
|
}
|
|
62
63
|
|
|
@@ -440,6 +440,7 @@ export class FlexHarnessReversions<TScope> {
|
|
|
440
440
|
): Promise<void> {
|
|
441
441
|
const pending = stored.pendingReversion;
|
|
442
442
|
if (pending?.kind !== 'apply') return;
|
|
443
|
+
return this.core.withRecoveryProducer(state, stored.session.sessionId, async () => {
|
|
443
444
|
const sessionGeneration = requireSessionGeneration(stored.session);
|
|
444
445
|
const segmentMap = new Map(stored.reversionSegments.map((segment) => [segment.runId, segment]));
|
|
445
446
|
if (
|
|
@@ -560,6 +561,7 @@ export class FlexHarnessReversions<TScope> {
|
|
|
560
561
|
stored.revertCursor = current.toCursor;
|
|
561
562
|
delete stored.pendingReversion;
|
|
562
563
|
}, true);
|
|
564
|
+
});
|
|
563
565
|
}
|
|
564
566
|
|
|
565
567
|
private async persistKnownNotApplied(
|
|
@@ -821,7 +823,8 @@ export class FlexHarnessReversions<TScope> {
|
|
|
821
823
|
let outcome = inspection.status === 'finalized' ? inspection.outcome : undefined;
|
|
822
824
|
if (inspection.status === 'prepared') {
|
|
823
825
|
try {
|
|
824
|
-
outcome = await this.
|
|
826
|
+
outcome = await this.core.withRecoveryProducer(state, sessionId, () =>
|
|
827
|
+
this.finalizeReversionCapture(
|
|
825
828
|
scopeId,
|
|
826
829
|
scope,
|
|
827
830
|
state.storageKey,
|
|
@@ -829,7 +832,7 @@ export class FlexHarnessReversions<TScope> {
|
|
|
829
832
|
sessionGeneration,
|
|
830
833
|
runId,
|
|
831
834
|
captureId,
|
|
832
|
-
);
|
|
835
|
+
));
|
|
833
836
|
} catch (error) {
|
|
834
837
|
const finalInspection = await this.inspectReversionCapture(
|
|
835
838
|
scopeId,
|
|
@@ -981,7 +984,8 @@ export class FlexHarnessReversions<TScope> {
|
|
|
981
984
|
);
|
|
982
985
|
}
|
|
983
986
|
try {
|
|
984
|
-
await this.core.
|
|
987
|
+
await this.core.withRecoveryProducer(run.state, run.sessionId,
|
|
988
|
+
() => Promise.resolve(this.core.turnReversionProvider!.prepare(context)));
|
|
985
989
|
} catch (error) {
|
|
986
990
|
let inspection;
|
|
987
991
|
try {
|
|
@@ -1386,8 +1390,9 @@ export class FlexHarnessReversions<TScope> {
|
|
|
1386
1390
|
);
|
|
1387
1391
|
}
|
|
1388
1392
|
try {
|
|
1389
|
-
await this.
|
|
1390
|
-
this.
|
|
1393
|
+
await this.core.withRecoveryProducer(state, stored.session.sessionId, () =>
|
|
1394
|
+
this.withReversionMaintenanceSignal((signal) =>
|
|
1395
|
+
this.core.turnReversionProvider!.release(Object.freeze({
|
|
1391
1396
|
scopeId,
|
|
1392
1397
|
scope,
|
|
1393
1398
|
storageKey: state.storageKey,
|
|
@@ -1397,7 +1402,7 @@ export class FlexHarnessReversions<TScope> {
|
|
|
1397
1402
|
captureId: release.captureId,
|
|
1398
1403
|
reference: cloneSerializable(release.reference),
|
|
1399
1404
|
signal,
|
|
1400
|
-
})));
|
|
1405
|
+
}))));
|
|
1401
1406
|
} catch (error) {
|
|
1402
1407
|
if (throwOnFailure) {
|
|
1403
1408
|
throw this.core.projectOperationError(
|
|
@@ -1436,6 +1441,7 @@ export class FlexHarnessReversions<TScope> {
|
|
|
1436
1441
|
stored?: IStoredSessionState,
|
|
1437
1442
|
): Promise<void> {
|
|
1438
1443
|
const sessionId = tombstone.sessionId;
|
|
1444
|
+
return this.core.withRecoveryProducer(state, sessionId, async () => {
|
|
1439
1445
|
const sessionGeneration = requireSessionGeneration(stored?.session ?? tombstone);
|
|
1440
1446
|
if (stored) await stored.projectionQueue;
|
|
1441
1447
|
let durableProjection = await this.core.stores.projections.load(storageKey, sessionId);
|
|
@@ -1742,6 +1748,7 @@ export class FlexHarnessReversions<TScope> {
|
|
|
1742
1748
|
});
|
|
1743
1749
|
}
|
|
1744
1750
|
}
|
|
1751
|
+
});
|
|
1745
1752
|
}
|
|
1746
1753
|
|
|
1747
1754
|
public completedReversionCandidates(stored: IStoredSessionState): IFlexReversionSegment[] {
|
|
@@ -289,6 +289,8 @@ export class FlexHarnessSessions<TScope> {
|
|
|
289
289
|
resolved.state.initializingSessions.delete(sessionId);
|
|
290
290
|
completeInitialization();
|
|
291
291
|
}
|
|
292
|
+
const stored = resolved.state.sessions.get(sessionId);
|
|
293
|
+
if (stored) await this.core.settleRecoveryObligation(resolved.state, stored);
|
|
292
294
|
const result = publicSnapshot(metadata);
|
|
293
295
|
this.core.emitEvent(scopeId, metadata, { type: 'session.created', session: result });
|
|
294
296
|
return result;
|
|
@@ -309,14 +311,28 @@ export class FlexHarnessSessions<TScope> {
|
|
|
309
311
|
signal.addEventListener('abort', onAbort, { once: true });
|
|
310
312
|
});
|
|
311
313
|
try {
|
|
312
|
-
const stateRead = this.core.resolveState(
|
|
313
|
-
|
|
314
|
+
const stateRead = this.core.resolveState(
|
|
315
|
+
scopeId,
|
|
316
|
+
this.core.cleanRecoveryConfigured ? undefined : sessionId,
|
|
317
|
+
);
|
|
318
|
+
const { state, scope } = await (aborted ? Promise.race([stateRead, aborted]) : stateRead);
|
|
314
319
|
throwIfAborted();
|
|
315
320
|
// Waits for the queued changes of this session only, never for another session's save.
|
|
316
321
|
const barrier = this.core.sessionReadBarrier(state, sessionId);
|
|
317
322
|
await (aborted ? Promise.race([barrier, aborted]) : barrier);
|
|
318
323
|
throwIfAborted();
|
|
319
324
|
this.core.assertStateAcceptingWork(state);
|
|
325
|
+
if (state.initializingSessions.has(sessionId)) this.core.requireSession(state, sessionId);
|
|
326
|
+
if (!state.sessions.has(sessionId)) {
|
|
327
|
+
const clean = this.core.readGuardedCleanSessionMetadata(state, sessionId);
|
|
328
|
+
const metadata = await (aborted ? Promise.race([clean, aborted]) : clean);
|
|
329
|
+
throwIfAborted();
|
|
330
|
+
if (metadata) return publicSnapshot(metadata);
|
|
331
|
+
const hydration = this.core.hydrateSession(state, sessionId, scopeId, scope.scope);
|
|
332
|
+
await (aborted ? Promise.race([hydration, aborted]) : hydration);
|
|
333
|
+
throwIfAborted();
|
|
334
|
+
}
|
|
335
|
+
await this.core.settleQuiescentSessionRecovery(state, sessionId);
|
|
320
336
|
return publicSnapshot(this.core.requireSession(state, sessionId).session);
|
|
321
337
|
} finally {
|
|
322
338
|
if (signal && onAbort) signal.removeEventListener('abort', onAbort);
|
|
@@ -245,10 +245,11 @@ export class FlexHarnessSubagents<TScope> {
|
|
|
245
245
|
const stored = this.core.requireSession(state, sessionId);
|
|
246
246
|
this.core.assertStateAcceptingWork(state);
|
|
247
247
|
try {
|
|
248
|
-
await this.core.
|
|
249
|
-
this.core.compaction.
|
|
250
|
-
|
|
251
|
-
|
|
248
|
+
await this.core.withRecoveryProducer(state, sessionId, () =>
|
|
249
|
+
this.core.compaction.withCompactorContext(
|
|
250
|
+
this.core.compaction.createCompactorContext(scopeId, scope.scope, state.storageKey, sessionId),
|
|
251
|
+
() => stored.agentSession.abortBackgroundExecution(executionId),
|
|
252
|
+
));
|
|
252
253
|
} catch (error) {
|
|
253
254
|
throw this.core.projectOperationError(error, 'agentSession', scopeId, sessionId, 'background-abort');
|
|
254
255
|
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import * as plugins from './plugins.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* One session's producer/publication exclusion. A producer owns the gate through its durable
|
|
5
|
+
* mark and effect. A publication reserves exclusion before it waits for existing producers, then
|
|
6
|
+
* admits no new producers until it settles. Nested work in an active producer remains admitted.
|
|
7
|
+
*/
|
|
8
|
+
export class FlexRecoveryProducerGate {
|
|
9
|
+
constructor(private readonly onQuiescent?: () => void) {}
|
|
10
|
+
|
|
11
|
+
private readonly context = new plugins.AsyncLocalStorage<{ leases: number }>();
|
|
12
|
+
private producers = 0;
|
|
13
|
+
private backgroundJobs = 0;
|
|
14
|
+
private backgroundJobFailed = false;
|
|
15
|
+
private runs = 0;
|
|
16
|
+
private runFailed = false;
|
|
17
|
+
private drained?: { promise: Promise<void>; resolve: () => void };
|
|
18
|
+
private publication?: Promise<void>;
|
|
19
|
+
|
|
20
|
+
public get activeProducers(): number { return this.producers; }
|
|
21
|
+
public get activeBackgroundJobs(): number { return this.backgroundJobs; }
|
|
22
|
+
public get activeRuns(): number { return this.runs; }
|
|
23
|
+
public get canPublish(): boolean {
|
|
24
|
+
return this.producers === 0 && this.backgroundJobs === 0 && !this.backgroundJobFailed
|
|
25
|
+
&& this.runs === 0 && !this.runFailed;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
public async waitForPublication(): Promise<void> {
|
|
29
|
+
await this.publication;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Generation execution may continue tool effects after prompt promotion has returned. */
|
|
33
|
+
public admitRun(completion: Promise<unknown>): void {
|
|
34
|
+
if (!this.context.getStore()?.leases) {
|
|
35
|
+
throw new Error('A run must be admitted by an active recovery producer.');
|
|
36
|
+
}
|
|
37
|
+
this.runs++;
|
|
38
|
+
void completion.then(
|
|
39
|
+
() => { this.runs--; this.notifyQuiescent(); },
|
|
40
|
+
() => { this.runs--; this.runFailed = true; },
|
|
41
|
+
);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** A start producer registers the job before returning its handle. Completion includes its
|
|
45
|
+
* terminal persistence; a failed completion leaves this runtime unable to publish clean. */
|
|
46
|
+
public admitBackgroundJob(completion: Promise<unknown>): void {
|
|
47
|
+
if (!this.context.getStore()?.leases) {
|
|
48
|
+
throw new Error('A background job must be admitted by an active recovery producer.');
|
|
49
|
+
}
|
|
50
|
+
this.backgroundJobs++;
|
|
51
|
+
void completion.then(
|
|
52
|
+
() => { this.backgroundJobs--; this.notifyQuiescent(); },
|
|
53
|
+
() => { this.backgroundJobs--; this.backgroundJobFailed = true; },
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
private notifyQuiescent(): void {
|
|
58
|
+
if (this.canPublish && !this.publication && this.onQuiescent) {
|
|
59
|
+
queueMicrotask(this.onQuiescent);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
public async producer<TValue>(operation: () => Promise<TValue>): Promise<TValue> {
|
|
64
|
+
const owner = this.context.getStore();
|
|
65
|
+
if (!owner?.leases) {
|
|
66
|
+
while (this.publication) await this.publication;
|
|
67
|
+
}
|
|
68
|
+
this.producers++;
|
|
69
|
+
const current = owner?.leases ? owner : { leases: 0 };
|
|
70
|
+
current.leases++;
|
|
71
|
+
try {
|
|
72
|
+
return await this.context.run(current, operation);
|
|
73
|
+
} finally {
|
|
74
|
+
current.leases--;
|
|
75
|
+
this.producers--;
|
|
76
|
+
if (this.producers === 0) {
|
|
77
|
+
this.drained?.resolve();
|
|
78
|
+
this.drained = undefined;
|
|
79
|
+
}
|
|
80
|
+
this.notifyQuiescent();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
public async publish<TValue>(operation: () => Promise<TValue>): Promise<TValue> {
|
|
85
|
+
if (this.context.getStore()?.leases) {
|
|
86
|
+
throw new Error('A recovery producer cannot publish its own clean proof.');
|
|
87
|
+
}
|
|
88
|
+
const previous = this.publication;
|
|
89
|
+
let release!: () => void;
|
|
90
|
+
const barrier = new Promise<void>((resolve) => { release = resolve; });
|
|
91
|
+
this.publication = barrier;
|
|
92
|
+
try {
|
|
93
|
+
if (previous) await previous;
|
|
94
|
+
if (this.producers > 0) {
|
|
95
|
+
this.drained ??= (() => {
|
|
96
|
+
let resolve!: () => void;
|
|
97
|
+
const promise = new Promise<void>((done) => { resolve = done; });
|
|
98
|
+
return { promise, resolve };
|
|
99
|
+
})();
|
|
100
|
+
await this.drained.promise;
|
|
101
|
+
}
|
|
102
|
+
if (this.backgroundJobs > 0 || this.backgroundJobFailed || this.runs > 0 || this.runFailed) {
|
|
103
|
+
throw new Error('Runs or background jobs have not reached a durable clean frontier.');
|
|
104
|
+
}
|
|
105
|
+
return await operation();
|
|
106
|
+
} finally {
|
|
107
|
+
if (this.publication === barrier) this.publication = undefined;
|
|
108
|
+
release();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|