@modelprofile.com/flexharness 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.flexharness.d.ts +14 -1
- package/dist_ts/classes.flexharness.js +211 -67
- package/package.json +2 -2
- package/readme.hints.md +2 -0
- package/readme.md +9 -0
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.flexharness.ts +298 -103
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@modelprofile.com/flexharness",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.1.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Provider-neutral model-session runtime with durable history, permissions, typed events, and pluggable local or remote tool execution.",
|
|
6
6
|
"main": "dist_ts/index.js",
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
"@git.zone/tsbuild": "4.4.2",
|
|
20
20
|
"@git.zone/tsrun": "2.0.6",
|
|
21
21
|
"@git.zone/tstest": "4.0.0",
|
|
22
|
-
"@types/node": "26.
|
|
22
|
+
"@types/node": "26.2.0"
|
|
23
23
|
},
|
|
24
24
|
"files": [
|
|
25
25
|
"ts/**/*",
|
package/readme.hints.md
CHANGED
|
@@ -21,6 +21,8 @@ Implementation findings for flexharness.
|
|
|
21
21
|
- Terminal persistence is the cancellation linearization point. Abort returns false once a run starts committing, while the active-run entry continues to block new prompts until the commit settles.
|
|
22
22
|
- Permission callbacks wait for the current save tail and then revalidate their run before reading remembered grants, so neither uncommitted grants nor captured callbacks can authorize later work.
|
|
23
23
|
- Detached tool-provider settlement remains tracked after prompt finalization. Disposal awaits late handle closure and reports cleanup failure.
|
|
24
|
+
- Scope retirement fences older resolver calls and late state admissions, drains the complete resolved storage namespace, and evicts only the exact cached state promise. It never deletes the durable snapshot. Applications must close admission across every resolver alias before retirement because aliases are unknowable before resolution.
|
|
25
|
+
- Detached tool-provider cleanup and retained cleanup errors are owned by the exact loaded storage state. Scope retirement cannot await or consume another namespace's cleanup, while full disposal settles all storage drains before aggregating failures.
|
|
24
26
|
|
|
25
27
|
## Tool output boundary
|
|
26
28
|
|
package/readme.md
CHANGED
|
@@ -161,6 +161,7 @@ await admission.completion;
|
|
|
161
161
|
await harness.abort(scopeId, sessionId);
|
|
162
162
|
await harness.listPendingPermissions(scopeId, sessionId);
|
|
163
163
|
await harness.respondToPermission(scopeId, sessionId, permissionId, 'once');
|
|
164
|
+
await harness.retireScope(scopeId);
|
|
164
165
|
await harness.dispose();
|
|
165
166
|
```
|
|
166
167
|
|
|
@@ -174,6 +175,12 @@ The reservation save is the admission point. A save failure produces no start ev
|
|
|
174
175
|
|
|
175
176
|
`abort()` returns `true` only while cancellation is still accepted. Terminal persistence is the run's commit point; once it starts, `abort()` returns `false` and the already-fixed terminal outcome completes while the session remains busy.
|
|
176
177
|
|
|
178
|
+
`retireScope()` stops runtime ownership for the complete resolved storage namespace without deleting its durable snapshot. It does not load a namespace that has no cached or in-flight state. For loaded state, it preserves and waits for persistence that has already started, while later queued reads, writes, and run admissions reject with `FlexHarnessAbortError`. It aborts cancellable runs, rejects pending permissions, waits for committing runs, terminal persistence, tool-handle closure, and detached tool-provider cleanup, then clears and evicts the cached state. Calls through storage-key aliases share the same retirement drain. A later call can load the durable namespace again if the application still resolves it.
|
|
179
|
+
|
|
180
|
+
Normal retirement-induced cancellation does not make `retireScope()` reject. Unexpected failures observed through run finalization or scoped detached cleanup are surfaced after the namespace has been drained and evicted. One such failure is thrown directly; multiple failures are reported through `FlexHarnessRunError`. State-load failures and already-started non-run persistence failures remain reported to their originating operations and are not reported a second time by retirement.
|
|
181
|
+
|
|
182
|
+
Applications removing a scope must stop and serialize new admission across every alias before calling `retireScope()`, await retirement, and only then remove or purge application-owned durable records. FlexHarness cannot discover aliases before the application resolver returns. Integrations must not use retirement itself as durable deletion.
|
|
183
|
+
|
|
177
184
|
## History And Audit Behavior
|
|
178
185
|
|
|
179
186
|
Successful model context is accumulated as:
|
|
@@ -281,6 +288,8 @@ Tool-handle close settles before a turn can be successful. Final persistence is
|
|
|
281
288
|
|
|
282
289
|
`dispose()` is asynchronous and idempotent. It marks the harness closed, prevents operations waiting on persistence from reserving a run, aborts cancellable active runs, rejects pending permissions, waits for committing runs, all run finalizers, state save tails, and tracked detached tool-provider cleanup, then clears listeners and loaded state caches. Multiple run or cleanup failures are reported through `FlexHarnessRunError`.
|
|
283
290
|
|
|
291
|
+
If `dispose()` overlaps a storage namespace already being retired, both calls await the same storage drain and cleanup runs once. A retirement call begun after disposal starts rejects with `FlexHarnessClosedError`.
|
|
292
|
+
|
|
284
293
|
Cancellation is cooperative: model resolvers, tool providers, runners, tools, and cleanup functions must observe the supplied `AbortSignal` and settle tracked work. After a sibling resolver fails, FlexHarness deliberately does not wait for an unresponsive model resolver; a detached tool provider remains tracked because any late handle must be closed. A process host that needs a hard shutdown deadline must enforce that deadline outside FlexHarness and terminate only the process it owns.
|
|
285
294
|
|
|
286
295
|
## License and Legal Information
|
package/ts/00_commitinfo_data.ts
CHANGED
|
@@ -3,6 +3,6 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export const commitinfo = {
|
|
5
5
|
name: '@modelprofile.com/flexharness',
|
|
6
|
-
version: '2.
|
|
6
|
+
version: '2.1.0',
|
|
7
7
|
description: 'Provider-neutral model-session runtime with durable history, permissions, typed events, and pluggable local or remote tool execution.'
|
|
8
8
|
}
|
|
@@ -82,9 +82,12 @@ interface IStorageState {
|
|
|
82
82
|
activeRuns: Map<string, IActiveRun>;
|
|
83
83
|
pendingPermissions: Map<string, IPendingPermission>;
|
|
84
84
|
saveTail: Promise<void>;
|
|
85
|
+
lifecycle: 'active' | 'retiring' | 'retired';
|
|
86
|
+
detachedToolCleanupErrors: Error[];
|
|
85
87
|
}
|
|
86
88
|
|
|
87
89
|
interface IActiveRun {
|
|
90
|
+
state: IStorageState;
|
|
88
91
|
scopeId: string;
|
|
89
92
|
sessionId: string;
|
|
90
93
|
runId: string;
|
|
@@ -122,6 +125,17 @@ interface IPendingPermission {
|
|
|
122
125
|
abortListener: () => void;
|
|
123
126
|
}
|
|
124
127
|
|
|
128
|
+
interface IDetachedToolCleanup {
|
|
129
|
+
state: IStorageState;
|
|
130
|
+
completion: Promise<void>;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
interface IScopeAdmissionState {
|
|
134
|
+
generation: number;
|
|
135
|
+
inFlightResolvers: number;
|
|
136
|
+
retiring: boolean;
|
|
137
|
+
}
|
|
138
|
+
|
|
125
139
|
interface IFinalMutationResult {
|
|
126
140
|
userMessage: IFlexMessage;
|
|
127
141
|
assistantMessage: IFlexMessage;
|
|
@@ -178,6 +192,7 @@ const externalErrorFallback: IFlexErrorInfo = Object.freeze({
|
|
|
178
192
|
code: 'FLEX_EXTERNAL_ERROR',
|
|
179
193
|
});
|
|
180
194
|
const maxDetachedCleanupErrors = 100;
|
|
195
|
+
const scopeRetirementMessage = 'The scope is being retired.';
|
|
181
196
|
|
|
182
197
|
function isAsyncIterable(value: unknown): value is AsyncIterable<unknown> {
|
|
183
198
|
return Boolean(
|
|
@@ -589,9 +604,11 @@ export class FlexHarness<TScope = unknown> {
|
|
|
589
604
|
private readonly callbackLimits: Required<IFlexCallbackLimits>;
|
|
590
605
|
private readonly externalErrorProjector?: TFlexExternalErrorProjector;
|
|
591
606
|
private readonly stateLoads = new Map<string, Promise<IStorageState>>();
|
|
607
|
+
private readonly scopeAdmissions = new Map<string, IScopeAdmissionState>();
|
|
608
|
+
private readonly scopeRetirements = new Map<string, Promise<void>>();
|
|
609
|
+
private readonly storageDrains = new Map<string, Promise<void>>();
|
|
592
610
|
private readonly listeners = new Set<TFlexHarnessEventListener>();
|
|
593
|
-
private readonly detachedToolCleanups = new Set<
|
|
594
|
-
private readonly detachedToolCleanupErrors: Error[] = [];
|
|
611
|
+
private readonly detachedToolCleanups = new Set<IDetachedToolCleanup>();
|
|
595
612
|
private readonly trustedInternalErrors = new WeakSet<object>();
|
|
596
613
|
private sequence = 0;
|
|
597
614
|
private closed = false;
|
|
@@ -610,7 +627,7 @@ export class FlexHarness<TScope = unknown> {
|
|
|
610
627
|
|
|
611
628
|
public async listSessions(scopeId: string): Promise<IFlexSession[]> {
|
|
612
629
|
const { state } = await this.resolveState(scopeId);
|
|
613
|
-
await state
|
|
630
|
+
await this.waitForReadableState(state);
|
|
614
631
|
return publicSnapshot(
|
|
615
632
|
[...state.sessions.values()]
|
|
616
633
|
.map((entry) => entry.session)
|
|
@@ -659,7 +676,7 @@ export class FlexHarness<TScope = unknown> {
|
|
|
659
676
|
|
|
660
677
|
public async getSession(scopeId: string, sessionId: string): Promise<IFlexSession> {
|
|
661
678
|
const { state } = await this.resolveState(scopeId);
|
|
662
|
-
await state
|
|
679
|
+
await this.waitForReadableState(state);
|
|
663
680
|
return publicSnapshot(this.requireSession(state, sessionId).session);
|
|
664
681
|
}
|
|
665
682
|
|
|
@@ -711,7 +728,7 @@ export class FlexHarness<TScope = unknown> {
|
|
|
711
728
|
|
|
712
729
|
public async getMessages(scopeId: string, sessionId: string): Promise<IFlexMessage[]> {
|
|
713
730
|
const { state } = await this.resolveState(scopeId);
|
|
714
|
-
await state
|
|
731
|
+
await this.waitForReadableState(state);
|
|
715
732
|
return publicSnapshot(this.requireSession(state, sessionId).messages);
|
|
716
733
|
}
|
|
717
734
|
|
|
@@ -750,7 +767,7 @@ export class FlexHarness<TScope = unknown> {
|
|
|
750
767
|
validateIdentifier(sessionId, 'sessionId');
|
|
751
768
|
requireTransferIdentifier(sessionId, 'sessionId');
|
|
752
769
|
const { state } = await this.resolveState(scopeId);
|
|
753
|
-
await state
|
|
770
|
+
await this.waitForReadableState(state);
|
|
754
771
|
const stored = this.requireSession(state, sessionId);
|
|
755
772
|
const namespace = this.messageCursorNamespace(state.storageKey);
|
|
756
773
|
let end = stored.messages.length;
|
|
@@ -810,7 +827,7 @@ export class FlexHarness<TScope = unknown> {
|
|
|
810
827
|
validateIdentifier(messageId, 'messageId');
|
|
811
828
|
requireTransferIdentifier(messageId, 'messageId');
|
|
812
829
|
const { state } = await this.resolveState(scopeId);
|
|
813
|
-
await state
|
|
830
|
+
await this.waitForReadableState(state);
|
|
814
831
|
const message = this.requireMessage(this.requireSession(state, sessionId), messageId);
|
|
815
832
|
return publicSnapshot(createBoundedTransferMessage(message));
|
|
816
833
|
}
|
|
@@ -838,6 +855,7 @@ export class FlexHarness<TScope = unknown> {
|
|
|
838
855
|
const resolved = await this.resolveState(scopeId);
|
|
839
856
|
await resolved.state.saveTail;
|
|
840
857
|
this.assertOpen();
|
|
858
|
+
this.assertStateAcceptingWork(resolved.state);
|
|
841
859
|
this.requireSession(resolved.state, sessionId);
|
|
842
860
|
if (resolved.state.activeRuns.has(sessionId)) {
|
|
843
861
|
throw new FlexHarnessSessionBusyError(sessionId);
|
|
@@ -857,6 +875,7 @@ export class FlexHarness<TScope = unknown> {
|
|
|
857
875
|
rejectCompletion = reject;
|
|
858
876
|
});
|
|
859
877
|
const run: IActiveRun = {
|
|
878
|
+
state: resolved.state,
|
|
860
879
|
scopeId,
|
|
861
880
|
sessionId,
|
|
862
881
|
runId: plugins.crypto.randomUUID(),
|
|
@@ -906,7 +925,7 @@ export class FlexHarness<TScope = unknown> {
|
|
|
906
925
|
sessionId?: string,
|
|
907
926
|
): Promise<IFlexPermissionRequest[]> {
|
|
908
927
|
const { state } = await this.resolveState(scopeId);
|
|
909
|
-
await state
|
|
928
|
+
await this.waitForReadableState(state);
|
|
910
929
|
return publicSnapshot(
|
|
911
930
|
[...state.pendingPermissions.values()]
|
|
912
931
|
.map((pending) => pending.request)
|
|
@@ -947,6 +966,27 @@ export class FlexHarness<TScope = unknown> {
|
|
|
947
966
|
};
|
|
948
967
|
}
|
|
949
968
|
|
|
969
|
+
public retireScope(scopeId: string): Promise<void> {
|
|
970
|
+
this.assertOpen();
|
|
971
|
+
validateIdentifier(scopeId, 'scopeId');
|
|
972
|
+
const existingRetirement = this.scopeRetirements.get(scopeId);
|
|
973
|
+
if (existingRetirement) return existingRetirement;
|
|
974
|
+
|
|
975
|
+
const admission = this.getScopeAdmission(scopeId);
|
|
976
|
+
admission.generation++;
|
|
977
|
+
admission.retiring = true;
|
|
978
|
+
let retirement!: Promise<void>;
|
|
979
|
+
retirement = this.retireScopeInternal(scopeId).finally(() => {
|
|
980
|
+
if (this.scopeRetirements.get(scopeId) === retirement) {
|
|
981
|
+
this.scopeRetirements.delete(scopeId);
|
|
982
|
+
}
|
|
983
|
+
admission.retiring = false;
|
|
984
|
+
this.pruneScopeAdmission(scopeId, admission);
|
|
985
|
+
});
|
|
986
|
+
this.scopeRetirements.set(scopeId, retirement);
|
|
987
|
+
return retirement;
|
|
988
|
+
}
|
|
989
|
+
|
|
950
990
|
public async dispose(): Promise<void> {
|
|
951
991
|
if (this.disposePromise) {
|
|
952
992
|
return this.disposePromise;
|
|
@@ -957,55 +997,31 @@ export class FlexHarness<TScope = unknown> {
|
|
|
957
997
|
}
|
|
958
998
|
|
|
959
999
|
private async disposeInternal(): Promise<void> {
|
|
960
|
-
const
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
const error = Object.freeze(
|
|
969
|
-
new FlexHarnessAbortError('The run was aborted because FlexHarness was disposed.'),
|
|
970
|
-
);
|
|
971
|
-
if (run.internalFailure === undefined) run.ownerCancellation ??= error;
|
|
972
|
-
this.rejectRunPermissions(state, run, error);
|
|
973
|
-
run.controller.abort(error);
|
|
974
|
-
}
|
|
975
|
-
for (const pending of [...state.pendingPermissions.values()]) {
|
|
976
|
-
this.rejectPending(
|
|
977
|
-
state,
|
|
978
|
-
pending,
|
|
979
|
-
Object.freeze(
|
|
980
|
-
new FlexHarnessAbortError('Permission was rejected because FlexHarness was disposed.'),
|
|
1000
|
+
const stateLoads = [...this.stateLoads.entries()];
|
|
1001
|
+
const results = await Promise.allSettled(
|
|
1002
|
+
stateLoads.map(([storageKey, stateLoad]) =>
|
|
1003
|
+
this.drainStorage(
|
|
1004
|
+
storageKey,
|
|
1005
|
+
stateLoad,
|
|
1006
|
+
this.trustInternalError(
|
|
1007
|
+
new FlexHarnessAbortError('The run was aborted because FlexHarness was disposed.'),
|
|
981
1008
|
),
|
|
982
|
-
)
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
await Promise.all(states.map((state) => state.saveTail));
|
|
987
|
-
await Promise.all([...this.detachedToolCleanups]);
|
|
1009
|
+
),
|
|
1010
|
+
),
|
|
1011
|
+
);
|
|
1012
|
+
await Promise.all([...this.detachedToolCleanups].map((cleanup) => cleanup.completion));
|
|
988
1013
|
this.listeners.clear();
|
|
989
1014
|
this.stateLoads.clear();
|
|
1015
|
+
this.scopeAdmissions.clear();
|
|
1016
|
+
this.scopeRetirements.clear();
|
|
1017
|
+
this.storageDrains.clear();
|
|
990
1018
|
this.detachedToolCleanups.clear();
|
|
991
|
-
for (const state of states) {
|
|
992
|
-
state.sessions.clear();
|
|
993
|
-
state.activeRuns.clear();
|
|
994
|
-
state.pendingPermissions.clear();
|
|
995
|
-
}
|
|
996
1019
|
const unexpectedErrors: unknown[] = [];
|
|
997
1020
|
for (const result of results) {
|
|
998
|
-
if (result.status === '
|
|
999
|
-
|
|
1000
|
-
for (const error of result.reason.errors) {
|
|
1001
|
-
if (!isAbortError(error)) unexpectedErrors.push(error);
|
|
1002
|
-
}
|
|
1003
|
-
} else if (!isAbortError(result.reason)) {
|
|
1004
|
-
unexpectedErrors.push(result.reason);
|
|
1021
|
+
if (result.status === 'rejected') {
|
|
1022
|
+
this.appendUnexpectedErrors(unexpectedErrors, result.reason);
|
|
1005
1023
|
}
|
|
1006
1024
|
}
|
|
1007
|
-
unexpectedErrors.push(...this.detachedToolCleanupErrors);
|
|
1008
|
-
this.detachedToolCleanupErrors.length = 0;
|
|
1009
1025
|
if (unexpectedErrors.length > 0) {
|
|
1010
1026
|
throw combineErrors(unexpectedErrors);
|
|
1011
1027
|
}
|
|
@@ -1423,17 +1439,20 @@ export class FlexHarness<TScope = unknown> {
|
|
|
1423
1439
|
originalError: unknown,
|
|
1424
1440
|
cancelled: boolean,
|
|
1425
1441
|
): Promise<IFinalMutationResult> {
|
|
1426
|
-
const final = await this.mutateAndSave(
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1442
|
+
const final = await this.mutateAndSave(
|
|
1443
|
+
state,
|
|
1444
|
+
() =>
|
|
1445
|
+
this.applyRunFinalState(
|
|
1446
|
+
state,
|
|
1447
|
+
run,
|
|
1448
|
+
prompt,
|
|
1449
|
+
model,
|
|
1450
|
+
result,
|
|
1451
|
+
serializedResultMessages,
|
|
1452
|
+
originalError,
|
|
1453
|
+
cancelled,
|
|
1454
|
+
),
|
|
1455
|
+
true,
|
|
1437
1456
|
);
|
|
1438
1457
|
this.emitFinalMutationEvents(run, final);
|
|
1439
1458
|
return final;
|
|
@@ -1822,22 +1841,26 @@ export class FlexHarness<TScope = unknown> {
|
|
|
1822
1841
|
let addedRememberKey = false;
|
|
1823
1842
|
pending.responding = true;
|
|
1824
1843
|
try {
|
|
1825
|
-
await this.mutateAndSave(
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1844
|
+
await this.mutateAndSave(
|
|
1845
|
+
state,
|
|
1846
|
+
() => {
|
|
1847
|
+
const stored = this.requireSession(state, pending.request.sessionId);
|
|
1848
|
+
if (decision === 'always' && rememberKey) {
|
|
1849
|
+
addedRememberKey = !stored.rememberedPermissionKeys.has(rememberKey);
|
|
1850
|
+
stored.rememberedPermissionKeys.add(rememberKey);
|
|
1851
|
+
}
|
|
1852
|
+
const hasOtherPending = [...state.pendingPermissions.values()].some(
|
|
1853
|
+
(entry) =>
|
|
1854
|
+
entry !== pending &&
|
|
1855
|
+
!entry.settled &&
|
|
1856
|
+
entry.request.sessionId === pending.request.sessionId,
|
|
1857
|
+
);
|
|
1858
|
+
stored.session.status = hasOtherPending ? 'waiting_permission' : 'running';
|
|
1859
|
+
stored.session.activity.status = hasOtherPending ? 'waiting_permission' : 'running';
|
|
1860
|
+
stored.session.updatedAt = new Date().toISOString();
|
|
1861
|
+
},
|
|
1862
|
+
true,
|
|
1863
|
+
);
|
|
1841
1864
|
} catch (error) {
|
|
1842
1865
|
const persistenceError = this.projectExternalError(pending.run, error, 'persistence');
|
|
1843
1866
|
pending.responding = false;
|
|
@@ -1852,12 +1875,16 @@ export class FlexHarness<TScope = unknown> {
|
|
|
1852
1875
|
if (pending.abortReason !== undefined) {
|
|
1853
1876
|
const abortReason = pending.abortReason;
|
|
1854
1877
|
const rollbackPromise = addedRememberKey && rememberKey
|
|
1855
|
-
? this.mutateAndSave(
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1878
|
+
? this.mutateAndSave(
|
|
1879
|
+
state,
|
|
1880
|
+
() => {
|
|
1881
|
+
this.requireSession(
|
|
1882
|
+
state,
|
|
1883
|
+
pending.request.sessionId,
|
|
1884
|
+
).rememberedPermissionKeys.delete(rememberKey);
|
|
1885
|
+
},
|
|
1886
|
+
true,
|
|
1887
|
+
)
|
|
1861
1888
|
: Promise.resolve();
|
|
1862
1889
|
pending.responding = false;
|
|
1863
1890
|
try {
|
|
@@ -2018,20 +2045,20 @@ export class FlexHarness<TScope = unknown> {
|
|
|
2018
2045
|
cleanup: () => Promise<void> | void,
|
|
2019
2046
|
run: IActiveRun,
|
|
2020
2047
|
): void {
|
|
2021
|
-
|
|
2022
|
-
|
|
2048
|
+
const record = { state: run.state } as IDetachedToolCleanup;
|
|
2049
|
+
record.completion = Promise.resolve()
|
|
2023
2050
|
.then(cleanup)
|
|
2024
2051
|
.catch((error) => {
|
|
2025
|
-
if (
|
|
2026
|
-
|
|
2052
|
+
if (run.state.detachedToolCleanupErrors.length < maxDetachedCleanupErrors) {
|
|
2053
|
+
run.state.detachedToolCleanupErrors.push(
|
|
2027
2054
|
this.projectExternalError(run, error, 'toolCleanup'),
|
|
2028
2055
|
);
|
|
2029
2056
|
}
|
|
2030
2057
|
})
|
|
2031
2058
|
.finally(() => {
|
|
2032
|
-
this.detachedToolCleanups.delete(
|
|
2059
|
+
this.detachedToolCleanups.delete(record);
|
|
2033
2060
|
});
|
|
2034
|
-
this.detachedToolCleanups.add(
|
|
2061
|
+
this.detachedToolCleanups.add(record);
|
|
2035
2062
|
}
|
|
2036
2063
|
|
|
2037
2064
|
private observeDetachedToolProvider(
|
|
@@ -2069,27 +2096,188 @@ export class FlexHarness<TScope = unknown> {
|
|
|
2069
2096
|
}
|
|
2070
2097
|
}
|
|
2071
2098
|
|
|
2072
|
-
private
|
|
2073
|
-
scopeId
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2099
|
+
private getScopeAdmission(scopeId: string): IScopeAdmissionState {
|
|
2100
|
+
let admission = this.scopeAdmissions.get(scopeId);
|
|
2101
|
+
if (!admission) {
|
|
2102
|
+
admission = { generation: 0, inFlightResolvers: 0, retiring: false };
|
|
2103
|
+
this.scopeAdmissions.set(scopeId, admission);
|
|
2104
|
+
}
|
|
2105
|
+
return admission;
|
|
2106
|
+
}
|
|
2107
|
+
|
|
2108
|
+
private pruneScopeAdmission(scopeId: string, admission: IScopeAdmissionState): void {
|
|
2109
|
+
if (
|
|
2110
|
+
!admission.retiring
|
|
2111
|
+
&& admission.inFlightResolvers === 0
|
|
2112
|
+
&& this.scopeAdmissions.get(scopeId) === admission
|
|
2113
|
+
) {
|
|
2114
|
+
this.scopeAdmissions.delete(scopeId);
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
private async retireScopeInternal(scopeId: string): Promise<void> {
|
|
2077
2119
|
const scope = await this.scopeResolver.resolveScope(scopeId);
|
|
2078
2120
|
this.assertOpen();
|
|
2079
2121
|
validateIdentifier(scope.storageKey, 'resolved storageKey');
|
|
2080
|
-
|
|
2122
|
+
const stateLoad = this.stateLoads.get(scope.storageKey);
|
|
2081
2123
|
if (!stateLoad) {
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2124
|
+
const existingDrain = this.storageDrains.get(scope.storageKey);
|
|
2125
|
+
if (existingDrain) await existingDrain;
|
|
2126
|
+
return;
|
|
2127
|
+
}
|
|
2128
|
+
await this.drainStorage(
|
|
2129
|
+
scope.storageKey,
|
|
2130
|
+
stateLoad,
|
|
2131
|
+
this.createScopeRetirementError(),
|
|
2132
|
+
);
|
|
2133
|
+
}
|
|
2134
|
+
|
|
2135
|
+
private drainStorage(
|
|
2136
|
+
storageKey: string,
|
|
2137
|
+
stateLoad: Promise<IStorageState>,
|
|
2138
|
+
abortReason: FlexHarnessAbortError,
|
|
2139
|
+
): Promise<void> {
|
|
2140
|
+
const existingDrain = this.storageDrains.get(storageKey);
|
|
2141
|
+
if (existingDrain) return existingDrain;
|
|
2142
|
+
let drain!: Promise<void>;
|
|
2143
|
+
drain = this.drainStorageInternal(stateLoad, abortReason).finally(() => {
|
|
2144
|
+
if (this.stateLoads.get(storageKey) === stateLoad) {
|
|
2145
|
+
this.stateLoads.delete(storageKey);
|
|
2146
|
+
}
|
|
2147
|
+
if (this.storageDrains.get(storageKey) === drain) {
|
|
2148
|
+
this.storageDrains.delete(storageKey);
|
|
2149
|
+
}
|
|
2150
|
+
});
|
|
2151
|
+
this.storageDrains.set(storageKey, drain);
|
|
2152
|
+
return drain;
|
|
2153
|
+
}
|
|
2154
|
+
|
|
2155
|
+
private async drainStorageInternal(
|
|
2156
|
+
stateLoad: Promise<IStorageState>,
|
|
2157
|
+
abortReason: FlexHarnessAbortError,
|
|
2158
|
+
): Promise<void> {
|
|
2159
|
+
let state: IStorageState;
|
|
2160
|
+
try {
|
|
2161
|
+
state = await stateLoad;
|
|
2162
|
+
} catch {
|
|
2163
|
+
return;
|
|
2164
|
+
}
|
|
2165
|
+
|
|
2166
|
+
const finalizers: Promise<IFlexPromptResult>[] = [];
|
|
2167
|
+
try {
|
|
2168
|
+
state.lifecycle = 'retiring';
|
|
2169
|
+
for (const run of state.activeRuns.values()) {
|
|
2170
|
+
finalizers.push(run.finalizer);
|
|
2171
|
+
if (run.phase === 'committing') continue;
|
|
2172
|
+
if (run.internalFailure === undefined) run.ownerCancellation ??= abortReason;
|
|
2173
|
+
this.rejectRunPermissions(state, run, abortReason);
|
|
2174
|
+
run.controller.abort(abortReason);
|
|
2175
|
+
}
|
|
2176
|
+
for (const pending of [...state.pendingPermissions.values()]) {
|
|
2177
|
+
this.rejectPending(state, pending, abortReason);
|
|
2178
|
+
}
|
|
2179
|
+
|
|
2180
|
+
const results = await Promise.allSettled(finalizers);
|
|
2181
|
+
await state.saveTail;
|
|
2182
|
+
await this.drainDetachedToolCleanups(state);
|
|
2183
|
+
const unexpectedErrors: unknown[] = [];
|
|
2184
|
+
for (const result of results) {
|
|
2185
|
+
if (result.status === 'rejected') {
|
|
2186
|
+
this.appendUnexpectedErrors(unexpectedErrors, result.reason);
|
|
2087
2187
|
}
|
|
2088
|
-
}
|
|
2188
|
+
}
|
|
2189
|
+
unexpectedErrors.push(...state.detachedToolCleanupErrors.splice(0));
|
|
2190
|
+
if (unexpectedErrors.length > 0) {
|
|
2191
|
+
throw combineErrors(unexpectedErrors);
|
|
2192
|
+
}
|
|
2193
|
+
} finally {
|
|
2194
|
+
state.lifecycle = 'retired';
|
|
2195
|
+
state.sessions.clear();
|
|
2196
|
+
state.activeRuns.clear();
|
|
2197
|
+
state.pendingPermissions.clear();
|
|
2198
|
+
state.detachedToolCleanupErrors.length = 0;
|
|
2089
2199
|
}
|
|
2090
|
-
|
|
2200
|
+
}
|
|
2201
|
+
|
|
2202
|
+
private async drainDetachedToolCleanups(state: IStorageState): Promise<void> {
|
|
2203
|
+
while (true) {
|
|
2204
|
+
const cleanups = [...this.detachedToolCleanups]
|
|
2205
|
+
.filter((cleanup) => cleanup.state === state)
|
|
2206
|
+
.map((cleanup) => cleanup.completion);
|
|
2207
|
+
if (cleanups.length === 0) return;
|
|
2208
|
+
await Promise.all(cleanups);
|
|
2209
|
+
}
|
|
2210
|
+
}
|
|
2211
|
+
|
|
2212
|
+
private appendUnexpectedErrors(target: unknown[], error: unknown): void {
|
|
2213
|
+
if (error instanceof FlexHarnessRunError) {
|
|
2214
|
+
for (const nestedError of error.errors) {
|
|
2215
|
+
this.appendUnexpectedErrors(target, nestedError);
|
|
2216
|
+
}
|
|
2217
|
+
} else if (!isAbortError(error)) {
|
|
2218
|
+
target.push(error);
|
|
2219
|
+
}
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
private async waitForReadableState(state: IStorageState): Promise<void> {
|
|
2223
|
+
await state.saveTail;
|
|
2224
|
+
this.assertStateAcceptingWork(state);
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
private assertStateAcceptingWork(state: IStorageState): void {
|
|
2228
|
+
if (state.lifecycle !== 'active' || this.storageDrains.has(state.storageKey)) {
|
|
2229
|
+
throw this.createScopeRetirementError();
|
|
2230
|
+
}
|
|
2231
|
+
}
|
|
2232
|
+
|
|
2233
|
+
private createScopeRetirementError(): FlexHarnessAbortError {
|
|
2234
|
+
return this.trustInternalError(new FlexHarnessAbortError(scopeRetirementMessage));
|
|
2235
|
+
}
|
|
2236
|
+
|
|
2237
|
+
private async resolveState(
|
|
2238
|
+
scopeId: string,
|
|
2239
|
+
): Promise<{ scope: IFlexResolvedScope<TScope>; state: IStorageState }> {
|
|
2091
2240
|
this.assertOpen();
|
|
2092
|
-
|
|
2241
|
+
validateIdentifier(scopeId, 'scopeId');
|
|
2242
|
+
const admission = this.getScopeAdmission(scopeId);
|
|
2243
|
+
if (admission.retiring) throw this.createScopeRetirementError();
|
|
2244
|
+
const generation = admission.generation;
|
|
2245
|
+
admission.inFlightResolvers++;
|
|
2246
|
+
try {
|
|
2247
|
+
const scope = await this.scopeResolver.resolveScope(scopeId);
|
|
2248
|
+
this.assertOpen();
|
|
2249
|
+
if (admission.retiring || admission.generation !== generation) {
|
|
2250
|
+
throw this.createScopeRetirementError();
|
|
2251
|
+
}
|
|
2252
|
+
validateIdentifier(scope.storageKey, 'resolved storageKey');
|
|
2253
|
+
if (this.storageDrains.has(scope.storageKey)) {
|
|
2254
|
+
throw this.createScopeRetirementError();
|
|
2255
|
+
}
|
|
2256
|
+
let stateLoad = this.stateLoads.get(scope.storageKey);
|
|
2257
|
+
if (!stateLoad) {
|
|
2258
|
+
stateLoad = this.loadState(scope.storageKey);
|
|
2259
|
+
this.stateLoads.set(scope.storageKey, stateLoad);
|
|
2260
|
+
void stateLoad.catch(() => {
|
|
2261
|
+
if (this.stateLoads.get(scope.storageKey) === stateLoad) {
|
|
2262
|
+
this.stateLoads.delete(scope.storageKey);
|
|
2263
|
+
}
|
|
2264
|
+
});
|
|
2265
|
+
}
|
|
2266
|
+
const state = await stateLoad;
|
|
2267
|
+
this.assertOpen();
|
|
2268
|
+
if (
|
|
2269
|
+
admission.retiring
|
|
2270
|
+
|| admission.generation !== generation
|
|
2271
|
+
|| this.storageDrains.has(scope.storageKey)
|
|
2272
|
+
|| state.lifecycle !== 'active'
|
|
2273
|
+
) {
|
|
2274
|
+
throw this.createScopeRetirementError();
|
|
2275
|
+
}
|
|
2276
|
+
return { scope, state };
|
|
2277
|
+
} finally {
|
|
2278
|
+
admission.inFlightResolvers--;
|
|
2279
|
+
this.pruneScopeAdmission(scopeId, admission);
|
|
2280
|
+
}
|
|
2093
2281
|
}
|
|
2094
2282
|
|
|
2095
2283
|
private async loadState(storageKey: string): Promise<IStorageState> {
|
|
@@ -2134,11 +2322,18 @@ export class FlexHarness<TScope = unknown> {
|
|
|
2134
2322
|
activeRuns: new Map<string, IActiveRun>(),
|
|
2135
2323
|
pendingPermissions: new Map<string, IPendingPermission>(),
|
|
2136
2324
|
saveTail: Promise.resolve(),
|
|
2325
|
+
lifecycle: 'active',
|
|
2326
|
+
detachedToolCleanupErrors: [],
|
|
2137
2327
|
};
|
|
2138
2328
|
}
|
|
2139
2329
|
|
|
2140
|
-
private mutateAndSave<T>(
|
|
2330
|
+
private mutateAndSave<T>(
|
|
2331
|
+
state: IStorageState,
|
|
2332
|
+
mutation: () => T,
|
|
2333
|
+
allowDuringDrain = false,
|
|
2334
|
+
): Promise<T> {
|
|
2141
2335
|
const operation = state.saveTail.then(async () => {
|
|
2336
|
+
if (!allowDuringDrain) this.assertStateAcceptingWork(state);
|
|
2142
2337
|
const beforeMutation = this.createSnapshot(state, state.revision);
|
|
2143
2338
|
try {
|
|
2144
2339
|
const result = mutation();
|