@modelprofile.com/flexharness 18.1.0 → 18.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/classes.flexharness.cachewarm.js +3 -3
- package/dist_ts/classes.flexharness.callbacks.d.ts +37 -0
- package/dist_ts/classes.flexharness.callbacks.js +129 -2
- package/dist_ts/classes.flexharness.compaction.js +2 -2
- package/dist_ts/classes.flexharness.configurations.js +4 -4
- package/dist_ts/classes.flexharness.core.d.ts +39 -4
- package/dist_ts/classes.flexharness.core.js +467 -134
- package/dist_ts/classes.flexharness.d.ts +4 -0
- package/dist_ts/classes.flexharness.deletion.d.ts +13 -0
- package/dist_ts/classes.flexharness.deletion.js +132 -48
- package/dist_ts/classes.flexharness.imports.js +26 -21
- package/dist_ts/classes.flexharness.js +9 -1
- package/dist_ts/classes.flexharness.lifecycle.js +9 -3
- package/dist_ts/classes.flexharness.projects.js +2 -2
- package/dist_ts/classes.flexharness.prompts.js +14 -8
- package/dist_ts/classes.flexharness.reversions.js +3 -3
- package/dist_ts/classes.flexharness.runs.js +18 -2
- package/dist_ts/classes.flexharness.sessions.d.ts +1 -1
- package/dist_ts/classes.flexharness.sessions.js +18 -18
- package/dist_ts/classes.flexharness.slashcommands.js +3 -3
- package/dist_ts/classes.flexharness.subagents.js +37 -16
- package/dist_ts/classes.flexharness.tooldispatch.js +3 -3
- package/dist_ts/classes.scoperecovery.d.ts +37 -0
- package/dist_ts/classes.scoperecovery.js +229 -0
- package/dist_ts/classes.stores.d.ts +14 -1
- package/dist_ts/classes.stores.js +150 -17
- package/dist_ts/index.d.ts +1 -0
- package/dist_ts/index.js +2 -1
- package/dist_ts/interfaces.d.ts +113 -2
- package/dist_ts/interfaces.flexharness.d.ts +41 -5
- package/dist_ts/interfaces.js +1 -1
- package/dist_ts/utils.harnessoptions.d.ts +2 -0
- package/dist_ts/utils.harnessoptions.js +16 -11
- package/dist_ts_stores_nosqldb/classes.nosqldbstores.d.ts +3 -2
- package/dist_ts_stores_nosqldb/classes.nosqldbstores.js +33 -2
- package/dist_ts_stores_nosqldb/classes.recordstore.d.ts +10 -1
- package/dist_ts_stores_nosqldb/classes.recordstore.js +75 -7
- package/package.json +3 -3
- package/readme.md +43 -5
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/classes.flexharness.cachewarm.ts +2 -2
- package/ts/classes.flexharness.callbacks.ts +142 -1
- package/ts/classes.flexharness.compaction.ts +1 -1
- package/ts/classes.flexharness.configurations.ts +3 -3
- package/ts/classes.flexharness.core.ts +463 -133
- package/ts/classes.flexharness.deletion.ts +178 -62
- package/ts/classes.flexharness.imports.ts +26 -21
- package/ts/classes.flexharness.lifecycle.ts +8 -2
- package/ts/classes.flexharness.projects.ts +1 -1
- package/ts/classes.flexharness.prompts.ts +11 -7
- package/ts/classes.flexharness.reversions.ts +2 -2
- package/ts/classes.flexharness.runs.ts +16 -1
- package/ts/classes.flexharness.sessions.ts +17 -17
- package/ts/classes.flexharness.slashcommands.ts +2 -2
- package/ts/classes.flexharness.subagents.ts +40 -15
- package/ts/classes.flexharness.tooldispatch.ts +2 -2
- package/ts/classes.flexharness.ts +13 -0
- package/ts/classes.scoperecovery.ts +291 -0
- package/ts/classes.stores.ts +150 -16
- package/ts/index.ts +2 -0
- package/ts/interfaces.flexharness.ts +44 -4
- package/ts/interfaces.ts +124 -2
- package/ts/readme.md +43 -5
- package/ts/utils.harnessoptions.ts +18 -12
- package/ts_stores_nosqldb/classes.nosqldbstores.ts +52 -2
- package/ts_stores_nosqldb/classes.recordstore.ts +78 -8
- package/ts_stores_nosqldb/readme.md +10 -0
|
@@ -17,6 +17,8 @@ import type {
|
|
|
17
17
|
IFlexPromptQueueEntry,
|
|
18
18
|
IFlexPromptOptions,
|
|
19
19
|
IFlexContinuePromptResult,
|
|
20
|
+
IFlexDeleteSessionGenerationCohortAbsent,
|
|
21
|
+
IFlexDeleteSessionGenerationCohortSuperseded,
|
|
20
22
|
IFlexReversionSegment,
|
|
21
23
|
IFlexPendingReversionRelease,
|
|
22
24
|
IFlexResolvedModel,
|
|
@@ -47,6 +49,32 @@ import type { INormalizedSessionImportOptions } from './utils.sessionimport.js';
|
|
|
47
49
|
import type { ISessionMetricsOperation } from './utils.sessionmetrics.js';
|
|
48
50
|
|
|
49
51
|
export type TCanonicalOutcome = 'accepted' | 'rejected' | 'interrupted';
|
|
52
|
+
/** When a projection save started and completed, on the checkpoint clock. */
|
|
53
|
+
export type TProjectionSaved = (startedAt: number, completedAt: number) => void;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* A run's projection checkpoints: the parts of its committed steps, saved on its streaming
|
|
57
|
+
* assistant message while the run goes on, so that a process stopping mid-run leaves them visible.
|
|
58
|
+
* A checkpoint starts no earlier than the larger of 10 s and 20 times the duration of the run's
|
|
59
|
+
* last save of that message after that save completed; one is outstanding at most.
|
|
60
|
+
*/
|
|
61
|
+
export interface IRunCheckpoint {
|
|
62
|
+
/** The parts of committed steps, by id: what a checkpoint saves, as the parts stand then. */
|
|
63
|
+
committedPartIds: Set<string>;
|
|
64
|
+
/** A step committed since the last checkpoint read the run's parts. */
|
|
65
|
+
dirty: boolean;
|
|
66
|
+
/** A checkpoint is queued in the projection FIFO or saving. */
|
|
67
|
+
outstanding: boolean;
|
|
68
|
+
/** The deadline timer on the checkpoint clock, while one is armed. */
|
|
69
|
+
timer?: unknown;
|
|
70
|
+
/** When the run's last save of its current assistant message completed, on the checkpoint clock. */
|
|
71
|
+
savedAt?: number;
|
|
72
|
+
/** How long that save took, in milliseconds of the checkpoint clock. */
|
|
73
|
+
saveMs: number;
|
|
74
|
+
/** No checkpoint starts any more: the run reached its terminal, closed or its namespace stopped. */
|
|
75
|
+
closed: boolean;
|
|
76
|
+
}
|
|
77
|
+
|
|
50
78
|
export type TRunPhase =
|
|
51
79
|
| 'admitting'
|
|
52
80
|
| 'scheduled'
|
|
@@ -82,6 +110,7 @@ export interface ICompactorInvocation<TScope> extends IFlexAgentContextInvocatio
|
|
|
82
110
|
|
|
83
111
|
export interface IStoredSessionState {
|
|
84
112
|
storageKey: string;
|
|
113
|
+
recovering?: boolean;
|
|
85
114
|
compactorContext?: IFlexAgentContextInvocation<unknown>;
|
|
86
115
|
session: IFlexSession;
|
|
87
116
|
messages: IFlexMessage[];
|
|
@@ -214,6 +243,13 @@ export interface IStorageState {
|
|
|
214
243
|
scopeIdHint: string;
|
|
215
244
|
scopeContext: { scopeId: string; scope: unknown };
|
|
216
245
|
revision: number;
|
|
246
|
+
/** Complete live metadata; independent of the hydrated runtime subset below. */
|
|
247
|
+
catalogue: Map<string, IFlexSession>;
|
|
248
|
+
sessionLocal: boolean;
|
|
249
|
+
qualifyingLegacy: boolean;
|
|
250
|
+
recoverySessionIds: Set<string>;
|
|
251
|
+
sessionLoads: Map<string, Promise<IStoredSessionState>>;
|
|
252
|
+
sharedRecovery?: Promise<void>;
|
|
217
253
|
sessions: Map<string, IStoredSessionState>;
|
|
218
254
|
retainedSessionCleanups: Map<string, IRetainedSessionCleanup>;
|
|
219
255
|
sessionDeletions: Map<string, Promise<void>>;
|
|
@@ -296,6 +332,8 @@ export interface IActiveRun {
|
|
|
296
332
|
stepStreamedCalls: Map<string, ReadonlySet<string>>;
|
|
297
333
|
/** The step's streamed calls the SDK could not parse or repair, with its error: they never start. */
|
|
298
334
|
stepInvalidCalls: Map<string, string>;
|
|
335
|
+
/** The run's projection checkpoints; see {@link IRunCheckpoint}. */
|
|
336
|
+
checkpoint: IRunCheckpoint;
|
|
299
337
|
textPartClosed: boolean;
|
|
300
338
|
toolPartIds: Map<string, string>;
|
|
301
339
|
trustedToolExecutionErrors: Map<string, Error>;
|
|
@@ -572,14 +610,16 @@ export interface IFlexSubagentAcquisition {
|
|
|
572
610
|
created: boolean;
|
|
573
611
|
}
|
|
574
612
|
|
|
575
|
-
export
|
|
576
|
-
matched:
|
|
577
|
-
|
|
578
|
-
|
|
613
|
+
export type TSessionDeletionPlan =
|
|
614
|
+
| { matched: true; deletion: Promise<void> }
|
|
615
|
+
| IFlexDeleteSessionGenerationCohortAbsent
|
|
616
|
+
| IFlexDeleteSessionGenerationCohortSuperseded;
|
|
579
617
|
|
|
580
618
|
export interface IReservedSessionDeletion {
|
|
581
619
|
deletedSessions: IFlexSession[];
|
|
582
620
|
descendantRoots: string[];
|
|
621
|
+
/** The settlement of the deleted sessions' configuration resolutions the reservation's fence aborted. */
|
|
622
|
+
configurationSettlement?: Promise<void>;
|
|
583
623
|
}
|
|
584
624
|
|
|
585
625
|
export type TProjectTaskToolInput =
|
package/ts/interfaces.ts
CHANGED
|
@@ -1569,10 +1569,39 @@ export interface IFlexDeleteSessionGenerationCohortInput {
|
|
|
1569
1569
|
authorizedCohort: readonly IFlexSessionGenerationCohortEntry[];
|
|
1570
1570
|
}
|
|
1571
1571
|
|
|
1572
|
-
|
|
1573
|
-
|
|
1572
|
+
/**
|
|
1573
|
+
* The root generation's deletion completed, including a retained cleanup tombstone of that
|
|
1574
|
+
* generation, which the call finishes rather than answering `absent`.
|
|
1575
|
+
*/
|
|
1576
|
+
export interface IFlexDeleteSessionGenerationCohortMatched {
|
|
1577
|
+
readonly matched: true;
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
/**
|
|
1581
|
+
* Neither a live session nor a retained cleanup tombstone of any generation holds the root's
|
|
1582
|
+
* session id: its generation was deleted completely, or never existed in this namespace.
|
|
1583
|
+
*/
|
|
1584
|
+
export interface IFlexDeleteSessionGenerationCohortAbsent {
|
|
1585
|
+
readonly matched: false;
|
|
1586
|
+
readonly reason: 'absent';
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
/**
|
|
1590
|
+
* The root's session id is held by another generation, as a live session or a retained cleanup
|
|
1591
|
+
* tombstone, which is left untouched. Generation sequences grow with the scope revision, so
|
|
1592
|
+
* `current` is newer than every generation the id held before it.
|
|
1593
|
+
*/
|
|
1594
|
+
export interface IFlexDeleteSessionGenerationCohortSuperseded {
|
|
1595
|
+
readonly matched: false;
|
|
1596
|
+
readonly reason: 'superseded';
|
|
1597
|
+
readonly current: Readonly<IFlexSessionGeneration>;
|
|
1574
1598
|
}
|
|
1575
1599
|
|
|
1600
|
+
export type IFlexDeleteSessionGenerationCohortResult =
|
|
1601
|
+
| IFlexDeleteSessionGenerationCohortMatched
|
|
1602
|
+
| IFlexDeleteSessionGenerationCohortAbsent
|
|
1603
|
+
| IFlexDeleteSessionGenerationCohortSuperseded;
|
|
1604
|
+
|
|
1576
1605
|
export const FLEX_PROJECT_MANAGEMENT_LIMITS = Object.freeze({
|
|
1577
1606
|
maxGoalBytes: 8 * 1024,
|
|
1578
1607
|
maxScratchpadBytes: 128 * 1024,
|
|
@@ -1688,7 +1717,92 @@ export interface IFlexProjectScratchpadResult extends IFlexProjectStateResult {
|
|
|
1688
1717
|
scratchpad: string;
|
|
1689
1718
|
}
|
|
1690
1719
|
|
|
1720
|
+
/** Compact recovery proof stored separately from the strict schema-1 scope snapshot. */
|
|
1721
|
+
export interface IFlexScopeRecoverySnapshot {
|
|
1722
|
+
schemaVersion: 1;
|
|
1723
|
+
revision: number;
|
|
1724
|
+
catalogueRevision: number;
|
|
1725
|
+
catalogueDigest: string;
|
|
1726
|
+
recoverySessionIds: string[];
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
/**
|
|
1730
|
+
* Additive sidecar persistence for `RecoverableFlexScopeStore`: the recovery proof and the
|
|
1731
|
+
* namespace's write provenance. A scope snapshot cannot show that nothing needs recovery: a writer
|
|
1732
|
+
* unaware of the proof (an older release, an eager harness, another tool) may write a projection's
|
|
1733
|
+
* pending capture or apply, run its workspace effect and stop before it writes the scope. The
|
|
1734
|
+
* provenance is a durable witness of the namespace's writes that the implementation keeps so that:
|
|
1735
|
+
*
|
|
1736
|
+
* - every write in every recovery-bearing domain of the namespace (scope, proof, projections,
|
|
1737
|
+
* permissions, agent events and archives, tool jobs, results, every deletion and purge of those)
|
|
1738
|
+
* changes it atomically with the write, by every writer that can open the namespace, releases
|
|
1739
|
+
* that predate this contract included: it is a counter or header those writers already change,
|
|
1740
|
+
* never a field only recovery-aware writers maintain. Project management is not recovery-bearing
|
|
1741
|
+
* and may stay outside the witness: qualification, repair and proof publication never read it,
|
|
1742
|
+
* each of its operations loads fresh and saves under its own revision compare-and-swap and
|
|
1743
|
+
* session-generation fence, and a retained deletion retries its tombstone idempotently from the
|
|
1744
|
+
* core cleanup tombstone. A witness that also counts it conforms too;
|
|
1745
|
+
* - a write of an activated instance is recorded as known atomically with the write; every other
|
|
1746
|
+
* write is unknown;
|
|
1747
|
+
* - it is read from metadata alone, never from history bodies, and never returns to an earlier
|
|
1748
|
+
* value while the namespace holds data.
|
|
1749
|
+
*/
|
|
1750
|
+
export interface IFlexScopeRecoveryStore {
|
|
1751
|
+
/**
|
|
1752
|
+
* Synchronous and local: afterwards this instance's writes for the key are unknown. Never changes
|
|
1753
|
+
* durable state. Called on release, eager loading, disposal, retirement and stale evidence.
|
|
1754
|
+
*/
|
|
1755
|
+
deactivateWriteProvenance(storageKey: string): void;
|
|
1756
|
+
/**
|
|
1757
|
+
* True only if every write since the last {@link qualifyWriteProvenance} is known; then activates
|
|
1758
|
+
* this instance for the key. Otherwise false, deactivated. Never repairs. While activated, each
|
|
1759
|
+
* write checks in the same atomic step that the witness still shows only known writes and fails
|
|
1760
|
+
* with `FlexHarnessStoreFormatError` instead of recording itself as known when it does not.
|
|
1761
|
+
*/
|
|
1762
|
+
loadWriteProvenance(storageKey: string): Promise<boolean>;
|
|
1763
|
+
/**
|
|
1764
|
+
* Marks every write so far as known and activates. Called only at the end of an explicit complete
|
|
1765
|
+
* qualification under exclusive ownership, after the proof was published; a failure leaves the
|
|
1766
|
+
* namespace unqualified.
|
|
1767
|
+
*/
|
|
1768
|
+
qualifyWriteProvenance(storageKey: string): Promise<void>;
|
|
1769
|
+
load(storageKey: string): Promise<IFlexScopeRecoverySnapshot | undefined>;
|
|
1770
|
+
/**
|
|
1771
|
+
* Durable compare-and-swap: writes only when the stored revision equals `expectedRevision`
|
|
1772
|
+
* (`FlexHarnessStoreConflictError` otherwise); an unknown outcome is
|
|
1773
|
+
* `FlexHarnessStoreCommitUncertainError`.
|
|
1774
|
+
*/
|
|
1775
|
+
save(storageKey: string, snapshot: IFlexScopeRecoverySnapshot, expectedRevision: number): Promise<void>;
|
|
1776
|
+
}
|
|
1777
|
+
|
|
1778
|
+
/** Requires exclusive namespace ownership, like the harness itself. */
|
|
1779
|
+
export interface IFlexSessionLocalScopeStore {
|
|
1780
|
+
/** Releases local writer activation only, not a distributed namespace lock. */
|
|
1781
|
+
releaseOwnership(storageKey: string): void;
|
|
1782
|
+
/** A missing/stale proof is unqualified, never an assertion that recovery is unnecessary. */
|
|
1783
|
+
loadCatalogue(storageKey: string): Promise<{
|
|
1784
|
+
snapshot: IFlexScopeSnapshot | undefined;
|
|
1785
|
+
qualified: boolean;
|
|
1786
|
+
recoverySessionIds: string[];
|
|
1787
|
+
}>;
|
|
1788
|
+
saveCatalogue(storageKey: string, snapshot: IFlexScopeSnapshot, expectedRevision: number): Promise<void>;
|
|
1789
|
+
setRecoveryRequired(storageKey: string, sessionId: string, required: boolean): Promise<void>;
|
|
1790
|
+
/** Called only after explicit complete legacy qualification under namespace ownership. */
|
|
1791
|
+
qualifyCatalogue(storageKey: string, snapshot: IFlexScopeSnapshot, recoverySessionIds: string[]): Promise<void>;
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1794
|
+
export interface IFlexPrepareScopeOptions {
|
|
1795
|
+
/** Explicitly permits the one-time complete legacy load/repair when proof is absent or stale. */
|
|
1796
|
+
qualifyLegacy?: boolean;
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1799
|
+
export interface IFlexScopeCatalogue {
|
|
1800
|
+
sessions: IFlexSession[];
|
|
1801
|
+
recoverySessionIds: string[];
|
|
1802
|
+
}
|
|
1803
|
+
|
|
1691
1804
|
export interface IFlexScopeStore {
|
|
1805
|
+
sessionLocal?: IFlexSessionLocalScopeStore;
|
|
1692
1806
|
load(storageKey: string): Promise<IFlexScopeSnapshot | undefined>;
|
|
1693
1807
|
save(
|
|
1694
1808
|
storageKey: string,
|
|
@@ -2197,6 +2311,8 @@ export interface IFlexHarnessOptions<TScope> {
|
|
|
2197
2311
|
/** Lets prompts name attachments by reference; see {@link IFlexAttachmentResolver}. */
|
|
2198
2312
|
attachmentResolver?: IFlexAttachmentResolver<TScope>;
|
|
2199
2313
|
stores?: IFlexHarnessStores;
|
|
2314
|
+
/** Opt-in: requires scopes.sessionLocal; never falls back to an implicit complete load. */
|
|
2315
|
+
sessionLoading?: 'eager' | 'session-local';
|
|
2200
2316
|
agentSessionPolicy?: IFlexAgentSessionPolicy<TScope>;
|
|
2201
2317
|
toolOutputLimits?: IFlexJsonLimits;
|
|
2202
2318
|
callbackLimits?: IFlexCallbackLimits;
|
|
@@ -2222,6 +2338,12 @@ export interface IFlexHarnessOptions<TScope> {
|
|
|
2222
2338
|
maxSubagentCallsPerRun?: number;
|
|
2223
2339
|
/** Bounds and the clock of the cache keeper; see `keepCacheWarm()`. */
|
|
2224
2340
|
cacheWarm?: IFlexCacheWarmOptions;
|
|
2341
|
+
/**
|
|
2342
|
+
* The clock a running turn's projection checkpoints read and set their timers on. Defaults to
|
|
2343
|
+
* the system clock, whose timers do not keep the process alive. Tests pass a fake one; the
|
|
2344
|
+
* spacing of checkpoints is fixed, not configurable.
|
|
2345
|
+
*/
|
|
2346
|
+
projectionCheckpointClock?: IFlexCacheWarmClock;
|
|
2225
2347
|
/**
|
|
2226
2348
|
* Called once each scope load ends, with what it took. An exception it throws, or a promise it
|
|
2227
2349
|
* rejects, is ignored.
|
package/ts/readme.md
CHANGED
|
@@ -709,7 +709,7 @@ await harness.dispose();
|
|
|
709
709
|
|
|
710
710
|
`getSession()` returns the session as saved after every queued write that may change it; it does not wait for queued writes of the scope's other sessions, so a slow save of one session never holds a read of another. `listSessions()` waits for every queued write.
|
|
711
711
|
|
|
712
|
-
`listSessionIdentities(scopeId)` lists a scope's live sessions as `IFlexSessionIdentity` (`sessionId`, `sessionGenerationId`, `sessionGenerationSequence`, `parentSessionId` when set), sorted by session id.
|
|
712
|
+
`listSessionIdentities(scopeId)` lists a scope's live sessions as `IFlexSessionIdentity` (`sessionId`, `sessionGenerationId`, `sessionGenerationSequence`, `parentSessionId` when set), sorted by session id. It reads complete scope metadata only, never a projection, event log or job store. Before the catalogue load settles it normalizes the stored scope snapshot just as loading does (including deterministic legacy generations and tombstone filtering); afterward it reads the complete in-memory catalogue, not the hydrated runtime subset.
|
|
713
713
|
|
|
714
714
|
The load gives the event loop a turn about every 25 ms of work, so timers, I/O and a caller's cancellation keep running beside it, and its repair takes time linear in a session's size. A request that stops waiting for the load leaves it running, and the next request of the scope joins it; `dispose()` and `retireScope()` stop it at its next turn, and the requests waiting for it fail with `FlexHarnessAbortError`. The `scopeLoadObserver` option is called once each load ends, with `IFlexScopeLoadMetrics`: `scopeId`, `storageKey`, the live `sessions` it read, `storeReadMs` (waiting for the scope snapshot and each session's projection, permissions, agent event store and job store), `agentSessionMs` (creating the sessions' agent sessions, each of which reads and checks its event log), `repairMs`, `totalMs`, all wall-clock milliseconds, and `outcome` (`'loaded'`, `'failed'`, or `'aborted'` by disposal or retirement). An exception the observer throws, or a promise it rejects, is ignored.
|
|
715
715
|
|
|
@@ -731,12 +731,48 @@ The reservation save is the admission point. A save failure produces no start ev
|
|
|
731
731
|
|
|
732
732
|
`listMessageMetadataPage()` pages like `listMessagePage()`, with the same options, `startIndex`, cursors (a cursor of either kind of page continues the other) and `includeUndone`, but each message carries only its identity and state (`TFlexMessageMetadata`: `messageId`, `sessionId`, `runId`, `role`, `status`, `createdAt`, `completedAt`, `steerId`, `promptId`, `keptInContext`, `undone`, `cacheWarm`), never its parts, model, usage or error. A full scan then reads the same few hundred bytes per message whatever its content, enough to prove which prompts and steers a session received and whether its runs kept them in context.
|
|
733
733
|
|
|
734
|
-
`updateSession()` supports title replacement, explicit title clearing with `null`, and archive state through `archived`. Title-only updates remain available while prompts are queued or running, while permission is pending, and after archival. Requests containing `archived` are rejected while the session has any outstanding prompt or pending permission; a mixed title-and-archive request is rejected atomically without changing the title. Archived sessions expose `archivedAt`. Every session exposes `preview` once a run with prompt text was admitted: its first prompt as one line (whitespace runs collapsed, cut at a character boundary to `maxTitleBytes`), with no model call, so a host can name a session nobody titled (`title || preview`). It is set once, a session stored before previews existed takes it from its first prompt at its next run, and no caller sets or clears it. Deleting a session cascades through its complete descendant subtree. One durable root-keyed tombstone group hides every newly affected live session, and the delete also joins any already-separate descendant cleanup groups without rewriting their roots. FlexHarness then cancels queued
|
|
734
|
+
`updateSession()` supports title replacement, explicit title clearing with `null`, and archive state through `archived`. Title-only updates remain available while prompts are queued or running, while permission is pending, and after archival. Requests containing `archived` are rejected while the session has any outstanding prompt or pending permission; a mixed title-and-archive request is rejected atomically without changing the title. Archived sessions expose `archivedAt`. Every session exposes `preview` once a run with prompt text was admitted: its first prompt as one line (whitespace runs collapsed, cut at a character boundary to `maxTitleBytes`), with no model call, so a host can name a session nobody titled (`title || preview`). It is set once, a session stored before previews existed takes it from its first prompt at its next run, and no caller sets or clears it. Deleting a session cascades through its complete descendant subtree. One durable root-keyed tombstone group hides every newly affected live session, and the delete also joins any already-separate descendant cleanup groups without rewriting their roots. FlexHarness cancels the subtree's active runs as soon as the group is applied, before its save completes, so an admission that has not reached its model call never does however slow the store; it then cancels queued subtree work, emits terminal queue events, waits for admitted initialization, and purges runtime queue status while cleaning runtime and persisted domains child-first. The requested root tombstone is removed last after every domain confirms cleanup; project-management cleanup confirmation is a retained durable project tombstone rather than physical removal. An imported session can hold no project-management state, so its deletion writes no project tombstone. A successful live `deleteSession()` call emits `session.deleted` for each session it newly tombstoned; retries of an existing tombstone and automatic load, retirement, or disposal cleanup emit no deletion events. Direct deletion of a descendant cascades only through that descendant's subtree. Cleanup authority follows the resolved storage namespace, so scope aliases share the same groups. A partial failure retains durable ownership for retry by a later `deleteSession()`, namespace load, `retireScope()`, or `dispose()` call.
|
|
735
735
|
|
|
736
|
-
`deleteSessionGenerationCohort()` is the generation-fenced destructive form for controllers. Its `root` and every `authorizedCohort` entry use the exact `{ sessionId, sessionGenerationId, sessionGenerationSequence }` shape. The cohort contains unique entries in strictly ascending `sessionId` order and is limited by `FLEX_SESSION_GENERATION_COHORT_MAX_ENTRIES` (2048). FlexHarness atomically validates the matching root, its complete live subtree, and every retained descendant tombstone group before reserving deletion. A missing or different root generation returns `{ matched: false }`
|
|
736
|
+
`deleteSessionGenerationCohort()` is the generation-fenced destructive form for controllers. Its `root` and every `authorizedCohort` entry use the exact `{ sessionId, sessionGenerationId, sessionGenerationSequence }` shape. The cohort contains unique entries in strictly ascending `sessionId` order and is limited by `FLEX_SESSION_GENERATION_COHORT_MAX_ENTRIES` (2048). FlexHarness atomically validates the matching root, its complete live subtree, and every retained descendant tombstone group before reserving deletion. A missing or different root generation returns `{ matched: false }` with a `reason` and touches nothing: `{ matched: false, reason: 'absent' }` only when neither a live session nor a retained cleanup tombstone of any generation holds the root's session id (its generation was deleted completely or never existed here), and `{ matched: false, reason: 'superseded', current }` when another generation holds it, as a live session or a retained cleanup tombstone, with `current` its `{ sessionGenerationId, sessionGenerationSequence }`. A retained cleanup tombstone of the root's own generation is never `absent`: the call finishes that deletion and answers `{ matched: true }`. Generation sequences grow with the scope revision, so `current` is newer than every generation the id held before; `absent` cannot say whether a newer generation existed and was deleted in between. Readers that check only `matched` keep working; a missing or mismatched cascade entry throws before new tombstoning or cleanup. Extra valid entries do not expand the cascade. A matched partial cleanup remains retryable with the same authority and resolves to `{ matched: true }` when cleanup completes. The existing `deleteSession()` method remains available for callers that intentionally own the complete reusable-ID namespace.
|
|
737
737
|
|
|
738
738
|
`abort()` returns `true` only while cancellation is still accepted. Terminal persistence is the run's commit point; once it starts, `abort()` returns `false` and the already-fixed terminal outcome completes while the session remains busy.
|
|
739
739
|
|
|
740
|
+
### Session-local loading and startup recovery
|
|
741
|
+
|
|
742
|
+
The default `sessionLoading: 'eager'` preserves existing store behavior. Opt into `sessionLoading: 'session-local'` to load a complete metadata catalogue and hydrate only the session a request uses. Concurrent requests, including scope aliases of one storage key, join that session's hydration and repair. Creating an empty session reads none of the unrelated sessions' projections, permissions, events or jobs. `listSessions()` deliberately requests every session's repaired metadata; use `getScopeCatalogue()` or `listSessionIdentities()` for metadata-only enumeration.
|
|
743
|
+
|
|
744
|
+
```typescript
|
|
745
|
+
const harness = new FlexHarness({
|
|
746
|
+
scopeResolver,
|
|
747
|
+
modelResolver,
|
|
748
|
+
stores,
|
|
749
|
+
sessionLoading: 'session-local',
|
|
750
|
+
});
|
|
751
|
+
// Normal startup: recover indexed shared-workspace obligations and deletion tombstones.
|
|
752
|
+
await harness.prepareScope(scopeId);
|
|
753
|
+
const { sessions, recoverySessionIds } = await harness.getScopeCatalogue(scopeId);
|
|
754
|
+
```
|
|
755
|
+
|
|
756
|
+
Session-local loading requires `stores.scopes.sessionLocal`. Missing capabilities fail at construction; absent or stale qualification evidence fails before ordinary session use. There is no implicit eager fallback. On a legacy namespace, the host must exclusively own the namespace, excluding every scope alias and every old writer, and explicitly run `await harness.prepareScope(scopeId, { qualifyLegacy: true })`. This permits one complete load/repair and establishes authoritative recovery evidence. It is not a flag to put on every request. Scope retirement and disposal join owned target hydrations without hydrating cold sessions.
|
|
757
|
+
|
|
758
|
+
Shared-workspace recovery is not merely target-local: durable sidecar obligations are registered before capture preparation, application, finalization and release effects, and cleared only after canonical durable resolution. Startup gates on those obligations and tombstones, while unrelated sessions remain cold. Per-session repair still uses the canonical Agent event history and projection reconciliation.
|
|
759
|
+
|
|
760
|
+
#### Implementing `scopes.sessionLocal` on your own storage
|
|
761
|
+
|
|
762
|
+
A host that keeps Flex data in its own storage composes `new RecoverableFlexScopeStore(existingScopeStore, recoveryStore)` and implements `IFlexScopeRecoveryStore` for that storage. The scope snapshot stays exactly `{ schemaVersion, revision, sessions, tombstones }`; the proof is a separate record, `IFlexScopeRecoverySnapshot` `{ schemaVersion: 1, revision, catalogueRevision, catalogueDigest, recoverySessionIds }`, which `RecoverableFlexScopeStore` computes and the recovery store persists. A session-local load trusts a namespace only when the proof names the stored scope's revision and digest and the write provenance is current. The digest alone is never enough: a writer that does not know the proof (an older release, an eager harness, another tool) can write a projection's pending capture or apply, run its workspace effect and stop before it writes the scope, which leaves scope and proof byte for byte unchanged. The recovery store must meet this contract:
|
|
763
|
+
|
|
764
|
+
1. `save(storageKey, proof, expectedRevision)` is a durable compare-and-swap: it writes only when the stored revision equals `expectedRevision` and throws `FlexHarnessStoreConflictError` otherwise; an unknown outcome throws `FlexHarnessStoreCommitUncertainError`.
|
|
765
|
+
2. The provenance is a durable witness that every write in every recovery-bearing domain of the namespace changes atomically with the write: scope, proof, projections, permissions, agent events and archives, tool jobs, results, and every deletion or purge of those. Every writer that can open the namespace must change it, including releases that predate this contract, so it has to be a counter or header those writers already change, never a field only recovery-aware writers maintain. `stores.projectManagement` is not recovery-bearing, and a host may keep it outside the witness: session-local qualification, repair and proof publication never read it, and a load touches it only to retry a retained deletion; every project-management operation loads its record fresh and saves under that record's own revision compare-and-swap and the session-generation fence, so a save from a stale read fails with `FlexHarnessStoreConflictError` instead of overwriting another writer's change; and a deletion is recovered from the core cleanup tombstone in the scope snapshot, whose retry writes the project-management tombstone idempotently (a record already tombstoned for that generation is left as it is). Project-management writes by other writers therefore never invalidate the proof. A bundle whose witness also counts project-management writes, as the NoSQL and in-memory stores do, still conforms. `NoSqlFlexHarnessStores` uses the namespace header's `creationCount`, which every write transaction increments since 12.1.1. Its releases 10.1.0 to 12.1.0 counted only record creations, so their projection, event and job updates change no witness: a namespace one of them may have written must be qualified explicitly before a session-local load.
|
|
766
|
+
3. A write of an activated instance is recorded as known atomically with the write (NoSQL: `recoveryWriteCount` in the same transaction); every other write is unknown. While activated, a write checks in the same atomic step that the witness still shows only known writes, and fails with `FlexHarnessStoreFormatError` instead of recording itself as known when it does not.
|
|
767
|
+
4. `loadWriteProvenance(storageKey)` reads metadata only, never history bodies, and returns `true` only if every write since the last qualification is known; it then activates the instance for that key. Otherwise it returns `false` and deactivates. It never repairs.
|
|
768
|
+
5. `deactivateWriteProvenance(storageKey)` is synchronous and local: afterwards the instance's writes are unknown. FlexHarness calls it on release, eager loading, disposal, retirement and whenever the evidence is not current.
|
|
769
|
+
6. `qualifyWriteProvenance(storageKey)` marks every write so far as known and activates. FlexHarness calls it only at the end of `prepareScope(scopeId, { qualifyLegacy: true })`, after the complete load and repair and after it published the new proof while inactive; when it fails, the namespace stays unqualified.
|
|
770
|
+
7. The witness never returns to an earlier value while the namespace holds data; an empty namespace (no scope, no proof, no records) may start over.
|
|
771
|
+
|
|
772
|
+
When the older releases of a host change no such witness, the host cannot detect their writes: it must not open the namespace with session-local loading after an older release may have written it, unless it qualifies explicitly first. Neither the provenance nor any compare-and-swap replaces the host's exclusive ownership of the namespace: no simultaneous writer, scope alias or older release.
|
|
773
|
+
|
|
774
|
+
In-memory and NoSQL stores implement the additive capability. Existing custom stores and `JsonFileFlexHarnessStores` keep their eager behavior; the JSON store creates no new recovery files and cannot opt into session-local loading without a separate capable backend.
|
|
775
|
+
|
|
740
776
|
### Continuing Without New Input
|
|
741
777
|
|
|
742
778
|
Use `continuePrompt()` after a failed or stopped run whose input remains in context
|
|
@@ -1055,7 +1091,7 @@ Resolved model identity contains provider and model IDs plus optional display na
|
|
|
1055
1091
|
|
|
1056
1092
|
Public audit history is safe to send to controllers: attachment parts contain source and size metadata, never inline base64, data URLs, or remote URL payloads. Canonical private Agent events retain those values solely for subsequent model turns.
|
|
1057
1093
|
|
|
1058
|
-
Sessions expose `idle`, `scheduled`, `running`, `waiting_permission`, `failed`, and `cancelled` status. A run stopped by `abort()`, `cancelPrompt()` or a rejected permission leaves its session `cancelled`; only a run that failed leaves it `failed`. Persisted non-terminal activity is repaired from canonical Agent generation outcomes after process restart; incomplete messages and parts normalize to `cancelled` unless an accepted hidden terminal stage can be promoted.
|
|
1094
|
+
Sessions expose `idle`, `scheduled`, `running`, `waiting_permission`, `failed`, and `cancelled` status. A run stopped by `abort()`, `cancelPrompt()` or a rejected permission leaves its session `cancelled`; only a run that failed leaves it `failed`. Persisted non-terminal activity is repaired from canonical Agent generation outcomes after process restart; incomplete messages and parts normalize to `cancelled` unless an accepted hidden terminal stage can be promoted; an interrupted turn's assistant message keeps the parts of its last checkpoint.
|
|
1059
1095
|
|
|
1060
1096
|
## Conversation Import
|
|
1061
1097
|
|
|
@@ -1163,7 +1199,9 @@ FlexHarness wraps every provided tool `execute` method before AgentSession recei
|
|
|
1163
1199
|
|
|
1164
1200
|
`toolOutputLimits` in the complete setup above bounds traversal depth and encoded bytes. The normalizer enforces its byte allowance incrementally: oversized strings are replaced before entering output, and arrays/objects stop reading entries once only truncation metadata fits.
|
|
1165
1201
|
|
|
1166
|
-
Streaming callbacks use run-local synchronous state rather than one persistence promise per source delta. Text and reasoning accumulate only in the run-local terminal projection while each source delta remains an immediate exact public event. Every distinct async-iterable tool output appears immediately as a bounded cumulative `part.updated` snapshot while the tool remains `running`, including the final yielded value before completion. Only the authoritative `part.completed` output enters the terminal projection, and failed or interrupted tools discard their transient output. `callbackLimits` bounds callback events, accumulated output bytes, and part count; overflow aborts internally with `FlexHarnessCallbackOverflowError` and the turn is recorded as failed. The calls of a `run_code` program never overflow it: they stop their program at its `callbacks` limit instead. Reservation and terminal finalization are the normal persistence
|
|
1202
|
+
Streaming callbacks use run-local synchronous state rather than one persistence promise per source delta. Text and reasoning accumulate only in the run-local terminal projection while each source delta remains an immediate exact public event. Every distinct async-iterable tool output appears immediately as a bounded cumulative `part.updated` snapshot while the tool remains `running`, including the final yielded value before completion. Only the authoritative `part.completed` output enters the terminal projection, and failed or interrupted tools discard their transient output. `callbackLimits` bounds callback events, accumulated output bytes, and part count; overflow aborts internally with `FlexHarnessCallbackOverflowError` and the turn is recorded as failed. The calls of a `run_code` program never overflow it: they stop their program at its `callbacks` limit instead. Reservation and terminal finalization are the normal persistence points, with permission state changes and the projection checkpoints of a running turn (below) as additional ones.
|
|
1203
|
+
|
|
1204
|
+
While a turn runs, its streaming assistant message is checkpointed. A checkpoint saves the parts of the model steps the turn committed (the engine saves a step's events before the step ends), each as it stands, in place of the message's parts, in one projection save in the session's projection FIFO. It starts no earlier than the larger of 10 s and 20 times the duration of the turn's last save of that message (its start, a steer split or the previous checkpoint), counted from that save's completion, so a turn shorter than that saves none. At most one is outstanding per turn; its deadline timer does not keep the process alive and fires also when no further step commits. The terminal, a steer split, the turn's close and the namespace's drain, disposal, fence or retirement end the pending one. A process that stops mid-turn therefore leaves the committed steps up to its last completed checkpoint visible, at most the current spacing plus one save behind; the next load repairs the message to `cancelled` as before, its running parts `cancelled`, and never rebuilds parts from events. The visible parts are always a prefix of the committed steps: nothing of the step in progress is saved, neither its text nor a tool call it streamed or started, although a call that started is in the model context (its call and execution intent are saved before it runs). Checkpoints emit no event. A failed checkpoint save fails the turn like its other saves, with a `persistence` error, and an uncertain one fences the namespace. `projectionCheckpointClock` (an `IFlexCacheWarmClock`) lets tests drive the timing; the spacing is fixed.
|
|
1167
1205
|
|
|
1168
1206
|
Model resolver, tool provider, delegated run admission provider, AgentSession, tool execution, tool callback, tool cleanup, and run-persistence failures cross an untrusted error boundary. By default they become a fixed immutable `FlexHarnessExternalError` before completion rejection, persistence, events, or detached-cleanup reporting. Raw external messages and aggregate members are not retained. A failed `onToolCallFinish` callback stores and accounts for only the bounded projected message; it does not otherwise reject completion, although exceeding the configured callback limits still fails the run. Scope resolution and the initial store load happen before a run exists and remain outside this boundary.
|
|
1169
1207
|
|
|
@@ -395,17 +395,32 @@ export function normalizeSlashCommands<TScope>(
|
|
|
395
395
|
return Object.freeze(normalized);
|
|
396
396
|
}
|
|
397
397
|
|
|
398
|
-
const
|
|
398
|
+
const systemClock: IFlexCacheWarmClock = Object.freeze({
|
|
399
399
|
now: () => Date.now(),
|
|
400
400
|
setTimeout: (callback: () => void, delayMs: number) => {
|
|
401
401
|
const handle = setTimeout(callback, delayMs);
|
|
402
|
-
//
|
|
402
|
+
// Neither a warm period nor a pending projection checkpoint holds the process open.
|
|
403
403
|
handle.unref?.();
|
|
404
404
|
return handle;
|
|
405
405
|
},
|
|
406
406
|
clearTimeout: (handle: unknown) => clearTimeout(handle as ReturnType<typeof setTimeout>),
|
|
407
407
|
});
|
|
408
408
|
|
|
409
|
+
/** The clock `name` gives, checked; the system clock, whose timers are unref'd, by default. */
|
|
410
|
+
export function normalizeClock(clock: IFlexCacheWarmClock | undefined, name: string): IFlexCacheWarmClock {
|
|
411
|
+
const resolved: unknown = clock ?? systemClock;
|
|
412
|
+
if (
|
|
413
|
+
!resolved
|
|
414
|
+
|| typeof resolved !== 'object'
|
|
415
|
+
|| typeof (resolved as IFlexCacheWarmClock).now !== 'function'
|
|
416
|
+
|| typeof (resolved as IFlexCacheWarmClock).setTimeout !== 'function'
|
|
417
|
+
|| typeof (resolved as IFlexCacheWarmClock).clearTimeout !== 'function'
|
|
418
|
+
) {
|
|
419
|
+
throw new FlexHarnessValidationError(`${name} must provide now, setTimeout and clearTimeout.`);
|
|
420
|
+
}
|
|
421
|
+
return resolved as IFlexCacheWarmClock;
|
|
422
|
+
}
|
|
423
|
+
|
|
409
424
|
export function normalizeCacheWarmOptions(
|
|
410
425
|
input: IFlexHarnessOptions<unknown>['cacheWarm'],
|
|
411
426
|
): INormalizedCacheWarmOptions {
|
|
@@ -416,16 +431,7 @@ export function normalizeCacheWarmOptions(
|
|
|
416
431
|
const unsupported = Object.keys(options)
|
|
417
432
|
.find((key) => !['maxDurationMs', 'maxOutputTokens', 'clock'].includes(key));
|
|
418
433
|
if (unsupported) throw new FlexHarnessValidationError(`cacheWarm does not support "${unsupported}".`);
|
|
419
|
-
const clock
|
|
420
|
-
if (
|
|
421
|
-
!clock
|
|
422
|
-
|| typeof clock !== 'object'
|
|
423
|
-
|| typeof clock.now !== 'function'
|
|
424
|
-
|| typeof clock.setTimeout !== 'function'
|
|
425
|
-
|| typeof clock.clearTimeout !== 'function'
|
|
426
|
-
) {
|
|
427
|
-
throw new FlexHarnessValidationError('cacheWarm.clock must provide now, setTimeout and clearTimeout.');
|
|
428
|
-
}
|
|
434
|
+
const clock = normalizeClock(options.clock, 'cacheWarm.clock');
|
|
429
435
|
return Object.freeze({
|
|
430
436
|
maxDurationMs: resolveBoundedPositiveInteger(
|
|
431
437
|
options.maxDurationMs,
|
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import * as plugins from './plugins.js';
|
|
2
|
+
import {
|
|
3
|
+
assertFlexScopeRecoverySnapshot,
|
|
4
|
+
RecoverableFlexScopeStore,
|
|
5
|
+
validateFlexScopeRecoverySave,
|
|
6
|
+
} from '../ts/classes.scoperecovery.js';
|
|
2
7
|
import { FlexHarnessStoreConflictError, FlexHarnessValidationError } from '../ts/errors.js';
|
|
3
8
|
import type {
|
|
4
9
|
IFlexAgentEventStoreProvider,
|
|
@@ -12,6 +17,8 @@ import type {
|
|
|
12
17
|
IFlexProjectionChange,
|
|
13
18
|
IFlexProjectionStore,
|
|
14
19
|
IFlexResultStore,
|
|
20
|
+
IFlexScopeRecoverySnapshot,
|
|
21
|
+
IFlexScopeRecoveryStore,
|
|
15
22
|
IFlexScopeSnapshot,
|
|
16
23
|
IFlexScopeStore,
|
|
17
24
|
IFlexToolJobStoreProvider,
|
|
@@ -122,6 +129,47 @@ class NoSqlScopeStore implements IFlexScopeStore {
|
|
|
122
129
|
}
|
|
123
130
|
}
|
|
124
131
|
|
|
132
|
+
class NoSqlScopeRecoveryStore implements IFlexScopeRecoveryStore {
|
|
133
|
+
constructor(private readonly records: FlexRecordStore) {}
|
|
134
|
+
|
|
135
|
+
public deactivateWriteProvenance(storageKey: string): void {
|
|
136
|
+
this.records.deactivateWriteProvenance(storageKey);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
public loadWriteProvenance(storageKey: string): Promise<boolean> {
|
|
140
|
+
validateIdentifier(storageKey, 'storageKey');
|
|
141
|
+
return this.records.loadWriteProvenance(storageKey);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
public qualifyWriteProvenance(storageKey: string): Promise<void> {
|
|
145
|
+
validateIdentifier(storageKey, 'storageKey');
|
|
146
|
+
return this.records.qualifyWriteProvenance(storageKey);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
public load(storageKey: string): Promise<IFlexScopeRecoverySnapshot | undefined> {
|
|
150
|
+
validateIdentifier(storageKey, 'storageKey');
|
|
151
|
+
return this.records.read({ domain: 'scopeRecovery', storageKey }, (value) => {
|
|
152
|
+
assertFlexScopeRecoverySnapshot(value);
|
|
153
|
+
return value;
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
public async save(
|
|
158
|
+
storageKey: string,
|
|
159
|
+
snapshot: IFlexScopeRecoverySnapshot,
|
|
160
|
+
expectedRevision: number,
|
|
161
|
+
): Promise<void> {
|
|
162
|
+
validateIdentifier(storageKey, 'storageKey');
|
|
163
|
+
const validated = validateFlexScopeRecoverySave(snapshot, expectedRevision);
|
|
164
|
+
const conflict = (actualRevision: number) =>
|
|
165
|
+
new FlexHarnessStoreConflictError(storageKey, expectedRevision, actualRevision);
|
|
166
|
+
await this.records.replace(
|
|
167
|
+
{ domain: 'scopeRecovery', storageKey },
|
|
168
|
+
revisionReplacement(validated, expectedRevision, conflict),
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
125
173
|
/** A projection is a log record: a save writes its change, checked in form. */
|
|
126
174
|
class NoSqlProjectionStore implements IFlexProjectionStore {
|
|
127
175
|
constructor(private readonly records: FlexRecordStore) {}
|
|
@@ -522,7 +570,7 @@ function validateResult(value: unknown): TJsonValue {
|
|
|
522
570
|
* `flexharness_changes`; `flexharness_namespaces` holds each storage namespace's fence.
|
|
523
571
|
*/
|
|
524
572
|
export class NoSqlFlexHarnessStores implements IFlexHarnessStores {
|
|
525
|
-
public readonly scopes:
|
|
573
|
+
public readonly scopes: RecoverableFlexScopeStore;
|
|
526
574
|
public readonly projections: IFlexProjectionStore;
|
|
527
575
|
public readonly permissions: IFlexPermissionStore;
|
|
528
576
|
public readonly projectManagement: IFlexProjectManagementStore;
|
|
@@ -536,7 +584,9 @@ export class NoSqlFlexHarnessStores implements IFlexHarnessStores {
|
|
|
536
584
|
throw new FlexHarnessValidationError('NoSqlFlexHarnessStores requires a SmartdataDb.');
|
|
537
585
|
}
|
|
538
586
|
this.records = new FlexRecordStore(options);
|
|
539
|
-
this.scopes = new
|
|
587
|
+
this.scopes = new RecoverableFlexScopeStore(
|
|
588
|
+
new NoSqlScopeStore(this.records), new NoSqlScopeRecoveryStore(this.records),
|
|
589
|
+
);
|
|
540
590
|
this.projections = new NoSqlProjectionStore(this.records);
|
|
541
591
|
this.permissions = new NoSqlPermissionStore(this.records);
|
|
542
592
|
this.projectManagement = new NoSqlProjectManagementStore(this.records);
|
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
|
|
9
9
|
export type TFlexRecordDomain =
|
|
10
10
|
| 'scope'
|
|
11
|
+
| 'scopeRecovery'
|
|
11
12
|
| 'projection'
|
|
12
13
|
| 'permission'
|
|
13
14
|
| 'projectManagement'
|
|
@@ -184,6 +185,8 @@ export class FlexNamespaceDocument extends plugins.SmartDataDbDoc<FlexNamespaceD
|
|
|
184
185
|
public namespaceKey!: string;
|
|
185
186
|
public fenced!: boolean;
|
|
186
187
|
public creationCount!: number;
|
|
188
|
+
/** Known session-local writes, aligned only by explicit complete qualification. */
|
|
189
|
+
public recoveryWriteCount?: number;
|
|
187
190
|
public purgeToken?: string | null;
|
|
188
191
|
public storageNamespaceKey?: string;
|
|
189
192
|
}
|
|
@@ -282,8 +285,8 @@ export function bindFlexRecordModels(db: plugins.SmartdataDb): IFlexRecordModels
|
|
|
282
285
|
});
|
|
283
286
|
plugins.defineCollectionModel(BoundNamespace, db, {
|
|
284
287
|
collectionName: FLEX_NAMESPACE_COLLECTION,
|
|
285
|
-
persistedFields: ['namespaceKey', 'fenced', 'creationCount', 'purgeToken', 'storageNamespaceKey'],
|
|
286
|
-
numericFields: ['creationCount'],
|
|
288
|
+
persistedFields: ['namespaceKey', 'fenced', 'creationCount', 'recoveryWriteCount', 'purgeToken', 'storageNamespaceKey'],
|
|
289
|
+
numericFields: ['creationCount', 'recoveryWriteCount'],
|
|
287
290
|
identityFields: ['namespaceKey'],
|
|
288
291
|
indexes: [
|
|
289
292
|
{ name: 'flexharness_namespace_key', key: { namespaceKey: 1 }, options: { unique: true } },
|
|
@@ -379,6 +382,7 @@ export class FlexRecordStore {
|
|
|
379
382
|
private readonly changes: TChangePageModel;
|
|
380
383
|
private readonly namespaces: TNamespaceModel;
|
|
381
384
|
private preparation?: Promise<void>;
|
|
385
|
+
private readonly recoveryNamespaces = new Set<string>();
|
|
382
386
|
|
|
383
387
|
constructor(options: IFlexRecordStoreOptions) {
|
|
384
388
|
const timeoutMs = options.operationTimeoutMs ?? defaultOperationTimeoutMs;
|
|
@@ -404,6 +408,52 @@ export class FlexRecordStore {
|
|
|
404
408
|
return this.preparation;
|
|
405
409
|
}
|
|
406
410
|
|
|
411
|
+
private assertWriteCounts(namespace: FlexNamespaceDocument): void {
|
|
412
|
+
for (const value of [namespace.creationCount,
|
|
413
|
+
...(namespace.recoveryWriteCount === undefined ? [] : [namespace.recoveryWriteCount])]) {
|
|
414
|
+
if (!Number.isSafeInteger(value) || value < 0) {
|
|
415
|
+
throw new FlexHarnessStoreFormatError('Namespace write provenance must contain non-negative safe integers.');
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
if ((namespace.recoveryWriteCount ?? 0) > namespace.creationCount) {
|
|
419
|
+
throw new FlexHarnessStoreFormatError('Namespace recovery write count exceeds its write generation.');
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/** Activates this owner's known writes only while no unqualified all-domain write is present. */
|
|
424
|
+
public deactivateWriteProvenance(storageKey: string): void {
|
|
425
|
+
this.recoveryNamespaces.delete(digest(storageKey));
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
public async loadWriteProvenance(storageKey: string): Promise<boolean> {
|
|
429
|
+
await this.prepare();
|
|
430
|
+
const namespaceKey = digest(storageKey);
|
|
431
|
+
const namespace = await this.namespaces.getInstance({ namespaceKey }, { timeoutMS: this.timeoutMs });
|
|
432
|
+
if (namespace) this.assertWriteCounts(namespace);
|
|
433
|
+
const qualified = !namespace || (!namespace.fenced && !namespace.purgeToken
|
|
434
|
+
&& namespace.creationCount === (namespace.recoveryWriteCount ?? 0));
|
|
435
|
+
if (qualified) this.recoveryNamespaces.add(namespaceKey);
|
|
436
|
+
else this.recoveryNamespaces.delete(namespaceKey);
|
|
437
|
+
return qualified;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/** Only the complete explicit qualification may reconcile unknown historical writes. */
|
|
441
|
+
public async qualifyWriteProvenance(storageKey: string): Promise<void> {
|
|
442
|
+
await this.prepare();
|
|
443
|
+
const namespaceKey = digest(storageKey);
|
|
444
|
+
await this.transaction(`${FLEX_NAMESPACE_COLLECTION}/${namespaceKey}`, 'write', async (transaction) => {
|
|
445
|
+
const namespace = await this.namespaces.getInstance({ namespaceKey }, { session: transaction });
|
|
446
|
+
if (!namespace || namespace.fenced || namespace.purgeToken) {
|
|
447
|
+
throw new FlexHarnessStoreFormatError('Namespace write provenance cannot be qualified without a current namespace.');
|
|
448
|
+
}
|
|
449
|
+
this.assertWriteCounts(namespace);
|
|
450
|
+
await this.namespaces.atomicUpdate({ namespaceKey }, {
|
|
451
|
+
$set: { recoveryWriteCount: namespace.creationCount },
|
|
452
|
+
}, { session: transaction });
|
|
453
|
+
});
|
|
454
|
+
this.recoveryNamespaces.add(namespaceKey);
|
|
455
|
+
}
|
|
456
|
+
|
|
407
457
|
private async prepareCollections(): Promise<void> {
|
|
408
458
|
const capabilities = this.db.capabilities;
|
|
409
459
|
if (!capabilities) {
|
|
@@ -632,9 +682,14 @@ export class FlexRecordStore {
|
|
|
632
682
|
uniqueField: identity, limit: deletionBatchSize, timeoutMS: this.timeoutMs });
|
|
633
683
|
if (page.documents.length === 0) break;
|
|
634
684
|
const active = await this.transaction(path, 'delete', async (transaction) => {
|
|
635
|
-
const
|
|
636
|
-
{
|
|
637
|
-
if (
|
|
685
|
+
const marker = await this.namespaces.getInstance({ namespaceKey: markerKey, purgeToken: token },
|
|
686
|
+
{ session: transaction });
|
|
687
|
+
if (!marker) return false;
|
|
688
|
+
await this.touchNamespace(namespaceKey, transaction);
|
|
689
|
+
if (markerKey !== namespaceKey) {
|
|
690
|
+
await this.namespaces.atomicUpdate({ namespaceKey: markerKey, purgeToken: token },
|
|
691
|
+
{ $inc: { creationCount: 1 } }, { session: transaction });
|
|
692
|
+
}
|
|
638
693
|
const identifiers = page.documents.map((document) => this.documentIdentity(document, identity));
|
|
639
694
|
await model.atomicDeleteMany<TDeletionDocument>({ ...filter, [identity]: { $in: identifiers } },
|
|
640
695
|
{ session: transaction });
|
|
@@ -687,9 +742,24 @@ export class FlexRecordStore {
|
|
|
687
742
|
return markers.some((marker) => marker.fenced || Boolean(marker.purgeToken));
|
|
688
743
|
}
|
|
689
744
|
|
|
690
|
-
private touchNamespace(namespaceKey: string, session: plugins.SmartdataSession): Promise<unknown> {
|
|
691
|
-
|
|
692
|
-
|
|
745
|
+
private async touchNamespace(namespaceKey: string, session: plugins.SmartdataSession): Promise<unknown> {
|
|
746
|
+
const recoveryActive = this.recoveryNamespaces.has(namespaceKey);
|
|
747
|
+
if (recoveryActive) {
|
|
748
|
+
const namespace = await this.namespaces.getInstance({ namespaceKey }, { session });
|
|
749
|
+
if (namespace) {
|
|
750
|
+
this.assertWriteCounts(namespace);
|
|
751
|
+
if (namespace.creationCount !== (namespace.recoveryWriteCount ?? 0)) {
|
|
752
|
+
throw new FlexHarnessStoreFormatError('Namespace write provenance requires explicit qualification.');
|
|
753
|
+
}
|
|
754
|
+
if (namespace.creationCount === Number.MAX_SAFE_INTEGER) {
|
|
755
|
+
throw new FlexHarnessStoreFormatError('Namespace write provenance exhausted its safe integer generation.');
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
return this.namespaces.atomicUpdate({ namespaceKey }, {
|
|
760
|
+
$inc: { creationCount: 1, ...(recoveryActive ? { recoveryWriteCount: 1 } : {}) },
|
|
761
|
+
$setOnInsert: { fenced: false },
|
|
762
|
+
}, { upsert: true, session });
|
|
693
763
|
}
|
|
694
764
|
|
|
695
765
|
private writeTransaction<TResult>(resolved: IResolvedAddress,
|