@modelprofile.com/flexharness 18.1.0 → 18.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/dist_ts/00_commitinfo_data.js +1 -1
  2. package/dist_ts/classes.flexharness.cachewarm.js +3 -3
  3. package/dist_ts/classes.flexharness.callbacks.d.ts +37 -0
  4. package/dist_ts/classes.flexharness.callbacks.js +129 -2
  5. package/dist_ts/classes.flexharness.compaction.js +2 -2
  6. package/dist_ts/classes.flexharness.configurations.js +4 -4
  7. package/dist_ts/classes.flexharness.core.d.ts +39 -4
  8. package/dist_ts/classes.flexharness.core.js +467 -134
  9. package/dist_ts/classes.flexharness.d.ts +4 -0
  10. package/dist_ts/classes.flexharness.deletion.d.ts +13 -0
  11. package/dist_ts/classes.flexharness.deletion.js +122 -40
  12. package/dist_ts/classes.flexharness.imports.js +26 -21
  13. package/dist_ts/classes.flexharness.js +9 -1
  14. package/dist_ts/classes.flexharness.lifecycle.js +9 -3
  15. package/dist_ts/classes.flexharness.projects.js +2 -2
  16. package/dist_ts/classes.flexharness.prompts.js +14 -8
  17. package/dist_ts/classes.flexharness.reversions.js +3 -3
  18. package/dist_ts/classes.flexharness.runs.js +18 -2
  19. package/dist_ts/classes.flexharness.sessions.d.ts +1 -1
  20. package/dist_ts/classes.flexharness.sessions.js +18 -18
  21. package/dist_ts/classes.flexharness.slashcommands.js +3 -3
  22. package/dist_ts/classes.flexharness.subagents.js +37 -16
  23. package/dist_ts/classes.flexharness.tooldispatch.js +3 -3
  24. package/dist_ts/classes.scoperecovery.d.ts +36 -0
  25. package/dist_ts/classes.scoperecovery.js +228 -0
  26. package/dist_ts/classes.stores.d.ts +14 -1
  27. package/dist_ts/classes.stores.js +150 -17
  28. package/dist_ts/index.d.ts +1 -0
  29. package/dist_ts/index.js +2 -1
  30. package/dist_ts/interfaces.d.ts +84 -0
  31. package/dist_ts/interfaces.flexharness.d.ts +36 -0
  32. package/dist_ts/interfaces.js +1 -1
  33. package/dist_ts/utils.harnessoptions.d.ts +2 -0
  34. package/dist_ts/utils.harnessoptions.js +16 -11
  35. package/dist_ts_stores_nosqldb/classes.nosqldbstores.d.ts +3 -2
  36. package/dist_ts_stores_nosqldb/classes.nosqldbstores.js +33 -2
  37. package/dist_ts_stores_nosqldb/classes.recordstore.d.ts +10 -1
  38. package/dist_ts_stores_nosqldb/classes.recordstore.js +75 -7
  39. package/package.json +3 -3
  40. package/readme.md +42 -4
  41. package/ts/00_commitinfo_data.ts +1 -1
  42. package/ts/classes.flexharness.cachewarm.ts +2 -2
  43. package/ts/classes.flexharness.callbacks.ts +142 -1
  44. package/ts/classes.flexharness.compaction.ts +1 -1
  45. package/ts/classes.flexharness.configurations.ts +3 -3
  46. package/ts/classes.flexharness.core.ts +463 -133
  47. package/ts/classes.flexharness.deletion.ts +161 -52
  48. package/ts/classes.flexharness.imports.ts +26 -21
  49. package/ts/classes.flexharness.lifecycle.ts +8 -2
  50. package/ts/classes.flexharness.projects.ts +1 -1
  51. package/ts/classes.flexharness.prompts.ts +11 -7
  52. package/ts/classes.flexharness.reversions.ts +2 -2
  53. package/ts/classes.flexharness.runs.ts +16 -1
  54. package/ts/classes.flexharness.sessions.ts +17 -17
  55. package/ts/classes.flexharness.slashcommands.ts +2 -2
  56. package/ts/classes.flexharness.subagents.ts +40 -15
  57. package/ts/classes.flexharness.tooldispatch.ts +2 -2
  58. package/ts/classes.flexharness.ts +13 -0
  59. package/ts/classes.scoperecovery.ts +290 -0
  60. package/ts/classes.stores.ts +150 -16
  61. package/ts/index.ts +2 -0
  62. package/ts/interfaces.flexharness.ts +38 -0
  63. package/ts/interfaces.ts +89 -0
  64. package/ts/readme.md +42 -4
  65. package/ts/utils.harnessoptions.ts +18 -12
  66. package/ts_stores_nosqldb/classes.nosqldbstores.ts +52 -2
  67. package/ts_stores_nosqldb/classes.recordstore.ts +78 -8
  68. package/ts_stores_nosqldb/readme.md +10 -0
@@ -1,4 +1,5 @@
1
1
  import * as plugins from './plugins.js';
2
+ import { RecoverableFlexScopeStore, validateFlexScopeRecoverySave } from './classes.scoperecovery.js';
2
3
  import {
3
4
  FlexHarnessStoreCommitUncertainError,
4
5
  FlexHarnessStoreConflictError,
@@ -17,6 +18,8 @@ import type {
17
18
  IFlexProjectionChange,
18
19
  IFlexProjectionStore,
19
20
  IFlexResultStore,
21
+ IFlexScopeRecoverySnapshot,
22
+ IFlexScopeRecoveryStore,
20
23
  IFlexScopeSnapshot,
21
24
  IFlexScopeStore,
22
25
  IFlexToolJobStoreProvider,
@@ -61,7 +64,56 @@ import {
61
64
  validateSnapshotSave,
62
65
  } from './utils.storerules.js';
63
66
 
67
+ interface IInMemoryWriteGeneration {
68
+ generation: number;
69
+ recoveryGeneration: number;
70
+ }
71
+
72
+ class InMemoryWriteProvenance {
73
+ private readonly generations = new Map<string, IInMemoryWriteGeneration>();
74
+ private readonly active = new Set<string>();
75
+
76
+ public deactivate(storageKey: string): void {
77
+ validateIdentifier(storageKey, 'storageKey');
78
+ this.active.delete(storageKey);
79
+ }
80
+
81
+ public load(storageKey: string): boolean {
82
+ validateIdentifier(storageKey, 'storageKey');
83
+ const generation = this.generations.get(storageKey);
84
+ const current = !generation || generation.generation === generation.recoveryGeneration;
85
+ if (current) this.active.add(storageKey);
86
+ else this.active.delete(storageKey);
87
+ return current;
88
+ }
89
+
90
+ public qualify(storageKey: string): void {
91
+ validateIdentifier(storageKey, 'storageKey');
92
+ const generation = this.generations.get(storageKey) ?? { generation: 0, recoveryGeneration: 0 };
93
+ generation.recoveryGeneration = generation.generation;
94
+ this.generations.set(storageKey, generation);
95
+ this.active.add(storageKey);
96
+ }
97
+
98
+ public touch(storageKey: string): void {
99
+ validateIdentifier(storageKey, 'storageKey');
100
+ const generation = this.generations.get(storageKey) ?? { generation: 0, recoveryGeneration: 0 };
101
+ const active = this.active.has(storageKey);
102
+ if (active && generation.generation !== generation.recoveryGeneration) {
103
+ throw new FlexHarnessStoreFormatError('Namespace write provenance requires explicit qualification.');
104
+ }
105
+ if (generation.generation === Number.MAX_SAFE_INTEGER) {
106
+ throw new FlexHarnessStoreFormatError('Namespace write provenance exhausted its safe integer generation.');
107
+ }
108
+ generation.generation++;
109
+ if (active) generation.recoveryGeneration++;
110
+ this.generations.set(storageKey, generation);
111
+ }
112
+ }
113
+
64
114
  class InMemoryScopeStore implements IFlexScopeStore {
115
+ constructor(private readonly writes: InMemoryWriteProvenance) {}
116
+
65
117
  private readonly snapshots = new Map<string, IFlexScopeSnapshot>();
66
118
 
67
119
  public async load(storageKey: string): Promise<IFlexScopeSnapshot | undefined> {
@@ -81,11 +133,53 @@ class InMemoryScopeStore implements IFlexScopeStore {
81
133
  if (actualRevision !== expectedRevision) {
82
134
  throw new FlexHarnessStoreConflictError(storageKey, expectedRevision, actualRevision);
83
135
  }
136
+ this.writes.touch(storageKey);
137
+ this.snapshots.set(storageKey, validated);
138
+ }
139
+ }
140
+
141
+ class InMemoryScopeRecoveryStore implements IFlexScopeRecoveryStore {
142
+ constructor(private readonly writes: InMemoryWriteProvenance) {}
143
+
144
+ public deactivateWriteProvenance(storageKey: string): void {
145
+ this.writes.deactivate(storageKey);
146
+ }
147
+
148
+ public async loadWriteProvenance(storageKey: string): Promise<boolean> {
149
+ return this.writes.load(storageKey);
150
+ }
151
+
152
+ public async qualifyWriteProvenance(storageKey: string): Promise<void> {
153
+ this.writes.qualify(storageKey);
154
+ }
155
+
156
+ private readonly snapshots = new Map<string, IFlexScopeRecoverySnapshot>();
157
+
158
+ public async load(storageKey: string): Promise<IFlexScopeRecoverySnapshot | undefined> {
159
+ validateIdentifier(storageKey, 'storageKey');
160
+ const snapshot = this.snapshots.get(storageKey);
161
+ return snapshot ? cloneSerializable(snapshot) : undefined;
162
+ }
163
+
164
+ public async save(
165
+ storageKey: string,
166
+ snapshot: IFlexScopeRecoverySnapshot,
167
+ expectedRevision: number,
168
+ ): Promise<void> {
169
+ validateIdentifier(storageKey, 'storageKey');
170
+ const validated = validateFlexScopeRecoverySave(snapshot, expectedRevision);
171
+ const actualRevision = this.snapshots.get(storageKey)?.revision ?? 0;
172
+ if (actualRevision !== expectedRevision) {
173
+ throw new FlexHarnessStoreConflictError(storageKey, expectedRevision, actualRevision);
174
+ }
175
+ this.writes.touch(storageKey);
84
176
  this.snapshots.set(storageKey, validated);
85
177
  }
86
178
  }
87
179
 
88
180
  class InMemoryProjectionStore implements IFlexProjectionStore {
181
+ constructor(private readonly writes: InMemoryWriteProvenance) {}
182
+
89
183
  private readonly snapshots = new Map<string, TFlexProjectionSnapshot>();
90
184
 
91
185
  public async load(
@@ -110,15 +204,21 @@ class InMemoryProjectionStore implements IFlexProjectionStore {
110
204
  if (actualRevision !== expectedRevision) {
111
205
  throw new FlexHarnessStoreConflictError(key, expectedRevision, actualRevision);
112
206
  }
113
- this.snapshots.set(key, createChangedProjectionSnapshot(current, validated, sessionId));
207
+ const next = createChangedProjectionSnapshot(current, validated, sessionId);
208
+ this.writes.touch(storageKey);
209
+ this.snapshots.set(key, next);
114
210
  }
115
211
 
116
212
  public async deleteSession(storageKey: string, sessionId: string): Promise<void> {
117
- this.snapshots.delete(providerKey(storageKey, sessionId));
213
+ const key = providerKey(storageKey, sessionId);
214
+ this.writes.touch(storageKey);
215
+ this.snapshots.delete(key);
118
216
  }
119
217
  }
120
218
 
121
219
  class InMemoryPermissionStore implements IFlexPermissionStore {
220
+ constructor(private readonly writes: InMemoryWriteProvenance) {}
221
+
122
222
  private readonly snapshots = new Map<string, IFlexPermissionSnapshot>();
123
223
 
124
224
  public async load(
@@ -141,15 +241,20 @@ class InMemoryPermissionStore implements IFlexPermissionStore {
141
241
  if (actualRevision !== expectedRevision) {
142
242
  throw new FlexHarnessStoreConflictError(key, expectedRevision, actualRevision);
143
243
  }
244
+ this.writes.touch(storageKey);
144
245
  this.snapshots.set(key, validated);
145
246
  }
146
247
 
147
248
  public async deleteSession(storageKey: string, sessionId: string): Promise<void> {
148
- this.snapshots.delete(providerKey(storageKey, sessionId));
249
+ const key = providerKey(storageKey, sessionId);
250
+ this.writes.touch(storageKey);
251
+ this.snapshots.delete(key);
149
252
  }
150
253
  }
151
254
 
152
255
  export class InMemoryFlexProjectManagementStore implements IFlexProjectManagementStore {
256
+ constructor(private readonly writes?: InMemoryWriteProvenance) {}
257
+
153
258
  private readonly snapshots = new Map<string, TFlexProjectManagementRecord>();
154
259
 
155
260
  public async load(
@@ -175,6 +280,7 @@ export class InMemoryFlexProjectManagementStore implements IFlexProjectManagemen
175
280
  assertFlexProjectManagementSnapshot,
176
281
  );
177
282
  assertProjectManagementSaveAdmitted(key, this.snapshots.get(key), validated, expectedRevision);
283
+ this.writes?.touch(storageKey);
178
284
  this.snapshots.set(key, validated);
179
285
  }
180
286
 
@@ -193,11 +299,13 @@ export class InMemoryFlexProjectManagementStore implements IFlexProjectManagemen
193
299
  if (!isProjectManagementTombstoneAdmitted(key, this.snapshots.get(key), validated, expectedRevision)) {
194
300
  return;
195
301
  }
302
+ this.writes?.touch(storageKey);
196
303
  this.snapshots.set(key, validated);
197
304
  }
198
305
 
199
306
  public async purgeNamespace(storageKey: string): Promise<void> {
200
307
  validateIdentifier(storageKey, 'storageKey');
308
+ this.writes?.touch(storageKey);
201
309
  for (const key of this.snapshots.keys()) {
202
310
  const [recordStorageKey] = JSON.parse(key) as [string, string];
203
311
  if (recordStorageKey === storageKey) this.snapshots.delete(key);
@@ -209,7 +317,11 @@ class InMemoryAgentEventStore implements plugins.IAgentEventStore {
209
317
  private snapshot: plugins.IAgentEventSnapshot | undefined;
210
318
  private readonly archives = new Map<string, plugins.IAgentEventArchive>();
211
319
 
212
- constructor(private readonly sessionId: string) {}
320
+ constructor(
321
+ private readonly sessionId: string,
322
+ private readonly storageKey: string,
323
+ private readonly writes: InMemoryWriteProvenance,
324
+ ) {}
213
325
 
214
326
  private assertSession(sessionId: string): void {
215
327
  if (sessionId !== this.sessionId) {
@@ -240,6 +352,7 @@ class InMemoryAgentEventStore implements plugins.IAgentEventStore {
240
352
  createChangedAgentEventSnapshot(sessionId, this.snapshot, normalized),
241
353
  sessionId,
242
354
  );
355
+ this.writes.touch(this.storageKey);
243
356
  this.snapshot = snapshot;
244
357
  return snapshot.revision;
245
358
  }
@@ -253,6 +366,7 @@ class InMemoryAgentEventStore implements plugins.IAgentEventStore {
253
366
  assertSameArchiveEvents(existing, validated);
254
367
  return;
255
368
  }
369
+ this.writes.touch(this.storageKey);
256
370
  this.archives.set(archive.archiveId, cloneSerializable(validated));
257
371
  }
258
372
 
@@ -267,19 +381,22 @@ class InMemoryAgentEventStore implements plugins.IAgentEventStore {
267
381
  }
268
382
 
269
383
  public clear(): void {
384
+ this.writes.touch(this.storageKey);
270
385
  this.snapshot = undefined;
271
386
  this.archives.clear();
272
387
  }
273
388
  }
274
389
 
275
390
  class InMemoryAgentEventStoreProvider implements IFlexAgentEventStoreProvider {
391
+ constructor(private readonly writes: InMemoryWriteProvenance) {}
392
+
276
393
  private readonly stores = new Map<string, InMemoryAgentEventStore>();
277
394
 
278
395
  public getStore(storageKey: string, sessionId: string): InMemoryAgentEventStore {
279
396
  const key = providerKey(storageKey, sessionId);
280
397
  let store = this.stores.get(key);
281
398
  if (!store) {
282
- store = new InMemoryAgentEventStore(sessionId);
399
+ store = new InMemoryAgentEventStore(sessionId, storageKey, this.writes);
283
400
  this.stores.set(key, store);
284
401
  }
285
402
  return store;
@@ -295,6 +412,8 @@ class InMemoryAgentEventStoreProvider implements IFlexAgentEventStoreProvider {
295
412
  }
296
413
 
297
414
  class InMemoryToolJobStore implements plugins.IToolJobStore {
415
+ constructor(private readonly writes: InMemoryWriteProvenance, private readonly storageKey: string) {}
416
+
298
417
  private snapshot: plugins.IToolJobSnapshot | undefined;
299
418
 
300
419
  public async load(abortSignal?: AbortSignal): Promise<plugins.IToolJobSnapshot | undefined> {
@@ -314,23 +433,27 @@ class InMemoryToolJobStore implements plugins.IToolJobStore {
314
433
  if (actualRevision !== expectedRevision) {
315
434
  throw new plugins.ToolJobStoreConflictError(expectedRevision, actualRevision);
316
435
  }
436
+ this.writes.touch(this.storageKey);
317
437
  this.snapshot = cloneSerializable(snapshot);
318
438
  return snapshot.revision;
319
439
  }
320
440
 
321
441
  public clear(): void {
442
+ this.writes.touch(this.storageKey);
322
443
  this.snapshot = undefined;
323
444
  }
324
445
  }
325
446
 
326
447
  class InMemoryToolJobStoreProvider implements IFlexToolJobStoreProvider {
448
+ constructor(private readonly writes: InMemoryWriteProvenance) {}
449
+
327
450
  private readonly stores = new Map<string, InMemoryToolJobStore>();
328
451
 
329
452
  public getStore(storageKey: string, sessionId: string): InMemoryToolJobStore {
330
453
  const key = providerKey(storageKey, sessionId);
331
454
  let store = this.stores.get(key);
332
455
  if (!store) {
333
- store = new InMemoryToolJobStore();
456
+ store = new InMemoryToolJobStore(this.writes, storageKey);
334
457
  this.stores.set(key, store);
335
458
  }
336
459
  return store;
@@ -346,6 +469,8 @@ class InMemoryToolJobStoreProvider implements IFlexToolJobStoreProvider {
346
469
  }
347
470
 
348
471
  class InMemoryResultStore implements IFlexResultStore {
472
+ constructor(private readonly writes: InMemoryWriteProvenance) {}
473
+
349
474
  private readonly sessions = new Map<string, Map<string, TJsonValue>>();
350
475
 
351
476
  public async save(storageKey: string, sessionId: string, handle: string, value: TJsonValue): Promise<void> {
@@ -353,7 +478,9 @@ class InMemoryResultStore implements IFlexResultStore {
353
478
  const key = providerKey(storageKey, sessionId);
354
479
  const results = this.sessions.get(key) ?? new Map<string, TJsonValue>();
355
480
  if (results.has(handle)) throw new FlexHarnessStoreConflictError(`${key}:${handle}`, 0, 1);
356
- results.set(handle, cloneSerializable(value));
481
+ const validated = cloneSerializable(value);
482
+ this.writes.touch(storageKey);
483
+ results.set(handle, validated);
357
484
  this.sessions.set(key, results);
358
485
  }
359
486
 
@@ -367,22 +494,29 @@ class InMemoryResultStore implements IFlexResultStore {
367
494
  }
368
495
 
369
496
  public async delete(storageKey: string, sessionId: string, handle: string): Promise<void> {
370
- this.sessions.get(providerKey(storageKey, sessionId))?.delete(handle);
497
+ const key = providerKey(storageKey, sessionId);
498
+ this.writes.touch(storageKey);
499
+ this.sessions.get(key)?.delete(handle);
371
500
  }
372
501
 
373
502
  public async deleteSession(storageKey: string, sessionId: string): Promise<void> {
374
- this.sessions.delete(providerKey(storageKey, sessionId));
503
+ const key = providerKey(storageKey, sessionId);
504
+ this.writes.touch(storageKey);
505
+ this.sessions.delete(key);
375
506
  }
376
507
  }
377
508
 
378
509
  export class InMemoryFlexHarnessStores implements IFlexHarnessStores {
379
- public readonly scopes: IFlexScopeStore = new InMemoryScopeStore();
380
- public readonly projections: IFlexProjectionStore = new InMemoryProjectionStore();
381
- public readonly permissions: IFlexPermissionStore = new InMemoryPermissionStore();
382
- public readonly agentEvents: IFlexAgentEventStoreProvider = new InMemoryAgentEventStoreProvider();
383
- public readonly jobs: IFlexToolJobStoreProvider = new InMemoryToolJobStoreProvider();
384
- public readonly projectManagement = new InMemoryFlexProjectManagementStore();
385
- public readonly results: IFlexResultStore = new InMemoryResultStore();
510
+ private readonly writes = new InMemoryWriteProvenance();
511
+ public readonly scopes = new RecoverableFlexScopeStore(
512
+ new InMemoryScopeStore(this.writes), new InMemoryScopeRecoveryStore(this.writes),
513
+ );
514
+ public readonly projections: IFlexProjectionStore = new InMemoryProjectionStore(this.writes);
515
+ public readonly permissions: IFlexPermissionStore = new InMemoryPermissionStore(this.writes);
516
+ public readonly agentEvents: IFlexAgentEventStoreProvider = new InMemoryAgentEventStoreProvider(this.writes);
517
+ public readonly jobs: IFlexToolJobStoreProvider = new InMemoryToolJobStoreProvider(this.writes);
518
+ public readonly projectManagement = new InMemoryFlexProjectManagementStore(this.writes);
519
+ public readonly results: IFlexResultStore = new InMemoryResultStore(this.writes);
386
520
  }
387
521
 
388
522
  const jsonDomains = [
package/ts/index.ts CHANGED
@@ -40,3 +40,5 @@ export {
40
40
  isValidSlashCommandName,
41
41
  parseSlashCommand,
42
42
  } from './utils.slashcommands.js';
43
+
44
+ export * from './classes.scoperecovery.js';
@@ -47,6 +47,32 @@ import type { INormalizedSessionImportOptions } from './utils.sessionimport.js';
47
47
  import type { ISessionMetricsOperation } from './utils.sessionmetrics.js';
48
48
 
49
49
  export type TCanonicalOutcome = 'accepted' | 'rejected' | 'interrupted';
50
+ /** When a projection save started and completed, on the checkpoint clock. */
51
+ export type TProjectionSaved = (startedAt: number, completedAt: number) => void;
52
+
53
+ /**
54
+ * A run's projection checkpoints: the parts of its committed steps, saved on its streaming
55
+ * assistant message while the run goes on, so that a process stopping mid-run leaves them visible.
56
+ * A checkpoint starts no earlier than the larger of 10 s and 20 times the duration of the run's
57
+ * last save of that message after that save completed; one is outstanding at most.
58
+ */
59
+ export interface IRunCheckpoint {
60
+ /** The parts of committed steps, by id: what a checkpoint saves, as the parts stand then. */
61
+ committedPartIds: Set<string>;
62
+ /** A step committed since the last checkpoint read the run's parts. */
63
+ dirty: boolean;
64
+ /** A checkpoint is queued in the projection FIFO or saving. */
65
+ outstanding: boolean;
66
+ /** The deadline timer on the checkpoint clock, while one is armed. */
67
+ timer?: unknown;
68
+ /** When the run's last save of its current assistant message completed, on the checkpoint clock. */
69
+ savedAt?: number;
70
+ /** How long that save took, in milliseconds of the checkpoint clock. */
71
+ saveMs: number;
72
+ /** No checkpoint starts any more: the run reached its terminal, closed or its namespace stopped. */
73
+ closed: boolean;
74
+ }
75
+
50
76
  export type TRunPhase =
51
77
  | 'admitting'
52
78
  | 'scheduled'
@@ -82,6 +108,7 @@ export interface ICompactorInvocation<TScope> extends IFlexAgentContextInvocatio
82
108
 
83
109
  export interface IStoredSessionState {
84
110
  storageKey: string;
111
+ recovering?: boolean;
85
112
  compactorContext?: IFlexAgentContextInvocation<unknown>;
86
113
  session: IFlexSession;
87
114
  messages: IFlexMessage[];
@@ -214,6 +241,13 @@ export interface IStorageState {
214
241
  scopeIdHint: string;
215
242
  scopeContext: { scopeId: string; scope: unknown };
216
243
  revision: number;
244
+ /** Complete live metadata; independent of the hydrated runtime subset below. */
245
+ catalogue: Map<string, IFlexSession>;
246
+ sessionLocal: boolean;
247
+ qualifyingLegacy: boolean;
248
+ recoverySessionIds: Set<string>;
249
+ sessionLoads: Map<string, Promise<IStoredSessionState>>;
250
+ sharedRecovery?: Promise<void>;
217
251
  sessions: Map<string, IStoredSessionState>;
218
252
  retainedSessionCleanups: Map<string, IRetainedSessionCleanup>;
219
253
  sessionDeletions: Map<string, Promise<void>>;
@@ -296,6 +330,8 @@ export interface IActiveRun {
296
330
  stepStreamedCalls: Map<string, ReadonlySet<string>>;
297
331
  /** The step's streamed calls the SDK could not parse or repair, with its error: they never start. */
298
332
  stepInvalidCalls: Map<string, string>;
333
+ /** The run's projection checkpoints; see {@link IRunCheckpoint}. */
334
+ checkpoint: IRunCheckpoint;
299
335
  textPartClosed: boolean;
300
336
  toolPartIds: Map<string, string>;
301
337
  trustedToolExecutionErrors: Map<string, Error>;
@@ -580,6 +616,8 @@ export interface ISessionDeletionPlan {
580
616
  export interface IReservedSessionDeletion {
581
617
  deletedSessions: IFlexSession[];
582
618
  descendantRoots: string[];
619
+ /** The settlement of the deleted sessions' configuration resolutions the reservation's fence aborted. */
620
+ configurationSettlement?: Promise<void>;
583
621
  }
584
622
 
585
623
  export type TProjectTaskToolInput =
package/ts/interfaces.ts CHANGED
@@ -1688,7 +1688,88 @@ export interface IFlexProjectScratchpadResult extends IFlexProjectStateResult {
1688
1688
  scratchpad: string;
1689
1689
  }
1690
1690
 
1691
+ /** Compact recovery proof stored separately from the strict schema-1 scope snapshot. */
1692
+ export interface IFlexScopeRecoverySnapshot {
1693
+ schemaVersion: 1;
1694
+ revision: number;
1695
+ catalogueRevision: number;
1696
+ catalogueDigest: string;
1697
+ recoverySessionIds: string[];
1698
+ }
1699
+
1700
+ /**
1701
+ * Additive sidecar persistence for `RecoverableFlexScopeStore`: the recovery proof and the
1702
+ * namespace's write provenance. A scope snapshot cannot show that nothing needs recovery: a writer
1703
+ * unaware of the proof (an older release, an eager harness, another tool) may write a projection's
1704
+ * pending capture or apply, run its workspace effect and stop before it writes the scope. The
1705
+ * provenance is a durable witness of the namespace's writes that the implementation keeps so that:
1706
+ *
1707
+ * - every write in every domain of the namespace (scope, proof, projections, permissions, agent
1708
+ * events and archives, tool jobs, results, project management, every deletion and purge) changes
1709
+ * it atomically with the write, by every writer that can open the namespace, releases that
1710
+ * predate this contract included: it is a counter or header those writers already change, never
1711
+ * a field only recovery-aware writers maintain;
1712
+ * - a write of an activated instance is recorded as known atomically with the write; every other
1713
+ * write is unknown;
1714
+ * - it is read from metadata alone, never from history bodies, and never returns to an earlier
1715
+ * value while the namespace holds data.
1716
+ */
1717
+ export interface IFlexScopeRecoveryStore {
1718
+ /**
1719
+ * Synchronous and local: afterwards this instance's writes for the key are unknown. Never changes
1720
+ * durable state. Called on release, eager loading, disposal, retirement and stale evidence.
1721
+ */
1722
+ deactivateWriteProvenance(storageKey: string): void;
1723
+ /**
1724
+ * True only if every write since the last {@link qualifyWriteProvenance} is known; then activates
1725
+ * this instance for the key. Otherwise false, deactivated. Never repairs. While activated, each
1726
+ * write checks in the same atomic step that the witness still shows only known writes and fails
1727
+ * with `FlexHarnessStoreFormatError` instead of recording itself as known when it does not.
1728
+ */
1729
+ loadWriteProvenance(storageKey: string): Promise<boolean>;
1730
+ /**
1731
+ * Marks every write so far as known and activates. Called only at the end of an explicit complete
1732
+ * qualification under exclusive ownership, after the proof was published; a failure leaves the
1733
+ * namespace unqualified.
1734
+ */
1735
+ qualifyWriteProvenance(storageKey: string): Promise<void>;
1736
+ load(storageKey: string): Promise<IFlexScopeRecoverySnapshot | undefined>;
1737
+ /**
1738
+ * Durable compare-and-swap: writes only when the stored revision equals `expectedRevision`
1739
+ * (`FlexHarnessStoreConflictError` otherwise); an unknown outcome is
1740
+ * `FlexHarnessStoreCommitUncertainError`.
1741
+ */
1742
+ save(storageKey: string, snapshot: IFlexScopeRecoverySnapshot, expectedRevision: number): Promise<void>;
1743
+ }
1744
+
1745
+ /** Requires exclusive namespace ownership, like the harness itself. */
1746
+ export interface IFlexSessionLocalScopeStore {
1747
+ /** Releases local writer activation only, not a distributed namespace lock. */
1748
+ releaseOwnership(storageKey: string): void;
1749
+ /** A missing/stale proof is unqualified, never an assertion that recovery is unnecessary. */
1750
+ loadCatalogue(storageKey: string): Promise<{
1751
+ snapshot: IFlexScopeSnapshot | undefined;
1752
+ qualified: boolean;
1753
+ recoverySessionIds: string[];
1754
+ }>;
1755
+ saveCatalogue(storageKey: string, snapshot: IFlexScopeSnapshot, expectedRevision: number): Promise<void>;
1756
+ setRecoveryRequired(storageKey: string, sessionId: string, required: boolean): Promise<void>;
1757
+ /** Called only after explicit complete legacy qualification under namespace ownership. */
1758
+ qualifyCatalogue(storageKey: string, snapshot: IFlexScopeSnapshot, recoverySessionIds: string[]): Promise<void>;
1759
+ }
1760
+
1761
+ export interface IFlexPrepareScopeOptions {
1762
+ /** Explicitly permits the one-time complete legacy load/repair when proof is absent or stale. */
1763
+ qualifyLegacy?: boolean;
1764
+ }
1765
+
1766
+ export interface IFlexScopeCatalogue {
1767
+ sessions: IFlexSession[];
1768
+ recoverySessionIds: string[];
1769
+ }
1770
+
1691
1771
  export interface IFlexScopeStore {
1772
+ sessionLocal?: IFlexSessionLocalScopeStore;
1692
1773
  load(storageKey: string): Promise<IFlexScopeSnapshot | undefined>;
1693
1774
  save(
1694
1775
  storageKey: string,
@@ -2197,6 +2278,8 @@ export interface IFlexHarnessOptions<TScope> {
2197
2278
  /** Lets prompts name attachments by reference; see {@link IFlexAttachmentResolver}. */
2198
2279
  attachmentResolver?: IFlexAttachmentResolver<TScope>;
2199
2280
  stores?: IFlexHarnessStores;
2281
+ /** Opt-in: requires scopes.sessionLocal; never falls back to an implicit complete load. */
2282
+ sessionLoading?: 'eager' | 'session-local';
2200
2283
  agentSessionPolicy?: IFlexAgentSessionPolicy<TScope>;
2201
2284
  toolOutputLimits?: IFlexJsonLimits;
2202
2285
  callbackLimits?: IFlexCallbackLimits;
@@ -2222,6 +2305,12 @@ export interface IFlexHarnessOptions<TScope> {
2222
2305
  maxSubagentCallsPerRun?: number;
2223
2306
  /** Bounds and the clock of the cache keeper; see `keepCacheWarm()`. */
2224
2307
  cacheWarm?: IFlexCacheWarmOptions;
2308
+ /**
2309
+ * The clock a running turn's projection checkpoints read and set their timers on. Defaults to
2310
+ * the system clock, whose timers do not keep the process alive. Tests pass a fake one; the
2311
+ * spacing of checkpoints is fixed, not configurable.
2312
+ */
2313
+ projectionCheckpointClock?: IFlexCacheWarmClock;
2225
2314
  /**
2226
2315
  * Called once each scope load ends, with what it took. An exception it throws, or a promise it
2227
2316
  * rejects, is ignored.
package/ts/readme.md CHANGED
@@ -709,7 +709,7 @@ await harness.dispose();
709
709
 
710
710
  `getSession()` returns the session as saved after every queued write that may change it; it does not wait for queued writes of the scope's other sessions, so a slow save of one session never holds a read of another. `listSessions()` waits for every queued write.
711
711
 
712
- `listSessionIdentities(scopeId)` lists a scope's live sessions as `IFlexSessionIdentity` (`sessionId`, `sessionGenerationId`, `sessionGenerationSequence`, `parentSessionId` when set), sorted by session id. The first request of a scope loads it: every session's projection, permissions, event log and job store, and the repair of what a stopped process left behind. `listSessions()` waits for that load; `listSessionIdentities()` does not. Until the load has settled it reads the stored scope snapshot alone, normalized as the load normalizes it (a session stored before session generations existed gets the generation the load derives for it, a deleted session is left out), and never reads a projection, an event log or a job store; nothing can change a scope before its load settles. Once the load has settled it answers from the loaded sessions. It gives the same answer before and after the load.
712
+ `listSessionIdentities(scopeId)` lists a scope's live sessions as `IFlexSessionIdentity` (`sessionId`, `sessionGenerationId`, `sessionGenerationSequence`, `parentSessionId` when set), sorted by session id. It reads complete scope metadata only, never a projection, event log or job store. Before the catalogue load settles it normalizes the stored scope snapshot just as loading does (including deterministic legacy generations and tombstone filtering); afterward it reads the complete in-memory catalogue, not the hydrated runtime subset.
713
713
 
714
714
  The load gives the event loop a turn about every 25 ms of work, so timers, I/O and a caller's cancellation keep running beside it, and its repair takes time linear in a session's size. A request that stops waiting for the load leaves it running, and the next request of the scope joins it; `dispose()` and `retireScope()` stop it at its next turn, and the requests waiting for it fail with `FlexHarnessAbortError`. The `scopeLoadObserver` option is called once each load ends, with `IFlexScopeLoadMetrics`: `scopeId`, `storageKey`, the live `sessions` it read, `storeReadMs` (waiting for the scope snapshot and each session's projection, permissions, agent event store and job store), `agentSessionMs` (creating the sessions' agent sessions, each of which reads and checks its event log), `repairMs`, `totalMs`, all wall-clock milliseconds, and `outcome` (`'loaded'`, `'failed'`, or `'aborted'` by disposal or retirement). An exception the observer throws, or a promise it rejects, is ignored.
715
715
 
@@ -731,12 +731,48 @@ The reservation save is the admission point. A save failure produces no start ev
731
731
 
732
732
  `listMessageMetadataPage()` pages like `listMessagePage()`, with the same options, `startIndex`, cursors (a cursor of either kind of page continues the other) and `includeUndone`, but each message carries only its identity and state (`TFlexMessageMetadata`: `messageId`, `sessionId`, `runId`, `role`, `status`, `createdAt`, `completedAt`, `steerId`, `promptId`, `keptInContext`, `undone`, `cacheWarm`), never its parts, model, usage or error. A full scan then reads the same few hundred bytes per message whatever its content, enough to prove which prompts and steers a session received and whether its runs kept them in context.
733
733
 
734
- `updateSession()` supports title replacement, explicit title clearing with `null`, and archive state through `archived`. Title-only updates remain available while prompts are queued or running, while permission is pending, and after archival. Requests containing `archived` are rejected while the session has any outstanding prompt or pending permission; a mixed title-and-archive request is rejected atomically without changing the title. Archived sessions expose `archivedAt`. Every session exposes `preview` once a run with prompt text was admitted: its first prompt as one line (whitespace runs collapsed, cut at a character boundary to `maxTitleBytes`), with no model call, so a host can name a session nobody titled (`title || preview`). It is set once, a session stored before previews existed takes it from its first prompt at its next run, and no caller sets or clears it. Deleting a session cascades through its complete descendant subtree. One durable root-keyed tombstone group hides every newly affected live session, and the delete also joins any already-separate descendant cleanup groups without rewriting their roots. FlexHarness then cancels queued and active subtree work, emits terminal queue events, waits for admitted initialization, and purges runtime queue status while cleaning runtime and persisted domains child-first. The requested root tombstone is removed last after every domain confirms cleanup; project-management cleanup confirmation is a retained durable project tombstone rather than physical removal. An imported session can hold no project-management state, so its deletion writes no project tombstone. A successful live `deleteSession()` call emits `session.deleted` for each session it newly tombstoned; retries of an existing tombstone and automatic load, retirement, or disposal cleanup emit no deletion events. Direct deletion of a descendant cascades only through that descendant's subtree. Cleanup authority follows the resolved storage namespace, so scope aliases share the same groups. A partial failure retains durable ownership for retry by a later `deleteSession()`, namespace load, `retireScope()`, or `dispose()` call.
734
+ `updateSession()` supports title replacement, explicit title clearing with `null`, and archive state through `archived`. Title-only updates remain available while prompts are queued or running, while permission is pending, and after archival. Requests containing `archived` are rejected while the session has any outstanding prompt or pending permission; a mixed title-and-archive request is rejected atomically without changing the title. Archived sessions expose `archivedAt`. Every session exposes `preview` once a run with prompt text was admitted: its first prompt as one line (whitespace runs collapsed, cut at a character boundary to `maxTitleBytes`), with no model call, so a host can name a session nobody titled (`title || preview`). It is set once, a session stored before previews existed takes it from its first prompt at its next run, and no caller sets or clears it. Deleting a session cascades through its complete descendant subtree. One durable root-keyed tombstone group hides every newly affected live session, and the delete also joins any already-separate descendant cleanup groups without rewriting their roots. FlexHarness cancels the subtree's active runs as soon as the group is applied, before its save completes, so an admission that has not reached its model call never does however slow the store; it then cancels queued subtree work, emits terminal queue events, waits for admitted initialization, and purges runtime queue status while cleaning runtime and persisted domains child-first. The requested root tombstone is removed last after every domain confirms cleanup; project-management cleanup confirmation is a retained durable project tombstone rather than physical removal. An imported session can hold no project-management state, so its deletion writes no project tombstone. A successful live `deleteSession()` call emits `session.deleted` for each session it newly tombstoned; retries of an existing tombstone and automatic load, retirement, or disposal cleanup emit no deletion events. Direct deletion of a descendant cascades only through that descendant's subtree. Cleanup authority follows the resolved storage namespace, so scope aliases share the same groups. A partial failure retains durable ownership for retry by a later `deleteSession()`, namespace load, `retireScope()`, or `dispose()` call.
735
735
 
736
736
  `deleteSessionGenerationCohort()` is the generation-fenced destructive form for controllers. Its `root` and every `authorizedCohort` entry use the exact `{ sessionId, sessionGenerationId, sessionGenerationSequence }` shape. The cohort contains unique entries in strictly ascending `sessionId` order and is limited by `FLEX_SESSION_GENERATION_COHORT_MAX_ENTRIES` (2048). FlexHarness atomically validates the matching root, its complete live subtree, and every retained descendant tombstone group before reserving deletion. A missing or different root generation returns `{ matched: false }` without touching a newer generation; a missing or mismatched cascade entry throws before new tombstoning or cleanup. Extra valid entries do not expand the cascade. A matched partial cleanup remains retryable with the same authority and resolves to `{ matched: true }` when cleanup completes. The existing `deleteSession()` method remains available for callers that intentionally own the complete reusable-ID namespace.
737
737
 
738
738
  `abort()` returns `true` only while cancellation is still accepted. Terminal persistence is the run's commit point; once it starts, `abort()` returns `false` and the already-fixed terminal outcome completes while the session remains busy.
739
739
 
740
+ ### Session-local loading and startup recovery
741
+
742
+ The default `sessionLoading: 'eager'` preserves existing store behavior. Opt into `sessionLoading: 'session-local'` to load a complete metadata catalogue and hydrate only the session a request uses. Concurrent requests, including scope aliases of one storage key, join that session's hydration and repair. Creating an empty session reads none of the unrelated sessions' projections, permissions, events or jobs. `listSessions()` deliberately requests every session's repaired metadata; use `getScopeCatalogue()` or `listSessionIdentities()` for metadata-only enumeration.
743
+
744
+ ```typescript
745
+ const harness = new FlexHarness({
746
+ scopeResolver,
747
+ modelResolver,
748
+ stores,
749
+ sessionLoading: 'session-local',
750
+ });
751
+ // Normal startup: recover indexed shared-workspace obligations and deletion tombstones.
752
+ await harness.prepareScope(scopeId);
753
+ const { sessions, recoverySessionIds } = await harness.getScopeCatalogue(scopeId);
754
+ ```
755
+
756
+ Session-local loading requires `stores.scopes.sessionLocal`. Missing capabilities fail at construction; absent or stale qualification evidence fails before ordinary session use. There is no implicit eager fallback. On a legacy namespace, the host must exclusively own the namespace, excluding every scope alias and every old writer, and explicitly run `await harness.prepareScope(scopeId, { qualifyLegacy: true })`. This permits one complete load/repair and establishes authoritative recovery evidence. It is not a flag to put on every request. Scope retirement and disposal join owned target hydrations without hydrating cold sessions.
757
+
758
+ Shared-workspace recovery is not merely target-local: durable sidecar obligations are registered before capture preparation, application, finalization and release effects, and cleared only after canonical durable resolution. Startup gates on those obligations and tombstones, while unrelated sessions remain cold. Per-session repair still uses the canonical Agent event history and projection reconciliation.
759
+
760
+ #### Implementing `scopes.sessionLocal` on your own storage
761
+
762
+ A host that keeps Flex data in its own storage composes `new RecoverableFlexScopeStore(existingScopeStore, recoveryStore)` and implements `IFlexScopeRecoveryStore` for that storage. The scope snapshot stays exactly `{ schemaVersion, revision, sessions, tombstones }`; the proof is a separate record, `IFlexScopeRecoverySnapshot` `{ schemaVersion: 1, revision, catalogueRevision, catalogueDigest, recoverySessionIds }`, which `RecoverableFlexScopeStore` computes and the recovery store persists. A session-local load trusts a namespace only when the proof names the stored scope's revision and digest and the write provenance is current. The digest alone is never enough: a writer that does not know the proof (an older release, an eager harness, another tool) can write a projection's pending capture or apply, run its workspace effect and stop before it writes the scope, which leaves scope and proof byte for byte unchanged. The recovery store must meet this contract:
763
+
764
+ 1. `save(storageKey, proof, expectedRevision)` is a durable compare-and-swap: it writes only when the stored revision equals `expectedRevision` and throws `FlexHarnessStoreConflictError` otherwise; an unknown outcome throws `FlexHarnessStoreCommitUncertainError`.
765
+ 2. The provenance is a durable witness that every write in every domain of the namespace changes atomically with the write: scope, proof, projections, permissions, agent events and archives, tool jobs, results, project management, and every deletion or purge. Every writer that can open the namespace must change it, including releases that predate this contract, so it has to be a counter or header those writers already change, never a field only recovery-aware writers maintain. `NoSqlFlexHarnessStores` uses the namespace header's `creationCount`, which every write transaction increments since 12.1.1. Its releases 10.1.0 to 12.1.0 counted only record creations, so their projection, event and job updates change no witness: a namespace one of them may have written must be qualified explicitly before a session-local load.
766
+ 3. A write of an activated instance is recorded as known atomically with the write (NoSQL: `recoveryWriteCount` in the same transaction); every other write is unknown. While activated, a write checks in the same atomic step that the witness still shows only known writes, and fails with `FlexHarnessStoreFormatError` instead of recording itself as known when it does not.
767
+ 4. `loadWriteProvenance(storageKey)` reads metadata only, never history bodies, and returns `true` only if every write since the last qualification is known; it then activates the instance for that key. Otherwise it returns `false` and deactivates. It never repairs.
768
+ 5. `deactivateWriteProvenance(storageKey)` is synchronous and local: afterwards the instance's writes are unknown. FlexHarness calls it on release, eager loading, disposal, retirement and whenever the evidence is not current.
769
+ 6. `qualifyWriteProvenance(storageKey)` marks every write so far as known and activates. FlexHarness calls it only at the end of `prepareScope(scopeId, { qualifyLegacy: true })`, after the complete load and repair and after it published the new proof while inactive; when it fails, the namespace stays unqualified.
770
+ 7. The witness never returns to an earlier value while the namespace holds data; an empty namespace (no scope, no proof, no records) may start over.
771
+
772
+ When the older releases of a host change no such witness, the host cannot detect their writes: it must not open the namespace with session-local loading after an older release may have written it, unless it qualifies explicitly first. Neither the provenance nor any compare-and-swap replaces the host's exclusive ownership of the namespace: no simultaneous writer, scope alias or older release.
773
+
774
+ In-memory and NoSQL stores implement the additive capability. Existing custom stores and `JsonFileFlexHarnessStores` keep their eager behavior; the JSON store creates no new recovery files and cannot opt into session-local loading without a separate capable backend.
775
+
740
776
  ### Continuing Without New Input
741
777
 
742
778
  Use `continuePrompt()` after a failed or stopped run whose input remains in context
@@ -1055,7 +1091,7 @@ Resolved model identity contains provider and model IDs plus optional display na
1055
1091
 
1056
1092
  Public audit history is safe to send to controllers: attachment parts contain source and size metadata, never inline base64, data URLs, or remote URL payloads. Canonical private Agent events retain those values solely for subsequent model turns.
1057
1093
 
1058
- Sessions expose `idle`, `scheduled`, `running`, `waiting_permission`, `failed`, and `cancelled` status. A run stopped by `abort()`, `cancelPrompt()` or a rejected permission leaves its session `cancelled`; only a run that failed leaves it `failed`. Persisted non-terminal activity is repaired from canonical Agent generation outcomes after process restart; incomplete messages and parts normalize to `cancelled` unless an accepted hidden terminal stage can be promoted.
1094
+ Sessions expose `idle`, `scheduled`, `running`, `waiting_permission`, `failed`, and `cancelled` status. A run stopped by `abort()`, `cancelPrompt()` or a rejected permission leaves its session `cancelled`; only a run that failed leaves it `failed`. Persisted non-terminal activity is repaired from canonical Agent generation outcomes after process restart; incomplete messages and parts normalize to `cancelled` unless an accepted hidden terminal stage can be promoted; an interrupted turn's assistant message keeps the parts of its last checkpoint.
1059
1095
 
1060
1096
  ## Conversation Import
1061
1097
 
@@ -1163,7 +1199,9 @@ FlexHarness wraps every provided tool `execute` method before AgentSession recei
1163
1199
 
1164
1200
  `toolOutputLimits` in the complete setup above bounds traversal depth and encoded bytes. The normalizer enforces its byte allowance incrementally: oversized strings are replaced before entering output, and arrays/objects stop reading entries once only truncation metadata fits.
1165
1201
 
1166
- Streaming callbacks use run-local synchronous state rather than one persistence promise per source delta. Text and reasoning accumulate only in the run-local terminal projection while each source delta remains an immediate exact public event. Every distinct async-iterable tool output appears immediately as a bounded cumulative `part.updated` snapshot while the tool remains `running`, including the final yielded value before completion. Only the authoritative `part.completed` output enters the terminal projection, and failed or interrupted tools discard their transient output. `callbackLimits` bounds callback events, accumulated output bytes, and part count; overflow aborts internally with `FlexHarnessCallbackOverflowError` and the turn is recorded as failed. The calls of a `run_code` program never overflow it: they stop their program at its `callbacks` limit instead. Reservation and terminal finalization are the normal persistence checkpoints, with permission state changes as explicit additional checkpoints.
1202
+ Streaming callbacks use run-local synchronous state rather than one persistence promise per source delta. Text and reasoning accumulate only in the run-local terminal projection while each source delta remains an immediate exact public event. Every distinct async-iterable tool output appears immediately as a bounded cumulative `part.updated` snapshot while the tool remains `running`, including the final yielded value before completion. Only the authoritative `part.completed` output enters the terminal projection, and failed or interrupted tools discard their transient output. `callbackLimits` bounds callback events, accumulated output bytes, and part count; overflow aborts internally with `FlexHarnessCallbackOverflowError` and the turn is recorded as failed. The calls of a `run_code` program never overflow it: they stop their program at its `callbacks` limit instead. Reservation and terminal finalization are the normal persistence points, with permission state changes and the projection checkpoints of a running turn (below) as additional ones.
1203
+
1204
+ While a turn runs, its streaming assistant message is checkpointed. A checkpoint saves the parts of the model steps the turn committed (the engine saves a step's events before the step ends), each as it stands, in place of the message's parts, in one projection save in the session's projection FIFO. It starts no earlier than the larger of 10 s and 20 times the duration of the turn's last save of that message (its start, a steer split or the previous checkpoint), counted from that save's completion, so a turn shorter than that saves none. At most one is outstanding per turn; its deadline timer does not keep the process alive and fires also when no further step commits. The terminal, a steer split, the turn's close and the namespace's drain, disposal, fence or retirement end the pending one. A process that stops mid-turn therefore leaves the committed steps up to its last completed checkpoint visible, at most the current spacing plus one save behind; the next load repairs the message to `cancelled` as before, its running parts `cancelled`, and never rebuilds parts from events. The visible parts are always a prefix of the committed steps: nothing of the step in progress is saved, neither its text nor a tool call it streamed or started, although a call that started is in the model context (its call and execution intent are saved before it runs). Checkpoints emit no event. A failed checkpoint save fails the turn like its other saves, with a `persistence` error, and an uncertain one fences the namespace. `projectionCheckpointClock` (an `IFlexCacheWarmClock`) lets tests drive the timing; the spacing is fixed.
1167
1205
 
1168
1206
  Model resolver, tool provider, delegated run admission provider, AgentSession, tool execution, tool callback, tool cleanup, and run-persistence failures cross an untrusted error boundary. By default they become a fixed immutable `FlexHarnessExternalError` before completion rejection, persistence, events, or detached-cleanup reporting. Raw external messages and aggregate members are not retained. A failed `onToolCallFinish` callback stores and accounts for only the bounded projected message; it does not otherwise reject completion, although exceeding the configured callback limits still fails the run. Scope resolution and the initial store load happen before a run exists and remain outside this boundary.
1169
1207