@modelprofile.com/flexharness 18.1.0 → 18.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/dist_ts/00_commitinfo_data.js +1 -1
  2. package/dist_ts/classes.flexharness.cachewarm.js +3 -3
  3. package/dist_ts/classes.flexharness.callbacks.d.ts +37 -0
  4. package/dist_ts/classes.flexharness.callbacks.js +129 -2
  5. package/dist_ts/classes.flexharness.compaction.js +2 -2
  6. package/dist_ts/classes.flexharness.configurations.js +4 -4
  7. package/dist_ts/classes.flexharness.core.d.ts +39 -4
  8. package/dist_ts/classes.flexharness.core.js +467 -134
  9. package/dist_ts/classes.flexharness.d.ts +4 -0
  10. package/dist_ts/classes.flexharness.deletion.d.ts +13 -0
  11. package/dist_ts/classes.flexharness.deletion.js +122 -40
  12. package/dist_ts/classes.flexharness.imports.js +26 -21
  13. package/dist_ts/classes.flexharness.js +9 -1
  14. package/dist_ts/classes.flexharness.lifecycle.js +9 -3
  15. package/dist_ts/classes.flexharness.projects.js +2 -2
  16. package/dist_ts/classes.flexharness.prompts.js +14 -8
  17. package/dist_ts/classes.flexharness.reversions.js +3 -3
  18. package/dist_ts/classes.flexharness.runs.js +18 -2
  19. package/dist_ts/classes.flexharness.sessions.d.ts +1 -1
  20. package/dist_ts/classes.flexharness.sessions.js +18 -18
  21. package/dist_ts/classes.flexharness.slashcommands.js +3 -3
  22. package/dist_ts/classes.flexharness.subagents.js +37 -16
  23. package/dist_ts/classes.flexharness.tooldispatch.js +3 -3
  24. package/dist_ts/classes.scoperecovery.d.ts +36 -0
  25. package/dist_ts/classes.scoperecovery.js +228 -0
  26. package/dist_ts/classes.stores.d.ts +14 -1
  27. package/dist_ts/classes.stores.js +150 -17
  28. package/dist_ts/index.d.ts +1 -0
  29. package/dist_ts/index.js +2 -1
  30. package/dist_ts/interfaces.d.ts +84 -0
  31. package/dist_ts/interfaces.flexharness.d.ts +36 -0
  32. package/dist_ts/interfaces.js +1 -1
  33. package/dist_ts/utils.harnessoptions.d.ts +2 -0
  34. package/dist_ts/utils.harnessoptions.js +16 -11
  35. package/dist_ts_stores_nosqldb/classes.nosqldbstores.d.ts +3 -2
  36. package/dist_ts_stores_nosqldb/classes.nosqldbstores.js +33 -2
  37. package/dist_ts_stores_nosqldb/classes.recordstore.d.ts +10 -1
  38. package/dist_ts_stores_nosqldb/classes.recordstore.js +75 -7
  39. package/package.json +3 -3
  40. package/readme.md +42 -4
  41. package/ts/00_commitinfo_data.ts +1 -1
  42. package/ts/classes.flexharness.cachewarm.ts +2 -2
  43. package/ts/classes.flexharness.callbacks.ts +142 -1
  44. package/ts/classes.flexharness.compaction.ts +1 -1
  45. package/ts/classes.flexharness.configurations.ts +3 -3
  46. package/ts/classes.flexharness.core.ts +463 -133
  47. package/ts/classes.flexharness.deletion.ts +161 -52
  48. package/ts/classes.flexharness.imports.ts +26 -21
  49. package/ts/classes.flexharness.lifecycle.ts +8 -2
  50. package/ts/classes.flexharness.projects.ts +1 -1
  51. package/ts/classes.flexharness.prompts.ts +11 -7
  52. package/ts/classes.flexharness.reversions.ts +2 -2
  53. package/ts/classes.flexharness.runs.ts +16 -1
  54. package/ts/classes.flexharness.sessions.ts +17 -17
  55. package/ts/classes.flexharness.slashcommands.ts +2 -2
  56. package/ts/classes.flexharness.subagents.ts +40 -15
  57. package/ts/classes.flexharness.tooldispatch.ts +2 -2
  58. package/ts/classes.flexharness.ts +13 -0
  59. package/ts/classes.scoperecovery.ts +290 -0
  60. package/ts/classes.stores.ts +150 -16
  61. package/ts/index.ts +2 -0
  62. package/ts/interfaces.flexharness.ts +38 -0
  63. package/ts/interfaces.ts +89 -0
  64. package/ts/readme.md +42 -4
  65. package/ts/utils.harnessoptions.ts +18 -12
  66. package/ts_stores_nosqldb/classes.nosqldbstores.ts +52 -2
  67. package/ts_stores_nosqldb/classes.recordstore.ts +78 -8
  68. package/ts_stores_nosqldb/readme.md +10 -0
@@ -395,17 +395,32 @@ export function normalizeSlashCommands<TScope>(
395
395
  return Object.freeze(normalized);
396
396
  }
397
397
 
398
- const systemCacheWarmClock: IFlexCacheWarmClock = Object.freeze({
398
+ const systemClock: IFlexCacheWarmClock = Object.freeze({
399
399
  now: () => Date.now(),
400
400
  setTimeout: (callback: () => void, delayMs: number) => {
401
401
  const handle = setTimeout(callback, delayMs);
402
- // A warm period never holds the process open.
402
+ // Neither a warm period nor a pending projection checkpoint holds the process open.
403
403
  handle.unref?.();
404
404
  return handle;
405
405
  },
406
406
  clearTimeout: (handle: unknown) => clearTimeout(handle as ReturnType<typeof setTimeout>),
407
407
  });
408
408
 
409
+ /** The clock `name` gives, checked; the system clock, whose timers are unref'd, by default. */
410
+ export function normalizeClock(clock: IFlexCacheWarmClock | undefined, name: string): IFlexCacheWarmClock {
411
+ const resolved: unknown = clock ?? systemClock;
412
+ if (
413
+ !resolved
414
+ || typeof resolved !== 'object'
415
+ || typeof (resolved as IFlexCacheWarmClock).now !== 'function'
416
+ || typeof (resolved as IFlexCacheWarmClock).setTimeout !== 'function'
417
+ || typeof (resolved as IFlexCacheWarmClock).clearTimeout !== 'function'
418
+ ) {
419
+ throw new FlexHarnessValidationError(`${name} must provide now, setTimeout and clearTimeout.`);
420
+ }
421
+ return resolved as IFlexCacheWarmClock;
422
+ }
423
+
409
424
  export function normalizeCacheWarmOptions(
410
425
  input: IFlexHarnessOptions<unknown>['cacheWarm'],
411
426
  ): INormalizedCacheWarmOptions {
@@ -416,16 +431,7 @@ export function normalizeCacheWarmOptions(
416
431
  const unsupported = Object.keys(options)
417
432
  .find((key) => !['maxDurationMs', 'maxOutputTokens', 'clock'].includes(key));
418
433
  if (unsupported) throw new FlexHarnessValidationError(`cacheWarm does not support "${unsupported}".`);
419
- const clock: IFlexCacheWarmClock = options.clock ?? systemCacheWarmClock;
420
- if (
421
- !clock
422
- || typeof clock !== 'object'
423
- || typeof clock.now !== 'function'
424
- || typeof clock.setTimeout !== 'function'
425
- || typeof clock.clearTimeout !== 'function'
426
- ) {
427
- throw new FlexHarnessValidationError('cacheWarm.clock must provide now, setTimeout and clearTimeout.');
428
- }
434
+ const clock = normalizeClock(options.clock, 'cacheWarm.clock');
429
435
  return Object.freeze({
430
436
  maxDurationMs: resolveBoundedPositiveInteger(
431
437
  options.maxDurationMs,
@@ -1,4 +1,9 @@
1
1
  import * as plugins from './plugins.js';
2
+ import {
3
+ assertFlexScopeRecoverySnapshot,
4
+ RecoverableFlexScopeStore,
5
+ validateFlexScopeRecoverySave,
6
+ } from '../ts/classes.scoperecovery.js';
2
7
  import { FlexHarnessStoreConflictError, FlexHarnessValidationError } from '../ts/errors.js';
3
8
  import type {
4
9
  IFlexAgentEventStoreProvider,
@@ -12,6 +17,8 @@ import type {
12
17
  IFlexProjectionChange,
13
18
  IFlexProjectionStore,
14
19
  IFlexResultStore,
20
+ IFlexScopeRecoverySnapshot,
21
+ IFlexScopeRecoveryStore,
15
22
  IFlexScopeSnapshot,
16
23
  IFlexScopeStore,
17
24
  IFlexToolJobStoreProvider,
@@ -122,6 +129,47 @@ class NoSqlScopeStore implements IFlexScopeStore {
122
129
  }
123
130
  }
124
131
 
132
+ class NoSqlScopeRecoveryStore implements IFlexScopeRecoveryStore {
133
+ constructor(private readonly records: FlexRecordStore) {}
134
+
135
+ public deactivateWriteProvenance(storageKey: string): void {
136
+ this.records.deactivateWriteProvenance(storageKey);
137
+ }
138
+
139
+ public loadWriteProvenance(storageKey: string): Promise<boolean> {
140
+ validateIdentifier(storageKey, 'storageKey');
141
+ return this.records.loadWriteProvenance(storageKey);
142
+ }
143
+
144
+ public qualifyWriteProvenance(storageKey: string): Promise<void> {
145
+ validateIdentifier(storageKey, 'storageKey');
146
+ return this.records.qualifyWriteProvenance(storageKey);
147
+ }
148
+
149
+ public load(storageKey: string): Promise<IFlexScopeRecoverySnapshot | undefined> {
150
+ validateIdentifier(storageKey, 'storageKey');
151
+ return this.records.read({ domain: 'scopeRecovery', storageKey }, (value) => {
152
+ assertFlexScopeRecoverySnapshot(value);
153
+ return value;
154
+ });
155
+ }
156
+
157
+ public async save(
158
+ storageKey: string,
159
+ snapshot: IFlexScopeRecoverySnapshot,
160
+ expectedRevision: number,
161
+ ): Promise<void> {
162
+ validateIdentifier(storageKey, 'storageKey');
163
+ const validated = validateFlexScopeRecoverySave(snapshot, expectedRevision);
164
+ const conflict = (actualRevision: number) =>
165
+ new FlexHarnessStoreConflictError(storageKey, expectedRevision, actualRevision);
166
+ await this.records.replace(
167
+ { domain: 'scopeRecovery', storageKey },
168
+ revisionReplacement(validated, expectedRevision, conflict),
169
+ );
170
+ }
171
+ }
172
+
125
173
  /** A projection is a log record: a save writes its change, checked in form. */
126
174
  class NoSqlProjectionStore implements IFlexProjectionStore {
127
175
  constructor(private readonly records: FlexRecordStore) {}
@@ -522,7 +570,7 @@ function validateResult(value: unknown): TJsonValue {
522
570
  * `flexharness_changes`; `flexharness_namespaces` holds each storage namespace's fence.
523
571
  */
524
572
  export class NoSqlFlexHarnessStores implements IFlexHarnessStores {
525
- public readonly scopes: IFlexScopeStore;
573
+ public readonly scopes: RecoverableFlexScopeStore;
526
574
  public readonly projections: IFlexProjectionStore;
527
575
  public readonly permissions: IFlexPermissionStore;
528
576
  public readonly projectManagement: IFlexProjectManagementStore;
@@ -536,7 +584,9 @@ export class NoSqlFlexHarnessStores implements IFlexHarnessStores {
536
584
  throw new FlexHarnessValidationError('NoSqlFlexHarnessStores requires a SmartdataDb.');
537
585
  }
538
586
  this.records = new FlexRecordStore(options);
539
- this.scopes = new NoSqlScopeStore(this.records);
587
+ this.scopes = new RecoverableFlexScopeStore(
588
+ new NoSqlScopeStore(this.records), new NoSqlScopeRecoveryStore(this.records),
589
+ );
540
590
  this.projections = new NoSqlProjectionStore(this.records);
541
591
  this.permissions = new NoSqlPermissionStore(this.records);
542
592
  this.projectManagement = new NoSqlProjectManagementStore(this.records);
@@ -8,6 +8,7 @@ import {
8
8
 
9
9
  export type TFlexRecordDomain =
10
10
  | 'scope'
11
+ | 'scopeRecovery'
11
12
  | 'projection'
12
13
  | 'permission'
13
14
  | 'projectManagement'
@@ -184,6 +185,8 @@ export class FlexNamespaceDocument extends plugins.SmartDataDbDoc<FlexNamespaceD
184
185
  public namespaceKey!: string;
185
186
  public fenced!: boolean;
186
187
  public creationCount!: number;
188
+ /** Known session-local writes, aligned only by explicit complete qualification. */
189
+ public recoveryWriteCount?: number;
187
190
  public purgeToken?: string | null;
188
191
  public storageNamespaceKey?: string;
189
192
  }
@@ -282,8 +285,8 @@ export function bindFlexRecordModels(db: plugins.SmartdataDb): IFlexRecordModels
282
285
  });
283
286
  plugins.defineCollectionModel(BoundNamespace, db, {
284
287
  collectionName: FLEX_NAMESPACE_COLLECTION,
285
- persistedFields: ['namespaceKey', 'fenced', 'creationCount', 'purgeToken', 'storageNamespaceKey'],
286
- numericFields: ['creationCount'],
288
+ persistedFields: ['namespaceKey', 'fenced', 'creationCount', 'recoveryWriteCount', 'purgeToken', 'storageNamespaceKey'],
289
+ numericFields: ['creationCount', 'recoveryWriteCount'],
287
290
  identityFields: ['namespaceKey'],
288
291
  indexes: [
289
292
  { name: 'flexharness_namespace_key', key: { namespaceKey: 1 }, options: { unique: true } },
@@ -379,6 +382,7 @@ export class FlexRecordStore {
379
382
  private readonly changes: TChangePageModel;
380
383
  private readonly namespaces: TNamespaceModel;
381
384
  private preparation?: Promise<void>;
385
+ private readonly recoveryNamespaces = new Set<string>();
382
386
 
383
387
  constructor(options: IFlexRecordStoreOptions) {
384
388
  const timeoutMs = options.operationTimeoutMs ?? defaultOperationTimeoutMs;
@@ -404,6 +408,52 @@ export class FlexRecordStore {
404
408
  return this.preparation;
405
409
  }
406
410
 
411
+ private assertWriteCounts(namespace: FlexNamespaceDocument): void {
412
+ for (const value of [namespace.creationCount,
413
+ ...(namespace.recoveryWriteCount === undefined ? [] : [namespace.recoveryWriteCount])]) {
414
+ if (!Number.isSafeInteger(value) || value < 0) {
415
+ throw new FlexHarnessStoreFormatError('Namespace write provenance must contain non-negative safe integers.');
416
+ }
417
+ }
418
+ if ((namespace.recoveryWriteCount ?? 0) > namespace.creationCount) {
419
+ throw new FlexHarnessStoreFormatError('Namespace recovery write count exceeds its write generation.');
420
+ }
421
+ }
422
+
423
+ /** Activates this owner's known writes only while no unqualified all-domain write is present. */
424
+ public deactivateWriteProvenance(storageKey: string): void {
425
+ this.recoveryNamespaces.delete(digest(storageKey));
426
+ }
427
+
428
+ public async loadWriteProvenance(storageKey: string): Promise<boolean> {
429
+ await this.prepare();
430
+ const namespaceKey = digest(storageKey);
431
+ const namespace = await this.namespaces.getInstance({ namespaceKey }, { timeoutMS: this.timeoutMs });
432
+ if (namespace) this.assertWriteCounts(namespace);
433
+ const qualified = !namespace || (!namespace.fenced && !namespace.purgeToken
434
+ && namespace.creationCount === (namespace.recoveryWriteCount ?? 0));
435
+ if (qualified) this.recoveryNamespaces.add(namespaceKey);
436
+ else this.recoveryNamespaces.delete(namespaceKey);
437
+ return qualified;
438
+ }
439
+
440
+ /** Only the complete explicit qualification may reconcile unknown historical writes. */
441
+ public async qualifyWriteProvenance(storageKey: string): Promise<void> {
442
+ await this.prepare();
443
+ const namespaceKey = digest(storageKey);
444
+ await this.transaction(`${FLEX_NAMESPACE_COLLECTION}/${namespaceKey}`, 'write', async (transaction) => {
445
+ const namespace = await this.namespaces.getInstance({ namespaceKey }, { session: transaction });
446
+ if (!namespace || namespace.fenced || namespace.purgeToken) {
447
+ throw new FlexHarnessStoreFormatError('Namespace write provenance cannot be qualified without a current namespace.');
448
+ }
449
+ this.assertWriteCounts(namespace);
450
+ await this.namespaces.atomicUpdate({ namespaceKey }, {
451
+ $set: { recoveryWriteCount: namespace.creationCount },
452
+ }, { session: transaction });
453
+ });
454
+ this.recoveryNamespaces.add(namespaceKey);
455
+ }
456
+
407
457
  private async prepareCollections(): Promise<void> {
408
458
  const capabilities = this.db.capabilities;
409
459
  if (!capabilities) {
@@ -632,9 +682,14 @@ export class FlexRecordStore {
632
682
  uniqueField: identity, limit: deletionBatchSize, timeoutMS: this.timeoutMs });
633
683
  if (page.documents.length === 0) break;
634
684
  const active = await this.transaction(path, 'delete', async (transaction) => {
635
- const touched = await this.namespaces.atomicUpdate({ namespaceKey: markerKey, purgeToken: token },
636
- { $inc: { creationCount: 1 } }, { session: transaction });
637
- if (touched.matchedCount !== 1) return false;
685
+ const marker = await this.namespaces.getInstance({ namespaceKey: markerKey, purgeToken: token },
686
+ { session: transaction });
687
+ if (!marker) return false;
688
+ await this.touchNamespace(namespaceKey, transaction);
689
+ if (markerKey !== namespaceKey) {
690
+ await this.namespaces.atomicUpdate({ namespaceKey: markerKey, purgeToken: token },
691
+ { $inc: { creationCount: 1 } }, { session: transaction });
692
+ }
638
693
  const identifiers = page.documents.map((document) => this.documentIdentity(document, identity));
639
694
  await model.atomicDeleteMany<TDeletionDocument>({ ...filter, [identity]: { $in: identifiers } },
640
695
  { session: transaction });
@@ -687,9 +742,24 @@ export class FlexRecordStore {
687
742
  return markers.some((marker) => marker.fenced || Boolean(marker.purgeToken));
688
743
  }
689
744
 
690
- private touchNamespace(namespaceKey: string, session: plugins.SmartdataSession): Promise<unknown> {
691
- return this.namespaces.atomicUpdate({ namespaceKey },
692
- { $inc: { creationCount: 1 }, $setOnInsert: { fenced: false } }, { upsert: true, session });
745
+ private async touchNamespace(namespaceKey: string, session: plugins.SmartdataSession): Promise<unknown> {
746
+ const recoveryActive = this.recoveryNamespaces.has(namespaceKey);
747
+ if (recoveryActive) {
748
+ const namespace = await this.namespaces.getInstance({ namespaceKey }, { session });
749
+ if (namespace) {
750
+ this.assertWriteCounts(namespace);
751
+ if (namespace.creationCount !== (namespace.recoveryWriteCount ?? 0)) {
752
+ throw new FlexHarnessStoreFormatError('Namespace write provenance requires explicit qualification.');
753
+ }
754
+ if (namespace.creationCount === Number.MAX_SAFE_INTEGER) {
755
+ throw new FlexHarnessStoreFormatError('Namespace write provenance exhausted its safe integer generation.');
756
+ }
757
+ }
758
+ }
759
+ return this.namespaces.atomicUpdate({ namespaceKey }, {
760
+ $inc: { creationCount: 1, ...(recoveryActive ? { recoveryWriteCount: 1 } : {}) },
761
+ $setOnInsert: { fenced: false },
762
+ }, { upsert: true, session });
693
763
  }
694
764
 
695
765
  private writeTransaction<TResult>(resolved: IResolvedAddress,
@@ -19,6 +19,16 @@ const harness = new FlexHarness({ scopeResolver, modelResolver, stores });
19
19
 
20
20
  `NoSqlFlexHarnessStores` implements every `IFlexHarnessStores` domain: scopes, projections with undo and redo history, remembered permissions, project management, Agent events with their archives, and background jobs. Several processes may open the same database; every save is a compare-and-swap on the stored record, so a stale writer gets the domain's conflict error instead of overwriting a newer snapshot.
21
21
 
22
+ ### Session-local catalogue and recovery
23
+
24
+ The bundle implements `scopes.sessionLocal`. Set `sessionLoading: 'session-local'` on `FlexHarness` and call `prepareScope(scopeId)` before admitting application traffic. Existing namespaces require one explicit `prepareScope(scopeId, { qualifyLegacy: true })` under exclusive namespace ownership; normal requests never silently hydrate every session. Use `getScopeCatalogue(scopeId)` for complete metadata enumeration instead of eager `listSessions()` followed by per-session reversion reads.
25
+
26
+ Scope snapshots retain exactly schema 1's `{ schemaVersion, revision, sessions, tombstones }`. Compact recovery proofs live in separate `scopeRecovery` records and bind the scope revision/digest to write-ahead recovery session IDs. The existing namespace `creationCount` advances on every record mutation since 12.1.1; releases 10.1.0 to 12.1.0 advanced it only on record creation, so a namespace one of them may have written needs explicit qualification before a session-local load. An additive namespace header `recoveryWriteCount` advances with it only for an activated recovery-aware owner, in the same transaction; legacy writes advance only `creationCount`. This catches an old writer's projection-only pending-apply/capture WAL or branch prune even when the scope snapshot is unchanged. A mismatch refuses active writes and ordinary startup until explicit qualification; no projection bodies are read to establish provenance.
27
+
28
+ Qualification publishes its proof while inactive, then explicitly qualifies the namespace witness. A failure between those steps remains unqualified. Disposal, retirement and eager activation release local known-writer activation, including when a bundle is reused. Neither those hooks nor cross-process CAS replace the host's exclusive namespace lifecycle ownership. All namespace header access uses the public document-model APIs of `@lossless.org/client/nosqldb`.
29
+
30
+ Engine qualification follows the store's client, `@lossless.org/client` ^1.17.0, which qualifies NoSQLDB 8.0.2 and 10.7.2; the store's tests run on NoSQLDB 10.x (10.11.0). NoSQLDB 11 is not qualified: on 11.1.2, two connections that create the same new collection at the same time can fail `prepare()` with `Created collection is not observable.` (one connection is told the collection exists while another still lists it as absent), and a fenced purge racing record creations can leave a record behind.
31
+
22
32
  The database is the application's. The stores never close it, and they need NoSQLDB or a MongoDB deployment with transactions. `prepare()` creates or verifies the `flexharness_records`, `flexharness_chunks`, `flexharness_changes` and `flexharness_namespaces` collections and their indexes; every store operation awaits the same preparation, so calling it at startup only moves that work and its failure forward. `operationTimeoutMs` (1 to 120000, default 30000) bounds every database call and transaction.
23
33
 
24
34
  A snapshot of up to `FLEX_RECORD_CHUNK_CODE_UNITS` UTF-16 code units of JSON is stored inside its record. A longer one is split into chunks of that size, each below 3 MiB of UTF-8, so a snapshot may exceed the 16 MiB document limit of NoSQLDB and MongoDB. Chunks are written before the record swaps to them, in small transactions that check the namespace and the record's current write token. The chunks a record no longer references are removed by a later write of that record, in bounded transactions that also touch the record. A writer that crashes between writing its chunks and swapping the record leaves only unreferenced chunks, never a record that points at missing ones. They are written against the record's current write token, which a concurrent writer may still win with, so the write after the next one removes them, or the deletion of the session or namespace does.