@modelprofile.com/flexharness 11.3.0 → 12.0.0

package/ts/utils.storerules.ts

@@ -144,8 +144,8 @@ export function isProjectManagementTombstoneAdmitted(
 
 /** An archive is written once; repeating it with the same events is a no-op. */
 export function assertSameArchiveEvents(
-  existing: plugins.IAgentEventArchiveV2,
-  next: plugins.IAgentEventArchiveV2,
+  existing: plugins.IAgentEventArchive,
+  next: plugins.IAgentEventArchive,
 ): void {
   if (
     JSON.stringify(existing.events.map((event) => event.id))

package/ts_migration/v3_legacyflexharness.ts

@@ -18,6 +18,7 @@ import {
   assertJsonSerializable,
   cloneSerializable,
 } from '../ts/utils.json.js';
+import { createWholeProjectionChange } from '../ts/utils.projectionchange.js';
 
 export interface IFlexLegacyStoredSessionSnapshot {
   session: IFlexSession;
@@ -59,8 +60,8 @@ interface IInspectedSessionDestination {
   plan: ILegacySessionMigrationPlan;
   projection: TFlexProjectionSnapshot | undefined;
   permission: IFlexPermissionSnapshot | undefined;
-  eventStore: plugins.IAgentEventStoreV2;
-  eventSnapshot: plugins.IAgentEventSnapshotV2 | undefined;
+  eventStore: plugins.IAgentEventStore;
+  eventSnapshot: plugins.IAgentEventSnapshot | undefined;
   jobSnapshot: plugins.IToolJobSnapshot | undefined;
 }
 
@@ -341,7 +342,7 @@ function createSessionAgentEvents(
 
   const jsonEvents = JSON.parse(JSON.stringify(events)) as plugins.TAgentEvent[];
   try {
-    plugins.validateAgentEventSnapshotV2({
+    plugins.validateAgentEventSnapshot({
       schemaVersion: 2,
       sessionId,
       revision: 1,
@@ -508,7 +509,7 @@ async function inspectDestinations(
       projection,
       permission,
       eventStore,
-      eventSnapshot: eventSnapshot as plugins.IAgentEventSnapshotV2 | undefined,
+      eventSnapshot,
       jobSnapshot,
     });
   }
@@ -596,7 +597,7 @@ export async function migrateLegacyFlexHarnessSnapshot(
         await stores.projections.save(
           storageKey,
           session.sessionId,
-          session.projectionSnapshot,
+          createWholeProjectionChange(session.projectionSnapshot),
           0,
         );
       }
@@ -609,7 +610,11 @@ export async function migrateLegacyFlexHarnessSnapshot(
         );
       }
       if (!destination.eventSnapshot) {
-        await destination.eventStore.save(session.sessionId, session.agentEvents, 0);
+        await destination.eventStore.save(
+          session.sessionId,
+          { remove: [], replace: [], append: session.agentEvents },
+          0,
+        );
       }
     }
     if (!destinations.scope) {

package/ts_stores_nosqldb/classes.nosqldbstores.ts

@@ -9,7 +9,7 @@ import type {
   IFlexProjectManagementStore,
   IFlexProjectManagementTombstone,
   IFlexProjectManagementWriteContext,
-  IFlexProjectionSnapshotV3,
+  IFlexProjectionChange,
   IFlexProjectionStore,
   IFlexResultStore,
   IFlexScopeSnapshot,
@@ -30,10 +30,16 @@ import {
   assertFlexProjectManagementSnapshot,
   assertFlexProjectManagementTombstone,
 } from '../ts/utils.projectmanagement.js';
+import {
+  applyFlexProjectionChange,
+  assertFlexProjectionChange,
+  validateProjectionChangeSave,
+} from '../ts/utils.projectionchange.js';
 import {
   assertFlexToolJobSnapshot,
-  createAgentEventSnapshot,
+  createChangedAgentEventSnapshot,
   createToolJobSnapshot,
+  normalizeAgentEventChange,
   normalizeJsonObjectUndefined,
   validateAgentEventArchive,
   validateAgentEventSnapshot,
@@ -49,17 +55,27 @@ import {
   validateResultIndex,
   validateSnapshotSave,
 } from '../ts/utils.storerules.js';
-import { FlexRecordStore, type IFlexRecordWrite } from './classes.recordstore.js';
+import {
+  FlexRecordStore,
+  type IFlexRecordLog,
+  type IFlexRecordReplacement,
+} from './classes.recordstore.js';
 
-function revisionWrite<TSnapshot extends { revision: number }>(
-  current: { revision: number } | undefined,
+/** Replaces a record with a snapshot while the record's revision is the expected one. */
+function revisionReplacement<TSnapshot extends { revision: number }>(
   snapshot: TSnapshot,
   expectedRevision: number,
   conflict: (actualRevision: number) => Error,
-): IFlexRecordWrite {
-  const actualRevision = current?.revision ?? 0;
-  if (actualRevision !== expectedRevision) throw conflict(actualRevision);
-  return { revision: snapshot.revision, value: snapshot };
+  abortSignal?: AbortSignal,
+): IFlexRecordReplacement {
+  return {
+    decide: (actualRevision) => {
+      abortSignal?.throwIfAborted();
+      if (actualRevision !== expectedRevision) throw conflict(actualRevision);
+      return { revision: snapshot.revision, value: snapshot };
+    },
+    contention: conflict,
+  };
 }
 
 function validateScope(value: unknown): IFlexScopeSnapshot {
@@ -99,50 +115,49 @@ class NoSqlScopeStore implements IFlexScopeStore {
     const validated = validateSnapshotSave(snapshot, expectedRevision, assertFlexScopeSnapshot);
     const conflict = (actualRevision: number) =>
       new FlexHarnessStoreConflictError(storageKey, expectedRevision, actualRevision);
-    await this.records.mutate({ domain: 'scope', storageKey }, {
-      validate: validateScope,
-      decide: (current) => revisionWrite(current, validated, expectedRevision, conflict),
-      contention: conflict,
-    });
+    await this.records.replace(
+      { domain: 'scope', storageKey },
+      revisionReplacement(validated, expectedRevision, conflict),
+    );
   }
 }
 
+/** A projection is a log record: a save writes its change, checked in form. */
 class NoSqlProjectionStore implements IFlexProjectionStore {
   constructor(private readonly records: FlexRecordStore) {}
 
-  private validator(sessionId: string): (value: unknown) => TFlexProjectionSnapshot {
-    return (value) => {
-      assertFlexProjectionSnapshot(value, sessionId);
-      return value;
+  private log(sessionId: string): IFlexRecordLog<TFlexProjectionSnapshot, IFlexProjectionChange> {
+    return {
+      validate: (value) => {
+        assertFlexProjectionSnapshot(value, sessionId);
+        return value;
+      },
+      validateChange: (value) => {
+        assertFlexProjectionChange(value);
+        return value;
+      },
+      apply: applyFlexProjectionChange,
     };
   }
 
   public load(storageKey: string, sessionId: string): Promise<TFlexProjectionSnapshot | undefined> {
     providerKey(storageKey, sessionId);
-    return this.records.read(
-      { domain: 'projection', storageKey, sessionId },
-      this.validator(sessionId),
-    );
+    return this.records.readLog({ domain: 'projection', storageKey, sessionId }, this.log(sessionId));
   }
 
   public async save(
     storageKey: string,
     sessionId: string,
-    snapshot: IFlexProjectionSnapshotV3,
+    change: IFlexProjectionChange,
     expectedRevision: number,
   ): Promise<void> {
     const key = providerKey(storageKey, sessionId);
-    const validated = validateSnapshotSave(
-      snapshot,
+    const validated = validateProjectionChangeSave(change, expectedRevision);
+    await this.records.append({ domain: 'projection', storageKey, sessionId }, this.log(sessionId), {
       expectedRevision,
-      (value) => assertFlexProjectionSnapshot(value, sessionId),
-    );
-    const conflict = (actualRevision: number) =>
-      new FlexHarnessStoreConflictError(key, expectedRevision, actualRevision);
-    await this.records.mutate({ domain: 'projection', storageKey, sessionId }, {
-      validate: this.validator(sessionId),
-      decide: (current) => revisionWrite(current, validated, expectedRevision, conflict),
-      contention: conflict,
+      value: validated,
+      conflict: (actualRevision) =>
+        new FlexHarnessStoreConflictError(key, expectedRevision, actualRevision),
     });
   }
 
@@ -170,11 +185,10 @@ class NoSqlPermissionStore implements IFlexPermissionStore {
     const validated = validateSnapshotSave(snapshot, expectedRevision, assertFlexPermissionSnapshot);
     const conflict = (actualRevision: number) =>
       new FlexHarnessStoreConflictError(key, expectedRevision, actualRevision);
-    await this.records.mutate({ domain: 'permission', storageKey, sessionId }, {
-      validate: validatePermission,
-      decide: (current) => revisionWrite(current, validated, expectedRevision, conflict),
-      contention: conflict,
-    });
+    await this.records.replace(
+      { domain: 'permission', storageKey, sessionId },
+      revisionReplacement(validated, expectedRevision, conflict),
+    );
   }
 
   public deleteSession(storageKey: string, sessionId: string): Promise<void> {
@@ -252,9 +266,8 @@ class NoSqlProjectManagementStore implements IFlexProjectManagementStore {
   }
 }
 
-class NoSqlAgentEventStore implements plugins.IAgentEventStoreV2 {
-  public readonly eventSchemaVersion = 2 as const;
-
+/** A session's event log is a log record: a save writes its change, checked in form. */
+class NoSqlAgentEventStore implements plugins.IAgentEventStore {
   constructor(
     private readonly records: FlexRecordStore,
     private readonly storageKey: string,
@@ -269,37 +282,38 @@ class NoSqlAgentEventStore implements plugins.IAgentEventStoreV2 {
     }
   }
 
-  private readonly validateSnapshot = (value: unknown): plugins.IAgentEventSnapshotV2 =>
-    validateAgentEventSnapshot(value, this.sessionId);
+  private readonly log: IFlexRecordLog<plugins.IAgentEventSnapshot, plugins.IAgentEventChange> = {
+    validate: (value) => validateAgentEventSnapshot(value, this.sessionId),
+    validateChange: normalizeAgentEventChange,
+    apply: (current, change) => createChangedAgentEventSnapshot(this.sessionId, current, change),
+  };
 
-  public load(sessionId: string): Promise<plugins.IAgentEventSnapshotV2 | undefined> {
+  public load(sessionId: string): Promise<plugins.IAgentEventSnapshot | undefined> {
     this.assertSession(sessionId);
-    return this.records.read(
+    return this.records.readLog(
       { domain: 'agentEvents', storageKey: this.storageKey, sessionId },
-      this.validateSnapshot,
+      this.log,
     );
   }
 
   public async save(
     sessionId: string,
-    events: readonly plugins.TAgentEvent[],
+    change: plugins.IAgentEventChange,
     expectedRevision: number,
   ): Promise<number> {
     this.assertSession(sessionId);
     validateExpectedRevision(expectedRevision);
-    const snapshot = createAgentEventSnapshot(sessionId, events, expectedRevision + 1);
-    validateAgentEventSnapshot(snapshot, sessionId);
-    const conflict = (actualRevision: number) =>
-      new plugins.AgentEventStoreConflictError(sessionId, expectedRevision, actualRevision);
-    await this.records.mutate({ domain: 'agentEvents', storageKey: this.storageKey, sessionId }, {
-      validate: this.validateSnapshot,
-      decide: (current) => revisionWrite(current, snapshot, expectedRevision, conflict),
-      contention: conflict,
+    const normalized = normalizeAgentEventChange(change);
+    await this.records.append({ domain: 'agentEvents', storageKey: this.storageKey, sessionId }, this.log, {
+      expectedRevision,
+      value: normalized,
+      conflict: (actualRevision) =>
+        new plugins.AgentEventStoreConflictError(sessionId, expectedRevision, actualRevision),
     });
-    return snapshot.revision;
+    return expectedRevision + 1;
   }
 
-  public async archive(archive: plugins.IAgentEventArchiveV2): Promise<void> {
+  public async archive(archive: plugins.IAgentEventArchive): Promise<void> {
     this.assertSession(archive.sessionId);
     const normalized = normalizeJsonObjectUndefined(archive, '$archive');
     const validated = validateAgentEventArchive(normalized, this.sessionId, archive.archiveId);
@@ -324,7 +338,7 @@ class NoSqlAgentEventStore implements plugins.IAgentEventStoreV2 {
   public loadArchive(
     sessionId: string,
     archiveId: string,
-  ): Promise<plugins.IAgentEventArchiveV2 | undefined> {
+  ): Promise<plugins.IAgentEventArchive | undefined> {
     this.assertSession(sessionId);
     validateIdentifier(archiveId, 'archiveId');
     return this.records.read(
@@ -387,16 +401,9 @@ class NoSqlToolJobStore implements plugins.IToolJobStore {
     const snapshot = createToolJobSnapshot(jobs, expectedRevision + 1);
     const conflict = (actualRevision: number) =>
       new plugins.ToolJobStoreConflictError(expectedRevision, actualRevision);
-    await this.records.mutate(
+    await this.records.replace(
       { domain: 'jobs', storageKey: this.storageKey, sessionId: this.sessionId },
-      {
-        validate: validateToolJobs,
-        decide: (current) => {
-          abortSignal?.throwIfAborted();
-          return revisionWrite(current, snapshot, expectedRevision, conflict);
-        },
-        contention: conflict,
-      },
+      revisionReplacement(snapshot, expectedRevision, conflict, abortSignal),
     );
     return snapshot.revision;
   }
@@ -510,8 +517,9 @@ function validateResult(value: unknown): TJsonValue {
 /**
  * Every FlexHarness persistence domain on `@lossless.org/client/nosqldb`, with compare-and-swap
  * that holds across processes. Records live in `flexharness_records`; payloads longer than one
- * chunk live in `flexharness_chunks`; `flexharness_namespaces` holds each storage namespace's
- * fence.
+ * chunk live in `flexharness_chunks`; the changes saved to a projection or an Agent event log
+ * since its last compaction live in its head and, beyond the head's inline room, in pages in
+ * `flexharness_changes`; `flexharness_namespaces` holds each storage namespace's fence.
  */
 export class NoSqlFlexHarnessStores implements IFlexHarnessStores {
   public readonly scopes: IFlexScopeStore;
@@ -538,7 +546,7 @@ export class NoSqlFlexHarnessStores implements IFlexHarnessStores {
   }
 
   /**
-   * Creates or verifies the three collections and their indexes. Every store operation awaits the
+   * Creates or verifies the four collections and their indexes. Every store operation awaits the
    * same preparation, so calling it at startup only moves that work, and its failure, forward.
    */
   public prepare(): Promise<void> {
@@ -546,7 +554,7 @@ export class NoSqlFlexHarnessStores implements IFlexHarnessStores {
   }
 
   /**
-   * Physically removes every record and chunk of one storage namespace in every domain: the
+   * Physically removes every record, chunk and change of one storage namespace in every domain: the
    * scope snapshot, each session's projection, permissions, Agent events and archives, jobs and
    * results, and the project-management records and tombstones. This is the host's durable
    * deletion of a scope. Stop new admission for every alias of the scope first. Without `fence`,