agl 40.5.1 → 41.0.0

package/ts/classes.flexstore.ts

@@ -23,7 +23,6 @@ import {
   assertFlexWriterLeaseDocument,
   flexDomainHeadId,
   flexMigrationId,
-  flexPrivateChunkBytes,
   flexWriterLeaseId,
   initFlexPrivateModels,
   registerFlexPrivateModels,
@@ -67,6 +66,14 @@ import {
   initFlexPublicModels,
   registerFlexPublicModels,
 } from './classes.flexmodels.public.js';
+import {
+  buildFlexPrivateSnapshot,
+  discardFlexPrivateChunks,
+  holdFlexPrivateLayout,
+  privateCandidateDocumentId,
+  releaseFlexPrivateLayout,
+  type IFlexPrivateLayout,
+} from './functions.flexprivatesnapshot.js';
 import { isFlexWriterId } from './interfaces.flexipc.js';
 
 type TFlexScopeSnapshot = plugins.flexharness.IFlexScopeSnapshot;
@@ -285,7 +292,25 @@ interface IFlexCachedSnapshot {
   storageKey: string;
   candidateId: string;
   snapshot: unknown;
+  /** The candidate's serialized bytes. */
   bytes: number;
+  /**
+   * How this writer stored the candidate, chunk by chunk with the chunks' bytes: the next save
+   * takes every chunk its change leaves in place. Absent for a snapshot read back.
+   */
+  layout?: IFlexPrivateLayout;
+  /** What the entry counts against `flexSnapshotCacheBytes`. */
+  cost: number;
+}
+
+/** A domain's committed snapshot a save took out of the cache or read, for the save alone. */
+interface IFlexDomainCheckout<TSnapshot> {
+  head: TStoredDomainHead;
+  /** Undefined when the save did not need the snapshot and none was cached. */
+  snapshot?: TSnapshot;
+  layout?: IFlexPrivateLayout;
+  /** Ends the save's hold on the layout; the cache holds what the save committed. */
+  release(): void;
 }
 
 interface IFlexCachedTranscript {
@@ -293,8 +318,19 @@ interface IFlexCachedTranscript {
   bytes: number;
 }
 
-/** What the snapshot and transcript caches may hold in total. */
-const flexSnapshotCacheBytes = 64 * 1024 * 1024;
+/**
+ * What the snapshot cache may hold in total. An entry counts its kept chunk bytes once and its
+ * parsed snapshot at `flexSnapshotHeapFactor` times the snapshot's serialized bytes, so the cache
+ * holds at most about 170 MiB of serialized snapshots stored by this writer (3 times their size
+ * counted). A snapshot that does not fit even alone is not cached: its saves take the full path.
+ */
+const flexSnapshotCacheBytes = 512 * 1024 * 1024;
+/**
+ * The V8 heap a parsed snapshot takes per byte of its JSON, measured on realistic projections
+ * and Agent event logs (1.79 and 1.83 on Node 25), rounded up.
+ */
+const flexSnapshotHeapFactor = 2;
+/** What the transcript cache may hold in total. */
 const flexTranscriptCacheBytes = 16 * 1024 * 1024;
 
 interface IFlexMaintenanceCandidate {
@@ -354,28 +390,6 @@ const randomCandidateId = (): string => plugins.crypto.randomBytes(16).toString(
 
 const randomOwnerToken = (): string => plugins.crypto.randomBytes(32).toString('base64url');
 
-const privateCandidateDocumentId = (candidateIdArg: string): string =>
-  `flex-private-candidate:${candidateIdArg}`;
-
-/**
- * Private chunks are addressed by their content within one domain incarnation, so a save stores
- * only the chunks it changed and shares the others with the snapshot it replaces.
- */
-const privateChunkDocumentId = (
-  controllerIdArg: string,
-  storageKeyArg: string,
-  sourceArg: IFlexPrivateDomainSource,
-  chunkDigestArg: string,
-): string => `flex-private-chunk:${sha256(JSON.stringify([
-  controllerIdArg,
-  storageKeyArg,
-  sourceArg.identity.domain,
-  sourceArg.identity.sessionId ?? null,
-  sourceArg.identity.archiveId ?? null,
-  sourceArg.incarnationId,
-  chunkDigestArg,
-]))}`;
-
 const publicCandidateDocumentId = (candidateIdArg: string): string =>
   `flex-public-candidate:${candidateIdArg}`;
 
@@ -920,136 +934,6 @@ const inBatches = async <TValue>(
   }
 };
 
-/** The lists of a domain snapshot that grow entry by entry, stored apart from the rest. */
-const privateSnapshotLists: Partial<Record<TFlexDomainKind, readonly string[]>> = {
-  projection: ['messages', 'reversionSegments'],
-};
-
-/** A chunk of a growing list stays under this size, so a change rewrites little of the list. */
-const privateListChunkBytes = 64 * 1024;
-
-/**
- * The JSON of a snapshot as runs of pieces. A chunk never spans two runs, and inside a run of
- * list entries chunks are filled from the list's start, so entries before a change keep their
- * chunks and a save that changes a list's newest entries rewrites only that list's last chunk
- * besides the small runs between the lists.
- */
-const privateSnapshotRuns = (snapshotArg: unknown, listKeysArg: readonly string[]): string[][] => {
-  if (
-    listKeysArg.length === 0
-    || snapshotArg === null
-    || typeof snapshotArg !== 'object'
-    || Array.isArray(snapshotArg)
-  ) return [[JSON.stringify(snapshotArg)]];
-  const runs: string[][] = [];
-  let frame = '{';
-  let first = true;
-  for (const [key, value] of Object.entries(snapshotArg)) {
-    if (value === undefined) continue;
-    frame += `${first ? '' : ','}${JSON.stringify(key)}:`;
-    first = false;
-    if (listKeysArg.includes(key) && Array.isArray(value) && value.length > 0) {
-      runs.push([`${frame}[`]);
-      runs.push(value.map((entry, index) => `${index === 0 ? '' : ','}${JSON.stringify(entry)}`));
-      frame = ']';
-    } else {
-      frame += JSON.stringify(value);
-    }
-  }
-  runs.push([`${frame}}`]);
-  return runs;
-};
-
-/** Cuts the runs into chunks of at most `flexPrivateChunkBytes`, never across two runs. */
-const privateSnapshotChunks = (runsArg: string[][]): Buffer[] => {
-  const chunks: Buffer[] = [];
-  for (const run of runsArg) {
-    let pending: Buffer[] = [];
-    let pendingBytes = 0;
-    const flush = (): void => {
-      if (pending.length === 0) return;
-      chunks.push(Buffer.concat(pending, pendingBytes));
-      pending = [];
-      pendingBytes = 0;
-    };
-    for (const piece of run) {
-      const bytes = Buffer.from(piece, 'utf8');
-      if (pendingBytes > 0 && pendingBytes + bytes.byteLength > privateListChunkBytes) flush();
-      if (bytes.byteLength <= flexPrivateChunkBytes - pendingBytes) {
-        pending.push(bytes);
-        pendingBytes += bytes.byteLength;
-        continue;
-      }
-      flush();
-      for (let offset = 0; offset < bytes.byteLength; offset += flexPrivateChunkBytes) {
-        chunks.push(bytes.subarray(offset, Math.min(offset + flexPrivateChunkBytes, bytes.byteLength)));
-      }
-    }
-    flush();
-  }
-  return chunks.length === 0 ? [Buffer.alloc(0)] : chunks;
-};
-
-const createPrivateProjection = (
-  controllerIdArg: string,
-  storageKeyArg: string,
-  candidateIdArg: string,
-  revisionArg: number,
-  snapshotArg: unknown,
-  createdAtArg: string,
-  sourceArg: IFlexPrivateDomainSource,
-): { candidate: IFlexPrivateCandidateDocument; chunks: IFlexPrivateChunkDocument[] } => {
-  const identity = {
-    domain: sourceArg.identity.domain,
-    incarnationId: sourceArg.incarnationId,
-    ...(sourceArg.identity.sessionId !== undefined
-      ? { sessionId: sourceArg.identity.sessionId }
-      : {}),
-    ...(sourceArg.identity.archiveId !== undefined
-      ? { archiveId: sourceArg.identity.archiveId }
-      : {}),
-  };
-  const buffers = privateSnapshotChunks(privateSnapshotRuns(
-    snapshotArg,
-    privateSnapshotLists[sourceArg.identity.domain] ?? [],
-  ));
-  const chunks: IFlexPrivateChunkDocument[] = buffers.map((chunk, index) => {
-    const digest = sha256(chunk);
-    return {
-      id: privateChunkDocumentId(controllerIdArg, storageKeyArg, sourceArg, digest),
-      controllerId: controllerIdArg,
-      scopeId: storageKeyArg,
-      candidateId: candidateIdArg,
-      revision: revisionArg,
-      index,
-      sha256: digest,
-      bytes: chunk.byteLength,
-      data: chunk.toString('base64'),
-      createdAt: createdAtArg,
-      ...identity,
-    };
-  });
-  const complete = Buffer.concat(buffers);
-  const candidate: IFlexPrivateCandidateDocument = {
-    id: privateCandidateDocumentId(candidateIdArg),
-    controllerId: controllerIdArg,
-    scopeId: storageKeyArg,
-    candidateId: candidateIdArg,
-    revision: revisionArg,
-    sha256: sha256(complete),
-    bytes: complete.byteLength,
-    chunks: chunks.map(({ id, sha256: digest, bytes }, index) => ({
-      id, index, sha256: digest, bytes,
-    })),
-    createdAt: createdAtArg,
-    ...identity,
-  };
-  complete.fill(0);
-  assertFlexPrivateCandidateDocument(candidate);
-  for (const chunk of chunks) assertFlexPrivateChunkDocument(chunk);
-  return { candidate, chunks };
-};
-
 export interface IFlexTranscriptLimits {
   messageLimit: number;
   messageBytesLimit: number;
@@ -1069,11 +953,39 @@ const defaultTranscriptLimits: IFlexTranscriptLimits = {
   singleMessageBytesLimit: flexPublicSingleMessageBytesLimit,
 };
 
+/**
+ * What a transcript build asks of each message: the id of its record, the size of its JSON and
+ * its stand-in when it is oversized. A store answers from what it cached per stored message.
+ */
+export interface IFlexTranscriptMessageRecords {
+  /** The id of the public record of `messageArg` at `messageIndexArg`. */
+  recordId(messageIndexArg: number, messageArg: TFlexMessage): string;
+  /** The UTF-8 bytes of `messageArg`'s JSON. */
+  messageBytes(messageArg: TFlexMessage): number;
+  /** The public stand-in of an oversized message, within `budgetArg` serialized bytes. */
+  placeholder(messageArg: TFlexMessage, budgetArg: number): TFlexMessage;
+}
+
+const computedMessageRecords = (
+  controllerIdArg: string,
+  storageKeyArg: string,
+): IFlexTranscriptMessageRecords => ({
+  recordId: (messageIndexArg, messageArg) => publicMessageDocumentId(
+    controllerIdArg,
+    storageKeyArg,
+    messageIndexArg,
+    messageArg,
+  ),
+  messageBytes: (messageArg) => serializedBytes(messageArg).byteLength,
+  placeholder: placeholderMessage,
+});
+
 /**
  * The public transcript of one session: its newest messages within the public bounds, a message
  * over the single-message bound reduced to a placeholder that keeps its metadata. Records are
  * addressed by their content, and `storedRecordsArg` names records already stored, so the
- * transcript lists the stored copy of every record it shares.
+ * transcript lists the stored copy of every record it shares; only the records it adds are
+ * serialized, hashed and checked.
  */
 export const createProjectionTranscript = (
   controllerIdArg: string,
@@ -1083,12 +995,13 @@ export const createProjectionTranscript = (
   preparationArg: IFlexCandidatePreparation,
   storedRecordsArg: ReadonlyMap<string, IFlexRecordManifest> = new Map(),
   limitsArg: IFlexTranscriptLimits = defaultTranscriptLimits,
+  recordsArg: IFlexTranscriptMessageRecords = computedMessageRecords(controllerIdArg, storageKeyArg),
 ): IFlexTranscriptBuild => {
   const createRecord = (
     messageIndexArg: number,
     messageArg: TFlexMessage,
   ): IFlexPublicMessageRecordDocument => ({
-    id: publicMessageDocumentId(controllerIdArg, storageKeyArg, messageIndexArg, messageArg),
+    id: recordsArg.recordId(messageIndexArg, messageArg),
     controllerId: controllerIdArg,
     scopeId: storageKeyArg,
     candidateId: preparationArg.candidateId,
@@ -1097,6 +1010,11 @@ export const createProjectionTranscript = (
     message: messageArg,
     createdAt: preparationArg.createdAt,
   });
+  // A record's JSON is its metadata's with the message's in place, so an oversized message is
+  // never serialized whole to learn that it does not fit.
+  const recordBytes = (recordArg: IFlexPublicMessageRecordDocument): number =>
+    serializedBytes({ ...recordArg, message: 0 }).byteLength - 1
+    + recordsArg.messageBytes(recordArg.message);
   const newest: Array<{ record: IFlexPublicMessageRecordDocument; manifest: IFlexRecordManifest }> = [];
   let selectedBytes = 0;
   let replaced = false;
@@ -1104,11 +1022,12 @@ export const createProjectionTranscript = (
   for (let messageIndex = messagesArg.length - 1; messageIndex >= oldestRetained; messageIndex--) {
     const message = messagesArg[messageIndex]!;
     let record = createRecord(messageIndex, message);
-    let manifest = storedRecordsArg.get(record.id) ?? recordManifest(record);
-    if (manifest.bytes > limitsArg.singleMessageBytesLimit) {
+    let manifest = storedRecordsArg.get(record.id)
+      ?? (recordBytes(record) > limitsArg.singleMessageBytesLimit ? undefined : recordManifest(record));
+    if (manifest === undefined || manifest.bytes > limitsArg.singleMessageBytesLimit) {
       record = createRecord(
         messageIndex,
-        placeholderMessage(message, Math.floor(limitsArg.singleMessageBytesLimit * 0.8)),
+        recordsArg.placeholder(message, Math.floor(limitsArg.singleMessageBytesLimit * 0.8)),
       );
       manifest = storedRecordsArg.get(record.id) ?? recordManifest(record);
       replaced = true;
@@ -1145,7 +1064,10 @@ export const createProjectionTranscript = (
     ...unaddressed,
   };
   assertFlexPublicTranscriptDocument(transcript);
-  for (const { record } of selected) assertFlexPublicMessageRecordDocument(record);
+  // A stored record was checked when it was written; only the records this build adds are new.
+  for (const { record } of selected) {
+    if (!storedRecordsArg.has(record.id)) assertFlexPublicMessageRecordDocument(record);
+  }
   return {
     transcript,
     manifest: {
@@ -1318,10 +1240,27 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
   /**
    * The snapshot of each domain head this writer last committed or loaded, keyed by head id and
    * valid while the head names the same private candidate: a save applies its change to it
-   * instead of reading the snapshot back. Only heads of storage keys this writer leases enter it.
+   * instead of reading the snapshot back, and reuses the chunks of a layout it committed. Only
+   * heads of storage keys this writer leases enter it; least recently used entries leave first.
    */
   private readonly snapshotCache = new Map<string, IFlexCachedSnapshot>();
   private snapshotCacheBytes = 0;
+  /** Heads whose snapshot exceeds the whole snapshot cache, logged once each. */
+  private readonly uncachedSnapshotHeads = new Set<string>();
+  /*
+   * What the public transcript needs of each stored message object, computed once: a stored
+   * message is never edited (a change replaces it), so its record id at an index of a scope, its
+   * JSON size and its oversized stand-in stay valid while the object lives.
+   */
+  private readonly publicMessageRecordIds = new WeakMap<
+    TFlexMessage,
+    { storageKey: string; messageIndex: number; recordId: string }
+  >();
+  private readonly publicMessageBytes = new WeakMap<TFlexMessage, number>();
+  private readonly publicMessagePlaceholders = new WeakMap<
+    TFlexMessage,
+    { budget: number; placeholder: TFlexMessage }
+  >();
   /** Transcripts by id; they are addressed by their content, so a cached one never goes stale. */
   private readonly transcriptCache = new Map<string, IFlexCachedTranscript>();
   private transcriptCacheBytes = 0;
@@ -1958,48 +1897,59 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
           expectedRevisionArg,
           actualRevisionArg,
         );
-      let stored: TFlexProjectionSnapshot | undefined;
-      if (change.messages.end > 0 || change.reversionSegments.end > 0) {
-        const current = await this.loadDomainDirect<TFlexProjectionSnapshot>(
+      // A change that keeps stored entries applies to the committed snapshot; one that sends
+      // whole lists needs none read.
+      const keepsEntries = change.messages.end > 0 || change.reversionSegments.end > 0;
+      const checkout = await this.checkoutDomain<TFlexProjectionSnapshot>(
+        identity,
+        (value) => this.validateProjectionSnapshot(value, sessionIdArg),
+        keepsEntries,
+      );
+      let transcriptChanged = true;
+      try {
+        if (keepsEntries) {
+          const actualRevision = checkout?.head.revision ?? 0;
+          if (actualRevision !== expectedRevisionArg) throw conflict(actualRevision);
+        }
+        // The committed snapshot was validated when it was loaded or saved and is never edited, and
+        // the change is this store's own copy: FlexHarness validates what the change carries and
+        // reads the kept entries for their identities only, never copying the snapshot again.
+        const snapshot = plugins.flexharness.createChangedProjectionSnapshot(
+          keepsEntries ? checkout?.snapshot : undefined,
+          change,
+          sessionIdArg,
+        );
+        await this.saveDomain(
+          lease,
           identity,
-          (value) => this.validateProjectionSnapshot(value, sessionIdArg),
+          snapshot,
+          expectedRevisionArg,
+          conflict,
+          undefined,
+          undefined,
+          async (preparation, current) => {
+            const previous = current?.transcript
+              ? await this.loadTranscript(storageKeyArg, current.transcript.id)
+              : undefined;
+            const build = await this.buildTranscript(
+              storageKeyArg,
+              sessionIdArg,
+              snapshot.messages,
+              preparation,
+              previous,
+            );
+            if (current?.transcript?.id === build.transcript.id) {
+              transcriptChanged = false;
+              return current.transcript;
+            }
+            await this.writeTranscript(build.transcript, build.added);
+            return build.manifest;
+          },
+          checkout,
         );
-        const actualRevision = current?.head.revision ?? 0;
-        if (actualRevision !== expectedRevisionArg) throw conflict(actualRevision);
-        stored = current?.snapshot;
+      } finally {
+        checkout?.release();
       }
-      const snapshot = this.validateProjectionSnapshot(
-        plugins.flexharness.applyFlexProjectionChange(stored, change),
-        sessionIdArg,
-      );
-      let transcriptChanged = true;
-      await this.saveDomain(
-        lease,
-        identity,
-        snapshot,
-        expectedRevisionArg,
-        conflict,
-        undefined,
-        undefined,
-        async (preparation, current) => {
-          const previous = current?.transcript
-            ? await this.loadTranscript(storageKeyArg, current.transcript.id)
-            : undefined;
-          const build = await this.buildTranscript(
-            storageKeyArg,
-            sessionIdArg,
-            snapshot.messages,
-            preparation,
-            previous,
-          );
-          if (current?.transcript?.id === build.transcript.id) {
-            transcriptChanged = false;
-            return current.transcript;
-          }
-          await this.writeTranscript(build.transcript, build.added);
-          return build.manifest;
-        },
-      );
       if (!transcriptChanged) return;
       if (!publishPublicProjectionArg) {
         // The publication no longer names this session's transcript; the next one reads every head.
@@ -2135,12 +2085,14 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
     sessionIdArg: string,
   ): Promise<TAgentEventSnapshot | undefined> {
     this.assertBoundSession(boundSessionIdArg, sessionIdArg);
-    return this.runStorage(storageKeyArg, async () => (
-      await this.loadDomainDirect<TAgentEventSnapshot>(
+    return this.runStorage(storageKeyArg, async () => {
+      const loaded = await this.loadDomainDirect<TAgentEventSnapshot>(
         { storageKey: storageKeyArg, domain: 'agentEvents', sessionId: sessionIdArg },
         (value) => this.validateAgentEventSnapshot(value, sessionIdArg),
-      )
-    )?.snapshot);
+      );
+      // The store keeps its own copy for the next save.
+      return loaded === undefined ? undefined : structuredClone(loaded.snapshot);
+    });
   }
 
   private async saveAgentEventSnapshot(
@@ -2169,25 +2121,45 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
           expectedRevisionArg,
           actualRevisionArg,
         );
-      const current = await this.loadDomainDirect<TAgentEventSnapshot>(
+      const checkout = await this.checkoutDomain<TAgentEventSnapshot>(
         identity,
         (value) => this.validateAgentEventSnapshot(value, sessionIdArg),
+        true,
       );
-      const actualRevision = current?.head.revision ?? 0;
-      if (actualRevision !== expectedRevisionArg) throw conflict(actualRevision);
-      const events = asStoreFormat(
-        'Agent event change does not apply to the stored log',
-        () => plugins.flexAgent.applyAgentEventChange(current?.snapshot.events ?? [], change),
-      );
-      const snapshot = this.validateAgentEventSnapshot({
-        schemaVersion: 2,
-        sessionId: sessionIdArg,
-        revision: expectedRevisionArg + 1,
-        updatedAt: events.reduce((latest, event) => Math.max(latest, event.timestamp), 0),
-        events,
-      }, sessionIdArg);
-      await this.saveDomain(lease, identity, snapshot, expectedRevisionArg, conflict);
-      return snapshot.revision;
+      try {
+        const actualRevision = checkout?.head.revision ?? 0;
+        if (actualRevision !== expectedRevisionArg) throw conflict(actualRevision);
+        const events = asStoreFormat(
+          'Agent event change does not apply to the stored log',
+          () => plugins.flexAgent.applyAgentEventChange(checkout?.snapshot?.events ?? [], change),
+        );
+        // The stored events were validated when they were stored and are never edited, and the
+        // change is this store's own copy: the log is checked, never copied again.
+        const snapshot = asStoreFormat(
+          'Invalid schema-2 Agent event snapshot',
+          () => plugins.flexAgent.validateAgentEventSnapshot({
+            schemaVersion: 2,
+            sessionId: sessionIdArg,
+            revision: expectedRevisionArg + 1,
+            updatedAt: events.reduce((latest, event) => Math.max(latest, event.timestamp), 0),
+            events,
+          }, sessionIdArg),
+        );
+        await this.saveDomain(
+          lease,
+          identity,
+          snapshot,
+          expectedRevisionArg,
+          conflict,
+          undefined,
+          undefined,
+          undefined,
+          checkout,
+        );
+        return snapshot.revision;
+      } finally {
+        checkout?.release();
+      }
     });
   }
 
@@ -2396,11 +2368,19 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
     sessionIdArg: string,
   ): TFlexProjectionSnapshot {
     const value = cloneJson(valueArg as TFlexProjectionSnapshot);
-    if (!Number.isSafeInteger(value?.revision) || value.revision < 0) {
+    this.assertProjectionSnapshot(value, sessionIdArg);
+    return value;
+  }
+
+  /** Checks a projection snapshot in place: FlexHarness owns what a valid projection is. */
+  private assertProjectionSnapshot(
+    valueArg: TFlexProjectionSnapshot,
+    sessionIdArg: string,
+  ): void {
+    if (!Number.isSafeInteger(valueArg?.revision) || valueArg.revision < 0) {
       throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid projection snapshot revision.');
     }
-    plugins.flexharness.assertFlexProjectionSnapshot(value, sessionIdArg);
-    return value;
+    plugins.flexharness.assertFlexProjectionSnapshot(valueArg, sessionIdArg);
   }
 
   private async validatePermissionSnapshot(valueArg: unknown): Promise<TFlexPermissionSnapshot> {
@@ -2530,6 +2510,7 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
       preparationArg: IFlexCandidatePreparation,
       currentArg: TStoredDomainHead | null,
     ) => Promise<IFlexPublicTranscriptManifest>,
+    checkoutArg?: IFlexDomainCheckout<TSnapshot>,
   ): Promise<TStoredDomainHead> {
     if (!Number.isSafeInteger(revisionArg) || Number(revisionArg) < 1) {
       throw new plugins.flexharness.FlexHarnessStoreFormatError('Invalid domain revision.');
@@ -2542,15 +2523,30 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
     const candidateId = randomCandidateId();
     const createdAt = new Date().toISOString();
     const incarnationId = current?.incarnationId ?? randomCandidateId();
-    const projection = createPrivateProjection(
-      this.controllerId,
-      identityArg.storageKey,
+    const sharedChunkIds = new Set(current?.privateCandidate.chunks.map((chunk) => chunk.id) ?? []);
+    // The checked-out layout describes the candidate the head names now, or it is not used.
+    const previous = current && checkoutArg?.layout
+      && checkoutArg.head.currentPrivateCandidateId === current.currentPrivateCandidateId
+      && checkoutArg.layout.sha256 === current.privateCandidate.sha256
+      && checkoutArg.layout.chunks.length === current.privateCandidate.chunks.length
+      ? checkoutArg.layout
+      : undefined;
+    const projection = buildFlexPrivateSnapshot({
+      owner: {
+        controllerId: this.controllerId,
+        storageKey: identityArg.storageKey,
+        domain: identityArg.domain,
+        ...(identityArg.sessionId !== undefined ? { sessionId: identityArg.sessionId } : {}),
+        ...(identityArg.archiveId !== undefined ? { archiveId: identityArg.archiveId } : {}),
+        incarnationId,
+      },
       candidateId,
       revision,
-      snapshotArg,
       createdAt,
-      { identity: identityArg, incarnationId },
-    );
+      snapshot: snapshotArg,
+      ...(previous === undefined ? {} : { previous }),
+      sharedChunkIds,
+    });
     const manifest: IFlexPrivateCandidateManifest = {
       candidateId,
       revision,
@@ -2564,7 +2560,7 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
       await this.writeAndVerifyPrivate(
         projection.candidate,
         projection.chunks,
-        new Set(current?.privateCandidate.chunks.map((chunk) => chunk.id) ?? []),
+        sharedChunkIds,
         abortSignalArg,
       );
       abortSignalArg?.throwIfAborted();
@@ -2603,7 +2599,14 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
         throw errorArg;
       }
       committed = true;
-      this.cacheSnapshot(next, snapshotArg);
+      // Only the domains whose saves take changes keep a layout to build the next save from.
+      this.cacheSnapshot(
+        next,
+        snapshotArg,
+        identityArg.domain === 'projection' || identityArg.domain === 'agentEvents'
+          ? projection.layout
+          : undefined,
+      );
       if (current) {
         await this.cleanupPrivateCandidate(
           current.currentPrivateCandidateId,
@@ -2618,6 +2621,7 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
         );
       })();
     } finally {
+      discardFlexPrivateChunks(projection.minted);
       if (!committed) this.queueOrphanMaintenance();
       this.activeCandidateIds.delete(candidateId);
     }
@@ -2654,6 +2658,48 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
     return { head, snapshot };
   }
 
+  /**
+   * The committed snapshot a save changes: taken out of the cache, so the save holds its layout
+   * alone, or read when `loadArg` asks for it. Undefined when the domain has no head. The caller
+   * releases it once the save ended, whatever its outcome.
+   */
+  private async checkoutDomain<TSnapshot>(
+    identityArg: IFlexDomainIdentity,
+    validateArg: (valueArg: unknown) => TSnapshot | Promise<TSnapshot>,
+    loadArg: boolean,
+  ): Promise<IFlexDomainCheckout<TSnapshot> | undefined> {
+    const head = await this.readDomainHead(identityArg);
+    if (!head) return undefined;
+    const entry = this.takeCachedSnapshot(head);
+    if (entry) {
+      let released = false;
+      return {
+        head,
+        snapshot: entry.snapshot as TSnapshot,
+        ...(entry.layout === undefined ? {} : { layout: entry.layout }),
+        release: () => {
+          if (released) return;
+          released = true;
+          if (entry.layout) releaseFlexPrivateLayout(entry.layout);
+        },
+      };
+    }
+    if (!loadArg) return { head, release: () => undefined };
+    const snapshot = await this.loadPrivateCandidate(
+      identityArg.storageKey,
+      head.privateCandidate,
+      validateArg,
+      undefined,
+      { identity: identityArg, incarnationId: head.incarnationId },
+    );
+    if ((snapshot as { revision?: unknown }).revision !== head.revision) {
+      throw new plugins.flexharness.FlexHarnessStoreFormatError(
+        'Flex domain snapshot revision does not match its head.',
+      );
+    }
+    return { head, snapshot, release: () => undefined };
+  }
+
   private async readMigration(
     storageKeyArg: string,
     optionsArg?: IFlexReadOptions,
@@ -2986,24 +3032,65 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
     return entry.snapshot as TSnapshot;
   }
 
-  /** Keeps a projection or scope snapshot this writer committed or loaded, never edited after. */
-  private cacheSnapshot(headArg: IFlexDomainHeadDocument, snapshotArg: unknown): void {
-    if (headArg.domain !== 'projection' && headArg.domain !== 'scope') return;
+  /**
+   * Takes the cached snapshot of `headArg` out of the cache for a save, which then holds its
+   * layout until it releases it; a stale entry is dropped.
+   */
+  private takeCachedSnapshot(headArg: IFlexDomainHeadDocument): IFlexCachedSnapshot | undefined {
+    const entry = this.snapshotCache.get(headArg.id);
+    if (!entry) return undefined;
+    this.snapshotCache.delete(headArg.id);
+    this.snapshotCacheBytes -= entry.cost;
+    if (
+      entry.candidateId === headArg.currentPrivateCandidateId
+      && entry.bytes === headArg.privateCandidate.bytes
+      && (entry.layout === undefined || entry.layout.sha256 === headArg.privateCandidate.sha256)
+    ) return entry;
+    if (entry.layout) releaseFlexPrivateLayout(entry.layout);
+    return undefined;
+  }
+
+  /**
+   * Keeps a projection, scope or Agent event snapshot this writer committed or loaded, never
+   * edited after, and the layout this writer stored it in.
+   */
+  private cacheSnapshot(
+    headArg: IFlexDomainHeadDocument,
+    snapshotArg: unknown,
+    layoutArg?: IFlexPrivateLayout,
+  ): void {
+    if (
+      headArg.domain !== 'projection'
+      && headArg.domain !== 'scope'
+      && headArg.domain !== 'agentEvents'
+    ) return;
     if (!this.heldLeases.has(headArg.storageKey)) return;
     this.forgetCachedSnapshot(headArg.id);
     const bytes = headArg.privateCandidate.bytes;
-    if (bytes > flexSnapshotCacheBytes / 4) return;
+    const cost = (layoutArg?.bytes ?? 0) + Math.ceil(bytes * flexSnapshotHeapFactor);
+    if (cost > flexSnapshotCacheBytes) {
+      if (!this.uncachedSnapshotHeads.has(headArg.id)) {
+        this.uncachedSnapshotHeads.add(headArg.id);
+        console.warn(
+          `Flex snapshot ${headArg.id} of ${bytes} bytes exceeds the ${flexSnapshotCacheBytes}-byte snapshot cache; its saves read and rewrite it whole.`,
+        );
+      }
+      return;
+    }
+    this.uncachedSnapshotHeads.delete(headArg.id);
+    if (layoutArg) holdFlexPrivateLayout(layoutArg);
     this.snapshotCache.set(headArg.id, {
       storageKey: headArg.storageKey,
       candidateId: headArg.currentPrivateCandidateId,
       snapshot: snapshotArg,
       bytes,
+      ...(layoutArg === undefined ? {} : { layout: layoutArg }),
+      cost,
     });
-    this.snapshotCacheBytes += bytes;
-    for (const [id, entry] of this.snapshotCache) {
+    this.snapshotCacheBytes += cost;
+    for (const id of this.snapshotCache.keys()) {
       if (this.snapshotCacheBytes <= flexSnapshotCacheBytes) break;
-      this.snapshotCache.delete(id);
-      this.snapshotCacheBytes -= entry.bytes;
+      this.forgetCachedSnapshot(id);
     }
   }
 
@@ -3011,7 +3098,8 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
     const entry = this.snapshotCache.get(headIdArg);
     if (!entry) return;
     this.snapshotCache.delete(headIdArg);
-    this.snapshotCacheBytes -= entry.bytes;
+    this.snapshotCacheBytes -= entry.cost;
+    if (entry.layout) releaseFlexPrivateLayout(entry.layout);
   }
 
   private forgetCachedSnapshots(storageKeyArg: string): void {
@@ -3037,6 +3125,7 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
         { id, sha256: digest, bytes },
       ]),
     );
+    const records = this.transcriptMessageRecords(storageKeyArg);
     let build = createProjectionTranscript(
       this.controllerId,
       storageKeyArg,
@@ -3044,6 +3133,8 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
       messagesArg,
       preparationArg,
       known,
+      undefined,
+      records,
     );
     const unknownIds = build.records.map((record) => record.id).filter((id) => !known.has(id));
     if (unknownIds.length > 0) {
@@ -3057,12 +3148,52 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
           messagesArg,
           preparationArg,
           known,
+          undefined,
+          records,
         );
       }
     }
     return { ...build, added: build.records.filter((record) => !known.has(record.id)) };
   }
 
+  /** The record facts of a scope's messages, from this store's per-message caches. */
+  private transcriptMessageRecords(storageKeyArg: string): IFlexTranscriptMessageRecords {
+    return {
+      recordId: (messageIndexArg, messageArg) => {
+        const cached = this.publicMessageRecordIds.get(messageArg);
+        if (cached?.storageKey === storageKeyArg && cached.messageIndex === messageIndexArg) {
+          return cached.recordId;
+        }
+        const recordId = publicMessageDocumentId(
+          this.controllerId,
+          storageKeyArg,
+          messageIndexArg,
+          messageArg,
+        );
+        this.publicMessageRecordIds.set(messageArg, {
+          storageKey: storageKeyArg,
+          messageIndex: messageIndexArg,
+          recordId,
+        });
+        return recordId;
+      },
+      messageBytes: (messageArg) => {
+        const cached = this.publicMessageBytes.get(messageArg);
+        if (cached !== undefined) return cached;
+        const bytes = serializedBytes(messageArg).byteLength;
+        this.publicMessageBytes.set(messageArg, bytes);
+        return bytes;
+      },
+      placeholder: (messageArg, budgetArg) => {
+        const cached = this.publicMessagePlaceholders.get(messageArg);
+        if (cached?.budget === budgetArg) return cached.placeholder;
+        const placeholder = placeholderMessage(messageArg, budgetArg);
+        this.publicMessagePlaceholders.set(messageArg, { budget: budgetArg, placeholder });
+        return placeholder;
+      },
+    };
+  }
+
   /**
    * Stores a transcript before the records it adds, so an interrupted write leaves a transcript no
    * head names that still names everything the write stored, and proves both.
@@ -4374,11 +4505,15 @@ export class FlexStore implements plugins.flexharness.IFlexHarnessStores {
       .flatMap((chunk: unknown) => (
         typeof (chunk as { id?: unknown })?.id === 'string' ? [(chunk as { id: string }).id] : []
       ));
-    await FlexPrivateChunkModel.atomicDeleteMany({
-      ...owner,
-      $or: [{ candidateId: candidateIdArg }, ...(listed.length > 0 ? [{ id: { $in: listed } }] : [])],
-      ...(keepArg.size > 0 ? { id: { $nin: [...keepArg] } } : {}),
-    }, options);
+    // The chunks the candidate wrote or lists, but those `keepArg` names: two deletes the indexes
+    // answer (by candidate, by id), so only the chunks that go are examined, never every chunk of
+    // the storage key.
+    const keep = keepArg.size > 0 ? { id: { $nin: [...keepArg] } } : {};
+    await FlexPrivateChunkModel.atomicDeleteMany({ ...owner, candidateId: candidateIdArg, ...keep }, options);
+    const unkept = listed.filter((id) => !keepArg.has(id));
+    if (unkept.length > 0) {
+      await FlexPrivateChunkModel.atomicDeleteMany({ ...owner, id: { $in: unkept } }, options);
+    }
     await FlexPrivateCandidateModel.atomicDeleteMany({ ...owner, candidateId: candidateIdArg }, options);
   }