@milaboratories/pl-middle-layer 1.67.6 → 1.68.0

package/src/middle_layer/project.ts

@@ -11,7 +11,10 @@ import {
   ensureSignedResourceIdNotNull,
   field,
   isNotFoundError,
+  isPermissionDenied,
   isTimeoutOrCancelError,
+  isUnauthenticated,
+  parseSignedResourceId,
   Pl,
   resourceIdToString,
   ResourceTypeName,
@@ -25,7 +28,12 @@ import type { BlockPackSpecAny } from "../model";
 import { randomUUID } from "node:crypto";
 import { withProject, withProjectAuthored } from "../mutator/project";
 import type { ExtendedResourceData, PruningFunction } from "@milaboratories/pl-tree";
-import { SynchronizedTreeState, treeDumpStats } from "@milaboratories/pl-tree";
+import {
+  SynchronizedTreeState,
+  treeDumpStats,
+  TreeStateUpdateError,
+} from "@milaboratories/pl-tree";
+import type { TreeSnapshotStore } from "./tree_snapshot_store";
 import { setTimeout } from "node:timers/promises";
 import { frontendData } from "./frontend_path";
 import type { NavigationState } from "@milaboratories/pl-model-common";
@@ -102,6 +110,21 @@ export class Project {
 
   private readonly abortController = new AbortController();
 
+  /** Tree change generation as of the snapshot currently on disk, or -1 when this session has
+   * not written one. Compared against the tree's current generation to skip writing a mirror
+   * that has not moved, which is what makes a project left open and idle go quiet. */
+  private snapshotGeneration: number;
+
+  /** When a snapshot was last attempted, for the periodic write's wall-clock gate.
+   *
+   * Zero, not the construction time, so the first write lands on the first maintenance pass
+   * after the tree has settled rather than a full interval later. Sessions shorter than one
+   * interval are the common case for a desktop app that is quit with a project still open, and
+   * seeding this to now would leave every one of them with nothing on disk. Set on every
+   * attempt, successful or not, so a persistently failing write retries at the interval rather
+   * than on every pass of the loop. */
+  private lastSnapshotAt = 0;
+
   private get destroyed() {
     return this.abortController.signal.aborted;
   }
@@ -111,7 +134,11 @@ export class Project {
     public readonly id: ProjectId /* Project ID, exposed to outer consumers, who work with ML */,
     readonly rid: SignedResourceId /* Contains signature, not exposed outside middle layer. */,
     private readonly projectTree: SynchronizedTreeState,
+    /** Whether this tree was seeded from a snapshot. When it was, the file on disk already
+     * holds generation 0, so an idle warm reopen writes nothing at all. */
+    restoredFromSnapshot: boolean = false,
   ) {
+    this.snapshotGeneration = restoredFromSnapshot ? 0 : -1;
     this.overview = projectOverview(
       projectTree.entry(),
       this.navigationStates,
@@ -129,6 +156,116 @@ export class Project {
     return "project:" + this.id.toString();
   }
 
+  /**
+   * Periodic snapshot write, carried on the maintenance loop rather than a timer of its own.
+   *
+   * Gated on the tree having changed since the last snapshot, so a project left open and idle
+   * writes once and then goes quiet, and on wall clock, so a project changing continuously
+   * writes at most once per interval.
+   */
+  private async maybeWriteSnapshot(): Promise<void> {
+    const store = this.env.treeSnapshots;
+    if (store === undefined) return;
+
+    const generation = this.projectTree.changeGeneration;
+    if (generation === this.snapshotGeneration) return;
+    if (Date.now() - this.lastSnapshotAt < this.env.ops.treeSnapshotOps.writeInterval) return;
+
+    await this.writeSnapshot(store, generation);
+  }
+
+  /**
+   * Starts the close-boundary snapshot and returns without waiting for the write.
+   *
+   * On top of the periodic write, since closing is a natural point to persist. Change-gated but
+   * not interval-gated: rewriting a mirror that has not moved is pure waste, but a mirror that
+   * has moved is worth keeping however recently the last write happened.
+   *
+   * The **capture is synchronous and happens here**, before the caller destroys the tree,
+   * because destroying it invalidates it and a later capture would be refused. Only the encode
+   * and the write are deferred: they are up to ten megabytes of work, and project switching
+   * should not wait for them. Deferring is safe only because a capture is a copy rather than a
+   * view of the tree.
+   *
+   * The returned promise never rejects. The caller is expected to keep it so it can be drained
+   * at shutdown, not to await it here.
+   */
+  public snapshotOnClose(): Promise<void> {
+    const store = this.env.treeSnapshots;
+    if (store === undefined) return Promise.resolve();
+
+    const generation = this.projectTree.changeGeneration;
+    if (generation === this.snapshotGeneration) return Promise.resolve();
+
+    let snapshot;
+    try {
+      snapshot = this.projectTree.capture(parseSignedResourceId(this.rid).signature);
+    } catch (e: unknown) {
+      this.env.logger.warn(
+        new Error(`failed to capture tree snapshot for project ${this.id} on close`, { cause: e }),
+      );
+      return Promise.resolve();
+    }
+
+    this.lastSnapshotAt = Date.now();
+
+    // Queued behind any in-flight periodic write rather than racing it. Both would land
+    // atomically, but the loser would be a wasted encode of the same mirror.
+    const previous = this.snapshotInFlight ?? Promise.resolve();
+    const write = previous.then(async () => {
+      if (this.snapshotGeneration >= generation) return; // the in-flight write covered it
+      if (await store.write(this.rid, snapshot)) this.snapshotGeneration = generation;
+    });
+
+    this.snapshotInFlight = write.finally(() => {
+      this.snapshotInFlight = undefined;
+    });
+    return this.snapshotInFlight;
+  }
+
+  /** In-flight snapshot write, if any. Both triggers can fire close together (the close write
+   *  lands while the loop is mid-write), and encoding ten megabytes twice for the same mirror
+   *  is worth avoiding. */
+  private snapshotInFlight: Promise<void> | undefined;
+
+  /** Serializes writes, and skips one that the in-flight write has already made redundant. */
+  private async writeSnapshot(store: TreeSnapshotStore, generation: number): Promise<void> {
+    // A loop, not a single check: with three or more callers, re-checking only once would let
+    // a waiter install its own promise over another's and clear the field while that write is
+    // still running. Two callers is the most that can happen today, so this is a guard against
+    // the next caller rather than a live fix.
+    while (this.snapshotInFlight !== undefined) {
+      await this.snapshotInFlight;
+      if (this.snapshotGeneration >= generation) return;
+    }
+
+    this.snapshotInFlight = this.captureAndWrite(store, generation).finally(() => {
+      this.snapshotInFlight = undefined;
+    });
+    await this.snapshotInFlight;
+  }
+
+  /** Captures and writes, never throwing: a snapshot is an optimisation and must not fail
+   *  whatever triggered it. */
+  private async captureAndWrite(store: TreeSnapshotStore, generation: number): Promise<void> {
+    // Recorded before the attempt and regardless of its outcome, so a failing disk is retried
+    // once per interval instead of on every pass of the maintenance loop.
+    this.lastSnapshotAt = Date.now();
+    try {
+      // The root's signature is the session witness a later open compares against.
+      const snapshot = this.projectTree.capture(parseSignedResourceId(this.rid).signature);
+
+      // Only a real write advances the change gate. Marking the generation persisted after a
+      // failed write would tell both triggers the tree is already on disk, so one transient
+      // I/O error would cost the rest of the session, close write included.
+      if (await store.write(this.rid, snapshot)) this.snapshotGeneration = generation;
+    } catch (e: unknown) {
+      this.env.logger.warn(
+        new Error(`failed to capture tree snapshot for project ${this.id}`, { cause: e }),
+      );
+    }
+  }
+
   private async refreshLoop(): Promise<void> {
     let retryState: InfiniteRetryState | undefined;
     while (!this.destroyed) {
@@ -147,6 +284,8 @@ export class Project {
           signal: this.abortController.signal,
         });
 
+        await this.maybeWriteSnapshot();
+
         // Block computables housekeeping
         const overviewLight = await this.overviewLight.getValue();
         const existingBlocks = new Set(overviewLight.listOfBlocks);
@@ -727,18 +866,8 @@ export class Project {
     // Doing a no-op mutation to apply all migration and schema fixes
     await withProject(env.projectHelper, env.pl, rid, (_) => {}, { name: "init" });
 
-    // Loading project tree
-    const projectTree = await SynchronizedTreeState.init(
-      env.pl,
-      rid,
-      {
-        ...env.ops.defaultTreeOptions,
-        pruning: projectTreePruning(env.logger),
-        fieldFilter: projectTreeFieldFilter(),
-        traverseStopRules: projectTreeTraverseStopRules(),
-      },
-      env.logger,
-    );
+    // Loading project tree, warm from a persisted mirror when one is usable
+    const { tree: projectTree, restored } = await loadProjectTree(env, rid);
 
     if (env.ops.debugOps.dumpInitialTreeState) {
       const state = projectTree.dumpState();
@@ -748,8 +877,99 @@ export class Project {
       await fs.writeFile(`${resourceIdToString(rid)}.stats.json`, stringifyForDump(stats));
     }
 
-    return new Project(env, id, rid, projectTree);
+    return new Project(env, id, rid, projectTree, restored);
+  }
+}
+
+/**
+ * Opens the project tree, seeded from a persisted mirror when there is a usable one.
+ *
+ * Carries the fail-safe: if the restored tree fails its first refresh on authentication,
+ * permission or an inconsistency, the snapshot is deleted and the open is retried cold. Once,
+ * and only for that first refresh, so a genuinely dead session still surfaces as itself rather
+ * than being masked as a slow open.
+ *
+ * The fail-safe is what bounds every case the cache key does not cover: a rotated master
+ * secret, a revoked grant, a snapshot valid in itself but no longer matching what the backend
+ * will serve. Without it, an explicit-root tree propagates the refresh failure rather than
+ * healing, so the project would fail to open on every attempt until someone deleted the cache
+ * directory by hand.
+ */
+async function loadProjectTree(
+  env: MiddleLayerEnvironment,
+  rid: SignedResourceId,
+): Promise<{ tree: SynchronizedTreeState; restored: boolean }> {
+  const treeOps = {
+    ...env.ops.defaultTreeOptions,
+    pruning: projectTreePruning(env.logger),
+    fieldFilter: projectTreeFieldFilter(),
+    traverseStopRules: projectTreeTraverseStopRules(),
+  };
+  const cold = async () => ({
+    tree: await SynchronizedTreeState.init(env.pl, rid, treeOps, env.logger),
+    restored: false,
+  });
+
+  const store = env.treeSnapshots;
+  if (store === undefined) return await cold();
+
+  const snapshot = await store.read(rid);
+  if (!snapshot.ok) {
+    env.logger.info(`project tree opening cold, snapshot miss: ${snapshot.miss}`);
+    return await cold();
+  }
+
+  try {
+    const tree = await SynchronizedTreeState.init(
+      env.pl,
+      rid,
+      { ...treeOps, restoreFrom: snapshot.tree },
+      env.logger,
+    );
+
+    // Read from the tree rather than assumed: a snapshot can be handed over and still be
+    // refused, in which case this open was cold and the file on disk does not describe the
+    // tree we now hold.
+    const restored = tree.wasRestoredFromSnapshot;
+    if (restored) store.noteRestored();
+    else env.logger.info("project tree opening cold: the snapshot was not applied");
+
+    return { tree, restored };
+  } catch (e: unknown) {
+    // Retry cold on ANY failure of the warm open, not only on the classified ones. A cold open
+    // is exactly what this code did before snapshots existed, so the retry cannot regress
+    // anything, whereas rethrowing here leaves a project that fails to open on every attempt
+    // until someone deletes the cache directory by hand: the snapshot stays on disk and the
+    // next open restores it and fails the same way. That is the outcome this fail-safe exists
+    // to prevent, and the error classes that can reach here are not a closed set.
+    env.logger.warn(
+      new Error("restored project tree failed its first refresh, opening cold", { cause: e }),
+    );
+
+    // Deleting is reserved for failures that implicate the snapshot itself. Anything else (a
+    // timeout, a dropped connection) says nothing about the file, and throwing it away would
+    // destroy a mirror that is still good, along with the evidence a later signature refresh
+    // would repair.
+    if (isSnapshotFailsafeError(e)) await store.discard(rid);
+
+    return await cold();
+  }
+}
+
+/** The failures that implicate the snapshot rather than the link or the session: a rotated
+ *  master secret, a revoked grant, or state the tree cannot reconcile. Only these delete the
+ *  file; every other failure still falls back to a cold open, it just keeps the file.
+ *
+ *  The cause chain is walked because a wrapper anywhere between the tree update and here would
+ *  otherwise silently disarm the inconsistency arm. `isUnauthenticated` and `isPermissionDenied`
+ *  do their own one-level unwrapping. */
+export function isSnapshotFailsafeError(e: unknown): boolean {
+  if (isUnauthenticated(e) || isPermissionDenied(e)) return true;
+  for (let cause: unknown = e, depth = 0; cause !== undefined && depth < 8; depth++) {
+    if (cause instanceof TreeStateUpdateError) return true;
+    cause = (cause as { cause?: unknown } | null)?.cause;
   }
+  return false;
 }
 
 export function projectTreePruning(logger: MiLogger): PruningFunction {

package/src/middle_layer/project_failsafe.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, test } from "vitest";
+import { TreeStateUpdateError } from "@milaboratories/pl-tree";
+import { PermissionDeniedError, UnauthenticatedError } from "@milaboratories/pl-client";
+import { isSnapshotFailsafeError } from "./project";
+
+/**
+ * Which first-refresh failures implicate the snapshot itself, and so delete it.
+ *
+ * Every failure retries cold regardless, so a mistake here cannot leave a project unopenable.
+ * What it can do is either destroy a good mirror over a transient link problem, or keep a bad
+ * one and pay a wasted warm attempt on every open.
+ */
+describe("the fail-safe classification", () => {
+  test("authentication and permission failures implicate the snapshot", () => {
+    expect(isSnapshotFailsafeError(new UnauthenticatedError("token expired"))).toBe(true);
+    expect(isSnapshotFailsafeError(new PermissionDeniedError("grant revoked"))).toBe(true);
+  });
+
+  test("a tree inconsistency implicates the snapshot", () => {
+    expect(isSnapshotFailsafeError(new TreeStateUpdateError("orphan resource"))).toBe(true);
+  });
+
+  test("a wrapped tree inconsistency still implicates it", () => {
+    // The cause chain is walked precisely so that a wrapper introduced anywhere between the
+    // tree update and the caller cannot silently disarm this arm of the fail-safe.
+    const wrapped = new Error("refresh failed", {
+      cause: new Error("while loading", { cause: new TreeStateUpdateError("orphan resource") }),
+    });
+    expect(isSnapshotFailsafeError(wrapped)).toBe(true);
+  });
+
+  test("a link failure does not, so the mirror is kept", () => {
+    expect(isSnapshotFailsafeError(new Error("socket hang up"))).toBe(false);
+    expect(isSnapshotFailsafeError(new Error("deadline exceeded"))).toBe(false);
+  });
+
+  test("nothing exotic throws", () => {
+    // A cause chain that loops must not hang the classifier.
+    const looped: { cause?: unknown } = {};
+    looped.cause = looped;
+
+    expect(isSnapshotFailsafeError(looped)).toBe(false);
+    expect(isSnapshotFailsafeError(undefined)).toBe(false);
+    expect(isSnapshotFailsafeError(null)).toBe(false);
+    expect(isSnapshotFailsafeError("a string")).toBe(false);
+  });
+});

package/src/middle_layer/tree_snapshot_scenarios.test.ts

@@ -0,0 +1,337 @@
+import { describe, expect, test } from "vitest";
+import { TestHelpers } from "@milaboratories/pl-client";
+import { randomUUID } from "node:crypto";
+import fsp from "node:fs/promises";
+import path from "node:path";
+import * as tp from "node:timers/promises";
+import { MiddleLayer } from "./middle_layer";
+import type { ProjectId } from "../model/project_model";
+import type { TreeSnapshotOps } from "./ops";
+
+/**
+ * The acceptance scenarios that need a live backend. Each one runs several middle layers in
+ * turn against one backend root and one work folder, which is what makes a reopen a reopen:
+ * the projects are the same projects and the snapshot directory is the same directory.
+ *
+ * `MiddleLayer.close()` closes the client it was given, so every middle layer here gets its
+ * own client. They share a session, because the test client reuses one cached token, and a
+ * shared session is exactly what a warm reopen needs.
+ */
+
+const WORK_ROOT = path.resolve(import.meta.dirname, "..", "..", "work");
+
+/** Short intervals so the periodic write is observable inside a test rather than in five
+ *  minutes. Everything else is left at its default. */
+function fastSnapshots(overrides: Partial<TreeSnapshotOps> = {}): TreeSnapshotOps {
+  return {
+    enabled: true,
+    writeInterval: 250,
+    maxSizeBytes: 256 * 1024 * 1024,
+    ...overrides,
+  };
+}
+
+type Scenario = {
+  /** Opens another middle layer, on its own client, over the same root and work folder. */
+  open: (treeSnapshotOps?: TreeSnapshotOps) => Promise<MiddleLayer>;
+  /** Closes one, so it is not closed twice during cleanup. */
+  close: (ml: MiddleLayer) => Promise<void>;
+  /** Creates a project, opens it, and lets its tree settle so there is a mirror worth writing.
+   *  Tracked for cleanup. */
+  project: (ml: MiddleLayer, label: string) => Promise<ProjectId>;
+  /** The shared snapshot directory. */
+  snapshotDir: string;
+};
+
+/**
+ * Each middle layer gets its own client, because `MiddleLayer.close()` closes the client it
+ * was given, and a reopen has to survive that.
+ *
+ * The clients use the caller's own root rather than a temporary one: `PlClient.init` with an
+ * `alternativeRoot` name always creates a fresh ephemeral root and overwrites the field, so a
+ * second client asking for the same name gets an empty project list, which is precisely the
+ * state a reopen must not start from. The projects created here are deleted afterwards.
+ */
+async function withScenario(body: (scenario: Scenario) => Promise<void>): Promise<void> {
+  const workFolder = path.resolve(WORK_ROOT, randomUUID());
+  const live = new Set<MiddleLayer>();
+  const projects = new Set<ProjectId>();
+
+  const openMl = async (treeSnapshotOps: TreeSnapshotOps) => {
+    const client = await TestHelpers.getTestClient();
+    const ml = await MiddleLayer.init(client, workFolder, {
+      defaultTreeOptions: { pollingInterval: 250, stopPollingDelay: 500 },
+      devBlockUpdateRecheckInterval: 300,
+      projectRefreshInterval: 250,
+      localSecret: MiddleLayer.generateLocalSecret(),
+      localProjections: [],
+      openFileDialogCallback: () => {
+        throw new Error("Not implemented.");
+      },
+      treeSnapshotOps,
+    });
+    live.add(ml);
+    return ml;
+  };
+
+  const scenario: Scenario = {
+    snapshotDir: path.join(workFolder, "treeSnapshots"),
+    open: async (treeSnapshotOps = fastSnapshots()) => await openMl(treeSnapshotOps),
+    close: async (ml: MiddleLayer) => {
+      live.delete(ml);
+      await ml.close();
+    },
+    project: async (ml: MiddleLayer, label: string) => {
+      const id = await ml.createProject({ label: `${label} ${randomUUID()}` });
+      projects.add(id);
+      await ml.openProject(id);
+      // Reading the overview forces the tree to load and the computables to resolve.
+      await ml.getOpenedProject(id).overview.awaitStableValue();
+      return id;
+    },
+  };
+
+  try {
+    await body(scenario);
+  } finally {
+    for (const ml of live) await ml.close().catch(() => {});
+
+    // The root outlives the test, so the projects have to be cleaned up explicitly.
+    if (projects.size > 0) {
+      const cleanup = await openMl({ ...fastSnapshots(), enabled: false });
+      try {
+        for (const id of projects) await cleanup.deleteProject(id).catch(() => {});
+      } finally {
+        await cleanup.close().catch(() => {});
+      }
+    }
+    await fsp.rm(workFolder, { recursive: true, force: true });
+  }
+}
+
+async function snapshotFiles(dir: string): Promise<string[]> {
+  try {
+    return (await fsp.readdir(dir)).sort();
+  } catch {
+    return [];
+  }
+}
+
+describe("reopening a project", () => {
+  test("the close write makes the next open warm", async () => {
+    await withScenario(async ({ open, close, project, snapshotDir }) => {
+      const first = await open();
+      const id = await project(first, "warm reopen");
+      await first.closeProject(id);
+
+      // The close write is started, not awaited, so that closing a project does not sit behind
+      // an encode. Shutdown drains it, which is what makes it observable here.
+      await close(first);
+
+      // One file for the one project, written at the close boundary.
+      expect(await snapshotFiles(snapshotDir)).toHaveLength(1);
+      expect(first.treeSnapshotStats?.writes).toBeGreaterThanOrEqual(1);
+
+      const second = await open();
+      await second.openProject(id);
+      // The claim: the reopen read the snapshot and restored from it.
+      expect(second.treeSnapshotStats?.hits).toBe(1);
+      expect(second.treeSnapshotStats?.misses.absent).toBe(0);
+      // Read is not enough: this is the tree actually accepting the mirror.
+      expect(second.treeSnapshotStats?.restores).toBe(1);
+
+      // And the project is genuinely usable, not merely restored.
+      const overview = await second.getOpenedProject(id).overview.awaitStableValue();
+      expect(overview.meta.label).toContain("warm reopen");
+      await close(second);
+    });
+  });
+
+  test("closing a project does not wait for its snapshot to be written", async () => {
+    await withScenario(async ({ open, close, project, snapshotDir }) => {
+      const ml = await open();
+      const id = await project(ml, "unblocked close");
+
+      await ml.closeProject(id);
+      // The capture happened synchronously inside closeProject, but the encode and write are
+      // deferred, so the file is normally not there yet. Asserted as "not blocked on it"
+      // rather than "definitely absent": a tiny mirror can beat us to the assertion, and the
+      // point is that close does not await, not that the write is slow.
+      const writesRightAfterClose = ml.treeSnapshotStats!.writes;
+
+      await close(ml); // drains
+      expect(ml.treeSnapshotStats!.writes).toBeGreaterThanOrEqual(writesRightAfterClose);
+      expect(await snapshotFiles(snapshotDir)).toHaveLength(1);
+    });
+  });
+
+  test("project switching: both returns hit", async () => {
+    await withScenario(async ({ open, close, project, snapshotDir }) => {
+      const first = await open();
+      const a = await project(first, "A");
+      const b = await project(first, "B");
+      await first.closeProject(a);
+      await first.closeProject(b);
+      await close(first); // drains both deferred close writes
+      expect(await snapshotFiles(snapshotDir)).toHaveLength(2);
+
+      const second = await open();
+      await second.openProject(a);
+      await second.openProject(b);
+      expect(second.treeSnapshotStats?.hits).toBe(2);
+      expect(second.treeSnapshotStats?.restores).toBe(2);
+      await close(second);
+    });
+  });
+
+  test("a killed process is covered by the periodic write", async () => {
+    await withScenario(async ({ open, close, project, snapshotDir }) => {
+      const first = await open(fastSnapshots({ writeInterval: 250 }));
+      const id = await project(first, "killed");
+
+      // Never closed, standing in for a reboot, a lost connection or a kill. The periodic
+      // write on the maintenance loop is the only thing that can have saved this.
+      //
+      // What this does NOT reproduce is the relaunch: both middle layers here share a session,
+      // because the test client reuses one cached token. In production the equivalent is the
+      // desktop app reconnecting with the JWT it persisted, which keeps the session and so the
+      // signatures; a change that made relaunch re-login instead would break the warm reopen
+      // and no assertion here would notice.
+      await tp.setTimeout(1500);
+      expect(first.treeSnapshotStats?.writes).toBeGreaterThanOrEqual(1);
+      expect(await snapshotFiles(snapshotDir)).toHaveLength(1);
+
+      // Closing the middle layer without closing the project: close() must not snapshot, so
+      // whatever is on disk came from the periodic write.
+      const writesBefore = first.treeSnapshotStats!.writes;
+      await close(first);
+      expect(first.treeSnapshotStats?.writes).toBe(writesBefore);
+
+      const second = await open();
+      await second.openProject(id);
+      expect(second.treeSnapshotStats?.hits).toBe(1);
+      expect(second.treeSnapshotStats?.restores).toBe(1);
+      await close(second);
+    });
+  });
+});
+
+describe("write cadence", () => {
+  test("open and idle writes once, then goes quiet", async () => {
+    await withScenario(async ({ open, close, project }) => {
+      const ml = await open(fastSnapshots({ writeInterval: 250 }));
+      await project(ml, "idle");
+
+      // Several intervals of nothing happening.
+      await tp.setTimeout(2000);
+
+      // Exactly one, not "at most one": a cold open loads a tree, so the change gate is open
+      // and the first maintenance pass writes. Asserting <= 1 would pass with zero writes and
+      // prove nothing about the periodic trigger existing at all.
+      expect(ml.treeSnapshotStats?.writes).toBe(1);
+
+      // And then quiet, because the gate closes on a mirror that has not moved.
+      await tp.setTimeout(1500);
+      expect(ml.treeSnapshotStats?.writes).toBe(1);
+      await close(ml);
+    });
+  });
+
+  test("a project that keeps changing writes at most once per interval", async () => {
+    await withScenario(async ({ open, close, project }) => {
+      const ml = await open(fastSnapshots({ writeInterval: 1000 }));
+      const id = await project(ml, "changing");
+
+      // Keep the tree moving for roughly three intervals.
+      const until = Date.now() + 3000;
+      let n = 0;
+      while (Date.now() < until) {
+        await ml.setProjectMeta(id, { label: `changing ${n++}` });
+        await tp.setTimeout(150);
+      }
+
+      // Bounded by wall clock, not by how often the tree changed.
+      expect(ml.treeSnapshotStats!.writes).toBeLessThanOrEqual(4);
+      expect(n).toBeGreaterThan(4);
+      await close(ml);
+    });
+  });
+});
+
+describe("when the snapshot cannot be used", () => {
+  test("a rotated signature is a miss, the file is kept, and the project still opens", async () => {
+    await withScenario(async ({ open, close, project, snapshotDir }) => {
+      const first = await open();
+      const id = await project(first, "rotated");
+      await first.closeProject(id);
+      await close(first);
+
+      // Rewrite the witness in the header to stand in for a session that has ended. Its
+      // offset is fixed: magic (4) + schema (2) + flags (2), then a u16 length and the bytes.
+      const [name] = await snapshotFiles(snapshotDir);
+      const file = path.join(snapshotDir, name);
+      const bytes = await fsp.readFile(file);
+      const witnessLength = bytes.readUInt16LE(8);
+      expect(witnessLength).toBeGreaterThan(0);
+      bytes[10] = bytes[10] ^ 0xff;
+      await fsp.writeFile(file, bytes);
+
+      const second = await open();
+      await second.openProject(id);
+      expect(second.treeSnapshotStats?.hits).toBe(0);
+      expect(second.treeSnapshotStats?.restores).toBe(0);
+      expect(second.treeSnapshotStats?.misses["session-rotated"]).toBe(1);
+
+      // Kept: the bodies are still good, only the signatures died.
+      expect(await snapshotFiles(snapshotDir)).toHaveLength(1);
+
+      const overview = await second.getOpenedProject(id).overview.awaitStableValue();
+      expect(overview.meta.label).toContain("rotated");
+      await close(second);
+    });
+  });
+
+  test("a truncated snapshot opens cold without raising", async () => {
+    await withScenario(async ({ open, close, project, snapshotDir }) => {
+      const first = await open();
+      const id = await project(first, "poisoned");
+      await first.closeProject(id);
+      await close(first);
+
+      const [name] = await snapshotFiles(snapshotDir);
+      const file = path.join(snapshotDir, name);
+      const bytes = await fsp.readFile(file);
+      await fsp.writeFile(file, bytes.subarray(0, Math.floor(bytes.length / 2)));
+
+      const second = await open();
+      await second.openProject(id);
+      expect(second.treeSnapshotStats?.hits).toBe(0);
+      expect(second.treeSnapshotStats?.restores).toBe(0);
+
+      const overview = await second.getOpenedProject(id).overview.awaitStableValue();
+      expect(overview.meta.label).toContain("poisoned");
+      await close(second);
+    });
+  });
+});
+
+describe("the kill switch", () => {
+  test("nothing is read or written when snapshots are off", async () => {
+    await withScenario(async ({ open, close, project, snapshotDir }) => {
+      const first = await open(fastSnapshots({ enabled: false }));
+      const id = await project(first, "disabled");
+      await tp.setTimeout(1000);
+      await first.closeProject(id);
+
+      expect(first.treeSnapshotStats).toBeUndefined();
+      expect(await snapshotFiles(snapshotDir)).toStrictEqual([]);
+      await close(first);
+
+      // And a project created while off still opens once it is back on.
+      const second = await open();
+      await second.openProject(id);
+      expect(second.treeSnapshotStats?.misses.absent).toBe(1);
+      await close(second);
+    });
+  });
+});