@valbuild/server 0.105.0 → 0.105.1

package/dist/valbuild-server.esm.js

@@ -1,7 +1,7 @@
 import ts from 'typescript';
 import { pipe, result, array } from '@valbuild/core/fp';
 import { PatchError, deepEqual, parseAndValidateArrayIndex, isNotRoot, applyPatch, JSONOps, deepClone, sourceToPatchPath } from '@valbuild/core/patch';
-import { derefPatch, RecordSchema, Internal, extractValModules, VAL_EXTENSION, ImageSchema, DEFAULT_CONTENT_HOST, hasRemoteFileSchema } from '@valbuild/core';
+import { derefPatch, RecordSchema, Internal, extractValModules, computeValModuleShas, VAL_EXTENSION, ImageSchema, DEFAULT_CONTENT_HOST, hasRemoteFileSchema } from '@valbuild/core';
 export { hasRemoteFileSchema } from '@valbuild/core';
 import * as path from 'path';
 import path__default from 'path';
@@ -1656,6 +1656,40 @@ class ValOps {
 
   /** The sha256 / hash of schema + config - if this changes users needs to reload */
 
+  /**
+   * What the SHAs above are a fold over, so they can be recomputed.
+   *
+   * See {@link promoteCommittedSources}: the one thing that changes sources
+   * without re-evaluating the modules is a save, and it has to be able to move
+   * the SHAs with them.
+   */
+
+  /**
+   * The extraction's OWN module errors, which are what the fold was given.
+   *
+   * Not the same list as {@link modulesErrors}: that one has the nested
+   * `.jsonValues()` errors concatenated on, and those were never part of the
+   * hash. Re-folding with the wrong list changes the base SHA for no reason.
+   */
+
+  /**
+   * What a save has told us each `.jsonValues()` entry now holds.
+   *
+   * The entry twin of {@link sources}, and it has to be separate because an
+   * entry's content is not IN the source: the source holds a marker, and
+   * {@link getJsonEntries} resolves it by awaiting the marker's own `import()`.
+   * That resolves from the module registry, so after `/save` rewrites a
+   * `*.val.json` the thunk keeps answering with the content from before — and
+   * unlike a module source there is nothing to re-extract, because the memo was
+   * never holding the content in the first place.
+   *
+   * `null` for an entry the commit deleted.
+   *
+   * Never cleared: it describes what is on disk. A host rebuild makes a new
+   * instance, which is the right reset. Bounded by the project's entry count,
+   * holding only the latest content per key.
+   */
+  adoptedJsonEntries = new Map();
   constructor(valModules, options) {
     this.valModules = valModules;
     this.options = options;
@@ -1666,6 +1700,8 @@ class ValOps {
     this.sourcesSha = null;
     this.configSha = null;
     this.modulesErrors = null;
+    this.shaEntries = null;
+    this.shaModuleErrors = null;
   }
 
   // #region stat
@@ -1691,6 +1727,8 @@ class ValOps {
       this.sourcesSha = extracted.sourcesSha;
       this.configSha = extracted.configSha;
       this.modulesErrors = moduleErrors;
+      this.shaEntries = extracted.shaEntries;
+      this.shaModuleErrors = extracted.moduleErrors;
       return {
         baseSha: this.baseSha,
         schemaSha: this.schemaSha,
@@ -1711,6 +1749,141 @@ class ValOps {
       moduleErrors: this.modulesErrors
     };
   }
+
+  /**
+   * These patches are on disk now: adopt what they produced as the committed
+   * sources.
+   *
+   * The entry point for the mechanism {@link promoteCommittedSources} describes,
+   * and the only one — a caller hands over the analysis it just committed and
+   * this works out the rest, so the rule about which sources are adopted lives
+   * in one place rather than at each save site.
+   *
+   * A module whose patches could not be applied cleanly is left alone. `/save`
+   * refuses the whole commit before reaching here if `prepare` found errors, so
+   * this cannot normally fire — but adopting a partially patched source would
+   * put content in the memo that is not what was written, which is worse than
+   * being stale.
+   */
+  async adoptCommittedSources(analysis, preparedCommit) {
+    // Read BEFORE anything is promoted: this applies the chain to the sources as
+    // they stand, and promoting first would apply the same patches twice.
+    const {
+      sources,
+      errors
+    } = await this.getSources(analysis);
+    const adopt = {};
+    for (const [moduleFilePathS, source] of Object.entries(sources)) {
+      const moduleFilePath = moduleFilePathS;
+      if (errors[moduleFilePath] !== undefined) {
+        console.error("Val: not adopting the committed source of a module whose patches " + "did not apply cleanly. Its content here stays as it was until the " + "modules are re-evaluated.", {
+          moduleFilePath,
+          errors: errors[moduleFilePath]
+        });
+        continue;
+      }
+      adopt[moduleFilePath] = source;
+    }
+    this.promoteCommittedSources(adopt);
+    /**
+     * And the `.jsonValues()` entry content, which the sources above do not
+     * carry — they hold markers. See {@link adoptedJsonEntries}.
+     *
+     * Only for a module whose source was adopted. The source is what frames an
+     * entry: it decides which keys exist at all, so adopting one without the
+     * other would leave the content and the key set describing different
+     * moments.
+     */
+    for (const [moduleFilePathS, entries] of Object.entries(preparedCommit.patchedJsonEntries)) {
+      const moduleFilePath = moduleFilePathS;
+      if (adopt[moduleFilePath] === undefined) {
+        continue;
+      }
+      const adopted = this.adoptedJsonEntries.get(moduleFilePath) ?? new Map();
+      for (const [entryKey, content] of Object.entries(entries)) {
+        adopted.set(entryKey, content);
+      }
+      this.adoptedJsonEntries.set(moduleFilePath, adopted);
+    }
+  }
+
+  /**
+   * Adopt sources that have just been written to disk, and move the SHAs with
+   * them.
+   *
+   * ## Why this exists rather than an invalidation
+   *
+   * The obvious thing — throw the memo away after a save so the next read
+   * re-extracts — does not work, and quietly. `extractValModules` gets a
+   * module's content by awaiting its `def`, which is the app's own `import()`:
+   * that resolves from the MODULE REGISTRY, not from the file on disk. Right
+   * after `/save` rewrites a `.val.ts`, the registry still holds the module as
+   * it was evaluated before, so a re-extraction returns the pre-save content and
+   * stores it as fresh. What actually replaces it is the host rebuilding its
+   * module graph and constructing a new `ValOps` — which happens on its own
+   * schedule, and until it does, every read is stale.
+   *
+   * Stale reads here are not abstract: `getJsonEntry` resolves a
+   * `.jsonValues()` entry from the committed source and then replays pending
+   * patches over it, so once a publish has removed the patches, a page rendering
+   * draft content gets the committed value — the one this memo is holding from
+   * before the publish.
+   *
+   * So the save tells us instead. It has just computed what the new committed
+   * sources are, and that answer does not depend on anything being
+   * re-evaluated.
+   *
+   * ## And the SHAs move
+   *
+   * Deliberately, and this is the part with consequences. `baseSha` and
+   * `sourcesSha` identify the sources being served; leaving them still while the
+   * sources move would put a value other code compares against into
+   * disagreement with what it describes. Moving them means a `fs`-mode base SHA
+   * changes within a server's lifetime for the first time, which is a signal the
+   * studio already knows how to read: `PatchStore.reconcileVanished` uses a
+   * moved base to tell "these patches were published" from "these patches were
+   * discarded", and takes them out of the chain without reverting the fields —
+   * which is what a second tab watching a publish needs and could not get
+   * before.
+   *
+   * A module the fold does not know is ignored rather than appended: the fold's
+   * order is `val.modules`, and a path that is not in it has no position, so
+   * there is no honest answer for where its hash would go. It also cannot happen
+   * — a save only ever writes modules it read from here.
+   */
+  promoteCommittedSources(patched) {
+    if (this.sources === null || this.shaEntries === null || this.shaModuleErrors === null) {
+      // Nothing has been read yet, so there is no stale answer to correct and
+      // no fold to replay. The first read extracts, as it always would.
+      return;
+    }
+    const known = new Set(this.shaEntries.map(entry => entry.path));
+    const adopt = Object.entries(patched).filter(([moduleFilePath, source]) => source !== undefined && known.has(moduleFilePath));
+    if (adopt.length === 0) {
+      return;
+    }
+    const bySource = new Map(adopt);
+    // A new object rather than a mutation: `getSources` hands this out, and a
+    // caller holding it must not have the ground move under it.
+    this.sources = {
+      ...this.sources
+    };
+    for (const [moduleFilePath, source] of adopt) {
+      this.sources[moduleFilePath] = source;
+    }
+    this.shaEntries = this.shaEntries.map(entry => {
+      const source = bySource.get(entry.path);
+      return source === undefined ? entry : {
+        ...entry,
+        source
+      };
+    });
+    const shas = computeValModuleShas(this.valModules.config, this.shaEntries, this.shaModuleErrors);
+    this.baseSha = shas.baseSha;
+    this.schemaSha = shas.schemaSha;
+    this.sourcesSha = shas.sourcesSha;
+    this.configSha = shas.configSha;
+  }
   async init() {
     const {
       baseSha,
@@ -1860,6 +2033,31 @@ class ValOps {
           baseContent: undefined
         };
       }
+      /**
+       * What a save told us this entry holds, ahead of the thunk.
+       *
+       * The thunk resolves from the module registry, so after `/save` rewrites
+       * a `*.val.json` it keeps answering with the content from before — and
+       * there is nothing to re-extract, because the committed content was
+       * never in the memoised source to begin with. See
+       * {@link adoptedJsonEntries}.
+       *
+       * `null` means the commit deleted the entry, which is reported the same
+       * way an absent key is. (Nearly unreachable — a `remove` also drops the
+       * thunk from the `.val.ts`, so the key is gone from `record` once the
+       * source is adopted — but the map says it, so this says it too.)
+       *
+       * The BASELINE only. Pending patches replay over it below exactly as
+       * they do over the thunk's answer.
+       */
+      const adopted = this.adoptedJsonEntries.get(moduleFilePath);
+      if (adopted !== undefined && adopted.has(entryKey)) {
+        const content = adopted.get(entryKey);
+        return {
+          entryKey,
+          baseContent: content === null ? undefined : content
+        };
+      }
       const thunk = Internal.getJsonImport(marker);
       if (!thunk) {
         return {
@@ -2562,6 +2760,22 @@ class ValOps {
 
       // jsonValues entry content, keyed by `*.val.json` path. `null` = delete.
       const jsonEntryContents = new Map();
+      /**
+       * The same content, keyed by ENTRY KEY rather than by file path.
+       *
+       * The path is what gets written; the key is what a reader asks for, and it
+       * is dropped at the flush below. Reconstructing it afterwards is not on:
+       * TWO producers turn a key into a path — `resolveEntryJsonPath` and
+       * `getNewJsonEntryPaths`, the latter a locked convention for `add` and a
+       * move's destination — and a marker does not carry its path at read time
+       * (see `jsonEntryFiles.ts`). So it is recorded where the key is known.
+       *
+       * Read by `ValOps.adoptCommittedSources`, so a save can tell this instance
+       * what an entry now holds. Nothing else can: an entry's committed content
+       * is resolved through the marker's own `import()`, which caches, so the
+       * memo cannot be refreshed by re-reading.
+       */
+      const jsonEntryContentsByKey = new Map();
       // Entries added in this commit → their new `*.val.json` path, so later
       // content ops in the same commit resolve to the freshly-created file.
       const entryKeyToJsonPath = new Map();
@@ -2722,6 +2936,7 @@ class ValOps {
               tsSourceFile = insRes.value;
               tsChanged = true;
               jsonEntryContents.set(jsonPath, op.value);
+              jsonEntryContentsByKey.set(cls.entryKey, op.value);
               entryKeyToJsonPath.set(cls.entryKey, jsonPath);
             } else if (op.op === "remove") {
               const jsonPathRes = resolveEntryJsonPath(cls.entryKey);
@@ -2739,6 +2954,7 @@ class ValOps {
               tsSourceFile = remRes.value;
               tsChanged = true;
               jsonEntryContents.set(jsonPathRes.value, null);
+              jsonEntryContentsByKey.set(cls.entryKey, null);
             } else if (op.op === "replace") {
               const jsonPathRes = resolveEntryJsonPath(cls.entryKey);
               if (result.isErr(jsonPathRes)) {
@@ -2747,6 +2963,7 @@ class ValOps {
                 break;
               }
               jsonEntryContents.set(jsonPathRes.value, op.value);
+              jsonEntryContentsByKey.set(cls.entryKey, op.value);
             } else if (op.op === "move" || op.op === "copy") {
               // Rename (move) or duplicate (copy) a whole entry. The new entry
               // gets its own `*.val.json` written with the source entry's
@@ -2806,9 +3023,11 @@ class ValOps {
               tsSourceFile = insRes.value;
               tsChanged = true;
               jsonEntryContents.set(jsonPath, content);
+              jsonEntryContentsByKey.set(cls.entryKey, content);
               entryKeyToJsonPath.set(cls.entryKey, jsonPath);
               if (op.op === "move" && fromPathRes.value !== jsonPath) {
                 jsonEntryContents.set(fromPathRes.value, null);
+                jsonEntryContentsByKey.set(fromKey, null);
               }
             } else {
               errors.push({
@@ -2859,6 +3078,7 @@ class ValOps {
               break;
             }
             jsonEntryContents.set(jsonPath, applied.value);
+            jsonEntryContentsByKey.set(cls.entryKey, applied.value);
           }
         }
         if (patchHadError) {
@@ -2929,7 +3149,8 @@ class ValOps {
             path,
             appliedPatches,
             result: sourceFileText,
-            extraFiles
+            extraFiles,
+            jsonEntries: Object.fromEntries(jsonEntryContentsByKey)
           };
         }
       }
@@ -2942,6 +3163,7 @@ class ValOps {
         errors
       };
     };
+    const patchedJsonEntries = {};
     const allResults = await Promise.all(Object.entries(patchesByModule).map(([path, patches]) => applySourceFilePatches(path, patches)));
     let hasErrors = false;
     const sourceFilePatchErrors = {};
@@ -2967,6 +3189,12 @@ class ValOps {
         for (const [extraPath, data] of Object.entries(res.extraFiles)) {
           patchedSourceFiles[extraPath] = data;
         }
+        // Kept per module and per entry key, not flattened into
+        // `patchedSourceFiles` beside the files: a reader of an entry has a
+        // module and a key, never a path. See `patchedJsonEntries`.
+        if (Object.keys(res.jsonEntries).length > 0) {
+          patchedJsonEntries[res.path] = res.jsonEntries;
+        }
         appliedPatches[res.path] = res.appliedPatches ?? [];
       }
       for (const patchId of res.appliedPatches ?? []) {
@@ -3007,6 +3235,7 @@ class ValOps {
       binaryFilePatchErrors,
       unappliablePatches,
       patchedSourceFiles,
+      patchedJsonEntries,
       previousSourceFiles,
       partiallyPatchedSourceFiles,
       patchedBinaryFilesDescriptors,
@@ -3599,11 +3828,180 @@ function patchRecordFile(patchesDir, patchId) {
 function patchBaseFile(patchesDir, patchId) {
   return path__default.join(patchDir(patchesDir, patchId), "base.json");
 }
+
+/**
+ * Where a patch's uploaded bytes wait for the record that will reference them.
+ *
+ * ## Why they cannot simply be written into the patch directory
+ *
+ * A patch that carries a file is written in TWO requests, and the bytes go
+ * first: the record's `file` op holds only a sha, so a record written before its
+ * bytes would point at nothing. Uploading straight into `<patchId>/files/` left
+ * the directory holding files and no `patch.json` for the length of a round
+ * trip — which is neither of the two shapes this store allows, so
+ * {@link readPatchStore} read it as a patch whose contents were lost and repair
+ * removed it, bytes and all.
+ *
+ * And that window is not passive: writing into the patches directory is exactly
+ * what ends `getStat`'s long poll, so the upload summoned the read that
+ * destroyed it. Replacing an image worked only when the two requests happened to
+ * land close enough together.
+ *
+ * So the bytes are not in the store until they belong to something.
+ * {@link appendPatch} moves them in after writing the record, under the lock, so
+ * no reader ever sees a half-built patch directory — and the invariant that a
+ * directory either holds a usable record or is named by the log holds again,
+ * with nothing to tolerate and no ambiguous state to classify.
+ *
+ * A SIBLING of the patches directory, for two reasons: nothing that reads the
+ * store lists it, and it is on the same filesystem, so moving into place is a
+ * rename rather than a copy.
+ */
+function uploadsDir(patchesDir) {
+  return path__default.join(path__default.dirname(patchesDir), "uploads");
+}
+
+/** Where one patch's uploads wait. See {@link uploadsDir}. */
+function patchUploadDir(patchesDir, patchId) {
+  return path__default.join(uploadsDir(patchesDir), patchId);
+}
+
+/**
+ * The binary layout, relative to whichever directory holds it.
+ *
+ * Shared by the patch directory and the staging directory so the two cannot
+ * drift — a move into place has to land the bytes exactly where a read expects
+ * them.
+ */
+function binaryFilesDir(dir) {
+  return path__default.join(dir, "files");
+}
+function binaryFileIn(dir, filePath) {
+  return path__default.join(binaryFilesDir(dir), filePath, path__default.basename(filePath));
+}
+function binaryFileMetadataIn(dir, filePath) {
+  return path__default.join(binaryFilesDir(dir), filePath, "metadata.json");
+}
 function patchBinaryFile(patchesDir, patchId, filePath) {
-  return path__default.join(patchDir(patchesDir, patchId), "files", filePath, path__default.basename(filePath));
+  return binaryFileIn(patchDir(patchesDir, patchId), filePath);
 }
 function patchBinaryFileMetadata(patchesDir, patchId, filePath) {
-  return path__default.join(patchDir(patchesDir, patchId), "files", filePath, "metadata.json");
+  return binaryFileMetadataIn(patchDir(patchesDir, patchId), filePath);
+}
+
+/** The staged twin of {@link patchBinaryFile}. */
+function stagedPatchBinaryFile(patchesDir, patchId, filePath) {
+  return binaryFileIn(patchUploadDir(patchesDir, patchId), filePath);
+}
+
+/** The staged twin of {@link patchBinaryFileMetadata}. */
+function stagedPatchBinaryFileMetadata(patchesDir, patchId, filePath) {
+  return binaryFileMetadataIn(patchUploadDir(patchesDir, patchId), filePath);
+}
+
+/**
+ * Move a patch's staged uploads into the patch directory.
+ *
+ * Called by {@link appendPatch} between the record and the log line, so it runs
+ * under the lock and no reader can observe the halfway state.
+ *
+ * The whole `files` tree in one rename where it can be — the common case, since
+ * a patch's files only ever arrive before its record — and per file otherwise,
+ * for the case where something is already there.
+ */
+function moveStagedUploadsIn(patchesDir, patchId) {
+  const from = binaryFilesDir(patchUploadDir(patchesDir, patchId));
+  if (!fs.existsSync(from)) {
+    return;
+  }
+  const to = binaryFilesDir(patchDir(patchesDir, patchId));
+  if (!fs.existsSync(to)) {
+    fs.mkdirSync(path__default.dirname(to), {
+      recursive: true
+    });
+    fs.renameSync(from, to);
+  } else {
+    moveTreeInto(from, to);
+  }
+  removeStagedUploads(patchesDir, patchId);
+}
+
+/** File-by-file, for when the destination already holds some of the tree. */
+function moveTreeInto(from, to) {
+  for (const entry of fs.readdirSync(from, {
+    withFileTypes: true
+  })) {
+    const source = path__default.join(from, entry.name);
+    const target = path__default.join(to, entry.name);
+    if (entry.isDirectory()) {
+      fs.mkdirSync(target, {
+        recursive: true
+      });
+      moveTreeInto(source, target);
+      continue;
+    }
+    fs.mkdirSync(path__default.dirname(target), {
+      recursive: true
+    });
+    fs.renameSync(source, target);
+  }
+}
+
+/** Drop a patch's staging directory, whatever is left of it. */
+function removeStagedUploads(patchesDir, patchId) {
+  try {
+    fs.rmSync(patchUploadDir(patchesDir, patchId), {
+      recursive: true,
+      force: true
+    });
+  } catch {
+    // Hygiene, not correctness: nothing reads a staging directory that no patch
+    // claims, and the sweep below gets it eventually.
+  }
+}
+
+/**
+ * How long an upload nobody claimed is kept.
+ *
+ * Only garbage collection, which is why it can be a guess at all: these bytes
+ * are outside the store, so no reader can mistake them for a patch and nothing
+ * is lost by keeping them a while. The old marker-based attempt at this problem
+ * had a TTL deciding whether to delete something INSIDE the store, where being
+ * wrong meant destroying a live upload.
+ */
+const STALE_UPLOAD_MS = 24 * 60 * 60 * 1000;
+
+/**
+ * Drop staged uploads whose patch never arrived.
+ *
+ * A client that dies between the upload and the `PUT` leaves its bytes here.
+ * Nothing references them — no record points at them and the log never named
+ * them — so they are removed without a word.
+ */
+function sweepStaleUploads(patchesDir, now = Date.now()) {
+  const dir = uploadsDir(patchesDir);
+  if (!fs.existsSync(dir)) {
+    return;
+  }
+  let names;
+  try {
+    names = fs.readdirSync(dir);
+  } catch {
+    return;
+  }
+  for (const name of names) {
+    const staged = path__default.join(dir, name);
+    try {
+      if (now - fs.statSync(staged).mtimeMs < STALE_UPLOAD_MS) continue;
+      fs.rmSync(staged, {
+        recursive: true,
+        force: true
+      });
+    } catch {
+      // Someone else is writing here, or it is already gone. Either way it is
+      // not this pass's business.
+    }
+  }
 }
 
 /** Names that live in the patches directory but are not patches. */
@@ -3839,6 +4237,18 @@ function writePatchRecord(patchesDir, patchId, record) {
 function appendPatch(patchesDir, record) {
   const patchId = record.patchId;
   writePatchRecord(patchesDir, patchId, record);
+  /*
+   * Then the bytes, then the log line — and the order is the whole point.
+   *
+   * The record goes first, so the directory never exists without one: that is
+   * what makes "files but no patch.json" a state this store cannot produce, and
+   * what lets a reader keep treating it as a patch whose contents are lost.
+   * The log line goes last, so an interrupted append leaves a directory the log
+   * does not name — the benign half of a crash, swept silently.
+   *
+   * See `uploadsDir`. Under the lock, like the rest of this function.
+   */
+  moveStagedUploadsIn(patchesDir, patchId);
   const entry = {
     patchId,
     createdAt: record.createdAt,
@@ -4488,13 +4898,24 @@ class ValOpsFS extends ValOps {
         };
       }
       const patches = announceRes.entries.map(entry => entry.patchId);
-      // Drained here, on the channel that always flows. A repair that removed
-      // everything leaves the studio nothing to fetch, so a notice riding on
-      // `GET /patches` would sit here unread.
-      const removed = this.removedPatchNotices.splice(0, this.removedPatchNotices.length);
-      const removedNotice = removed.length > 0 ? {
-        removed
-      } : {};
+      /**
+       * Drained on the channel that always flows.
+       *
+       * A repair that removed everything leaves the studio nothing to fetch, so
+       * a notice riding on `GET /patches` would sit here unread.
+       *
+       * Accumulating rather than assigning, because the long poll below drains
+       * again before it answers: a repair during the wait must not have to sit
+       * out another whole stat.
+       */
+      const removed = [];
+      const drainRemoved = () => {
+        removed.push(...this.removedPatchNotices.splice(0, this.removedPatchNotices.length));
+        return removed.length > 0 ? {
+          removed
+        } : {};
+      };
+      const removedNotice = drainRemoved();
       // something changed: return immediately
       const didChange = !params ||
       // An entry file changed on disk: nothing else here can see that, since a
@@ -4575,6 +4996,14 @@ class ValOpsFS extends ValOps {
             if (Date.now() - start > interval) {
               console.warn("Val: polling interval of files exceeded");
             }
+            // Checked BEFORE rescheduling, as the patches-directory poller
+            // above does. Without it this walk goes on forever: the `finally`
+            // that ends the race clears the handle it can see, and a `go`
+            // already on the queue then schedules one nothing will ever clear —
+            // a leaked timer, re-stat-ing the whole project, per stat poll.
+            if (stopPolling) {
+              return;
+            }
             setHandle(setTimeout(() => go(resolve), interval));
           };
           if (stopPolling) {
@@ -4587,6 +5016,11 @@ class ValOpsFS extends ValOps {
       const disableFilePolling = ((_this$options2 = this.options) === null || _this$options2 === void 0 ? void 0 : _this$options2.disableFilePolling) || false;
       let patchesDirHandle;
       let valFilesIntervalHandle;
+      // Held so the `finally` can clear it. A race the timeout LOSES still
+      // leaves its timer armed, and it is the long one — so every stat that
+      // returned on a file change kept the whole poll interval alive behind it,
+      // doing nothing but holding the event loop open.
+      let noChangeHandle;
       const type = await Promise.race([
       // we poll the patches directory for changes since fs.watch does not work reliably on all system (in particular on WSL) and just checking the patches dir is relatively cheap
       disableFilePolling ? new Promise(() => {}) : didDirectoryChangeUsingPolling(this.getPatchesDir(), statFilePollingInterval, handle => {
@@ -4619,7 +5053,7 @@ class ValOpsFS extends ValOps {
         });
       }), new Promise(resolve => {
         var _this$options3;
-        return setTimeout(() => resolve("no-change"), ((_this$options3 = this.options) === null || _this$options3 === void 0 ? void 0 : _this$options3.statPollingInterval) || 20000);
+        noChangeHandle = setTimeout(() => resolve("no-change"), ((_this$options3 = this.options) === null || _this$options3 === void 0 ? void 0 : _this$options3.statPollingInterval) || 20000);
       })]).finally(() => {
         if (fsWatcher) {
           fsWatcher.close();
@@ -4627,14 +5061,52 @@ class ValOpsFS extends ValOps {
         stopPolling = true;
         clearInterval(patchesDirHandle);
         clearInterval(valFilesIntervalHandle);
+        clearTimeout(noChangeHandle);
       });
+      /**
+       * Read the store AGAIN, because `patches` above describes the moment this
+       * poll OPENED — up to a whole polling interval ago.
+       *
+       * That staleness is not a detail: what ends the wait is a write, and the
+       * write the studio does most is `/save`, which in `fs` mode commits the
+       * patches and DELETES them. Answering with the list read before it names
+       * patches that no longer exist, and the studio then puts those ids back in
+       * its chain and fetches them from a server that correctly no longer has
+       * them — "unpublished changes could not be loaded", for changes that were
+       * published a moment earlier. With auto-save on, that is every pause in
+       * typing.
+       *
+       * So a stat describes the moment it ANSWERS. `request-again` and
+       * `no-change` differ in why the wait ended, not in how current the answer
+       * has to be, so both come through here.
+       *
+       * The patch list is the only part that CAN have moved: the shas and the
+       * schemas come from `initSources`, which is memoised for the lifetime of
+       * this instance and never invalidated — a module change makes a new
+       * `ValOpsFS` rather than updating this one. Recomputing them would be a
+       * second full schema serialization per poll for a value that cannot have
+       * changed.
+       *
+       * A read error is returned as one rather than papered over with the list
+       * from the open: a patch store that cannot be read is what the error path
+       * is for, and the studio asks again.
+       */
+      const answerRes = await this.readStore();
+      if (answerRes.status === "error") {
+        return {
+          type: "error",
+          error: {
+            message: answerRes.message
+          }
+        };
+      }
       return {
         type,
         baseSha: currentBaseSha,
         schemaSha: currentSchemaSha,
         sourcesSha: currentSourcesSha,
-        patches,
-        ...removedNotice,
+        patches: answerRes.entries.map(entry => entry.patchId),
+        ...drainRemoved(),
         jsonEntriesSha: currentJsonEntriesSha
       };
     } catch (err) {
@@ -4966,14 +5438,28 @@ class ValOpsFS extends ValOps {
   }
   async saveBase64EncodedBinaryFileFromPatch(filePath, _parentRef, patchId, data, _type, metadata) {
     // Keyed by the patch's own id, so the parent is not needed and is not asked
-    // for. Uploads arrive before the patch record does, which is fine: the
-    // directory sits there unreferenced until the log line that names it lands,
-    // and repair sweeps it up if that never happens.
+    // for.
+    //
+    // Written OUTSIDE the store, and moved in by `appendPatch` once the record
+    // exists. Uploads arrive before the patch record does — the record's `file`
+    // op carries only a sha, so it would otherwise point at nothing — and
+    // writing them straight into `<patchId>/files/` left a directory holding
+    // files and no `patch.json` for a whole round trip. Nothing can read that as
+    // anything but a patch whose contents are lost, so repair removed it, bytes
+    // and all, and the image 404ed.
+    //
+    // Staging is what makes that state unreachable rather than tolerated. See
+    // `uploadsDir`.
     const patchesDir = this.getPatchesDir();
-    const patchFilePath = patchBinaryFile(patchesDir, patchId, filePath);
-    const metadataFilePath = patchBinaryFileMetadata(patchesDir, patchId, filePath);
+    const patchFilePath = stagedPatchBinaryFile(patchesDir, patchId, filePath);
+    const metadataFilePath = stagedPatchBinaryFileMetadata(patchesDir, patchId, filePath);
     try {
       if (data === null) {
+        // A delete is the other order — the record is written first — so the
+        // bytes are already in the store. Both locations, because a file staged
+        // and then removed before its record never got there.
+        this.host.deleteFile(patchBinaryFile(patchesDir, patchId, filePath));
+        this.host.deleteFile(patchBinaryFileMetadata(patchesDir, patchId, filePath));
         this.host.deleteFile(patchFilePath);
         this.host.deleteFile(metadataFilePath);
         return {
@@ -4981,6 +5467,9 @@ class ValOpsFS extends ValOps {
           filePath
         };
       }
+      // Cheap, and this is the one path that creates staging directories, so it
+      // is where the ones nobody claimed get noticed.
+      sweepStaleUploads(patchesDir);
       const buffer = bufferFromDataUrl(data);
       if (!buffer) {
         return {
@@ -5010,9 +5499,28 @@ class ValOpsFS extends ValOps {
       };
     }
   }
+
+  /**
+   * Which of the two places a patch's file can be, if either.
+   *
+   * The bytes are in the store once the patch's record is, and in the staging
+   * area before that — see `uploadsDir`. Every reader has to accept both, and
+   * they decide it here rather than each on its own, so two readers of the same
+   * file cannot disagree about whether it exists.
+   */
+  wherePatchFileIs(inStore, staged) {
+    if (this.host.fileExists(inStore)) {
+      return inStore;
+    }
+    if (this.host.fileExists(staged)) {
+      return staged;
+    }
+    return null;
+  }
   async getBase64EncodedBinaryFileMetadataFromPatch(filePath, type, patchId) {
-    const metadataFilePath = patchBinaryFileMetadata(this.getPatchesDir(), patchId, filePath);
-    if (!this.host.fileExists(metadataFilePath)) {
+    const patchesDir = this.getPatchesDir();
+    const metadataFilePath = this.wherePatchFileIs(patchBinaryFileMetadata(patchesDir, patchId, filePath), stagedPatchBinaryFileMetadata(patchesDir, patchId, filePath));
+    if (metadataFilePath === null) {
       return {
         errors: [{
           message: "Metadata file not found",
@@ -5049,8 +5557,9 @@ class ValOpsFS extends ValOps {
   async getBase64EncodedBinaryFileFromPatch(filePath, patchId) {
     // Straight from the id. This used to read and parse every patch on disk to
     // work out which directory the file was under, on every single image request.
-    const absPath = patchBinaryFile(this.getPatchesDir(), patchId, filePath);
-    if (!this.host.fileExists(absPath)) {
+    const patchesDir = this.getPatchesDir();
+    const absPath = this.wherePatchFileIs(patchBinaryFile(patchesDir, patchId, filePath), stagedPatchBinaryFile(patchesDir, patchId, filePath));
+    if (absPath === null) {
       return null;
     }
     return this.host.readBinaryFile(absPath);
@@ -5095,6 +5604,9 @@ class ValOpsFS extends ValOps {
             recursive: true,
             force: true
           });
+          // And anything this patch had staged but never moved in, which is
+          // the case where a delete arrives before the record does.
+          removeStagedUploads(patchesDir, patchId);
           deleted.push(patchId);
         } catch (err) {
           // Reported. This endpoint used to answer "deleted" unconditionally —
@@ -8325,6 +8837,26 @@ const ValServer = (valModules, options, callbacks) => {
               }
             };
           }
+          /*
+           * The files on disk are the committed content now, so say so here too.
+           *
+           * Nothing else will: the sources are memoised per `ValOps` instance
+           * and are re-read by awaiting each module's `def`, which is the app's
+           * own `import()` — that resolves from the module registry, not from
+           * the file this save just rewrote. So until the host rebuilds its
+           * module graph, every read of committed content answers with what was
+           * there before the save. A page rendering draft content sees exactly
+           * that once the patches below are gone: `getJsonEntry` resolves the
+           * committed entry and has no patches left to replay over it.
+           *
+           * Before `deletePatches`, because it needs the patches it is adopting
+           * the result of. See `ValOps.promoteCommittedSources` for why the SHAs
+           * move with them.
+           */
+          await serverOps.adoptCommittedSources({
+            ...analysis,
+            ...patches
+          }, preparedCommit);
           /*
            * Only what this request consumed.
            *