@joystack/cli 0.1.45 → 0.1.46

package/lib/freeze.mjs

@@ -1,15 +1,17 @@
-// `joystack dev --no-reload`: the game as it is at startup, for the whole run.
+// A copy of the game as it is at one moment: one version of it, as `joystack
+// dev` serves it (live-versions.mjs).
 //
-// The dev server reads a game's files from disk on every request and rebuilds
-// its room when they change, so switching the watchers off alone does not
-// freeze anything: a page opened after an edit gets the edited files, while the
-// room and the injected entity layout stay as they were, and an edit that adds
-// an entity makes that page fail to boot. Freezing therefore serves, and builds
-// the room from, a copy of the game taken once at startup.
-import { constants, cpSync, existsSync, mkdirSync, readdirSync, rmSync, symlinkSync, unlinkSync } from 'node:fs';
+// Serving the game's own folder cannot give a page one version: the page
+// fetches files for as long as it runs, while the room and the injected entity
+// layout are built once, so an edit lands in some of them and not the others.
+// A copy is served and built from instead, and replaced by a new copy, never
+// edited.
+import { constants, copyFileSync, cpSync, existsSync, linkSync, mkdirSync, readdirSync, rmSync, symlinkSync, unlinkSync } from 'node:fs';
 import { randomBytes } from 'node:crypto';
-import { basename, join } from 'node:path';
+import { basename, dirname, isAbsolute, join, relative, sep } from 'node:path';
+import { collectGameFiles } from './game-files.mjs';
 import { gameBuild } from './build-id.mjs';
+import { snapshotFiles } from './live-versions.mjs';
 import { STATE_DIR_NAMES, stateDir } from './legacy-names.mjs';
 
 // Left out of the copy: recordings and generated build output (.joystack), which
@@ -27,21 +29,39 @@ export function frozenRoot(gameDir) {
   return join(stateDir(gameDir), 'frozen');
 }
 
+// The files a copy of the game at `dir` is taken from, as path → `mtime:size`.
+//
+// Small files also carry a hash of their content. On Windows a file's
+// modified time moves in steps of about 15 ms, so a rewrite to the same size
+// within one step would otherwise go unseen; large files are left to time and
+// size, since reading them on every poll would cost more than it finds.
+export function frozenFiles(dir) {
+  return snapshotFiles([dir], (name, depth) => depth === 0 && LEFT_OUT.has(name), { hashBelow: 256 * 1024 });
+}
+
 /**
  * Copies the game at `dir` into `.joystack/frozen/<pid>-<random>/<folder name>`.
- * `build` is the game's build at the moment of the copy, the one every session
- * played on the copy is stamped with. `remove()` deletes the copy; it also runs
- * when the process exits normally, and a copy whose process died without
- * running it is swept by the next freeze of the same game.
+ * `build()` is the game's build as the copy has it, the one every session
+ * played on the copy is stamped with; it is worked out the first time it is
+ * asked for. `remove()` deletes the copy; it also runs when the process exits
+ * normally, and a copy whose process died without running it is swept by the
+ * next freeze of the same game.
+ *
+ * `files` is the snapshot the copy is taken from (frozenFiles). With `from`, a
+ * previous copy and its snapshot, a file whose entry is the same in both is
+ * hard-linked to the previous copy's instead of copied, so a copy costs the
+ * files that changed. Copies are never written to after they are made, which
+ * is what makes sharing their files safe; the game's own files are always
+ * copied, never linked, since the game's are edited in place.
  *
  * Symlinks inside the game are copied as what they point to, so a linked
  * folder is frozen with the rest.
  * @param {string} dir
- * @returns {{ dir: string, build: object, remove: () => void }}
+ * @param {{ files?: Map<string, string>, from?: { dir: string, files: Map<string, string> } }} [opts]
+ * @returns {{ dir: string, build: () => object, remove: () => void }}
  */
-export function freezeGame(dir) {
+export function freezeGame(dir, { files = null, from = null } = {}) {
   sweepStale(dir);
-  const build = gameBuild(dir);
   const run = join(frozenRoot(dir), `${process.pid}-${randomBytes(3).toString('hex')}`);
   // The folder name is the game's slug unless package.json names one, so the
   // copy keeps it.
@@ -50,21 +70,54 @@ export function freezeGame(dir) {
   let removed = false;
   const remove = () => {
     if (removed) return;
-    removed = true;
-    process.off('exit', remove);
     // The link first, on its own: on Windows it is a junction, and a recursive
     // delete that followed it would empty the game's real node_modules.
     try { unlinkSync(modulesLink); } catch { /* never made */ }
-    rmSync(run, { recursive: true, force: true });
+    // On Windows a file a process still has open cannot be deleted, and a
+    // stopped room server lets go of its files a moment after it exits. A copy
+    // that still cannot be deleted is tried again when this process exits,
+    // and swept by the next `joystack dev` of the game after that.
+    try {
+      rmSync(run, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 });
+    } catch {
+      return;
+    }
+    removed = true;
+    process.off('exit', remove);
   };
   process.on('exit', remove);
   try {
     mkdirSync(copy, { recursive: true });
-    // Entry by entry, because the copy lives inside the game (under .joystack),
-    // and a copy of a folder into its own subfolder is refused outright.
-    for (const entry of readdirSync(dir)) {
-      if (LEFT_OUT.has(entry)) continue;
-      cpSync(join(dir, entry), join(copy, entry), { recursive: true, dereference: true, mode: constants.COPYFILE_FICLONE });
+    if (files) {
+      const made = new Set();
+      for (const [path, entry] of files) {
+        const rel = relative(dir, path);
+        if (rel.startsWith('..') || isAbsolute(rel)) continue;
+        // The snapshot also names files the copy leaves out: an engine
+        // installed in the game's node_modules is watched along with the game,
+        // and the copy reaches it through the node_modules link made below.
+        if (LEFT_OUT.has(rel.split(sep)[0])) continue;
+        const to = join(copy, rel);
+        const parent = dirname(to);
+        if (!made.has(parent)) {
+          mkdirSync(parent, { recursive: true });
+          made.add(parent);
+        }
+        if (from && from.files.get(path) === entry) {
+          try {
+            linkSync(join(from.dir, rel), to);
+            continue;
+          } catch { /* a disk without hard links, or the previous file is gone: copied below */ }
+        }
+        copyFileSync(path, to, constants.COPYFILE_FICLONE);
+      }
+    } else {
+      // Entry by entry, because the copy lives inside the game (under
+      // .joystack), and a copy of a folder into its own subfolder is refused.
+      for (const entry of readdirSync(dir)) {
+        if (LEFT_OUT.has(entry)) continue;
+        cpSync(join(dir, entry), join(copy, entry), { recursive: true, dereference: true, mode: constants.COPYFILE_FICLONE });
+      }
     }
     const modules = join(dir, 'node_modules');
     if (existsSync(modules)) symlinkSync(modules, modulesLink, process.platform === 'win32' ? 'junction' : 'dir');
@@ -72,7 +125,10 @@ export function freezeGame(dir) {
     remove();
     throw err;
   }
-  return { dir: copy, build, remove };
+  // The id is the copy's files; the commit and whether the files differ from
+  // it are read from the game's folder when the build is first asked for.
+  let build = null;
+  return { dir: copy, build: () => (build ??= gameBuild(dir, { files: collectGameFiles(copy) })), remove };
 }
 
 // Deletes the copies of runs whose process no longer exists: a `joystack dev`

package/lib/live-versions.mjs

@@ -0,0 +1,212 @@
+// The versions `joystack dev` serves.
+//
+// A version is a copy of the game taken at one moment (freeze.mjs), with the
+// room and the entity layout built from that copy. The page, every file it
+// fetches later and the room all come from one version, so a page never runs
+// against files that changed under it.
+//
+// This module decides when a new version is made and keeps the state every
+// page and the control endpoints report:
+//
+//   state    'ready'     the current version is served
+//            'updating'  a new version is being made; game pages show "Updating…"
+//            'failed'    the last attempt failed; game pages show `error`
+//   auto     whether a version is made as soon as the files settle
+//   pending  whether the files differ from the last version attempted
+//
+// The operating system says when a file in a watched folder changes, and a
+// version is made once no change has arrived for `quietMs`. The events only
+// say when to look: what a version is made from, and whether anything is
+// pending, is always a fresh snapshot of the files. A slower poll covers
+// changes the events miss (a folder the system cannot watch, an engine
+// checkout); a poll counts the files as settled when it finds what the
+// previous poll found.
+import { createHash } from 'node:crypto';
+import { readdirSync, readFileSync, statSync, watch as watchFolder } from 'node:fs';
+import { join } from 'node:path';
+
+// Every file under each root, as path → `mtime:size`, with a hash of the
+// content appended for files smaller than `hashBelow` bytes. `skip(name,
+// depth)` leaves an entry out; depth 0 is a root's own entries.
+export function snapshotFiles(roots, skip = () => false, { hashBelow = 0 } = {}) {
+  const out = new Map();
+  const walk = (dir, depth) => {
+    let names;
+    try {
+      names = readdirSync(dir);
+    } catch {
+      return;
+    }
+    for (const name of names) {
+      if (skip(name, depth)) continue;
+      const p = join(dir, name);
+      let st;
+      try {
+        st = statSync(p);
+      } catch {
+        continue;
+      }
+      if (st.isDirectory()) walk(p, depth + 1);
+      else out.set(p, `${st.mtimeMs}:${st.size}${st.size < hashBelow ? `:${contentHash(p)}` : ''}`);
+    }
+  };
+  for (const root of roots) walk(root, 0);
+  return out;
+}
+
+function contentHash(path) {
+  try {
+    return createHash('sha1').update(readFileSync(path)).digest('base64');
+  } catch {
+    return 'unreadable';
+  }
+}
+
+export function sameFiles(a, b) {
+  if (!a || !b || a.size !== b.size) return false;
+  for (const [k, v] of a) if (b.get(k) !== v) return false;
+  return true;
+}
+
+/**
+ * @param {object} o
+ * @param {() => Map} o.snapshot      the files a version is made from, now
+ * @param {(o: { files: Map, previous: object }) => Promise<object>} o.make
+ *        makes a version from the files as they are; `files` is the snapshot
+ *        taken just before, `previous` the version served now
+ * @param {(v: object) => Promise<void>} o.show  makes `v` the served version; throws to fail it
+ * @param {(v: object) => void} o.retire   the previous version, once it is no longer served
+ * @param {object} o.first          the version served at startup, with its `files` snapshot
+ * @param {{ dir: string, ignore?: (rel: string) => boolean }[]} [o.watch]
+ *        folders whose change events start a version; `rel` is the changed
+ *        path within the folder, with `/` separators
+ */
+export function createLiveVersions({ snapshot, make, show, retire, first, auto = true, watch = [], quietMs = 50, intervalMs = 1000, log = () => {}, onStatus = () => {} }) {
+  let current = first;
+  // The files the last attempt was made from, whether it was shown or failed.
+  let attempted = first.files;
+  let last = null;
+  let busy = false;
+  let closed = false;
+  let wanted = false;
+  let quiet = null;
+  let status = { state: 'ready', auto, pending: false, error: null };
+
+  const set = (next) => {
+    const changed = Object.keys(next).some((k) => status[k] !== next[k]);
+    status = { ...status, ...next };
+    if (changed) onStatus(status);
+  };
+
+  async function update() {
+    if (busy) {
+      wanted = true;
+      return;
+    }
+    busy = true;
+    try {
+      do {
+        wanted = false;
+        set({ state: 'updating', error: null });
+        const before = snapshot();
+        let version = null;
+        let error = null;
+        try {
+          version = await make({ files: before, previous: current });
+        } catch (err) {
+          error = err;
+        }
+        if (closed) {
+          version?.remove();
+          return;
+        }
+        // Files that changed while the copy was taken may have been copied
+        // half-written: the attempt does not count, and is made again once
+        // they settle.
+        if (!sameFiles(before, snapshot())) {
+          version?.remove();
+          await new Promise((r) => setTimeout(r, quietMs));
+          wanted = true;
+          continue;
+        }
+        attempted = before;
+        if (version) {
+          version.files = before;
+          try {
+            await show(version);
+          } catch (err) {
+            error = err;
+            version.remove();
+            version = null;
+          }
+        }
+        if (closed) return;
+        if (version) {
+          const previous = current;
+          current = version;
+          retire(previous);
+          set({ state: 'ready', error: null, pending: false });
+          log('→ reloaded: serving the files as they are now');
+        } else {
+          const message = String(error?.message ?? error);
+          set({ state: 'failed', error: message, pending: false });
+          log(`✗ the game does not build — the page shows this until the files change:\n${message}`);
+        }
+      } while (wanted && !closed);
+    } finally {
+      busy = false;
+    }
+    // Changes that arrived while this update ran.
+    check({ settled: true });
+  }
+
+  // Compares the files with the last attempt; `settled` says they have
+  // stopped changing.
+  function check({ settled }) {
+    if (closed || busy) return;
+    const now = snapshot();
+    const stable = settled || sameFiles(now, last);
+    last = now;
+    const pending = !sameFiles(now, attempted);
+    if (pending !== status.pending) set({ pending });
+    if (status.auto && pending && stable) update();
+  }
+
+  const watchers = [];
+  for (const { dir, ignore = () => false } of watch) {
+    try {
+      const w = watchFolder(dir, { recursive: true }, (_, name) => {
+        if (name && ignore(String(name).replace(/\\/g, '/'))) return;
+        clearTimeout(quiet);
+        quiet = setTimeout(() => check({ settled: true }), quietMs);
+      });
+      w.on('error', () => w.close());
+      watchers.push(w);
+    } catch { /* this folder is left to the poll */ }
+  }
+  const timer = setInterval(() => check({ settled: false }), intervalMs);
+  timer.unref();
+
+  return {
+    get status() {
+      return status;
+    },
+    get current() {
+      return current;
+    },
+    // Makes a version of the files as they are now, whatever the mode.
+    reload() {
+      return update();
+    },
+    setAuto(on) {
+      set({ auto: !!on });
+      if (on) check({ settled: true });
+    },
+    close() {
+      closed = true;
+      clearInterval(timer);
+      clearTimeout(quiet);
+      for (const w of watchers) w.close();
+    },
+  };
+}