@mmerterden/multi-agent-pipeline 20.8.1 → 20.8.2

package/install/_common.mjs

@@ -1,10 +1,9 @@
 /**
  * Shared filesystem helpers for install/* modules.
  *
- * These primitives are 1:1 ports of the helpers that lived inline in the
- * pre-v8.0.0 monolithic install.js. Behaviour MUST stay byte-equivalent -
- * the install layout smoke test (smoke-install-layout.sh) compares before/after
- * trees and fails on any drift.
+ * The install layout smoke (smoke-install-layout.sh) compares the installed
+ * tree against a committed fixture, so a helper that changes what lands on
+ * disk changes that fixture too.
  *
  * @module install/_common
  */
@@ -25,8 +24,9 @@ import {
   unlinkSync,
   writeFileSync,
 } from "fs";
-import { dirname } from "path";
-import { join } from "path";
+import { createHash } from "crypto";
+import { dirname, join, relative, sep } from "path";
+import { fileURLToPath } from "url";
 
 let dryRun = false;
 
@@ -55,32 +55,476 @@ export function ensureDir(dir) {
   if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
 }
 
+const PACKAGE_ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
+const PACKAGE_NAME = (() => {
+  try {
+    return JSON.parse(readFileSync(join(PACKAGE_ROOT, "package.json"), "utf-8")).name || "";
+  } catch {
+    return "";
+  }
+})();
+
+function isWithin(path, dir) {
+  const d = dir.endsWith(sep) ? dir : dir + sep;
+  return path === dir || path.startsWith(d);
+}
+
+function realOr(p) {
+  try {
+    return realpathSync(p);
+  } catch {
+    return p;
+  }
+}
+
+/**
+ * True when `real` lies inside a pipeline source tree: the one this install
+ * runs from (`pipelineSrc`'s package root), or any checkout of this package,
+ * recognised by an ancestor package.json carrying the package name. A --link
+ * install points its trees at a checkout, and that checkout need not be the
+ * one the current install runs from.
+ *
+ * @param {string} real resolved path
+ * @param {string} [pipelineSrc]
+ * @returns {boolean}
+ */
+export function isInsidePipelineSource(real, pipelineSrc) {
+  const roots = [PACKAGE_ROOT];
+  if (pipelineSrc) roots.push(dirname(realOr(pipelineSrc)), realOr(pipelineSrc));
+  if (roots.some((r) => isWithin(real, realOr(r)))) return true;
+  if (!PACKAGE_NAME) return false;
+  for (let dir = real; dir !== dirname(dir); dir = dirname(dir)) {
+    try {
+      const pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf-8"));
+      if (pkg?.name === PACKAGE_NAME) return true;
+    } catch {
+      /* no package.json here */
+    }
+  }
+  return false;
+}
+
 /**
- * Replace a symlinked path with a real (empty) directory before installing
- * into it. After a `--link` install, an owned dir (or a shared parent like
- * `~/.claude/commands`) can be a symlink into the developer's repo checkout;
- * wiping or copying through it would land inside the repo and destroy the
- * pipeline sources. Non-symlink paths are left untouched.
+ * What an install target currently is.
  *
  * @param {string} dir
- * @returns {boolean} true when a symlink was replaced
+ * @param {string} [pipelineSrc]
+ * @returns {"missing"|"real"|"pipeline-link"|"foreign-link"}
  */
-export function ensureRealDir(dir) {
+export function classifyInstallDir(dir, pipelineSrc) {
   let st;
   try {
     st = lstatSync(dir);
   } catch {
-    return false;
+    return "missing";
+  }
+  if (!st.isSymbolicLink()) return "real";
+  let real;
+  try {
+    real = realpathSync(dir);
+  } catch {
+    // A dangling link points at nothing, so replacing it loses nothing.
+    return "pipeline-link";
+  }
+  return isInsidePipelineSource(real, pipelineSrc) ? "pipeline-link" : "foreign-link";
+}
+
+/**
+ * Resolve the directory an install step should write into.
+ *
+ * - A real directory, or a missing one: returned as is.
+ * - A symlink into a pipeline source tree (left by `--link`): replaced with an
+ *   empty real directory, because wiping or copying through it would land in
+ *   the checkout and destroy the sources.
+ * - A symlink anywhere else belongs to the user (a dotfiles repo, typically).
+ *   For a user-owned tree (`rules/`, `agents/`, `commands/`) the link target
+ *   is returned and the install writes through it; for a pipeline-managed
+ *   tree the link is left alone, a warning is printed and null is returned so
+ *   the caller skips the tree.
+ *
+ * @param {string} dir
+ * @param {{pipelineSrc?: string, userOwned?: boolean}} [opts]
+ * @returns {string|null} the path to write into, or null to skip
+ */
+export function ensureRealDir(dir, { pipelineSrc, userOwned = false } = {}) {
+  const kind = classifyInstallDir(dir, pipelineSrc);
+  if (kind === "missing" || kind === "real") return dir;
+  if (kind === "foreign-link") {
+    const target = realOr(dir);
+    if (userOwned) {
+      console.log(`    -> ${dir} is a symlink to ${target}; installing through it`);
+      return target;
+    }
+    console.log(
+      `    -> WARNING: skipped ${dir}: it is a symlink to ${target}, outside the pipeline source. ` +
+        `Remove the link (or point it at a pipeline checkout) and re-run the install to update it.`,
+    );
+    return null;
   }
-  if (!st.isSymbolicLink()) return false;
   if (dryRun) {
     console.log(`    [dry-run] would replace symlink ${dir} with a real directory`);
-    return true;
+    return dir;
   }
   unlinkSync(dir);
   mkdirSync(dir, { recursive: true });
-  console.log(`    -> replaced symlink ${dir} with a real directory (was --link mode)`);
-  return true;
+  console.log(
+    `    -> replaced symlink ${dir} with a real directory (it pointed into a pipeline checkout)`,
+  );
+  return dir;
+}
+
+/**
+ * Raised when an install cannot finish. `index.mjs` turns it into one line and
+ * a non-zero exit; the stack trace prints only with MULTI_AGENT_INSTALL_DEBUG=1.
+ */
+export class InstallIncompleteError extends Error {
+  /**
+   * @param {string} message
+   * @param {{cause?: unknown, untouched?: boolean}} [opts]
+   */
+  constructor(message, { cause, untouched = false } = {}) {
+    super(message);
+    this.name = "InstallIncompleteError";
+    this.code = "INSTALL_INCOMPLETE";
+    this.untouched = untouched;
+    if (cause) this.cause = cause;
+  }
+}
+
+function describeFsError(err) {
+  if (!err) return "unknown error";
+  if (err.code && err.path) return `${err.code} on ${err.path}`;
+  return String(err.message || err).split("\n")[0];
+}
+
+/**
+ * Stage every managed tree under `swapRoot` (a hidden directory of the host
+ * dir), then swap them all in by rename. Staging lives outside the managed
+ * trees so a half-written or undeletable copy never sits inside `commands/`,
+ * where the host would load it as a command namespace. Nothing at a
+ * destination changes until every tree staged cleanly; a failed swap puts
+ * back the trees already swapped. Previous copies wait under `swapRoot` until
+ * every swap succeeded.
+ *
+ * A leftover staged copy from an interrupted run is discarded, a leftover
+ * previous copy is restored when its destination is missing, and anything
+ * that cannot be deleted is moved aside inside `swapRoot`.
+ *
+ * @param {Array<{dest: string, label: string, build: (staged: string) => void}>} trees
+ * @param {{swapRoot: string}} opts
+ * @throws {InstallIncompleteError}
+ */
+export function stageAndSwapTrees(trees, { swapRoot }) {
+  if (dryRun) {
+    for (const t of trees) console.log(`    [dry-run] would stage and swap in ${t.dest}`);
+    return;
+  }
+  const paths = new Map(trees.map(({ dest }) => [dest, swapPaths(swapRoot, dest)]));
+  mkdirSync(swapRoot, { recursive: true });
+  for (const { dest } of trees) recoverInterruptedSwap(dest, paths.get(dest));
+
+  const staged = [];
+  try {
+    for (const t of trees) {
+      const tmp = paths.get(t.dest).staged;
+      staged.push(tmp);
+      t.build(tmp);
+    }
+  } catch (err) {
+    for (const tmp of staged) rmQuiet(tmp);
+    throw new InstallIncompleteError(
+      `could not stage the new files: ${describeFsError(err)}. The previous install was left in place`,
+      { cause: err, untouched: true },
+    );
+  }
+
+  const swapped = [];
+  try {
+    for (const { dest } of trees) {
+      const { staged: tmp, old } = paths.get(dest);
+      const had = pathExists(dest);
+      mkdirSync(dirname(dest), { recursive: true });
+      if (had) renameSync(dest, old);
+      try {
+        renameSync(tmp, dest);
+      } catch (err) {
+        if (had) renameSync(old, dest);
+        throw err;
+      }
+      swapped.push({ dest, had, old });
+    }
+  } catch (err) {
+    for (const { dest, had, old } of swapped.reverse()) {
+      try {
+        rmSync(dest, { recursive: true, force: true });
+        if (had) renameSync(old, dest);
+      } catch {
+        /* best effort: the next run's recovery restores a leftover previous copy */
+      }
+    }
+    for (const tmp of staged) rmQuiet(tmp);
+    throw new InstallIncompleteError(
+      `could not swap in the new files: ${describeFsError(err)}. The previous install was restored`,
+      { cause: err, untouched: true },
+    );
+  }
+
+  for (const { had, old } of swapped) {
+    if (!had) continue;
+    try {
+      rmSync(old, { recursive: true, force: true });
+    } catch (err) {
+      console.log(
+        `    -> note: could not remove ${old} (${describeFsError(err)}); ` +
+          "the next install moves it aside",
+      );
+    }
+  }
+  try {
+    if (readdirSync(swapRoot).length === 0) rmSync(swapRoot, { recursive: true, force: true });
+  } catch {
+    /* best effort */
+  }
+}
+
+/**
+ * Staging and previous-copy paths for one destination: the destination's path
+ * relative to the host dir, flattened into one name under `swapRoot`.
+ *
+ * @param {string} swapRoot
+ * @param {string} dest
+ * @returns {{staged: string, old: string}}
+ */
+export function swapPaths(swapRoot, dest) {
+  const key = relative(dirname(swapRoot), dest).split(sep).join("__");
+  return { staged: join(swapRoot, `${key}.new`), old: join(swapRoot, `${key}.old`) };
+}
+
+function pathExists(p) {
+  try {
+    lstatSync(p);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function rmQuiet(p) {
+  try {
+    rmSync(p, { recursive: true, force: true });
+  } catch {
+    /* best effort */
+  }
+}
+
+function recoverInterruptedSwap(dest, { staged, old }) {
+  rmQuiet(staged);
+  rmQuiet(`${dest}.new`);
+  for (const prev of [old, `${dest}.old`]) {
+    if (!pathExists(prev)) continue;
+    if (!pathExists(dest)) {
+      try {
+        renameSync(prev, dest);
+        continue;
+      } catch {
+        /* fall through and clear it */
+      }
+    }
+    rmQuiet(prev);
+    if (!pathExists(prev)) continue;
+    // Something inside it cannot be deleted (a read-only subdirectory). Move it
+    // under the swap root so it is out of the managed trees, and say where.
+    const aside = `${old}-${Date.now()}`;
+    try {
+      renameSync(prev, aside);
+      console.log(`    -> note: could not delete ${prev}; moved it to ${aside}, remove it by hand`);
+    } catch {
+      /* the swap below reports the failure */
+    }
+  }
+}
+
+/** File name of the install manifest, at the root of the host dir. */
+export const INSTALL_MANIFEST = ".pipeline-manifest.json";
+
+/**
+ * @param {string} path
+ * @returns {string} sha256 of the file's bytes
+ */
+export function hashFile(path) {
+  return createHash("sha256").update(readFileSync(path)).digest("hex");
+}
+
+/**
+ * The install manifest: every co-owned file the installer wrote, keyed by its
+ * path relative to the host dir, with the hash it had when written, plus the
+ * unknown files already reported so each is reported once.
+ *
+ * @param {string} hostDir e.g. `$HOME/.claude`
+ * @returns {{present: boolean, files: Record<string,string>, warned: string[]}}
+ */
+export function readInstallManifest(hostDir) {
+  try {
+    const doc = JSON.parse(readFileSync(join(hostDir, INSTALL_MANIFEST), "utf-8"));
+    return {
+      present: true,
+      files: doc && typeof doc.files === "object" && doc.files ? { ...doc.files } : {},
+      warned: Array.isArray(doc?.warned) ? doc.warned.filter((x) => typeof x === "string") : [],
+    };
+  } catch {
+    return { present: false, files: {}, warned: [] };
+  }
+}
+
+/**
+ * @param {string} hostDir
+ * @param {{files: Record<string,string>, warned: string[]}} manifest
+ */
+export function writeInstallManifest(hostDir, manifest) {
+  const doc = {
+    schema: 1,
+    files: Object.fromEntries(
+      Object.entries(manifest.files).sort(([a], [b]) => a.localeCompare(b)),
+    ),
+    warned: [...new Set(manifest.warned)].sort(),
+  };
+  writeFile(join(hostDir, INSTALL_MANIFEST), JSON.stringify(doc, null, 2) + "\n");
+}
+
+/**
+ * Relative paths of every regular file under `dir`, junk excluded.
+ * @param {string} dir
+ * @returns {string[]}
+ */
+export function listFiles(dir) {
+  const out = [];
+  const walk = (abs, rel) => {
+    let entries;
+    try {
+      entries = readdirSync(abs, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const e of entries) {
+      const r = rel ? `${rel}/${e.name}` : e.name;
+      if (isCopyJunk(r)) continue;
+      if (e.isDirectory()) walk(join(abs, e.name), r);
+      else if (e.isFile()) out.push(r);
+    }
+  };
+  walk(dir, "");
+  return out.sort();
+}
+
+/**
+ * Bring the pipeline's files in a co-owned directory up to date without
+ * touching anything the user owns.
+ *
+ * A shipped file is written when it is absent, or when its current hash is the
+ * one the manifest recorded (unchanged since the last install). `adopt` covers
+ * the first run after an install that predates the manifest: that installer
+ * overwrote these files on every run, so one carrying a shipped name is its
+ * copy. Any other existing file is the user's: it is kept and reported. A file
+ * the manifest records but the source no longer ships is removed when it is
+ * still unchanged. Files neither shipped nor recorded are the user's; with
+ * `reportUnknown` they are reported once.
+ *
+ * @param {{
+ *   srcDir: string,
+ *   destDir: string,
+ *   displayDir?: string,
+ *   rels: string[],
+ *   prefix: string,
+ *   manifest: {files: Record<string,string>, warned: string[]},
+ *   adopt: boolean,
+ *   reportUnknown?: boolean,
+ *   label: string,
+ * }} opts
+ * @returns {{written: number, kept: string[], removed: number}}
+ */
+export function syncTrackedFiles(opts) {
+  const { srcDir, destDir, rels, prefix, manifest, adopt, reportUnknown = false, label } = opts;
+  const shownDir = opts.displayDir || destDir;
+  const shipped = new Set(rels);
+  const kept = [];
+  let written = 0;
+  let removed = 0;
+  const warn = (key, text) => {
+    if (manifest.warned.includes(key)) return;
+    manifest.warned.push(key);
+    console.log(`    -> WARNING: ${text}`);
+  };
+  for (const rel of rels) {
+    const key = `${prefix}${rel}`;
+    const from = join(srcDir, rel);
+    const to = join(destDir, rel);
+    const want = hashFile(from);
+    let have;
+    try {
+      have = lstatSync(to).isFile() ? hashFile(to) : "not-a-file";
+    } catch {
+      have = null;
+    }
+    const recorded = manifest.files[key];
+    if (have === want) {
+      manifest.files[key] = want;
+      continue;
+    }
+    if (have === null || have === recorded || (adopt && recorded === undefined)) {
+      if (dryRun) {
+        console.log(`    [dry-run] would write ${join(shownDir, rel)}`);
+      } else {
+        mkdirSync(dirname(to), { recursive: true });
+        cpSync(from, to, { force: true });
+      }
+      manifest.files[key] = want;
+      manifest.warned = manifest.warned.filter((w) => w !== key);
+      written++;
+      continue;
+    }
+    delete manifest.files[key];
+    kept.push(rel);
+    warn(
+      key,
+      `kept your ${join(shownDir, rel)}: it differs from what the pipeline installed there. ` +
+        `Delete it and re-run the install to get the ${label} version.`,
+    );
+  }
+  for (const [key, hash] of Object.entries(manifest.files)) {
+    if (!key.startsWith(prefix)) continue;
+    const rel = key.slice(prefix.length);
+    if (shipped.has(rel)) continue;
+    const to = join(destDir, rel);
+    let have;
+    try {
+      have = hashFile(to);
+    } catch {
+      have = null;
+    }
+    if (have === hash) {
+      if (dryRun) console.log(`    [dry-run] would remove ${join(shownDir, rel)}`);
+      else rmSync(to, { force: true });
+      removed++;
+    }
+    delete manifest.files[key];
+  }
+  if (reportUnknown && existsSync(destDir)) {
+    const unknown = listFiles(destDir).filter(
+      (rel) => !shipped.has(rel) && manifest.files[`${prefix}${rel}`] === undefined,
+    );
+    const fresh = unknown.filter((rel) => !manifest.warned.includes(`${prefix}${rel}`));
+    if (fresh.length > 0) {
+      for (const rel of fresh) manifest.warned.push(`${prefix}${rel}`);
+      console.log(
+        `    -> note: kept ${fresh.length} file(s) in ${shownDir} the pipeline did not install: ` +
+          fresh.join(", "),
+      );
+    }
+    manifest.warned = manifest.warned.filter(
+      (w) => !w.startsWith(prefix) || existsSync(join(destDir, w.slice(prefix.length))),
+    );
+  }
+  return { written, kept, removed };
 }
 
 /**
@@ -105,8 +549,8 @@ function pipelineAgentFileNames(agentsSrc) {
 
 /**
  * Remove ONLY pipeline-owned agent files from an installed agents dir.
- * The dir also holds user-authored agent definitions, so it must never be
- * wiped wholesale (that destroyed user files pre-v11.4.1).
+ * The dir also holds user-authored agent definitions, so it is never wiped
+ * wholesale.
  *
  * @param {string} destDir - installed agents dir (~/.claude/agents etc.)
  * @param {string} agentsSrc - pipeline/agents source directory
@@ -134,25 +578,15 @@ export function removePipelineAgentFiles(destDir, agentsSrc) {
   return removed;
 }
 
-/**
- * Recursively copy a directory (or symlink it in `--link` mode).
- *
- * @param {string} src
- * @param {string} dest
- * @param {{ exclude?: Array<string|RegExp>, useSymlinks?: boolean }} [opts]
- */
 /**
  * Build and editor droppings that live inside a source tree and must never
  * reach a user's install. Python writes `__pycache__/<mod>.cpython-NNN.pyc`
- * next to any script it IMPORTS, so a tree that has merely been used has them;
+ * next to any script it imports, so a tree that has merely been used has them;
  * the Finder writes .DS_Store into any directory it has displayed.
  *
- * This is not cosmetic. The installed file count is a gate in this repo, and a
- * CI runner on Python 3.14 shipped 147 files where every developer tree shipped
- * 146 - for four rounds the difference looked like a fixture bug, because the
- * extra file was untracked and therefore invisible to every diff of the SOURCE.
- * Filtering here fixes the user-visible half too: a .pyc compiled against one
- * interpreter is worse than useless on another.
+ * Filtering them keeps the installed file count equal to the tracked source
+ * count (the layout smoke compares the two), and a .pyc compiled against one
+ * interpreter is of no use on another.
  *
  * @param {string} rel  path relative to the copy root, posix-separated
  * @returns {boolean}
@@ -169,6 +603,13 @@ function isCopyJunk(rel) {
   );
 }
 
+/**
+ * Recursively copy a directory (or symlink it in `--link` mode).
+ *
+ * @param {string} src
+ * @param {string} dest
+ * @param {{ exclude?: Array<string|RegExp>, useSymlinks?: boolean, skipExisting?: boolean }} [opts]
+ */
 export function copyDir(src, dest, opts = {}) {
   const { exclude = [], useSymlinks = false, skipExisting = false } = opts;
   if (useSymlinks) {
@@ -269,25 +710,17 @@ export function pruneLegacyMultiAgentSkills(skillsDir) {
 /**
  * Trees an older installer created and no current one manages.
  *
- * A tree that stops being copied does NOT stop existing: the wipe-before-copy
- * pattern only runs for trees the installer still writes, so dropping one from
- * the installer leaves whatever it last wrote on every existing machine forever.
- *
- * `eval/` is the case that prompted this. An older version copied the eval
- * corpora into `~/.claude/eval/`; the harnesses that read them are maintainer CI
- * tooling and no longer ship, so neither installer references `eval` any more and
- * neither did uninstall - 31 stale files sat there with nothing left that could
- * ever read or remove them.
- *
- * `~/.multi-agent/` is the larger case: the "shared runtime" the Cursor /
- * Antigravity / VS Code Copilot Chat adapters needed, so their emitted agents
- * could reach the gate scripts by absolute path. Those adapters were deleted in
- * v10.7.0 along with `installSharedRuntime`, `_base.mjs`, `rewriteScriptRefs` and
- * `smoke-shared-runtime.sh` - but not the tree they wrote. 221 files (174 scripts,
- * 23 lib, 24 schemas) frozen at whatever the last adapter-era install produced,
- * including a migrations directory missing `prefs-2.3.0-to-2.4.0.mjs`. Browsing it
- * looks exactly like browsing a current install, which is how it comes to be
- * mistaken for one.
+ * A tree that stops being copied does not stop existing: the wipe-before-copy
+ * pattern only runs for trees the installer still writes, so a tree dropped
+ * from the installer keeps whatever it last wrote until something removes it.
+ *
+ * `eval/` held eval corpora for harnesses that are maintainer CI tooling and do
+ * not ship, so nothing installed can read or remove it.
+ *
+ * `~/.multi-agent/` is the shared runtime the retired Cursor / Antigravity /
+ * VS Code Copilot Chat adapters used to reach the gate scripts by absolute
+ * path: scripts, lib and schemas that no current installer refreshes, and that
+ * look like a live install when browsed.
  *
  * `root: "home"` entries sit beside `~/.claude`, not inside it.
  *