@mnaimfaizy/agent-kit 1.0.0-alpha.12 → 1.0.0-beta.2

package/src/kit-tag.mjs

@@ -1,6 +1,8 @@
-// Fetches one Agent-kit tag into a throwaway bare repository and reads files
-// and tree hashes from it. Nothing is bundled in the npm package: the git tag
-// Consumers pin in `uses:` is the only source of kit files (ADR 0001).
+// Fetches Agent-kit tags into a throwaway bare repository and reads files,
+// file lists and tree hashes from them. Nothing is bundled in the npm
+// package: the git tag Consumers pin in `uses:` is the only source of kit
+// files (ADR 0001). One repository can hold two tags, so `upgrade` can diff
+// them.
 
 import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
@@ -8,12 +10,28 @@ import { join } from "node:path";
 import { REPO_PATTERN } from "./checks/result.mjs";
 import { firstLine } from "./exec.mjs";
 import { parseKitOwnedPaths } from "./kit-owned-paths.mjs";
+import { caseCollisions, checkSafePath, fileDirConflicts } from "./safe-paths.mjs";
 import { FIRST_MANIFEST_TAG, isReleaseTag } from "./tags.mjs";
 
 const SAFE_PATH = /^[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*\/?$/;
 export const DEFAULT_SERVER = "https://github.com";
 const SERVER = /^https:\/\/[A-Za-z0-9.-]+(?::\d+)?$/;
 
+/** GITHUB_SERVER_URL when it is a valid server URL (set on GitHub Enterprise Server too), else github.com. */
+export function serverFromEnv(env) {
+  const fromEnv = String(env.GITHUB_SERVER_URL ?? "").replace(/\/$/, "");
+  return SERVER.test(fromEnv) ? fromEnv : DEFAULT_SERVER;
+}
+
+/** The git modes of an executable and of a regular file. */
+export const EXECUTABLE_MODE = "100755";
+export const REGULAR_MODE = "100644";
+
+/** File modes the CLI copies: a regular file, or an executable one. */
+const FILE_MODES = new Set([REGULAR_MODE, EXECUTABLE_MODE]);
+const MODE_NAMES = { 120000: "a symbolic link", 160000: "a submodule" };
+const OID = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/;
+
 export class KitTagError extends Error {}
 
 /** A tag with no kit-owned-paths.txt (up to v1.0.0-alpha.10): the CLI refuses it. */
@@ -37,46 +55,64 @@ function checkPath(path) {
   return path.replace(/\/$/, "");
 }
 
+function checkArgs({ repo, serverUrl }) {
+  if (!REPO_PATTERN.test(repo)) {
+    throw new KitTagError(`not a GitHub repository name: ${JSON.stringify(repo)}`);
+  }
+  if (!SERVER.test(serverUrl)) {
+    throw new KitTagError(`not a GitHub server URL: ${JSON.stringify(serverUrl)}`);
+  }
+}
+
 /**
- * Fetch `tag` of `repo`. The tag must carry kit-owned-paths.txt, or this throws
- * PreManifestTagError. Call `cleanup()` when done.
+ * Open a throwaway bare repository for `repo`, to fetch one or more tags
+ * into. Call `cleanup()` when done.
  *
  * @param {{
  *   runner: import("./exec.mjs").Runner,
  *   repo: string,
- *   tag: string,
  *   serverUrl?: string,   // https://github.com, or a GitHub Enterprise Server host
  *   remoteUrl?: string,   // tests: fetch from here instead
- *   forDeletion?: boolean // strict manifest shapes, for code that deletes paths
  * }} opts
  */
-export function openKitTag({
-  runner,
-  repo,
-  tag,
-  serverUrl = DEFAULT_SERVER,
-  remoteUrl,
-  forDeletion = true,
-}) {
-  if (!REPO_PATTERN.test(repo)) {
-    throw new KitTagError(`not a GitHub repository name: ${JSON.stringify(repo)}`);
-  }
-  if (!isReleaseTag(tag)) throw new KitTagError(`not a Release-line tag: ${JSON.stringify(tag)}`);
-  if (!SERVER.test(serverUrl)) {
-    throw new KitTagError(`not a GitHub server URL: ${JSON.stringify(serverUrl)}`);
-  }
+export function openKitRepo({ runner, repo, serverUrl = DEFAULT_SERVER, remoteUrl }) {
+  checkArgs({ repo, serverUrl });
   const url = remoteUrl ?? `${serverUrl}/${repo}.git`;
   const tmp = mkdtempSync(join(tmpdir(), "agent-kit-tag-"));
   const dir = join(tmp, "repo.git");
   const noHooks = join(tmp, "no-hooks");
   mkdirSync(noHooks);
-  const cleanup = () => rmSync(tmp, { recursive: true, force: true });
+  // The retries are a fallback only: with background maintenance off (below)
+  // no git process is left writing in `tmp` once its command has returned.
+  const cleanup = () =>
+    rmSync(tmp, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
+  // No background maintenance: `git fetch` ends with `git maintenance run
+  // --auto`, which newer git detaches (maintenance.autoDetach, gc.autoDetach).
+  // The detached process outlives the fetch and can still be writing in the
+  // repository (objects/maintenance.lock, gc.pid) when cleanup() removes it,
+  // which failed cleanup with ENOTEMPTY.
+  const noBackground = ["-c", "maintenance.auto=false", "-c", "gc.auto=0"];
   // No hooks run, whatever the adopter's global git config says.
-  const git = (...args) => runner("git", ["-C", dir, "-c", `core.hooksPath=${noHooks}`, ...args]);
+  const base = ["-C", dir, "-c", `core.hooksPath=${noHooks}`, ...noBackground];
+  const git = (...args) => runner("git", [...base, ...args]);
+  const gitBytes = (...args) => runner("git", [...base, ...args], { encoding: "buffer" });
   try {
-    let r = runner("git", ["init", "--bare", "--quiet", "--template=", dir]);
+    const r = runner("git", [...noBackground, "init", "--bare", "--quiet", "--template=", dir]);
     if (r.status !== 0) throw new KitTagError(`git init failed: ${firstLine(r.stderr)}`);
-    r = git(
+  } catch (err) {
+    cleanup();
+    throw err;
+  }
+
+  /**
+   * Fetch `tag` into the repository. It must carry kit-owned-paths.txt, or
+   * this throws PreManifestTagError.
+   */
+  const fetchTag = (tag, { forDeletion = true } = {}) => {
+    if (!isReleaseTag(tag)) throw new KitTagError(`not a Release-line tag: ${JSON.stringify(tag)}`);
+    let r = git(
+      "-c",
+      "fetch.fsckObjects=true",
       "fetch",
       "--quiet",
       "--depth=1",
@@ -108,12 +144,123 @@ export function openKitTag({
       return type.status === 0 && type.stdout.trim() === "tree" ? hash : null;
     };
 
+    /**
+     * Every file under folder `path` at the tag, as `{ path, mode, oid }`.
+     * Throws KitTagError for anything that is not a regular or executable
+     * file (a symbolic link, a submodule), for a file path that is not safe
+     * to write on every platform, and for two paths that would be one file
+     * on a case-insensitive file system.
+     */
+    const listFiles = (path) => {
+      const folder = checkPath(path);
+      const res = git("ls-tree", "-r", "-z", "--full-tree", commit, "--", folder);
+      if (res.status !== 0) {
+        throw new KitTagError(`could not list ${folder}/ at ${tag}: ${firstLine(res.stderr)}`);
+      }
+      const files = [];
+      for (const entry of res.stdout.split("\0").filter(Boolean)) {
+        const m = /^(\d{6}) (\w+) ([0-9a-f]+)\t(.+)$/s.exec(entry);
+        if (!m) throw new KitTagError(`unexpected git ls-tree output at ${tag}`);
+        const [, mode, type, oid, file] = m;
+        if (!file.startsWith(`${folder}/`)) continue;
+        if (!FILE_MODES.has(mode) || type !== "blob" || !OID.test(oid)) {
+          const kind = MODE_NAMES[mode] ?? `mode ${mode}`;
+          throw new KitTagError(
+            `${tag} has ${JSON.stringify(file)} as ${kind}; the CLI copies only regular files`,
+          );
+        }
+        try {
+          checkSafePath(file, `${tag} file`);
+        } catch (err) {
+          throw new KitTagError(err.message);
+        }
+        files.push({ path: file, mode, oid });
+      }
+      const names = files.map((f) => f.path);
+      const collisions = caseCollisions(names);
+      if (collisions.length) {
+        throw new KitTagError(
+          `${tag} has files that differ only in case, which would overwrite each other on ` +
+            `Windows and macOS: ${collisions.map((g) => g.join(" and ")).join("; ")}`,
+        );
+      }
+      const conflicts = fileDirConflicts(names);
+      if (conflicts.length) {
+        throw new KitTagError(
+          `${tag} has ${conflicts[0][0]} as both a file and a folder (${conflicts[0][1]})`,
+        );
+      }
+      return files;
+    };
+
+    /** The exact bytes of a blob listed by listFiles, as a Buffer. */
+    const readBlob = (oid) => {
+      if (!OID.test(oid)) throw new KitTagError(`not a git object id: ${JSON.stringify(oid)}`);
+      const res = gitBytes("cat-file", "blob", oid);
+      if (res.status !== 0 || !Buffer.isBuffer(res.stdout)) {
+        throw new KitTagError(`could not read ${oid} at ${tag}: ${firstLine(res.stderr)}`);
+      }
+      return res.stdout;
+    };
+
     const manifest = readFile("kit-owned-paths.txt");
     if (manifest === null) throw new PreManifestTagError(repo, tag, serverUrl);
     const kitOwnedPaths = parseKitOwnedPaths(manifest, { forDeletion });
-    return { repo, tag, commit, serverUrl, kitOwnedPaths, readFile, treeHash, cleanup };
+    return { repo, tag, commit, serverUrl, kitOwnedPaths, readFile, treeHash, listFiles, readBlob };
+  };
+
+  /**
+   * `git diff` of folder `path` between two fetched tags, as text. No
+   * external diff tool or textconv filter from the adopter's git config runs.
+   */
+  const diff = (oldSnapshot, newSnapshot, path) => {
+    const folder = checkPath(path);
+    const res = git(
+      "diff",
+      "--no-ext-diff",
+      "--no-textconv",
+      "--no-color",
+      oldSnapshot.commit,
+      newSnapshot.commit,
+      "--",
+      `${folder}/`,
+    );
+    if (res.status !== 0) throw new KitTagError(`git diff failed: ${firstLine(res.stderr)}`);
+    return res.stdout;
+  };
+
+  return { repo, serverUrl, fetchTag, diff, cleanup };
+}
+
+/**
+ * Fetch `tag` of `repo`. The tag must carry kit-owned-paths.txt, or this throws
+ * PreManifestTagError. Call `cleanup()` when done.
+ *
+ * @param {{
+ *   runner: import("./exec.mjs").Runner,
+ *   repo: string,
+ *   tag: string,
+ *   serverUrl?: string,   // https://github.com, or a GitHub Enterprise Server host
+ *   remoteUrl?: string,   // tests: fetch from here instead
+ *   forDeletion?: boolean // strict manifest shapes, for code that deletes paths
+ * }} opts
+ */
+export function openKitTag({
+  runner,
+  repo,
+  tag,
+  serverUrl = DEFAULT_SERVER,
+  remoteUrl,
+  forDeletion = true,
+}) {
+  checkArgs({ repo, serverUrl });
+  // Checked before any git runs, so a bad tag never reaches the command line.
+  if (!isReleaseTag(tag)) throw new KitTagError(`not a Release-line tag: ${JSON.stringify(tag)}`);
+  const kit = openKitRepo({ runner, repo, serverUrl, remoteUrl });
+  try {
+    return { ...kit.fetchTag(tag, { forDeletion }), cleanup: kit.cleanup };
   } catch (err) {
-    cleanup();
+    kit.cleanup();
     throw err;
   }
 }

package/src/local-tree.mjs

@@ -13,10 +13,21 @@
 // configured clean/process filter drivers, so hashing a checkout runs no
 // command that the checkout's or the adopter's git config names.
 
-import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import {
+  existsSync,
+  lstatSync,
+  mkdirSync,
+  mkdtempSync,
+  readdirSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
 import { tmpdir } from "node:os";
 import { isAbsolute, join, resolve } from "node:path";
 import { firstLine } from "./exec.mjs";
+import { EXECUTABLE_MODE, REGULAR_MODE } from "./kit-tag.mjs";
+
+const MODES = new Set([EXECUTABLE_MODE, REGULAR_MODE]);
 
 const SAFE_PATH = /^[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*\/$/;
 
@@ -37,11 +48,98 @@ export function filterDrivers(configNames) {
   return [...drivers];
 }
 
+/**
+ * True when core.fileMode is off (the default on Windows): git then cannot
+ * see an executable bit on disk and takes a file's mode from the index.
+ */
+export function fileModeIsOff(runner, root) {
+  const r = runner("git", ["-C", root, "config", "--type=bool", "core.fileMode"]);
+  return r.status === 0 && r.stdout.trim() === "false";
+}
+
+/**
+ * `-c` options that keep git from running anything the checkout's or the
+ * adopter's git config names: no hooks (`noHooks` is an empty folder), no
+ * fsmonitor, no background maintenance, and no clean/process filter drivers.
+ */
+export function gitSafetyArgs({ runner, root, noHooks }) {
+  const safety = [
+    "-c",
+    `core.hooksPath=${noHooks}`,
+    "-c",
+    "core.fsmonitor=false",
+    "-c",
+    "gc.auto=0",
+    "-c",
+    "maintenance.auto=false",
+  ];
+  const listed = runner("git", ["-C", root, "config", "--name-only", "--get-regexp", "^filter\\."]);
+  for (const driver of filterDrivers(listed.status === 0 ? listed.stdout.split("\n") : [])) {
+    if (/[=\r\n]/.test(driver)) {
+      throw new LocalTreeError(`cannot turn off git filter driver ${JSON.stringify(driver)}`);
+    }
+    safety.push("-c", `filter.${driver}.clean=`, "-c", `filter.${driver}.process=`);
+    safety.push("-c", `filter.${driver}.required=false`);
+  }
+  return safety;
+}
+
+/** `lstat`, or null when nothing is there. */
+export function lstatOrNull(path) {
+  try {
+    return lstatSync(path);
+  } catch (err) {
+    if (err.code === "ENOENT" || err.code === "ENOTDIR") return null;
+    throw err;
+  }
+}
+
+/**
+ * The first symbolic link or junction at, above or inside the folder `path`
+ * (relative to `root`, ending in /), as a path without a trailing slash, or
+ * null. git looks straight through a Windows junction, stops at a symlink
+ * with "beyond a symbolic link", and stores a symlink inside a folder as a
+ * link entry, so this is checked before git ever sees the path.
+ */
+export function linkAt(root, path) {
+  const folder = path.replace(/\/$/, "");
+  const parts = folder.split("/");
+  for (let i = 1; i <= parts.length; i++) {
+    const sub = parts.slice(0, i).join("/");
+    const st = lstatOrNull(join(root, sub));
+    if (!st) return null;
+    if (st.isSymbolicLink()) return sub;
+  }
+  return linksUnder(root, folder)[0] ?? null;
+}
+
+/**
+ * Every symbolic link or junction at or inside `rel` (relative to `root`, no
+ * trailing slash), in sorted walk order: never followed or guessed at. doctor
+ * reports what it finds and upgrade refuses to delete through it, so both
+ * agree on what is linked.
+ */
+export function linksUnder(root, rel) {
+  const found = [];
+  const walk = (sub) => {
+    const st = lstatOrNull(join(root, sub));
+    if (!st) return;
+    if (st.isSymbolicLink()) found.push(sub);
+    else if (st.isDirectory()) {
+      for (const name of readdirSync(join(root, sub)).sort()) walk(`${sub}/${name}`);
+    }
+  };
+  walk(rel);
+  return found;
+}
+
 /**
  * @param {{ runner: import("./exec.mjs").Runner, cwd: string, paths: string[] }} opts
- * @returns {{ root: string, hashes: Map<string, string | null>, skipped: Map<string, string>, notes: Map<string, string> }}
+ * @returns {{ root: string, hashes: Map<string, string | null>, skipped: Map<string, string>, notes: Map<string, string>, links: Map<string, string> }}
  *   A null hash means the path is missing locally; `skipped` holds paths that
- *   could not be compared, with the reason; `notes` explains how a path was compared.
+ *   could not be compared, with the reason; `notes` explains how a path was compared;
+ *   `links` maps a path that is, sits under, or holds a symbolic link or junction to
+ *   that link. A linked path is never hashed.
  */
 export function localTreeHashes({ runner, cwd, paths }) {
   for (const p of paths) {
@@ -75,29 +173,7 @@ export function localTreeHashes({ runner, cwd, paths }) {
     // splits on ':' on POSIX, so a path containing one would break it.
     writeFileSync(join(objects, "info", "alternates"), `${realObjects}\n`);
 
-    const safety = [
-      "-c",
-      `core.hooksPath=${noHooks}`,
-      "-c",
-      "core.fsmonitor=false",
-      "-c",
-      "gc.auto=0",
-    ];
-    const listed = runner("git", [
-      "-C",
-      root,
-      "config",
-      "--name-only",
-      "--get-regexp",
-      "^filter\\.",
-    ]);
-    for (const driver of filterDrivers(listed.status === 0 ? listed.stdout.split("\n") : [])) {
-      if (/[=\r\n]/.test(driver)) {
-        throw new LocalTreeError(`cannot turn off git filter driver ${JSON.stringify(driver)}`);
-      }
-      safety.push("-c", `filter.${driver}.clean=`, "-c", `filter.${driver}.process=`);
-      safety.push("-c", `filter.${driver}.required=false`);
-    }
+    const safety = gitSafetyArgs({ runner, root, noHooks });
     const base = ["-C", root, ...safety, "--literal-pathspecs"];
     const env = { GIT_INDEX_FILE: index, GIT_OBJECT_DIRECTORY: objects };
     const git = (...args) => runner("git", [...base, ...args], { env });
@@ -113,10 +189,20 @@ export function localTreeHashes({ runner, cwd, paths }) {
       }
     }
 
+    // Without core.fileMode (the default on Windows) git cannot see an
+    // executable bit on disk and takes the mode from the index: so does this.
+    const trustIndexModes = fileModeIsOff(runner, root);
+
     const hashes = new Map();
     const skipped = new Map();
     const notes = new Map();
+    const links = new Map();
     for (const path of paths) {
+      const link = linkAt(root, path);
+      if (link !== null) {
+        links.set(path, link);
+        continue;
+      }
       // The real index (read only) knows skip-worktree entries (`S`), which
       // have no files on disk. Everything else comes from the working copy
       // (`git add`); only the S files are then reset to HEAD's copy, so local
@@ -160,6 +246,35 @@ export function localTreeHashes({ runner, cwd, paths }) {
           );
         }
       }
+      if (trustIndexModes) {
+        const modes = (res) =>
+          new Map(
+            (res.status === 0 ? res.stdout.split("\0") : [])
+              .filter(Boolean)
+              .map((e) => [e.slice(e.indexOf("\t") + 1), e.slice(0, 6)]),
+          );
+        const inReal = modes(real("ls-files", "-s", "-z", "--", path));
+        const inTemp = modes(git("ls-files", "-s", "-z", "--", path));
+        // Both directions: the real index may have gained or lost the bit.
+        for (const [want, flag] of [
+          [EXECUTABLE_MODE, "--chmod=+x"],
+          [REGULAR_MODE, "--chmod=-x"],
+        ]) {
+          const fix = [...inReal]
+            .filter(
+              ([f, mode]) => mode === want && MODES.has(inTemp.get(f)) && inTemp.get(f) !== want,
+            )
+            .map(([f]) => f);
+          for (let i = 0; i < fix.length; i += 200) {
+            const chmod = git("update-index", flag, "--", ...fix.slice(i, i + 200));
+            if (chmod.status !== 0) {
+              throw new LocalTreeError(
+                `git update-index ${flag} failed: ${firstLine(chmod.stderr)}`,
+              );
+            }
+          }
+        }
+      }
       const inIndex = git("ls-files", "-z", "--", path);
       const staged = inIndex.status === 0 ? inIndex.stdout.split("\0").filter(Boolean) : [];
       if (staged.length === 0) {
@@ -183,7 +298,7 @@ export function localTreeHashes({ runner, cwd, paths }) {
       }
       hashes.set(path, tree.stdout.trim());
     }
-    return { root, hashes, skipped, notes };
+    return { root, hashes, skipped, notes, links };
   } finally {
     rmSync(tmp, { recursive: true, force: true });
   }