@mnaimfaizy/agent-kit 0.0.0-stage → 1.0.0-beta.1

package/src/kit-tag.mjs

@@ -0,0 +1,257 @@
+// Fetches Agent-kit tags into a throwaway bare repository and reads files,
+// file lists and tree hashes from them. Nothing is bundled in the npm
+// package: the git tag Consumers pin in `uses:` is the only source of kit
+// files (ADR 0001). One repository can hold two tags, so `upgrade` can diff
+// them.
+
+import { mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { REPO_PATTERN } from "./checks/result.mjs";
+import { firstLine } from "./exec.mjs";
+import { parseKitOwnedPaths } from "./kit-owned-paths.mjs";
+import { caseCollisions, checkSafePath, fileDirConflicts } from "./safe-paths.mjs";
+import { FIRST_MANIFEST_TAG, isReleaseTag } from "./tags.mjs";
+
+const SAFE_PATH = /^[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*\/?$/;
+export const DEFAULT_SERVER = "https://github.com";
+const SERVER = /^https:\/\/[A-Za-z0-9.-]+(?::\d+)?$/;
+
+/** GITHUB_SERVER_URL when it is a valid server URL (set on GitHub Enterprise Server too), else github.com. */
+export function serverFromEnv(env) {
+  const fromEnv = String(env.GITHUB_SERVER_URL ?? "").replace(/\/$/, "");
+  return SERVER.test(fromEnv) ? fromEnv : DEFAULT_SERVER;
+}
+
+/** The git modes of an executable and of a regular file. */
+export const EXECUTABLE_MODE = "100755";
+export const REGULAR_MODE = "100644";
+
+/** File modes the CLI copies: a regular file, or an executable one. */
+const FILE_MODES = new Set([REGULAR_MODE, EXECUTABLE_MODE]);
+const MODE_NAMES = { 120000: "a symbolic link", 160000: "a submodule" };
+const OID = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/;
+
+export class KitTagError extends Error {}
+
+/** A tag with no kit-owned-paths.txt (up to v1.0.0-alpha.10): the CLI refuses it. */
+export class PreManifestTagError extends KitTagError {
+  constructor(repo, tag, serverUrl = DEFAULT_SERVER) {
+    super(
+      `${tag} has no kit-owned-paths.txt, so the CLI cannot handle it ` +
+        `(the manifest ships from ${FIRST_MANIFEST_TAG}). Adopt or upgrade ${tag} by hand ` +
+        `with that tag's ${serverUrl}/${repo}/blob/${tag}/docs/adopt.md, ` +
+        `or move your pins to ${FIRST_MANIFEST_TAG} or later.`,
+    );
+    this.repo = repo;
+    this.tag = tag;
+  }
+}
+
+function checkPath(path) {
+  if (!SAFE_PATH.test(path) || path.split("/").includes("..")) {
+    throw new KitTagError(`refusing to read ${JSON.stringify(path)} from the tag`);
+  }
+  return path.replace(/\/$/, "");
+}
+
+function checkArgs({ repo, serverUrl }) {
+  if (!REPO_PATTERN.test(repo)) {
+    throw new KitTagError(`not a GitHub repository name: ${JSON.stringify(repo)}`);
+  }
+  if (!SERVER.test(serverUrl)) {
+    throw new KitTagError(`not a GitHub server URL: ${JSON.stringify(serverUrl)}`);
+  }
+}
+
+/**
+ * Open a throwaway bare repository for `repo`, to fetch one or more tags
+ * into. Call `cleanup()` when done.
+ *
+ * @param {{
+ *   runner: import("./exec.mjs").Runner,
+ *   repo: string,
+ *   serverUrl?: string,   // https://github.com, or a GitHub Enterprise Server host
+ *   remoteUrl?: string,   // tests: fetch from here instead
+ * }} opts
+ */
+export function openKitRepo({ runner, repo, serverUrl = DEFAULT_SERVER, remoteUrl }) {
+  checkArgs({ repo, serverUrl });
+  const url = remoteUrl ?? `${serverUrl}/${repo}.git`;
+  const tmp = mkdtempSync(join(tmpdir(), "agent-kit-tag-"));
+  const dir = join(tmp, "repo.git");
+  const noHooks = join(tmp, "no-hooks");
+  mkdirSync(noHooks);
+  const cleanup = () => rmSync(tmp, { recursive: true, force: true });
+  // No hooks run, whatever the adopter's global git config says.
+  const base = ["-C", dir, "-c", `core.hooksPath=${noHooks}`];
+  const git = (...args) => runner("git", [...base, ...args]);
+  const gitBytes = (...args) => runner("git", [...base, ...args], { encoding: "buffer" });
+  try {
+    const r = runner("git", ["init", "--bare", "--quiet", "--template=", dir]);
+    if (r.status !== 0) throw new KitTagError(`git init failed: ${firstLine(r.stderr)}`);
+  } catch (err) {
+    cleanup();
+    throw err;
+  }
+
+  /**
+   * Fetch `tag` into the repository. It must carry kit-owned-paths.txt, or
+   * this throws PreManifestTagError.
+   */
+  const fetchTag = (tag, { forDeletion = true } = {}) => {
+    if (!isReleaseTag(tag)) throw new KitTagError(`not a Release-line tag: ${JSON.stringify(tag)}`);
+    let r = git(
+      "-c",
+      "fetch.fsckObjects=true",
+      "fetch",
+      "--quiet",
+      "--depth=1",
+      "--no-tags",
+      "--",
+      url,
+      `+refs/tags/${tag}:refs/tags/${tag}`,
+    );
+    if (r.status !== 0) {
+      throw new KitTagError(
+        `could not fetch ${tag} from ${repo}: ${firstLine(r.stderr) || "git fetch failed"}`,
+      );
+    }
+    r = git("rev-parse", "--verify", "--quiet", `refs/tags/${tag}^{commit}`);
+    if (r.status !== 0) throw new KitTagError(`${tag} does not point at a commit`);
+    const commit = r.stdout.trim();
+
+    const readFile = (path) => {
+      const res = git("cat-file", "blob", `${commit}:${checkPath(path)}`);
+      return res.status === 0 ? res.stdout : null;
+    };
+    const treeHash = (path) => {
+      // `<rev>:<path>` takes the rest as a path, so `^{tree}` cannot follow it:
+      // resolve the object, then check its type.
+      const res = git("rev-parse", "--verify", "--quiet", `${commit}:${checkPath(path)}`);
+      if (res.status !== 0) return null;
+      const hash = res.stdout.trim();
+      const type = git("cat-file", "-t", hash);
+      return type.status === 0 && type.stdout.trim() === "tree" ? hash : null;
+    };
+
+    /**
+     * Every file under folder `path` at the tag, as `{ path, mode, oid }`.
+     * Throws KitTagError for anything that is not a regular or executable
+     * file (a symbolic link, a submodule), for a file path that is not safe
+     * to write on every platform, and for two paths that would be one file
+     * on a case-insensitive file system.
+     */
+    const listFiles = (path) => {
+      const folder = checkPath(path);
+      const res = git("ls-tree", "-r", "-z", "--full-tree", commit, "--", folder);
+      if (res.status !== 0) {
+        throw new KitTagError(`could not list ${folder}/ at ${tag}: ${firstLine(res.stderr)}`);
+      }
+      const files = [];
+      for (const entry of res.stdout.split("\0").filter(Boolean)) {
+        const m = /^(\d{6}) (\w+) ([0-9a-f]+)\t(.+)$/s.exec(entry);
+        if (!m) throw new KitTagError(`unexpected git ls-tree output at ${tag}`);
+        const [, mode, type, oid, file] = m;
+        if (!file.startsWith(`${folder}/`)) continue;
+        if (!FILE_MODES.has(mode) || type !== "blob" || !OID.test(oid)) {
+          const kind = MODE_NAMES[mode] ?? `mode ${mode}`;
+          throw new KitTagError(
+            `${tag} has ${JSON.stringify(file)} as ${kind}; the CLI copies only regular files`,
+          );
+        }
+        try {
+          checkSafePath(file, `${tag} file`);
+        } catch (err) {
+          throw new KitTagError(err.message);
+        }
+        files.push({ path: file, mode, oid });
+      }
+      const names = files.map((f) => f.path);
+      const collisions = caseCollisions(names);
+      if (collisions.length) {
+        throw new KitTagError(
+          `${tag} has files that differ only in case, which would overwrite each other on ` +
+            `Windows and macOS: ${collisions.map((g) => g.join(" and ")).join("; ")}`,
+        );
+      }
+      const conflicts = fileDirConflicts(names);
+      if (conflicts.length) {
+        throw new KitTagError(
+          `${tag} has ${conflicts[0][0]} as both a file and a folder (${conflicts[0][1]})`,
+        );
+      }
+      return files;
+    };
+
+    /** The exact bytes of a blob listed by listFiles, as a Buffer. */
+    const readBlob = (oid) => {
+      if (!OID.test(oid)) throw new KitTagError(`not a git object id: ${JSON.stringify(oid)}`);
+      const res = gitBytes("cat-file", "blob", oid);
+      if (res.status !== 0 || !Buffer.isBuffer(res.stdout)) {
+        throw new KitTagError(`could not read ${oid} at ${tag}: ${firstLine(res.stderr)}`);
+      }
+      return res.stdout;
+    };
+
+    const manifest = readFile("kit-owned-paths.txt");
+    if (manifest === null) throw new PreManifestTagError(repo, tag, serverUrl);
+    const kitOwnedPaths = parseKitOwnedPaths(manifest, { forDeletion });
+    return { repo, tag, commit, serverUrl, kitOwnedPaths, readFile, treeHash, listFiles, readBlob };
+  };
+
+  /**
+   * `git diff` of folder `path` between two fetched tags, as text. No
+   * external diff tool or textconv filter from the adopter's git config runs.
+   */
+  const diff = (oldSnapshot, newSnapshot, path) => {
+    const folder = checkPath(path);
+    const res = git(
+      "diff",
+      "--no-ext-diff",
+      "--no-textconv",
+      "--no-color",
+      oldSnapshot.commit,
+      newSnapshot.commit,
+      "--",
+      `${folder}/`,
+    );
+    if (res.status !== 0) throw new KitTagError(`git diff failed: ${firstLine(res.stderr)}`);
+    return res.stdout;
+  };
+
+  return { repo, serverUrl, fetchTag, diff, cleanup };
+}
+
+/**
+ * Fetch `tag` of `repo`. The tag must carry kit-owned-paths.txt, or this throws
+ * PreManifestTagError. Call `cleanup()` when done.
+ *
+ * @param {{
+ *   runner: import("./exec.mjs").Runner,
+ *   repo: string,
+ *   tag: string,
+ *   serverUrl?: string,   // https://github.com, or a GitHub Enterprise Server host
+ *   remoteUrl?: string,   // tests: fetch from here instead
+ *   forDeletion?: boolean // strict manifest shapes, for code that deletes paths
+ * }} opts
+ */
+export function openKitTag({
+  runner,
+  repo,
+  tag,
+  serverUrl = DEFAULT_SERVER,
+  remoteUrl,
+  forDeletion = true,
+}) {
+  checkArgs({ repo, serverUrl });
+  // Checked before any git runs, so a bad tag never reaches the command line.
+  if (!isReleaseTag(tag)) throw new KitTagError(`not a Release-line tag: ${JSON.stringify(tag)}`);
+  const kit = openKitRepo({ runner, repo, serverUrl, remoteUrl });
+  try {
+    return { ...kit.fetchTag(tag, { forDeletion }), cleanup: kit.cleanup };
+  } catch (err) {
+    kit.cleanup();
+    throw err;
+  }
+}

package/src/local-tree.mjs

@@ -0,0 +1,240 @@
+// Git tree hashes of paths in the local working copy, as they would be if the
+// working copy were committed now (tracked and untracked files, minus ignored
+// ones, after git's own line-ending rules).
+//
+// Read-only: git runs against a temporary index built from HEAD, and writes
+// any new objects to a temporary object directory that borrows the
+// repository's objects through objects/info/alternates. The Consumer's index,
+// object store and refs are never written. Because the temporary index holds
+// only HEAD plus the Kit-owned paths, a merge conflict elsewhere in the
+// repository does not get in the way.
+//
+// git runs no hooks (core.hooksPath is an empty folder), no fsmonitor, and no
+// configured clean/process filter drivers, so hashing a checkout runs no
+// command that the checkout's or the adopter's git config names.
+
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { isAbsolute, join, resolve } from "node:path";
+import { firstLine } from "./exec.mjs";
+import { EXECUTABLE_MODE, REGULAR_MODE } from "./kit-tag.mjs";
+
+const MODES = new Set([EXECUTABLE_MODE, REGULAR_MODE]);
+
+const SAFE_PATH = /^[A-Za-z0-9._-]+(?:\/[A-Za-z0-9._-]+)*\/$/;
+
+export const SKIP_WORKTREE_REASON =
+  "some of its files are marked skip-worktree (for example by a sparse checkout) and HEAD has no copy of them to compare";
+export const HEAD_COPY_NOTE =
+  "its files are marked skip-worktree (for example by a sparse checkout), so the copy committed at HEAD was compared";
+
+export class LocalTreeError extends Error {}
+
+/** `filter.<driver>.<key>` config names → the driver names. */
+export function filterDrivers(configNames) {
+  const drivers = new Set();
+  for (const raw of configNames) {
+    const m = /^filter\.(.+)\.[^.]+$/i.exec(raw.trim());
+    if (m) drivers.add(m[1]);
+  }
+  return [...drivers];
+}
+
+/**
+ * True when core.fileMode is off (the default on Windows): git then cannot
+ * see an executable bit on disk and takes a file's mode from the index.
+ */
+export function fileModeIsOff(runner, root) {
+  const r = runner("git", ["-C", root, "config", "--type=bool", "core.fileMode"]);
+  return r.status === 0 && r.stdout.trim() === "false";
+}
+
+/**
+ * `-c` options that keep git from running anything the checkout's or the
+ * adopter's git config names: no hooks (`noHooks` is an empty folder), no
+ * fsmonitor, no background maintenance, and no clean/process filter drivers.
+ */
+export function gitSafetyArgs({ runner, root, noHooks }) {
+  const safety = [
+    "-c",
+    `core.hooksPath=${noHooks}`,
+    "-c",
+    "core.fsmonitor=false",
+    "-c",
+    "gc.auto=0",
+    "-c",
+    "maintenance.auto=false",
+  ];
+  const listed = runner("git", ["-C", root, "config", "--name-only", "--get-regexp", "^filter\\."]);
+  for (const driver of filterDrivers(listed.status === 0 ? listed.stdout.split("\n") : [])) {
+    if (/[=\r\n]/.test(driver)) {
+      throw new LocalTreeError(`cannot turn off git filter driver ${JSON.stringify(driver)}`);
+    }
+    safety.push("-c", `filter.${driver}.clean=`, "-c", `filter.${driver}.process=`);
+    safety.push("-c", `filter.${driver}.required=false`);
+  }
+  return safety;
+}
+
+/**
+ * @param {{ runner: import("./exec.mjs").Runner, cwd: string, paths: string[] }} opts
+ * @returns {{ root: string, hashes: Map<string, string | null>, skipped: Map<string, string>, notes: Map<string, string> }}
+ *   A null hash means the path is missing locally; `skipped` holds paths that
+ *   could not be compared, with the reason; `notes` explains how a path was compared.
+ */
+export function localTreeHashes({ runner, cwd, paths }) {
+  for (const p of paths) {
+    if (!SAFE_PATH.test(p) || p.split("/").includes("..")) {
+      throw new LocalTreeError(`refusing to hash ${JSON.stringify(p)}`);
+    }
+  }
+  const top = runner("git", ["-C", cwd, "rev-parse", "--show-toplevel"]);
+  if (top.status !== 0) throw new LocalTreeError(`not a git repository: ${firstLine(top.stderr)}`);
+  const root = top.stdout.trim();
+  // --git-common-dir (not --path-format=absolute, which needs git 2.31) may
+  // be relative to `root`; the objects of a linked worktree live there too.
+  const common = runner("git", ["-C", root, "rev-parse", "--git-common-dir"]);
+  if (common.status !== 0) {
+    throw new LocalTreeError(`git rev-parse failed: ${firstLine(common.stderr)}`);
+  }
+  const commonDir = common.stdout.trim();
+  const realObjects = join(isAbsolute(commonDir) ? commonDir : resolve(root, commonDir), "objects");
+  if (/[\r\n]/.test(realObjects) || realObjects.startsWith('"')) {
+    throw new LocalTreeError("the repository path cannot be written to an alternates file");
+  }
+
+  const tmp = mkdtempSync(join(tmpdir(), "agent-kit-tree-"));
+  try {
+    const index = join(tmp, "index");
+    const objects = join(tmp, "objects");
+    const noHooks = join(tmp, "no-hooks");
+    mkdirSync(join(objects, "info"), { recursive: true });
+    mkdirSync(noHooks);
+    // An alternates file, not GIT_ALTERNATE_OBJECT_DIRECTORIES: that variable
+    // splits on ':' on POSIX, so a path containing one would break it.
+    writeFileSync(join(objects, "info", "alternates"), `${realObjects}\n`);
+
+    const safety = gitSafetyArgs({ runner, root, noHooks });
+    const base = ["-C", root, ...safety, "--literal-pathspecs"];
+    const env = { GIT_INDEX_FILE: index, GIT_OBJECT_DIRECTORY: objects };
+    const git = (...args) => runner("git", [...base, ...args], { env });
+    const real = (...args) => runner("git", [...base, ...args]);
+
+    const head = real("rev-parse", "--verify", "--quiet", "HEAD^{commit}");
+    // HEAD gives tracked files their recorded modes (core.fileMode=false on
+    // Windows would otherwise drop executable bits).
+    if (head.status === 0) {
+      const read = git("read-tree", "HEAD");
+      if (read.status !== 0) {
+        throw new LocalTreeError(`git read-tree HEAD failed: ${firstLine(read.stderr)}`);
+      }
+    }
+
+    // Without core.fileMode (the default on Windows) git cannot see an
+    // executable bit on disk and takes the mode from the index: so does this.
+    const trustIndexModes = fileModeIsOff(runner, root);
+
+    const hashes = new Map();
+    const skipped = new Map();
+    const notes = new Map();
+    for (const path of paths) {
+      // The real index (read only) knows skip-worktree entries (`S`), which
+      // have no files on disk. Everything else comes from the working copy
+      // (`git add`); only the S files are then reset to HEAD's copy, so local
+      // edits and new files elsewhere in the path still count.
+      const tags = real("ls-files", "-t", "-z", "--", path);
+      const entries = tags.status === 0 ? tags.stdout.split("\0").filter(Boolean) : [];
+      const skipFiles = entries.filter((e) => e.startsWith("S ")).map((e) => e.slice(2));
+      if (skipFiles.length) {
+        const inHead =
+          head.status === 0
+            ? real("ls-tree", "-r", "-z", "--name-only", "HEAD", "--", path)
+            : { status: 1, stdout: "" };
+        const committed = new Set(inHead.stdout.split("\0").filter(Boolean));
+        if (inHead.status !== 0 || skipFiles.some((f) => !committed.has(f))) {
+          skipped.set(path, SKIP_WORKTREE_REASON);
+          continue;
+        }
+      }
+      // --sparse: in a sparse checkout, a path outside the cone may still have
+      // stray files on disk, and plain `git add` refuses to touch it. git
+      // before 2.34 has no --sparse; it then gets a plain add.
+      let add = git("add", "--all", "--sparse", "--", path);
+      if (add.status !== 0 && /unknown option|usage: git add/i.test(add.stderr)) {
+        add = git("add", "--all", "--", path);
+      }
+      // Two refusals that still stage what can be staged, then exit 1: files
+      // force-added under an ignored folder, and (old git) a path outside
+      // the sparse-checkout definition.
+      const benign =
+        /ignored by one of your \.gitignore files|outside of your sparse-checkout definition/i.test(
+          add.stderr,
+        );
+      if (add.status !== 0 && !benign && existsSync(join(root, path))) {
+        throw new LocalTreeError(`git add ${path} failed: ${firstLine(add.stderr)}`);
+      }
+      for (let i = 0; i < skipFiles.length; i += 200) {
+        const reset = git("reset", "--quiet", "HEAD", "--", ...skipFiles.slice(i, i + 200));
+        if (reset.status !== 0) {
+          throw new LocalTreeError(
+            `git reset of skip-worktree files failed: ${firstLine(reset.stderr)}`,
+          );
+        }
+      }
+      if (trustIndexModes) {
+        const modes = (res) =>
+          new Map(
+            (res.status === 0 ? res.stdout.split("\0") : [])
+              .filter(Boolean)
+              .map((e) => [e.slice(e.indexOf("\t") + 1), e.slice(0, 6)]),
+          );
+        const inReal = modes(real("ls-files", "-s", "-z", "--", path));
+        const inTemp = modes(git("ls-files", "-s", "-z", "--", path));
+        // Both directions: the real index may have gained or lost the bit.
+        for (const [want, flag] of [
+          [EXECUTABLE_MODE, "--chmod=+x"],
+          [REGULAR_MODE, "--chmod=-x"],
+        ]) {
+          const fix = [...inReal]
+            .filter(
+              ([f, mode]) => mode === want && MODES.has(inTemp.get(f)) && inTemp.get(f) !== want,
+            )
+            .map(([f]) => f);
+          for (let i = 0; i < fix.length; i += 200) {
+            const chmod = git("update-index", flag, "--", ...fix.slice(i, i + 200));
+            if (chmod.status !== 0) {
+              throw new LocalTreeError(
+                `git update-index ${flag} failed: ${firstLine(chmod.stderr)}`,
+              );
+            }
+          }
+        }
+      }
+      const inIndex = git("ls-files", "-z", "--", path);
+      const staged = inIndex.status === 0 ? inIndex.stdout.split("\0").filter(Boolean) : [];
+      if (staged.length === 0) {
+        hashes.set(path, null);
+        continue;
+      }
+      if (skipFiles.length) {
+        // The HEAD-only wording only when nothing came from the working copy.
+        notes.set(
+          path,
+          staged.length === skipFiles.length
+            ? HEAD_COPY_NOTE
+            : `${skipFiles.length} of its ${staged.length} files are marked skip-worktree and were compared as committed at HEAD; the rest came from the working copy`,
+        );
+      }
+      const tree = git("write-tree", `--prefix=${path}`);
+      if (tree.status !== 0) {
+        throw new LocalTreeError(
+          `git write-tree --prefix=${path} failed: ${firstLine(tree.stderr)}`,
+        );
+      }
+      hashes.set(path, tree.stdout.trim());
+    }
+    return { root, hashes, skipped, notes };
+  } finally {
+    rmSync(tmp, { recursive: true, force: true });
+  }
+}