@saadnvd1/agent-os 0.3.605 → 0.3.606

package/lib/env-setup.ts

@@ -9,6 +9,8 @@
 import * as fs from "fs";
 import * as path from "path";
 import { bringDependencies, runCommand } from "./worktree-deps";
+import { inside, realExisting } from "./worktree-deps/paths";
+import { seedBuildCaches } from "./worktree-deps/caches";
 import { loadProjectConfig, projectEnv, withPorts } from "./project-config";
 import { recordPlaced } from "./worktree-placed";
 import { databaseEnv, ensureSessionDatabase } from "./project-config/database";
@@ -79,18 +81,6 @@ export async function copyEnvFiles(
   return copied;
 }
 
-const inside = (root: string, p: string) =>
-  p === root || p.startsWith(root + path.sep);
-
-// The real path of `p`'s deepest part that exists.
-async function realExisting(p: string): Promise<string> {
-  for (let at = p; ; at = path.dirname(at)) {
-    const real = await fs.promises.realpath(at).catch(() => null);
-    if (real) return real;
-    if (path.dirname(at) === at) return at;
-  }
-}
-
 /**
  * agentos.json's `copy`: files or folders from the main checkout, each one
  * resolved (symlinks included) to a place inside it, so a project can't
@@ -272,6 +262,31 @@ export async function setupWorktree(options: {
     );
   }
 
+  // Build caches, so the first typecheck, test or build is warm. An
+  // optimisation: it never fails a setup.
+  if (config.caches?.length !== 0 && sourcePath !== worktreePath) {
+    progress?.onStage?.("deps");
+    try {
+      const seeded = await seedBuildCaches({
+        sourcePath,
+        worktreePath,
+        paths: config.caches,
+      });
+      if (seeded.seeded.length || seeded.skipped.length)
+        result.steps.push({
+          name: "Seed build caches",
+          command: "cp (build caches)",
+          success: true,
+          output: [
+            ...seeded.seeded.map((c) => `${c.rel} (${c.how})`),
+            ...seeded.skipped.map((c) => `skipped ${c.rel}: ${c.why}`),
+          ].join("\n"),
+        });
+    } catch (error) {
+      console.error("Seeding build caches failed:", error);
+    }
+  }
+
   // 4. The config's setup commands
   if (config.setup && config.setup.length > 0) {
     progress?.onStage?.("script");

package/lib/project-config/agentos.schema.json

@@ -48,6 +48,20 @@
         "pattern": "^(?![/~\\\\])(?!(.*[/\\\\])?\\.\\.([/\\\\]|$)).+$"
       }
     },
+    "caches": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "allOf": [
+          {
+            "pattern": "^(?![/~\\\\])(?!(.*[/\\\\])?\\.\\.([/\\\\]|$)).+$"
+          },
+          {
+            "pattern": "^([^*]*[/\\\\])?[^*/\\\\]*\\*?[^*/\\\\]*$"
+          }
+        ]
+      }
+    },
     "dev": {
       "type": "string",
       "minLength": 1

package/lib/project-config/schema.ts

@@ -28,6 +28,15 @@ const projectPath = z
     "must be a path inside the project (relative, no '..')"
   );
 
+// Build caches to seed into a worktree: a path inside the project, with `*`
+// allowed in the last segment (`*.tsbuildinfo`), never `.git`.
+const cachePath = projectPath
+  .regex(
+    /^([^*]*[/\\])?[^*/\\]*\*?[^*/\\]*$/,
+    "`*` is allowed in the last segment only"
+  )
+  .refine((p) => !/(^|[/\\])\.git([/\\]|$)/.test(p), "not .git");
+
 const port = z.number().int().min(1024).max(65000);
 const notes = z.union([z.string(), z.array(z.string())]);
 
@@ -120,6 +129,7 @@ const shape = {
   setup: z.array(z.string().min(1)).optional(),
   copy: z.array(projectPath).optional(),
   clone: z.array(projectPath).optional(),
+  caches: z.array(cachePath).optional(),
   dev: z.string().min(1).optional(),
   ready: ready.optional(),
   deploy: deploy.optional(),

package/lib/worktree-deps/caches.ts

@@ -0,0 +1,250 @@
+/**
+ * Build caches seeded into a new worktree from the main checkout, so its
+ * first typecheck, test and build start warm: `.next/cache`, `.next/dev/cache`
+ * and `*.tsbuildinfo` for a Node project, or exactly agentos.json's `caches`.
+ * (Vitest's own cache is not a default: it only orders tests, and seeding it
+ * measured no gain.) A cache is seeded only when it is fresh, built since the
+ * worktree's base commit, and not huge: an old one makes the first run
+ * slower, not faster. Best effort: a cache that can't be seeded is skipped and never
+ * fails a setup.
+ *
+ * Only ever a copy: an APFS clone, a reflink, or (on a filesystem with
+ * neither) a plain copy of a small cache. Never a hardlink or a symlink, so
+ * nothing a worktree writes can reach the main checkout, and never from a
+ * place other than the main checkout.
+ */
+
+import { execFile } from "child_process";
+import * as fs from "fs";
+import * as path from "path";
+import { promisify } from "util";
+import { gitResult, isGitError } from "../git-exec";
+import { findNodeModules } from "./clone";
+import { fastCopyMode } from "./fastcopy";
+import { inside, realExisting } from "./paths";
+import { copyTree } from "./spare";
+import type { CopyMode } from "./spare";
+
+const execFileAsync = promisify(execFile);
+
+// Past these sizes a cache is skipped. A copy for real (no clone, no reflink)
+// costs the disk; a clone of a huge one costs time, since the build then
+// loads it: a main checkout's .next/cache that has grown to gigabytes built
+// slower for a new worktree than starting empty (measured).
+const DEFAULT_MAX_COPY_MB = 512;
+const DEFAULT_MAX_CLONE_MB = 1024;
+
+interface Stack {
+  name: string;
+  // Cache paths, relative to the checkout, for a project of this stack.
+  paths(sourcePath: string): Promise<string[]>;
+}
+
+const hasNext = (dir: string) => {
+  if (fs.readdirSync(dir).some((f) => /^next\.config\./.test(f))) return true;
+  try {
+    const pkg = JSON.parse(
+      fs.readFileSync(path.join(dir, "package.json"), "utf-8")
+    );
+    return !!(pkg.dependencies?.next ?? pkg.devDependencies?.next);
+  } catch {
+    return false;
+  }
+};
+
+async function nodePaths(sourcePath: string): Promise<string[]> {
+  const dirs = new Set<string>();
+  for (const rel of await findNodeModules(sourcePath))
+    dirs.add(path.dirname(rel));
+  if (fs.existsSync(path.join(sourcePath, "package.json"))) dirs.add(".");
+  const out: string[] = [];
+  for (const dir of dirs) {
+    const abs = path.join(sourcePath, dir);
+    if (!fs.existsSync(path.join(abs, "package.json"))) continue;
+    out.push(path.join(dir, "*.tsbuildinfo"));
+    if (hasNext(abs))
+      out.push(
+        path.join(dir, ".next", "cache"),
+        // Next 16's dev server keeps its Turbopack cache here.
+        path.join(dir, ".next", "dev", "cache")
+      );
+  }
+  return out;
+}
+
+// One entry per stack. Add a stack here to give it defaults; a project can
+// always name its own in agentos.json's `caches`.
+export const STACKS: Stack[] = [
+  { name: "node", paths: nodePaths },
+  // iOS (DerivedData, Pods, ccache) is the agentos-mobile "Faster native iOS
+  // builds" task's: it plugs in here.
+  { name: "ios", paths: async () => [] },
+  // Rails is deliberately absent: bootsnap's load-path cache holds absolute
+  // paths, so a copy in another directory is stale on arrival.
+];
+
+export async function defaultCaches(sourcePath: string): Promise<string[]> {
+  const all = await Promise.all(STACKS.map((s) => s.paths(sourcePath)));
+  return all.flat();
+}
+
+// `*` is allowed in the last segment only.
+async function expand(sourcePath: string, rel: string): Promise<string[]> {
+  const base = path.basename(rel);
+  if (!base.includes("*")) return [rel];
+  const re = new RegExp(
+    "^" + base.split("*").map(escapeRe).join("[^/]*") + "$"
+  );
+  const dir = path.dirname(rel);
+  const names = await fs.promises
+    .readdir(path.join(sourcePath, dir))
+    .catch(() => [] as string[]);
+  return names.filter((n) => re.test(n)).map((n) => path.join(dir, n));
+}
+const escapeRe = (s: string) => s.replace(/[.+?^${}()|[\]\\]/g, "\\$&");
+
+// When the worktree's HEAD was committed; null when it can't be told.
+async function headTimeMs(worktreePath: string): Promise<number | null> {
+  try {
+    const { stdout } = await gitResult(worktreePath, [
+      "log",
+      "-1",
+      "--format=%ct",
+    ]);
+    const seconds = parseInt(stdout, 10);
+    return Number.isFinite(seconds) ? seconds * 1000 : null;
+  } catch {
+    return null;
+  }
+}
+
+// The last write to a file, or to a folder or its direct children.
+async function newestMs(p: string): Promise<number> {
+  const st = await fs.promises.stat(p);
+  let newest = st.mtimeMs;
+  if (st.isDirectory())
+    for (const name of await fs.promises.readdir(p)) {
+      const child = await fs.promises
+        .stat(path.join(p, name))
+        .catch(() => null);
+      if (child) newest = Math.max(newest, child.mtimeMs);
+    }
+  return newest;
+}
+
+async function sizeMb(p: string): Promise<number> {
+  const { stdout } = await execFileAsync("du", ["-sk", p]);
+  return parseInt(stdout, 10) / 1024;
+}
+
+// Git ignores it, or seeding it would leave an uncommitted file behind.
+// Outside a repository there is no answer, and it is allowed.
+async function notIgnored(worktreePath: string, rel: string): Promise<boolean> {
+  try {
+    await gitResult(worktreePath, ["check-ignore", "-q", "--", rel]);
+    return false;
+  } catch (error) {
+    return isGitError(error) && error.exitCode === 1;
+  }
+}
+
+export interface SeedResult {
+  seeded: Array<{ rel: string; how: CopyMode }>;
+  skipped: Array<{ rel: string; why: string }>;
+}
+
+export async function seedBuildCaches(opts: {
+  sourcePath: string;
+  worktreePath: string;
+  // agentos.json's `caches`; the stack defaults without it.
+  paths?: string[];
+  mode?: CopyMode;
+  platform?: NodeJS.Platform;
+  maxCopyMb?: number;
+  maxCloneMb?: number;
+  // Caches last written before this (ms) are stale. Default: the worktree's
+  // HEAD commit time; without one every cache counts as fresh.
+  freshAfterMs?: number;
+}): Promise<SeedResult> {
+  const { sourcePath, worktreePath } = opts;
+  const result: SeedResult = { seeded: [], skipped: [] };
+  // The switch for measuring a start without seeding.
+  if (process.env.AGENTOS_NO_BUILD_CACHES) return result;
+  const declared = opts.paths ?? (await defaultCaches(sourcePath));
+  if (!declared.length) return result;
+
+  const fast = await fastCopyMode({
+    sourcePath,
+    worktreePath,
+    platform: opts.platform,
+  });
+  // Hardlinks are for dependencies from the store, never for a cache.
+  const mode: CopyMode =
+    opts.mode ?? (fast === "clone" || fast === "reflink" ? fast : "copy");
+  const max =
+    opts.maxCopyMb ??
+    Number(process.env.AGENTOS_CACHE_COPY_MAX_MB || DEFAULT_MAX_COPY_MB);
+  const maxClone =
+    opts.maxCloneMb ??
+    Number(process.env.AGENTOS_CACHE_CLONE_MAX_MB || DEFAULT_MAX_CLONE_MB);
+  const freshAfter = opts.freshAfterMs ?? (await headTimeMs(worktreePath));
+  const root = await fs.promises.realpath(sourcePath);
+  const target = await fs.promises.realpath(worktreePath);
+
+  const skip = (rel: string, why: string) => result.skipped.push({ rel, why });
+
+  for (const rel of (
+    await Promise.all(declared.map((d) => expand(sourcePath, d)))
+  ).flat()) {
+    const real = await fs.promises
+      .realpath(path.join(sourcePath, rel))
+      .catch(() => null);
+    if (!real) continue; // not built in the main checkout yet
+    if (!inside(root, real)) {
+      skip(rel, "resolves outside the project");
+      continue;
+    }
+    const dir = path.join(target, path.dirname(rel));
+    if (!inside(target, await realExisting(dir))) {
+      skip(rel, "its folder in the worktree resolves outside it");
+      continue;
+    }
+    const dest = path.join(dir, path.basename(rel));
+    if (await fs.promises.lstat(dest).catch(() => null)) continue; // already there
+    if (await notIgnored(worktreePath, rel)) {
+      skip(rel, "not ignored by git");
+      continue;
+    }
+    if (freshAfter !== null && (await newestMs(real)) < freshAfter) {
+      skip(rel, "older than the worktree's base commit");
+      continue;
+    }
+    const limit = mode === "copy" ? max : maxClone;
+    if ((await sizeMb(real).catch(() => Infinity)) > limit) {
+      skip(
+        rel,
+        mode === "copy"
+          ? `bigger than ${limit} MB and this filesystem can't clone`
+          : `bigger than ${limit} MB`
+      );
+      continue;
+    }
+    await fs.promises.mkdir(dir, { recursive: true });
+    if (!inside(target, await fs.promises.realpath(dir))) {
+      skip(rel, "its folder in the worktree resolves outside it");
+      continue;
+    }
+    const partial = `${dest}.aos-partial`;
+    await fs.promises.rm(partial, { recursive: true, force: true });
+    const ok =
+      (await copyTree(real, partial, mode)) &&
+      (await fs.promises.rename(partial, dest).then(
+        () => true,
+        () => false
+      ));
+    await fs.promises.rm(partial, { recursive: true, force: true });
+    if (ok) result.seeded.push({ rel, how: mode });
+    else skip(rel, "the copy failed");
+  }
+  return result;
+}

package/lib/worktree-deps/clone.ts

@@ -10,6 +10,8 @@ import { createHash } from "crypto";
 import * as fs from "fs";
 import * as path from "path";
 import { LOCKFILES } from "./install";
+import { STORE_ROOT, fastCopyMode, unlinkMutable } from "./fastcopy";
+import type { FastMode } from "./fastcopy";
 import {
   SPARE_ROOT,
   copyTree,
@@ -72,11 +74,15 @@ export async function findNodeModules(sourcePath: string): Promise<string[]> {
 
 // Cloned beside the target and renamed into place, so a clone cut off by a
 // restart or a failed cp never passes for a finished one.
-async function cloneWhole(from: string, target: string): Promise<boolean> {
+async function cloneWhole(
+  from: string,
+  target: string,
+  mode: FastMode
+): Promise<boolean> {
   const partial = `${target}.aos-partial`;
   await fs.promises.rm(partial, { recursive: true, force: true });
   const ok =
-    (await copyTree(from, partial)) && (await takeSpare(partial, target));
+    (await copyTree(from, partial, mode)) && (await takeSpare(partial, target));
   await fs.promises.rm(partial, { recursive: true, force: true });
   return ok;
 }
@@ -86,11 +92,14 @@ export interface CloneResult {
   // Why it didn't clone, when it didn't.
   reason?: string;
   cloned: Array<{ rel: string; from: "spare" | "clone" }>;
+  mode?: FastMode;
+  skipped?: string[];
   // The spares being made for the next start. Nothing waits on it but tests.
   refill?: Promise<unknown>;
 }
 
-export const NOT_MAC = "clones are macOS only";
+export const NO_FAST_COPY =
+  "no clone, reflink or hardlink store is possible here";
 
 export async function cloneDeps(opts: {
   sourcePath: string;
@@ -99,13 +108,26 @@ export async function cloneDeps(opts: {
   // node_modules findNodeModules finds.
   only?: string[];
   spareRoot?: string;
+  // Where Linux keeps the hardlinked store (fastcopy.ts).
+  storeRoot?: string;
   platform?: NodeJS.Platform;
+  // Forced instead of detected; null is "none works" (tests).
+  mode?: FastMode | null;
 }): Promise<CloneResult> {
   const { sourcePath, worktreePath, only } = opts;
   const root = opts.spareRoot ?? SPARE_ROOT;
   const cloned: CloneResult["cloned"] = [];
-  if ((opts.platform ?? process.platform) !== "darwin")
-    return { ok: false, reason: NOT_MAC, cloned };
+  const storeRoot = opts.storeRoot ?? STORE_ROOT;
+  const mode =
+    opts.mode !== undefined
+      ? opts.mode
+      : await fastCopyMode({
+          sourcePath,
+          worktreePath,
+          storeRoot,
+          platform: opts.platform,
+        });
+  if (!mode) return { ok: false, reason: NO_FAST_COPY, cloned };
 
   const isModules = (rel: string) => path.basename(rel) === "node_modules";
   const all = only
@@ -124,6 +146,13 @@ export async function cloneDeps(opts: {
       };
     if (!lock)
       return { ok: false, reason: "the main checkout has no lockfile", cloned };
+    if (!(await nodeModulesMatchLock(sourcePath)))
+      return {
+        ok: false,
+        reason:
+          "the main checkout's node_modules is not what its lockfile lists",
+        cloned,
+      };
     if (lock !== (await lockfileHash(worktreePath)))
       return {
         ok: false,
@@ -133,16 +162,22 @@ export async function cloneDeps(opts: {
   }
 
   const spares: Array<{ rel: string; lock: string }> = [];
+  const skipped: string[] = [];
   for (const rel of all) {
     const from = path.join(sourcePath, rel);
     const target = path.join(worktreePath, rel);
     if (!(await isDir(from))) continue;
     // A directory this branch doesn't have, or the files already there.
     if (!(await isDir(path.dirname(target))) || fs.existsSync(target)) continue;
+    if (!isModules(rel) && mode === "hardlink") {
+      // Mutable, and hardlinks would write back: left to the setup commands.
+      skipped.push(`${rel} (not linked)`);
+      continue;
+    }
     if (!isModules(rel)) {
       // Anything else declared (a virtualenv, vendor/bundle) is cloned as
       // is: there is no lockfile AgentOS knows to check it against.
-      if (!(await cloneWhole(from, target)))
+      if (!(await cloneWhole(from, target, mode)))
         return { ok: false, reason: `cloning ${rel} failed`, cloned };
       cloned.push({ rel, from: "clone" });
       continue;
@@ -154,10 +189,30 @@ export async function cloneDeps(opts: {
     if (rel !== "node_modules" && own !== ownHere) continue;
     const key = own ?? lock;
     if (!key) continue;
+    if (mode === "hardlink") {
+      // Linked from the store, never from the checkout; the store is made
+      // once per lockfile with a plain copy.
+      const stored = sparePath(storeRoot, sourcePath, rel, key);
+      if (!fs.existsSync(stored))
+        await replenishSpare(storeRoot, sourcePath, rel, key, "copy").catch(
+          () => undefined
+        );
+      if (!fs.existsSync(stored))
+        return {
+          ok: false,
+          reason: `making the store for ${rel} failed`,
+          cloned,
+        };
+      if (!(await cloneWhole(stored, target, mode)))
+        return { ok: false, reason: `linking ${rel} failed`, cloned };
+      await unlinkMutable(target).catch(() => undefined);
+      cloned.push({ rel, from: "clone" });
+      continue;
+    }
     const spare = sparePath(root, sourcePath, rel, key);
     if (await takeSpare(spare, target)) {
       cloned.push({ rel, from: "spare" });
-    } else if (await cloneWhole(from, target)) {
+    } else if (await cloneWhole(from, target, mode)) {
       cloned.push({ rel, from: "clone" });
     } else {
       return { ok: false, reason: `cloning ${rel} failed`, cloned };
@@ -167,10 +222,40 @@ export async function cloneDeps(opts: {
 
   const refill = Promise.all(
     spares.map(({ rel, lock: key }) =>
-      replenishSpare(root, sourcePath, rel, key).catch((error) =>
+      replenishSpare(root, sourcePath, rel, key, mode).catch((error) =>
         console.error(`Spare for ${rel} failed:`, error)
       )
     )
   );
-  return { ok: true, cloned, refill };
+  return { ok: true, cloned, refill, mode, skipped };
+}
+
+// Whether the checkout's node_modules is what its npm lockfile says: every
+// package the lockfile lists (optional ones, which a platform may skip, aside)
+// is installed at that version. A checkout pulled but never reinstalled is
+// not, and cloning it would hand the worktree missing dependencies. Other
+// package managers have no hidden lockfile to check, so they pass.
+export async function nodeModulesMatchLock(dir: string): Promise<boolean> {
+  const read = (file: string) =>
+    fs.promises
+      .readFile(file, "utf-8")
+      .then(
+        (text) =>
+          JSON.parse(text) as {
+            packages?: Record<string, { version?: string; optional?: boolean }>;
+          }
+      )
+      .catch(() => null);
+  const lock = await read(path.join(dir, "package-lock.json"));
+  if (!lock?.packages) return true;
+  const installed = await read(
+    path.join(dir, "node_modules", ".package-lock.json")
+  );
+  if (!installed?.packages) return false;
+  return Object.entries(lock.packages).every(
+    ([name, pkg]) =>
+      name === "" ||
+      pkg.optional ||
+      installed.packages![name]?.version === pkg.version
+  );
 }

package/lib/worktree-deps/fastcopy.ts

@@ -0,0 +1,89 @@
+/**
+ * How a Linux (or macOS) worktree can be given a copy of the main checkout's
+ * files without paying for every byte: an APFS clone, a reflink where the
+ * filesystem has them, or hardlinks from a store AgentOS owns. Null when
+ * none works, and the caller installs or copies plainly instead.
+ */
+
+import { execFile } from "child_process";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import { promisify } from "util";
+import type { CopyMode } from "./spare";
+
+const execFileAsync = promisify(execFile);
+
+export type FastMode = Exclude<CopyMode, "copy">;
+
+// Where hardlinked dependencies are kept: one real copy per lockfile, so a
+// write through a link can only ever reach this, never the main checkout.
+export const STORE_ROOT = path.join(os.homedir(), ".agent-os", "deps-store");
+
+// Files tools rewrite in place. A hardlink would carry the write back into
+// the store, so these are real copies in every worktree.
+export const MUTABLE_IN_MODULES = [".package-lock.json", ".cache", ".vite"];
+
+const probed = new Map<string, boolean>();
+
+// Whether `dir`'s filesystem reflinks: tried on a scratch file made there.
+export async function canReflink(dir: string): Promise<boolean> {
+  const dev = String((await fs.promises.stat(dir)).dev);
+  const known = probed.get(dev);
+  if (known !== undefined) return known;
+  const probe = path.join(dir, `.aos-reflink-probe-${process.pid}`);
+  let ok = false;
+  try {
+    await fs.promises.writeFile(probe, "x");
+    await execFileAsync("cp", ["--reflink=always", probe, `${probe}.copy`]);
+    ok = true;
+  } catch {
+    ok = false;
+  } finally {
+    await fs.promises.rm(probe, { force: true });
+    await fs.promises.rm(`${probe}.copy`, { force: true });
+  }
+  probed.set(dev, ok);
+  return ok;
+}
+
+const device = (p: string) =>
+  fs.promises.stat(p).then(
+    (s) => s.dev,
+    () => null
+  );
+
+export async function fastCopyMode(opts: {
+  sourcePath: string;
+  worktreePath: string;
+  storeRoot?: string;
+  platform?: NodeJS.Platform;
+}): Promise<FastMode | null> {
+  const platform = opts.platform ?? process.platform;
+  if (platform === "darwin") return "clone";
+  if (platform !== "linux") return null;
+  const wt = await device(opts.worktreePath);
+  if (wt === null) return null;
+  if (
+    (await device(opts.sourcePath)) === wt &&
+    (await canReflink(opts.worktreePath))
+  )
+    return "reflink";
+  const store = opts.storeRoot ?? STORE_ROOT;
+  await fs.promises.mkdir(store, { recursive: true }).catch(() => undefined);
+  return (await device(store)) === wt ? "hardlink" : null;
+}
+
+// Replace the entries of a hardlinked node_modules that tools rewrite in
+// place with real copies.
+export async function unlinkMutable(modules: string): Promise<void> {
+  for (const name of MUTABLE_IN_MODULES) {
+    const entry = path.join(modules, name);
+    if (!(await fs.promises.lstat(entry).catch(() => null))) continue;
+    const copy = `${entry}.aos-copy`;
+    await fs.promises.rm(copy, { recursive: true, force: true });
+    await fs.promises.cp(entry, copy, { recursive: true });
+    await fs.promises.rm(entry, { recursive: true, force: true });
+    await fs.promises.rename(copy, entry);
+  }
+}

package/lib/worktree-deps/index.ts

@@ -6,11 +6,17 @@
 import { exec } from "child_process";
 import { promisify } from "util";
 import type { SetupResult } from "../env-setup";
-import { NOT_MAC, cloneDeps } from "./clone";
+import { NO_FAST_COPY, cloneDeps } from "./clone";
 import { detectPackageManager, setupEnv } from "./install";
 
 const execAsync = promisify(exec);
 
+const COMMANDS = {
+  clone: "cp -Rc",
+  reflink: "cp -a --reflink=always",
+  hardlink: "cp -al (from the store)",
+};
+
 /**
  * Run a setup command in the worktree directory
  */
@@ -57,15 +63,18 @@ export async function bringDependencies(
   if (pm) result.packageManager = pm.name;
 
   const clone = await deps.clone({ sourcePath, worktreePath, only });
-  // Off macOS only node_modules has a fallback (the install below).
-  if (!pm && !clone.ok && !clone.cloned.length && clone.reason === NOT_MAC)
+  // Without a fast copy only node_modules has a fallback (the install below).
+  if (!pm && !clone.ok && !clone.cloned.length && clone.reason === NO_FAST_COPY)
     return;
   if (clone.ok || !pm) {
     result.steps.push({
       name: "Clone dependencies",
-      command: "cp -Rc",
+      command: clone.mode ? COMMANDS[clone.mode] : "cp",
       success: clone.ok,
-      output: clone.cloned.map((c) => `${c.rel} (${c.from})`).join(", "),
+      output: [
+        ...clone.cloned.map((c) => `${c.rel} (${c.from})`),
+        ...(clone.skipped ?? []).map((r) => `skipped ${r}`),
+      ].join(", "),
       error: clone.ok ? undefined : clone.reason,
     });
     if (!clone.ok) result.success = false;

package/lib/worktree-deps/paths.ts

@@ -0,0 +1,20 @@
+/**
+ * Path checks shared by what setup places into a worktree (agentos.json's
+ * `copy`, `clone` and `caches`): everything resolved through symlinks to a
+ * place inside its checkout, so a link can't carry a copy somewhere else.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+
+export const inside = (root: string, p: string) =>
+  p === root || p.startsWith(root + path.sep);
+
+// The real path of `p`'s deepest part that exists.
+export async function realExisting(p: string): Promise<string> {
+  for (let at = p; ; at = path.dirname(at)) {
+    const real = await fs.promises.realpath(at).catch(() => null);
+    if (real) return real;
+    if (path.dirname(at) === at) return at;
+  }
+}

package/lib/worktree-deps/spare.ts

@@ -49,18 +49,35 @@ export async function takeSpare(
   }
 }
 
-// macOS clones (APFS, no extra disk); elsewhere this is a plain copy, which
-// production never asks for: Linux worktrees install instead.
-export const cloneArgs = (from: string, to: string) =>
-  process.platform === "darwin" ? ["-Rc", from, to] : ["-R", from, to];
+// How a tree is copied: an APFS clone (macOS), a reflink (Linux on a
+// copy-on-write filesystem), hardlinks (Linux, from AgentOS's own store), or
+// a plain copy.
+export type CopyMode = "clone" | "reflink" | "hardlink" | "copy";
+
+export const copyArgs = (mode: CopyMode, from: string, to: string) =>
+  mode === "clone"
+    ? [process.platform === "darwin" ? "-Rc" : "-R", from, to]
+    : mode === "reflink"
+      ? ["-a", "--reflink=always", from, to]
+      : mode === "hardlink"
+        ? ["-al", from, to]
+        : ["-a", from, to];
+
+// What a copy with no mode given does: clones on macOS, a plain copy elsewhere.
+const defaultMode = (): CopyMode =>
+  process.platform === "darwin" ? "clone" : "copy";
 
 // A clone that stalls (a locked or dying volume) is killed rather than
 // holding a task's start forever.
 const COPY_TIMEOUT_MS = 10 * 60 * 1000;
 
-export function copyTree(from: string, to: string): Promise<boolean> {
+export function copyTree(
+  from: string,
+  to: string,
+  mode: CopyMode = defaultMode()
+): Promise<boolean> {
   return new Promise((resolve) => {
-    const child = spawn("cp", cloneArgs(from, to), {
+    const child = spawn("cp", copyArgs(mode, from, to), {
       stdio: "ignore",
       timeout: COPY_TIMEOUT_MS,
       killSignal: "SIGKILL",
@@ -104,7 +121,8 @@ export async function replenishSpare(
   root: string,
   sourcePath: string,
   rel: string,
-  lockHash: string
+  lockHash: string,
+  mode?: CopyMode
 ): Promise<"made" | "exists" | "failed"> {
   const spare = sparePath(root, sourcePath, rel, lockHash);
   if (building.has(spare) || fs.existsSync(spare)) return "exists";
@@ -113,7 +131,7 @@ export async function replenishSpare(
     await fs.promises.mkdir(root, { recursive: true });
     await pruneSpares(root, sourcePath, rel, spare);
     const tmp = `${spare}.${process.pid}-${Date.now()}`;
-    let made = await copyTree(path.join(sourcePath, rel), tmp);
+    let made = await copyTree(path.join(sourcePath, rel), tmp, mode);
     if (made) made = await takeSpare(tmp, spare);
     await fs.promises.rm(tmp, { recursive: true, force: true });
     return made ? "made" : "failed";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saadnvd1/agent-os",
-  "version": "0.3.605",
+  "version": "0.3.606",
   "description": "Self-hosted web UI for managing Claude Code sessions",
   "bin": {
     "agent-os": "scripts/agent-os"

package/scripts/bench/worktree-setup.ts

@@ -0,0 +1,101 @@
+/**
+ * Times a new worktree's setup and its first typecheck, test and build, and
+ * the disk it takes, against a checkout. Run it with the seeding on, then
+ * with AGENTOS_NO_BUILD_CACHES=1 (and, to see an install on Linux, --install).
+ *
+ *   npx tsx scripts/bench/worktree-setup.ts <checkout> [--test <file>]
+ *     [--build] [--install] [--cwd <subdir>] [--drop <seeded path>]
+ *
+ * It makes a throwaway git worktree of the checkout's HEAD in the temp
+ * directory and removes it at the end; it touches nothing else.
+ */
+
+import { execFileSync, spawnSync } from "child_process";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import { setupWorktree } from "../../lib/env-setup";
+
+const args = process.argv.slice(2);
+const flag = (name: string) => args.includes(name);
+const value = (name: string) => args[args.indexOf(name) + 1];
+const source = path.resolve(args[0]);
+const sub = flag("--cwd") ? value("--cwd") : ".";
+
+const freeKb = (dir: string) =>
+  Number(
+    execFileSync("df", ["-Pk", dir], { encoding: "utf-8" })
+      .trim()
+      .split("\n")[1]
+      .split(/\s+/)[3]
+  );
+
+function timed(command: string, cwd: string) {
+  const start = Date.now();
+  const run = spawnSync(command, { cwd, shell: true, stdio: "ignore" });
+  return { seconds: (Date.now() - start) / 1000, ok: run.status === 0 };
+}
+
+async function main() {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "aos-bench-"));
+  const worktree = path.join(root, "wt");
+  execFileSync("git", ["worktree", "add", "-q", "--detach", worktree, "HEAD"], {
+    cwd: source,
+  });
+  const out: Record<string, unknown> = {
+    source: path.basename(source),
+    platform: process.platform,
+    seeding: process.env.AGENTOS_NO_BUILD_CACHES ? "off" : "on",
+  };
+  try {
+    const before = freeKb(root);
+    const start = Date.now();
+    let steps: string[] = [];
+    if (flag("--install")) {
+      const r = timed("npm ci --include=dev", worktree);
+      steps = [`npm ci ok=${r.ok}`];
+    } else {
+      const result = await setupWorktree({
+        worktreePath: worktree,
+        sourcePath: source,
+      });
+      steps = result.steps.map(
+        (s) => `${s.name}: ${s.success ? "ok" : "failed"}`
+      );
+    }
+    out.setupSeconds = (Date.now() - start) / 1000;
+    out.setupSteps = steps;
+    out.diskMbAfterSetup = Math.round((before - freeKb(root)) / 1024);
+    // Take a seeded cache back out to measure without it.
+    if (flag("--drop"))
+      fs.rmSync(path.join(worktree, value("--drop")), {
+        recursive: true,
+        force: true,
+      });
+    const dir = path.join(worktree, sub);
+    const pkg = path.join(dir, "package.json");
+    const typecheck = fs.existsSync(pkg)
+      ? JSON.parse(fs.readFileSync(pkg, "utf-8")).scripts?.typecheck
+      : undefined;
+    if (typecheck) out.typecheck = timed("npm run typecheck", dir);
+    else if (fs.existsSync(path.join(dir, "tsconfig.json")))
+      out.typecheck = timed("npx tsc --noEmit", dir);
+    if (flag("--test"))
+      out.test = timed(`npx vitest run ${value("--test")}`, dir);
+    if (flag("--build")) {
+      out.build = timed("npx next build", dir);
+      out.diskMbAfterBuild = Math.round((before - freeKb(root)) / 1024);
+    }
+  } finally {
+    execFileSync("git", ["worktree", "remove", "--force", worktree], {
+      cwd: source,
+    });
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+  console.log(JSON.stringify(out));
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});