@saadnvd1/agent-os 0.3.660 → 0.3.664

package/lib/worktree-sweep/caches.ts

@@ -0,0 +1,123 @@
+/**
+ * What a task leaves outside its checkout and the database: dependency
+ * spares nobody took, its temp folder, a crashed preview browser's profile,
+ * and Xcode's build folder for a worktree that no longer exists. Each is
+ * rebuilt or recreated on demand, so the only rule is "old enough, and not
+ * in use".
+ */
+
+import fs from "fs";
+import path from "path";
+import { execFile } from "child_process";
+import { SESSION_TMP_PREFIX, tmpName } from "../sessions/leftovers";
+import type { Candidate, SweepSkipped } from "./types";
+
+const DAY_MS = 24 * 3600_000;
+const SPARE_MIN_AGE_MS = 7 * DAY_MS;
+const BUILDING_MIN_AGE_MS = 3600_000;
+
+export interface CacheDeps {
+  now: number;
+  spareRoot: string;
+  tmpRoot: string;
+  derivedRoot: string;
+  worktreesRoot: string;
+  // Ids of the sessions still running on this machine.
+  liveIds: Set<string>;
+  workspaceOf: (derivedDir: string) => Promise<string | null>;
+  size: (dir: string) => Promise<number>;
+}
+
+export const workspaceFromPlist = (dir: string): Promise<string | null> =>
+  new Promise((resolve) =>
+    execFile(
+      "plutil",
+      [
+        "-extract",
+        "WorkspacePath",
+        "raw",
+        "-o",
+        "-",
+        path.join(dir, "info.plist"),
+      ],
+      { timeout: 10_000 },
+      (error, stdout) => resolve(error ? null : String(stdout).trim() || null)
+    )
+  );
+
+const subdirs = (root: string): string[] => {
+  try {
+    return fs
+      .readdirSync(root, { withFileTypes: true })
+      .filter((e) => e.isDirectory())
+      .map((e) => path.join(root, e.name));
+  } catch {
+    return [];
+  }
+};
+
+const mtime = (p: string) => {
+  try {
+    return fs.statSync(p).mtimeMs;
+  } catch {
+    return 0;
+  }
+};
+
+const inside = (root: string, p: string) => {
+  const rel = path.relative(path.resolve(root), path.resolve(p));
+  return !!rel && !rel.startsWith("..") && !path.isAbsolute(rel);
+};
+
+export async function findCacheLeftovers(deps: CacheDeps): Promise<{
+  candidates: Candidate[];
+  skipped: SweepSkipped[];
+}> {
+  const candidates: Candidate[] = [];
+  const skipped: SweepSkipped[] = [];
+  const add = async (dir: string, kind: Candidate["kind"], why: string) =>
+    candidates.push({ path: dir, kind, why, bytes: await deps.size(dir) });
+
+  for (const dir of subdirs(deps.spareRoot)) {
+    const building = path.basename(dir).includes(".");
+    const min = building ? BUILDING_MIN_AGE_MS : SPARE_MIN_AGE_MS;
+    if (deps.now - mtime(dir) >= min)
+      await add(
+        dir,
+        "spare",
+        building ? "an abandoned build" : "unused for a week"
+      );
+  }
+
+  const livePrefixes = [...deps.liveIds].map(tmpName);
+  for (const dir of subdirs(deps.tmpRoot)) {
+    const name = path.basename(dir);
+    const isSession = name.startsWith(SESSION_TMP_PREFIX);
+    if (!isSession && !name.startsWith("agentos-preview-")) continue;
+    if (deps.now - mtime(dir) < DAY_MS) continue;
+    if (
+      isSession &&
+      livePrefixes.includes(name.slice(SESSION_TMP_PREFIX.length))
+    ) {
+      skipped.push({
+        path: dir,
+        kind: "temp",
+        reason: "its session is running",
+      });
+      continue;
+    }
+    await add(
+      dir,
+      "temp",
+      isSession ? "its task is over" : "a preview browser's profile"
+    );
+  }
+
+  for (const dir of subdirs(deps.derivedRoot)) {
+    const workspace = await deps.workspaceOf(dir);
+    if (!workspace || !inside(deps.worktreesRoot, workspace)) continue;
+    if (fs.existsSync(workspace)) continue;
+    await add(dir, "derived-data", "its worktree is gone");
+  }
+  return { candidates, skipped };
+}

package/lib/worktree-sweep/databases.ts

@@ -0,0 +1,222 @@
+/**
+ * Task databases left behind: copies and what projects derived from them
+ * (`<copy>_test`), found by the `_aos_<12 hex>` token every session's name
+ * carries. A database goes only if its token belongs to no live session,
+ * nobody is connected, it isn't a project's declared source, and it is more
+ * than a day old. Anything without the token is never looked at.
+ *
+ * A session is over only when it is merged, dropped, done or moved away, or
+ * its row is gone: archiving and settling can be undone, and an interactive
+ * session has no end of its own, so neither keeps a database off the list.
+ *
+ * Proof of who made it: a copy must carry the `agentos session <id>` comment
+ * AgentOS gave it, and the id must hash to the token. Something a project
+ * derived from a copy (`<copy>_test`) has no such comment, so it also needs
+ * its token to belong to no session this machine has ever known, and the
+ * server to be local (a host that isn't localhost or a socket is never
+ * reaped on a name alone).
+ */
+
+import { db } from "../db";
+import { loadProjectConfig } from "../project-config";
+import { expandHome } from "../project-config/session";
+import {
+  LOCAL_HOSTS,
+  MARK,
+  SOCKET_DIR,
+  pgEnv,
+  pgRun,
+  sessionHash,
+  taskHashOf,
+  type PgRun,
+} from "../project-config/database";
+import type { Candidate, SweepSkipped } from "./types";
+
+const DAY_MS = 24 * 3600_000;
+
+export interface PgServer {
+  host?: string;
+  port?: number;
+}
+
+export interface DatabaseDeps {
+  now: number;
+  run: PgRun;
+  servers: PgServer[];
+  // Hashes of the sessions that still need their database.
+  live: Set<string>;
+  // Hashes of every session this machine has a row for, over or not.
+  known: Set<string>;
+  // Databases a project declares as its source, never reaped.
+  froms: Set<string>;
+}
+
+const where = (s: PgServer) =>
+  `${s.host ?? "local server"}${s.port ? `:${s.port}` : ""}`;
+
+// Sessions whose work isn't over: everything with a row except a task that
+// merged, was dropped, finished or moved away.
+export function liveSessionIds(): Set<string> {
+  const rows = db
+    .prepare(
+      `SELECT id FROM session_rows
+       WHERE task_status IS NULL
+          OR task_status NOT IN ('merged', 'dropped', 'done', 'moved')`
+    )
+    .all() as Array<{ id: string }>;
+  return new Set(rows.map((r) => r.id));
+}
+
+export const knownSessionIds = (): Set<string> =>
+  new Set(
+    (
+      db.prepare(`SELECT id FROM session_rows`).all() as Array<{ id: string }>
+    ).map((r) => r.id)
+  );
+
+export const hashesOf = (ids: Iterable<string>): Set<string> =>
+  new Set([...ids].map(sessionHash));
+
+export const liveHashes = (ids = liveSessionIds()): Set<string> =>
+  hashesOf(ids);
+
+const isLocal = (s: PgServer) =>
+  !s.host || LOCAL_HOSTS.has(s.host) || SOCKET_DIR.test(s.host);
+
+// Every project's declared database source and server.
+export function declaredDatabases(): {
+  servers: PgServer[];
+  froms: Set<string>;
+} {
+  const rows = db
+    .prepare(
+      `SELECT working_directory FROM projects WHERE working_directory IS NOT NULL`
+    )
+    .all() as Array<{ working_directory: string }>;
+  const servers = new Map<string, PgServer>([[":", {}]]);
+  const froms = new Set<string>();
+  for (const r of rows) {
+    try {
+      const d = loadProjectConfig(expandHome(r.working_directory)).config
+        .database;
+      if (!d) continue;
+      froms.add(d.from);
+      servers.set(`${d.host ?? ""}:${d.port ?? ""}`, {
+        host: d.host,
+        port: d.port,
+      });
+    } catch {
+      // A project whose config can't be read declares nothing.
+    }
+  }
+  return { servers: [...servers.values()], froms };
+}
+
+const LIST = (age: boolean) =>
+  `select d.datname, pg_database_size(d.oid), coalesce(shobj_description(d.oid, 'pg_database'), ''),
+     (select count(*) from pg_stat_activity a where a.datname = d.datname),
+     ${age ? `extract(epoch from (pg_stat_file('base/' || d.oid || '/PG_VERSION')).modification)::bigint` : "0"}
+   from pg_database d where d.datname ~ '_aos_[0-9a-f]{12}'`;
+
+// Why this database can't be shown to be a session's, or null when it can.
+function whoMade(
+  hash: string,
+  note: string,
+  known: Set<string>,
+  server: PgServer
+): string | null {
+  const m = /^agentos session (\S+?)(?: \(copying\))?$/.exec(note);
+  if (m)
+    return sessionHash(m[1]) === hash
+      ? null
+      : "its comment names another session";
+  if (note) return "it has a comment that isn't AgentOS's";
+  if (known.has(hash))
+    return "no AgentOS comment on it, and its session is known here";
+  return isLocal(server) ? null : "no AgentOS comment, and not a local server";
+}
+
+async function list(deps: DatabaseDeps, server: PgServer) {
+  const env = pgEnv(server);
+  for (const age of [true, false]) {
+    const { code, out } = await deps.run(
+      "psql",
+      ["-X", "-d", "postgres", "-tAq", "-F", "\t", "-c", LIST(age)],
+      env
+    );
+    if (code === 0)
+      return out
+        .split("\n")
+        .filter(Boolean)
+        .map((line) => {
+          const [name, size, note, conns, at] = line.split("\t");
+          return {
+            name,
+            bytes: Number(size) || 0,
+            note: note ?? "",
+            conns: Number(conns) || 0,
+            madeAt: Number(at) * 1000 || 0,
+          };
+        });
+    if (code === 127) return "psql isn't installed";
+    if (!age) return `Postgres isn't answering (${out.slice(0, 120)})`;
+  }
+  return "Postgres isn't answering";
+}
+
+export async function findTaskDatabases(deps: DatabaseDeps): Promise<{
+  candidates: Candidate[];
+  skipped: SweepSkipped[];
+}> {
+  const candidates: Candidate[] = [];
+  const skipped: SweepSkipped[] = [];
+  for (const server of deps.servers) {
+    const found = await list(deps, server);
+    if (typeof found === "string") {
+      skipped.push({
+        path: where(server),
+        kind: "database",
+        reason: found,
+      });
+      continue;
+    }
+    for (const d of found) {
+      const hash = taskHashOf(d.name);
+      const skip = (reason: string) =>
+        skipped.push({ path: d.name, kind: "database", reason });
+      if (!hash) continue;
+      const proof = whoMade(hash, d.note, deps.known, server);
+      if (deps.froms.has(d.name)) skip("a project's declared source");
+      else if (proof) skip(proof);
+      else if (deps.live.has(hash)) skip("its session is still running");
+      else if (d.conns > 0) skip(`${d.conns} connection(s) open`);
+      else if (d.madeAt && deps.now - d.madeAt < DAY_MS)
+        skip("made less than a day ago");
+      else
+        candidates.push({
+          path: d.name,
+          kind: "database",
+          server,
+          bytes: d.bytes,
+          why: `no session needs it${d.madeAt ? "" : " (age unknown)"}`,
+        });
+    }
+  }
+  return { candidates, skipped };
+}
+
+// Re-checked here as well: only a name carrying the token is ever dropped,
+// and a plain drop, which Postgres refuses while anyone is connected.
+export async function dropTaskDatabase(
+  c: Candidate,
+  froms: Set<string>,
+  run: PgRun = pgRun
+): Promise<boolean> {
+  if (!taskHashOf(c.path) || froms.has(c.path)) return false;
+  const { code } = await run(
+    "dropdb",
+    ["--if-exists", "--", c.path],
+    pgEnv(c.server ?? {})
+  );
+  return code === 0;
+}

package/lib/worktree-sweep/du.ts

@@ -0,0 +1,13 @@
+import { execFile } from "child_process";
+
+// Disk use of a folder in bytes; 0 when it can't be measured in time.
+export default function duBytes(
+  dir: string,
+  timeout = 300_000
+): Promise<number> {
+  return new Promise((resolve) =>
+    execFile("du", ["-sk", dir], { timeout }, (error, stdout) =>
+      resolve(error ? 0 : (parseInt(String(stdout), 10) || 0) * 1024)
+    )
+  );
+}

package/lib/worktree-sweep/extras.ts

@@ -0,0 +1,188 @@
+/**
+ * The second half of a run: what tasks leave beside their worktrees. Found
+ * by databases.ts and caches.ts, removed here only when the full clean-up is
+ * on; otherwise listed as pending, with sizes, for a dry run's report.
+ */
+
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { SPARE_ROOT } from "../worktree-deps/spare";
+import duBytes from "./du";
+import { pgRun } from "../project-config/database";
+import {
+  declaredDatabases,
+  dropTaskDatabase,
+  findTaskDatabases,
+  hashesOf,
+  knownSessionIds,
+  liveHashes,
+  liveSessionIds,
+} from "./databases";
+import { findCacheLeftovers, workspaceFromPlist } from "./caches";
+import type { SweepDeps } from "./runSweep";
+import type {
+  Candidate,
+  SweepRemoved,
+  SweepSkipped,
+  SweepSummary,
+  SweepTotals,
+} from "./types";
+import { WORKTREES_DIR } from "../worktrees";
+
+export interface Extras {
+  candidates: Candidate[];
+  skipped: SweepSkipped[];
+  drop: (c: Candidate) => Promise<boolean>;
+  spares: { count: number; bytes: number };
+}
+
+const dirCount = (root: string) => {
+  try {
+    return fs
+      .readdirSync(root, { withFileTypes: true })
+      .filter((e) => e.isDirectory() && !e.name.startsWith(".")).length;
+  } catch {
+    return 0;
+  }
+};
+
+// The size of a whole root (every worktree, every spare) takes minutes of
+// disk reading, so it is measured at most every six hours, or again on a
+// dry run, which is what asks for fresh numbers.
+const ROOT_SIZE_TTL_MS = 6 * 3600_000;
+const rootSizes = new Map<string, { at: number; bytes: number }>();
+
+export async function cachedRootSize(
+  dir: string,
+  size: (dir: string) => Promise<number>,
+  fresh: boolean,
+  now = Date.now()
+): Promise<number> {
+  const hit = rootSizes.get(dir);
+  if (hit && !fresh && now - hit.at < ROOT_SIZE_TTL_MS) return hit.bytes;
+  const bytes = await size(dir);
+  // 0 is what a failed or timed-out du reports: try again next time.
+  if (bytes > 0) rootSizes.set(dir, { at: now, bytes });
+  return bytes;
+}
+
+export const forgetRootSizes = () => rootSizes.clear();
+
+export async function defaultExtras(
+  opts: { fresh?: boolean } = {}
+): Promise<Extras> {
+  const { servers, froms } = declaredDatabases();
+  const ids = liveSessionIds();
+  const size = (dir: string) => duBytes(dir);
+  const dbs = await findTaskDatabases({
+    now: Date.now(),
+    run: pgRun,
+    servers,
+    live: liveHashes(ids),
+    known: hashesOf(knownSessionIds()),
+    froms,
+  });
+  const derivedRoot = path.join(
+    os.homedir(),
+    "Library",
+    "Developer",
+    "Xcode",
+    "DerivedData"
+  );
+  const caches = await findCacheLeftovers({
+    now: Date.now(),
+    spareRoot: SPARE_ROOT,
+    tmpRoot: os.tmpdir(),
+    derivedRoot: process.platform === "darwin" ? derivedRoot : "",
+    worktreesRoot: WORKTREES_DIR,
+    liveIds: ids,
+    workspaceOf: workspaceFromPlist,
+    size,
+  });
+  const roots = [SPARE_ROOT, os.tmpdir(), derivedRoot];
+  return {
+    candidates: [...dbs.candidates, ...caches.candidates],
+    skipped: [...dbs.skipped, ...caches.skipped],
+    spares: {
+      count: dirCount(SPARE_ROOT),
+      bytes: await cachedRootSize(SPARE_ROOT, size, !!opts.fresh),
+    },
+    drop: async (c) => {
+      if (c.kind === "database") return dropTaskDatabase(c, froms);
+      const target = path.resolve(c.path);
+      if (!roots.some((r) => r && path.dirname(target) === path.resolve(r)))
+        return false;
+      await fs.promises.rm(target, { recursive: true, force: true });
+      return !fs.existsSync(target);
+    },
+  };
+}
+
+// Removes (or, off or dry, only lists) what the extras found.
+export async function findExtras(
+  deps: SweepDeps,
+  summary: SweepSummary,
+  opts: { armed: boolean; dryRun: boolean }
+): Promise<{ totals: SweepTotals }> {
+  const x = await deps.extras({ fresh: opts.dryRun });
+  summary.skipped.push(...x.skipped);
+  for (const c of x.candidates) {
+    const item: SweepRemoved = {
+      path: c.path,
+      kind: c.kind,
+      why: c.why ?? "",
+      bytes: c.bytes ?? (await deps.size(c.path)),
+    };
+    if (!opts.armed) {
+      summary.pending!.push(item);
+      continue;
+    }
+    if (deps.busy()) {
+      summary.skipped.push({
+        path: c.path,
+        kind: c.kind,
+        reason: "the machine is busy; left for the next run",
+      });
+      continue;
+    }
+    if (!opts.dryRun && !(await x.drop(c).catch(() => false))) {
+      summary.skipped.push({
+        path: c.path,
+        kind: c.kind,
+        reason: "removing it failed",
+      });
+      continue;
+    }
+    summary.removed.push(item);
+    summary.freedBytes += item.bytes;
+  }
+  const found = (kind: Candidate["kind"]) => {
+    const items = [...summary.removed, ...(summary.pending ?? [])].filter(
+      (i) => i.kind === kind
+    );
+    return {
+      count: items.length,
+      bytes: items.reduce((n, i) => n + i.bytes, 0),
+    };
+  };
+  return {
+    totals: {
+      worktrees: {
+        count: dirCount(deps.root),
+        bytes: await cachedRootSize(deps.root, deps.size, opts.dryRun),
+      },
+      databases: found("database"),
+      spares: x.spares,
+      temp: found("temp"),
+    },
+  };
+}
+
+// For tests, and for a machine without the extras: nothing found.
+export const noExtras = async (): Promise<Extras> => ({
+  candidates: [],
+  skipped: [],
+  drop: async () => false,
+  spares: { count: 0, bytes: 0 },
+});

package/lib/worktree-sweep/findOrphans.ts

@@ -61,8 +61,16 @@ function gitdirOf(dir: string): string | null {
   }
 }
 
+// The server's own temp folder differs from a test run's (launchd, systemd),
+// so the usual system ones count too.
+const TEMP_ROOTS = [
+  /^\/(?:private\/)?tmp(?:\/|$)/,
+  /^\/(?:private\/)?var\/folders\/[^/]+\/[^/]+\/T(?:\/|$)/,
+  /^\/var\/tmp(?:\/|$)/,
+];
 const inTemp = (p: string) =>
-  [os.tmpdir(), real(os.tmpdir())].some((t) => inFolder(p, t));
+  [os.tmpdir(), real(os.tmpdir())].some((t) => inFolder(p, t)) ||
+  TEMP_ROOTS.some((r) => r.test(p));
 
 // Nothing in it but what AgentOS placed or rebuilds: no work to lose.
 export const holdsNothing = (dir: string) =>
@@ -103,7 +111,16 @@ export async function findOrphans(deps: OrphanDeps): Promise<{
     if (mine.has(real(dir))) continue;
     const skip = (reason: string) =>
       skipped.push({ path: dir, kind: "orphan", reason });
-    if (deps.now - newestMs(dir) < days * DAY_MS) {
+    const gitdir = gitdirOf(dir);
+    const registered = !!gitdir && fs.existsSync(gitdir);
+    const leak =
+      TEST_LEAK.test(entry.name) &&
+      !!gitdir &&
+      inTemp(real(path.dirname(gitdir)));
+    // A test's folder whose repository is gone is nobody's, however young:
+    // new ones arrive with every run of an old checkout's tests.
+    const abandoned = leak && !registered;
+    if (!abandoned && deps.now - newestMs(dir) < days * DAY_MS) {
       skip(`touched less than ${days} day${days === 1 ? "" : "s"} ago`);
       continue;
     }
@@ -111,12 +128,6 @@ export async function findOrphans(deps: OrphanDeps): Promise<{
       skip("a terminal is open in it");
       continue;
     }
-    const gitdir = gitdirOf(dir);
-    const registered = !!gitdir && fs.existsSync(gitdir);
-    const leak =
-      TEST_LEAK.test(entry.name) &&
-      !!gitdir &&
-      inTemp(real(path.dirname(gitdir)));
     if (leak) {
       candidates.push({
         path: dir,

package/lib/worktree-sweep/report.ts

@@ -0,0 +1,85 @@
+/**
+ * The list a dry run writes before anything is removed: what would go, what
+ * stays and why, with sizes, for a person to read before turning the full
+ * clean-up on. One file per day under ~/.agent-os/audits/disk-cleanup.
+ */
+
+import fs from "fs";
+import path from "path";
+import { agentosHome } from "../agentos-home";
+import type { SweepKind, SweepRemoved, SweepSummary } from "./types";
+
+export const mb = (b: number) =>
+  b >= 1024 ** 3
+    ? `${(b / 1024 ** 3).toFixed(2)} GB`
+    : `${(b / 1024 ** 2).toFixed(1)} MB`;
+
+const TITLES: Record<SweepKind, string> = {
+  settled: "Worktrees of finished tasks",
+  orphan: "Worktree folders no session owns",
+  "test-leak": "Throwaway repositories left by tests",
+  database: "Task databases",
+  spare: "Dependency spares",
+  temp: "Temp folders",
+  "derived-data": "Xcode build folders of removed worktrees",
+};
+
+function section(kind: SweepKind, items: SweepRemoved[]): string {
+  const bytes = items.reduce((n, i) => n + i.bytes, 0);
+  const rows = items
+    .sort((a, b) => b.bytes - a.bytes)
+    .map((i) => `- \`${i.path}\` · ${mb(i.bytes)} · ${i.why}`);
+  return `## ${TITLES[kind]}: ${items.length} (${mb(bytes)})\n\n${rows.join("\n")}\n`;
+}
+
+export function renderDiskReport(summary: SweepSummary, at: Date): string {
+  const all = [...summary.removed, ...(summary.pending ?? [])];
+  const kinds = Object.keys(TITLES) as SweepKind[];
+  const total = all.reduce((n, i) => n + i.bytes, 0);
+  const why = new Map<string, number>();
+  for (const k of summary.skipped) {
+    const r = k.reason.replace(/\(.*\)/, "").trim();
+    why.set(r, (why.get(r) ?? 0) + 1);
+  }
+  return [
+    `# Disk clean-up dry run, ${at.toISOString()}`,
+    "",
+    `Would remove ${all.length} items, ${mb(total)}. Nothing was removed.`,
+    `Full clean-up is ${summary.armed ? "on" : "off"} on this machine.`,
+    "",
+    ...kinds
+      .map((k) => all.filter((i) => i.kind === k))
+      .filter((items) => items.length)
+      .map((items) => section(items[0].kind, items)),
+    `## Kept: ${summary.skipped.length}`,
+    "",
+    ...[...why].map(([r, n]) => `- ${n} × ${r}`),
+    "",
+    "### Kept, in detail",
+    "",
+    ...summary.skipped
+      .filter((s) => !/less than \d+ day/.test(s.reason))
+      .slice(0, 300)
+      .map((s) => `- \`${s.path}\` · ${s.reason}`),
+    "",
+  ].join("\n");
+}
+
+// A second dry run the same day gets -2, -3, never overwrites the first.
+export function writeDiskReport(
+  summary: SweepSummary,
+  at = new Date(),
+  dir = agentosHome("audits", "disk-cleanup")
+): string {
+  fs.mkdirSync(dir, { recursive: true });
+  const day = [
+    at.getFullYear(),
+    String(at.getMonth() + 1).padStart(2, "0"),
+    String(at.getDate()).padStart(2, "0"),
+  ].join("-");
+  let file = path.join(dir, `${day}.md`);
+  for (let n = 2; fs.existsSync(file); n++)
+    file = path.join(dir, `${day}-${n}.md`);
+  fs.writeFileSync(file, renderDiskReport(summary, at), { mode: 0o600 });
+  return file;
+}