@tiens.nguyen/gu-cli 1.0.787 → 1.0.789

package/clear-logs.mjs

@@ -0,0 +1,352 @@
+/**
+ * "clear logs" (#329) — what is taking the disk under ~/.gonext, and a way to give it back.
+ *
+ * Asked for after this Mac's disk filled three times in one day and took the local MongoDB down
+ * with it (#328). A click on the notice row's "clear logs" (or /clear-logs) DELETES NOTHING at
+ * first: it measures, shows the logs beside the two things that are actually big — the RAG indexes
+ * and the downloaded API versions — with a total, and then asks about each of the three.
+ *
+ * THREE DIFFERENT KINDS OF "CLEAR", because losing each costs something different:
+ *
+ *   logs  nothing. But the worker and the local API hold their log OPEN: a deleted open file frees
+ *         no space on macOS/Linux until the holder closes it (and the holder keeps writing into
+ *         the unlinked file), and on Windows the delete fails outright. So the live "<name>.log"
+ *         is TRUNCATED IN PLACE and only rotations and finished run-logs are deleted — the same
+ *         rule rotateIfLarge (log-policy.mjs) already lives by.
+ *   rag   minutes of embedding per folder: an index is rebuilt the next time its folder is used.
+ *   api   nothing, PROVIDED the version in use stays. One folder per installed version, none ever
+ *         removed; deleting the running one would break every open terminal's turn. So "clear
+ *         api" only ever means the OLDER versions.
+ *
+ * SELECTED BY AN EXPLICIT LIST, never by a "*.log*" / "*.jsonl" sweep of the home: sessions/archive
+ * is conversation history (#278) and rag/ shards are also .jsonl. Anything not named here is not
+ * touched, whatever the answers.
+ *
+ * Everything takes `home` so the tests run against a real temp folder, not this machine's.
+ */
+import { readdir, readFile, rm, rmdir, stat, truncate, unlink } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { pickUsableVersion, compareVersions } from "./version-check.mjs";
+
+/** The logs gu writes at the top of its home. `<name>` is live; `<name>.N` are its rotations. */
+export const LIVE_LOGS = ["worker.log", "local-api.log", "model.log", "embed.log"];
+/** Folders that hold nothing but finished log files. */
+export const LOG_DIRS = ["run-logs", "logs"];
+/** The link appears once the logs pass this — below it the row stays quiet. */
+export const LOGS_NOTICE_BYTES = 50 * 1024 * 1024;
+
+const guHome = (home = homedir()) => join(home, ".gonext");
+
+/** "740 KB", "15 MB", "4.9 GB" — two significant figures, the way `du -h` reads. */
+export function humanSize(bytes) {
+  const n = Math.max(0, Number(bytes) || 0);
+  const units = [["GB", 1024 ** 3], ["MB", 1024 ** 2], ["KB", 1024]];
+  for (const [unit, size] of units) {
+    if (n < size) continue;
+    const v = n / size;
+    // 9.96 would print as "10.0" with one decimal — let it round up to the integer form instead.
+    return `${v < 9.95 ? v.toFixed(1) : Math.round(v)} ${unit}`;
+  }
+  return `${Math.round(n)} B`;
+}
+
+const plural = (n, one, many = one + "s") => `${n} ${n === 1 ? one : many}`;
+
+/**
+ * What a file takes ON DISK — its blocks, which is what clearing gives back and what `du` reports.
+ * The byte length undercounts thousands of small files (an API version read as 85 MB here against
+ * du's 120 MB). Windows reports no blocks; the length is the best there is.
+ */
+const onDisk = (s) => (s.blocks > 0 ? s.blocks * 512 : s.size);
+
+async function fileSize(p) {
+  try {
+    const s = await stat(p);
+    return s.isFile() ? onDisk(s) : -1;
+  } catch {
+    return -1;
+  }
+}
+
+/** Bytes under a folder. Children are walked together: an API version is thousands of files. */
+async function dirSize(p) {
+  let entries;
+  try {
+    entries = await readdir(p, { withFileTypes: true });
+  } catch {
+    return 0;
+  }
+  const sizes = await Promise.all(
+    entries.map(async (e) => {
+      const full = join(p, e.name);
+      if (e.isDirectory()) return dirSize(full);
+      if (e.isSymbolicLink()) return 0; // never followed: a link out of the home is not ours to count
+      try {
+        return onDisk(await stat(full));
+      } catch {
+        return 0;
+      }
+    })
+  );
+  return sizes.reduce((a, b) => a + b, 0);
+}
+
+/**
+ * The logs: each live file and its rotations by name, each log folder as a count.
+ * → { items: [{ name, bytes, kind: "live" | "rotation" | "dir", files?, paths? }], bytes, files }
+ */
+export async function surveyLogs({ home = homedir() } = {}) {
+  const root = guHome(home);
+  let names = [];
+  try {
+    names = await readdir(root);
+  } catch {
+    /* no home yet — nothing to report */
+  }
+  const items = [];
+  for (const live of LIVE_LOGS) {
+    const bytes = await fileSize(join(root, live));
+    if (bytes >= 0) items.push({ name: live, bytes, kind: "live", path: join(root, live) });
+    const rotations = names
+      .filter((n) => n.startsWith(live + ".") && /^\d+$/.test(n.slice(live.length + 1)))
+      .sort((a, b) => Number(a.slice(live.length + 1)) - Number(b.slice(live.length + 1)));
+    for (const r of rotations) {
+      const b = await fileSize(join(root, r));
+      if (b >= 0) items.push({ name: r, bytes: b, kind: "rotation", path: join(root, r) });
+    }
+  }
+  for (const dir of LOG_DIRS) {
+    let files = [];
+    try {
+      files = (await readdir(join(root, dir), { withFileTypes: true })).filter(
+        (e) => e.isFile() && /\.log(\.\d+)?$/.test(e.name)
+      );
+    } catch {
+      continue;
+    }
+    const paths = files.map((e) => join(root, dir, e.name));
+    let bytes = 0;
+    for (const p of paths) bytes += Math.max(0, await fileSize(p));
+    if (paths.length) items.push({ name: dir, bytes, kind: "dir", files: paths.length, paths });
+  }
+  return {
+    items,
+    bytes: items.reduce((n, i) => n + i.bytes, 0),
+    files: items.reduce((n, i) => n + (i.kind === "dir" ? i.files : 1), 0),
+  };
+}
+
+/**
+ * Every local index: ~/.gonext/rag/<folder hash>/<source hash>/. The hashes say nothing to a
+ * person, so the name comes from the manifest the agent wrote (`sourceUrl`). `missing` = that
+ * source was a folder on this machine and is gone — the index can never be used again.
+ */
+export async function surveyRag({ home = homedir() } = {}) {
+  const root = join(guHome(home), "rag");
+  const indexes = [];
+  let folders = [];
+  try {
+    folders = (await readdir(root, { withFileTypes: true })).filter((e) => e.isDirectory());
+  } catch {
+    return { indexes, bytes: 0 };
+  }
+  for (const f of folders) {
+    let sources = [];
+    try {
+      sources = (await readdir(join(root, f.name), { withFileTypes: true })).filter((e) => e.isDirectory());
+    } catch {
+      continue;
+    }
+    for (const s of sources) {
+      const dir = join(root, f.name, s.name);
+      let source = "";
+      try {
+        source = String(JSON.parse(await readFile(join(dir, "manifest.json"), "utf8"))?.sourceUrl ?? "");
+      } catch {
+        /* no manifest — an interrupted index; still counted, still clearable */
+      }
+      const isPath = source.startsWith("/") || /^[A-Za-z]:[\\/]/.test(source);
+      indexes.push({
+        dir,
+        parent: join(root, f.name),
+        source,
+        label: source ? source.replace(/[\\/]+$/, "").split(/[\\/]/).pop() || source : `${s.name.slice(0, 8)}…`,
+        missing: isPath && !existsSync(source),
+        bytes: await dirSize(dir),
+      });
+    }
+  }
+  indexes.sort((a, b) => b.bytes - a.bytes);
+  return { indexes, bytes: indexes.reduce((n, i) => n + i.bytes, 0) };
+}
+
+/**
+ * The installed API versions, and which must stay.
+ *
+ * KEPT: the version answering on the port right now (`runningVersion`, from /api/health), the one
+ * `gu-cli api start` would pick (newest that has its dependencies — #181), and the newest folder
+ * even when it is not runnable yet (an upgrade in progress). Everything else is "old".
+ */
+export async function surveyApi({ home = homedir(), runningVersion = "" } = {}) {
+  const root = join(guHome(home), "api");
+  let names = [];
+  try {
+    names = (await readdir(root, { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => e.name);
+  } catch {
+    return { versions: [], bytes: 0, oldBytes: 0, old: [], inUse: "" };
+  }
+  names.sort((a, b) => compareVersions(a, b));
+  const usable = pickUsableVersion(names, (v) => existsSync(join(root, v, "node_modules", "mongodb", "package.json")));
+  const running = names.includes(runningVersion) ? runningVersion : "";
+  const keep = new Set([running, usable, names[names.length - 1]].filter(Boolean));
+  const sizes = await Promise.all(names.map((v) => dirSize(join(root, v))));
+  const versions = names.map((version, i) => ({ version, dir: join(root, version), bytes: sizes[i], keep: keep.has(version) }));
+  const old = versions.filter((v) => !v.keep);
+  return {
+    versions,
+    old,
+    bytes: versions.reduce((n, v) => n + v.bytes, 0),
+    oldBytes: old.reduce((n, v) => n + v.bytes, 0),
+    inUse: running || usable || "",
+  };
+}
+
+/** All three, measured together. */
+export async function surveyDisk({ home = homedir(), runningVersion = "" } = {}) {
+  const [logs, rag, api] = await Promise.all([surveyLogs({ home }), surveyRag({ home }), surveyApi({ home, runningVersion })]);
+  return { logs, rag, api, total: logs.bytes + rag.bytes + api.bytes };
+}
+
+/** Lay `parts` out after `lead`, breaking only between parts; later rows start at `indent`. */
+function wrapParts(lead, parts, width, { sep = " · ", indent = lead.length } = {}) {
+  const pad = " ".repeat(indent);
+  const rows = [];
+  let cur = lead;
+  let empty = true;
+  for (const part of parts) {
+    if (!empty && cur.length + sep.length + part.length > width) {
+      rows.push(cur);
+      cur = pad + part;
+    } else {
+      cur += (empty ? "" : sep) + part;
+    }
+    empty = false;
+  }
+  rows.push(cur.trimEnd());
+  // a single part longer than the row is cut, never wrapped — a wrapped row moves everything under it
+  return rows.map((r) => (r.length > width ? r.slice(0, Math.max(1, width - 1)) + "…" : r));
+}
+
+/**
+ * What the click shows before it asks anything: the logs with their files, RAG and API with what
+ * they are, and the total. Plain text rows that fit `width` — the caller paints and indents them.
+ * A part that is not on this machine (no api/ outside Local mode) has no row.
+ */
+export function summaryRows(survey, { width = 76, top = 3 } = {}) {
+  const w = Math.max(30, width | 0);
+  const { logs, rag, api } = survey;
+  const parts = [
+    { name: "Logs", bytes: logs.bytes, detail: logs.items.map((i) => i.kind === "dir"
+      ? `${i.name} ${plural(i.files, "file")} ${humanSize(i.bytes)}`
+      : `${i.name} ${humanSize(i.bytes)}`), none: "none" },
+  ];
+  if (rag.indexes.length) {
+    const shown = rag.indexes.slice(0, top);
+    const rest = rag.indexes.slice(top);
+    const detail = shown.map((i) => `${i.label} ${humanSize(i.bytes)}`);
+    if (rest.length) detail.push(`${rest.length} more ${humanSize(rest.reduce((n, i) => n + i.bytes, 0))}`);
+    parts.push({ name: "RAG", bytes: rag.bytes, intro: plural(rag.indexes.length, "folder index", "folder indexes") + ":", detail });
+  }
+  if (api.versions.length) {
+    const detail = [plural(api.versions.length, "version")];
+    if (api.inUse) detail.push(`${api.inUse} in use`);
+    if (api.old.length) detail.push(`${plural(api.old.length, "older one")} ${humanSize(api.oldBytes)}`);
+    parts.push({ name: "API", bytes: api.bytes, detail });
+  }
+  const nameW = Math.max(5, ...parts.map((p) => p.name.length));
+  const sizeW = Math.max(humanSize(survey.total).length, ...parts.map((p) => humanSize(p.bytes).length));
+  const lead = (name, bytes) => `${name.padEnd(nameW)}  ${humanSize(bytes).padStart(sizeW)}  `;
+  const rows = [];
+  for (const p of parts) {
+    const head = lead(p.name, p.bytes);
+    // "22 folder indexes:" introduces the list — the list wraps under the first index, the
+    // other parts under their own first word
+    rows.push(...wrapParts(head + (p.intro ? p.intro + " " : ""), p.detail.length ? p.detail : [p.none ?? ""], w, { indent: head.length }));
+  }
+  rows.push(lead("Total", survey.total).trimEnd());
+  return rows;
+}
+
+/**
+ * Clear the logs `surveyLogs` found: truncate the live ones, delete rotations and log folders'
+ * files. → { freed, truncated, deleted, failed: [name] } — `freed` is what really went, so a file
+ * Windows refused to let go of is reported, not counted.
+ */
+export async function clearLogs(logs) {
+  const out = { freed: 0, truncated: 0, deleted: 0, failed: [] };
+  for (const item of logs?.items ?? []) {
+    if (item.kind === "live") {
+      try {
+        const before = Math.max(0, await fileSize(item.path));
+        await truncate(item.path, 0);
+        out.freed += before;
+        out.truncated += 1;
+      } catch {
+        out.failed.push(item.name);
+      }
+      continue;
+    }
+    for (const p of item.kind === "dir" ? item.paths : [item.path]) {
+      try {
+        const before = Math.max(0, await fileSize(p));
+        await unlink(p);
+        out.freed += before;
+        out.deleted += 1;
+      } catch (e) {
+        if (e?.code === "ENOENT") continue; // pruned meanwhile — gone is what was wanted
+        if (!out.failed.includes(item.name)) out.failed.push(item.name);
+      }
+    }
+  }
+  return out;
+}
+
+/** Remove the given indexes (surveyRag's rows). An emptied folder-hash directory goes with them. */
+export async function clearRagIndexes(indexes) {
+  const out = { freed: 0, removed: 0, failed: [] };
+  for (const idx of indexes ?? []) {
+    try {
+      await rm(idx.dir, { recursive: true, force: true });
+      out.freed += idx.bytes;
+      out.removed += 1;
+      await rmdir(idx.parent).catch(() => {}); // only succeeds when nothing else lives there
+    } catch {
+      out.failed.push(idx.label);
+    }
+  }
+  return out;
+}
+
+/** Remove the OLD API versions (surveyApi's `old`). A row marked keep is refused, whoever asks. */
+export async function clearOldApiVersions(versions) {
+  const out = { freed: 0, removed: 0, failed: [] };
+  for (const v of versions ?? []) {
+    if (v.keep) continue;
+    try {
+      await rm(v.dir, { recursive: true, force: true });
+      out.freed += v.bytes;
+      out.removed += 1;
+    } catch {
+      out.failed.push(v.version);
+    }
+  }
+  return out;
+}
+
+/** The notice-row item's text, or "" while the logs are too small to be worth a row. */
+export function clearLogsNoticeText(logBytes, { threshold = LOGS_NOTICE_BYTES } = {}) {
+  const n = Number(logBytes) || 0;
+  return n >= threshold ? `clear logs · ${humanSize(n)} (click)` : "";
+}