@indigoai-us/hq-cloud 6.15.0 → 6.15.2

package/src/cli/reindex-knowledge.ts

@@ -1,450 +0,0 @@
-/**
- * Legacy knowledge-repo materialization — the migration arm of policy
- * `knowledge-repositories-never-symlink` (hq-core).
- *
- * Every knowledge repository must be a REAL directory at its canonical HQ
- * path (`core/knowledge/`, `personal/knowledge/`, `companies/{co}/knowledge/`),
- * with git initialized inside that directory when independent history is
- * needed. The legacy layout — a repo under `repos/` (or any separate git
- * repo) with the knowledge path symlinked to it — breaks cloud sync: sync
- * records a directory symlink as a ~50-byte vault marker, so teammates
- * receive nothing.
- *
- * On every reindex this module scours the canonical knowledge locations for
- * that legacy shape and migrates each occurrence in place:
- *
- *   1. `git pull --ff-only` the legacy repo (best-effort — offline or dirty
- *      repos still migrate with their local content),
- *   2. copy the link target to a temp sibling — INCLUDING `.git/` when the
- *      target is the repo root, so history is preserved as a supported
- *      embedded repo at the canonical path,
- *   3. swap the symlink for the copied real directory, and
- *   4. remove the legacy repo under `repos/` once every symlink that
- *      referenced it has been materialized and nothing else lives in it
- *      (a repo whose knowledge was only a subdirectory is left in place).
- *
- * Deliberately NOT migrated:
- *   - package-manager links from `core/knowledge/` into the pack knowledge
- *     dirs under `core/packages/` — immutable pack mounts, a supported
- *     non-repo link;
- *   - symlinks that resolve inside the HQ tree itself (aliases tracked by HQ
- *     git are not separate repositories);
- *   - real directories with an embedded `.git/` — already the supported form;
- *   - broken symlinks — warned about and left for a human, since there is no
- *     content to materialize.
- *
- * Steady-state cost is a handful of lstats: git subprocesses are spawned only
- * when a symlink candidate is actually present. Everything is best-effort —
- * a failed migration warns to stderr and never fails the reindex. Escape
- * hatch: HQ_REINDEX_SKIP_KNOWLEDGE_MIGRATION=1 disables the whole pass.
- */
-import { spawnSync } from "child_process";
-import * as fs from "fs";
-import * as path from "path";
-
-function warn(msg: string): void {
-  process.stderr.write(`${msg}\n`);
-}
-
-function lstatOrNull(p: string): fs.Stats | null {
-  try {
-    return fs.lstatSync(p);
-  } catch {
-    return null;
-  }
-}
-
-function isDir(p: string): boolean {
-  try {
-    return fs.statSync(p).isDirectory();
-  } catch {
-    return false;
-  }
-}
-
-/** Non-hidden entries of `dir`, sorted; [] when missing/unreadable. */
-function entries(dir: string): string[] {
-  try {
-    return fs
-      .readdirSync(dir)
-      .filter((n) => !n.startsWith("."))
-      .sort();
-  } catch {
-    return [];
-  }
-}
-
-function realpathOrNull(p: string): string | null {
-  try {
-    return fs.realpathSync(p);
-  } catch {
-    return null;
-  }
-}
-
-/** True when `child` is `parent` or lexically inside it. */
-function isUnder(child: string, parent: string): boolean {
-  const rel = path.relative(parent, child);
-  return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
-}
-
-/** `git rev-parse --show-toplevel` for `dir`, realpath'd; null when not in a repo. */
-function gitToplevel(dir: string): string | null {
-  try {
-    const res = spawnSync("git", ["-C", dir, "rev-parse", "--show-toplevel"], {
-      encoding: "utf8",
-      stdio: ["ignore", "pipe", "pipe"],
-      timeout: 15_000,
-    });
-    if (res.status !== 0) return null;
-    const top = res.stdout.trim();
-    return top ? realpathOrNull(top) ?? top : null;
-  } catch {
-    return null;
-  }
-}
-
-/** One legacy symlink occurrence: canonical link path → resolved target. */
-interface LegacyLink {
-  /** The canonical knowledge path currently occupied by the symlink. */
-  linkPath: string;
-  /** Link path relative to the HQ root (for messages). */
-  linkRel: string;
-  /** Resolved (real) target directory the symlink points at. */
-  target: string;
-  /** Toplevel of the separate git repo the target belongs to, if any. */
-  repoTop: string | null;
-}
-
-/**
- * Candidate knowledge locations, per the HQ layout contract. Only these are
- * scanned — knowledge repositories are defined to live at canonical paths, so
- * a full-tree walk would add cost without adding coverage.
- */
-function knowledgeCandidates(root: string): string[] {
-  const out: string[] = [];
-  const push = (p: string): void => {
-    if (lstatOrNull(p)) out.push(p);
-  };
-
-  for (const slug of entries(path.join(root, "companies"))) {
-    const knowledge = path.join(root, "companies", slug, "knowledge");
-    push(knowledge);
-    // One level deep: /newcompany-era docs showed per-base links such as
-    // companies/acme/knowledge/acme → repos/private/knowledge-acme.
-    const st = lstatOrNull(knowledge);
-    if (st?.isDirectory()) {
-      for (const base of entries(knowledge)) push(path.join(knowledge, base));
-    }
-  }
-
-  const personal = path.join(root, "personal", "knowledge");
-  push(personal);
-  if (lstatOrNull(personal)?.isDirectory()) {
-    for (const base of entries(personal)) push(path.join(personal, base));
-  }
-
-  const coreKnowledge = path.join(root, "core", "knowledge");
-  for (const entry of entries(coreKnowledge)) {
-    const p = path.join(coreKnowledge, entry);
-    if (entry === "public" || entry === "private") {
-      for (const base of entries(p)) push(path.join(p, base));
-    } else {
-      push(p); // flat starter-kit layout
-    }
-  }
-  return out;
-}
-
-/**
- * Classify a symlink candidate. Returns the legacy occurrence to migrate, or
- * null when the link is supported (pack mount, in-tree alias) or unusable
- * (broken — warned here).
- */
-function classifyLink(root: string, linkPath: string, rootTop: string | null): LegacyLink | null {
-  const linkRel = path.relative(root, linkPath);
-  const target = realpathOrNull(linkPath);
-  if (target === null) {
-    warn(
-      `reindex: knowledge symlink '${linkRel}' is broken (target missing) — nothing to materialize; remove or repair it by hand`,
-    );
-    return null;
-  }
-  if (!isDir(target)) return null; // file links are not knowledge repositories
-
-  // Supported pack mount: core/knowledge/* → core/packages/*/knowledge/*.
-  if (isUnder(target, path.join(root, "core", "packages"))) return null;
-
-  const repoTop = gitToplevel(target);
-  const underRepos = isUnder(target, path.join(root, "repos"));
-  const separateRepo = repoTop !== null && repoTop !== rootTop;
-
-  // Legacy iff the target lives under repos/ (code-only tree) or is a separate
-  // git repo OUTSIDE the HQ tree. Anything else is left alone: an in-HQ alias
-  // is not a knowledge repository, and an in-HQ symlink to an embedded repo at
-  // another canonical path (e.g. a retired-release mirror core/knowledge/<x> →
-  // personal/knowledge/<x> whose personal side has its own .git) is the
-  // mirror-cleanup pass's to prune — materializing it here would double-surface
-  // the same knowledge as a real duplicate before that cleanup runs.
-  const insideHq = isUnder(target, root);
-  if (!underRepos && (insideHq || !separateRepo)) return null;
-
-  return { linkPath, linkRel, target, repoTop: separateRepo ? repoTop : null };
-}
-
-/** Best-effort `git pull --ff-only`; failures warn and never block migration. */
-function pullLegacyRepo(root: string, repoTop: string): void {
-  try {
-    // A repo with no remote has nothing to pull; migrate its local content
-    // without the noise of a doomed pull attempt.
-    const remotes = spawnSync("git", ["-C", repoTop, "remote"], {
-      encoding: "utf8",
-      stdio: ["ignore", "pipe", "pipe"],
-      timeout: 15_000,
-    });
-    if (remotes.status !== 0 || !remotes.stdout.trim()) return;
-    const res = spawnSync("git", ["-C", repoTop, "pull", "--ff-only", "--quiet"], {
-      encoding: "utf8",
-      stdio: ["ignore", "pipe", "pipe"],
-      timeout: 60_000,
-    });
-    if (res.status !== 0) {
-      const detail = `${res.stderr ?? ""}`.trim().split("\n")[0] || `exit ${res.status}`;
-      warn(
-        `reindex: could not pull legacy knowledge repo '${path.relative(root, repoTop)}' (${detail}); migrating its local content as-is`,
-      );
-    }
-  } catch (err) {
-    warn(
-      `reindex: could not pull legacy knowledge repo '${path.relative(root, repoTop)}' (${(err as Error).message}); migrating its local content as-is`,
-    );
-  }
-}
-
-/**
- * Paths under a link's target that must NOT be materialized, and whether their
- * presence means the legacy repo has to survive the migration.
- */
-interface CopyExclusions {
-  /** Absolute paths (files or dirs) to skip during the copy. */
-  paths: Set<string>;
-  /** Human reasons the legacy repo must be kept in place (empty → deletable). */
-  keepLegacyReasons: string[];
-}
-
-/**
- * Work out what the copy must skip.
- *
- * - A `.git` FILE (not directory) marks a linked git worktree / gitdir
- *   pointer: copying it verbatim would leave the materialized directory
- *   pointing at metadata inside the main repository, and deleting the legacy
- *   path would strand that registration. Skip the pointer file and keep the
- *   legacy worktree for `git worktree remove` by hand.
- * - Files the legacy repo git-ignores (e.g. a local `credentials.json`) stay
- *   out of the copy: cloud sync's ignore layer reads only the HQ ROOT
- *   `.gitignore`, so materializing them under a synced knowledge path would
- *   upload files their own repo deliberately kept local. The legacy repo is
- *   kept so those local-only files are never destroyed.
- */
-function copyExclusions(root: string, link: LegacyLink): CopyExclusions {
-  const paths = new Set<string>();
-  const keepLegacyReasons: string[] = [];
-
-  const gitEntry = path.join(link.target, ".git");
-  if (lstatOrNull(gitEntry)?.isFile()) {
-    paths.add(gitEntry);
-    keepLegacyReasons.push(
-      "it is a linked git worktree (its history lives in the main repository; remove it with `git worktree remove` after verifying the migrated content)",
-    );
-  }
-
-  if (link.repoTop) {
-    try {
-      const res = spawnSync(
-        "git",
-        ["-C", link.repoTop, "ls-files", "--others", "--ignored", "--exclude-standard", "-z"],
-        { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 30_000 },
-      );
-      if (res.status === 0) {
-        const ignored = res.stdout
-          .split("\0")
-          .filter(Boolean)
-          .map((rel) => path.join(link.repoTop as string, rel))
-          .filter((abs) => isUnder(abs, link.target));
-        if (ignored.length > 0) {
-          for (const abs of ignored) paths.add(abs);
-          keepLegacyReasons.push(
-            `it holds ${ignored.length} git-ignored local file(s) that must not enter a cloud-synced knowledge path`,
-          );
-        }
-      } else {
-        keepLegacyReasons.push(
-          "its git-ignored local files could not be enumerated, so the copy may be incomplete",
-        );
-      }
-    } catch (err) {
-      keepLegacyReasons.push(
-        `its git-ignored local files could not be enumerated (${(err as Error).message})`,
-      );
-    }
-  }
-  return { paths, keepLegacyReasons };
-}
-
-/**
- * Replace the symlink at `link.linkPath` with a real directory holding a full
- * copy of its target (`.git/` included when the target is the repo root).
- * Relative symlinks inside the content are copied verbatim so they still
- * resolve after the legacy repo is removed. Returns true when the canonical
- * path ends as a real, non-symlink directory.
- */
-function materializeLink(link: LegacyLink, exclude: Set<string>): boolean {
-  // PID-suffixed so a standalone `hq reindex` overlapping the post-sync/rescue
-  // inline reindex (skipLock — an overlap that pass tolerates by design) can
-  // never rename/delete a temp directory out from under the other process.
-  const tmp = path.join(
-    path.dirname(link.linkPath),
-    `.${path.basename(link.linkPath)}.hq-knowledge-migrate.${process.pid}.tmp`,
-  );
-  try {
-    fs.rmSync(tmp, { recursive: true, force: true });
-    fs.cpSync(link.target, tmp, {
-      recursive: true,
-      verbatimSymlinks: true,
-      filter: (src) => !exclude.has(src),
-    });
-  } catch (err) {
-    warn(
-      `reindex: could not copy knowledge content for '${link.linkRel}' from '${link.target}' (${(err as Error).message}); leaving the legacy symlink in place`,
-    );
-    fs.rmSync(tmp, { recursive: true, force: true });
-    return false;
-  }
-
-  const previousTarget = (() => {
-    try {
-      return fs.readlinkSync(link.linkPath);
-    } catch {
-      return null;
-    }
-  })();
-
-  try {
-    fs.rmSync(link.linkPath);
-    fs.renameSync(tmp, link.linkPath);
-  } catch (err) {
-    // Restore the link if the swap died between rm and rename, so the
-    // knowledge path never ends up missing entirely.
-    if (previousTarget !== null && lstatOrNull(link.linkPath) === null) {
-      try {
-        fs.symlinkSync(previousTarget, link.linkPath);
-      } catch {
-        /* best-effort restore */
-      }
-    }
-    fs.rmSync(tmp, { recursive: true, force: true });
-    warn(
-      `reindex: could not swap knowledge symlink '${link.linkRel}' for its materialized copy (${(err as Error).message}); leaving the legacy layout in place`,
-    );
-    return false;
-  }
-
-  const st = lstatOrNull(link.linkPath);
-  return st !== null && st.isDirectory() && !st.isSymbolicLink();
-}
-
-/**
- * Scour the HQ tree for legacy knowledge-repo layouts and migrate each to a
- * real canonical directory, removing fully-migrated legacy repos under
- * `repos/`. Best-effort throughout; never throws.
- */
-export function materializeLegacyKnowledgeRepos(root: string): void {
-  if (process.env.HQ_REINDEX_SKIP_KNOWLEDGE_MIGRATION === "1") return;
-  try {
-    const symlinkCandidates = knowledgeCandidates(root).filter((p) =>
-      lstatOrNull(p)?.isSymbolicLink(),
-    );
-    if (symlinkCandidates.length === 0) return;
-
-    const rootTop = gitToplevel(root);
-    const links = symlinkCandidates
-      .map((p) => classifyLink(root, p, rootTop))
-      .filter((l): l is LegacyLink => l !== null);
-    if (links.length === 0) return;
-
-    // Pull each distinct legacy repo once, before any copies are taken.
-    const pulled = new Set<string>();
-    for (const link of links) {
-      if (link.repoTop && !pulled.has(link.repoTop)) {
-        pulled.add(link.repoTop);
-        pullLegacyRepo(root, link.repoTop);
-      }
-    }
-
-    // Materialize every link first; legacy repos are only removed once no
-    // remaining symlink references them (aliases to one repo each get a copy).
-    const byRepo = new Map<
-      string,
-      { targets: Set<string>; allMigrated: boolean; keepReasons: Set<string> }
-    >();
-    for (const link of links) {
-      const exclusions = copyExclusions(root, link);
-      const ok = materializeLink(link, exclusions.paths);
-      if (ok) {
-        warn(
-          `reindex: materialized knowledge repo at '${link.linkRel}' (was a symlink to '${path.relative(root, link.target)}')`,
-        );
-      }
-      if (link.repoTop) {
-        const rec = byRepo.get(link.repoTop) ?? {
-          targets: new Set<string>(),
-          allMigrated: true,
-          keepReasons: new Set<string>(),
-        };
-        rec.targets.add(link.target);
-        rec.allMigrated = rec.allMigrated && ok;
-        for (const reason of exclusions.keepLegacyReasons) rec.keepReasons.add(reason);
-        byRepo.set(link.repoTop, rec);
-      }
-    }
-
-    // Remove a legacy repo only when it (a) lives under repos/ inside this HQ,
-    // (b) had every referencing symlink migrated, and (c) held nothing beyond
-    // the knowledge itself — i.e. every link targeted the repo root. A repo
-    // whose knowledge was a subdirectory (e.g. <repo>/docs) is left in place.
-    for (const [repoTop, rec] of byRepo) {
-      const repoRel = path.relative(root, repoTop);
-      if (!isUnder(repoTop, path.join(root, "repos"))) {
-        warn(
-          `reindex: legacy knowledge repo '${repoTop}' lives outside this HQ's repos/ — content was materialized; remove the old repo yourself once verified`,
-        );
-        continue;
-      }
-      if (!rec.allMigrated) continue;
-      if (rec.keepReasons.size > 0) {
-        for (const reason of rec.keepReasons) {
-          warn(`reindex: keeping legacy knowledge repo '${repoRel}' — ${reason}`);
-        }
-        continue;
-      }
-      if (![...rec.targets].every((t) => t === repoTop)) {
-        warn(
-          `reindex: legacy repo '${repoRel}' also holds content beyond the migrated knowledge subdirectory; leaving it in place`,
-        );
-        continue;
-      }
-      try {
-        fs.rmSync(repoTop, { recursive: true, force: true });
-        warn(`reindex: removed legacy knowledge repo '${repoRel}' (fully materialized inline)`);
-      } catch (err) {
-        warn(
-          `reindex: could not remove legacy knowledge repo '${repoRel}' (${(err as Error).message}); its content was already materialized`,
-        );
-      }
-    }
-  } catch (err) {
-    warn(
-      `reindex: legacy knowledge-repo migration pass failed (${(err as Error).message}); skipping`,
-    );
-  }
-}