@indigoai-us/hq-cloud 6.14.49 → 6.15.0

package/src/cli/reindex-knowledge.ts

@@ -0,0 +1,450 @@
+/**
+ * Legacy knowledge-repo materialization — the migration arm of policy
+ * `knowledge-repositories-never-symlink` (hq-core).
+ *
+ * Every knowledge repository must be a REAL directory at its canonical HQ
+ * path (`core/knowledge/`, `personal/knowledge/`, `companies/{co}/knowledge/`),
+ * with git initialized inside that directory when independent history is
+ * needed. The legacy layout — a repo under `repos/` (or any separate git
+ * repo) with the knowledge path symlinked to it — breaks cloud sync: sync
+ * records a directory symlink as a ~50-byte vault marker, so teammates
+ * receive nothing.
+ *
+ * On every reindex this module scours the canonical knowledge locations for
+ * that legacy shape and migrates each occurrence in place:
+ *
+ *   1. `git pull --ff-only` the legacy repo (best-effort — offline or dirty
+ *      repos still migrate with their local content),
+ *   2. copy the link target to a temp sibling — INCLUDING `.git/` when the
+ *      target is the repo root, so history is preserved as a supported
+ *      embedded repo at the canonical path,
+ *   3. swap the symlink for the copied real directory, and
+ *   4. remove the legacy repo under `repos/` once every symlink that
+ *      referenced it has been materialized and nothing else lives in it
+ *      (a repo whose knowledge was only a subdirectory is left in place).
+ *
+ * Deliberately NOT migrated:
+ *   - package-manager links from `core/knowledge/` into the pack knowledge
+ *     dirs under `core/packages/` — immutable pack mounts, a supported
+ *     non-repo link;
+ *   - symlinks that resolve inside the HQ tree itself (aliases tracked by HQ
+ *     git are not separate repositories);
+ *   - real directories with an embedded `.git/` — already the supported form;
+ *   - broken symlinks — warned about and left for a human, since there is no
+ *     content to materialize.
+ *
+ * Steady-state cost is a handful of lstats: git subprocesses are spawned only
+ * when a symlink candidate is actually present. Everything is best-effort —
+ * a failed migration warns to stderr and never fails the reindex. Escape
+ * hatch: HQ_REINDEX_SKIP_KNOWLEDGE_MIGRATION=1 disables the whole pass.
+ */
+import { spawnSync } from "child_process";
+import * as fs from "fs";
+import * as path from "path";
+
+function warn(msg: string): void {
+  process.stderr.write(`${msg}\n`);
+}
+
+function lstatOrNull(p: string): fs.Stats | null {
+  try {
+    return fs.lstatSync(p);
+  } catch {
+    return null;
+  }
+}
+
+function isDir(p: string): boolean {
+  try {
+    return fs.statSync(p).isDirectory();
+  } catch {
+    return false;
+  }
+}
+
+/** Non-hidden entries of `dir`, sorted; [] when missing/unreadable. */
+function entries(dir: string): string[] {
+  try {
+    return fs
+      .readdirSync(dir)
+      .filter((n) => !n.startsWith("."))
+      .sort();
+  } catch {
+    return [];
+  }
+}
+
+function realpathOrNull(p: string): string | null {
+  try {
+    return fs.realpathSync(p);
+  } catch {
+    return null;
+  }
+}
+
+/** True when `child` is `parent` or lexically inside it. */
+function isUnder(child: string, parent: string): boolean {
+  const rel = path.relative(parent, child);
+  return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
+}
+
+/** `git rev-parse --show-toplevel` for `dir`, realpath'd; null when not in a repo. */
+function gitToplevel(dir: string): string | null {
+  try {
+    const res = spawnSync("git", ["-C", dir, "rev-parse", "--show-toplevel"], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+      timeout: 15_000,
+    });
+    if (res.status !== 0) return null;
+    const top = res.stdout.trim();
+    return top ? realpathOrNull(top) ?? top : null;
+  } catch {
+    return null;
+  }
+}
+
+/** One legacy symlink occurrence: canonical link path → resolved target. */
+interface LegacyLink {
+  /** The canonical knowledge path currently occupied by the symlink. */
+  linkPath: string;
+  /** Link path relative to the HQ root (for messages). */
+  linkRel: string;
+  /** Resolved (real) target directory the symlink points at. */
+  target: string;
+  /** Toplevel of the separate git repo the target belongs to, if any. */
+  repoTop: string | null;
+}
+
+/**
+ * Candidate knowledge locations, per the HQ layout contract. Only these are
+ * scanned — knowledge repositories are defined to live at canonical paths, so
+ * a full-tree walk would add cost without adding coverage.
+ */
+function knowledgeCandidates(root: string): string[] {
+  const out: string[] = [];
+  const push = (p: string): void => {
+    if (lstatOrNull(p)) out.push(p);
+  };
+
+  for (const slug of entries(path.join(root, "companies"))) {
+    const knowledge = path.join(root, "companies", slug, "knowledge");
+    push(knowledge);
+    // One level deep: /newcompany-era docs showed per-base links such as
+    // companies/acme/knowledge/acme → repos/private/knowledge-acme.
+    const st = lstatOrNull(knowledge);
+    if (st?.isDirectory()) {
+      for (const base of entries(knowledge)) push(path.join(knowledge, base));
+    }
+  }
+
+  const personal = path.join(root, "personal", "knowledge");
+  push(personal);
+  if (lstatOrNull(personal)?.isDirectory()) {
+    for (const base of entries(personal)) push(path.join(personal, base));
+  }
+
+  const coreKnowledge = path.join(root, "core", "knowledge");
+  for (const entry of entries(coreKnowledge)) {
+    const p = path.join(coreKnowledge, entry);
+    if (entry === "public" || entry === "private") {
+      for (const base of entries(p)) push(path.join(p, base));
+    } else {
+      push(p); // flat starter-kit layout
+    }
+  }
+  return out;
+}
+
+/**
+ * Classify a symlink candidate. Returns the legacy occurrence to migrate, or
+ * null when the link is supported (pack mount, in-tree alias) or unusable
+ * (broken — warned here).
+ */
+function classifyLink(root: string, linkPath: string, rootTop: string | null): LegacyLink | null {
+  const linkRel = path.relative(root, linkPath);
+  const target = realpathOrNull(linkPath);
+  if (target === null) {
+    warn(
+      `reindex: knowledge symlink '${linkRel}' is broken (target missing) — nothing to materialize; remove or repair it by hand`,
+    );
+    return null;
+  }
+  if (!isDir(target)) return null; // file links are not knowledge repositories
+
+  // Supported pack mount: core/knowledge/* → core/packages/*/knowledge/*.
+  if (isUnder(target, path.join(root, "core", "packages"))) return null;
+
+  const repoTop = gitToplevel(target);
+  const underRepos = isUnder(target, path.join(root, "repos"));
+  const separateRepo = repoTop !== null && repoTop !== rootTop;
+
+  // Legacy iff the target lives under repos/ (code-only tree) or is a separate
+  // git repo OUTSIDE the HQ tree. Anything else is left alone: an in-HQ alias
+  // is not a knowledge repository, and an in-HQ symlink to an embedded repo at
+  // another canonical path (e.g. a retired-release mirror core/knowledge/<x> →
+  // personal/knowledge/<x> whose personal side has its own .git) is the
+  // mirror-cleanup pass's to prune — materializing it here would double-surface
+  // the same knowledge as a real duplicate before that cleanup runs.
+  const insideHq = isUnder(target, root);
+  if (!underRepos && (insideHq || !separateRepo)) return null;
+
+  return { linkPath, linkRel, target, repoTop: separateRepo ? repoTop : null };
+}
+
+/** Best-effort `git pull --ff-only`; failures warn and never block migration. */
+function pullLegacyRepo(root: string, repoTop: string): void {
+  try {
+    // A repo with no remote has nothing to pull; migrate its local content
+    // without the noise of a doomed pull attempt.
+    const remotes = spawnSync("git", ["-C", repoTop, "remote"], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+      timeout: 15_000,
+    });
+    if (remotes.status !== 0 || !remotes.stdout.trim()) return;
+    const res = spawnSync("git", ["-C", repoTop, "pull", "--ff-only", "--quiet"], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+      timeout: 60_000,
+    });
+    if (res.status !== 0) {
+      const detail = `${res.stderr ?? ""}`.trim().split("\n")[0] || `exit ${res.status}`;
+      warn(
+        `reindex: could not pull legacy knowledge repo '${path.relative(root, repoTop)}' (${detail}); migrating its local content as-is`,
+      );
+    }
+  } catch (err) {
+    warn(
+      `reindex: could not pull legacy knowledge repo '${path.relative(root, repoTop)}' (${(err as Error).message}); migrating its local content as-is`,
+    );
+  }
+}
+
+/**
+ * Paths under a link's target that must NOT be materialized, and whether their
+ * presence means the legacy repo has to survive the migration.
+ */
+interface CopyExclusions {
+  /** Absolute paths (files or dirs) to skip during the copy. */
+  paths: Set<string>;
+  /** Human reasons the legacy repo must be kept in place (empty → deletable). */
+  keepLegacyReasons: string[];
+}
+
+/**
+ * Work out what the copy must skip.
+ *
+ * - A `.git` FILE (not directory) marks a linked git worktree / gitdir
+ *   pointer: copying it verbatim would leave the materialized directory
+ *   pointing at metadata inside the main repository, and deleting the legacy
+ *   path would strand that registration. Skip the pointer file and keep the
+ *   legacy worktree for `git worktree remove` by hand.
+ * - Files the legacy repo git-ignores (e.g. a local `credentials.json`) stay
+ *   out of the copy: cloud sync's ignore layer reads only the HQ ROOT
+ *   `.gitignore`, so materializing them under a synced knowledge path would
+ *   upload files their own repo deliberately kept local. The legacy repo is
+ *   kept so those local-only files are never destroyed.
+ */
+function copyExclusions(root: string, link: LegacyLink): CopyExclusions {
+  const paths = new Set<string>();
+  const keepLegacyReasons: string[] = [];
+
+  const gitEntry = path.join(link.target, ".git");
+  if (lstatOrNull(gitEntry)?.isFile()) {
+    paths.add(gitEntry);
+    keepLegacyReasons.push(
+      "it is a linked git worktree (its history lives in the main repository; remove it with `git worktree remove` after verifying the migrated content)",
+    );
+  }
+
+  if (link.repoTop) {
+    try {
+      const res = spawnSync(
+        "git",
+        ["-C", link.repoTop, "ls-files", "--others", "--ignored", "--exclude-standard", "-z"],
+        { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 30_000 },
+      );
+      if (res.status === 0) {
+        const ignored = res.stdout
+          .split("\0")
+          .filter(Boolean)
+          .map((rel) => path.join(link.repoTop as string, rel))
+          .filter((abs) => isUnder(abs, link.target));
+        if (ignored.length > 0) {
+          for (const abs of ignored) paths.add(abs);
+          keepLegacyReasons.push(
+            `it holds ${ignored.length} git-ignored local file(s) that must not enter a cloud-synced knowledge path`,
+          );
+        }
+      } else {
+        keepLegacyReasons.push(
+          "its git-ignored local files could not be enumerated, so the copy may be incomplete",
+        );
+      }
+    } catch (err) {
+      keepLegacyReasons.push(
+        `its git-ignored local files could not be enumerated (${(err as Error).message})`,
+      );
+    }
+  }
+  return { paths, keepLegacyReasons };
+}
+
+/**
+ * Replace the symlink at `link.linkPath` with a real directory holding a full
+ * copy of its target (`.git/` included when the target is the repo root).
+ * Relative symlinks inside the content are copied verbatim so they still
+ * resolve after the legacy repo is removed. Returns true when the canonical
+ * path ends as a real, non-symlink directory.
+ */
+function materializeLink(link: LegacyLink, exclude: Set<string>): boolean {
+  // PID-suffixed so a standalone `hq reindex` overlapping the post-sync/rescue
+  // inline reindex (skipLock — an overlap that pass tolerates by design) can
+  // never rename/delete a temp directory out from under the other process.
+  const tmp = path.join(
+    path.dirname(link.linkPath),
+    `.${path.basename(link.linkPath)}.hq-knowledge-migrate.${process.pid}.tmp`,
+  );
+  try {
+    fs.rmSync(tmp, { recursive: true, force: true });
+    fs.cpSync(link.target, tmp, {
+      recursive: true,
+      verbatimSymlinks: true,
+      filter: (src) => !exclude.has(src),
+    });
+  } catch (err) {
+    warn(
+      `reindex: could not copy knowledge content for '${link.linkRel}' from '${link.target}' (${(err as Error).message}); leaving the legacy symlink in place`,
+    );
+    fs.rmSync(tmp, { recursive: true, force: true });
+    return false;
+  }
+
+  const previousTarget = (() => {
+    try {
+      return fs.readlinkSync(link.linkPath);
+    } catch {
+      return null;
+    }
+  })();
+
+  try {
+    fs.rmSync(link.linkPath);
+    fs.renameSync(tmp, link.linkPath);
+  } catch (err) {
+    // Restore the link if the swap died between rm and rename, so the
+    // knowledge path never ends up missing entirely.
+    if (previousTarget !== null && lstatOrNull(link.linkPath) === null) {
+      try {
+        fs.symlinkSync(previousTarget, link.linkPath);
+      } catch {
+        /* best-effort restore */
+      }
+    }
+    fs.rmSync(tmp, { recursive: true, force: true });
+    warn(
+      `reindex: could not swap knowledge symlink '${link.linkRel}' for its materialized copy (${(err as Error).message}); leaving the legacy layout in place`,
+    );
+    return false;
+  }
+
+  const st = lstatOrNull(link.linkPath);
+  return st !== null && st.isDirectory() && !st.isSymbolicLink();
+}
+
+/**
+ * Scour the HQ tree for legacy knowledge-repo layouts and migrate each to a
+ * real canonical directory, removing fully-migrated legacy repos under
+ * `repos/`. Best-effort throughout; never throws.
+ */
+export function materializeLegacyKnowledgeRepos(root: string): void {
+  if (process.env.HQ_REINDEX_SKIP_KNOWLEDGE_MIGRATION === "1") return;
+  try {
+    const symlinkCandidates = knowledgeCandidates(root).filter((p) =>
+      lstatOrNull(p)?.isSymbolicLink(),
+    );
+    if (symlinkCandidates.length === 0) return;
+
+    const rootTop = gitToplevel(root);
+    const links = symlinkCandidates
+      .map((p) => classifyLink(root, p, rootTop))
+      .filter((l): l is LegacyLink => l !== null);
+    if (links.length === 0) return;
+
+    // Pull each distinct legacy repo once, before any copies are taken.
+    const pulled = new Set<string>();
+    for (const link of links) {
+      if (link.repoTop && !pulled.has(link.repoTop)) {
+        pulled.add(link.repoTop);
+        pullLegacyRepo(root, link.repoTop);
+      }
+    }
+
+    // Materialize every link first; legacy repos are only removed once no
+    // remaining symlink references them (aliases to one repo each get a copy).
+    const byRepo = new Map<
+      string,
+      { targets: Set<string>; allMigrated: boolean; keepReasons: Set<string> }
+    >();
+    for (const link of links) {
+      const exclusions = copyExclusions(root, link);
+      const ok = materializeLink(link, exclusions.paths);
+      if (ok) {
+        warn(
+          `reindex: materialized knowledge repo at '${link.linkRel}' (was a symlink to '${path.relative(root, link.target)}')`,
+        );
+      }
+      if (link.repoTop) {
+        const rec = byRepo.get(link.repoTop) ?? {
+          targets: new Set<string>(),
+          allMigrated: true,
+          keepReasons: new Set<string>(),
+        };
+        rec.targets.add(link.target);
+        rec.allMigrated = rec.allMigrated && ok;
+        for (const reason of exclusions.keepLegacyReasons) rec.keepReasons.add(reason);
+        byRepo.set(link.repoTop, rec);
+      }
+    }
+
+    // Remove a legacy repo only when it (a) lives under repos/ inside this HQ,
+    // (b) had every referencing symlink migrated, and (c) held nothing beyond
+    // the knowledge itself — i.e. every link targeted the repo root. A repo
+    // whose knowledge was a subdirectory (e.g. <repo>/docs) is left in place.
+    for (const [repoTop, rec] of byRepo) {
+      const repoRel = path.relative(root, repoTop);
+      if (!isUnder(repoTop, path.join(root, "repos"))) {
+        warn(
+          `reindex: legacy knowledge repo '${repoTop}' lives outside this HQ's repos/ — content was materialized; remove the old repo yourself once verified`,
+        );
+        continue;
+      }
+      if (!rec.allMigrated) continue;
+      if (rec.keepReasons.size > 0) {
+        for (const reason of rec.keepReasons) {
+          warn(`reindex: keeping legacy knowledge repo '${repoRel}' — ${reason}`);
+        }
+        continue;
+      }
+      if (![...rec.targets].every((t) => t === repoTop)) {
+        warn(
+          `reindex: legacy repo '${repoRel}' also holds content beyond the migrated knowledge subdirectory; leaving it in place`,
+        );
+        continue;
+      }
+      try {
+        fs.rmSync(repoTop, { recursive: true, force: true });
+        warn(`reindex: removed legacy knowledge repo '${repoRel}' (fully materialized inline)`);
+      } catch (err) {
+        warn(
+          `reindex: could not remove legacy knowledge repo '${repoRel}' (${(err as Error).message}); its content was already materialized`,
+        );
+      }
+    }
+  } catch (err) {
+    warn(
+      `reindex: legacy knowledge-repo migration pass failed (${(err as Error).message}); skipping`,
+    );
+  }
+}

package/src/cli/reindex.ts

@@ -37,6 +37,7 @@ import {
   type TelemetryClaims,
 } from "../telemetry-events.js";
 import { decodeLocalVaultSegment, encodeLocalVaultSegment } from "../local-path-codec.js";
+import { materializeLegacyKnowledgeRepos } from "./reindex-knowledge.js";
 
 export interface ReindexOptions {
   /** HQ root to operate on. Defaults to process.cwd(). */
@@ -536,6 +537,14 @@ export function reindex(opts: ReindexOptions = {}): ReindexResult {
   }
   try {
 
+  // --- Legacy knowledge-repo materialization --------------------------------
+  // Knowledge repositories must be real directories at their canonical paths
+  // (policy knowledge-repositories-never-symlink); a symlink to a repo under
+  // repos/ syncs as an empty vault marker. Scour the canonical knowledge
+  // locations, pull + copy each legacy repo inline, and drop the fully
+  // migrated legacy repo. Best-effort and a pure no-op on compliant trees.
+  materializeLegacyKnowledgeRepos(root);
+
   // Overlay/skill-wrapper mirroring is best-effort and, on a normal synced HQ,
   // legitimately no-ops for MANY entries (a real core file already sits where a
   // symlink would go, or a link already points elsewhere). Emitting one stderr

package/src/cli/rescue-classify-ordering.test.ts

@@ -497,11 +497,17 @@ exec ${JSON.stringify(realGit)} "$@"
       expect(r.threw, out).toBeUndefined();
       expect(r.status, out).toBe(0);
       // The core fixture proves a real rescue apply occurred, while this
-      // release-owned setting proves `.claude` reached the outpost.
+      // release-owned setting proves `.claude` reached the outpost. The
+      // post-overlay settings reconcile keeps the release baseline (`release:
+      // true`) and additionally composes the machine env.PATH onto it (see
+      // reconcileClaudeSettings), so the file is the release marker plus a
+      // machine-specific env.PATH rather than the verbatim template.
       expect(fs.readFileSync(path.join(hqRoot, "core/a.md"), "utf-8")).toBe("v2\n");
-      expect(fs.readFileSync(path.join(hqRoot, ".claude/settings.json"), "utf-8")).toBe(
-        '{"release":true}\n',
+      const laidSettings = JSON.parse(
+        fs.readFileSync(path.join(hqRoot, ".claude/settings.json"), "utf-8"),
       );
+      expect(laidSettings.release).toBe(true);
+      expect(typeof laidSettings.env?.PATH).toBe("string");
       // `.local` survives untouched and is not copied into the safety snapshot.
       expect(fs.statSync(deviceId).mode & 0o777).toBe(0o000);
       const snapshots = fs.readdirSync(backupRoot).filter((name) => name.startsWith("pre-update-"));