@indigoai-us/hq-cli 5.71.0 → 5.72.0

package/src/commands/outposts.ts

@@ -29,6 +29,7 @@ import * as os from "node:os";
 import * as path from "node:path";
 import * as readline from "node:readline";
 import { randomBytes } from "node:crypto";
+import * as yaml from "js-yaml";
 import { loadCachedTokens } from "@indigoai-us/hq-cloud";
 import { ensureCognitoToken } from "../utils/cognito-session.js";
 import { vaultApiFetch } from "../utils/vault-api.js";
@@ -795,12 +796,246 @@ async function selfDeployOutpost(
   console.log(chalk.dim("It is unregistered and unmanaged by hq-pro (no console entry or remote management)."));
 }
 
+// ---------------------------------------------------------------------------
+// Hidden replica-sync command (box-side full-replica timer target)
+// ---------------------------------------------------------------------------
+//
+// A MANAGED outpost (provisioned via `hq outposts provision`) runs this on a
+// systemd timer to make the box a full replica of the operator's HQ:
+//   1. hq auth refresh        — freshen the seeded Cognito session
+//   2. hq sync pull --personal — the personal vault (repos.yaml, policies, …)
+//   3. hq rescue              — re-lay the core kernel (needs companies/)
+//   4. clone repos.yaml       — the operator's OWN repos, gh-authed via vault
+//
+// Unlike `self-deploy` (which is a manual, unmanaged, all-company watcher on a
+// box you own), replica-sync is GENERIC: it works for every HQ user off their
+// own personal vault — no per-operator setup script. It is deliberately
+// best-effort end to end: a repeating timer must never leave the systemd unit
+// in `failed` just because one transient step (a mirror blip, a not-yet-signed
+// box) didn't complete this cycle.
+
+/**
+ * The extra local surface `replica-sync` needs on top of the self-deploy set:
+ * two filesystem probes for the repos clone. Kept as an extension so the
+ * `self-deploy` dependency surface stays minimal.
+ */
+export interface ReplicaSyncDependencies extends SelfDeployDependencies {
+  pathExists: (p: string) => boolean;
+  mkdirp: (p: string) => void;
+}
+
+const defaultReplicaSyncDependencies: ReplicaSyncDependencies = {
+  ...defaultSelfDeployDependencies,
+  pathExists: (p) => fs.existsSync(p),
+  mkdirp: (p) => {
+    fs.mkdirSync(p, { recursive: true });
+  },
+};
+
+/** One repo entry resolved from `personal/data/repos.yaml`. */
+export interface ReplicaRepo {
+  url: string;
+  visibility: "public" | "private";
+  name: string;
+}
+
+function repoNameFromUrl(url: string): string {
+  const tail = url.split("/").pop() ?? "";
+  return tail.replace(/\.git$/, "").trim();
+}
+
+/**
+ * A repo `name` becomes a path segment under repos/<visibility>/. A synced
+ * repos.yaml is operator data, but a malformed or hostile name must never
+ * escape that directory, so constrain it to a single, non-traversal component.
+ */
+function isSafePathComponent(value: string): boolean {
+  return (
+    value.length > 0 &&
+    !value.includes("/") &&
+    !value.includes("\\") &&
+    value !== "." &&
+    value !== ".."
+  );
+}
+
+/**
+ * Parse `personal/data/repos.yaml` into a clone list. Pure + exported for
+ * tests. Mirrors setup.sh's `clone_repos`: `visibility` defaults to public,
+ * `name` defaults to the URL basename. Malformed entries are skipped, never
+ * thrown.
+ */
+export function parseReposManifest(yamlText: string): ReplicaRepo[] {
+  let doc: unknown;
+  try {
+    doc = yaml.load(yamlText);
+  } catch {
+    return [];
+  }
+  const rawRepos = (doc as { repos?: unknown } | null)?.repos;
+  if (!Array.isArray(rawRepos)) return [];
+
+  const out: ReplicaRepo[] = [];
+  for (const entry of rawRepos) {
+    if (!entry || typeof entry !== "object") continue;
+    const e = entry as Record<string, unknown>;
+    const url = typeof e.url === "string" ? e.url.trim() : "";
+    if (!url) continue;
+    const visibility = e.visibility === "private" ? "private" : "public";
+    const name =
+      typeof e.name === "string" && e.name.trim()
+        ? e.name.trim()
+        : repoNameFromUrl(url);
+    if (!isSafePathComponent(name)) continue;
+    out.push({ url, visibility, name });
+  }
+  return out;
+}
+
+/**
+ * Best-effort spawn: log a one-line outcome and carry on. NEVER throws and
+ * NEVER echoes child output — the same secret-safety stance as `runChecked`,
+ * since these children (auth refresh, secrets exec) touch credentials.
+ */
+function runBestEffort(
+  deps: ReplicaSyncDependencies,
+  command: string,
+  args: string[],
+  description: string,
+  options?: Parameters<typeof spawnSync>[2],
+): boolean {
+  let result: ReturnType<typeof spawnSync>;
+  try {
+    result = deps.spawnSync(command, args, options ?? { stdio: "inherit" });
+  } catch {
+    console.warn(`replica-sync: ${description} could not run — continuing.`);
+    return false;
+  }
+  if (!result || result.error || result.status !== 0) {
+    console.warn(`replica-sync: ${description} did not complete cleanly — continuing.`);
+    return false;
+  }
+  return true;
+}
+
+/**
+ * Authenticate `gh` for private clones by streaming GITHUB_TOKEN through the
+ * vault into the child's ENV ONLY, unsetting it in-shell, and piping it to
+ * `gh auth login --with-token` over stdin. The token never touches argv or
+ * this process's logs. Mirrors personal/scripts/setup.sh's `configure_gh`.
+ */
+function authGitHubViaVault(deps: ReplicaSyncDependencies): void {
+  const ghLogin = [
+    'TOKEN="$GITHUB_TOKEN"',
+    "unset GITHUB_TOKEN",
+    'printf "%s" "$TOKEN" | gh auth login --with-token',
+  ].join("\n");
+  const authed = runBestEffort(
+    deps,
+    "hq",
+    ["secrets", "--personal", "exec", "--only", "GITHUB_TOKEN", "--", "bash", "-c", ghLogin],
+    "GitHub auth via vault",
+  );
+  if (authed) {
+    // Let plain `git clone https://github.com/...` reuse gh's token so private
+    // repos don't prompt for a username/password.
+    runBestEffort(deps, "gh", ["auth", "setup-git"], "gh auth setup-git");
+  }
+}
+
+/** Clone any repos from `personal/data/repos.yaml` that aren't on disk yet. */
+function replicateRepos(deps: ReplicaSyncDependencies, hqRoot: string): void {
+  const manifestPath = path.join(hqRoot, "personal", "data", "repos.yaml");
+  let manifestText: string;
+  try {
+    manifestText = deps.readTextFile(manifestPath);
+  } catch {
+    console.log("replica-sync: no personal/data/repos.yaml — skipping repo clone.");
+    return;
+  }
+
+  const repos = parseReposManifest(manifestText);
+  if (repos.length === 0) {
+    console.log("replica-sync: repos.yaml lists no repos — skipping.");
+    return;
+  }
+
+  const missing = repos.filter(
+    (repo) =>
+      !deps.pathExists(path.join(hqRoot, "repos", repo.visibility, repo.name, ".git")),
+  );
+  if (missing.length === 0) {
+    console.log("replica-sync: all repos already present.");
+    return;
+  }
+
+  // Private repos need gh auth first; public-only manifests skip the vault call.
+  if (missing.some((repo) => repo.visibility === "private")) {
+    authGitHubViaVault(deps);
+  }
+
+  for (const repo of missing) {
+    deps.mkdirp(path.join(hqRoot, "repos", repo.visibility));
+    const dest = path.join(hqRoot, "repos", repo.visibility, repo.name);
+    console.log(`replica-sync: cloning ${repo.visibility}/${repo.name}…`);
+    runBestEffort(
+      deps,
+      "git",
+      ["clone", "--recurse-submodules", repo.url, dest],
+      `clone ${repo.visibility}/${repo.name}`,
+    );
+  }
+}
+
+async function replicaSyncOutpost(
+  opts: { hqRoot?: string },
+  deps: ReplicaSyncDependencies,
+): Promise<void> {
+  const hqRoot = opts.hqRoot ?? deps.defaultHqRoot();
+
+  // Gate: only replicate on a box that has an HQ session. A box that was never
+  // signed in has nothing to pull — exit cleanly so the timer stays green.
+  const session = deps.loadCachedTokens();
+  if (!session?.refreshToken) {
+    console.log("replica-sync: no HQ session on this box yet — nothing to replicate.");
+    return;
+  }
+
+  // Auth refresh gates the run: if the seeded refresh token is dead, every
+  // authenticated step below (pull, secrets exec) would just cascade 401s.
+  // Stop the cycle cleanly instead — the timer retries next tick.
+  if (!runBestEffort(deps, "hq", ["auth", "refresh"], "HQ auth refresh")) {
+    console.warn(
+      "replica-sync: auth refresh failed — skipping this cycle (token likely expired).",
+    );
+    return;
+  }
+  runBestEffort(
+    deps,
+    "hq",
+    ["sync", "pull", "--personal", "--hq-root", hqRoot, "--on-conflict", "keep"],
+    "personal vault pull",
+  );
+
+  // rescue lays down the core kernel and expects companies/ to exist.
+  deps.mkdirp(path.join(hqRoot, "companies"));
+  runBestEffort(
+    deps,
+    "hq",
+    ["rescue", "--hq-root", hqRoot, "--yes"],
+    "HQ kernel rescue",
+  );
+
+  replicateRepos(deps, hqRoot);
+  console.log("replica-sync: complete.");
+}
+
 export function registerOutpostsCommand(
   program: Command,
-  selfDeployOverrides: Partial<SelfDeployDependencies> = {},
+  selfDeployOverrides: Partial<ReplicaSyncDependencies> = {},
 ): void {
-  const selfDeployDependencies: SelfDeployDependencies = {
-    ...defaultSelfDeployDependencies,
+  const deps: ReplicaSyncDependencies = {
+    ...defaultReplicaSyncDependencies,
     ...selfDeployOverrides,
   };
   const outposts = program
@@ -811,15 +1046,33 @@ export function registerOutpostsCommand(
     .command("self-deploy", { hidden: true })
     .description("Configure this EC2 host as a locally self-hosted HQ outpost")
     .option("--yes", "Skip the self-hosting confirmation")
-    .option("--hq-root <path>", "HQ root to sync", selfDeployDependencies.defaultHqRoot())
+    .option("--hq-root <path>", "HQ root to sync", deps.defaultHqRoot())
     .action(async (opts: { yes?: boolean; hqRoot?: string }) => {
       try {
-        await selfDeployOutpost(opts, selfDeployDependencies);
+        await selfDeployOutpost(opts, deps);
       } catch (err) {
         fail(err);
       }
     });
 
+  outposts
+    .command("replica-sync", { hidden: true })
+    .description(
+      "Replicate your full HQ onto this box (personal pull + core rescue + repos). Runs on a timer.",
+    )
+    .option("--hq-root <path>", "HQ root to replicate into", deps.defaultHqRoot())
+    .action(async (opts: { hqRoot?: string }) => {
+      // Best-effort background job: never hard-fail the systemd unit on a
+      // transient step. Swallow, log, and exit 0.
+      try {
+        await replicaSyncOutpost(opts, deps);
+      } catch (err) {
+        console.warn(
+          `replica-sync: unexpected error — ${err instanceof Error ? err.message : String(err)}`,
+        );
+      }
+    });
+
   outposts
     .command("provision")
     .alias("create")

package/src/packaging.test.ts

@@ -0,0 +1,55 @@
+/**
+ * Packaging guard: every declared `bin` must ship EXECUTABLE.
+ *
+ * Regression pin for the "outpost is not CLI-ready" class of failure: `tsc`
+ * emits dist bins as 0644, and a root global install on a fresh outpost has
+ * been observed to leave the linked target non-executable, so `/usr/bin/hq`
+ * dies with "Permission denied" and the box never becomes usable. The build
+ * runs `scripts/chmod-bins.mjs` to 0755 the bins; this test locks that wiring
+ * in and, when a build has already produced `dist/`, asserts the bits directly.
+ */
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vitest";
+
+const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const pkg = JSON.parse(
+  readFileSync(resolve(repoRoot, "package.json"), "utf8"),
+) as {
+  bin?: Record<string, string> | string;
+  scripts?: Record<string, string>;
+};
+
+const binMap =
+  typeof pkg.bin === "string" ? { hq: pkg.bin } : (pkg.bin ?? {});
+
+describe("packaging: bin executable bit", () => {
+  it("wires the chmod-bins step into build and prepublishOnly", () => {
+    expect(pkg.scripts?.build ?? "").toContain("chmod-bins");
+    // prepublishOnly must rebuild so a publish always re-chmods, regardless of
+    // the publisher's local dist state.
+    expect(pkg.scripts?.prepublishOnly ?? "").toContain("build");
+  });
+
+  it("declares at least one bin, all pointing at built .js files", () => {
+    const entries = Object.entries(binMap);
+    expect(entries.length).toBeGreaterThan(0);
+    for (const [, relPath] of entries) {
+      expect(relPath.endsWith(".js")).toBe(true);
+    }
+  });
+
+  it("ships every bin executable once dist is built", () => {
+    for (const [name, relPath] of Object.entries(binMap)) {
+      const target = resolve(repoRoot, relPath);
+      // Pre-build (e.g. a bare `vitest` before `tsc`) → nothing to assert yet.
+      if (!existsSync(target)) continue;
+      const mode = statSync(target).mode;
+      // Owner-executable bit is the one that matters for `npm` bin linking.
+      expect(mode & 0o100, `${name} (${relPath}) must be executable`).toBe(
+        0o100,
+      );
+    }
+  });
+});