@indigoai-us/hq-cli 5.77.13 → 5.78.0

package/src/commands/outposts-heartbeat.ts

@@ -1,310 +0,0 @@
-/**
- * `hq outposts heartbeat` — publish this box's agent-session summary to the
- * realtime fabric.
- *
- * Runs ON an Outpost, under systemd. It enumerates the box's local Claude Code
- * and Codex sessions with cheap scandir + stat + bounded reads, projects them
- * down to the compact secret-free `AgentSession[]` payload, and publishes to
- * `hq/{personUid}/sessions` so Mission Control can see what the box is doing.
- *
- * ## Why the cadence lives in here
- *
- * The original box-side design was a bash `while` loop that, every 5 seconds,
- * ran `hq auth refresh` and then `npx -y --package=@indigoai-us/hq-cloud@latest
- * outpost-session-heartbeat-runner`. That is ~35k process launches a day, and
- * every `npx` re-resolves the package and re-reads it off disk — on a t3 box
- * whose EBS budget is the scarce resource, that is a meaningful, permanent tax
- * for a job that should be nearly free.
- *
- * So the loop lives in-process instead:
- *   - identity is resolved ONCE per process, not once per tick;
- *   - the Cognito session refreshes itself on demand (`ensureCognitoToken`
- *     returns the cached token until it is close to expiry), so there is no
- *     separate per-tick `hq auth refresh`;
- *   - the IoT client is cached across ticks by the publisher and only rebuilt
- *     when its credentials near expiry.
- *
- * The loop is deliberately unkillable-by-transients: a failed publish, a failed
- * enumeration, or a not-yet-warm session are all logged and retried on the next
- * tick. Nothing here should ever take the service down, because a service that
- * exits looks identical to a service that is quietly failing.
- */
-
-import type { Command } from "commander";
-import { mkdir, writeFile } from "node:fs/promises";
-import { homedir } from "node:os";
-import { dirname, join } from "node:path";
-import { ensureCognitoToken } from "../utils/cognito-session.js";
-import { DEFAULT_VAULT_API_URL } from "../utils/cognito-session.js";
-import { resolveCallerPersonUid } from "../utils/vault-api.js";
-import {
-  DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
-  collectSessions,
-  nodeFileSystem,
-  sessionsTopicForPerson,
-  type FileSystemPort,
-  type PublishPort,
-  type SessionsHeartbeatPayload,
-} from "../outpost/session-heartbeat.js";
-import {
-  createIotPublishPort,
-  defaultRealtimeCredentialsFetcher,
-} from "../outpost/session-heartbeat-publisher.js";
-
-/** Everything the loop touches, injected so it is testable without a box. */
-export interface HeartbeatLoopDeps {
-  fs: FileSystemPort;
-  publish: PublishPort;
-  /** Resolve the caller's canonical `prs_*` id. Called once per process. */
-  getPersonUid: () => Promise<string>;
-  /**
-   * Wait between ticks. MUST return early when `signal` aborts — otherwise a
-   * SIGTERM arriving mid-sleep is not noticed until the full interval elapses,
-   * and systemd force-kills the process instead of it stopping cooperatively.
-   */
-  sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
-  now: () => Date;
-  log: (line: string) => void;
-  /**
-   * Record that a heartbeat actually LANDED. Called only after a successful
-   * publish — never on failure, or the freshness signal it feeds would always
-   * read fresh and be worth nothing.
-   */
-  onPublished?: (payload: SessionsHeartbeatPayload) => Promise<void>;
-}
-
-export interface HeartbeatLoopOptions {
-  /** Emit a single heartbeat and return. */
-  once?: boolean;
-  /** Cadence between ticks. Defaults to {@link DEFAULT_HEARTBEAT_INTERVAL_SECONDS}. */
-  intervalSeconds?: number;
-  /** Home directory to scan (defaults to the process HOME). */
-  home?: string;
-  /** Print each published payload as JSON. */
-  json?: boolean;
-  /** Stop the loop cooperatively. */
-  signal?: AbortSignal;
-  /** Bound the loop — used by tests; production runs until aborted. */
-  maxTicks?: number;
-}
-
-/**
- * Resolve the cadence from the flag, then the environment, then the default.
- * Floored at 1s: a sub-second cadence hammers the fabric for no benefit, and a
- * zero/negative value would spin the loop hot.
- */
-export function resolveIntervalSeconds(
-  flag: string | undefined,
-  env: NodeJS.ProcessEnv = process.env,
-): number {
-  for (const raw of [flag, env.OUTPOST_SESSIONS_HEARTBEAT_INTERVAL_SECONDS]) {
-    if (raw === undefined || raw === null || raw === "") continue;
-    const parsed = Number.parseInt(raw, 10);
-    if (!Number.isFinite(parsed)) return DEFAULT_HEARTBEAT_INTERVAL_SECONDS;
-    return Math.max(1, parsed);
-  }
-  return DEFAULT_HEARTBEAT_INTERVAL_SECONDS;
-}
-
-/**
- * Where the box records its last LANDED heartbeat. The post-provision audit
- * reads this file's freshness, because "systemd says the unit is active" is not
- * evidence that anything was published — that gap is what let a heartbeat fail
- * every five seconds for nine days while every dashboard stayed green.
- *
- * Lives under the service user's home so the unit (User=ec2-user) can write it
- * without extra tmpfiles.d wiring; the SSM collector reads it as root.
- */
-export const DEFAULT_HEARTBEAT_STATE_FILE = join(
-  homedir(),
-  ".hq",
-  "outpost-session-heartbeat.json",
-);
-
-/** Persist the liveness marker the box audit reads. */
-export async function writeHeartbeatState(
-  path: string,
-  payload: SessionsHeartbeatPayload,
-): Promise<void> {
-  await mkdir(dirname(path), { recursive: true });
-  await writeFile(
-    path,
-    `${JSON.stringify({
-      lastPublishAt: payload.emittedAt,
-      sessions: payload.sessions.length,
-    })}\n`,
-    "utf8",
-  );
-}
-
-/** Structured one-line log — journald gets JSON, not prose. */
-function logLine(
-  step: string,
-  outcome: "ok" | "error",
-  extra: Record<string, unknown>,
-  now: () => Date,
-): string {
-  return JSON.stringify({
-    service: "outpost-session-heartbeat",
-    step,
-    outcome,
-    ...extra,
-    timestamp: now().toISOString(),
-  });
-}
-
-/**
- * Run the heartbeat loop. Returns the number of ticks performed.
- *
- * A "tick" is one attempt, whether or not it published — the count is the
- * loop's liveness, not its success rate.
- */
-export async function runHeartbeatLoop(
-  options: HeartbeatLoopOptions,
-  deps: HeartbeatLoopDeps,
-): Promise<number> {
-  const intervalMs =
-    (options.intervalSeconds ?? DEFAULT_HEARTBEAT_INTERVAL_SECONDS) * 1000;
-  const maxTicks = options.once ? 1 : options.maxTicks;
-
-  // Resolved once and reused. Re-resolving per tick was the original sin.
-  let personUid: string | null = null;
-  let ticks = 0;
-
-  for (;;) {
-    ticks += 1;
-
-    try {
-      if (!personUid) personUid = await deps.getPersonUid();
-      const payload = await collectSessions(
-        { personUid, home: options.home, now: deps.now },
-        { fs: deps.fs },
-      );
-      const topic = sessionsTopicForPerson(personUid);
-      await deps.publish(topic, payload);
-      // Ordering matters: the marker is written only once the publish has
-      // resolved, so a stale marker means "nothing landed", not "nothing ran".
-      await deps.onPublished?.(payload);
-      if (options.json) deps.log(JSON.stringify(payload));
-      deps.log(
-        logLine(
-          "publish",
-          "ok",
-          { topic, sessions: payload.sessions.length },
-          deps.now,
-        ),
-      );
-    } catch (err) {
-      // Every failure mode lands here on purpose: a transient must never take
-      // the service down, because systemd restarting a hot-failing loop looks
-      // exactly like a healthy one.
-      deps.log(
-        logLine(
-          "tick",
-          "error",
-          { message: err instanceof Error ? err.message : String(err) },
-          deps.now,
-        ),
-      );
-    }
-
-    if (options.once) break;
-    if (maxTicks !== undefined && ticks >= maxTicks) break;
-    if (options.signal?.aborted) break;
-
-    await deps.sleep(intervalMs, options.signal);
-    if (options.signal?.aborted) break;
-  }
-
-  return ticks;
-}
-
-/** Wire the production dependencies and register the subcommand. */
-export function registerHeartbeatCommand(outposts: Command): void {
-  outposts
-    .command("heartbeat", { hidden: true })
-    .description(
-      "Publish this box's agent-session summary to the realtime fabric (runs on the Outpost, under systemd)",
-    )
-    .option("--once", "Emit a single heartbeat and exit")
-    .option(
-      "--interval <seconds>",
-      "Cadence in seconds (default 5; env OUTPOST_SESSIONS_HEARTBEAT_INTERVAL_SECONDS)",
-    )
-    .option("--home <path>", "Home directory to scan for sessions")
-    .option("--api-base-url <url>", "HQ API base URL")
-    .option(
-      "--state-file <path>",
-      `Liveness marker read by the box audit (default ${DEFAULT_HEARTBEAT_STATE_FILE})`,
-    )
-    .option("--json", "Print each published payload as JSON")
-    .action(
-      async (opts: {
-        once?: boolean;
-        interval?: string;
-        home?: string;
-        apiBaseUrl?: string;
-        stateFile?: string;
-        json?: boolean;
-      }) => {
-        const apiBaseUrl =
-          opts.apiBaseUrl ??
-          process.env.HQAPI_BASE_URL ??
-          DEFAULT_VAULT_API_URL;
-
-        const getJwt = () => ensureCognitoToken({ interactive: false });
-
-        const deps: HeartbeatLoopDeps = {
-          fs: nodeFileSystem,
-          publish: createIotPublishPort({
-            fetchCredentials: defaultRealtimeCredentialsFetcher({
-              apiBaseUrl,
-              getJwt,
-            }),
-          }),
-          // Same base as the credentials fetch. Resolving identity against the
-          // default plane while vending credentials from another sends a
-          // deployment-specific token to the wrong control plane, which is
-          // rejected — and the loop then retries forever without publishing.
-          getPersonUid: async () =>
-            resolveCallerPersonUid(await getJwt(), apiBaseUrl),
-          sleep: (ms, signal) =>
-            new Promise((resolve) => {
-              if (signal?.aborted) return resolve();
-              const t = setTimeout(done, ms);
-              function done() {
-                clearTimeout(t);
-                signal?.removeEventListener("abort", done);
-                resolve();
-              }
-              signal?.addEventListener("abort", done, { once: true });
-            }),
-          now: () => new Date(),
-          log: (line) => console.error(line),
-          onPublished: (payload) =>
-            writeHeartbeatState(
-              opts.stateFile ?? DEFAULT_HEARTBEAT_STATE_FILE,
-              payload,
-            ),
-        };
-
-        // systemd sends SIGTERM on stop/restart — finish the in-flight tick,
-        // then exit cleanly rather than being killed mid-publish.
-        const controller = new AbortController();
-        const stop = () => controller.abort();
-        process.once("SIGTERM", stop);
-        process.once("SIGINT", stop);
-
-        await runHeartbeatLoop(
-          {
-            once: opts.once,
-            intervalSeconds: resolveIntervalSeconds(opts.interval),
-            home: opts.home,
-            json: opts.json,
-            signal: controller.signal,
-          },
-          deps,
-        );
-      },
-    );
-}

package/src/commands/outposts-replica-sync.test.ts

@@ -1,272 +0,0 @@
-import { Command } from "commander";
-import {
-  afterEach,
-  beforeEach,
-  describe,
-  expect,
-  it,
-  vi,
-  type MockInstance,
-} from "vitest";
-import {
-  parseReposManifest,
-  registerOutpostsCommand,
-  type ReplicaSyncDependencies,
-} from "./outposts.js";
-
-const HQ_ROOT = "/home/ec2-user/hq";
-const SECRET = "refresh-token-that-must-never-be-printed";
-
-const REPOS_YAML = `repos:
-  - url: https://github.com/indigoai-us/hq-pro.git
-    visibility: private
-    name: hq-pro
-  - url: https://github.com/indigoai-us/public-widget.git
-    visibility: public
-  - url: https://github.com/indigoai-us/hq-cli.git
-    visibility: private
-    name: hq-cli
-`;
-
-type SpawnCall = {
-  command: string;
-  args: string[];
-  options: Parameters<ReplicaSyncDependencies["spawnSync"]>[2];
-};
-
-function ok(stdout = "") {
-  return {
-    status: 0,
-    stdout,
-    stderr: "",
-    error: undefined,
-  } as ReturnType<ReplicaSyncDependencies["spawnSync"]>;
-}
-
-function replicaDependencies(input: {
-  session?: { refreshToken?: string; idToken?: string } | undefined;
-  reposYaml?: string | null;
-  existing?: Set<string>;
-  failRescue?: boolean;
-  failAuthRefresh?: boolean;
-} = {}): {
-  deps: Partial<ReplicaSyncDependencies>;
-  calls: SpawnCall[];
-  mkdirp: MockInstance;
-} {
-  const calls: SpawnCall[] = [];
-  const existing = input.existing ?? new Set<string>();
-  const mkdirp = vi.fn();
-  const deps: Partial<ReplicaSyncDependencies> = {
-    defaultHqRoot: () => HQ_ROOT,
-    loadCachedTokens: () =>
-      "session" in input ? input.session : { refreshToken: SECRET },
-    readTextFile: (file: string) => {
-      if (file.endsWith("repos.yaml")) {
-        if (input.reposYaml === null) throw new Error("ENOENT");
-        return input.reposYaml ?? REPOS_YAML;
-      }
-      throw new Error(`unexpected readTextFile(${file})`);
-    },
-    pathExists: (p: string) => existing.has(p),
-    mkdirp: mkdirp as unknown as ReplicaSyncDependencies["mkdirp"],
-    spawnSync: ((command, args, options) => {
-      calls.push({ command, args, options });
-      if (input.failRescue && command === "hq" && args[0] === "rescue") {
-        return { ...ok(), status: 1 };
-      }
-      if (
-        input.failAuthRefresh &&
-        command === "hq" &&
-        args[0] === "auth" &&
-        args[1] === "refresh"
-      ) {
-        return { ...ok(), status: 1 };
-      }
-      return ok();
-    }) as ReplicaSyncDependencies["spawnSync"],
-  };
-  return { deps, calls, mkdirp };
-}
-
-function buildProgram(deps: Partial<ReplicaSyncDependencies>): Command {
-  const program = new Command();
-  program.name("hq").exitOverride();
-  registerOutpostsCommand(program, deps);
-  return program;
-}
-
-async function run(
-  deps: Partial<ReplicaSyncDependencies>,
-  args: string[],
-): Promise<void> {
-  await buildProgram(deps).parseAsync(["node", "hq", ...args]);
-}
-
-let logSpy: MockInstance<typeof console.log>;
-let warnSpy: MockInstance<typeof console.warn>;
-let fetchSpy: MockInstance<typeof fetch>;
-
-beforeEach(() => {
-  logSpy = vi.spyOn(console, "log").mockImplementation(() => {});
-  warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
-  fetchSpy = vi.spyOn(globalThis, "fetch");
-});
-
-afterEach(() => {
-  vi.restoreAllMocks();
-});
-
-function printed(): string {
-  return [
-    ...logSpy.mock.calls.map((c) => c.map(String).join(" ")),
-    ...warnSpy.mock.calls.map((c) => c.map(String).join(" ")),
-  ].join("\n");
-}
-
-function hqArgs(calls: SpawnCall[]): string[][] {
-  return calls.filter((c) => c.command === "hq").map((c) => c.args);
-}
-
-describe("parseReposManifest", () => {
-  it("resolves visibility + name, defaulting to public + URL basename", () => {
-    expect(parseReposManifest(REPOS_YAML)).toEqual([
-      { url: "https://github.com/indigoai-us/hq-pro.git", visibility: "private", name: "hq-pro" },
-      { url: "https://github.com/indigoai-us/public-widget.git", visibility: "public", name: "public-widget" },
-      { url: "https://github.com/indigoai-us/hq-cli.git", visibility: "private", name: "hq-cli" },
-    ]);
-  });
-
-  it("returns [] for junk, empty, or repo-less YAML", () => {
-    expect(parseReposManifest("not: valid: [")).toEqual([]);
-    expect(parseReposManifest("")).toEqual([]);
-    expect(parseReposManifest("other: 1")).toEqual([]);
-  });
-
-  it("skips entries with no url", () => {
-    expect(parseReposManifest("repos:\n  - visibility: public\n  - url: https://x/y.git")).toEqual([
-      { url: "https://x/y.git", visibility: "public", name: "y" },
-    ]);
-  });
-
-  it("rejects names that would escape repos/<visibility>/ (path traversal)", () => {
-    const yaml =
-      "repos:\n" +
-      "  - url: https://x/evil.git\n    name: ../../etc\n" +
-      "  - url: https://x/evil2.git\n    name: sub/dir\n" +
-      "  - url: https://x/ok.git\n    name: ok\n";
-    expect(parseReposManifest(yaml)).toEqual([
-      { url: "https://x/ok.git", visibility: "public", name: "ok" },
-    ]);
-  });
-});
-
-describe("hq outposts replica-sync", () => {
-  it("is hidden from outposts help but remains invokable", async () => {
-    const { deps } = replicaDependencies();
-    const program = buildProgram(deps);
-    const outposts = program.commands.find((c) => c.name() === "outposts");
-    expect(outposts?.helpInformation()).not.toContain("replica-sync");
-    await run(deps, ["outposts", "replica-sync"]);
-  });
-
-  it("no HQ session → replicates nothing and runs no commands", async () => {
-    const { deps, calls } = replicaDependencies({ session: undefined });
-    await run(deps, ["outposts", "replica-sync"]);
-    expect(calls).toEqual([]);
-    expect(printed()).toContain("nothing to replicate");
-  });
-
-  it("runs the full replica chain in order on a signed-in box", async () => {
-    const { deps, calls, mkdirp } = replicaDependencies();
-    await run(deps, ["outposts", "replica-sync"]);
-
-    const args = hqArgs(calls);
-    expect(args).toContainEqual(["auth", "refresh"]);
-    // The pull is pinned to the SAME --hq-root the command was invoked with,
-    // not the default root.
-    expect(args).toContainEqual([
-      "sync",
-      "pull",
-      "--personal",
-      "--hq-root",
-      HQ_ROOT,
-      "--on-conflict",
-      "keep",
-    ]);
-    expect(args).toContainEqual(["rescue", "--hq-root", HQ_ROOT, "--yes"]);
-    // rescue needs companies/ to pre-exist.
-    expect(mkdirp).toHaveBeenCalledWith(`${HQ_ROOT}/companies`);
-  });
-
-  it("clones only the missing repos, gh-authing via the vault for privates", async () => {
-    const { deps, calls } = replicaDependencies({
-      // hq-cli already present → should be skipped.
-      existing: new Set([`${HQ_ROOT}/repos/private/hq-cli/.git`]),
-    });
-    await run(deps, ["outposts", "replica-sync"]);
-
-    const clones = calls
-      .filter((c) => c.command === "git" && c.args[0] === "clone")
-      .map((c) => c.args[c.args.length - 1]);
-    expect(clones).toContain(`${HQ_ROOT}/repos/private/hq-pro`);
-    expect(clones).toContain(`${HQ_ROOT}/repos/public/public-widget`);
-    expect(clones).not.toContain(`${HQ_ROOT}/repos/private/hq-cli`);
-
-    // A private repo is missing → gh auth flows through the vault (token in ENV,
-    // never argv).
-    const vaultAuth = calls.find(
-      (c) => c.command === "hq" && c.args[0] === "secrets" && c.args.includes("GITHUB_TOKEN"),
-    );
-    expect(vaultAuth).toBeTruthy();
-    expect(vaultAuth?.args).toContain("--personal");
-    expect(vaultAuth?.args).toContain("--only");
-  });
-
-  it("skips the vault gh auth when every missing repo is public", async () => {
-    const { deps, calls } = replicaDependencies({
-      reposYaml: "repos:\n  - url: https://github.com/x/pub.git\n    visibility: public\n",
-    });
-    await run(deps, ["outposts", "replica-sync"]);
-    const vaultAuth = calls.find((c) => c.command === "hq" && c.args[0] === "secrets");
-    expect(vaultAuth).toBeUndefined();
-  });
-
-  it("no repos.yaml → skips the clone step without error", async () => {
-    const { deps, calls } = replicaDependencies({ reposYaml: null });
-    await run(deps, ["outposts", "replica-sync"]);
-    expect(calls.some((c) => c.command === "git")).toBe(false);
-    expect(printed()).toContain("skipping repo clone");
-  });
-
-  it("aborts the cycle when auth refresh fails — no pull, rescue, or clone", async () => {
-    const { deps, calls } = replicaDependencies({ failAuthRefresh: true });
-    await run(deps, ["outposts", "replica-sync"]);
-    const args = hqArgs(calls);
-    expect(args).toContainEqual(["auth", "refresh"]);
-    expect(args.some((a) => a[0] === "sync")).toBe(false);
-    expect(args.some((a) => a[0] === "rescue")).toBe(false);
-    expect(calls.some((c) => c.command === "git")).toBe(false);
-    expect(printed()).toContain("auth refresh failed");
-  });
-
-  it("a failed step never throws or hard-fails the unit", async () => {
-    const { deps } = replicaDependencies({ failRescue: true });
-    await expect(run(deps, ["outposts", "replica-sync"])).resolves.toBeUndefined();
-    expect(printed()).toContain("HQ kernel rescue did not complete cleanly");
-  });
-
-  it("never leaks the session token or makes hq-pro requests", async () => {
-    const { deps, calls } = replicaDependencies();
-    await run(deps, ["outposts", "replica-sync"]);
-    expect(JSON.stringify(calls)).not.toContain(SECRET);
-    expect(printed()).not.toContain(SECRET);
-    expect(fetchSpy).not.toHaveBeenCalled();
-    // The GITHUB_TOKEN never appears as a bare argv value — only the var name
-    // inside the bash script and the --only selector.
-    const bashScripts = calls
-      .filter((c) => c.command === "hq" && c.args.includes("bash"))
-      .flatMap((c) => c.args);
-    expect(bashScripts.some((a) => a.includes("gh auth login --with-token"))).toBe(true);
-  });
-});