@indigoai-us/hq-cli 5.345.168 → 5.345.169

package/dist/lib/doctor/fix/process-identity.js

@@ -0,0 +1,34 @@
+/**
+ * Process start times, used to tell the owner of a pid from a later process
+ * that reused it (feedback cd56a501).
+ */
+import { spawnSync } from "node:child_process";
+/**
+ * A process's start time, or null if it cannot be read: `ps -o lstart=` on
+ * POSIX, the process StartTime from PowerShell on Windows.
+ */
+export function processStartTime(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return null;
+    try {
+        const result = process.platform === "win32"
+            ? spawnSync("powershell.exe", [
+                "-NoProfile",
+                "-NonInteractive",
+                "-Command",
+                `(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().ToString('o')`,
+            ], { encoding: "utf8", timeout: 10_000, windowsHide: true })
+            : spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8", timeout: 5_000 });
+        const text = (result.stdout ?? "").trim();
+        return result.status === 0 && text ? text : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Real process-table identity for the refresh spawn lock. */
+export const DEFAULT_OWNER_IDENTITY = {
+    selfStart: () => processStartTime(process.pid),
+    startOf: (pid) => processStartTime(pid),
+};
+//# sourceMappingURL=process-identity.js.map

package/dist/lib/doctor/fix/refresh-sync-handoff.d.ts

@@ -0,0 +1,201 @@
+/**
+ * Supervised `hq doctor --fix` journal pulls (feedback cd56a501).
+ *
+ * A pull records `lastSync` only when its whole pass finishes. `--fix` used to
+ * kill its pull at 90s, so a large vault never refreshed its journal and the
+ * client-health hook reported it every day.
+ *
+ * Every `--fix` pull now runs under a small supervisor process. The
+ * supervisor writes a heartbeat while the pull runs, the pull's pid, and an
+ * exit file with the pull's real status when it ends. It stops the pull at
+ * {@link REFRESH_BACKGROUND_MAX_MS}. If the pull is still running at the
+ * foreground deadline, `--fix` leaves the supervisor running.
+ *
+ * One pull per HQ root. A run claims the root before it spawns
+ * ({@link claimRefreshRun}): holding an O_EXCL spawn lock, it checks that no
+ * other pull occupies the root and creates the pull's record exclusively. A
+ * second `--fix` started at the same moment either cannot get the lock or
+ * sees that record, and does not spawn.
+ *
+ * State lives in one file per company under
+ * `workspace/doctor/refresh-in-progress/`, and each run's output, heartbeat,
+ * pid and exit files are named by a per-run id.
+ *
+ * The next `--fix` harvests finished runs first: it reads the exit file and
+ * the log tail, reports the real outcome, and removes the record. A pull whose
+ * supervisor died is still alive while its process group is, until the cap.
+ * Past the cap the next run removes it as capped, and kills it only when the
+ * caller's identity check proves the pid is still that pull.
+ */
+/** The supervisor stops a background pull after this long. */
+export declare const REFRESH_BACKGROUND_MAX_MS: number;
+/** The supervisor refreshes its heartbeat this often. */
+export declare const REFRESH_HEARTBEAT_INTERVAL_MS = 10000;
+/** A heartbeat older than this means the supervisor is gone. */
+export declare const REFRESH_HEARTBEAT_STALE_MS = 60000;
+/** After SIGTERM, the supervisor waits this long before SIGKILL of the pull's process group. */
+export declare const REFRESH_KILL_GRACE_MS = 5000;
+/**
+ * A record older than this is over the cap whatever its heartbeat says. The
+ * supervisor stops the pull at {@link REFRESH_BACKGROUND_MAX_MS} and kills it
+ * {@link REFRESH_KILL_GRACE_MS} later; this adds a heartbeat interval of slack.
+ */
+export declare const REFRESH_RECORD_MAX_AGE_MS: number;
+/**
+ * Reads a process's start time, to tell the owner of a pid from a later
+ * process that reused it. Supplied by the caller (it needs a process table
+ * probe). `null` means the start time could not be read.
+ */
+export interface RefreshOwnerIdentity {
+    /** This process's start time. */
+    selfStart: () => string | null;
+    /** Start time of `pid`, or null if unreadable. */
+    startOf: (pid: number) => string | null;
+}
+/** Files for one supervised pull. */
+export interface RefreshRunPaths {
+    id: string;
+    log: string;
+    exit: string;
+    heartbeat: string;
+    /** The supervisor writes the pull's pid (its process group on POSIX) here. */
+    pid: string;
+    /** The supervisor writes the pull's process start time (`ps -o lstart=`) here, on POSIX. */
+    start: string;
+    /** Create this file to ask the supervisor to stop the pull. */
+    cancel: string;
+}
+/** Exit file the supervisor writes when the pull ends. */
+export interface RefreshRunExit {
+    status: number | null;
+    signal: string | null;
+    /** True when the supervisor stopped the pull because another operation waited on the lock. */
+    preempted?: boolean;
+    /** True when the pull was stopped at the background time cap. */
+    capped?: boolean;
+}
+export declare function safeRefreshSlug(slug: string): string;
+/** Fresh per-run file names under workspace/doctor/refresh-runs/. */
+export declare function newRefreshRunPaths(hqRoot: string, slug: string): RefreshRunPaths;
+/** Write (or replace) the record for a pull of `slug`. Tests and recovery only; claims use {@link claimRefreshRun}. */
+export declare function recordRefreshHandoff(hqRoot: string, slug: string, run: RefreshRunPaths, nowMs: number): void;
+/**
+ * An unreadable record younger than this may still be mid-write (the
+ * exclusive-create fallback), so harvest does not delete it yet.
+ */
+export declare const UNREADABLE_RECORD_SETTLE_MS = 10000;
+/** Remove the record for `slug` if it still names `run`. */
+export declare function clearRefreshRecord(hqRoot: string, slug: string, run: RefreshRunPaths): void;
+/** Read a supervisor exit file, or null if the run has not ended. */
+export declare function readRefreshRunExit(run: RefreshRunPaths): RefreshRunExit | null;
+/** Last 64 KiB of a run's output: enough to classify, bounded in memory. */
+export declare function readRefreshRunLog(run: RefreshRunPaths): string;
+/** What an identity-checked killer needs to know about an orphaned pull. */
+export interface OrphanPull {
+    pid: number;
+    /** Start time the supervisor recorded for the pull, or null if none. */
+    recordedStart: string | null;
+    /** When the record was written (the pull started at or before this). */
+    recordStartedAtMs: number;
+}
+/**
+ * Kills an orphaned pull only if it can prove the pid is still that pull.
+ * Supplied by the caller (it needs a process table probe); without one,
+ * harvest never kills.
+ */
+export type OrphanPullKiller = (orphan: OrphanPull, nowMs: number) => void;
+/** An orphan is only ever killed within this long of its record. */
+export declare const REFRESH_ORPHAN_KILL_WINDOW_MS: number;
+/** Delete a run's files. */
+export declare function removeRefreshRun(run: RefreshRunPaths): void;
+/** A run that has ended (or died) and been removed. */
+export interface HarvestedRefreshRun {
+    slug: string;
+    exit: RefreshRunExit | null;
+    output: string;
+}
+/**
+ * Remove every run that is no longer alive and return what each one ended
+ * with. `exit` is null when the supervisor died without writing one. A run
+ * over the cap is reported as capped, and an orphaned pull still running past
+ * the cap is killed. Live runs are left in place. Unreadable records are
+ * deleted.
+ */
+export declare function harvestRefreshHandoffs(hqRoot: string, nowMs: number, killOrphan?: OrphanPullKiller): HarvestedRefreshRun[];
+/** Start time of the live pull for `slug`, or null. */
+export declare function refreshHandoffStartedAt(hqRoot: string, slug: string, nowMs: number): number | null;
+/**
+ * True when any `--fix` pull for this HQ root still occupies it: running,
+ * starting, orphaned, or capped and still inside its stop grace window.
+ */
+export declare function anyRefreshHandoffAlive(hqRoot: string, nowMs: number): boolean;
+/**
+ * Take the per-root spawn lock: an O_EXCL file holding this process's pid,
+ * its start time and a random token.
+ *
+ * An existing lock is taken over only when identity proves its owner is gone:
+ * the pid is dead, or the pid is alive but its start time differs from the
+ * recorded one (the pid was reused). Age never counts. If the owner's start
+ * time cannot be verified the lock is left alone and this returns null.
+ */
+/** Releases the spawn lock; `stillHeld` re-reads the lock file for this holder's token. */
+export type SpawnLockRelease = (() => void) & {
+    stillHeld: () => boolean;
+};
+export declare function acquireRefreshSpawnLock(hqRoot: string, nowMs: number, identity?: RefreshOwnerIdentity): SpawnLockRelease | null;
+/** A spawn lock still held after this long is reported by `hq doctor`. */
+export declare const REFRESH_SPAWN_LOCK_WARN_MS: number;
+/**
+ * The current spawn lock holder and how long it has held the lock, or null
+ * when no lock is held. Read-only: never removes or changes the lock.
+ */
+export declare function inspectRefreshSpawnLock(hqRoot: string, nowMs: number, identity?: RefreshOwnerIdentity): {
+    pid: number;
+    heldMs: number | null;
+    ownerGone: boolean;
+} | null;
+/**
+ * Release a spawn lock whose owner is proven gone (pid dead, or start time
+ * changed). A live or unverifiable owner's lock is left alone. Uses the same
+ * atomic takeover as a claim, then releases at once.
+ */
+export declare function releaseDeadRefreshSpawnLock(hqRoot: string, nowMs: number, identity?: RefreshOwnerIdentity): void;
+/**
+ * Claim this HQ root for a pull of `slug`; true means spawn.
+ *
+ * - The spawn lock is required. If another live or unverifiable process holds
+ *   it, nothing is written and the claim fails.
+ * - Under the lock: no other pull may occupy the root, and the record is
+ *   created exclusively, never overwritten. An existing record for `slug`
+ *   wins unless it no longer occupies the root (its pull exited, or its
+ *   owner and pull are proven gone); then it is moved aside atomically and
+ *   the create is retried once.
+ * - The claim holds only if, after writing, the record still carries this
+ *   claim's token and this process still holds the lock.
+ */
+export declare function claimRefreshRun(hqRoot: string, slug: string, run: RefreshRunPaths, nowMs: number, identity?: RefreshOwnerIdentity): boolean;
+/** How often the supervisor looks for another operation waiting on the lock. */
+export declare const REFRESH_PREEMPT_POLL_MS = 500;
+/**
+ * Supervisor program, run as
+ * `node -e SUPERVISOR log exit heartbeat pidFile cancel locksDir maxMs cmd ...args`.
+ *
+ * It appends the pull's output to `log`, writes the pull's pid to `pidFile`,
+ * refreshes `heartbeat`, and writes `{status, signal, preempted, capped}` to
+ * `exit` when the pull ends.
+ *
+ * Stopping: at `maxMs`, when `cancel` exists, or when another live process
+ * publishes a waiter for the operation lock the pull holds
+ * (`<lock>.wait-<pid>-*`; `sync` holds the lock shared, so only exclusive
+ * callers such as rescue and repairs wait), the supervisor stops the pull.
+ * On POSIX it sends SIGTERM to the pull's process group, and SIGKILL to the
+ * group after {@link REFRESH_KILL_GRACE_MS} in every case, so a grandchild
+ * cannot outlive it. SIGTERM is the same stop as Ctrl-C on `hq sync`: the
+ * lock's signal handler releases the lock, downloads are staged and renamed
+ * into place, and the journal is written by atomic rename. On Windows the
+ * stop is `taskkill /T /F` on the tree, which is a forced kill, not a
+ * cooperative one; the same staged writes and atomic journal keep completed
+ * work.
+ */
+export declare const REFRESH_SUPERVISOR_SOURCE = "\nconst fs = require(\"node:fs\");\nconst path = require(\"node:path\");\nconst { spawn, spawnSync } = require(\"node:child_process\");\nconst [logPath, exitPath, hbPath, pidPath, cancelPath, locksDir, maxMs, cmd, ...args] = process.argv.slice(1);\nconst isWin = process.platform === \"win32\";\nconst beat = () => { try { fs.writeFileSync(hbPath, String(Date.now())); } catch {} };\nbeat();\nconst hb = setInterval(beat, 10000);\nlet fd;\ntry { fd = fs.openSync(logPath, \"a\", 0o600); } catch { fd = \"ignore\"; }\nlet finished = false;\nlet preempted = false;\nlet capped = false;\nlet stopAt = 0;\nlet watch;\n// Its own process group on POSIX, so a stop reaches every process the pull started.\nconst child = spawn(cmd, args, { stdio: [\"ignore\", fd, fd], windowsHide: true, detached: !isWin });\ntry { fs.writeFileSync(pidPath, String(child.pid)); } catch {}\nif (!isWin && child.pid) {\n  try {\n    const r = spawnSync(\"ps\", [\"-o\", \"lstart=\", \"-p\", String(child.pid)], { encoding: \"utf8\" });\n    const started = (r.stdout || \"\").trim();\n    if (started) fs.writeFileSync(path.join(path.dirname(pidPath), path.basename(pidPath, \".pid\") + \".start\"), started);\n  } catch {}\n}\nconst killTree = (sig) => {\n  if (isWin) {\n    try { spawnSync(\"taskkill\", [\"/PID\", String(child.pid), \"/T\", \"/F\"], { stdio: \"ignore\", windowsHide: true }); } catch {}\n    return;\n  }\n  try { process.kill(-child.pid, sig); } catch {}\n};\nconst stop = () => {\n  if (stopAt) return;\n  stopAt = Date.now();\n  killTree(\"SIGTERM\");\n  setTimeout(() => killTree(\"SIGKILL\"), 5000);\n};\nconst exitNow = (status) => process.exit(typeof status === \"number\" ? status : 1);\nconst done = (status, signal) => {\n  if (finished) return;\n  finished = true;\n  clearInterval(hb);\n  clearInterval(watch);\n  clearTimeout(stopper);\n  try {\n    fs.writeFileSync(exitPath + \".tmp\", JSON.stringify({ status, signal, preempted, capped }));\n    fs.renameSync(exitPath + \".tmp\", exitPath);\n  } catch {}\n  if (!stopAt) return exitNow(status);\n  // A stopped pull's group is always SIGKILLed after the grace period, even\n  // when the pull itself exited on SIGTERM, so no grandchild survives.\n  const left = Math.max(0, stopAt + 5000 - Date.now());\n  setTimeout(() => { killTree(\"SIGKILL\"); exitNow(status); }, left);\n};\nconst stopper = setTimeout(() => {\n  capped = true;\n  // At the cap the pull is no longer live: stop heartbeating so nothing reads it as in progress.\n  clearInterval(hb);\n  try { fs.writeFileSync(hbPath, \"capped:\" + Date.now()); } catch {}\n  stop();\n}, Number(maxMs));\nconst alive = (pid) => { try { process.kill(pid, 0); return true; } catch (e) { return Boolean(e && e.code === \"EPERM\"); } };\nlet lockName = null;\nconst findLock = () => {\n  let names;\n  try { names = fs.readdirSync(locksDir); } catch { return null; }\n  for (const name of names) {\n    // A sync holds shared membership: <lock>.shared-<pid>-<hex>.\n    const shared = /^(.+\\.lock)\\.shared-(\\d+)-/.exec(name);\n    if (shared && Number(shared[2]) === child.pid) return shared[1];\n    if (!name.endsWith(\".lock\")) continue;\n    // An exclusive holder writes its record into the lock file itself.\n    try {\n      const info = JSON.parse(fs.readFileSync(path.join(locksDir, name), \"utf8\"));\n      if (info && info.pid === child.pid) return name;\n    } catch {}\n  }\n  return null;\n};\nconst someoneWaiting = () => {\n  if (fs.existsSync(cancelPath)) return true;\n  if (!lockName) lockName = findLock();\n  if (!lockName) return false;\n  let names;\n  try { names = fs.readdirSync(locksDir); } catch { return false; }\n  const prefix = lockName + \".wait-\";\n  for (const name of names) {\n    if (!name.startsWith(prefix)) continue;\n    const waiter = Number(name.slice(prefix.length).split(\"-\")[0]);\n    if (Number.isInteger(waiter) && waiter > 0 && waiter !== child.pid && waiter !== process.pid && alive(waiter)) return true;\n  }\n  return false;\n};\nwatch = setInterval(() => {\n  if (finished || stopAt) return;\n  if (someoneWaiting()) {\n    preempted = true;\n    stop();\n  }\n}, 500);\nchild.on(\"error\", (error) => {\n  try { fs.appendFileSync(logPath, String((error && error.code) || error) + \"\\n\"); } catch {}\n  done(null, null);\n});\nchild.on(\"exit\", (status, signal) => done(status, signal));\n";
+//# sourceMappingURL=refresh-sync-handoff.d.ts.map