@indigoai-us/hq-cli 5.345.168 → 5.345.170

package/dist/lib/doctor/fix/apply.js

@@ -37,9 +37,10 @@
  * (the command resolves it once), keeping this module decoupled from the CLI and
  * trivially testable against a fake tree.
  */
-import { spawnSync } from "node:child_process";
+import { spawn, spawnSync } from "node:child_process";
 import * as fs from "node:fs";
 import * as path from "node:path";
+import { getStateDir } from "@indigoai-us/hq-cloud";
 import { isPersonalJournalSlug } from "../checks/sync-health.js";
 import { createDefaultRegistry } from "../registry.js";
 import { flattenFamilies } from "../report.js";
@@ -51,6 +52,8 @@ import { HQ_CLI_BINS, removeHqCliPs1Shims } from "../../windows/ps1-shims.js";
 import { botDir } from "../../bot/paths.js";
 import { readBotConfig } from "../../bot/config.js";
 import { buildBotDaemonPaths, installedDaemonHqBinary, repairBotDaemonIfStale, rewriteBotDaemonNodePath, } from "../../bot/daemon.js";
+import { DEFAULT_OWNER_IDENTITY, processStartTime } from "./process-identity.js";
+import { REFRESH_BACKGROUND_MAX_MS, REFRESH_SUPERVISOR_SOURCE, anyRefreshHandoffAlive, harvestRefreshHandoffs, releaseDeadRefreshSpawnLock, newRefreshRunPaths, readRefreshRunExit, readRefreshRunLog, claimRefreshRun, clearRefreshRecord, removeRefreshRun, } from "./refresh-sync-handoff.js";
 import { classifyRefreshSyncOutcome, } from "./refresh-sync-outcome.js";
 import { pullOutputShowsCompanyNotResolved, recordCompanyResolution, } from "../company-resolution-state.js";
 import { readRefreshManifestCursor, readRefreshSyncProgress, writeRefreshManifestCursor, writeRefreshSyncCursor, writeRefreshSyncLastRun, } from "./refresh-sync-progress.js";
@@ -184,6 +187,10 @@ export async function applyFixes(options) {
     };
     const refreshSync = async (target) => {
         const result = await performRefreshSync(target);
+        // Still running in the background: no evidence about the company yet.
+        // The next --fix harvests its real exit before it plans anything.
+        if (result.outcome === "pull-handed-off")
+            return result;
         if (target.personal)
             return result;
         noteCompanyOutcome(target.slug, result.ok, result.companyNotResolved === true);
@@ -205,6 +212,17 @@ export async function applyFixes(options) {
     // 2. Collect findings, then plan each newly unlocked allowlisted repair.
     //    Identity is deliberately the check id + target pair: an unsuccessful
     //    repair must not be applied repeatedly in one invocation.
+    // Background pulls from an earlier --fix: record how each really ended
+    // before planning, then drop the finished records (feedback cd56a501).
+    // A spawn lock left by a crashed claimer is released here when its owner is
+    // proven gone, even if no journal is stale this run.
+    releaseDeadRefreshSpawnLock(hqRoot, Date.now(), DEFAULT_OWNER_IDENTITY);
+    for (const harvested of harvestRefreshHandoffs(hqRoot, Date.now(), killOrphanPullIfSame)) {
+        const outcome = harvestedRefreshSyncResult(harvested);
+        if (harvested.slug === "personal" || isPersonalJournalSlug(harvested.slug))
+            continue;
+        noteCompanyOutcome(harvested.slug, outcome.ok, outcome.companyNotResolved === true);
+    }
     let findings = await collect(hqRoot);
     let lastAttemptedRefreshSyncSlug = readRefreshSyncProgress(hqRoot)?.lastAttemptedSlug ?? null;
     let refreshSyncAppliedThisRun = 0;
@@ -333,6 +351,11 @@ export async function applyFixes(options) {
             // left over and reported "not attempted" rather than as a failed repair.
             if (network && !networkBudgetLeft())
                 continue;
+            // One background pull per HQ root. While it runs it holds the operation
+            // lock, so another pull would only wait on it; leave the rest for the
+            // next run.
+            if (plan.fixClass === "refresh-sync" && anyRefreshHandoffAlive(hqRoot, Date.now()))
+                continue;
             if (plan.fixClass === "refresh-sync") {
                 // Persist before starting the bounded pull. If this process is killed,
                 // the next invocation resumes after this company instead of starving
@@ -540,101 +563,267 @@ async function defaultRunChecks(hqRoot) {
 /**
  * Default `refresh-sync` repair: a bounded, pull-only, keep-conflicts
  * `hq sync pull --company <slug>` (or `--personal`) against the tree
- * `--fix` is repairing. Never a push, never interactive. A timeout or
- * non-zero exit is reported rather than thrown so later plans still run.
+ * `--fix` is repairing. Never a push, never interactive. A non-zero exit is
+ * reported rather than thrown so later plans still run.
+ *
+ * A pull still running after {@link REFRESH_SYNC_TIMEOUT_MS} is NOT killed
+ * (feedback cd56a501). A pull writes `lastSync` only when its pass finishes,
+ * so killing it at the deadline left a large vault, or a pull queued behind
+ * the sync runner's lock, stale on every run. It is left running detached and
+ * recorded in refresh-sync-handoff.ts; the journal check reports it as
+ * "refresh in progress" until it finishes.
  */
-export function defaultRefreshSync(target) {
+export async function defaultRefreshSync(target, deps = {}) {
     const hqBin = process.argv[1];
     const selector = target.personal || isPersonalJournalSlug(target.slug)
         ? ["--personal"]
         : ["--company", target.slug];
-    const args = hqBin
-        ? [
-            hqBin,
-            "sync",
-            "pull",
-            ...selector,
-            "--hq-root",
-            target.hqRoot,
-            "--on-conflict",
-            "keep",
-            "--lock-timeout",
-            "30",
-        ]
-        : [
-            "sync",
-            "pull",
-            ...selector,
-            "--hq-root",
-            target.hqRoot,
-            "--on-conflict",
-            "keep",
-            "--lock-timeout",
-            "30",
-        ];
-    const result = spawnSync(hqBin ? process.execPath : "hq", args, {
-        encoding: "utf8",
-        timeout: REFRESH_SYNC_TIMEOUT_MS,
-        cwd: target.hqRoot,
-    });
-    if (result.error) {
-        const spawnError = result.error;
-        const outcome = classifyRefreshSyncOutcome({
-            errorCode: spawnError.code,
-            ...(result.error.message.includes("ETIMEDOUT")
-                ? { errorCode: "ETIMEDOUT" }
-                : {}),
-            signal: result.signal,
-            status: result.status,
-            stdout: result.stdout,
-            stderr: result.stderr,
-        });
-        return Promise.resolve({
+    const pullArgs = [
+        "sync",
+        "pull",
+        ...selector,
+        "--hq-root",
+        target.hqRoot,
+        "--on-conflict",
+        "keep",
+        "--lock-timeout",
+        String(REFRESH_SYNC_LOCK_TIMEOUT_SEC),
+    ];
+    // Run the same CLI script under this node. Without a script path there is no
+    // safe way to start `hq` on Windows (it is a .cmd shim that needs a shell),
+    // so only POSIX falls back to `hq` on PATH.
+    if (!deps.command && !hqBin && process.platform === "win32") {
+        return {
             ok: false,
-            outcome,
-            message: outcome === "pull-timeout"
-                ? `timed out pulling '${target.slug}'`
-                : `could not start pull for '${target.slug}': ${result.error.message}`,
-        });
+            outcome: "pull-failed:other",
+            message: `could not start pull for '${target.slug}': the hq CLI script path is unknown`,
+        };
     }
-    if (result.status !== 0) {
-        const outcome = classifyRefreshSyncOutcome({
-            status: result.status,
-            stdout: result.stdout,
-            stderr: result.stderr,
-        });
-        const detail = (result.stderr || result.stdout || "").trim().split("\n").at(-1) ?? "";
-        return Promise.resolve({
+    const command = deps.command ?? (hqBin ? process.execPath : "hq");
+    const args = deps.args ?? (hqBin ? [hqBin, ...pullArgs] : pullArgs);
+    const timeoutMs = deps.timeoutMs ?? REFRESH_SYNC_TIMEOUT_MS;
+    const maxBackgroundMs = deps.maxBackgroundMs ?? REFRESH_BACKGROUND_MAX_MS;
+    const nowMs = deps.nowMs ?? (() => Date.now());
+    // The pull runs under a supervisor that records its real exit status, so a
+    // pull left running past the deadline is still classified by the next run.
+    const run = newRefreshRunPaths(target.hqRoot, target.slug);
+    // One pull per HQ root: claim it (record written under the spawn lock)
+    // before spawning, so a concurrent --fix sees this pull and backs off.
+    if (!claimRefreshRun(target.hqRoot, target.slug, run, nowMs(), deps.identity ?? DEFAULT_OWNER_IDENTITY)) {
+        return {
             ok: false,
-            outcome,
-            message: `pull of '${target.slug}' exited ${result.status}${detail ? `: ${detail}` : ""}`,
-            companyNotResolved: !target.personal &&
-                !isPersonalJournalSlug(target.slug) &&
-                pullOutputShowsCompanyNotResolved(`${result.stdout ?? ""}\n${result.stderr ?? ""}`),
-        });
+            outcome: "pull-failed:lock-busy",
+            message: `another hq doctor --fix pull is already running; '${target.slug}' is left for the next run`,
+        };
+    }
+    let supervisor;
+    try {
+        fs.mkdirSync(path.dirname(run.log), { recursive: true });
+        supervisor = spawn(process.execPath, [
+            "-e",
+            REFRESH_SUPERVISOR_SOURCE,
+            run.log,
+            run.exit,
+            run.heartbeat,
+            run.pid,
+            run.cancel,
+            deps.locksDir ?? defaultLocksDir(),
+            String(maxBackgroundMs),
+            command,
+            ...args,
+        ], { cwd: target.hqRoot, detached: true, stdio: "ignore", windowsHide: true });
     }
-    return Promise.resolve({
-        ok: true,
-        outcome: "pull-succeeded",
-        message: `pulled '${target.slug}'`,
+    catch (error) {
+        clearRefreshRecord(target.hqRoot, target.slug, run);
+        removeRefreshRun(run);
+        return {
+            ok: false,
+            outcome: classifyRefreshSyncOutcome({ errorCode: error.code }),
+            message: `could not start pull for '${target.slug}': ${error.message}`,
+        };
+    }
+    const finished = await new Promise((resolve) => {
+        let settled = false;
+        const settle = (value) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            resolve(value);
+        };
+        supervisor.once("error", () => settle(true));
+        supervisor.once("exit", () => settle(true));
+        const timer = setTimeout(() => settle(false), timeoutMs);
     });
+    if (!finished) {
+        // Leave the pull running on its own. It records lastSync when it
+        // finishes; until then the journal check reports "refresh in progress".
+        supervisor.removeAllListeners();
+        supervisor.unref();
+        return {
+            ok: false,
+            outcome: "pull-handed-off",
+            message: `still pulling '${target.slug}' after ${timeoutMs / 1000}s; it keeps running in the background`,
+        };
+    }
+    const result = harvestedRefreshSyncResult({
+        slug: target.slug,
+        exit: readRefreshRunExit(run),
+        output: readRefreshRunLog(run),
+    });
+    clearRefreshRecord(target.hqRoot, target.slug, run);
+    removeRefreshRun(run);
+    return result;
+}
+/** Map a finished supervised pull to a {@link RefreshSyncResult}. */
+export function harvestedRefreshSyncResult(harvested) {
+    const { slug, exit, output } = harvested;
+    if (!exit) {
+        return { ok: false, outcome: "pull-failed:other", message: `pull of '${slug}' ended without a result` };
+    }
+    if (exit.capped) {
+        // Stopped at the background time cap. Completed downloads are kept.
+        return { ok: false, outcome: "pull-timeout-capped", message: `stopped pulling '${slug}' at the background time limit` };
+    }
+    if (exit.preempted) {
+        // Stopped so a sync the user or the runner started could take the lock.
+        // Completed downloads are kept; the next --fix resumes the journal.
+        return { ok: false, outcome: "pull-preempted", message: `paused pulling '${slug}' for another sync` };
+    }
+    if (exit.status === 0 && !exit.signal) {
+        return { ok: true, outcome: "pull-succeeded", message: `pulled '${slug}'` };
+    }
+    const outcome = classifyRefreshSyncOutcome({ status: exit.status, signal: exit.signal, stdout: output });
+    const detail = output.trim().split("\n").at(-1) ?? "";
+    const personal = slug === "personal" || isPersonalJournalSlug(slug);
+    return {
+        ok: false,
+        outcome,
+        message: `pull of '${slug}' exited ${exit.status ?? exit.signal}${detail ? `: ${detail}` : ""}`,
+        companyNotResolved: !personal && pullOutputShowsCompanyNotResolved(output),
+    };
 }
+export { processStartTime };
+/**
+ * Kill an orphaned --fix pull past its cap, but only when the pid still names
+ * that pull: its process start time must match the one recorded when it
+ * started. A reused pid has a different start time and is left alone. On
+ * Windows no start time is recorded, so nothing is killed there.
+ */
+export function killOrphanPullIfSame(orphan, _nowMs, probe = processStartTime) {
+    if (process.platform === "win32" || !orphan.recordedStart)
+        return;
+    const current = probe(orphan.pid);
+    if (!current || current !== orphan.recordedStart)
+        return;
+    try {
+        process.kill(-orphan.pid, "SIGKILL");
+    }
+    catch {
+        // gone
+    }
+}
+/** hq-cloud keeps operation locks in `<stateDir>/locks`. */
+function defaultLocksDir() {
+    return path.join(getStateDir(), "locks");
+}
+/**
+ * How long a `--fix` pull waits for the operation lock. Unchanged from before
+ * the hand-off: a background pull must never queue on the lock longer than a
+ * foreground one did.
+ */
+export const REFRESH_SYNC_LOCK_TIMEOUT_SEC = 30;
 /**
  * Default `refresh-manifest` repair: a bounded `hq sync manifest --scope <s>`
- * against the tree `--fix` is repairing. Exit 0 also covers throttled,
- * disabled, and skipped passes, so the post-fix re-check decides whether the
- * row cleared. A timeout or non-zero exit is reported, not thrown.
+ * against the tree `--fix` is repairing, with `--json` so a pass that exits 0
+ * without uploading (soft_skipped, throttled, disabled, locked,
+ * resend_full_scheduled) is reported as not uploaded. A `resend_full` answer
+ * is retried once with `--full`. The post-fix re-check still decides whether
+ * the row cleared. A timeout or non-zero exit is reported, not thrown.
  */
 export function defaultRefreshManifest(target) {
-    const hqBin = process.argv[1];
-    const args = ["sync", "manifest", "--scope", target.scope, "--hq-root", target.hqRoot];
-    const result = spawnSync(hqBin ? process.execPath : "hq", hqBin ? [hqBin, ...args] : args, {
-        encoding: "utf8",
-        timeout: REFRESH_SYNC_TIMEOUT_MS,
-        cwd: target.hqRoot,
-    });
-    return Promise.resolve(manifestUploadResult(target.scope, result));
+    return Promise.resolve(refreshManifestWithRetry(target.scope, (full, json) => {
+        const hqBin = process.argv[1];
+        const args = [
+            "sync",
+            "manifest",
+            "--scope",
+            target.scope,
+            "--hq-root",
+            target.hqRoot,
+            ...(json ? ["--json"] : []),
+            ...(full ? ["--full"] : []),
+        ];
+        return spawnSync(hqBin ? process.execPath : "hq", hqBin ? [hqBin, ...args] : args, {
+            encoding: "utf8",
+            timeout: REFRESH_SYNC_TIMEOUT_MS,
+            cwd: target.hqRoot,
+        });
+    }));
+}
+/**
+ * Run one upload and, when the server answered `resend_full` (it dropped or
+ * refused the delta base), run it once more with `--full`. Without the retry
+ * the scope stays stale until the next day's --fix run, and the health hook
+ * reports it as unrepaired in between (feedback_84d7a862).
+ */
+export function refreshManifestWithRetry(scope, run) {
+    const firstRaw = run(false, true);
+    // `hq` from PATH can be an older binary without `--json`. Fall back to the
+    // exit-code handling it always had instead of reporting a failure.
+    if (manifestJsonUnsupported(firstRaw))
+        return manifestUploadResult(scope, run(false, false));
+    const first = manifestUploadResult(scope, firstRaw);
+    if (first.passStatus !== "resend_full_scheduled")
+        return first;
+    const retry = manifestUploadResult(scope, run(true, true));
+    if (retry.passStatus !== "resend_full_scheduled")
+        return retry;
+    return {
+        ...retry,
+        reason: "the server asked for a full manifest again after a --full retry; a full resend is queued, so run `hq sync` again later",
+    };
+}
+/** Commander's rejection of `--json` by a binary that predates it. */
+export function manifestJsonUnsupported(result) {
+    if (result.status === 0 || result.status === null)
+        return false;
+    return /unknown option ['"]?--json/i.test(`${result.stderr ?? ""}\n${result.stdout ?? ""}`);
 }
+/**
+ * The status token from the `hq sync manifest --json` line, or undefined when
+ * the output has none (an older binary, or a pass that printed only a summary).
+ */
+export function manifestPassStatus(stdout) {
+    const lines = (stdout ?? "").split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+    for (let index = lines.length - 1; index >= 0; index -= 1) {
+        const line = lines[index];
+        if (!line.startsWith("{"))
+            continue;
+        try {
+            const parsed = JSON.parse(line);
+            if (typeof parsed.status === "string" && /^[a-z_]{1,40}$/.test(parsed.status)) {
+                return parsed.status;
+            }
+        }
+        catch {
+            // Not the outcome line.
+        }
+    }
+    return undefined;
+}
+/**
+ * Plain-words reason for a pass that exited 0 without uploading. Each one
+ * leaves `lastUploadAt` unchanged, so the stale row cannot clear.
+ */
+const MANIFEST_NOT_UPLOADED_REASONS = {
+    soft_skipped: "the sync service did not accept manifest uploads (the pass was skipped)",
+    throttled: "the pass was throttled and sent nothing",
+    disabled: "manifest uploads are turned off on this machine",
+    locked: "another manifest pass for this scope was already running",
+    resend_full_scheduled: "the server asked for a fresh full manifest and the upload did not finish",
+    printed: "the pass only printed the manifest",
+};
 /**
  * Map one `hq sync manifest` spawn result to a {@link RefreshManifestResult}.
  * A spawn killed by the timeout has a null status, so it is reported as a
@@ -664,6 +853,15 @@ export function manifestUploadResult(scope, result) {
                 : {}),
         };
     }
+    const passStatus = manifestPassStatus(result.stdout);
+    if (passStatus && passStatus !== "uploaded") {
+        return {
+            ok: false,
+            passStatus,
+            reason: MANIFEST_NOT_UPLOADED_REASONS[passStatus] ?? `the pass ended '${passStatus}' without uploading`,
+            message: `manifest upload for '${scope}' did not upload (${passStatus})`,
+        };
+    }
     return { ok: true, message: `uploaded the manifest for '${scope}'` };
 }
 /** `hq sync manifest` output for a company this identity is not a member of. */

package/dist/lib/doctor/fix/process-identity.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Process start times, used to tell the owner of a pid from a later process
+ * that reused it (feedback cd56a501).
+ */
+import type { RefreshOwnerIdentity } from "./refresh-sync-handoff.js";
+/**
+ * A process's start time, or null if it cannot be read: `ps -o lstart=` on
+ * POSIX, the process StartTime from PowerShell on Windows.
+ */
+export declare function processStartTime(pid: number): string | null;
+/** Real process-table identity for the refresh spawn lock. */
+export declare const DEFAULT_OWNER_IDENTITY: RefreshOwnerIdentity;
+//# sourceMappingURL=process-identity.d.ts.map

package/dist/lib/doctor/fix/process-identity.js

@@ -0,0 +1,34 @@
+/**
+ * Process start times, used to tell the owner of a pid from a later process
+ * that reused it (feedback cd56a501).
+ */
+import { spawnSync } from "node:child_process";
+/**
+ * A process's start time, or null if it cannot be read: `ps -o lstart=` on
+ * POSIX, the process StartTime from PowerShell on Windows.
+ */
+export function processStartTime(pid) {
+    if (!Number.isInteger(pid) || pid <= 0)
+        return null;
+    try {
+        const result = process.platform === "win32"
+            ? spawnSync("powershell.exe", [
+                "-NoProfile",
+                "-NonInteractive",
+                "-Command",
+                `(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().ToString('o')`,
+            ], { encoding: "utf8", timeout: 10_000, windowsHide: true })
+            : spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8", timeout: 5_000 });
+        const text = (result.stdout ?? "").trim();
+        return result.status === 0 && text ? text : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Real process-table identity for the refresh spawn lock. */
+export const DEFAULT_OWNER_IDENTITY = {
+    selfStart: () => processStartTime(process.pid),
+    startOf: (pid) => processStartTime(pid),
+};
+//# sourceMappingURL=process-identity.js.map