@indigoai-us/hq-cli 5.345.170 → 5.345.172

package/dist/utils/sync-watchdog.js

@@ -0,0 +1,250 @@
+/**
+ * Opt-in watchdog for `hq sync now` (feedback_b7c00a62).
+ *
+ * A scheduled `hq sync now --all --on-conflict keep` held a CPU core for
+ * hours and never exited. The cause, a LIST that re-requested the same page
+ * forever, is fixed in hq-cloud and hq-pro. This watchdog is a safety net
+ * for scheduled or unattended jobs, and it is OFF by default: a healthy sync
+ * is never stopped unless the caller asks for a limit.
+ *
+ * - Idle limit (`--max-idle` / HQ_SYNC_MAX_IDLE, default off): stop the run
+ *   after this long with no observed progress. Progress is a change in this
+ *   process's hq-cloud sync-progress snapshot: a leg starting, its plan, or a
+ *   file transferred, skipped, or in conflict. The snapshot has no counter
+ *   for LIST pages or planned rows, so the remote LIST and planning phase of
+ *   a leg counts as no progress; set the idle limit longer than the slowest
+ *   expected planning phase. One file transfer also counts only when it
+ *   completes.
+ * - Total cap (`--max-runtime` / HQ_SYNC_MAX_RUNTIME, default off).
+ *
+ * Time is measured on the monotonic clock (`process.hrtime`), which does not
+ * advance while a Mac or Linux laptop sleeps, so sleep is not idle time.
+ *
+ * Two layers enforce the limits:
+ *
+ * 1. An in-process check. When a limit passes, the run rejects with
+ *    {@link SyncWatchdogTimeoutError}; the command prints the message,
+ *    records a failed sync (bounded to {@link SYNC_WATCHDOG_REPORT_GRACE_MS}),
+ *    and exits {@link SYNC_WATCHDOG_EXIT_CODE} (124).
+ * 2. A worker-thread backstop. A timer on the main thread cannot fire while
+ *    the main thread spins in a loop that never yields. The worker reads the
+ *    same progress clock through shared memory and, once a limit is exceeded
+ *    by {@link SYNC_WATCHDOG_BACKSTOP_GRACE_MS}, writes the message to stderr
+ *    and SIGKILLs the process. The backstop stays armed after an in-process
+ *    timeout until the process exits, so hung post-timeout work is still
+ *    ended. A backstop kill exits 137 on POSIX (128 + SIGKILL), not 124; on
+ *    Windows the process ends with exit code 1.
+ */
+import { Worker } from "node:worker_threads";
+/** Default no-progress limit: off. Opt in with --max-idle / HQ_SYNC_MAX_IDLE. */
+export const DEFAULT_SYNC_MAX_IDLE_SECONDS = 0;
+/** Default total cap: off. */
+export const DEFAULT_SYNC_MAX_RUNTIME_SECONDS = 0;
+/** How long the worker backstop waits past a limit before it kills. */
+export const SYNC_WATCHDOG_BACKSTOP_GRACE_MS = 60_000;
+/** Upper bound on the failed-sync report after a timeout, before exit 124. */
+export const SYNC_WATCHDOG_REPORT_GRACE_MS = 10_000;
+/** Exit code for a run the in-process watchdog ended (same as coreutils `timeout`). */
+export const SYNC_WATCHDOG_EXIT_CODE = 124;
+export class SyncWatchdogTimeoutError extends Error {
+    code = "SYNC_WATCHDOG_TIMEOUT";
+    reason;
+    seconds;
+    constructor(reason, seconds, label) {
+        super(syncWatchdogMessage(reason, seconds, label));
+        this.name = "SyncWatchdogTimeoutError";
+        this.reason = reason;
+        this.seconds = seconds;
+    }
+}
+export function syncWatchdogMessage(reason, seconds, label) {
+    const what = reason === "idle"
+        ? `${label} made no progress for ${seconds}s`
+        : `${label} did not finish within ${seconds}s`;
+    return (`${what} and was stopped by the sync watchdog. Run it again; if it stops ` +
+        "here every time, report it with `hq feedback bug`. Change the limits with " +
+        "--max-idle / HQ_SYNC_MAX_IDLE and --max-runtime / HQ_SYNC_MAX_RUNTIME " +
+        "(0 turns a limit off).");
+}
+function parseSeconds(value, flag) {
+    const seconds = Number(value);
+    if (!Number.isInteger(seconds) || seconds < 0) {
+        throw new Error(`${flag} must be a non-negative integer number of seconds`);
+    }
+    return seconds;
+}
+/** Resolve both limits. A flag wins, then the environment, then the default. */
+export function resolveSyncWatchdogLimits(raw, env = process.env) {
+    return {
+        idleSeconds: parseSeconds(raw.maxIdle ?? env.HQ_SYNC_MAX_IDLE ?? String(DEFAULT_SYNC_MAX_IDLE_SECONDS), "--max-idle"),
+        maxRuntimeSeconds: parseSeconds(raw.maxRuntime ?? env.HQ_SYNC_MAX_RUNTIME ?? String(DEFAULT_SYNC_MAX_RUNTIME_SECONDS), "--max-runtime"),
+    };
+}
+// ── Progress clock ──────────────────────────────────────────────────────────
+// One per process. Float64 slot holds the last progress time (ms since epoch)
+// in shared memory, so the worker backstop sees updates made on the main
+// thread without any message passing.
+let progressClock = null;
+/** Monotonic milliseconds, shared by every thread in the process. */
+function hrtimeMs() {
+    return Number(process.hrtime.bigint() / 1000000n);
+}
+let monotonicNow = hrtimeMs;
+/** Test seam: replace the monotonic clock. `null` restores process.hrtime. */
+export function setSyncWatchdogClockForTesting(clock) {
+    monotonicNow = clock ?? hrtimeMs;
+}
+let lastProgressAt = monotonicNow();
+/** Record sync progress. Cheap; safe to call on every row or event. */
+export function noteSyncProgress() {
+    const now = monotonicNow();
+    lastProgressAt = now;
+    if (progressClock)
+        progressClock[0] = now;
+}
+/** Last recorded progress time, for tests and the in-process check. */
+export function lastSyncProgressAt() {
+    return lastProgressAt;
+}
+/**
+ * Build a probe that calls {@link noteSyncProgress} whenever THIS process's
+ * sync-progress snapshot changes. hq-cloud rewrites that snapshot when a leg
+ * starts, when it plans, and for every file transferred or skipped, for both
+ * push and pull, so a moving sync keeps resetting the idle limit without any
+ * change to the engine's progress recorders. Snapshots written by another
+ * process (the menubar runner) are ignored. A single file transfer is not
+ * seen until it completes; the engine's own per-body idle timeouts cover a
+ * transfer that stalls.
+ */
+export function createSnapshotProgressProbe(read, pid = process.pid) {
+    let last = null;
+    return () => {
+        let snap;
+        try {
+            snap = read();
+        }
+        catch {
+            return;
+        }
+        if (!snap || snap.pid !== pid)
+            return;
+        const key = [snap.startedAt, snap.phase, snap.company, snap.filesTotal, snap.filesDone, snap.conflicts].join("|");
+        if (last !== null && key !== last)
+            noteSyncProgress();
+        last = key;
+    };
+}
+// Runs in a worker. Wakes every `checkMs` (or on cancel), reads the shared
+// progress clock, and kills the whole process once a limit plus grace passes.
+const BACKSTOP_SOURCE = `
+const { workerData } = require("node:worker_threads");
+const fs = require("node:fs");
+const flag = new Int32Array(workerData.flag);
+const clock = new Float64Array(workerData.clock);
+const { startedAt, idleMs, maxRuntimeMs, graceMs, idleMessage, runtimeMessage } = workerData;
+const checkMs = Math.max(10, Math.min(5000, Math.floor(graceMs / 4)));
+for (;;) {
+  Atomics.wait(flag, 0, 0, checkMs);
+  if (Atomics.load(flag, 0) !== 0) break;
+  const now = Number(process.hrtime.bigint() / 1000000n);
+  let message = null;
+  if (idleMs > 0 && now - clock[0] > idleMs + graceMs) message = idleMessage;
+  else if (maxRuntimeMs > 0 && now - startedAt > maxRuntimeMs + graceMs) message = runtimeMessage;
+  if (message !== null) {
+    try { fs.writeSync(2, "\\n\\u2717 " + message + " (forced exit)\\n"); } catch {}
+    process.kill(process.pid, "SIGKILL");
+  }
+}
+`;
+export const startWorkerBackstop = (options) => {
+    const flagBuffer = new SharedArrayBuffer(4);
+    const flag = new Int32Array(flagBuffer);
+    const worker = new Worker(BACKSTOP_SOURCE, {
+        eval: true,
+        workerData: {
+            flag: flagBuffer,
+            clock: options.clock.buffer,
+            startedAt: options.startedAt,
+            idleMs: options.idleMs,
+            maxRuntimeMs: options.maxRuntimeMs,
+            graceMs: options.graceMs,
+            idleMessage: options.idleMessage,
+            runtimeMessage: options.runtimeMessage,
+        },
+    });
+    worker.unref();
+    worker.on("error", () => {
+        // Best-effort; the in-process check still applies.
+    });
+    return {
+        cancel() {
+            Atomics.store(flag, 0, 1);
+            Atomics.notify(flag, 0);
+            void worker.terminate().catch(() => undefined);
+        },
+    };
+};
+let backstopStarterForTest = null;
+/** Test seam: replace the worker backstop. `null` restores the real one. */
+export function setSyncBackstopStarterForTesting(start) {
+    backstopStarterForTest = start;
+}
+/**
+ * Run `run` under the watchdog. Resolves or rejects with `run`'s outcome when
+ * it finishes within the limits. Otherwise rejects with
+ * {@link SyncWatchdogTimeoutError} and leaves the backstop armed: the caller
+ * is expected to exit the process. With both limits 0 it just runs `run`.
+ */
+export async function withSyncWatchdog(run, options) {
+    const { idleMs, maxRuntimeMs, label } = options;
+    if (idleMs <= 0 && maxRuntimeMs <= 0)
+        return run();
+    const startedAt = monotonicNow();
+    if (!progressClock)
+        progressClock = new Float64Array(new SharedArrayBuffer(8));
+    noteSyncProgress();
+    const idleSeconds = Math.round(idleMs / 1000);
+    const runtimeSeconds = Math.round(maxRuntimeMs / 1000);
+    const backstop = (options.startBackstop ?? backstopStarterForTest ?? startWorkerBackstop)({
+        clock: progressClock,
+        startedAt,
+        idleMs,
+        maxRuntimeMs,
+        graceMs: options.backstopGraceMs ?? SYNC_WATCHDOG_BACKSTOP_GRACE_MS,
+        idleMessage: syncWatchdogMessage("idle", idleSeconds, label),
+        runtimeMessage: syncWatchdogMessage("runtime", runtimeSeconds, label),
+    });
+    const limits = [idleMs, maxRuntimeMs].filter((ms) => ms > 0);
+    const checkMs = options.checkIntervalMs ?? Math.max(10, Math.min(30_000, Math.floor(Math.min(...limits) / 4)));
+    let interval;
+    const deadline = new Promise((_, reject) => {
+        interval = setInterval(() => {
+            options.probeProgress?.();
+            const now = monotonicNow();
+            if (maxRuntimeMs > 0 && now - startedAt >= maxRuntimeMs) {
+                reject(new SyncWatchdogTimeoutError("runtime", runtimeSeconds, label));
+            }
+            else if (idleMs > 0 && now - lastProgressAt >= idleMs) {
+                reject(new SyncWatchdogTimeoutError("idle", idleSeconds, label));
+            }
+        }, checkMs);
+    });
+    let timedOut = false;
+    try {
+        return await Promise.race([run(), deadline]);
+    }
+    catch (err) {
+        if (err instanceof SyncWatchdogTimeoutError)
+            timedOut = true;
+        throw err;
+    }
+    finally {
+        if (interval !== undefined)
+            clearInterval(interval);
+        // After a timeout the backstop stays armed until the process exits, so
+        // a report or cleanup step that hangs cannot keep the process alive.
+        if (!timedOut)
+            backstop.cancel();
+    }
+}
+//# sourceMappingURL=sync-watchdog.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.345.170",
+  "version": "5.345.172",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {
@@ -52,7 +52,7 @@
     "@aws-sdk/client-iot-data-plane": "^3.1096.0",
     "@aws-sdk/client-s3": "^3.1049.0",
     "@aws-sdk/dsql-signer": "^3.1146.0",
-    "@indigoai-us/hq-cloud": "6.18.101",
+    "@indigoai-us/hq-cloud": "6.18.103",
     "@indigoai-us/hq-flags-client": "^0.1.2",
     "@modelcontextprotocol/sdk": "1.29.0",
     "@sentry/node": "^10.49.0",
@@ -114,11 +114,11 @@
     "*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}": "eslint --cache --no-error-on-unmatched-pattern"
   },
   "optionalDependencies": {
-    "@indigoai-us/hq-hook-win32-x64": "5.345.170",
-    "@indigoai-us/hq-hook-win32-arm64": "5.345.170",
-    "@indigoai-us/hq-hook-darwin-x64": "5.345.170",
-    "@indigoai-us/hq-hook-darwin-arm64": "5.345.170",
-    "@indigoai-us/hq-hook-linux-x64": "5.345.170",
-    "@indigoai-us/hq-hook-linux-arm64": "5.345.170"
+    "@indigoai-us/hq-hook-win32-x64": "5.345.172",
+    "@indigoai-us/hq-hook-win32-arm64": "5.345.172",
+    "@indigoai-us/hq-hook-darwin-x64": "5.345.172",
+    "@indigoai-us/hq-hook-darwin-arm64": "5.345.172",
+    "@indigoai-us/hq-hook-linux-x64": "5.345.172",
+    "@indigoai-us/hq-hook-linux-arm64": "5.345.172"
   }
 }

package/dist/lib/daemon/sync-core-canary.d.ts

@@ -1,9 +0,0 @@
-import type { SyncCoreCanaryConfig } from "./config.js";
-export interface SyncCoreCanaryPlan {
-    binaryPath?: string;
-    vaultUid?: string;
-    canaryPath?: string;
-    blockedReason?: string;
-}
-export declare function inspectSyncCoreCanary(home: string, hqRoot: string | undefined, config: SyncCoreCanaryConfig, platform: NodeJS.Platform): SyncCoreCanaryPlan;
-//# sourceMappingURL=sync-core-canary.d.ts.map

package/dist/lib/daemon/sync-core-canary.js

@@ -1,56 +0,0 @@
-import * as fs from "node:fs";
-import * as path from "node:path";
-import { parse as parseToml } from "smol-toml";
-function blocked(config, blockedReason) {
-    return { ...config, blockedReason };
-}
-function record(value) {
-    return value && typeof value === "object" && !Array.isArray(value)
-        ? value
-        : undefined;
-}
-export function inspectSyncCoreCanary(home, hqRoot, config, platform) {
-    if (platform !== "darwin" && platform !== "linux") {
-        return blocked(config, "the sync-core canary requires macOS or Linux");
-    }
-    try {
-        if (!fs.statSync(config.binaryPath).isFile())
-            return blocked(config, "the sync-core binary is missing");
-        fs.accessSync(config.binaryPath, fs.constants.X_OK);
-    }
-    catch {
-        return blocked(config, "the sync-core binary is missing or not executable");
-    }
-    const configPath = path.join(home, ".hq", "sync-core", "config.toml");
-    let sync;
-    try {
-        sync = record(record(parseToml(fs.readFileSync(configPath, "utf8")))?.sync);
-    }
-    catch {
-        return blocked(config, `the sync-core config at ${configPath} is missing or invalid`);
-    }
-    if (sync?.engine !== "rust-test" || sync.mode !== "full") {
-        return blocked(config, "the sync-core config must select engine rust-test in full mode");
-    }
-    const vaults = sync.vaults;
-    if (!Array.isArray(vaults) || vaults.length !== 1 || vaults[0] !== config.vaultUid) {
-        return blocked(config, `the sync-core config must name exactly ${config.vaultUid}`);
-    }
-    if (!hqRoot || typeof sync.hq_root !== "string" || path.resolve(sync.hq_root) !== path.resolve(hqRoot)) {
-        return blocked(config, "the sync-core config must use the daemon's HQ folder");
-    }
-    const canaryPath = typeof sync.canary_path === "string" ? sync.canary_path.trim() : "";
-    if (!canaryPath)
-        return blocked(config, "the sync-core config must name one canary path");
-    const ownedPath = path.join(home, ".hq", "sync-core", "owned", `${config.vaultUid}.json`);
-    try {
-        const owned = record(JSON.parse(fs.readFileSync(ownedPath, "utf8")));
-        if (owned?.vault !== config.vaultUid)
-            throw new Error("mismatch");
-    }
-    catch {
-        return blocked(config, `the sync-core ownership record for ${config.vaultUid} is missing or invalid`);
-    }
-    return { ...config, canaryPath };
-}
-//# sourceMappingURL=sync-core-canary.js.map