@indigoai-us/hq-cli 5.321.0 → 5.322.0

package/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## [Unreleased]
 
+## [5.322.0] — 2026-10-02
+
+- Retried idempotent vault API GETs once when API Gateway returned its generic Lambda-invoke 504 response without a function request ID.
+
+- Defer CLI-update daemon restarts until the sync runner finishes its pass, coalesce pending restart requests, and bound mesh daemon shutdown steps with clearer SIGTERM escalation diagnostics.
+
 ## [5.321.0] — 2026-10-02
 
 - Project catalog cache keys include their source directory, and runtime-probe tests isolate the host CLI and scope checks to hooks.

package/dist/commands/daemon.js

@@ -632,7 +632,7 @@ async function runServiceSelfUpdate(d) {
         tools: TOOL_NAMES.map((name) => hostToolUpdater(name, env, d.home)),
         updateHqCloud: async () => updateManagedHqCloud(paths.root, env),
         hqCloudUnits: hqCloudUnits(detected.type, enabledServices(detected.type, readDaemonConfig(paths.configPath))),
-        requestDaemonRestart: () => writeControlRequest(paths.requestsDir, { action: "restart-daemon" }),
+        requestDaemonRestart: () => writeControlRequest(paths.requestsDir, { action: "restart-daemon", deferUntilSyncIdle: true }),
         requestRestart: (unit) => writeControlRequest(paths.requestsDir, { action: "restart", unit }),
         log: d.out,
     });

package/dist/lib/daemon/requests.d.ts

@@ -22,6 +22,7 @@ export type ControlRequest = {
 /** Restart the whole daemon, for example after the updater installed a new CLI. */
  | {
     action: "restart-daemon";
+    deferUntilSyncIdle?: boolean;
 };
 export declare function writeControlRequest(dir: string, request: ControlRequest, now?: () => number): string;
 export declare function takeControlRequests(dir: string): ControlRequest[];

package/dist/lib/daemon/requests.js

@@ -18,7 +18,7 @@ function isControlRequest(value) {
     if (v.action === "trigger" || v.action === "stop")
         return typeof v.unit === "string";
     if (v.action === "restart-daemon")
-        return true;
+        return v.deferUntilSyncIdle === undefined || typeof v.deferUntilSyncIdle === "boolean";
     if (v.action === "reconcile")
         return true;
     return false;

package/dist/lib/daemon/run.js

@@ -594,7 +594,7 @@ export async function runDaemon(opts) {
         }
     };
     // Set once the shutdown path exists below; a restart request before then is dropped.
-    let restartDaemon = () => { };
+    let restartDaemon = () => false;
     const supervisor = createSupervisor({
         machineType,
         host: opts.host,
@@ -730,12 +730,12 @@ export async function runDaemon(opts) {
         restartDaemon = () => {
             if (!restartsForParent(opts)) {
                 log("warn", "a restart was requested, but no service manager would start this daemon again; restart it yourself");
-                return;
+                return false;
             }
             if (stopping || restartPending)
-                return;
+                return true;
             restartPending = true;
-            void handoffWanted()
+            return handoffWanted()
                 .catch((err) => {
                 log("warn", `could not decide on a handoff; restarting every service: ${err.message}`);
                 return false;
@@ -745,6 +745,7 @@ export async function runDaemon(opts) {
                     handOver();
                 else
                     shutdown("restarting onto the updated CLI", RESTART_EXIT_CODE);
+                return true;
             });
         };
         process.on("SIGTERM", () => shutdown("received SIGTERM"));

package/dist/lib/daemon/succession.js

@@ -202,7 +202,7 @@ export async function stopOrphans(orphans, deps) {
         remaining = remaining.filter(same);
     }
     for (const r of remaining) {
-        deps.log("warn", `${r.id} (pid ${r.pid}) ignored SIGTERM; sending SIGKILL`);
+        deps.log("warn", `${r.id} (pid ${r.pid}) did not exit within ${graceMs / 1_000} s of SIGTERM; sending SIGKILL`);
         send(r, "SIGKILL");
     }
 }

package/dist/lib/daemon/supervisor.d.ts

@@ -26,6 +26,7 @@ import type { MachineType } from "./machine-type.js";
 import type { ControlRequest } from "./requests.js";
 import { syncPassResult } from "./sync-runner.js";
 export declare const STOP_ESCALATION_MS = 10000;
+export declare const CLI_UPDATE_SYNC_IDLE_MAX_MS: number;
 /** After a handoff, how long a replaced unit must stay up before the next one is replaced. */
 export declare const HANDOFF_STAGGER_MS = 15000;
 /** How long a draining unit may take to finish its work before it gets SIGTERM. */
@@ -94,7 +95,7 @@ export interface SupervisorDeps {
     runInternal: (task: string) => Promise<boolean>;
     takeRequests: () => ControlRequest[];
     /** Restart the whole daemon process (a `restart-daemon` request). */
-    restartDaemon?: () => void;
+    restartDaemon?: () => void | boolean | Promise<boolean | void>;
     writeState: (state: DaemonState) => void;
     log: (level: "info" | "warn" | "error", msg: string) => void;
     now?: () => number;

package/dist/lib/daemon/supervisor.js

@@ -27,6 +27,7 @@ const BACKOFF_BASE_MS = 1_000;
 const BACKOFF_MAX_MS = 60_000;
 const HEALTHY_RUN_MS = 60_000;
 export const STOP_ESCALATION_MS = 10_000;
+export const CLI_UPDATE_SYNC_IDLE_MAX_MS = 10 * 60_000;
 const MESH_OWNER_RECHECK_MS = 15_000;
 const MESH_OWNER_LOG_INTERVAL_MS = 10 * 60_000;
 /** After a handoff, how long a replaced unit must stay up before the next one is replaced. */
@@ -61,6 +62,13 @@ export function planKey(plan, cliVersion) {
 function errorMessage(err) {
     return err instanceof Error ? err.message : String(err);
 }
+function lastProtocolHint(slot) {
+    if (slot.lastProtocolStatus)
+        return ` (last status: ${slot.lastProtocolStatus})`;
+    if (slot.lastProtocolLine)
+        return ` (last protocol line: ${slot.lastProtocolLine})`;
+    return "";
+}
 export function createSupervisor(deps) {
     const now = deps.now ?? Date.now;
     const setT = deps.setTimeout ?? ((fn, ms) => setTimeout(fn, ms));
@@ -81,6 +89,9 @@ export function createSupervisor(deps) {
     const live = new Set();
     let drained = null;
     let onDrained = null;
+    let deferredCliRestart = false;
+    let deferredCliRestartTimer;
+    let cliRestartInProgress = false;
     const snapshot = () => ({
         version: 1,
         pid: deps.pid,
@@ -108,6 +119,71 @@ export function createSupervisor(deps) {
             clearT(slot.heartbeatTimer);
         slot.heartbeatTimer = undefined;
     };
+    const clearDeferredCliRestart = () => {
+        deferredCliRestart = false;
+        if (deferredCliRestartTimer !== undefined)
+            clearT(deferredCliRestartTimer);
+        deferredCliRestartTimer = undefined;
+    };
+    const runDeferredCliRestart = () => {
+        if (!deferredCliRestart)
+            return;
+        clearDeferredCliRestart();
+        cliRestartInProgress = true;
+        const sync = [...slots.values()].find((slot) => slot.plan.service === "sync" && slot.child);
+        if (sync)
+            terminate(sync);
+        deps.log("info", "sync runner is between passes; restarting daemon onto the updated CLI");
+        const releaseHold = () => {
+            cliRestartInProgress = false;
+            const currentSync = [...slots.values()].find((slot) => slot.plan.service === "sync");
+            if (!currentSync)
+                return;
+            if (currentSync.child)
+                terminate(currentSync, () => activate(currentSync));
+            else
+                activate(currentSync);
+        };
+        try {
+            const result = deps.restartDaemon?.();
+            if (result === false)
+                releaseHold();
+            else if (result && typeof result === "object" && "then" in result) {
+                void Promise.resolve(result).then((accepted) => {
+                    if (accepted === false)
+                        releaseHold();
+                }).catch((err) => {
+                    deps.log("error", `CLI-update restart handoff failed: ${errorMessage(err)}`);
+                    releaseHold();
+                });
+            }
+        }
+        catch (err) {
+            deps.log("error", `CLI-update restart handoff failed: ${errorMessage(err)}`);
+            releaseHold();
+        }
+    };
+    const requestCliUpdateRestart = () => {
+        if (!deps.restartDaemon)
+            return;
+        const sync = [...slots.values()].find((slot) => slot.plan.service === "sync" && slot.child);
+        if (!sync) {
+            void deps.restartDaemon();
+            return;
+        }
+        if (sync.syncPassActive === false) {
+            deferredCliRestart = true;
+            runDeferredCliRestart();
+            return;
+        }
+        if (deferredCliRestart) {
+            deps.log("info", "coalescing CLI-update restart while waiting for sync to become idle");
+            return;
+        }
+        deferredCliRestart = true;
+        deps.log("info", "deferring CLI-update restart until sync is idle (maximum 10 minutes)");
+        deferredCliRestartTimer = setT(runDeferredCliRestart, CLI_UPDATE_SYNC_IDLE_MAX_MS);
+    };
     const setBlocked = (slot) => {
         const block = slot.plan.blocked;
         slot.state.status = block.state;
@@ -136,6 +212,12 @@ export function createSupervisor(deps) {
             then?.();
             return;
         }
+        if (slot.expectingExit) {
+            // Keep the original stop deadline and signal; only update the action
+            // that runs after exit (for example, start the changed command).
+            slot.afterExit = then;
+            return;
+        }
         slot.expectingExit = true;
         slot.afterExit = then;
         if (slot.killTimer !== undefined)
@@ -145,7 +227,7 @@ export function createSupervisor(deps) {
                 slot.killTimer = undefined;
                 if (slot.child !== child)
                     return;
-                deps.log("warn", `${slot.plan.id} ignored SIGTERM; sending SIGKILL`);
+                deps.log("warn", `${slot.plan.id} did not exit within ${STOP_ESCALATION_MS / 1_000} s of SIGTERM; sending SIGKILL${lastProtocolHint(slot)}`);
                 signalChild(slot, child, "SIGKILL");
             }, STOP_ESCALATION_MS);
         };
@@ -211,6 +293,25 @@ export function createSupervisor(deps) {
             }
             if (!event)
                 return;
+            slot.lastProtocolLine = line.length > 240 ? `${line.slice(0, 237)}...` : line;
+            if (slot.plan.service === "sync") {
+                if (event.type === "fanout-plan")
+                    slot.syncPassActive = true;
+                else if (event.type === "all-complete")
+                    slot.syncPassActive = false;
+                else if (event.type === "status" && typeof event.state === "string") {
+                    slot.lastProtocolStatus = event.state;
+                    if (event.state === "stopped" || event.state === "waiting-for-lock")
+                        slot.syncPassActive = false;
+                    else if (slot.syncPassActive === undefined && (event.state === "starting" || event.state === "running")) {
+                        // Before the first pass boundary, status heartbeats cannot prove
+                        // that startup work has not already entered a pass.
+                        slot.syncPassActive = true;
+                    }
+                }
+                if (deferredCliRestart && slot.syncPassActive === false)
+                    runDeferredCliRestart();
+            }
             const at = now();
             slot.state.lastHeartbeatAt = new Date(at).toISOString();
             const pass = syncPassResult(event, slot.state.lastHeartbeatAt);
@@ -358,6 +459,15 @@ export function createSupervisor(deps) {
     const startLong = (slot) => {
         if (stopped)
             return;
+        if (cliRestartInProgress && slot.plan.service === "sync") {
+            slot.state.status = "waiting";
+            slot.state.reason = "daemon restart pending";
+            delete slot.state.nextRunAt;
+            return;
+        }
+        delete slot.lastProtocolLine;
+        delete slot.lastProtocolStatus;
+        delete slot.syncPassActive;
         let child;
         try {
             child = deps.spawn(slot.plan);
@@ -550,7 +660,10 @@ export function createSupervisor(deps) {
                 deps.log("info", `restarting ${plan.id}: its command changed`);
                 existing.attempts = 0;
                 existing.state.restarts = 0;
-                terminate(existing, () => activate(existing));
+                if (cliRestartInProgress && plan.service === "sync")
+                    terminate(existing);
+                else
+                    terminate(existing, () => activate(existing));
             }
         }
         slots = next;
@@ -650,11 +763,19 @@ export function createSupervisor(deps) {
             return;
         }
         if (req.action === "restart-daemon") {
-            deps.log("info", "restarting the daemon (requested)");
-            if (deps.restartDaemon)
-                deps.restartDaemon();
-            else
+            if (req.deferUntilSyncIdle && deps.restartDaemon) {
+                deps.log("info", "CLI-update daemon restart requested");
+                requestCliUpdateRestart();
+            }
+            else if (deps.restartDaemon) {
+                deps.log("info", "restarting the daemon (requested)");
+                clearDeferredCliRestart();
+                void deps.restartDaemon();
+            }
+            else {
+                deps.log("info", "restarting the daemon (requested)");
                 deps.log("warn", "this daemon cannot restart itself; restart it to load the new version");
+            }
             return;
         }
         // A bot created since the last plan read is asked for before the timer
@@ -756,7 +877,11 @@ export function createSupervisor(deps) {
                 clearT(requestTimer);
             if (handoffTimer !== undefined)
                 clearT(handoffTimer);
+            clearDeferredCliRestart();
+            const childOwners = new Map();
             for (const slot of slots.values()) {
+                if (slot.child)
+                    childOwners.set(slot.child, slot);
                 clearTimer(slot);
                 clearHeartbeatTimer(slot);
                 slot.child = undefined;
@@ -784,7 +909,8 @@ export function createSupervisor(deps) {
                 }
                 const escalation = setT(() => {
                     for (const child of live) {
-                        deps.log("warn", `pid ${child.pid ?? "?"} ignored SIGTERM; sending SIGKILL`);
+                        const owner = childOwners.get(child);
+                        deps.log("warn", `${owner?.plan.id ?? `pid ${child.pid ?? "?"}`} did not exit within ${STOP_ESCALATION_MS / 1_000} s of SIGTERM; sending SIGKILL${owner ? lastProtocolHint(owner) : ""}`);
                         try {
                             child.kill("SIGKILL");
                         }
@@ -818,6 +944,7 @@ export function createSupervisor(deps) {
                 clearT(requestTimer);
             if (handoffTimer !== undefined)
                 clearT(handoffTimer);
+            clearDeferredCliRestart();
             for (const slot of slots.values()) {
                 clearTimer(slot);
                 if (slot.killTimer !== undefined)

package/dist/lib/mesh/live/daemon/run.d.ts

@@ -24,6 +24,8 @@ export declare const FLUSH_INTERVAL_MS = 10000;
 /** Keep startup recovery bounded: the next daemon start continues the backlog. */
 export declare const HELD_BACKFILL_STARTUP_BATCH_SIZE = 100;
 export declare const DEVICE_DEFAULT_MEMBERSHIP_TIMEOUT_MS = 2000;
+/** Three sequential async shutdown steps must finish within six seconds. */
+export declare const SHUTDOWN_STEP_TIMEOUT_MS = 2000;
 /** Non-zero exit reserved for CLI version drift so legacy Restart=on-failure units restart. */
 export declare const DAEMON_VERSION_DRIFT_EXIT_CODE = 75;
 /**

package/dist/lib/mesh/live/daemon/run.js

@@ -45,6 +45,8 @@ export const FLUSH_INTERVAL_MS = 10_000;
 /** Keep startup recovery bounded: the next daemon start continues the backlog. */
 export const HELD_BACKFILL_STARTUP_BATCH_SIZE = 100;
 export const DEVICE_DEFAULT_MEMBERSHIP_TIMEOUT_MS = 2_000;
+/** Three sequential async shutdown steps must finish within six seconds. */
+export const SHUTDOWN_STEP_TIMEOUT_MS = 2_000;
 /** Non-zero exit reserved for CLI version drift so legacy Restart=on-failure units restart. */
 export const DAEMON_VERSION_DRIFT_EXIT_CODE = 75;
 /**
@@ -871,29 +873,48 @@ export async function runMeshDaemon(deps = {}) {
         catch {
             /* ignore */
         }
-        // Final flush, then clean MQTT DISCONNECT so the broker emits the
-        // lifecycle event; offline is published server-side only.
-        await doFlushCycle("shutdown", { force: true });
-        if (workCache) {
+        const stopStep = async (name, action) => {
+            let deadline;
+            let timedOut = false;
+            const timeout = new Promise((resolve) => {
+                deadline = timers.setTimeout(() => {
+                    timedOut = true;
+                    resolve();
+                }, SHUTDOWN_STEP_TIMEOUT_MS);
+            });
             try {
-                await workCache.stop();
+                await Promise.race([Promise.resolve().then(action), timeout]);
+                if (timedOut)
+                    log(dir, `${name} did not finish within ${SHUTDOWN_STEP_TIMEOUT_MS / 1_000} s; continuing shutdown`);
             }
             catch (err) {
                 const msg = err instanceof Error ? err.message : String(err);
-                log(dir, `work cache stop error: ${msg.slice(0, 120)}`);
+                log(dir, `${name} error: ${msg.slice(0, 120)}`);
+            }
+            finally {
+                if (deadline !== undefined)
+                    timers.clearTimeout(deadline);
             }
+        };
+        // Final flush, then clean MQTT DISCONNECT so the broker emits the
+        // lifecycle event; offline is published server-side only.
+        await stopStep("shutdown flush", () => doFlushCycle("shutdown", { force: true }));
+        if (workCache) {
+            await stopStep("work cache stop", () => workCache.stop());
         }
         if (presence) {
-            try {
-                await presence.stop();
-            }
-            catch (err) {
-                const msg = err instanceof Error ? err.message : String(err);
-                log(dir, `presence stop error: ${msg.slice(0, 120)}`);
-            }
+            await stopStep("presence stop", () => presence.stop());
+        }
+        try {
+            patchDaemonState(dir, { mqttState: "closed", companiesOnline: [] }, now);
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            log(dir, `daemon shutdown state update error: ${msg.slice(0, 120)}`);
+        }
+        finally {
+            releasePidLock(dir, lockDeps);
         }
-        patchDaemonState(dir, { mqttState: "closed", companiesOnline: [] }, now);
-        releasePidLock(dir, lockDeps);
         log(dir, "daemon stopped");
     };
     const onSignal = () => {

package/dist/utils/vault-api.js

@@ -23,6 +23,57 @@ import { readCachedScopeUid, writeCachedScopeUid } from './secrets-cache.js';
  * whole-process watchdog.
  */
 const IDENTITY_LOOKUP_TIMEOUT_MS = 15_000;
+const LAMBDA_INVOKE_RETRY_MAX_DELAY_MS = 100;
+function isLambdaInvokeServiceError(response, bodyText) {
+    if (response.status !== 504) {
+        return false;
+    }
+    try {
+        const body = JSON.parse(bodyText);
+        return (typeof body === 'object' &&
+            body !== null &&
+            !Array.isArray(body) &&
+            Object.keys(body).length === 1 &&
+            body.message === 'Internal server error');
+    }
+    catch {
+        return false;
+    }
+}
+async function fetchWithLambdaInvokeRetry(fetchOnce, retryEligible, signal) {
+    const first = await fetchOnce();
+    if (!retryEligible || first.status !== 504)
+        return first;
+    const bodyText = await first.text();
+    if (!isLambdaInvokeServiceError(first, bodyText)) {
+        return new Response(bodyText, {
+            status: first.status,
+            statusText: first.statusText,
+            headers: first.headers,
+        });
+    }
+    const delay = 25 + Math.floor(Math.random() * (LAMBDA_INVOKE_RETRY_MAX_DELAY_MS - 24));
+    await jitteredDelayWithAbort(delay, signal);
+    return fetchOnce();
+}
+function jitteredDelayWithAbort(ms, signal) {
+    if (signal?.aborted) {
+        return Promise.reject(signal.reason ?? new DOMException('The operation was aborted', 'AbortError'));
+    }
+    return new Promise((resolve, reject) => {
+        const cleanup = () => signal?.removeEventListener('abort', abort);
+        const timer = setTimeout(() => {
+            cleanup();
+            resolve();
+        }, ms);
+        const abort = () => {
+            clearTimeout(timer);
+            cleanup();
+            reject(signal?.reason ?? new DOMException('The operation was aborted', 'AbortError'));
+        };
+        signal?.addEventListener('abort', abort, { once: true });
+    });
+}
 /**
  * The identity-lookup abort bound, in milliseconds. Defaults to the shipped
  * IDENTITY_LOOKUP_TIMEOUT_MS (15s). An optional HQ_IDENTITY_LOOKUP_TIMEOUT_MS
@@ -192,41 +243,40 @@ export async function vaultApiFetch(opts) {
         level: "info",
         data: { url: safeUrl, method },
     });
-    let response;
-    try {
-        response = await fetch(url.toString(), {
-            method,
-            headers: {
-                Authorization: `Bearer ${opts.token}`,
-                'Content-Type': 'application/json',
-                'x-hq-client-name': HQ_CLIENT_NAME,
-                // US-003: version alongside the client family, mirroring the desktop
-                // app's client_info.rs headers, so the server can see client-version
-                // skew on EVERY authenticated request, not only health heartbeats.
-                // Fixed build constant, never caller-supplied; attribution only.
-                'x-hq-client-version': CLI_VERSION,
-                ...opts.headers,
-            },
-            body: opts.body ? JSON.stringify(opts.body) : undefined,
-            signal: opts.signal,
-        });
-    }
-    catch (err) {
-        // A transport failure never reaches the server, so the non-2xx breadcrumb
-        // below never fires and the run's last HTTP trace is the request that
-        // silently vanished (exactly what made HQ-CLI-G's event unreadable). Leave
-        // a bounded, non-secret trail — the already-redacted safeUrl plus the
-        // recognized transport code — then RE-THROW the original error untouched,
-        // so `main.ts`'s classifier stays the single decision point.
-        const code = networkTransportErrorCode(err);
-        Sentry.addBreadcrumb({
-            category: "http",
-            message: `${method} ${path} → transport error`,
-            level: "warning",
-            data: { url: safeUrl, method, ...(code ? { code } : {}) },
-        });
-        throw err;
-    }
+    const requestInit = {
+        method,
+        headers: {
+            Authorization: `Bearer ${opts.token}`,
+            'Content-Type': 'application/json',
+            'x-hq-client-name': HQ_CLIENT_NAME,
+            // US-003: version alongside the client family, mirroring the desktop
+            // app's client_info.rs headers, so the server can see client-version
+            // skew on EVERY authenticated request, not only health heartbeats.
+            // Fixed build constant, never caller-supplied; attribution only.
+            'x-hq-client-version': CLI_VERSION,
+            ...opts.headers,
+        },
+        body: opts.body ? JSON.stringify(opts.body) : undefined,
+        signal: opts.signal,
+    };
+    const fetchWithTransportBreadcrumb = async () => {
+        try {
+            return await fetch(url.toString(), requestInit);
+        }
+        catch (err) {
+            // Keep the original transport failure and its bounded, redacted trail on
+            // both the first request and the single retry.
+            const code = networkTransportErrorCode(err);
+            Sentry.addBreadcrumb({
+                category: "http",
+                message: `${method} ${path} → transport error`,
+                level: "warning",
+                data: { url: safeUrl, method, ...(code ? { code } : {}) },
+            });
+            throw err;
+        }
+    };
+    const response = await fetchWithLambdaInvokeRetry(fetchWithTransportBreadcrumb, method.toUpperCase() === 'GET' && opts.body === undefined, opts.signal);
     if (!response.ok) {
         Sentry.addBreadcrumb({
             category: "http",
@@ -287,10 +337,10 @@ export async function vaultApiFetchPublic(opts) {
         level: 'info',
         data: { url: safeUrl, method: 'GET' },
     });
-    const response = await fetch(url.toString(), {
+    const response = await fetchWithLambdaInvokeRetry(() => fetch(url.toString(), {
         method: 'GET',
         headers: { 'Content-Type': 'application/json' },
-    });
+    }), true);
     if (!response.ok) {
         Sentry.addBreadcrumb({
             category: 'http',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.321.0",
+  "version": "5.322.0",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {