@indigoai-us/hq-cli 5.345.47 → 5.345.48

package/dist/lib/daemon/supervisor.js

@@ -13,7 +13,8 @@
  * - restart/trigger/stop requests from the CLI are polled and applied; a
  *   request for a unit that is not planned yet re-reads the plan first;
  * - stop() sends SIGTERM to every child, escalates to SIGKILL after
- *   STOP_ESCALATION_MS, and resolves once every child has exited;
+ *   STOP_ESCALATION_MS except for a sync runner already stopping, which it lets
+ *   finish persisting pending pushes before resolving;
  * - release() instead leaves every child running and reports the long-running
  *   ones, so the next daemon can adopt them (`adopt`). Adopted units whose
  *   command is unchanged keep running; changed ones are replaced one at a
@@ -27,6 +28,7 @@ const BACKOFF_BASE_MS = 1_000;
 const BACKOFF_MAX_MS = 60_000;
 const HEALTHY_RUN_MS = 60_000;
 export const STOP_ESCALATION_MS = 10_000;
+export const SYNC_RUNNER_PERSIST_GRACE_MS = 60_000;
 const CLI_UPDATE_SYNC_IDLE_MAX_MS = 10 * 60_000;
 export const CLI_UPDATE_SYNC_IDLE_KILL_GRACE_MS = 60_000;
 const MESH_OWNER_RECHECK_MS = 15_000;
@@ -91,6 +93,7 @@ export function createSupervisor(deps) {
     let handoffTimer;
     /** Every child process not yet known to have exited, including ones the slots have let go of. */
     const live = new Set();
+    const liveOwners = new Map();
     let drained = null;
     let onDrained = null;
     let deferredCliRestart = false;
@@ -247,6 +250,16 @@ export function createSupervisor(deps) {
             deps.log("error", `could not send ${signal} to ${slot.plan.id}: ${errorMessage(err)}`);
         }
     };
+    const syncRunnerIsStopping = (slot) => slot?.plan.service === "sync" && slot.lastProtocolStatus === "stopping";
+    const scheduleSyncRunnerPersistEscalation = (slot, child) => {
+        slot.killTimer = setT(() => {
+            slot.killTimer = undefined;
+            if (slot.child !== child)
+                return;
+            deps.log("warn", `${slot.plan.id} did not finish persisting pending pushes within ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s; sending SIGKILL${lastProtocolHint(slot)}`);
+            signalChild(slot, child, "SIGKILL");
+        }, SYNC_RUNNER_PERSIST_GRACE_MS);
+    };
     /**
      * Stop the slot's child (if any), then run `then` once it has exited. With
      * `drain`, a unit that has a drain signal gets it first and SIGTERM only
@@ -276,6 +289,11 @@ export function createSupervisor(deps) {
                 slot.killTimer = undefined;
                 if (slot.child !== child)
                     return;
+                if (syncRunnerIsStopping(slot)) {
+                    deps.log("info", `${slot.plan.id} is stopping; allowing ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s for pending pushes to persist`);
+                    scheduleSyncRunnerPersistEscalation(slot, child);
+                    return;
+                }
                 deps.log("warn", `${slot.plan.id} did not exit within ${killGraceMs / 1_000} s of SIGTERM; sending SIGKILL${lastProtocolHint(slot)}`);
                 signalChild(slot, child, "SIGKILL");
             }, killGraceMs);
@@ -313,6 +331,7 @@ export function createSupervisor(deps) {
     };
     const watch = (slot, child) => {
         live.add(child);
+        liveOwners.set(child, slot);
         let ended = false;
         const timeoutMs = slot.plan.heartbeatTimeoutMs;
         const decoder = timeoutMs ? new StringDecoder("utf8") : null;
@@ -350,6 +369,8 @@ export function createSupervisor(deps) {
                     slot.syncPassActive = false;
                 else if (event.type === "status" && typeof event.state === "string") {
                     slot.lastProtocolStatus = event.state;
+                    if (event.state === "stopping")
+                        deps.onSyncRunnerStopping?.();
                     if (event.state === "stopped" || event.state === "waiting-for-lock")
                         slot.syncPassActive = false;
                     else if (slot.syncPassActive === undefined && (event.state === "starting" || event.state === "running")) {
@@ -405,6 +426,7 @@ export function createSupervisor(deps) {
                 return;
             ended = true;
             live.delete(child);
+            liveOwners.delete(child);
             if (live.size === 0)
                 onDrained?.();
             if (slot.child !== child)
@@ -991,10 +1013,8 @@ export function createSupervisor(deps) {
                 clearT(handoffTimer);
             clearDeferredCliRestart();
             clearPendingSyncCommandRestart();
-            const childOwners = new Map();
+            const childOwners = new Map(liveOwners);
             for (const slot of slots.values()) {
-                if (slot.child)
-                    childOwners.set(slot.child, slot);
                 clearTimer(slot);
                 clearHeartbeatTimer(slot);
                 slot.child = undefined;
@@ -1020,9 +1040,16 @@ export function createSupervisor(deps) {
                     resolve();
                     return;
                 }
+                const persistingSyncRunners = new Set();
+                let persistEscalation;
                 const escalation = setT(() => {
                     for (const child of live) {
                         const owner = childOwners.get(child);
+                        if (owner && syncRunnerIsStopping(owner)) {
+                            deps.log("info", `${owner.plan.id} is stopping; allowing ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s for pending pushes to persist`);
+                            persistingSyncRunners.add(child);
+                            continue;
+                        }
                         deps.log("warn", `${owner?.plan.id ?? `pid ${child.pid ?? "?"}`} did not exit within ${STOP_ESCALATION_MS / 1_000} s of SIGTERM; sending SIGKILL${owner ? lastProtocolHint(owner) : ""}`);
                         try {
                             child.kill("SIGKILL");
@@ -1031,10 +1058,28 @@ export function createSupervisor(deps) {
                             deps.log("error", `could not SIGKILL pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
                         }
                     }
+                    if (persistingSyncRunners.size > 0) {
+                        persistEscalation = setT(() => {
+                            for (const child of persistingSyncRunners) {
+                                if (!live.has(child))
+                                    continue;
+                                const owner = childOwners.get(child);
+                                deps.log("warn", `${owner?.plan.id ?? `pid ${child.pid ?? "?"}`} did not finish persisting pending pushes within ${SYNC_RUNNER_PERSIST_GRACE_MS / 1_000} s; sending SIGKILL${owner ? lastProtocolHint(owner) : ""}`);
+                                try {
+                                    child.kill("SIGKILL");
+                                }
+                                catch (err) {
+                                    deps.log("error", `could not SIGKILL pid ${child.pid ?? "?"}: ${errorMessage(err)}`);
+                                }
+                            }
+                        }, SYNC_RUNNER_PERSIST_GRACE_MS);
+                    }
                 }, STOP_ESCALATION_MS);
                 onDrained = () => {
                     onDrained = null;
                     clearT(escalation);
+                    if (persistEscalation !== undefined)
+                        clearT(persistEscalation);
                     resolve();
                 };
             });
@@ -1042,6 +1087,7 @@ export function createSupervisor(deps) {
         },
         reconcile,
         state: snapshot,
+        hasStoppingSyncRunner: () => [...liveOwners.values()].some(syncRunnerIsStopping),
         release() {
             const released = [];
             for (const slot of slots.values()) {

package/dist/utils/resident-package-preload.js

@@ -83,6 +83,8 @@ export const RESIDENT_PACKAGE_MODULES = {
         "../utils/cli-telemetry.js",
         "../utils/cognito-session.js",
         "../lib/first-sync-next-step.js",
+        "../lib/process-flag-company-context.js",
+        "../lib/flag-snapshot-file.js",
         "@indigoai-us/hq-cloud",
     ],
 };
@@ -352,6 +354,8 @@ const LOADERS = {
         () => import("../utils/cli-telemetry.js"),
         () => import("../utils/cognito-session.js"),
         () => import("../lib/first-sync-next-step.js"),
+        () => import("../lib/process-flag-company-context.js"),
+        () => import("../lib/flag-snapshot-file.js"),
         () => import("@indigoai-us/hq-cloud"),
     ],
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.345.47",
+  "version": "5.345.48",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {