@indigoai-us/hq-cli 5.345.87 → 5.345.88

package/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## [Unreleased]
 
+## [5.345.88] — 2026-10-11
+- Work Mesh no longer crash-loops through memory during an internet outage:
+  failed presence and membership lookups back off and log once per drop, and a
+  downed network stops the held-event flush instead of retrying every session.
+  Held events that can be matched to the device's default company are delivered
+  instead of being re-queued forever.
+
 ## [5.345.87] — 2026-10-11
 - `hq bot create setup` no longer fails when HQ's assistant is already on this
   computer. It answers `ok` with `outcome` `existing`, `adopted` or `created`,

package/dist/lib/mesh/live/daemon/presence.d.ts

@@ -101,6 +101,12 @@ export interface PresenceClientOptions {
  * backoff counters.
  */
 export declare function backoffDelayMs(attempt: number, baseMs: number, maxMs: number, random: () => number, previousMs?: number, minMs?: number): number;
+/**
+ * Stable episode key for transport failures so a network drop logs once
+ * instead of once per reconnect/error event (those Error objects otherwise
+ * pin the heap while the daemon crash-loops).
+ */
+export declare function networkErrorEpisodeKey(err: unknown): string | null;
 export declare function buildPresencePayload(input: {
     status: "online" | "offline";
     actorUid: string;
@@ -129,6 +135,10 @@ export declare class PresenceClient {
     private publishedTopics;
     /** Last refusal code we logged via onError (episode dedupe). */
     private loggedRefusalKey;
+    /** Last transport-failure episode logged via onError (network-drop dedupe). */
+    private loggedNetworkErrorKey;
+    /** Serialize connect/vend so a drop cannot stack MQTT clients or fetches. */
+    private connectPromise;
     private refusal;
     private preferredHandle;
     private preferredSnapshot;
@@ -163,6 +173,9 @@ export declare class PresenceClient {
     /** Force a reconnect cycle (network change / sleep-wake). */
     reconnectNow(): void;
     private teardownClient;
+    private emitError;
+    private clearNetworkErrorEpisode;
+    private enqueueConnect;
     private onCredentialsRenewed;
     private handleRenewalError;
     private refusalKey;

package/dist/lib/mesh/live/daemon/presence.js

@@ -54,6 +54,27 @@ export function backoffDelayMs(attempt, baseMs, maxMs, random, previousMs = 0, m
     const raw = Math.max(0, random() * cap);
     return Math.min(maxMs, Math.max(previousMs, raw, minMs));
 }
+const NETWORK_ERROR_RE = /fetch failed|ETIMEDOUT|ENOTFOUND|ECONNREFUSED|EAI_AGAIN|ENETUNREACH|ECONNRESET|ENETDOWN|EHOSTUNREACH/i;
+/**
+ * Stable episode key for transport failures so a network drop logs once
+ * instead of once per reconnect/error event (those Error objects otherwise
+ * pin the heap while the daemon crash-loops).
+ */
+export function networkErrorEpisodeKey(err) {
+    const code = err !== null && typeof err === "object" && "code" in err &&
+        typeof err.code === "string"
+        ? err.code
+        : "";
+    const msg = err instanceof Error ? err.message : String(err);
+    if (code && NETWORK_ERROR_RE.test(code))
+        return code;
+    if (!NETWORK_ERROR_RE.test(msg))
+        return null;
+    if (/fetch failed/i.test(msg))
+        return "fetch_failed";
+    const match = msg.match(/ETIMEDOUT|ENOTFOUND|ECONNREFUSED|EAI_AGAIN|ENETUNREACH|ECONNRESET|ENETDOWN|EHOSTUNREACH/i);
+    return match ? match[0].toUpperCase() : "network";
+}
 export function buildPresencePayload(input) {
     return {
         v: 1,
@@ -92,6 +113,10 @@ export class PresenceClient {
     publishedTopics = [];
     /** Last refusal code we logged via onError (episode dedupe). */
     loggedRefusalKey = null;
+    /** Last transport-failure episode logged via onError (network-drop dedupe). */
+    loggedNetworkErrorKey = null;
+    /** Serialize connect/vend so a drop cannot stack MQTT clients or fetches. */
+    connectPromise = null;
     refusal = null;
     preferredHandle = null;
     preferredSnapshot = [];
@@ -202,7 +227,7 @@ export class PresenceClient {
         if (this.stopped)
             throw new Error("PresenceClient already stopped");
         this.setState("connecting");
-        await this.connectOnce();
+        await this.enqueueConnect();
     }
     /**
      * Graceful stop: disconnect cleanly (MQTT DISCONNECT so the broker emits
@@ -234,13 +259,20 @@ export class PresenceClient {
         }
         // Backoff counters reset only after a stable connection — not here.
         this.clearReconnect();
-        void this.connectOnce();
+        void this.enqueueConnect();
     }
     teardownClient(force) {
         const client = this.client;
         if (!client)
             return;
         this.client = null;
+        try {
+            const removable = client;
+            removable.removeAllListeners?.();
+        }
+        catch {
+            /* best-effort */
+        }
         try {
             if (force) {
                 client.end(true);
@@ -253,22 +285,43 @@ export class PresenceClient {
             /* best-effort */
         }
     }
+    emitError(err) {
+        const key = networkErrorEpisodeKey(err);
+        if (key) {
+            if (this.loggedNetworkErrorKey === key)
+                return;
+            this.loggedNetworkErrorKey = key;
+        }
+        this.options.onError?.(err);
+    }
+    clearNetworkErrorEpisode() {
+        this.loggedNetworkErrorKey = null;
+    }
+    enqueueConnect() {
+        if (this.connectPromise)
+            return this.connectPromise;
+        this.connectPromise = this.connectOnce().finally(() => {
+            this.connectPromise = null;
+        });
+        return this.connectPromise;
+    }
     async onCredentialsRenewed(bundle) {
         this.bundle = bundle;
         this.notePreferredVend();
         // Successful vend ends a refusal episode (renewal skips connectOnce's vend path).
         this.clearRefusal(true);
+        this.clearNetworkErrorEpisode();
         if (this.stopped)
             return;
         // Reconnect with the new presigned URL; backoff resets only after stable connect.
-        await this.connectOnce();
+        await this.enqueueConnect();
     }
     handleRenewalError(err, info) {
         if (err instanceof CredentialVendError && err.kind === "refused") {
             this.noteRefusal(err, info?.nextRetryAt);
             return;
         }
-        this.options.onError?.(err);
+        this.emitError(err);
     }
     refusalKey(err) {
         return err.code ?? `HTTP_${err.status}`;
@@ -314,6 +367,7 @@ export class PresenceClient {
                 this.bundle = await this.options.fetchCredentials();
                 // Successful vend ends a refusal episode (even before MQTT connects).
                 this.clearRefusal(true);
+                this.clearNetworkErrorEpisode();
                 this.renewal.schedule(this.bundle);
                 this.notePreferredVend();
                 this.armPreferredWatch();
@@ -334,7 +388,7 @@ export class PresenceClient {
                 });
             }
             catch (err) {
-                this.options.onError?.(err);
+                this.emitError(err);
                 this.scheduleReconnect();
                 return;
             }
@@ -342,6 +396,7 @@ export class PresenceClient {
             client.on("connect", () => {
                 if (this.stopped || generation !== this.generation)
                     return;
+                this.clearNetworkErrorEpisode();
                 // Do not reset attempt here — broker may accept then close immediately
                 // (denied publish / policy). Reset only after CONNECT_STABLE_GRACE_MS.
                 this.connectedAtMs = this.timers.now();
@@ -362,7 +417,7 @@ export class PresenceClient {
             client.on("error", (err) => {
                 if (this.stopped || generation !== this.generation)
                     return;
-                this.options.onError?.(err);
+                this.emitError(err);
             });
             client.on("close", () => {
                 if (this.stopped || generation !== this.generation)
@@ -381,7 +436,7 @@ export class PresenceClient {
                 this.scheduleReconnect({ refused: err });
                 return;
             }
-            this.options.onError?.(err);
+            this.emitError(err);
             this.scheduleReconnect();
         }
     }
@@ -401,7 +456,7 @@ export class PresenceClient {
         for (const company of bundle.companies) {
             const topic = company.presenceTopic;
             if (!isOwnPresenceTopic(topic, bundle.actorUid, companyUids)) {
-                this.options.onError?.(new Error(`refusing to publish to non-presence topic`));
+                this.emitError(new Error(`refusing to publish to non-presence topic`));
                 continue;
             }
             await new Promise((resolve) => {
@@ -412,7 +467,7 @@ export class PresenceClient {
                     }
                 }
                 catch (err) {
-                    this.options.onError?.(err);
+                    this.emitError(err);
                     resolve();
                 }
             });
@@ -451,7 +506,7 @@ export class PresenceClient {
             if (opts?.refused) {
                 this.bundle = null;
             }
-            void this.connectOnce();
+            void this.enqueueConnect();
         }, delay);
         return delay;
     }

package/dist/lib/mesh/live/daemon/run.d.ts

@@ -24,6 +24,9 @@ export declare const FLUSH_INTERVAL_MS = 10000;
 /** Keep startup recovery bounded: the next daemon start continues the backlog. */
 export declare const HELD_BACKFILL_STARTUP_BATCH_SIZE = 100;
 export declare const DEVICE_DEFAULT_MEMBERSHIP_TIMEOUT_MS = 2000;
+/** Skip further membership probes after a transport failure so a network drop
+ *  cannot fan out one hanging lookup per held session. */
+export declare const DEVICE_DEFAULT_MEMBERSHIP_UNAVAILABLE_COOLDOWN_MS = 30000;
 /** Three sequential async shutdown steps must finish within six seconds. */
 export declare const SHUTDOWN_STEP_TIMEOUT_MS = 2000;
 /** Non-zero exit reserved for CLI version drift so legacy Restart=on-failure units restart. */

package/dist/lib/mesh/live/daemon/run.js

@@ -45,6 +45,9 @@ export const FLUSH_INTERVAL_MS = 10_000;
 /** Keep startup recovery bounded: the next daemon start continues the backlog. */
 export const HELD_BACKFILL_STARTUP_BATCH_SIZE = 100;
 export const DEVICE_DEFAULT_MEMBERSHIP_TIMEOUT_MS = 2_000;
+/** Skip further membership probes after a transport failure so a network drop
+ *  cannot fan out one hanging lookup per held session. */
+export const DEVICE_DEFAULT_MEMBERSHIP_UNAVAILABLE_COOLDOWN_MS = 30_000;
 /** Three sequential async shutdown steps must finish within six seconds. */
 export const SHUTDOWN_STEP_TIMEOUT_MS = 2_000;
 /** Non-zero exit reserved for CLI version drift so legacy Restart=on-failure units restart. */
@@ -172,6 +175,9 @@ export async function runMeshDaemon(deps = {}) {
     const resolveMembership = deps.resolveMembership ?? resolveActiveMembershipCompany;
     const machineIdentity = deps.machineIdentity ?? isMachineIdentity();
     /** The sole authorization seam for daemon tenant I/O derived from a device default. */
+    let membershipUnavailableUntilMs = 0;
+    let membershipUnavailableError = "membership lookup unavailable";
+    let membershipUnavailableLogged = false;
     const tenantGate = {
         authorize: async (input) => {
             const state = input.state ?? (input.sessionId
@@ -190,6 +196,9 @@ export async function runMeshDaemon(deps = {}) {
             const lookupKey = slug || state?.companyUid || (usingDefaultFallback
                 ? (defaultCompany?.slug || defaultCompany?.uid)
                 : undefined);
+            if (lookupKey && now().getTime() < membershipUnavailableUntilMs) {
+                return { status: "unavailable", error: membershipUnavailableError };
+            }
             let authorization;
             if (!lookupKey) {
                 authorization = { status: "denied" };
@@ -240,14 +249,21 @@ export async function runMeshDaemon(deps = {}) {
                 }
             }
             if (authorization.status === "unavailable") {
-                const prior = readDaemonState(dir)?.deviceDefaultMembershipLookupFailures ?? 0;
-                patchDaemonState(dir, {
-                    lastErrorCode: "DEVICE_DEFAULT_MEMBERSHIP_LOOKUP_FAILED",
-                    deviceDefaultMembershipLookupFailures: prior + 1,
-                }, now);
-                log(dir, `device-default membership lookup unavailable${slug ? ` company=${slug}` : ""}: ${authorization.error.slice(0, 160)}`);
+                membershipUnavailableUntilMs = now().getTime() + DEVICE_DEFAULT_MEMBERSHIP_UNAVAILABLE_COOLDOWN_MS;
+                membershipUnavailableError = authorization.error;
+                if (!membershipUnavailableLogged) {
+                    membershipUnavailableLogged = true;
+                    const prior = readDaemonState(dir)?.deviceDefaultMembershipLookupFailures ?? 0;
+                    patchDaemonState(dir, {
+                        lastErrorCode: "DEVICE_DEFAULT_MEMBERSHIP_LOOKUP_FAILED",
+                        deviceDefaultMembershipLookupFailures: prior + 1,
+                    }, now);
+                    log(dir, `device-default membership lookup unavailable${slug ? ` company=${slug}` : ""}: ${authorization.error.slice(0, 160)}`);
+                }
                 return authorization;
             }
+            membershipUnavailableUntilMs = 0;
+            membershipUnavailableLogged = false;
             if (!input.deviceDefault)
                 return authorization;
             if (authorization.status === "authorized" &&
@@ -351,12 +367,17 @@ export async function runMeshDaemon(deps = {}) {
                     lastHeldRetryAtMs = atMs;
                 },
                 log: (message) => log(dir, message),
-                authorizeDeviceDefaultTenant: async (state) => (await tenantGate.authorize({
-                    state,
-                    companySlug: state.companySlug,
-                    companyUid: state.companyUid,
-                    deviceDefault: true,
-                })).status === "authorized",
+                authorizeDeviceDefaultTenant: async (state) => {
+                    const authorization = await tenantGate.authorize({
+                        state,
+                        companySlug: state.companySlug,
+                        companyUid: state.companyUid,
+                        deviceDefault: true,
+                    });
+                    if (authorization.status === "unavailable")
+                        return "unavailable";
+                    return authorization.status === "authorized";
+                },
             });
         });
     const replayFn = deps.replayOutbox ??

package/dist/lib/mesh/live/flush.d.ts

@@ -45,8 +45,11 @@ export interface FlushDeps {
     heldMaxLines?: number;
     /** Optional logger (daemon wires this); used once per overflow trim. */
     log?: (message: string) => void;
-    /** Authoritative gate for state derived from a device default. */
-    authorizeDeviceDefaultTenant?: (state: SessionStateFile) => Promise<boolean>;
+    /** Authoritative gate for state derived from a device default.
+     *  `true` allow, `false` deny (hold NEEDS_COMPANY), `"unavailable"` skip the
+     *  network this cycle and keep the original held envelope.
+     */
+    authorizeDeviceDefaultTenant?: (state: SessionStateFile) => Promise<boolean | "unavailable">;
     /** Passed through to bind-best (catalog cache root and HQ_WORK_MESH_CACHE). */
     env?: NodeJS.ProcessEnv;
     home?: string;

package/dist/lib/mesh/live/flush.js

@@ -142,6 +142,10 @@ async function postBatchWithRetries(poster, body, deps) {
         lastBody = res.body;
         if (res.ok)
             return { ok: true, body: res.body };
+        if (res.status === 0) {
+            // Transport down (fetch failed): do not hot-loop five more attempts.
+            return { ok: false, retryable: true, status: 0, body: res.body };
+        }
         if (res.status === 429 || res.status === 503) {
             const retryAfterMs = typeof res.retryAfterMs === "number" &&
                 Number.isFinite(res.retryAfterMs) &&
@@ -279,12 +283,13 @@ export async function flushSessionEvents(inputDeps) {
             if (leaveResult.abort)
                 continue;
             const remainingItems = leaveResult.items;
+            state = bindHeldCompanyIfResolved(sessionId, remainingItems, state, deps, now);
             if (state?.companyConfidence === "device_default" &&
                 deps.authorizeDeviceDefaultTenant) {
                 const allowed = await deps.authorizeDeviceDefaultTenant(state);
                 // Authorization may restore a company the gate previously cleared.
                 state = readSessionState(sessionId, deps.workContextRoot) ?? state;
-                if (!allowed) {
+                if (allowed === false || allowed === "unavailable") {
                     const heldAt = now().toISOString();
                     summary.held += appendHeldLinesIfAbsent(remainingItems.map((item) => heldEnvelope(item.event, "NEEDS_COMPANY", item.heldAt ?? heldAt)), deps.workMeshRoot);
                     for (const item of remainingItems)
@@ -392,6 +397,8 @@ export async function flushSessionEvents(inputDeps) {
                     summary.restored += 1;
                     markDisposed(item.event);
                 }
+                if (result.status === 0)
+                    break sessionLoop;
                 if (result.status === 429 || result.status === 503)
                     break;
                 if (deps.signal?.aborted)
@@ -604,6 +611,72 @@ const SESSION_EVENT_HARNESSES = new Set([
     "agent-box",
     "agents-v2",
 ]);
+function bindHeldCompanyIfResolved(sessionId, items, state, deps, now) {
+    if (state?.companyUid)
+        return state;
+    const sample = items[0]?.event;
+    if (!sample)
+        return state;
+    const cwd = eventText(sample, "cwd");
+    const hqRoot = eventText(sample, "hqRoot");
+    const remoteOwnerSlug = cwd
+        ? deriveRemoteOwnerSlug({ cwd, hqRoot }) ?? undefined
+        : undefined;
+    const repoIdentityKey = cwd ? deriveRepoIdentityKey({ cwd }) : null;
+    const resolution = resolveCompany({
+        root: deps.workContextRoot,
+        sessionId,
+        existingState: state,
+        env: {
+            ...(deps.env ?? {}),
+            HQ_SPAWN_COMPANY: "",
+            HQ_AGENT_IDENTITY_FILE: "",
+        },
+        trusted: eventText(sample, "companySlug")
+            ? { companySlug: eventText(sample, "companySlug") }
+            : undefined,
+        cwd,
+        hqRoot,
+        remoteOwnerSlug,
+        repoIdentityKey,
+    });
+    if (resolution.status !== "resolved" || !resolution.company.uid)
+        return state;
+    if (resolution.company.source === "trusted_explicit" &&
+        !eventText(sample, "companySlug")) {
+        return state;
+    }
+    const uid = resolution.company.uid;
+    const slug = resolution.company.slug;
+    const nowIso = now().toISOString();
+    const next = updateSessionState(sessionId, deps.workContextRoot, (current) => {
+        const base = current ?? state;
+        if (!base)
+            return null;
+        if (base.companyUid && base.companyUid !== uid)
+            return null;
+        const contextStatus = base.contextStatus === "needs_company" ||
+            base.contextStatus === "unresolved" ||
+            !base.contextStatus
+            ? (base.projectId
+                ? (base.taskId ? "bound" : "needs_task")
+                : "needs_project")
+            : base.contextStatus;
+        return {
+            ...base,
+            sessionId,
+            contractVersion: base.contractVersion ?? 1,
+            companyUid: uid,
+            ...(slug ? { companySlug: slug } : {}),
+            ...(resolution.company.source === "device_default"
+                ? { companyConfidence: "device_default" }
+                : {}),
+            contextStatus,
+            updatedAt: nowIso,
+        };
+    });
+    return next ?? readSessionState(sessionId, deps.workContextRoot) ?? state;
+}
 function applySessionMetaCompanyRebind(sessionId, items, state, deps, now) {
     if (!state)
         return state;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.345.87",
+  "version": "5.345.88",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {