@indigoai-us/hq-cli 5.108.15 → 5.108.16

package/CHANGELOG.md

@@ -2,6 +2,19 @@
 
 ## [Unreleased]
 
+## [5.108.16] — 2026-09-07
+
+### Fixed
+
+- Work Mesh Live daemon no longer retries every queued work-context outbox
+  operation on every flush cycle. Outbox replay now applies per-operation
+  exponential backoff (`nextAttemptAt`, 30s base → 6h cap with jitter), a
+  per-cycle cap (200 due ops), an attempt ceiling (50 → quarantine as
+  `<code>_MAX_ATTEMPTS`), cheaper listing that skips re-reading unchanged
+  outbox files, and classifies HTTP 404/409/410/412 (and other 4xx) from
+  transport errors as non-retryable instead of retrying them forever as
+  `TRANSPORT_ERROR`.
+
 ## [5.108.15] — 2026-09-07
 
 ### Fixed

package/dist/lib/mesh/api.js

@@ -43,7 +43,12 @@ export async function meshJson(token, path, init = {}) {
     }
     if (!res.ok) {
         const err = data;
-        throw new Error(err.error || err.message || `${res.status} ${res.statusText}`);
+        // Always lead with the HTTP status: callers classify retryable vs permanent
+        // failures from the message (see createWorkSessionDeliverer). Without it a
+        // 400 whose body text matched no pattern was retried forever.
+        const detail = err.error || err.message || res.statusText || "request failed";
+        const code = typeof err.code === "string" && err.code ? `${err.code}: ` : "";
+        throw new Error(`${res.status} ${code}${detail}`);
     }
     return data;
 }

package/dist/lib/mesh/client.js

@@ -46,17 +46,37 @@ export function createWorkSessionDeliverer(opts) {
         }
         catch (err) {
             const message = err instanceof Error ? err.message : String(err);
-            // meshJson throws on !ok with error text; classify roughly.
-            if (/^401\b|^403\b|unauthorized|forbidden|auth/i.test(message)) {
+            // meshJson throws on !ok with error text; classify by leading status when present.
+            const statusMatch = /^(\d{3})\b/.exec(message);
+            if (statusMatch) {
+                const status = Number(statusMatch[1]);
+                if (status === 401 || status === 403) {
+                    return { ok: false, retryable: false, code: "AUTH_DENIED" };
+                }
+                if (status === 400 || status === 422) {
+                    return { ok: false, retryable: false, code: "VALIDATION_FAILED" };
+                }
+                if (status === 429 || (status >= 500 && status <= 599)) {
+                    return { ok: false, retryable: true, code: `HTTP_${status}` };
+                }
+                if (status === 404 ||
+                    status === 409 ||
+                    status === 410 ||
+                    status === 412 ||
+                    (status >= 400 && status < 500)) {
+                    return { ok: false, retryable: false, code: `HTTP_${status}` };
+                }
+            }
+            if (/unauthorized|forbidden|auth/i.test(message)) {
                 return { ok: false, retryable: false, code: "AUTH_DENIED" };
             }
-            if (/^400\b|^422\b|invalid|validation/i.test(message)) {
+            if (/invalid|validation/i.test(message)) {
                 return { ok: false, retryable: false, code: "VALIDATION_FAILED" };
             }
-            if (/^5\d\d\b|ECONN|ENOTFOUND|ETIMEDOUT|network|fetch failed/i.test(message)) {
+            if (/ECONN|ENOTFOUND|ETIMEDOUT|network|fetch failed/i.test(message)) {
                 return { ok: false, retryable: true, code: "NETWORK_OR_5XX" };
             }
-            // Unknown: keep queued (fail open to retry, not quarantine).
+            // No status in the message: keep queued (fail open to retry, not quarantine).
             return { ok: false, retryable: true, code: "TRANSPORT_ERROR" };
         }
     };
@@ -73,7 +93,7 @@ function mapRegisterResponse(status, body, operationId) {
     if (status >= 500 || status === 429) {
         return { ok: false, retryable: true, code: `HTTP_${status}` };
     }
-    if (status === 401 || status === 403 || status === 400 || status === 422) {
+    if (status >= 400 && status < 500) {
         return { ok: false, retryable: false, code: `HTTP_${status}` };
     }
     if (status === 0) {

package/dist/lib/mesh/live/daemon/run.d.ts

@@ -41,6 +41,7 @@ export interface DaemonRunDeps {
         delivered: number;
         queued: number;
         quarantined: number;
+        skipped?: number;
     }>;
     /** Injected board refresh (tests). */
     refreshBoards?: () => Promise<{

package/dist/lib/mesh/live/daemon/run.js

@@ -255,7 +255,14 @@ export async function runMeshDaemon(deps = {}) {
                 noteHookSessionsFromSpoolFile(workMeshHeldPath(meshRoot), lastHookEventAt, at);
                 const summary = await flushFn();
                 try {
-                    await replayFn();
+                    const replay = await replayFn();
+                    const delivered = replay.delivered ?? 0;
+                    const queued = replay.queued ?? 0;
+                    const quarantined = replay.quarantined ?? 0;
+                    const skipped = replay.skipped ?? 0;
+                    if (delivered > 0 || queued > 0 || quarantined > 0 || skipped > 0) {
+                        log(dir, `outbox replay: delivered=${delivered} queued=${queued} quarantined=${quarantined} skipped=${skipped}`);
+                    }
                 }
                 catch (err) {
                     const msg = err instanceof Error ? err.message : String(err);

package/dist/lib/work-context/outbox.d.ts

@@ -5,7 +5,15 @@
 import type { DeliveryState } from "./contract.js";
 import { WORK_CONTEXT_CONTRACT_VERSION } from "./contract.js";
 /** Fields allowed on a durable outbox operation (privacy allowlist). */
-export declare const OUTBOX_ALLOWLIST: readonly ["contractVersion", "operationId", "clientOperationId", "sessionId", "companyUid", "companySlug", "projectId", "taskId", "kind", "digest", "delivery", "createdAt", "updatedAt", "attemptCount", "lastErrorCode", "receiptId", "destinationCompanyUid", "destinationCompanySlug"];
+export declare const OUTBOX_ALLOWLIST: readonly ["contractVersion", "operationId", "clientOperationId", "sessionId", "companyUid", "companySlug", "projectId", "taskId", "kind", "digest", "delivery", "createdAt", "updatedAt", "attemptCount", "lastErrorCode", "receiptId", "destinationCompanyUid", "destinationCompanySlug", "nextAttemptAt"];
+/** Base delay for outbox retry backoff (attempt 1 → 30s). */
+export declare const OUTBOX_RETRY_BASE_MS = 30000;
+/** Cap for outbox retry backoff (6 hours). */
+export declare const OUTBOX_RETRY_MAX_MS: number;
+/** Quarantine after this many delivery attempts on retryable failures. */
+export declare const OUTBOX_MAX_ATTEMPTS = 50;
+/** Max due operations processed per replayOutbox call. */
+export declare const OUTBOX_REPLAY_MAX_OPS = 200;
 export type OutboxKind = "register" | "reconcile" | "migrate";
 export interface OutboxOperation {
     contractVersion: typeof WORK_CONTEXT_CONTRACT_VERSION;
@@ -27,6 +35,8 @@ export interface OutboxOperation {
     /** Destination company for kind=migrate (source is companyUid). */
     destinationCompanyUid?: string;
     destinationCompanySlug?: string;
+    /** ISO time when the next delivery attempt is due (absent → due immediately). */
+    nextAttemptAt?: string;
 }
 export interface OutboxEnqueueInput {
     clientOperationId: string;
@@ -40,6 +50,7 @@ export interface OutboxEnqueueInput {
     destinationCompanySlug?: string;
     now?: () => Date;
 }
+export declare function clearOutboxListCache(): void;
 export declare function stableOperationId(clientOperationId: string, sessionId: string): string;
 export declare function digestOperation(parts: {
     sessionId: string;
@@ -52,6 +63,8 @@ export declare function digestOperation(parts: {
     destinationCompanyUid?: string;
     destinationCompanySlug?: string;
 }): string;
+/** Delay before next attempt: min(BASE * 2^(attemptCount-1), MAX), then [0.5, 1.0] jitter. */
+export declare function outboxRetryDelayMs(attemptCount: number, random?: () => number): number;
 /**
  * Atomically enqueue (or idempotently return) an outbox operation.
  * Disk/permission/lock failure → NotTrackingError (no network-only send).
@@ -60,9 +73,12 @@ export declare function enqueueOutbox(input: OutboxEnqueueInput, root: string):
 export declare function readOutboxOperation(operationId: string, root: string): OutboxOperation | null;
 export declare function updateOutboxOperation(op: OutboxOperation, root: string): void;
 export declare function markOutboxAcked(operationId: string, root: string, receiptId: string, now?: () => Date): OutboxOperation | null;
-export declare function markOutboxQueued(operationId: string, root: string, errorCode: string, now?: () => Date): OutboxOperation | null;
+export declare function markOutboxQueued(operationId: string, root: string, errorCode: string, now?: () => Date, random?: () => number): OutboxOperation | null;
 export declare function markOutboxQuarantined(operationId: string, root: string, errorCode: string, now?: () => Date): OutboxOperation | null;
-export declare function listOutboxOperations(root: string): OutboxOperation[];
+export type OutboxListReadFile = (filePath: string) => string;
+export declare function listOutboxOperations(root: string, deps?: {
+    readFile?: OutboxListReadFile;
+}): OutboxOperation[];
 export interface OutboxStats {
     depth: number;
     quarantined: number;
@@ -81,14 +97,18 @@ export type DeliverFn = (op: OutboxOperation) => Promise<{
 /**
  * Replay queued (and recover lost-receipt) operations.
  * FIFO per sessionId; bounded parallelism across sessions.
+ * Skips ops whose nextAttemptAt is in the future; caps work per call.
  */
 export declare function replayOutbox(root: string, deliver: DeliverFn, opts?: {
     parallel?: number;
     now?: () => Date;
+    maxOps?: number;
+    random?: () => number;
 }): Promise<{
     delivered: number;
     queued: number;
     quarantined: number;
+    skipped: number;
 }>;
 /** Remove acked ops older than retention (optional GC; not required by AC). */
 export declare function removeAckedOutbox(root: string, olderThanMs: number, now?: number): number;

package/dist/lib/work-context/outbox.js

@@ -29,8 +29,24 @@ export const OUTBOX_ALLOWLIST = [
     "receiptId",
     "destinationCompanyUid",
     "destinationCompanySlug",
+    "nextAttemptAt",
 ];
+/** Base delay for outbox retry backoff (attempt 1 → 30s). */
+export const OUTBOX_RETRY_BASE_MS = 30_000;
+/** Cap for outbox retry backoff (6 hours). */
+export const OUTBOX_RETRY_MAX_MS = 6 * 60 * 60 * 1000;
+/** Quarantine after this many delivery attempts on retryable failures. */
+export const OUTBOX_MAX_ATTEMPTS = 50;
+/** Max due operations processed per replayOutbox call. */
+export const OUTBOX_REPLAY_MAX_OPS = 200;
 const DEFAULT_PARALLEL = 4;
+const outboxListCache = new Map();
+function invalidateOutboxListCacheEntry(filePath) {
+    outboxListCache.delete(filePath);
+}
+export function clearOutboxListCache() {
+    outboxListCache.clear();
+}
 export function stableOperationId(clientOperationId, sessionId) {
     const h = crypto
         .createHash("sha256")
@@ -85,8 +101,18 @@ function projectOutbox(op) {
     if (op.destinationCompanySlug) {
         out.destinationCompanySlug = op.destinationCompanySlug;
     }
+    if (op.nextAttemptAt)
+        out.nextAttemptAt = op.nextAttemptAt;
     return out;
 }
+/** Delay before next attempt: min(BASE * 2^(attemptCount-1), MAX), then [0.5, 1.0] jitter. */
+export function outboxRetryDelayMs(attemptCount, random = Math.random) {
+    const exp = Math.max(0, attemptCount - 1);
+    const delay = Math.min(OUTBOX_RETRY_BASE_MS * 2 ** exp, OUTBOX_RETRY_MAX_MS);
+    const unit = random();
+    const factor = 0.5 + 0.5 * Math.min(1, Math.max(0, unit));
+    return Math.floor(delay * factor);
+}
 /**
  * Atomically enqueue (or idempotently return) an outbox operation.
  * Disk/permission/lock failure → NotTrackingError (no network-only send).
@@ -182,6 +208,7 @@ export function enqueueOutbox(input, root) {
         catch {
             /* ignore */
         }
+        invalidateOutboxListCacheEntry(dest);
         return op;
     }
     catch (err) {
@@ -204,7 +231,9 @@ export function readOutboxOperation(operationId, root) {
     }
 }
 export function updateOutboxOperation(op, root) {
-    atomicWriteJson(workContextOutboxPath(op.operationId, root), projectOutbox(op));
+    const filePath = workContextOutboxPath(op.operationId, root);
+    atomicWriteJson(filePath, projectOutbox(op));
+    invalidateOutboxListCacheEntry(filePath);
 }
 export function markOutboxAcked(operationId, root, receiptId, now = () => new Date()) {
     const op = readOutboxOperation(operationId, root);
@@ -216,14 +245,17 @@ export function markOutboxAcked(operationId, root, receiptId, now = () => new Da
     updateOutboxOperation(op, root);
     return op;
 }
-export function markOutboxQueued(operationId, root, errorCode, now = () => new Date()) {
+export function markOutboxQueued(operationId, root, errorCode, now = () => new Date(), random = Math.random) {
     const op = readOutboxOperation(operationId, root);
     if (!op)
         return null;
     op.delivery = "queued";
     op.lastErrorCode = errorCode;
     op.attemptCount += 1;
-    op.updatedAt = now().toISOString();
+    const at = now();
+    op.updatedAt = at.toISOString();
+    const delayMs = outboxRetryDelayMs(op.attemptCount, random);
+    op.nextAttemptAt = new Date(at.getTime() + delayMs).toISOString();
     updateOutboxOperation(op, root);
     return op;
 }
@@ -238,17 +270,58 @@ export function markOutboxQuarantined(operationId, root, errorCode, now = () =>
     updateOutboxOperation(op, root);
     return op;
 }
-export function listOutboxOperations(root) {
+export function listOutboxOperations(root, deps = {}) {
+    const readFile = deps.readFile ?? ((filePath) => fs.readFileSync(filePath, "utf8"));
     const dir = workContextOutboxDir(root);
     if (!fs.existsSync(dir))
         return [];
     const ops = [];
+    const seen = new Set();
     for (const name of fs.readdirSync(dir)) {
         if (!name.endsWith(".json") || name.startsWith("."))
             continue;
-        const op = readOutboxOperation(name.replace(/\.json$/, ""), root);
-        if (op)
+        const operationId = name.replace(/\.json$/, "");
+        if (!isSafeWorkContextSegment(operationId))
+            continue;
+        const filePath = workContextOutboxPath(operationId, root);
+        seen.add(filePath);
+        let st;
+        try {
+            st = fs.statSync(filePath);
+        }
+        catch {
+            continue;
+        }
+        const cached = outboxListCache.get(filePath);
+        if (cached &&
+            cached.ino === st.ino &&
+            cached.mtimeMs === st.mtimeMs &&
+            cached.ctimeMs === st.ctimeMs &&
+            cached.size === st.size) {
+            ops.push(cached.op);
+            continue;
+        }
+        try {
+            const op = JSON.parse(readFile(filePath));
+            outboxListCache.set(filePath, {
+                ino: st.ino,
+                mtimeMs: st.mtimeMs,
+                ctimeMs: st.ctimeMs,
+                size: st.size,
+                op,
+            });
             ops.push(op);
+        }
+        catch {
+            /* ignore corrupt */
+        }
+    }
+    // Drop cache entries for paths no longer present under this listing pass
+    // only when they sit in this outbox dir (other roots may share the module cache).
+    for (const key of outboxListCache.keys()) {
+        if (key.startsWith(dir + path.sep) && !seen.has(key)) {
+            outboxListCache.delete(key);
+        }
     }
     return ops.sort((a, b) => {
         const byTime = a.createdAt.localeCompare(b.createdAt);
@@ -266,21 +339,68 @@ export function outboxStats(root) {
         acked: ops.filter((o) => o.delivery === "acked").length,
     };
 }
+function isOutboxDue(op, nowIso) {
+    return !op.nextAttemptAt || op.nextAttemptAt <= nowIso;
+}
+function outboxFifoCompare(a, b) {
+    const byCreated = a.createdAt.localeCompare(b.createdAt);
+    if (byCreated !== 0)
+        return byCreated;
+    return a.operationId.localeCompare(b.operationId);
+}
+/** Contiguous leading due ops; stops at the first not-yet-due op (no skip-ahead). */
+function duePrefixForSession(ops, nowIso) {
+    const prefix = [];
+    for (const op of ops) {
+        if (!isOutboxDue(op, nowIso))
+            break;
+        prefix.push(op);
+    }
+    return prefix;
+}
 /**
  * Replay queued (and recover lost-receipt) operations.
  * FIFO per sessionId; bounded parallelism across sessions.
+ * Skips ops whose nextAttemptAt is in the future; caps work per call.
  */
 export async function replayOutbox(root, deliver, opts = {}) {
     const parallel = opts.parallel ?? DEFAULT_PARALLEL;
     const now = opts.now ?? (() => new Date());
+    const random = opts.random ?? Math.random;
+    const maxOps = opts.maxOps ?? OUTBOX_REPLAY_MAX_OPS;
+    const nowIso = now().toISOString();
     const pending = listOutboxOperations(root).filter((o) => o.delivery === "queued" || (o.delivery === "acked" && !o.receiptId));
-    // Group by session, preserve FIFO within each.
-    const bySession = new Map();
+    // Group ALL pending by session first (FIFO), then take each session's due prefix.
+    const pendingBySession = new Map();
     for (const op of pending) {
-        const list = bySession.get(op.sessionId) ?? [];
+        const list = pendingBySession.get(op.sessionId) ?? [];
         list.push(op);
-        bySession.set(op.sessionId, list);
+        pendingBySession.set(op.sessionId, list);
+    }
+    for (const list of pendingBySession.values()) {
+        list.sort(outboxFifoCompare);
     }
+    const sessionPrefixes = [];
+    for (const [sessionId, list] of pendingBySession) {
+        const prefix = duePrefixForSession(list, nowIso);
+        if (prefix.length > 0) {
+            sessionPrefixes.push({ sessionId, ops: prefix });
+        }
+    }
+    // Cap by head age: oldest session heads first; take leading due ops only.
+    sessionPrefixes.sort((a, b) => outboxFifoCompare(a.ops[0], b.ops[0]));
+    const bySession = new Map();
+    let selectedCount = 0;
+    let eligibleDue = 0;
+    for (const { sessionId, ops } of sessionPrefixes) {
+        eligibleDue += ops.length;
+        if (selectedCount >= maxOps)
+            continue;
+        const take = Math.min(ops.length, maxOps - selectedCount);
+        bySession.set(sessionId, ops.slice(0, take));
+        selectedCount += take;
+    }
+    const skipped = Math.max(0, eligibleDue - selectedCount);
     let delivered = 0;
     let queued = 0;
     let quarantined = 0;
@@ -297,8 +417,14 @@ export async function replayOutbox(root, deliver, opts = {}) {
                 delivered += 1;
             }
             else if (result.retryable) {
-                markOutboxQueued(op.operationId, root, result.code, now);
-                queued += 1;
+                if (op.attemptCount + 1 >= OUTBOX_MAX_ATTEMPTS) {
+                    markOutboxQuarantined(op.operationId, root, `${result.code}_MAX_ATTEMPTS`, now);
+                    quarantined += 1;
+                }
+                else {
+                    markOutboxQueued(op.operationId, root, result.code, now, random);
+                    queued += 1;
+                }
                 // Stop this session's FIFO on transient failure.
                 break;
             }
@@ -320,7 +446,7 @@ export async function replayOutbox(root, deliver, opts = {}) {
     }
     const workers = Array.from({ length: Math.min(parallel, sessions.length) }, () => worker());
     await Promise.all(workers);
-    return { delivered, queued, quarantined };
+    return { delivered, queued, quarantined, skipped };
 }
 /** Remove acked ops older than retention (optional GC; not required by AC). */
 export function removeAckedOutbox(root, olderThanMs, now = Date.now()) {
@@ -330,8 +456,10 @@ export function removeAckedOutbox(root, olderThanMs, now = Date.now()) {
             continue;
         const age = now - Date.parse(op.updatedAt);
         if (Number.isFinite(age) && age > olderThanMs) {
+            const filePath = workContextOutboxPath(op.operationId, root);
             try {
-                fs.unlinkSync(workContextOutboxPath(op.operationId, root));
+                fs.unlinkSync(filePath);
+                invalidateOutboxListCacheEntry(filePath);
                 removed += 1;
             }
             catch {
@@ -349,6 +477,7 @@ export function quarantineCorruptOutboxFile(filePath, root) {
     const dest = path.join(workContextOutboxDir(root), `.quarantine.${base}`);
     try {
         fs.renameSync(filePath, dest);
+        invalidateOutboxListCacheEntry(filePath);
     }
     catch {
         /* ignore */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.108.15",
+  "version": "5.108.16",
   "description": "HQ by Indigo management CLI — modules and cloud sync",
   "main": "dist/index.js",
   "bin": {