@indigoai-us/hq-cli 5.345.187 → 5.345.189

package/dist/lib/bot/run.js

@@ -107,6 +107,49 @@ export const API_BACKOFF_CAP_MS = 5 * 60_000;
 /** Honor Retry-After up to this; a bogus 30-day header must not park the bot. */
 export const API_RETRY_AFTER_CAP_MS = 15 * 60_000;
 export const API_RETRY_AFTER_FLOOR_MS = 1_000;
+/**
+ * Delays between startup identity-check tries when HQ cannot be reached
+ * (five tries over about three minutes). After the last one the bot stops in
+ * a visible failed state the person can retry.
+ */
+export const STARTUP_VERIFY_RETRY_DELAYS_MS = [5_000, 15_000, 45_000, 120_000];
+/**
+ * True for failures worth retrying: transport errors (no HQ response at all),
+ * request timeouts, rate limits and server errors. Other 4xx answers are real
+ * refusals and are not retried.
+ */
+const TRANSIENT_NETWORK_CODES = new Set([
+    "ECONNREFUSED",
+    "ECONNRESET",
+    "ENOTFOUND",
+    "ETIMEDOUT",
+    "EAI_AGAIN",
+    "UND_ERR_CONNECT_TIMEOUT",
+    "UND_ERR_SOCKET",
+]);
+function isNetworkShapedError(err, depth = 0) {
+    if (depth > 4 || !err || typeof err !== "object")
+        return false;
+    const e = err;
+    if (typeof e.code === "string" && TRANSIENT_NETWORK_CODES.has(e.code))
+        return true;
+    if (e.name === "AbortError" || e.name === "TimeoutError")
+        return true;
+    if (err instanceof TypeError && e.message === "fetch failed")
+        return true;
+    return isNetworkShapedError(e.cause, depth + 1);
+}
+export function isTransientApiError(err) {
+    if (err instanceof BotApiError)
+        return err.status === 408 || err.status === 429 || err.status >= 500;
+    // Only network-shaped failures (no HQ answer at all). Auth and programming
+    // errors fail at once.
+    return isNetworkShapedError(err);
+}
+/** Plain failed-state reason after the startup retries run out. */
+export function startupUnreachableReason(name) {
+    return `could not reach HQ to start; check the network, then restart the bot (hq bot start ${name})`;
+}
 /**
  * How long to wait after a failed inbox poll or heartbeat.
  * 429/5xx (and transport errors) back off exponentially, honouring Retry-After
@@ -566,8 +609,57 @@ export async function runBot(deps) {
             setStatus({ lastError: `own ${runtime.id} login missing: run hq bot login ${config.name} --runtime ${runtime.id}` });
     }
     deps.cliUpdate?.startup(log);
+    // Stop handling is registered before the startup identity retries, which
+    // can wait about three minutes: a SIGTERM during a wait must still release
+    // the pid lock and record "stopped" with exit 0. Once the loops start,
+    // signals route to the full stop().
+    const startupAbort = new AbortController();
+    let onStopRequest = () => {
+        stopping = true;
+        startupAbort.abort();
+    };
+    const events = deps.processEvents ?? process;
+    events.once("SIGTERM", () => onStopRequest("abort"));
+    events.once("SIGINT", () => onStopRequest("abort"));
+    events.once("SIGUSR2", () => onStopRequest("drain"));
+    const stoppedDuringStartup = async () => {
+        await finish("stopped", 0, "stop requested");
+        return { stop: async () => { }, done, status: () => null, idle: () => true };
+    };
     try {
-        const record = await api.getAgent(config.agentUid);
+        let record;
+        // A network drop at login used to fail the bot with exit 0, which launchd
+        // KeepAlive{SuccessfulExit:false} never restarts, so one blip kept bots
+        // down for a week. Transient failures now retry with a growing delay
+        // before the bot settles into a visible failed state.
+        for (let attempt = 0;; attempt++) {
+            try {
+                record = await api.getAgent(config.agentUid);
+                if (attempt > 0)
+                    setStatus({ lastError: undefined });
+                break;
+            }
+            catch (err) {
+                const detail = err instanceof Error ? err.message : String(err);
+                const delayMs = STARTUP_VERIFY_RETRY_DELAYS_MS[attempt];
+                if (stopping)
+                    return await stoppedDuringStartup();
+                if (!isTransientApiError(err))
+                    throw err;
+                if (delayMs === undefined) {
+                    log("error", `could not reach HQ to verify bot identity: ${detail}`);
+                    await finish("failed", 0, startupUnreachableReason(config.name));
+                    return { stop: async () => { }, done, status: () => null, idle: () => true };
+                }
+                log("warn", `could not reach HQ to verify bot identity (try ${attempt + 1} of ${STARTUP_VERIFY_RETRY_DELAYS_MS.length + 1}): ${detail}; retrying in ${Math.round(delayMs / 1000)}s`);
+                setStatus({ lastError: "Could not reach HQ yet; retrying" });
+                await sleep(delayMs, startupAbort.signal);
+                if (stopping)
+                    return await stoppedDuringStartup();
+            }
+        }
+        if (stopping)
+            return await stoppedDuringStartup();
         // Promotion keeps the identity and owner. Ownership alone must not let
         // an old local installation start consuming the cloud bot's inbox.
         if (record.uid !== config.agentUid || record.computeMode !== "local" || record.botKind !== kind) {
@@ -650,6 +742,10 @@ export async function runBot(deps) {
     // Read before the kickoff can write its own marker: whatever is here now was
     // left by a previous run that was cut off mid-turn.
     const leftovers = readInflightTurns(dir);
+    // A signal that arrived anywhere in the startup stretch above ends the bot
+    // here, before any outbound DM.
+    if (stopping)
+        return await stoppedDuringStartup();
     // ── Intro DM (once) ────────────────────────────────────────────────────────
     // The kickoff turn is tied to the intro: it runs only in the start that sent
     // the intro, so introSentAt (persisted before the turn) guarantees it never
@@ -666,6 +762,8 @@ export async function runBot(deps) {
         let offered = false;
         if (offerTool) {
             try {
+                if (stopping)
+                    return await stoppedDuringStartup();
                 await api.sendDm({ toPersonUid: config.ownerUid, body: setupToolOfferText(offerTool, setupWelcomeOpening(config.intro)) });
                 patchBotConfig(dir, { introSentAt: now().toISOString() });
                 log("info", "intro DM sent to owner, leading into the app offer");
@@ -676,6 +774,8 @@ export async function runBot(deps) {
                 log("warn", `coding tool offer could not be sent as the welcome (${err instanceof Error ? err.message : String(err)}); sending the usual welcome`);
             }
         }
+        if (stopping)
+            return await stoppedDuringStartup();
         if (!offered)
             try {
                 const sent = await api.sendDm({ toPersonUid: config.ownerUid, body: introDmText(config.name, config.runtime, config.intro, { kind: effectiveBotKind(config), companies: effectiveBotCompanies(config) }) });
@@ -701,6 +801,8 @@ export async function runBot(deps) {
                 log("warn", `intro DM failed (will retry next start): ${err instanceof Error ? err.message : String(err)}`);
             }
     }
+    if (stopping)
+        return await stoppedDuringStartup();
     // ── Heartbeat loop ─────────────────────────────────────────────────────────
     const failures = new FailureWindow(FAILURE_WINDOW_MS, FAILURE_LIMIT);
     let modelHealth = "ok";
@@ -2639,6 +2741,10 @@ export async function runBot(deps) {
     // app know before any message arrives. A tool that cannot be asked changes
     // nothing; the first turn then reports for itself.
     await runSignInCheck("start");
+    // A signal during the sign-in probe arrives before the full stop() is wired
+    // up below: end here, before the kickoff can send a "needs a sign-in" DM.
+    if (stopping)
+        return await stoppedDuringStartup();
     // The kickoff runs alongside the heartbeat, before the first inbox poll, so
     // an owner message sent meanwhile is answered after it in the same session.
     const kickoff = kickoffAfterIntro
@@ -2837,9 +2943,15 @@ export async function runBot(deps) {
         // bot then exits 0 and stays down instead of being restarted.
         cliUpdatePending = null;
         if (stopping) {
+            // Already stopping (an earlier stop, a startup signal, or an update
+            // restart): still wait for the loops and finish, so this call ends the
+            // bot too. finish() is a no-op once the bot has stopped.
             if (mode === "abort")
                 for (const turn of currentTurns)
                     turn.abort();
+            activeInboxWait?.abort();
+            await Promise.allSettled([heartbeatLoop, inboxLoop]);
+            await finish("stopped", 0, "stop requested");
             return;
         }
         stopping = true;
@@ -2853,13 +2965,7 @@ export async function runBot(deps) {
         await Promise.allSettled([heartbeatLoop, inboxLoop]);
         await finish("stopped", 0, mode === "drain" ? "drain completed (running turns finished)" : "stop requested");
     };
-    const onSignal = () => {
-        void stop();
-    };
-    const events = deps.processEvents ?? process;
-    events.once("SIGTERM", onSignal);
-    events.once("SIGINT", onSignal);
-    events.once("SIGUSR2", () => { void stop("drain"); });
+    onStopRequest = (mode) => { void stop(mode); };
     void inboxLoop.then(() => { if (hasPromotionHold(dir))
         void stop("drain"); });
     setStatus({ canDrainForPromotion: true });

package/dist/lib/bot/runtime-failure.d.ts

@@ -39,6 +39,9 @@ export declare const RUNTIME_REPAIR_ACTIONS: Record<RuntimeRepairClass, RuntimeR
 export declare function isRuntimeRepairClass(cls: RuntimeFailureClass): cls is RuntimeRepairClass;
 /** Sort a runtime's failure text into one class. Order matters: most specific first. */
 export declare function classifyRuntimeFailure(text: string | undefined | null): RuntimeFailureClass;
+export declare function isRuntimeUsageLimit(text: string | undefined | null): boolean;
+/** When a usage limit resets, as the tool wrote it ("9pm (Asia/Dhaka)"), or undefined. */
+export declare function usageLimitResetsAt(text: string | undefined | null): string | undefined;
 export declare function isBunProcessError(text: string | undefined | null): boolean;
 /** A process that exits non-zero this soon, with no stream event, never started its turn. */
 export declare const RUNTIME_CRASH_MAX_MS = 5000;

package/dist/lib/bot/runtime-failure.js

@@ -44,7 +44,15 @@ const VERSION_TOO_OLD_MODEL = /\bthe ['"`]([^'"`\s]{1,128})['"`] model requires
  * Codex with a ChatGPT account".
  */
 const MODEL_UNSUPPORTED = /there'?s an issue with the selected model|\bnot_found_error\b[^\n]{0,200}\bmodel:\s|\bmodel_not_found\b|\b(?:unknown|invalid|unsupported) model\b|\bthe (?:requested )?model ['"`]?[^'"`\s]{1,128}['"`]? (?:does not exist|is not supported|is not available|was not found|has been deprecated)\b|\bmodel ['"`][^'"`\s]{1,128}['"`] (?:does not exist|is not supported|is not available|not found)\b|\bdo(?:es)? not have access to (?:the )?model\b|\bmodel is not supported when using codex with a chatgpt account\b/i;
-const TRANSIENT = /\brate[_ ]limit|\busage[_ ]limit|\byou've hit your usage limit\b|\btoo many requests\b|\b429\b|\boverloaded\b|\b(?:502|503|504|529)\b|\bservice unavailable\b|\bbad gateway\b|\bgateway time-?out\b|\binternal server error\b|\bECONNRESET\b|\bECONNREFUSED\b|\bETIMEDOUT\b|\bENOTFOUND\b|\bEAI_AGAIN\b|\bENETUNREACH\b|\bsocket hang up\b|\bnetwork (?:error|is unreachable)\b|\bstream (?:disconnected|error)\b|\bconnection (?:reset|refused|closed|error)\b|\btimed out\b|\bexceeded \d+ms \(HQ_BOT_TURN_TIMEOUT_MS\)/i;
+/**
+ * The account's plan ran out of usage. Claude Code: "You've hit your weekly
+ * limit · resets 9pm (Asia/Dhaka)", "You've hit your session limit". Codex:
+ * "You've hit your usage limit".
+ */
+const USAGE_LIMIT = /\byou'?ve (?:hit|reached) your (?:[\w-]+ ){0,3}limit\b|\b(?:usage|session|weekly|daily|monthly|5-hour) limit (?:reached|exceeded)\b/i;
+/** "9pm (Asia/Dhaka)" from "... · resets 9pm (Asia/Dhaka)". */
+const USAGE_LIMIT_RESET = /\bresets?(?: at| on)? ([^\n|·]{1,60}?)\s*(?:$|[\n|·])/im;
+const TRANSIENT = /\brate[_ ]limit|\busage[_ ]limit|\btoo many requests\b|\b429\b|\boverloaded\b|\b(?:502|503|504|529)\b|\bservice unavailable\b|\bbad gateway\b|\bgateway time-?out\b|\binternal server error\b|\bECONNRESET\b|\bECONNREFUSED\b|\bETIMEDOUT\b|\bENOTFOUND\b|\bEAI_AGAIN\b|\bENETUNREACH\b|\bsocket hang up\b|\bnetwork (?:error|is unreachable)\b|\bstream (?:disconnected|error)\b|\bconnection (?:reset|refused|closed|error)\b|\btimed out\b|\bexceeded \d+ms \(HQ_BOT_TURN_TIMEOUT_MS\)/i;
 /** Sort a runtime's failure text into one class. Order matters: most specific first. */
 export function classifyRuntimeFailure(text) {
     const t = (text ?? "").trim();
@@ -58,10 +66,20 @@ export function classifyRuntimeFailure(text) {
         return "signed-out";
     if (MODEL_UNSUPPORTED.test(t))
         return "model-unsupported";
-    if (TRANSIENT.test(t))
+    if (TRANSIENT.test(t) || USAGE_LIMIT.test(t))
         return "transient";
     return "unknown";
 }
+export function isRuntimeUsageLimit(text) {
+    return USAGE_LIMIT.test((text ?? "").trim());
+}
+/** When a usage limit resets, as the tool wrote it ("9pm (Asia/Dhaka)"), or undefined. */
+export function usageLimitResetsAt(text) {
+    const t = text ?? "";
+    if (!USAGE_LIMIT.test(t))
+        return undefined;
+    return USAGE_LIMIT_RESET.exec(t)?.[1]?.trim() || undefined;
+}
 /**
  * Bun's process-level error line (Claude Code is a Bun-compiled binary):
  * "error: An unknown error occurred (Unexpected)". The excerpt joins stderr