@indigoai-us/hq-cli 5.345.76 → 5.345.78

package/dist/lib/bot/run.js

@@ -73,7 +73,8 @@ import { isAgentSender, isDmItem, isMetaNonReply, resolveReplyTarget } from "./r
 import { buildJobTurnPrompt, isCatchUpRun, isJobItem, jobReportHeader, jobRunOutput, machineTimeZone, planJobRuns, stripJobHeader, SUPERSEDED_DISPOSITION } from "./job-run.js";
 import { botJobQuietOptionEnabled } from "./job-quiet-flag.js";
 import { clearPendingJobCompletion, isCompletionDue, isCompletionExpired, isCompletionForOtherAgent, readPendingJobCompletions, recordPendingJobCompletion } from "./job-completions.js";
-import { isRuntimeSignInFailure, SIGN_IN_RETRY_MS, signInNeededText } from "./runtime-sign-in.js";
+import { SIGN_IN_RETRY_MS, signInNeededText } from "./runtime-sign-in.js";
+import { classifyRuntimeError, classifyRuntimeFailure, runtimeDidNotFinishText, runtimeFailureReason, runtimeFailureText, runtimeRepairNotice, } from "./runtime-failure.js";
 import { HQ_BOT_AGENT_UID_ENV } from "../../commands/onboard-bot-membership.js";
 import { captureHeartbeatIdentityMismatch, identityMismatchMessage, isHeartbeatIdentityMismatch, } from "../heartbeat-identity.js";
 import { clearBotSession, readBotSession, writeBotSession } from "./session.js";
@@ -181,13 +182,19 @@ export const SETUP_WORKER_ID = SETUP_BOT_WORKER_ID;
 /** The one message a person gets when a turn on their message did not finish. */
 export function didNotFinishText(reason, progressPosted) {
     const r = reason.trim().replace(/[.\s]+$/, "");
-    return `I didn't finish that — ${r}.${progressPosted ? " The messages above show how far I got." : ""}`;
+    const sentence = r ? `${r[0].toUpperCase()}${r.slice(1)}` : r;
+    return `I didn't finish that. ${sentence}.${progressPosted ? " The messages above show how far I got." : ""}`;
 }
 export { containsNoReplySentinel };
 /** Grok emits is_error with this message when the model produces only reasoning tokens. */
 export function isReasoningOnlyError(excerpt) {
     return excerpt.includes("reasoning_only");
 }
+/** The one note a person gets when the bot pauses after repeated failures. */
+export function holdNoticeText(holdMs, messagePending) {
+    const minutes = Math.max(1, Math.round(holdMs / 60_000));
+    return `I'm pausing for ${minutes} minute${minutes === 1 ? "" : "s"} after a few failed tries, then I'll ${messagePending ? "pick your message up" : "pick up from your next message"}.`;
+}
 /** Owner stop, `hq bot stop` or `hq bot restart`: all arrive as the same signal. */
 export const STOPPED_BY_OWNER_REASON = "I was stopped before I was done";
 export const RESTARTED_REASON = "I was restarted while I was working on it";
@@ -228,6 +235,9 @@ class FailureWindow {
     exceeded(nowMs) {
         return this.count(nowMs) >= this.limit;
     }
+    clear() {
+        this.stamps.length = 0;
+    }
 }
 function defaultSleep(ms, signal) {
     return new Promise((resolve) => {
@@ -493,7 +503,9 @@ export async function runBot(deps) {
         return { stop: async () => { }, done, status: () => null, idle: () => true };
     }
     state = "running";
-    setStatus({ state: "running", lastActivityAt: now().toISOString() });
+    // A fresh process has a fresh failure window; whatever the last one recorded
+    // (a bot that exited after repeated failures on an older hq-cli) is over.
+    setStatus({ state: "running", lastActivityAt: now().toISOString(), recentFailures: 0, modelHold: null });
     log("info", "running");
     // Read before the kickoff can write its own marker: whatever is here now was
     // left by a previous run that was cut off mid-turn.
@@ -539,12 +551,49 @@ export async function runBot(deps) {
     // ── Heartbeat loop ─────────────────────────────────────────────────────────
     const failures = new FailureWindow(FAILURE_WINDOW_MS, FAILURE_LIMIT);
     let modelHealth = "ok";
+    /**
+     * FAILURE_LIMIT model failures inside the window: hold model turns for a
+     * cooldown instead of exiting. The bot used to exit 0 here, which every
+     * supervisor (launchd KeepAlive{SuccessfulExit:false}, systemd on-failure,
+     * the hq daemon units) treats as "stay down", so a bot whose coding tool was
+     * briefly broken stayed dead until someone ran `hq bot start`, and never saw
+     * the updated hq-cli that fixed it. Holding keeps the process (and its
+     * hq-cli update watcher) alive; after the cooldown the window is cleared and
+     * the next inbound message runs a fresh turn.
+     */
+    const modelHoldMs = Math.max(0, deps.modelHoldMs ?? FAILURE_WINDOW_MS);
+    let modelHoldUntil = 0;
+    const holdAfterFailures = async (excerpt, tell, messagePending = false) => {
+        const until = now().getTime() + modelHoldMs;
+        modelHoldUntil = until;
+        failures.clear();
+        modelHealth = "failed";
+        const reason = `${FAILURE_LIMIT} model failures in 10 minutes; last: ${excerpt}`;
+        setStatus({ modelHold: { until: new Date(until).toISOString(), reason: reason.slice(0, 400) }, lastError: excerpt.slice(0, 400) });
+        log("error", `${reason}. Holding model turns for ${Math.round(modelHoldMs / 60_000)} min, then trying again on the next message`);
+        // One plain note to the person, never silence: they would otherwise see
+        // nothing while their next messages wait out the hold.
+        if (tell) {
+            try {
+                await tell(holdNoticeText(modelHoldMs, messagePending));
+            }
+            catch (err) {
+                log("warn", `could not say that turns are on hold: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    };
+    const liftModelHold = () => {
+        modelHoldUntil = 0;
+        setStatus({ modelHold: null });
+        log("info", "model hold over; the next message runs a fresh turn");
+    };
     // ── Coding tool sign-in ────────────────────────────────────────────────────
     const signInRetryMs = Math.max(0, deps.signInRetryMs ?? SIGN_IN_RETRY_MS);
     let signInIssue = null;
     let nextSignInRetryAt = 0;
     /** Conversations already told this time that the bot needs a sign-in. */
     const toldAboutSignIn = new Set();
+    const isSignInFailure = (excerpt) => classifyRuntimeFailure(excerpt) === "signed-out";
     const markSignInBroken = (excerpt) => {
         nextSignInRetryAt = now().getTime() + signInRetryMs;
         modelHealth = "degraded";
@@ -563,11 +612,18 @@ export async function runBot(deps) {
         log("info", `${runtime.id} sign-in works again; resuming`);
     };
     /** Say once per conversation that the bot cannot work until its tool signs in. */
-    const tellSignInNeeded = async (conversation, post, kickoff = false) => {
+    const tellSignInNeeded = async (conversation, post, kickoff = false, retryOf) => {
         if (toldAboutSignIn.has(conversation))
             return;
         try {
-            await post(signInNeededText(config.runtime, kickoff));
+            await post(runtimeRepairNotice({
+                cls: "signed-out",
+                runtime: config.runtime,
+                botName: config.name,
+                context: kickoff ? "kickoff" : "message",
+                text: signInNeededText(config.runtime, kickoff),
+                ...(retryOf ? { retryOf } : {}),
+            }));
             toldAboutSignIn.add(conversation);
         }
         catch (err) {
@@ -774,6 +830,16 @@ export async function runBot(deps) {
         }
         modelHealth = "ok";
         markSignInWorks();
+        // A good turn: the last recorded failure is no longer the bot's state.
+        setStatus({ recentFailures: failures.count(now().getTime()), lastError: undefined });
+        if (turn.modelFallback) {
+            const { from, to } = turn.modelFallback;
+            log("warn", `${runtime.id} on this computer is too old for model ${from || "(its configured default)"}; this turn ran on ${to} instead. Update ${runtime.id} to use ${from || "that model"} again`);
+            setStatus({ runtimeModelFallback: { from, to, at: now().toISOString() } });
+        }
+        else if (readStatus()?.runtimeModelFallback) {
+            setStatus({ runtimeModelFallback: null });
+        }
         log("info", `model turn done in ${Math.round((now().getTime() - tTurnStart) / 1000)}s (${Math.round((tTurnStart - tReceived) / 1000)}s before it)`);
         if (turn.sessionId) {
             writeBotSession(dir, {
@@ -792,7 +858,7 @@ export async function runBot(deps) {
         const count = failures.record(now().getTime());
         modelHealth = count >= FAILURE_LIMIT ? "failed" : "degraded";
         setStatus({ restarts: (readStatus()?.restarts ?? 0) + 1, recentFailures: count, lastError: excerpt.slice(0, 400) });
-        log("error", `model turn failed (${count}/${FAILURE_LIMIT} in window): ${excerpt}`);
+        log("error", `model turn failed (${count}/${FAILURE_LIMIT} in window, ${classifyRuntimeError(err)}): ${excerpt}`);
         // A resumed session that no longer exists must not poison every turn.
         const resumed = Boolean(err?.resumedSession);
         if (resumed && /session|resume|not found/i.test(excerpt))
@@ -892,7 +958,30 @@ export async function runBot(deps) {
             currentTurns.delete(controller);
         }
     };
-    const failureReason = (outcome) => outcome.aborted ? STOPPED_BY_OWNER_REASON : `${runtime.id} stopped with an error: ${outcome.excerpt.slice(0, 160)}`;
+    const failureClass = (outcome) => classifyRuntimeError(outcome.error);
+    /** Plain reason for a failed turn: never the runtime's raw text (that goes to the log). */
+    const failureReason = (outcome) => outcome.aborted ? STOPPED_BY_OWNER_REASON : runtimeFailureReason(failureClass(outcome), runtime.id);
+    /**
+     * Post a failure notice: the plain body, plus its repair block in a DM.
+     * A server that refuses the block (400) still gets the plain body.
+     */
+    const postRepair = async (target, notice, rootEventId) => {
+        if (target.kind === "dm" && notice.richContent) {
+            const base = { toPersonUid: target.peerUid, body: notice.text, audience: "human", ...(rootEventId ? { rootEventId } : {}) };
+            try {
+                await api.sendDm({ ...base, richContent: notice.richContent });
+                return;
+            }
+            catch (err) {
+                if (!(err instanceof BotApiError && err.status === 400))
+                    throw err;
+                log("warn", `repair card refused (${err.code ?? err.status}); sent as plain text`);
+            }
+            await api.sendDm(base);
+            return;
+        }
+        await sendToTarget(target, notice.text, rootEventId, "human");
+    };
     const postOne = (target, body, rootEventId, audience) => {
         const root = rootEventId ? { rootEventId } : {};
         return target.kind === "dm"
@@ -1061,7 +1150,8 @@ export async function runBot(deps) {
                 ? {
                     status: "failed",
                     disposition: failure.reason.slice(0, 256),
-                    ...(failure.excerpt ? { output: jobRunOutput(failure.excerpt) } : {}),
+                    // The raw runtime text stays in the bot log and status.json only.
+                    output: jobRunOutput(failure.output ?? `${failure.reason[0]?.toUpperCase() ?? ""}${failure.reason.slice(1)}.`),
                 }
                 : { status: "failed", disposition: "the run was cut off when the bot restarted" };
         await completeJobRun(marker.jobId, marker.messageId, completion);
@@ -1232,12 +1322,12 @@ export async function runBot(deps) {
             log("info", `kickoff reply sent (${outcome.reply.length} chars)`);
             return;
         }
-        if (!outcome.aborted && poster.postedCount === 0 && isRuntimeSignInFailure(outcome.excerpt)) {
+        if (!outcome.aborted && poster.postedCount === 0 && isSignInFailure(outcome.excerpt)) {
             // Nothing ran: hold the kickoff until the tool can sign in, then run it.
             clearInflight(dir, KICKOFF_MESSAGE_ID);
             patchBotConfig(dir, { kickoffPendingSignIn: true });
             markSignInBroken(outcome.excerpt);
-            await tellSignInNeeded("dm", sendToPerson, true);
+            await tellSignInNeeded("dm", (notice) => postRepair(target, notice), true);
             return;
         }
         if (readBotConfig(dir)?.kickoffPendingSignIn)
@@ -1245,11 +1335,20 @@ export async function runBot(deps) {
         let count = 0;
         if (!outcome.aborted)
             count = recordTurnFailure(outcome.error, "dm").count;
-        const body = outcome.started && (outcome.aborted || poster.postedCount > 0)
-            ? `${didNotFinishText(failureReason(outcome), poster.postedCount > 0)} Send me any message and I'll pick up from there.`
-            : `Sorry — I couldn't get started on my own (${runtime.id} failed: ${outcome.excerpt.slice(0, 160)}). Send me any message and I'll pick up from there, or check that ${runtime.id} is signed in on this computer.`;
+        // The person sees one plain sentence and one next step; the raw runtime
+        // text is in the log (recordTurnFailure) and status.json only.
+        const cls = failureClass(outcome);
+        const notice = outcome.aborted
+            ? { cls, text: `${didNotFinishText(failureReason(outcome), poster.postedCount > 0)} Send me any message and I'll pick up from there.` }
+            : runtimeRepairNotice({
+                cls,
+                runtime: runtime.id,
+                botName: config.name,
+                context: "kickoff",
+                ...(outcome.started && poster.postedCount > 0 ? { text: runtimeDidNotFinishText(cls, runtime.id, true) } : {}),
+            });
         try {
-            await sendToPerson(body);
+            await postRepair(target, notice);
             clearInflight(dir, KICKOFF_MESSAGE_ID);
         }
         catch {
@@ -1257,7 +1356,7 @@ export async function runBot(deps) {
         }
         log("error", `kickoff turn ${outcome.aborted ? "stopped by the owner" : `failed (${count}/${FAILURE_LIMIT} in window)`}; not retried`);
         if (!outcome.aborted && failures.exceeded(now().getTime())) {
-            await finish("failed", 0, `${FAILURE_LIMIT} model failures in 10 minutes; last: ${outcome.excerpt}`);
+            await holdAfterFailures(outcome.excerpt, sendToPerson);
         }
     };
     /**
@@ -1392,13 +1491,13 @@ export async function runBot(deps) {
             log("info", `job ${jobId} run ${id} done (${outcome.reply.length} chars${catchUp ? ", catch-up" : ""})`);
             return;
         }
-        if (!outcome.aborted && poster.postedCount === 0 && isRuntimeSignInFailure(outcome.excerpt)) {
+        if (!outcome.aborted && poster.postedCount === 0 && isSignInFailure(outcome.excerpt)) {
             // Nothing ran: hold the run until the coding tool can sign in again. It
             // is neither acked nor reported succeeded — the work is still owed.
             clearInflight(dir, id);
             attempts.delete(id);
             markSignInBroken(outcome.excerpt);
-            await tellSignInNeeded(`job:${jobId}`, (body) => sendToTarget(markerTarget, body, undefined, "human"));
+            await tellSignInNeeded(`job:${jobId}`, (notice) => postRepair(markerTarget, notice));
             log("warn", `kept scheduled run ${id} of job ${jobId} until ${runtime.id} can sign in again`);
             return; // un-acked, un-reported: run once the sign-in works
         }
@@ -1407,12 +1506,13 @@ export async function runBot(deps) {
             clearInflight(dir, id);
             const { excerpt } = recordTurnFailure(outcome.error, target.sessionScope);
             if (failures.exceeded(now().getTime())) {
-                await finish("failed", 0, `${FAILURE_LIMIT} model failures in 10 minutes; last: ${excerpt}`);
+                await holdAfterFailures(excerpt, sendToPerson, true);
                 return;
             }
             if (attempt >= MAX_ATTEMPTS_PER_MESSAGE) {
-                await sendToPerson(`I couldn't run this one — ${runtime.id} kept failing: ${excerpt.slice(0, 160)}.`).catch(() => undefined);
-                await completeJobRun(jobId, id, { status: "failed", output: jobRunOutput(excerpt), disposition: `${runtime.id} could not be started` });
+                const plain = runtimeFailureText(classifyRuntimeError(outcome.error), runtime.id);
+                await sendToPerson(`I couldn't run this one. ${plain}`).catch(() => undefined);
+                await completeJobRun(jobId, id, { status: "failed", output: jobRunOutput(plain), disposition: `${runtime.id} could not be started` });
                 markReported("failed");
                 markProcessed(dir, id, now().toISOString());
                 await api.ackInbox(config.agentUid, id);
@@ -1428,16 +1528,21 @@ export async function runBot(deps) {
         const reason = failureReason(outcome);
         // Saved BEFORE the post: if "did not finish" cannot be posted, recovery
         // reports and says this real reason, not "the bot was restarted".
-        patchInflight(dir, id, { jobFailure: { reason, ...(outcome.excerpt ? { excerpt: outcome.excerpt } : {}) } });
+        // What the cloud stores as the run's output: plain words, never the raw
+        // runtime text (that is in the bot log and status.json).
+        const jobOutput = outcome.aborted ? `${STOPPED_BY_OWNER_REASON}.` : runtimeFailureText(failureClass(outcome), runtime.id);
+        patchInflight(dir, id, { jobFailure: { reason, output: jobOutput } });
         try {
-            await sendToPerson(didNotFinishText(reason, poster.postedCount > 0));
+            await sendToPerson(outcome.aborted
+                ? didNotFinishText(reason, poster.postedCount > 0)
+                : runtimeDidNotFinishText(failureClass(outcome), runtime.id, poster.postedCount > 0));
         }
         catch (err) {
             // Leave it un-acked with the marker: the next poll reports it.
             log("warn", `could not post "did not finish" for run ${id} of job ${jobId}: ${err instanceof Error ? err.message : String(err)}`);
             return;
         }
-        await completeJobRun(jobId, id, { status: "failed", output: jobRunOutput(outcome.excerpt), disposition: reason.slice(0, 256) });
+        await completeJobRun(jobId, id, { status: "failed", output: jobRunOutput(jobOutput), disposition: reason.slice(0, 256) });
         markReported("failed");
         markProcessed(dir, id, now().toISOString());
         await api.ackInbox(config.agentUid, id);
@@ -1445,7 +1550,7 @@ export async function runBot(deps) {
         attempts.delete(id);
         log("error", `scheduled run ${id} of job ${jobId} ${outcome.aborted ? "stopped by the owner" : "failed"} after it started; not re-run`);
         if (!outcome.aborted && failures.exceeded(now().getTime())) {
-            await finish("failed", 0, `${FAILURE_LIMIT} model failures in 10 minutes; last: ${outcome.excerpt}`);
+            await holdAfterFailures(outcome.excerpt, sendToPerson);
         }
     };
     /**
@@ -1657,7 +1762,7 @@ export async function runBot(deps) {
         });
         // The model saw the note once its turn ran. A turn that never started, or
         // one held for a coding-tool sign-in, runs again and keeps the note for it.
-        if (setupNote && !toolOffer && (outcome.ok || (outcome.started && !isRuntimeSignInFailure(outcome.excerpt)))) {
+        if (setupNote && !toolOffer && (outcome.ok || (outcome.started && !isSignInFailure(outcome.excerpt)))) {
             try {
                 patchSetupIntroState(dir, { notePending: false });
             }
@@ -1666,7 +1771,7 @@ export async function runBot(deps) {
             }
         }
         // Same rule for the offer: answered once the model saw the answer.
-        if (toolOffer && toolAnswer && (outcome.ok || (outcome.started && !isRuntimeSignInFailure(outcome.excerpt)))) {
+        if (toolOffer && toolAnswer && (outcome.ok || (outcome.started && !isSignInFailure(outcome.excerpt)))) {
             try {
                 patchSetupIntroState(dir, { kickoffDeferred: false, toolOfferAnswer: toolAnswer });
                 log("info", `coding tool offer answered (${toolAnswer === "kept_here" ? "kept setup here" : "wrote a message"}); setup runs here`);
@@ -1710,14 +1815,14 @@ export async function runBot(deps) {
             return;
         }
         await clearThinking();
-        if (!outcome.aborted && poster.postedCount === 0 && isRuntimeSignInFailure(outcome.excerpt)) {
+        if (!outcome.aborted && poster.postedCount === 0 && isSignInFailure(outcome.excerpt)) {
             // The tool refused before doing anything: keep the message for when it
             // can sign in again, and say so once in this conversation.
             clearInflight(dir, id);
             attempts.delete(id);
             markSignInBroken(outcome.excerpt);
             const conversation = markerTarget.kind === "dm" ? sessionScope : `room:${markerTarget.channelId}:${item.rootEventId ?? ""}`;
-            await tellSignInNeeded(conversation, sendToPerson);
+            await tellSignInNeeded(conversation, (notice) => postRepair(markerTarget, notice, item.rootEventId), false, item.eventId ?? id);
             log("warn", `kept ${id} until ${runtime.id} can sign in again`);
             return; // un-acked: answered once the sign-in works
         }
@@ -1727,12 +1832,18 @@ export async function runBot(deps) {
             clearInflight(dir, id);
             const { excerpt } = recordTurnFailure(outcome.error, sessionScope);
             if (failures.exceeded(now().getTime())) {
-                await finish("failed", 0, `${FAILURE_LIMIT} model failures in 10 minutes; last: ${excerpt}`);
+                await holdAfterFailures(excerpt, sendToPerson, true);
                 return;
             }
             if (attempt >= MAX_ATTEMPTS_PER_MESSAGE) {
                 try {
-                    await sendToPerson(`Sorry — I couldn't answer that one (${runtime.id} kept failing: ${excerpt.slice(0, 160)}). Try again in a bit, or check that ${runtime.id} is signed in on this computer.`);
+                    await postRepair(markerTarget, runtimeRepairNotice({
+                        cls: classifyRuntimeError(outcome.error),
+                        runtime: runtime.id,
+                        botName: config.name,
+                        context: "message",
+                        retryOf: item.eventId ?? id,
+                    }), item.rootEventId);
                 }
                 catch {
                     /* best-effort */
@@ -1762,8 +1873,9 @@ export async function runBot(deps) {
             attempts.delete(id);
             await clearThinking();
             log("info", `unmentioned room turn ${id} failed with no post (${where}); acked silently`);
+            // Nobody asked the bot here, so the hold note goes to the owner's DM, not the room.
             if (failures.exceeded(now().getTime())) {
-                await finish("failed", 0, `${FAILURE_LIMIT} model failures in 10 minutes; last: ${outcome.excerpt}`);
+                await holdAfterFailures(outcome.excerpt, (body) => sendToTarget({ kind: "dm", peerUid: config.ownerUid }, body, undefined, "human"));
             }
             return;
         }
@@ -1771,8 +1883,21 @@ export async function runBot(deps) {
             // DM or @-mentioned room reasoning_only: the person asked directly, so
             // still say we did not finish — but with a plain reason, never the raw
             // internal error string.
-            const reason = reasoningOnly ? "the model returned no answer" : failureReason(outcome);
-            await sendToPerson(didNotFinishText(reason, poster.postedCount > 0));
+            if (reasoningOnly || outcome.aborted) {
+                const reason = reasoningOnly ? "the model returned no answer" : failureReason(outcome);
+                await sendToPerson(didNotFinishText(reason, poster.postedCount > 0));
+            }
+            else {
+                const cls = failureClass(outcome);
+                await postRepair(markerTarget, runtimeRepairNotice({
+                    cls,
+                    runtime: runtime.id,
+                    botName: config.name,
+                    context: "message",
+                    text: runtimeDidNotFinishText(cls, runtime.id, poster.postedCount > 0),
+                    retryOf: item.eventId ?? id,
+                }), item.rootEventId);
+            }
         }
         catch (err) {
             // Leave it un-acked with the marker: the next poll says it did not finish.
@@ -1785,7 +1910,7 @@ export async function runBot(deps) {
         attempts.delete(id);
         log("error", `turn on ${id} ${outcome.aborted ? "stopped by the owner" : "failed"} after it started; told the owner, not re-run`);
         if (!outcome.aborted && failures.exceeded(now().getTime())) {
-            await finish("failed", 0, `${FAILURE_LIMIT} model failures in 10 minutes; last: ${outcome.excerpt}`);
+            await holdAfterFailures(outcome.excerpt, sendToPerson);
         }
     };
     // The kickoff runs alongside the heartbeat, before the first inbox poll, so
@@ -1874,6 +1999,18 @@ export async function runBot(deps) {
             // Queued items were never started or acked; the next process gets them.
             if (pollForCliUpdate())
                 break;
+            // Completions the cloud has not accepted yet: re-send the report (never
+            // the work), alongside the poll rather than in front of it. Also during
+            // a hold: a report owed to the cloud has nothing to do with the model.
+            kickCompletionRetries();
+            // Holding after repeated model failures: no inbox pulls until the
+            // cooldown is over (the update watcher above keeps running meanwhile).
+            if (modelHoldUntil > now().getTime()) {
+                await sleep(Math.max(1, Math.min(inboxIntervalMs, modelHoldUntil - now().getTime())));
+                continue;
+            }
+            if (modelHoldUntil)
+                liftModelHold();
             // A kickoff held for a sign-in retries on the same timer as messages.
             if (busyLanes.size === 0 &&
                 (!signInIssue || now().getTime() >= nextSignInRetryAt) &&
@@ -1884,9 +2021,6 @@ export async function runBot(deps) {
                     log("error", `kickoff retry crashed: ${err instanceof Error ? err.message : String(err)}`);
                 });
             }
-            // Completions the cloud has not accepted yet: re-send the report (never
-            // the work), alongside the poll rather than in front of it.
-            kickCompletionRetries();
             const useInboxBackoff = await inboxPollBackoffEnabled().catch((error) => {
                 log("warn", `inbox poll backoff flag lookup failed (${safeErrorClass(error)}); using default-off`);
                 return false;

package/dist/lib/bot/runtime/codex.d.ts

@@ -13,7 +13,51 @@
  * Verified against Codex CLI on 2026-09-10 (event names; the model itself
  * cannot run under this machine's account).
  */
-import { type BotRuntime, type RuntimeStreamEvent, type RuntimeTurnInput } from "./types.js";
+import { type BotRuntime, type RuntimeStreamEvent, type RuntimeTurnInput, type SpawnResult } from "./types.js";
+/**
+ * The model a turn falls back to when the installed Codex is too old for the
+ * one it was asked to use (usually the person's own `model` in
+ * ~/.codex/config.toml, set by a newer Codex app) and the first choice from
+ * Codex's bundled catalog is refused as well. Every Codex CLI since mid 2026
+ * ships it.
+ */
+export declare const CODEX_PINNED_FALLBACK_MODEL = "gpt-5.5";
+/**
+ * The generation number in a Codex model slug: 6 for "gpt-6-astra", 6.1 for
+ * "gpt-6.1-sol", 5.6 for "gpt-5.6-terra". Undefined when the slug has none
+ * ("codex-auto-review").
+ */
+export declare function codexModelGeneration(slug: string): number | undefined;
+/** [0, 147, 0] from "codex-cli 0.147.0". */
+export declare function parseCodexVersion(text: string | undefined): number[] | undefined;
+/**
+ * The model Codex itself would offer first that this Codex is most likely to
+ * run, from the JSON of `codex debug models --bundled` (`{ "models": [...] }`
+ * or a bare array). Lowest `priority` wins among the models that are listed
+ * in the picker, not `exclude`d, not newer than `installed` (when the catalog
+ * carries `minimal_client_version`; the real one does not), and of an OLDER
+ * generation than `refused` (gpt-6.1-sol is no better than gpt-6-astra for a
+ * Codex that refuses gpt-6; gpt-5.6-sol is). The bundled catalog of an old
+ * Codex still lists models it cannot run, because the "requires a newer
+ * version" check is the server's, so the generation rule is what matters.
+ */
+export declare function pickCodexFallbackModel(catalogJson: string, opts?: {
+    exclude?: string[];
+    installed?: number[];
+    refused?: string;
+}): string | undefined;
+/**
+ * The models to try, best first, after this Codex refused `failing` as too
+ * new: the best older-generation model from the bundled catalog, then the
+ * pinned fallback. At most MAX_MODEL_FALLBACK_RETRIES, never `failing` itself.
+ * `codex debug models --bundled` reads only the catalog inside the binary: no
+ * network, no sign-in, no cache write.
+ */
+export declare function codexFallbackModels(args: {
+    reason: string;
+    requestedModel: string | undefined;
+    run: (args: string[]) => Promise<SpawnResult>;
+}): Promise<string[]>;
 /** Progress events in one codex JSONL line: finished agent messages and tool starts. */
 export declare function codexStreamEvents(line: string): RuntimeStreamEvent[];
 export declare function codexPrompt(input: RuntimeTurnInput): string;

package/dist/lib/bot/runtime/codex.js

@@ -16,8 +16,137 @@
 import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
-import { parseJsonLine, RuntimeError } from "./types.js";
+import { MAX_MODEL_FALLBACK_RETRIES, parseJsonLine, RuntimeError } from "./types.js";
 import { resolveRuntimeBinary } from "./windows.js";
+import { versionTooOldModel } from "../runtime-failure.js";
+/**
+ * The model a turn falls back to when the installed Codex is too old for the
+ * one it was asked to use (usually the person's own `model` in
+ * ~/.codex/config.toml, set by a newer Codex app) and the first choice from
+ * Codex's bundled catalog is refused as well. Every Codex CLI since mid 2026
+ * ships it.
+ */
+export const CODEX_PINNED_FALLBACK_MODEL = "gpt-5.5";
+/**
+ * The generation number in a Codex model slug: 6 for "gpt-6-astra", 6.1 for
+ * "gpt-6.1-sol", 5.6 for "gpt-5.6-terra". Undefined when the slug has none
+ * ("codex-auto-review").
+ */
+export function codexModelGeneration(slug) {
+    const m = /^gpt-(\d+(?:\.\d+)?)(?:$|[-.])/i.exec(slug.trim());
+    return m ? Number(m[1]) : undefined;
+}
+/** [0, 147, 0] from "codex-cli 0.147.0". */
+export function parseCodexVersion(text) {
+    const m = /(\d+)\.(\d+)\.(\d+)/.exec(text ?? "");
+    return m ? [Number(m[1]), Number(m[2]), Number(m[3])] : undefined;
+}
+function versionOf(value) {
+    if (typeof value === "string")
+        return parseCodexVersion(value);
+    if (Array.isArray(value) && value.length > 0 && value.every((n) => typeof n === "number" && Number.isFinite(n))) {
+        return value;
+    }
+    return undefined;
+}
+function compareVersions(a, b) {
+    for (let i = 0; i < Math.max(a.length, b.length); i += 1) {
+        const d = (a[i] ?? 0) - (b[i] ?? 0);
+        if (d !== 0)
+            return d;
+    }
+    return 0;
+}
+/**
+ * The model Codex itself would offer first that this Codex is most likely to
+ * run, from the JSON of `codex debug models --bundled` (`{ "models": [...] }`
+ * or a bare array). Lowest `priority` wins among the models that are listed
+ * in the picker, not `exclude`d, not newer than `installed` (when the catalog
+ * carries `minimal_client_version`; the real one does not), and of an OLDER
+ * generation than `refused` (gpt-6.1-sol is no better than gpt-6-astra for a
+ * Codex that refuses gpt-6; gpt-5.6-sol is). The bundled catalog of an old
+ * Codex still lists models it cannot run, because the "requires a newer
+ * version" check is the server's, so the generation rule is what matters.
+ */
+export function pickCodexFallbackModel(catalogJson, opts = {}) {
+    let root;
+    try {
+        root = JSON.parse(catalogJson);
+    }
+    catch {
+        return undefined;
+    }
+    const list = Array.isArray(root)
+        ? root
+        : root && typeof root === "object" && Array.isArray(root.models)
+            ? root.models
+            : [];
+    const exclude = new Set((opts.exclude ?? []).filter(Boolean));
+    const refusedGen = opts.refused ? codexModelGeneration(opts.refused) : undefined;
+    const candidates = [];
+    list.forEach((entry, index) => {
+        if (!entry || typeof entry !== "object")
+            return;
+        const m = entry;
+        const slug = typeof m.slug === "string" ? m.slug.trim() : "";
+        if (!slug || exclude.has(slug) || !/^[\w.:/-]{1,128}$/.test(slug))
+            return;
+        if (typeof m.visibility === "string" && m.visibility !== "list")
+            return;
+        if (refusedGen !== undefined) {
+            const gen = codexModelGeneration(slug);
+            if (gen === undefined || gen >= refusedGen)
+                return;
+        }
+        const min = versionOf(m.minimal_client_version);
+        if (min && opts.installed && compareVersions(min, opts.installed) > 0)
+            return;
+        const priority = typeof m.priority === "number" && Number.isFinite(m.priority) ? m.priority : Number.POSITIVE_INFINITY;
+        candidates.push({ slug, priority, index });
+    });
+    candidates.sort((a, b) => a.priority - b.priority || a.index - b.index);
+    return candidates[0]?.slug;
+}
+/**
+ * The models to try, best first, after this Codex refused `failing` as too
+ * new: the best older-generation model from the bundled catalog, then the
+ * pinned fallback. At most MAX_MODEL_FALLBACK_RETRIES, never `failing` itself.
+ * `codex debug models --bundled` reads only the catalog inside the binary: no
+ * network, no sign-in, no cache write.
+ */
+export async function codexFallbackModels(args) {
+    const failing = versionTooOldModel(args.reason) ?? args.requestedModel;
+    const exclude = [failing ?? "", args.requestedModel ?? ""];
+    // With no refused model named, or one whose slug carries no generation, the
+    // catalog cannot say which models are older: the pinned fallback alone.
+    if (!failing || codexModelGeneration(failing) === undefined) {
+        return exclude.includes(CODEX_PINNED_FALLBACK_MODEL) ? [] : [CODEX_PINNED_FALLBACK_MODEL];
+    }
+    const out = [];
+    let installed;
+    try {
+        const v = await args.run(["--version"]);
+        if (v.code === 0)
+            installed = parseCodexVersion(v.stdout);
+    }
+    catch {
+        /* unknown version: the catalog is still filtered by generation and visibility */
+    }
+    try {
+        const catalog = await args.run(["debug", "models", "--bundled"]);
+        if (catalog.code === 0) {
+            const picked = pickCodexFallbackModel(catalog.stdout, { exclude, ...(installed ? { installed } : {}), ...(failing ? { refused: failing } : {}) });
+            if (picked)
+                out.push(picked);
+        }
+    }
+    catch {
+        /* older Codex without `debug models`: the pinned fallback alone */
+    }
+    if (!exclude.includes(CODEX_PINNED_FALLBACK_MODEL) && !out.includes(CODEX_PINNED_FALLBACK_MODEL))
+        out.push(CODEX_PINNED_FALLBACK_MODEL);
+    return out.slice(0, MAX_MODEL_FALLBACK_RETRIES);
+}
 /** Progress events in one codex JSONL line: finished agent messages and tool starts. */
 export function codexStreamEvents(line) {
     const ev = parseJsonLine(line);
@@ -200,7 +329,7 @@ export function createCodexRuntime() {
             const fromFile = extra.outputFileText?.trim();
             const text = (fromFile || lastMessage || "").trim();
             if (!text) {
-                throw new RuntimeError(`codex produced no reply${errorMessage ? `: ${errorMessage.slice(0, 200)}` : ""}`, (errorMessage ?? stdout.slice(-400)).slice(0, 400), null);
+                throw new RuntimeError(`codex produced no reply${errorMessage ? `: ${errorMessage.slice(0, 200)}` : ""}`, (errorMessage ?? stdout.slice(-400)).slice(0, 400), null, errorMessage ? {} : { excerptIsOutputTail: true });
             }
             return { text, sessionId };
         },
@@ -210,6 +339,9 @@ export function createCodexRuntime() {
         failureReason(stdout) {
             return codexFailureReason(stdout);
         },
+        fallbackModelsAfterVersionMismatch(args) {
+            return codexFallbackModels(args);
+        },
     };
 }
 //# sourceMappingURL=codex.js.map