@nexrall/code-core 1.4.13 → 1.4.15

package/dist/api/client.js

@@ -6,14 +6,69 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.API_BASE = void 0;
 exports.chooseFinalContent = chooseFinalContent;
 exports.streamChat = streamChat;
+exports.cancelTurn = cancelTurn;
 exports.getBalance = getBalance;
 exports.exchangeVscodeCode = exchangeVscodeCode;
 exports.login = login;
 const eventsource_parser_1 = require("eventsource-parser");
 const node_fetch_1 = __importDefault(require("node-fetch"));
+const crypto_1 = require("crypto");
 const index_1 = require("../auth/index");
 // ─── Constants ────────────────────────────────────────────────────────────────
-exports.API_BASE = 'https://api.nexrall.com';
+const DEFAULT_API_BASE = 'https://api.nexrall.com';
+/**
+ * Resolve the backend origin, honouring NEXRALL_API_BASE.
+ *
+ * Everything sensitive in this module goes to API_BASE: the bearer token on
+ * `/api/code/chat`, the user's plaintext password on `/api/auth/login/email`, and the
+ * full source-code context of every turn.
+ *
+ * Loopback is allowed freely (it cannot exfiltrate, and the transport-failure tests need
+ * it). A REMOTE host requires NEXRALL_ALLOW_API_OVERRIDE=1 in addition, so redirecting a
+ * user's credentials is a deliberate act rather than something a stray env var in a CI
+ * config or an npm lifecycle script can do silently. Every override is announced on stderr,
+ * so it can never be in effect unnoticed. This is still not a hard security boundary —
+ * something that can set two env vars can set one — but it removes the ambient-risk case,
+ * which is the one that actually happens.
+ *
+ * An invalid value falls back to production rather than throwing: this runs at module
+ * load, and `@nexrall/code-core` is imported statically by the VS Code extension, so a
+ * throw here would abort activation and bury the reason in the extension-host log.
+ */
+function resolveApiBase() {
+    const raw = process.env.NEXRALL_API_BASE?.trim();
+    if (!raw)
+        return DEFAULT_API_BASE;
+    const isLoopback = /^http:\/\/(127\.0\.0\.1|\[::1\]|localhost)(:\d+)?(\/|$)/i.test(raw);
+    if (isLoopback) {
+        console.error(`⚠️  Nexrall API override in effect (loopback): ${raw.replace(/\/+$/, '')}`);
+        return raw.replace(/\/+$/, '');
+    }
+    // A REMOTE override additionally requires an explicit opt-in flag.
+    //
+    // Loopback can't exfiltrate anything, so it stays freely available (it is what makes the
+    // transport-failure tests possible). A remote host is different: this variable receives the
+    // bearer token, the plaintext login password, and the full source context of every turn. In
+    // a published package, honouring it from a bare env var means anything that can set one
+    // variable — a CI config, a shell profile, an npm lifecycle script that spawns `nex` — can
+    // silently redirect all of that, with nothing visible in the UI. Requiring a second,
+    // purpose-named variable makes redirection a deliberate act rather than an ambient one.
+    if (process.env.NEXRALL_ALLOW_API_OVERRIDE !== '1') {
+        console.error(`⚠️  Ignoring NEXRALL_API_BASE="${raw}": redirecting to a remote host also requires ` +
+            `NEXRALL_ALLOW_API_OVERRIDE=1, because this variable receives your auth token and ` +
+            `source code. Falling back to ${DEFAULT_API_BASE}.`);
+        return DEFAULT_API_BASE;
+    }
+    if (!/^https:\/\//i.test(raw)) {
+        console.error(`⚠️  Ignoring NEXRALL_API_BASE="${raw}": a remote override must be https:// so your ` +
+            `token is never sent in plaintext. Falling back to ${DEFAULT_API_BASE}.`);
+        return DEFAULT_API_BASE;
+    }
+    const base = raw.replace(/\/+$/, '');
+    console.error(`⚠️  Nexrall API override in effect: ${base}`);
+    return base;
+}
+exports.API_BASE = resolveApiBase();
 // ─── Helpers ──────────────────────────────────────────────────────────────────
 /**
  * Decide the final assistant `content` array to store in history for a completed
@@ -37,7 +92,18 @@ function chooseFinalContent(rebuilt, rawContent) {
     if (!Array.isArray(rawContent))
         return rebuilt;
     const hasServerSideBlocks = rawContent.some((b) => b && b.type !== 'text' && b.type !== 'tool_use');
-    return hasServerSideBlocks ? rawContent : rebuilt;
+    if (hasServerSideBlocks)
+        return rawContent;
+    // The rebuild came out EMPTY even though the authoritative message has content. That
+    // means the incremental events this reconstruction depends on never landed — a `text`
+    // frame split across a chunk boundary and dropped as malformed, or a mid-stream
+    // reconnect that resumed past them. Preferring `rebuilt` here would silently return an
+    // empty assistant turn and the agent loop would treat the whole turn as a no-op, losing
+    // an answer the server had already delivered in full. The raw array is authoritative,
+    // so fall back to it.
+    if (rebuilt.length === 0 && rawContent.length > 0)
+        return rawContent;
+    return rebuilt;
 }
 function authHeaders() {
     const token = (0, index_1.getToken)();
@@ -53,6 +119,27 @@ function authHeaders() {
 const MAX_RETRIES = 5;
 const RETRY_BASE_MS = 1000;
 const RETRY_MAX_MS = 30000;
+/**
+ * Wall-clock ceiling on ALL reconnect effort for a single streamChat() call.
+ *
+ * The retry loops are nested — a connect-time loop inside an outer stream loop —
+ * so their attempt counts MULTIPLY: 6 × 6 = 36 network attempts, each able to
+ * sleep up to 30 s of backoff. In a genuine outage that meant the UI could sit
+ * "reconnecting…" for the better part of an hour, silently resending the full
+ * conversation (and paying cache-write on it) every single time, before finally
+ * admitting failure.
+ *
+ * A time budget is the honest limit — the user cares about "how long until you
+ * tell me it's broken", not how many sockets were opened. Once it's spent, the
+ * next failure surfaces immediately. Attempt caps stay as a secondary bound.
+ */
+const MAX_TOTAL_RETRY_MS = (() => {
+    // Overridable so tests can exercise budget EXHAUSTION, which is otherwise unreachable
+    // in under five minutes and is exactly where the tricky control flow lives (the
+    // connect-time catch must break rather than fall back into its loop unthrottled).
+    const raw = Number(process.env.NEXRALL_MAX_RETRY_MS);
+    return Number.isFinite(raw) && raw > 0 ? raw : 5 * 60000;
+})();
 /** Exponential backoff with full jitter, capped at RETRY_MAX_MS. The jitter spreads
  *  retries out so a fleet of clients hitting the SAME overloaded API (529) don't all
  *  reconnect in lockstep and re-trigger the overload (thundering herd). Returns a delay
@@ -64,30 +151,105 @@ function backoffMs(attempt) {
 function sleep(ms) {
     return new Promise((r) => setTimeout(r, ms));
 }
-/** Total number of network-level attempts a single streamChat() call may make
- *  (connect-time retries × outer stream-retries), used only to report a
- *  meaningful "attempt N of M" to the UI — not a hard limit by itself. */
+/** Upper bound on network-level attempts (connect-time retries × outer stream-retries),
+ *  reported to the UI as the "of M" in "attempt N of M". It is NOT the real stopping
+ *  condition — MAX_TOTAL_RETRY_MS almost always runs out first, and deliberately so:
+ *  the nested product (36) exists to bound pathological cases, not to be reached. */
 const MAX_TOTAL_ATTEMPTS = (MAX_RETRIES + 1) * (MAX_RETRIES + 1);
 async function streamChat(messages, options, onEvent) {
-    const { model, env, editorContext, nexrallMd, mode, effort, clientType, abortSignal, extraTools, agents, skills } = options;
+    const { model, env, editorContext, nexrallMd, mode, effort, clientType, abortSignal, extraTools, agents, skills, allowRestartAfterRender } = options;
+    // IDEMPOTENCY KEY for this logical turn, generated ONCE and reused by every retry
+    // below (that reuse is the entire point — a fresh id per attempt would be useless).
+    //
+    // The failure it closes is expensive and invisible: the backend finishes the model
+    // call, bills it, and writes `done` — but the socket dies before the client reads it.
+    // The client sees a dropped connection and retries, so the backend runs the SAME turn
+    // through the model a SECOND time and bills it again. The first, perfectly good answer
+    // is discarded. The user paid twice for one question and nothing on either side could
+    // tell. With a stable key the backend recognises the replay and returns the stored
+    // result instead of re-invoking the model.
+    let turnId = (0, crypto_1.randomUUID)();
     // Bridge custom abort signal → native AbortController so fetch is cancelled immediately
     const controller = new AbortController();
     if (abortSignal?.aborted)
         throw Object.assign(new Error('Aborted'), { name: 'AbortError' });
     let abortPoll;
+    // Guards against the 50ms poll firing a fresh cancel request on every tick between the
+    // signal flipping and streamChat returning — each one a real POST doing Redis work, i.e.
+    // a self-inflicted burst on the exact path a user hits when things are already wrong.
+    let cancelSent = false;
     if (abortSignal) {
-        abortPoll = setInterval(() => { if (abortSignal.aborted)
-            controller.abort(); }, 50);
+        abortPoll = setInterval(() => {
+            if (!abortSignal.aborted)
+                return;
+            if (cancelSent)
+                return; // the poll keeps ticking until the outer finally clears it
+            cancelSent = true;
+            // Tell the SERVER to stop, not just this socket.
+            //
+            // Closing the socket used to be sufficient — the backend aborted the model when the
+            // request closed. That is exactly what changed to make streams resumable: a dropped
+            // connection now DETACHES and keeps generating, because a blip must not throw the
+            // turn away. But a deliberate Stop arrives as the very same dead socket. Without an
+            // explicit cancel the model would run to completion and bill in full while the user
+            // believed they had stopped it.
+            //
+            // Fire-and-forget, before aborting: we don't want Stop to feel slow, and if the
+            // request fails the server's detach grace still bounds the damage.
+            void cancelTurn(turnId);
+            controller.abort();
+        }, 50);
     }
-    const fetchArgs = [
-        `${exports.API_BASE}/api/code/chat`,
-        {
-            method: 'POST',
-            headers: authHeaders(),
-            body: JSON.stringify({ messages, model, env, editorContext, nexrallMd, mode, effort, clientType, extraTools, agents, skills }),
-            signal: controller.signal,
-        },
-    ];
+    // ── Resume state ───────────────────────────────────────────────────────────
+    // Set when the backend announces (via a `resumable` frame) that this turn's SSE frames
+    // are numbered and buffered server-side. Once true, a dropped connection can be RESUMED
+    // — reattached mid-response — instead of restarting the turn.
+    //
+    // Why that matters: restarting re-sends the whole conversation, pays for prompt
+    // processing and time-to-first-token again, and re-bills every token already generated.
+    // Resuming costs one buffer read and wastes nothing. Older backends never send the frame,
+    // so this stays false and the existing restart path is used unchanged.
+    let serverResumable = false;
+    // Highest frame id received. The resume cursor handed back as Last-Event-ID.
+    let lastEventId = 0;
+    // Whether the CURRENT attempt is a resume (GET …/resume) rather than a fresh POST.
+    let resuming = false;
+    // Output received BEFORE the drop, carried across a resume. A resume delivers only the
+    // frames past `lastEventId`, so without this the rebuilt message would consist of just
+    // the tail and the start of the answer would be lost.
+    let carryText = [];
+    let carryToolUse = [];
+    // Whether ANY output has reached the caller across ALL attempts of this turn, and how
+    // much. `emittedToCaller` inside runOnce is per-attempt and resets, but a rollback
+    // decision has to consider everything the user can currently see — including output
+    // delivered by an earlier attempt before a resume.
+    let emittedAnythingAcrossAttempts = false;
+    let emittedCharsAcrossAttempts = 0;
+    // Built per attempt, because an attempt is EITHER a fresh turn (POST /chat) or a resume
+    // of the one already generating server-side (GET /chat/resume + Last-Event-ID). The
+    // resume carries no body at all — the conversation is already being processed; re-sending
+    // it is exactly the waste this avoids.
+    const buildFetchArgs = () => {
+        if (resuming) {
+            return [
+                `${exports.API_BASE}/api/code/chat/resume?turnId=${encodeURIComponent(turnId)}`,
+                {
+                    method: 'GET',
+                    headers: { ...authHeaders(), 'Last-Event-ID': String(lastEventId) },
+                    signal: controller.signal,
+                },
+            ];
+        }
+        return [
+            `${exports.API_BASE}/api/code/chat`,
+            {
+                method: 'POST',
+                headers: authHeaders(),
+                body: JSON.stringify({ messages, model, env, editorContext, nexrallMd, mode, effort, clientType, extraTools, agents, skills, turnId }),
+                signal: controller.signal,
+            },
+        ];
+    };
     // A transient mid-stream failure (Anthropic "overloaded_error") arrives as an SSE error
     // event over an ALREADY-200 response, so the connect-time HTTP-status retry below can't
     // see it. We detect it in the parser and retry the WHOLE attempt with exponential backoff
@@ -102,7 +264,33 @@ async function streamChat(messages, options, onEvent) {
     // happened (this is the fix for that gap).
     let didRetry = false;
     let totalAttemptsMade = 0;
+    // Shared wall-clock budget for every reconnect across BOTH nested retry loops.
+    // `retryBudgetLeft()` gates each retry decision and also clamps the backoff sleep,
+    // so we never wait longer than the budget we have left.
+    //
+    // The clock starts at the FIRST retry, not at streamChat() entry. This matters: a
+    // healthy first attempt on a huge context can legitimately spend minutes in
+    // prompt-processing before the first token (hence FIRST_EVENT_TIMEOUT_MS = 300 s).
+    // Anchoring the deadline at entry would let that normal, successful work consume the
+    // entire reconnect budget, so the first real network blip would get zero retries —
+    // precisely backwards. Measuring only time spent RECONNECTING keeps the guarantee
+    // meaningful: "at most 5 minutes of retrying before we tell you it's broken".
+    let retryDeadline = 0;
+    const retryBudgetLeft = () => retryDeadline === 0 ? MAX_TOTAL_RETRY_MS : retryDeadline - Date.now();
+    // Require a MEANINGFUL remainder, not just "> 0". With a bare positive check, 1 ms of
+    // budget was enough to launch a whole fresh attempt — which can then sit in
+    // FIRST_EVENT_TIMEOUT_MS (300 s) of prompt-processing wait, blowing past the very
+    // deadline it was checked against. It also turned a fast-failing connect error
+    // (ENOTFOUND) into a tight loop of near-zero sleeps. One base backoff of headroom
+    // makes the budget an approximate ceiling instead of a suggestion.
+    const canRetry = () => retryBudgetLeft() > RETRY_BASE_MS;
+    /** Sleep for `ms`, but never past the shared retry deadline. */
+    const sleepWithinBudget = (ms) => sleep(Math.max(0, Math.min(ms, retryBudgetLeft())));
     const reportRetry = (reason) => {
+        // Start the reconnect budget on the first retry (see retryDeadline above). Every
+        // retry path funnels through here, so this is the one place it needs to happen.
+        if (retryDeadline === 0)
+            retryDeadline = Date.now() + MAX_TOTAL_RETRY_MS;
         totalAttemptsMade++;
         didRetry = true;
         onEvent({ type: 'retry', attempt: totalAttemptsMade, maxAttempts: MAX_TOTAL_ATTEMPTS, reason });
@@ -120,24 +308,149 @@ async function streamChat(messages, options, onEvent) {
         // Retry loop — handles connect-time 429 (rate limit) and 5xx (server errors, incl. 529)
         let response;
         let lastErr;
+        // How many times we've waited on a `turnInFlight` 409. Drives that path's own backoff,
+        // separately from `attempt` (a wait is not a failed attempt — see the 409 branch).
+        let inFlightWaits = 0;
+        // A 409's body, already consumed to classify it. node-fetch bodies are single-use, so
+        // the !response.ok handler below reads this instead of calling response.text() again.
+        let errorBodyOverride = null;
         for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
             try {
-                response = await (0, node_fetch_1.default)(...fetchArgs);
-                if (response.status === 429 && attempt < MAX_RETRIES) {
+                response = await (0, node_fetch_1.default)(...buildFetchArgs());
+                if (response.status === 429 && attempt < MAX_RETRIES && canRetry()) {
                     reportRetry('Rate limited by the API — retrying');
                     const retryAfter = parseInt(response.headers.get('retry-after') ?? '0', 10);
-                    await sleep(retryAfter > 0 ? retryAfter * 1000 : backoffMs(attempt));
+                    await sleepWithinBudget(retryAfter > 0 ? retryAfter * 1000 : backoffMs(attempt));
                     continue;
                 }
-                if (response.status >= 500 && response.status < 600 && attempt < MAX_RETRIES) {
+                if (response.status >= 500 && response.status < 600 && attempt < MAX_RETRIES && canRetry()) {
                     reportRetry(`Server error (${response.status}) — retrying`);
-                    await sleep(backoffMs(attempt));
+                    await sleepWithinBudget(backoffMs(attempt));
                     continue;
                 }
-                // 4xx (incl. 413 Payload Too Large, 400 Bad Request, 402 Insufficient balance)
-                // are DETERMINISTIC given the same body — retrying resends the identical oversized
-                // conversation and can never succeed. Fall through to the non-retryable !response.ok
-                // handler below instead of burning retries (this was the 16-request "retry storm").
+                // 403 `{"error":{"type":"forbidden","message":"Request not allowed"}}` is
+                // Anthropic's own upstream response shape (reproduced verbatim through the AI
+                // Gateway passthrough — see services/aiGatewayClient.js), not something Nexrall's
+                // auth layer produces (an expired/invalid Nexrall JWT is a 401/403 with a
+                // completely different body — see middleware/auth.js). Reports of this exact
+                // error on Anthropic's own CLI describe it as intermittent and gone on retry, not
+                // a real permissions failure — so unlike every OTHER 4xx below (which are
+                // deterministic given the same body and must not be retried), this one gets a
+                // short, capped number of attempts before being treated as terminal. Capped well
+                // below MAX_RETRIES so a genuinely revoked/banned key still fails fast instead of
+                // burning the whole reconnect budget on something that will never succeed.
+                const FORBIDDEN_RETRY_LIMIT = 2;
+                if (response.status === 403 && attempt < FORBIDDEN_RETRY_LIMIT && canRetry()) {
+                    const bodyText = await response.text().catch(() => '');
+                    let isUpstreamForbidden = false;
+                    try {
+                        const parsed = JSON.parse(bodyText);
+                        isUpstreamForbidden = parsed?.error?.type === 'forbidden';
+                    }
+                    catch { /* not JSON — not this shape, fall through as terminal below */ }
+                    if (isUpstreamForbidden) {
+                        reportRetry('Upstream rejected the request — retrying');
+                        await sleepWithinBudget(backoffMs(attempt));
+                        continue;
+                    }
+                    // Not the known transient shape — treat as terminal, but reuse the body we
+                    // already consumed (node-fetch bodies are single-use) instead of losing it.
+                    errorBodyOverride = bodyText;
+                    break;
+                }
+                // 410 Gone — a resume was refused because the buffered window no longer contains
+                // our position (we were away too long, or the turn produced a huge amount in the
+                // meantime). Continuity can't be proven, and silently skipping frames would corrupt
+                // the message being rebuilt. So abandon resuming and fall back to restarting the
+                // turn from scratch: correct, just not free.
+                if (response.status === 410 && resuming) {
+                    await response.text().catch(() => '');
+                    // A 410 means the BUFFER WINDOW moved past our cursor — it does NOT mean the turn
+                    // ended. It is almost certainly still generating and still holding its idempotency
+                    // claim, so re-POSTing with the same turnId would be rejected as `turnInFlight`,
+                    // and we would wait out the entire retry budget only to fail — while the orphaned
+                    // turn kept generating and billing.
+                    //
+                    // So: stop the old turn (awaited — the claim must be released BEFORE we re-POST),
+                    // and mint a fresh id, because a restart is semantically a new turn.
+                    await cancelTurn(turnId);
+                    turnId = (0, crypto_1.randomUUID)();
+                    resuming = false;
+                    serverResumable = false;
+                    lastEventId = 0;
+                    carryText = [];
+                    carryToolUse = [];
+                    // Snapshot BEFORE resetting: the rollback decision below needs to know what the
+                    // user can currently see, but the counters must not carry into the restarted turn
+                    // or a second restart would report characters this one already discarded.
+                    const hadRendered = emittedAnythingAcrossAttempts;
+                    const renderedChars = emittedCharsAcrossAttempts;
+                    emittedAnythingAcrossAttempts = false;
+                    emittedCharsAcrossAttempts = 0;
+                    if (hadRendered && !allowRestartAfterRender) {
+                        // Output is already on screen and the caller can't take it back, so a restart
+                        // would duplicate it. Surface the failure instead of corrupting the transcript.
+                        throw new Error('Connection lost and this turn could no longer be resumed. Send your message again.');
+                    }
+                    if (hadRendered) {
+                        onEvent({
+                            type: 'stream_restart',
+                            reason: 'Connection lost too long to resume',
+                            discardedChars: renderedChars,
+                        });
+                    }
+                    if (attempt < MAX_RETRIES && canRetry()) {
+                        reportRetry('Could not resume — restarting this turn');
+                        await sleepWithinBudget(backoffMs(attempt));
+                        continue;
+                    }
+                    errorBodyOverride = JSON.stringify({ error: 'Could not resume this turn — send your message again.' });
+                    break;
+                }
+                // 409 from the backend's turn-idempotency gate. Two distinct meanings, and the
+                // body distinguishes them:
+                //
+                //   turnInFlight  — another attempt carrying the same turnId is STILL RUNNING
+                //                   (two sockets racing; typically a reconnect landing before the
+                //                   original finished tearing down). Running it anyway would
+                //                   double-charge one turn, which is what that gate prevents.
+                //                   WAITING is correct: the original either completes (and this
+                //                   attempt is then served the stored answer for free) or fails
+                //                   (and this attempt runs for real).
+                //   turnCompleted — the turn finished and was billed, but its response is too
+                //                   large to resend. DETERMINISTIC; retrying can never help.
+                if (response.status === 409) {
+                    const body = await response.text();
+                    let inFlight = false;
+                    try {
+                        inFlight = JSON.parse(body)?.turnInFlight === true;
+                    }
+                    catch { /* treat as terminal */ }
+                    // A wait is NOT a failed attempt, so it must not consume the network-error
+                    // budget: a real coding turn runs for minutes, and spending `MAX_RETRIES` on
+                    // fixed short sleeps burned the whole allowance in ~10 s and hard-failed while
+                    // the original turn was perfectly healthy — leaving the user billed for an
+                    // answer they never received. Only the wall-clock budget bounds this loop.
+                    if (inFlight && canRetry()) {
+                        inFlightWaits++;
+                        reportRetry('This turn is already being processed — waiting for it');
+                        const retryAfter = parseInt(response.headers.get('retry-after') ?? '0', 10);
+                        // Honour the server's hint but keep growing our own backoff, so a wildly
+                        // optimistic Retry-After can't turn this into a hot loop.
+                        await sleepWithinBudget(Math.max(retryAfter * 1000, backoffMs(inFlightWaits)));
+                        attempt--; // refund the attempt — this was a wait, not a failure
+                        continue;
+                    }
+                    // Terminal 409 (or the wait budget is spent): rebuild a response-like shape so
+                    // the shared !response.ok handler below can surface the server's own message.
+                    errorBodyOverride = body;
+                    break;
+                }
+                // Every other 4xx (413 Payload Too Large, 400 Bad Request, 402 Insufficient
+                // balance) is DETERMINISTIC given the same body — retrying resends the identical
+                // oversized conversation and can never succeed. Fall through to the non-retryable
+                // !response.ok handler below instead of burning retries (this was the 16-request
+                // "retry storm").
                 break; // success or non-retryable error
             }
             catch (err) {
@@ -145,19 +458,31 @@ async function streamChat(messages, options, onEvent) {
                 if (err.name === 'AbortError')
                     throw err;
                 lastErr = err;
-                if (attempt < MAX_RETRIES) {
+                if (attempt < MAX_RETRIES && canRetry()) {
                     // Connection couldn't even be established — network drop / DNS blip / the
                     // machine just woke from sleep and Wi-Fi hasn't reconnected yet.
                     reportRetry('Connection lost — attempting to reconnect');
-                    await sleep(backoffMs(attempt));
+                    await sleepWithinBudget(backoffMs(attempt));
                     continue;
                 }
+                // Out of attempts, or out of reconnect budget. MUST break: falling off the end
+                // of the catch re-enters the for-loop with no backoff and no further budget
+                // check, firing the remaining attempts back-to-back in milliseconds — a burst
+                // of full-conversation POSTs, invisible to the UI. (The 429/5xx branches above
+                // already break for the same reason.) Tag the error so the outer loop can
+                // report budget exhaustion rather than a bare transport failure.
+                // Guard the shape: a patched fetch/undici that throws a string or null would turn
+                // a network failure into a TypeError raised by the error handler itself.
+                if (!canRetry() && lastErr && typeof lastErr === 'object') {
+                    Object.assign(lastErr, { retryable: true });
+                }
+                break;
             }
         }
         if (!response)
             throw lastErr ?? new Error('Request failed after max retries');
         if (!response.ok) {
-            const errText = await response.text();
+            const errText = errorBodyOverride ?? await response.text();
             let errMsg = `API error ${response.status}`;
             let balance;
             try {
@@ -178,13 +503,60 @@ async function streamChat(messages, options, onEvent) {
         if (!response.body) {
             throw new Error('Response body is null');
         }
-        // Accumulated state for building the assistant message
-        const textParts = [];
-        const toolUseBlocks = [];
+        // Accumulated state for building the assistant message.
+        //
+        // Seeded from the previous attempt when RESUMING. This is essential and easy to miss:
+        // a resume delivers only the frames after `lastEventId`, so the text and tool calls
+        // received before the drop live in the outer carry-over. Starting empty here would
+        // rebuild the message from the tail alone and silently truncate the answer — the
+        // beginning of the response would simply vanish.
+        const textParts = resuming ? [...carryText] : [];
+        const toolUseBlocks = resuming ? [...carryToolUse] : [];
         let completedMessage = null;
-        // Once anything has been shown to the caller (text / tool_use / thinking), a retry would
-        // duplicate it — so from that point on a stream error is surfaced, never retried.
+        // Publish the accumulators outward continuously, so if THIS attempt dies the next one
+        // can resume on top of what it already received. Assigning the same array references is
+        // enough — the resume path copies them.
+        carryText = textParts;
+        carryToolUse = toolUseBlocks;
+        // Once anything has been shown to the caller (text / tool_use / thinking), a naive retry
+        // would duplicate it on screen. Two outcomes are possible from here:
+        //   • caller CAN'T undo its rendering → the failure stays fatal (legacy behaviour);
+        //   • caller CAN  (allowRestartAfterRender) → we tag the error `needsRestart`, the outer
+        //     loop emits `stream_restart` so the caller drops the partial render, and the turn
+        //     is retried from scratch. Nothing else needs undoing: tool calls only execute after
+        //     `message_complete`, which by definition never arrived on a failed attempt.
         let emittedToCaller = false;
+        // Visible characters emitted this attempt — reported in `stream_restart` purely so the
+        // UI can say how much output is being discarded.
+        let emittedChars = 0;
+        /**
+         * Classify a transport/stream failure. `retryable` means "safe to retry with nothing
+         * shown yet"; `needsRestart` means "safe to retry ONLY if the caller rolls back what
+         * it already rendered". Anything else is terminal.
+         */
+        const tagTransient = (err) => {
+            if (!emittedToCaller)
+                return Object.assign(err, { retryable: true });
+            if (allowRestartAfterRender) {
+                return Object.assign(err, { retryable: true, needsRestart: true, discardedChars: emittedChars });
+            }
+            return err;
+        };
+        /**
+         * True once the turn's complete assistant message has arrived.
+         *
+         * There is a real window between `message_complete` and `done`: the backend awaits
+         * its billing promise (a DB round-trip) before sending `done`, so a socket that
+         * dies in between produces a transport error on a turn that has ALREADY fully
+         * succeeded. Retrying there would be the worst possible outcome — it discards a
+         * complete, correct answer, re-runs the model, and bills the user twice.
+         *
+         * Every failure path therefore checks this and resolves instead of rejecting: we
+         * have the message, and `done` carries no information we need. This also restores
+         * the premise the restart design rests on — a restart now genuinely implies
+         * `message_complete` never arrived.
+         */
+        const haveCompleteMessage = () => completedMessage !== null;
         // Partial input accumulator keyed by tool_use id
         const partialInputs = {};
         await new Promise((resolve, reject) => {
@@ -192,11 +564,19 @@ async function streamChat(messages, options, onEvent) {
             // Declared here so the heartbeat watchdog closure can reference it.
             const stream = response.body;
             // ── Heartbeat watchdog ────────────────────────────────────────────────────
-            // The backend sends ": ping" comments every 20 s. If we receive no data at
-            // all for 45 s the connection has silently dropped (proxy timeout, network
+            // The backend sends ": ping" comments every 10 s. If we receive no data at
+            // all for 90 s the connection has silently dropped (proxy timeout, network
             // blip). Reject so the caller sees an actionable error instead of hanging.
-            const HEARTBEAT_TIMEOUT_MS = 45000;
-            // Progress watchdog. The backend's ": ping" keepalive (every 20 s) resets lastDataAt,
+            //
+            // Sizing: this must tolerate a burst of consecutive LOST pings, not just one.
+            // The old pairing (20 s ping / 45 s timeout) survived exactly two misses —
+            // routinely exceeded on flaky Wi-Fi, tethered 4G, or the seconds right after a
+            // laptop wakes, so a perfectly healthy connection got torn down and the turn
+            // restarted for nothing. At 10 s / 90 s we ride out eight consecutive misses
+            // while still detecting a truly dead socket faster in absolute terms than the
+            // proxy's own idle timeout.
+            const HEARTBEAT_TIMEOUT_MS = 90000;
+            // Progress watchdog. The backend's ": ping" keepalive (every 10 s) resets lastDataAt,
             // so the heartbeat above only catches a fully dead connection — NOT an upstream model
             // stall, where Anthropic stops emitting tokens but the backend keeps pinging. That case
             // would otherwise hang on "Thinking…" forever. lastProgressAt is bumped ONLY on real
@@ -210,13 +590,34 @@ async function streamChat(messages, options, onEvent) {
             //     attempt warmed the prompt cache, so the retry's TTFT was short. Allow 300 s.
             //   • AFTER the first model event, tokens are flowing; a 150 s silent gap mid-stream
             //     is a genuine stall.
-            const FIRST_EVENT_TIMEOUT_MS = 300000;
-            const PROGRESS_TIMEOUT_MS = 150000;
+            // Overridable so tests can reach the stall path in seconds instead of minutes.
+            // That path is where the subtlest bug in this file lives: `lastProgressAt` stops
+            // advancing at `message_complete` while heartbeat pings keep `lastDataAt` fresh, so
+            // this watchdog — and only this one — can fire on a turn that already succeeded.
+            const FIRST_EVENT_TIMEOUT_MS = Number(process.env.NEXRALL_FIRST_EVENT_TIMEOUT_MS) || 300000;
+            const PROGRESS_TIMEOUT_MS = Number(process.env.NEXRALL_PROGRESS_TIMEOUT_MS) || 150000;
             let sawModelEvent = false;
             let lastDataAt = Date.now();
             let lastProgressAt = Date.now();
             const heartbeatWatchdog = setInterval(() => {
                 const now = Date.now();
+                // The turn already succeeded and we're only waiting on the trailing `done`.
+                // Finish with what we have rather than discarding a complete message and
+                // re-running the model.
+                //
+                // The silence condition is essential, not decorative: the backend AWAITS its
+                // billing write before emitting `balance_status` and `done`, so the gap after
+                // `message_complete` is normal, expected, and can run for seconds on a slow DB.
+                // Firing on the next tick regardless would tear down a perfectly healthy socket
+                // and swallow the low-balance nudge — trading the bug this guard was written to
+                // fix for a different one. Wait out the full heartbeat timeout first; only then
+                // is the trailing frame genuinely never coming.
+                if (haveCompleteMessage() && now - lastDataAt > HEARTBEAT_TIMEOUT_MS) {
+                    clearInterval(heartbeatWatchdog);
+                    stream.destroy?.();
+                    resolve();
+                    return;
+                }
                 if (now - lastDataAt > HEARTBEAT_TIMEOUT_MS) {
                     clearInterval(heartbeatWatchdog);
                     stream.destroy?.();
@@ -227,24 +628,36 @@ async function streamChat(messages, options, onEvent) {
                     // `now - lastDataAt` immediately reads as a large gap even though nothing is
                     // actually wrong with the connection; without this it used to hard-fail
                     // instead of silently reconnecting.
-                    const deadErr = new Error(emittedToCaller
-                        ? 'Connection lost — no data received for 45 s. Retry your message.'
-                        : 'Connection lost — no data received for 45 s. Reconnecting…');
-                    reject(emittedToCaller ? deadErr : Object.assign(deadErr, { retryable: true }));
+                    const recoverable = !emittedToCaller || !!allowRestartAfterRender;
+                    const secs = Math.round(HEARTBEAT_TIMEOUT_MS / 1000);
+                    reject(tagTransient(new Error(recoverable
+                        ? `Connection lost — no data received for ${secs} s. Reconnecting…`
+                        : `Connection lost — no data received for ${secs} s. Retry your message.`)));
                     return;
                 }
                 const stallLimitMs = sawModelEvent ? PROGRESS_TIMEOUT_MS : FIRST_EVENT_TIMEOUT_MS;
                 if (now - lastProgressAt > stallLimitMs) {
                     clearInterval(heartbeatWatchdog);
                     stream.destroy?.();
+                    // Same rule as the heartbeat branch above, and MORE easily reached here: this
+                    // watchdog reads `lastProgressAt`, which only real model events bump. Once
+                    // `message_complete` lands, nothing bumps it again — so while the backend awaits
+                    // its billing write, the heartbeat's `: ping` keeps `lastDataAt` fresh (both
+                    // branches above correctly stay silent) but THIS clock runs out and reports a
+                    // "stall" on a turn that already succeeded. Restarting there would discard a
+                    // correct answer and re-bill the model call, repeatedly. The turn is complete;
+                    // finish with it.
+                    if (haveCompleteMessage()) {
+                        resolve();
+                        return;
+                    }
                     // If nothing has been shown to the caller yet, a fresh attempt can't duplicate
                     // output — tag it retryable so the outer loop transparently retries the turn
                     // instead of killing it. Once output has been emitted, a retry would duplicate
                     // rendered text/tool calls, so surface the stall as a terminal error.
-                    const stallErr = new Error(sawModelEvent
-                        ? `The model stopped responding mid-stream (no output for ${Math.round(stallLimitMs / 1000)} s). Retry your message.`
-                        : `The model did not start responding within ${Math.round(stallLimitMs / 1000)} s (large context can take a while to process). Retry your message.`);
-                    reject(emittedToCaller ? stallErr : Object.assign(stallErr, { retryable: true }));
+                    reject(tagTransient(new Error(sawModelEvent
+                        ? `The model stopped responding mid-stream (no output for ${Math.round(stallLimitMs / 1000)} s).`
+                        : `The model did not start responding within ${Math.round(stallLimitMs / 1000)} s (large context can take a while to process).`)));
                     return;
                 }
             }, 5000);
@@ -255,6 +668,15 @@ async function streamChat(messages, options, onEvent) {
                     return;
                 }
                 {
+                    // Track the server-assigned frame id. This is the resume cursor: on a drop we
+                    // hand it back via Last-Event-ID and receive exactly the frames we missed,
+                    // instead of re-running the whole turn. Only present on resumable turns.
+                    const frameId = event.id;
+                    if (frameId) {
+                        const n = Number(frameId);
+                        if (Number.isFinite(n) && n > lastEventId)
+                            lastEventId = n;
+                    }
                     const raw = event.data;
                     if (!raw || raw === '[DONE]') {
                         resolve();
@@ -283,6 +705,9 @@ async function streamChat(messages, options, onEvent) {
                             const text = typeof evt.text === 'string' ? evt.text : '';
                             textParts.push(text);
                             emittedToCaller = true;
+                            emittedChars += text.length;
+                            emittedAnythingAcrossAttempts = true;
+                            emittedCharsAcrossAttempts += text.length;
                             onEvent({ type: 'text', text });
                             break;
                         }
@@ -305,7 +730,14 @@ async function streamChat(messages, options, onEvent) {
                                 }
                             }
                             toolUseBlocks.push(block);
+                            // Deliberately does NOT bump `emittedChars`: a streamed tool_use is a
+                            // preview that no consumer renders. The agent loop's `tool_use` case is a
+                            // no-op and the UI's tool row is only drawn later, from `options.onToolUse`
+                            // during execution — which happens after streamChat() returns and so never
+                            // runs on a failed attempt. `emittedToCaller` is still set, conservatively,
+                            // so a restart continues to require an explicit rollback handler.
                             emittedToCaller = true;
+                            emittedAnythingAcrossAttempts = true;
                             onEvent({ type: 'tool_use', id: block.id, name: block.name, input: block.input });
                             break;
                         }
@@ -360,6 +792,17 @@ async function streamChat(messages, options, onEvent) {
                             if (typeof evt.input_tokens === 'number' && typeof evt.output_tokens === 'number') {
                                 onEvent({
                                     type: 'usage',
+                                    // Forwarded, not dropped: the backend tags the usage of an attempt it
+                                    // billed but never completed (its stream `abort` path). Without this
+                                    // the flag dies here and every consumer that sums usage silently folds
+                                    // a discarded attempt's tokens into the successful turn's total.
+                                    ...(evt.partial === true ? { partial: true } : {}),
+                                    // `replayed` means these tokens are being reported a SECOND time: the
+                                    // turn completed and was billed on an earlier attempt whose `done` never
+                                    // reached us, and the backend served this one from its idempotency cache
+                                    // instead of re-running the model. Nothing new was charged, so a cost
+                                    // display must not add them again.
+                                    ...(evt.replayed === true ? { replayed: true } : {}),
                                     usage: {
                                         input_tokens: evt.input_tokens,
                                         output_tokens: evt.output_tokens,
@@ -380,6 +823,9 @@ async function streamChat(messages, options, onEvent) {
                             const text = typeof evt.text === 'string' ? evt.text : '';
                             if (text) {
                                 emittedToCaller = true;
+                                emittedChars += text.length;
+                                emittedAnythingAcrossAttempts = true;
+                                emittedCharsAcrossAttempts += text.length;
                                 onEvent({ type: 'thinking', text });
                             }
                             break;
@@ -397,10 +843,20 @@ async function streamChat(messages, options, onEvent) {
                             const text = typeof evt.text === 'string' ? evt.text : '';
                             if (text) {
                                 emittedToCaller = true;
+                                emittedChars += text.length;
+                                emittedAnythingAcrossAttempts = true;
+                                emittedCharsAcrossAttempts += text.length;
                                 onEvent({ type: 'thinking_delta', text });
                             }
                             break;
                         }
+                        case 'resumable': {
+                            // The backend numbers this turn's frames and buffers them, so a dropped
+                            // connection can reattach mid-response rather than re-running the turn.
+                            // Purely a capability announcement — nothing to show the user.
+                            serverResumable = true;
+                            break;
+                        }
                         case 'balance_status': {
                             const balance = typeof evt.balance === 'number' ? evt.balance : 0;
                             const zero = !!evt.zero;
@@ -415,12 +871,36 @@ async function streamChat(messages, options, onEvent) {
                         case 'error': {
                             const message = typeof evt.message === 'string' ? evt.message :
                                 typeof evt.error === 'string' ? evt.error : 'Unknown SSE error';
+                            // The GET /chat/resume tail loop sends this when the turn's Redis frame
+                            // buffer developed a gap (trimmed window, a swallowed flush error, or the
+                            // owning pod died) or the turn is simply gone — see routes/code.js. Unlike
+                            // `isRetryableStreamMsg` below, this is NOT a fuzzy text match: the backend
+                            // sets the flag explicitly, so it can never silently stop matching if either
+                            // side rewrites the English message. There is no cursor left to resume
+                            // from — a plain "retryable" resume-with-Last-Event-ID would just 410 again
+                            // — so this always needs a fresh turnId, which the outer loop mints via
+                            // `forceRestart`.
+                            const notResumable = evt.notResumable === true;
                             // Transient upstream failure before any output → let the outer loop retry it
                             // transparently instead of killing the turn (this is the "Overloaded" case).
-                            if (isRetryableStreamMsg(message) && !emittedToCaller) {
+                            // But never retry once the turn's message has already been delivered in
+                            // full: a late error frame (e.g. the backend's billing write failing after
+                            // `message_complete`) must not discard a correct answer and re-bill the
+                            // model call. Surface it as a notice and finish with what we have.
+                            if (haveCompleteMessage()) {
+                                clearInterval(heartbeatWatchdog);
+                                onEvent({ type: 'error', message });
+                                resolve();
+                            }
+                            else if (notResumable && (!emittedToCaller || allowRestartAfterRender)) {
+                                clearInterval(heartbeatWatchdog);
+                                stream.destroy?.();
+                                reject(Object.assign(tagTransient(new Error(message)), { forceRestart: true }));
+                            }
+                            else if (isRetryableStreamMsg(message) && (!emittedToCaller || allowRestartAfterRender)) {
                                 clearInterval(heartbeatWatchdog);
                                 stream.destroy?.();
-                                reject(Object.assign(new Error(message), { retryable: true }));
+                                reject(tagTransient(new Error(message)));
                             }
                             else {
                                 onEvent({ type: 'error', message });
@@ -459,16 +939,21 @@ async function streamChat(messages, options, onEvent) {
                     resolve();
                     return;
                 }
+                // Socket died AFTER the turn was fully delivered — the only thing still
+                // outstanding was the trailing `done`. Treat as success: rejecting here would
+                // throw away a complete answer and (with restart enabled) re-run and re-bill
+                // the entire turn. This is the widest such window in practice, because the
+                // backend does a DB write between `message_complete` and `done`.
+                if (haveCompleteMessage()) {
+                    resolve();
+                    return;
+                }
                 // Transport-level failures ("Premature close" / ECONNRESET / socket hang up)
                 // happen whenever the backend restarts mid-deploy or a proxy drops the socket.
                 // They are exactly as transient as an overloaded_error — if nothing has been
                 // emitted to the caller yet, retry the attempt transparently instead of
                 // surfacing "Stream failed: Premature close" and killing the whole turn.
-                if (!emittedToCaller) {
-                    reject(Object.assign(err, { retryable: true }));
-                    return;
-                }
-                reject(err);
+                reject(tagTransient(err));
             });
         });
         if (completedMessage) {
@@ -497,11 +982,80 @@ async function streamChat(messages, options, onEvent) {
             catch (err) {
                 if (abortSignal?.aborted || controller.signal.aborted || err.name === 'AbortError')
                     throw err;
-                if (err.retryable && sAttempt < MAX_RETRIES) {
+                const e = err;
+                if (e.retryable && sAttempt < MAX_RETRIES && canRetry()) {
+                    // The backend explicitly said this turnId's buffer is unrecoverable (gap in the
+                    // Redis frame stream, or the owning pod is gone — see the `notResumable` case in
+                    // client.ts's SSE parser). Resuming with the same `lastEventId` would just hit
+                    // the same wall again, so — exactly like the inline 410-while-resuming handling
+                    // above — release the old claim, mint a fresh turnId, and restart clean instead
+                    // of falling into the "prefer resuming" branch below.
+                    if (e.forceRestart) {
+                        await cancelTurn(turnId);
+                        turnId = (0, crypto_1.randomUUID)();
+                        resuming = false;
+                        serverResumable = false;
+                        lastEventId = 0;
+                        carryText = [];
+                        carryToolUse = [];
+                        const hadRendered = emittedAnythingAcrossAttempts;
+                        const renderedChars = emittedCharsAcrossAttempts;
+                        emittedAnythingAcrossAttempts = false;
+                        emittedCharsAcrossAttempts = 0;
+                        if (hadRendered && !allowRestartAfterRender) {
+                            throw new Error('Connection lost and this turn could no longer be resumed. Send your message again.');
+                        }
+                        if (hadRendered) {
+                            onEvent({
+                                type: 'stream_restart',
+                                reason: err.message || 'Could not resume — restarting this turn',
+                                discardedChars: renderedChars,
+                            });
+                        }
+                        reportRetry('Could not resume — restarting this turn');
+                        await sleepWithinBudget(backoffMs(sAttempt));
+                        continue;
+                    }
+                    // PREFER RESUMING over restarting.
+                    //
+                    // The turn is still generating on the server (a dropped socket no longer aborts
+                    // it), and every frame it emits is numbered and buffered. So instead of throwing
+                    // the work away we reattach at `lastEventId` and receive only what we missed:
+                    // nothing re-processed, nothing re-billed, nothing re-rendered — which also means
+                    // NO `stream_restart` and no rollback for the caller, because the output already
+                    // on screen remains valid.
+                    //
+                    // Requires having seen at least one numbered frame; without a cursor there is
+                    // nothing to resume from and a fresh attempt is correct anyway.
+                    if (serverResumable && lastEventId > 0) {
+                        resuming = true;
+                        reportRetry(err.message || 'Connection interrupted — resuming');
+                        await sleepWithinBudget(backoffMs(sAttempt));
+                        continue;
+                    }
+                    // Not resumable — fall back to restarting the whole turn. The dead attempt had
+                    // already rendered output, so the caller must throw that partial render away
+                    // BEFORE the replacement starts streaming or the re-sent text appears twice.
+                    // Emitted first so the UI is clean by the time "reconnecting…" goes up.
+                    if (e.needsRestart) {
+                        onEvent({
+                            type: 'stream_restart',
+                            reason: err.message || 'Connection interrupted',
+                            discardedChars: e.discardedChars ?? 0,
+                        });
+                    }
                     reportRetry(err.message || 'Connection interrupted — reconnecting');
-                    await sleep(backoffMs(sAttempt));
+                    await sleepWithinBudget(backoffMs(sAttempt));
                     continue;
                 }
+                // Out of retries (or out of time budget) — make the distinction explicit so
+                // "it just gave up" is never a mystery. A budget exhaustion means the network
+                // was down for the entire window, which is actionable in a way that a bare
+                // "Stream failed" is not.
+                if (e.retryable && !canRetry()) {
+                    throw Object.assign(new Error(`${err.message} — gave up after ${Math.round(MAX_TOTAL_RETRY_MS / 60000)} minutes of reconnect attempts. ` +
+                        `Check your connection and resend; completed work in this turn is preserved.`), { retryBudgetExhausted: true });
+                }
                 throw err;
             }
         }
@@ -512,6 +1066,31 @@ async function streamChat(messages, options, onEvent) {
             clearInterval(abortPoll);
     }
 }
+// ─── Cancel a turn ────────────────────────────────────────────────────────────
+/**
+ * Ask the server to stop generating a turn.
+ *
+ * This is REQUIRED for Stop to work, not an optimisation. A dropped socket used to abort
+ * the model; now it detaches and keeps generating so a network blip doesn't discard the
+ * turn (see the resume machinery above). Since a deliberate Stop looks identical to a
+ * blip from the server's side, it has to be signalled explicitly — otherwise the model
+ * would run to completion and bill in full after the user pressed Stop.
+ *
+ * Best-effort and never throws: Stop must feel instant, and the server's detach grace
+ * bounds the cost if this request never lands.
+ */
+async function cancelTurn(turnId) {
+    try {
+        await (0, node_fetch_1.default)(`${exports.API_BASE}/api/code/chat/cancel`, {
+            method: 'POST',
+            headers: { ...authHeaders(), 'Content-Type': 'application/json' },
+            body: JSON.stringify({ turnId }),
+        });
+    }
+    catch {
+        // Ignored deliberately — see above.
+    }
+}
 // ─── Get Balance ──────────────────────────────────────────────────────────────
 async function getBalance() {
     const response = await (0, node_fetch_1.default)(`${exports.API_BASE}/api/code/balance`, {