@askalf/dario 6.0.53 → 6.2.0

package/dist/proxy.d.ts

@@ -317,6 +317,15 @@ interface ProxyOptions {
     sessionStartJitterMs?: number;
     stealth?: boolean;
     drainOnClose?: boolean;
+    /**
+     * Finish a streamed answer that dies mid-way from the other subscription
+     * (v6.1, src/midstream.ts). On by default: it only ever acts where the
+     * alternative is a truncated stream, and the resume runs through dario's
+     * own front door at whatever `--pool-fallback` names for the other
+     * provider. `--no-midstream-continue` / `DARIO_MIDSTREAM_CONTINUE=0` turns
+     * it off; with no fallback chain it is inert and says so on the first miss.
+     */
+    midstreamContinue?: boolean;
     sessionIdleRotateMs?: number;
     sessionRotateJitterMs?: number;
     sessionMaxAgeMs?: number;

package/dist/proxy.js

@@ -26,6 +26,7 @@ import { createTokenBucket } from './rate-limit.js';
 import { getOpenAIBackend, isOpenAIModel, forwardToOpenAI } from './openai-backend.js';
 import { forwardToCodex, getCodexModelSlugs, peekCodexModelSlugs, isCodexModel, pickCodexFallback, pickClaudeTarget, CODEX_BACKEND_BASE_URL } from './codex-backend.js';
 import { effortForCodex } from './effort.js';
+import { MidstreamGuard, guardFor, loopbackBaseFor, CONTINUATION_HEADER, MAX_CONTINUATION_DEPTH, continuationDepth } from './midstream.js';
 import { isClaudeServableModel } from './claude-model.js';
 import { MODEL_UNROUTABLE } from './upstream-rejection.js';
 import { readCompareTarget, teeResponse, runCompare, writeCompareRecord, COMPARE_RESULT_HEADER } from './compare.js';
@@ -1834,6 +1835,12 @@ export async function startProxy(opts = {}) {
     // src/stream-drain.ts for the rationale + tradeoff.
     const { decideOnClientClose, resolveDrainOnClose, waitForClientDrain } = await import('./stream-drain.js');
     const drainOnClose = resolveDrainOnClose(opts.drainOnClose);
+    const midstreamContinue = opts.midstreamContinue !== false;
+    const loopbackBase = loopbackBaseFor(host, port);
+    if (!midstreamContinue)
+        console.log('[dario] mid-stream continuation: disabled (--no-midstream-continue)');
+    else if (verbose)
+        console.log(`[dario] mid-stream continuation: enabled (resume via ${loopbackBase}, target = the other provider's --pool-fallback entry)`);
     if (verbose) {
         console.log(`[dario] drain-on-close: ${drainOnClose ? 'enabled' : 'disabled'}`);
     }
@@ -2625,8 +2632,20 @@ export async function startProxy(opts = {}) {
         // replaces the v3.30.x-and-earlier unbounded semaphore — dario#80. A
         // queue-full condition returns an explicit 429 with a `"queue-full"`
         // marker in the body; a queue-timeout returns 504 with `"queue-timeout"`.
+        // Released in the handler's finally — or EARLY by a mid-stream
+        // continuation, whose loopback request needs the slot this request no
+        // longer uses once its upstream is dead (a one-slot proxy would otherwise
+        // wait on itself until the queue timeout).
+        let queueSlotHeld = false;
+        const releaseQueueSlot = () => {
+            if (!queueSlotHeld)
+                return;
+            queueSlotHeld = false;
+            queue.release(consumerFromHeaders);
+        };
         try {
             await queue.acquire(consumerFromHeaders);
+            queueSlotHeld = true;
         }
         catch (err) {
             if (err instanceof QueueFullError) {
@@ -2928,6 +2947,75 @@ export async function startProxy(opts = {}) {
                 clearTimeout(bodyTimeout);
             }
             let body = Buffer.concat(chunks);
+            // The request exactly as the client sent it. `body` is rewritten below
+            // (aliases, prefixes, the CC template); a mid-stream continuation
+            // re-issues the CLIENT's request, not the rewritten one, so dario's own
+            // rules apply to the resume the same way they applied to the original.
+            const clientBodyBytes = body;
+            // How deep in a continuation chain this request sits: 0 for a client
+            // request, 1 for its resume, 2 for the resume of that resume — which is
+            // never continued itself (MAX_CONTINUATION_DEPTH).
+            const requestDepth = continuationDepth(req.headers[CONTINUATION_HEADER]);
+            const isContinuation = requestDepth >= MAX_CONTINUATION_DEPTH;
+            /**
+             * First hop: the SAME model again, through the front door. The pool
+             * picks a seat (sticky binding keeps the prompt cache warm), and if the
+             * provider cannot take it at all the existing pre-byte failover already
+             * hands it to the other one. Null when the client named no model.
+             */
+            const sameModelTarget = () => {
+                const m = parseClientBody()?.model;
+                return typeof m === 'string' && m.length > 0 ? { model: m, label: `${m} (same model)` } : null;
+            };
+            const loopbackHeaders = () => {
+                const h = {};
+                if (apiKey)
+                    h['x-api-key'] = apiKey;
+                const c = req.headers[CONSUMER_HEADER];
+                if (typeof c === 'string' && c.length > 0)
+                    h[CONSUMER_HEADER] = c;
+                return h;
+            };
+            const parseClientBody = () => {
+                try {
+                    const v = JSON.parse(clientBodyBytes.toString('utf-8'));
+                    return v !== null && typeof v === 'object' && !Array.isArray(v) ? v : null;
+                }
+                catch {
+                    return null;
+                }
+            };
+            /**
+             * Where a Claude stream that died mid-way resumes: the codex half of the
+             * `--pool-fallback` chain, exactly the entry a mid-flight 429 would use.
+             * Resolved at failure time — an account that was cooling or missing at
+             * selection may be fine now.
+             */
+            const codexContinuationTarget = async () => {
+                const models = selectPoolFallbackForBody(body);
+                if (models.length === 0)
+                    return null;
+                if (!(await hasAnyCodexAccount().catch(() => false)))
+                    return null;
+                const stored = await selectCodexAccount().catch(() => null);
+                if (!stored)
+                    return null;
+                let creds;
+                try {
+                    creds = await getFreshCodexAccount(stored);
+                }
+                catch {
+                    return null;
+                }
+                const slugs = await getCodexModelSlugs(creds).catch(() => []);
+                const pick = pickCodexFallback(models, slugs);
+                if (!pick)
+                    return null;
+                return {
+                    model: `codex:${pick.model}${pick.effort ? `:${pick.effort}` : ''}`,
+                    label: `${pick.model} (codex ${creds.alias})`,
+                };
+            };
             // A body that is not a JSON object cannot be routed — every decision
             // below (alias, provider prefix, codex slug, template) peeks at `.model`
             // and each peek swallows its parse error and falls through. So `{` used
@@ -3285,6 +3373,30 @@ export async function startProxy(opts = {}) {
                             return;
                         }
                         const codexReq = requestCount;
+                        // A codex stream that dies mid-way resumes on the Claude half of
+                        // the chain — the same target a declined codex request defers to.
+                        const claudeContinuation = claudeTarget && pool.size > 0 && !upstreamApiKey
+                            ? { model: `claude:${claudeTarget.model}${claudeTarget.effort ? `:${claudeTarget.effort}` : ''}`, label: `${claudeTarget.model} (claude pool)` }
+                            : null;
+                        const codexGuard = midstreamContinue && !isContinuation
+                            ? guardFor(res, {
+                                shape: isOpenAI ? 'openai' : 'anthropic',
+                                write: (chunk) => { if (!res.destroyed)
+                                    res.write(chunk); },
+                                isClientGone: () => res.destroyed || res.writableEnded,
+                                requestNo: codexReq,
+                                depth: requestDepth,
+                                verbose,
+                                resume: {
+                                    clientBody: parseClientBody,
+                                    loopbackBase,
+                                    loopbackHeaders: loopbackHeaders(),
+                                    resolveTarget: async (hop) => hop === 1 ? sameModelTarget() : claudeContinuation,
+                                    onBeforeResume: releaseQueueSlot,
+                                    timeoutMs: upstreamTimeoutMs,
+                                },
+                            })
+                            : null;
                         const served = codexAvailable && await forwardToCodex(req, res, body, codexCreds, corsOrigin, SECURITY_HEADERS, upstreamTimeoutMs, verbose, isOpenAI ? 'openai' : 'anthropic', fetch, canDefer, 
                         // Before this hook a codex request left no trace: nothing in
                         // /analytics, nothing in the request log, no per-account count.
@@ -3330,7 +3442,7 @@ export async function startProxy(opts = {}) {
                         // dario#1260 — the effort named by the model-name suffix stripped
                         // above. Undefined for every request that did not name one, which
                         // leaves the outbound body exactly as it was.
-                        effortForCodex(requestEffort));
+                        effortForCodex(requestEffort), codexGuard);
                         if (served) {
                             // A provider that just served is not rate-limited.
                             providerCooldowns.clear('codex');
@@ -4587,12 +4699,37 @@ export async function startProxy(opts = {}) {
                 // false the client's socket buffer is full, so pause the read loop until
                 // it drains instead of buffering the whole (fast) upstream in memory.
                 let needsDrain = false;
-                const writeToClient = (chunk) => {
+                const writeToClientRaw = (chunk) => {
                     if (clientDisconnected)
                         return;
                     if (res.write(chunk) === false)
                         needsDrain = true;
                 };
+                // Mid-stream continuation (v6.1): every client-bound frame passes
+                // through the guard so a stream that dies with content on the wire can
+                // be finished from the other subscription instead of truncated. Only a
+                // 2xx stream is guarded — an upstream error body is not a message.
+                const guard = midstreamContinue && !isContinuation && upstream.status >= 200 && upstream.status < 300
+                    ? new MidstreamGuard({
+                        shape: isOpenAI ? 'openai' : 'anthropic',
+                        write: writeToClientRaw,
+                        end: () => { if (!res.writableEnded)
+                            res.end(); },
+                        isClientGone: () => clientDisconnected || res.destroyed || upstreamAbortReason === 'client_closed' || upstreamAbortReason === 'sse_overflow',
+                        requestNo: requestCount,
+                        depth: requestDepth,
+                        verbose,
+                        resume: {
+                            clientBody: parseClientBody,
+                            loopbackBase,
+                            loopbackHeaders: loopbackHeaders(),
+                            resolveTarget: async (hop) => hop === 1 ? sameModelTarget() : codexContinuationTarget(),
+                            onBeforeResume: releaseQueueSlot,
+                            timeoutMs: upstreamTimeoutMs,
+                        },
+                    })
+                    : null;
+                const writeToClient = guard ? (chunk) => guard.write(chunk) : writeToClientRaw;
                 // Resolves on 'close' (vanished client) AND on upstream abort. The
                 // abort arm is what keeps a connected-but-not-reading client from
                 // parking this handler forever and leaking its queue slot — the
@@ -4712,7 +4849,10 @@ export async function startProxy(opts = {}) {
                     if (!upstreamAbort.signal.aborted)
                         upstreamAbort.abort();
                 }
-                res.end();
+                if (guard)
+                    await guard.finish();
+                else
+                    res.end();
                 // Stamp the response-completion timestamp + token count so the
                 // next request's think-time delay can model human read time.
                 // Only on 2xx — error responses don't represent content the user
@@ -4875,7 +5015,7 @@ export async function startProxy(opts = {}) {
                 clearTimeout(upstreamTimeout);
             if (onClientClose !== null)
                 req.off('close', onClientClose);
-            queue.release(consumerFromHeaders);
+            releaseQueueSlot();
         }
     });
     server.on('error', async (err) => {

package/docs/midstream-continuation.md

@@ -0,0 +1,155 @@
+# Mid-stream continuation
+
+The answer does not stop when the plan does.
+
+A streamed answer that dies part-way through used to end the way the transport
+ended it: the upstream socket resets, Anthropic sends an in-band
+`overloaded_error`, the codex backend answers `response.failed`, and the client
+gets a stream with no `message_stop` and no `[DONE]`. An SDK throws "stream
+ended without producing a Message"; a chat UI shows half a paragraph and a
+spinner; an agent loses the tool call it was two tokens away from. Every word
+already on screen is wasted, and the failover chain never fires, because once
+bytes were on the wire the request was treated as too late to hand to anyone
+else.
+
+Since 6.1 dario finishes the same client stream from the other subscription.
+
+```
+client ──▶ dario ──▶ Claude pool   ✗ dies after 1,240 chars
+                 └─▶ ChatGPT plan  ✓ resumes at char 1,241, same stream
+```
+
+The client sees one message: the same `message_start`, the same content block
+still open, then the rest of the text, then a clean close. There is an SSE
+comment at the seam (`: dario continuation gpt-5.6-sol (codex live) after 1240
+chars`) that every parser ignores and every raw capture shows.
+
+## When it fires
+
+Only on a stream that is already partly delivered and then breaks — never on a
+request that failed before its first byte (the existing failover covers those),
+never on a stream that finished. Concretely, on `/v1/messages` and
+`/v1/chat/completions` with `stream: true`, when a 2xx stream ends without its
+terminal event, or carries an `error` event after content, or the codex
+translator flags a failed turn:
+
+| cut fell inside | what happens |
+|---|---|
+| a text block | the resume continues **that block** — no new `content_block_start` |
+| thinking, or before any block | the open block is closed and the resume starts a fresh text block |
+| between blocks | the resume starts a fresh text block |
+| a `tool_use` block, or an OpenAI `tool_calls` delta | **not continued** — half a JSON argument is not resumable; the stream ends as before |
+
+Non-streaming requests are untouched; nothing was on the wire.
+
+## Where it resumes
+
+Through dario's own front door. The resume is a loopback `POST` to the same
+proxy, so the pool, the codex translator, cch, the template, every rule that
+applied to the original request applies to the resume.
+
+Two choices, in order:
+
+1. **The same model again.** A fresh request for the model the client asked
+   for. The pool picks a seat — the sticky binding keeps the prompt cache warm
+   — and if the provider cannot take the request at all, the existing pre-byte
+   failover already hands it to the other one. A transient reset therefore
+   finishes on the model the user chose, with no chain configured. This is
+   what most streams that die get, and most users have one plan.
+2. **The other provider's entry in `--pool-fallback`**, exactly the entry a
+   mid-flight 429 would use: a Claude stream goes to the codex half of the
+   chain (`gpt-5.6-sol` in `--pool-fallback=gpt-5.6-sol,claude:claude-sonnet-5`),
+   resolved at failure time against the account's live model list; a codex
+   stream goes to the Claude half, resolved against the live catalog.
+
+Choice 2 is taken when choice 1 delivers nothing — refused, unreachable, dead
+before its first byte — or when the resume itself dies mid-way. In the second
+case the resume's own guard makes the hop, so the client stream carries two
+seams: `(same model)` then `(codex live)`. The loopback carries
+`x-dario-continuation: <depth>`; a request at depth 2 is never continued. Two
+hops, never three: a third would be a third attempt at whatever is failing.
+
+With only one plan and no chain, a stream whose same-model resume also fails
+ends where the resume stopped, and the log says why once:
+
+```
+[dario] #42 continuation as claude-opus-5 (same model) delivered nothing — trying the next choice
+[dario] #42 stream died after 1240 chars — no continuation target (set --pool-fallback with an entry for the other provider)
+```
+
+The request's queue slot is released before the loopback is made, so a
+`--max-concurrent=1` proxy resumes instead of waiting on itself.
+
+## The seam
+
+The resume is the client's own request with two turns appended: the partial
+answer as the assistant turn, and a user turn asking for the rest. There is
+**no assistant prefill** — Claude 4.6+ answers a trailing assistant turn with a
+400, and the Responses API never had the concept — so the resume is
+instruction-driven on both providers.
+
+The user turn is written as the person whose connection dropped, not as an
+operator notice. The first live run is why: told `[transport notice] … resume
+now`, claude-sonnet-5 answered that the notice "isn't an actual system message
+— it's just text in your prompt" and stopped, which is the injection-awareness
+it should have. "My connection dropped while you were writing that reply, so I
+only received it up to this point: «…». Please pick up exactly where you left
+off" is an ordinary request and gets the ordinary answer.
+
+It asks the model to begin by repeating, verbatim, the last ~40 characters of
+the cut-off text, then continue. dario holds the first ~240
+characters of the resume, finds that repeat with a whitespace- and
+quote-normalized match, cuts it, and streams everything after it. The model
+renders the seam — the space between two words, the four-space indent, the
+second half of a split word — inside its own token stream, and dario only trims.
+Told merely to "continue", a model drops the boundary whitespace often enough
+to notice (`replies<cut>with`); told to repeat the anchor, it does not.
+
+One rule on top: prose cut mid-sentence whose continuation opens with a
+paragraph break gets one space instead. Inside a code fence a newline is
+content and is left alone.
+
+If the model does not repeat the anchor, the longest exact overlap between the
+partial's tail and the resume's head is trimmed; if there is none, nothing is.
+
+## What it costs
+
+One extra request on the other subscription, carrying the whole conversation
+plus the partial. The original request is logged and counted as it was (a
+502 on the codex path, a truncated 200 on the Claude path); the resume is
+logged as its own request. A resume that fails before producing anything hands
+the stream back to end as it would have. A resume that fails after producing
+something forwards its error frame if it sent one, and otherwise simply stops —
+whatever it had already written is on the wire, and the stream is left without
+its terminal event so the client sees the truncation. A twice-truncated answer
+is never closed with a synthetic `end_turn`; only the resume's own
+`message_stop` / `[DONE]` finishes the message.
+
+## Switches
+
+| | |
+|---|---|
+| `--no-midstream-continue` | off for this proxy |
+| `DARIO_MIDSTREAM_CONTINUE=0` | same, for the container |
+| `--pool-fallback=…` | where the second hop goes; without an entry for the other provider a stream gets the same-model resume only |
+
+On by default: it only ever acts where the alternative is a broken stream.
+
+## How it was proven
+
+`test/midstream-continuation-wiring.mjs` runs a real proxy against a fake
+Anthropic upstream and a codex stub, kills the stream in every way listed
+above, and replays what the client received through a strict grammar check of
+both wire shapes. Before that, the same splice ran outside the proxy against
+production dario 6.0.51 with real Opus 5 and a real ChatGPT Plus account, both
+directions, prose and code: ten runs, zero restarts, zero preamble, zero
+repetition, anchor matched exactly five times out of five, and the official
+`@anthropic-ai/sdk` accepted every spliced stream as one message. The seam,
+verbatim, from one of them:
+
+```
+…concatenation of one or more unit components in the
+    order hours, min<CUT>utes, and seconds. At least one component is required.
+```
+
+Claude wrote the left half, GPT the right.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@askalf/dario",
-  "version": "6.0.53",
+  "version": "6.2.0",
   "description": "Use your Claude and ChatGPT subscriptions in Cursor, Cline, Aider, Claude Code and the Agent SDK — at subscription pricing, not per-token API bills. One local Anthropic + OpenAI-compatible endpoint: either plan answers either wire shape, with automatic failover when one hits its limit.",
   "type": "module",
   "bin": {