billion-context 0.1.88-pr.558.418 → 0.1.88-pr.571.415

package/README.md

@@ -112,6 +112,25 @@ turn is a semantic mismatch the model tolerates (it is clearly marked
   summaries (their tool call isn't in the agent's history and the agent's view
   skips `acp_summary`) — but that needs the id match above, which doesn't occur.
 
+### Preflight hold (#568)
+
+Preflight compression runs **before** the proxy sends any response bytes, and on
+big sessions it takes minutes (summarization round-trips to the same upstream).
+That exceeds client-side header timeouts (undici's default is 300s): the client
+aborts, the proxy drops the in-flight compression and never forwards — a
+repeating death loop every few minutes.
+
+So when preflight outlives a grace period (default **30s**, override with
+`BILI_PREFLIGHT_HOLD_MS`), the proxy commits the response early: streams get
+`200` + `text/event-stream` + SSE comment keep-alives (`: bili-preflight`, every
+15s); non-stream requests get `200` + `application/json` + whitespace padding.
+Early commits carry an `x-bili-preflight: compressing` header. Once the status
+is committed, late failures can no longer change it, so they are delivered
+**in-band** instead: a protocol error event (`preflight_compress_failed`) for
+streams, the identical structured error body under the already-sent 200 for
+JSON. Fast paths (preflight finishes within the grace) are unchanged and keep
+full status-code fidelity.
+
 ## Which do I need?
 
 Pick by your client:

package/dist/index.js

@@ -51147,11 +51147,6 @@ function estimateRawBodyTokens(parsed) {
   walk(parsed);
   return tokens;
 }
-function estimateCoreMessagesUpper(messages) {
-  let chars = 0;
-  for (const m2 of messages) chars += (m2.text ?? "").length;
-  return chars;
-}
 function rangeChars2(messages, startIdx, endIdx) {
   let chars = 0;
   for (let i = startIdx; i <= endIdx && i < messages.length; i++) {
@@ -51171,16 +51166,16 @@ ${text}`);
   }
   return parts.join("\n\n");
 }
-function splitChunks(messages, startIdx, endIdx, budget, minUnits, countText = defaultCountTokens) {
+function splitChunks(messages, startIdx, endIdx, budgetTokens) {
   const chunks = [];
   let cur = startIdx;
   while (cur <= endIdx) {
-    let total = 0;
+    let tokens = 0;
     let last = cur;
     for (let i = cur; i <= endIdx; i++) {
-      const t = countText(messages[i].text ?? "");
-      if (total + t > budget && i > cur && (minUnits <= 0 || total >= minUnits)) break;
-      total += t;
+      const t = defaultCountTokens(messages[i].text ?? "");
+      if (tokens + t > budgetTokens && i > cur) break;
+      tokens += t;
       last = i;
     }
     chunks.push([cur, last]);
@@ -51267,14 +51262,11 @@ function noEmergencyTruncate(config) {
 }
 async function preflightCompress(deps, messages) {
   const limit = deps.config.modelContextLimit;
-  const result = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0), fitsWindow: true };
+  const result = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0) };
   if (limit <= 0) return result;
   const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION));
   const minChars = deps.config.compress.minCompressRange;
-  const baselineKnown = deps.unknownBaseline !== true;
-  const countText = baselineKnown ? defaultCountTokens : (text) => text.length;
-  let currentTokens = baselineKnown ? deps.session.stats.lastInputTokens : estimateCoreMessagesUpper(messages);
-  let finalUpper = baselineKnown ? 0 : estimateCoreMessagesUpper(messages);
+  let currentTokens = deps.session.stats.lastInputTokens;
   let startTokens = -1;
   let failure;
   let activeConfig = deps.config;
@@ -51298,10 +51290,6 @@ async function preflightCompress(deps, messages) {
     });
     deps.session.state = turn.state;
     currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0));
-    if (!baselineKnown) {
-      finalUpper = estimateCoreMessagesUpper(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0);
-      currentTokens = Math.max(currentTokens, finalUpper);
-    }
     result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0);
     if (startTokens < 0) startTokens = currentTokens;
     if (currentTokens < limit) break;
@@ -51337,7 +51325,7 @@ async function preflightCompress(deps, messages) {
         continue;
       }
       rangesTried += 1;
-      for (const [cs2, ce2] of splitChunks(messages, startIdx, endIdx, budget, baselineKnown ? 0 : minChars, countText)) {
+      for (const [cs2, ce2] of splitChunks(messages, startIdx, endIdx, budget)) {
         if (currentTokens < limit) break;
         if (deps.signal?.aborted) {
           failure = ABORTED_FAILURE;
@@ -51396,8 +51384,7 @@ async function preflightCompress(deps, messages) {
           break;
         }
         const compressed = deps.session.stats.compressCreditTokens - creditBefore;
-        const folded = baselineKnown ? compressed : rangeChars2(messages, cs2, ce2);
-        currentTokens = Math.max(0, currentTokens - folded + countText(summary));
+        currentTokens = Math.max(0, currentTokens - compressed + defaultCountTokens(summary));
         deps.session.stats.lastInputTokens += defaultCountTokens(summary);
         appliedThisRound += 1;
         result.compressedRanges += 1;
@@ -51422,7 +51409,6 @@ async function preflightCompress(deps, messages) {
   if (result.compressedRanges > 0) deps.session.stats.lastInputTokens = currentTokens;
   result.savedTokens = Math.max(0, startTokens - currentTokens);
   if (currentTokens >= limit) result.failure = failure;
-  result.fitsWindow = baselineKnown ? result.payloadEstimate < limit : finalUpper < limit;
   return result;
 }
 
@@ -54717,6 +54703,35 @@ data: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn"
       safeWrite(res, `event: message_stop
 data: ${JSON.stringify({ type: "message_stop" })}
 
+`);
+    }
+  } catch {
+  } finally {
+    try {
+      res.end();
+    } catch {
+    }
+  }
+}
+function emitPreflightError(res, protocol, error, log2) {
+  const err2 = { type: "server_error", code: "preflight_compress_failed", message: error.message, retryable: error.retryable };
+  log2?.(`[acp-proxy: preflight failed after early response commit \u2014 delivering in-band: ${error.message}]`);
+  try {
+    if (protocol === "openai") {
+      safeWrite(res, `data: ${JSON.stringify({ error: err2 })}
+
+data: [DONE]
+
+`);
+    } else if (protocol === "responses") {
+      safeWrite(res, `event: error
+data: ${JSON.stringify({ type: "error", code: err2.code, message: err2.message })}
+
+`);
+    } else {
+      safeWrite(res, `event: error
+data: ${JSON.stringify({ type: "error", error: { type: "server_error", code: err2.code, message: err2.message } })}
+
 `);
     }
   } catch {
@@ -58356,24 +58371,41 @@ ${bodyBuffer.toString("utf8")}`);
             parsed.model,
             route,
             affinity,
-            anonAffinity !== null,
             log2,
             instanceId
           );
           if (isPreflightFailFast(outcome)) {
-            if (outcome.respond && !res.headersSent && !res.destroyed) {
-              res.writeHead(outcome.status, {
-                "content-type": "application/json",
-                ...outcome.status === 503 ? { "retry-after": "30" } : {}
-              });
-              res.end(JSON.stringify({
-                error: {
-                  type: "server_error",
-                  code: "preflight_compress_failed",
-                  message: outcome.message,
-                  retryable: outcome.retryable
+            if (outcome.respond && !res.destroyed) {
+              if (res.headersSent) {
+                if (prepared.stream) {
+                  emitPreflightError(res, prepared.protocol, { message: outcome.message, retryable: outcome.retryable }, (m2) => log2("warn", m2));
+                } else {
+                  try {
+                    res.end(JSON.stringify({
+                      error: {
+                        type: "server_error",
+                        code: "preflight_compress_failed",
+                        message: outcome.message,
+                        retryable: outcome.retryable
+                      }
+                    }));
+                  } catch {
+                  }
                 }
-              }));
+              } else {
+                res.writeHead(outcome.status, {
+                  "content-type": "application/json",
+                  ...outcome.status === 503 ? { "retry-after": "30" } : {}
+                });
+                res.end(JSON.stringify({
+                  error: {
+                    type: "server_error",
+                    code: "preflight_compress_failed",
+                    message: outcome.message,
+                    retryable: outcome.retryable
+                  }
+                }));
+              }
             }
             return;
           }
@@ -58492,11 +58524,6 @@ function armHostUsageCredit(session, originalMessages, processedMessages, log2)
     log2("info", `[${session.id}] host usage backfill armed: +${session.hostCreditTokens} tok (forwarded view is folded); host usage will report the uncompressed baseline`);
   }
 }
-function effectiveTokenCount(session, msgs) {
-  if (session.stats.lastInputTokens > 0) return session.stats.lastInputTokens;
-  if (!session.metadata.anonymousPrefixAffinity) return 0;
-  return estimateCoreMessagesUpper(msgs);
-}
 function prepareAnthropic(parsed, req, opts, core, config, prompts, log2, session, pluginMode) {
   const sessionId = session.id;
   const stream2 = parsed.stream === true;
@@ -58521,7 +58548,7 @@ function prepareAnthropic(parsed, req, opts, core, config, prompts, log2, sessio
     const { msgs, cacheControls } = anthropicToCore(parsed);
     originalMessages = msgs;
     extractSystem(parsed.system);
-    const tokenCount = effectiveTokenCount(session, msgs);
+    const tokenCount = session.stats.lastInputTokens;
     const activeBefore = new Set(session.state.blocks.filter((b2) => b2.active).map((b2) => b2.blockId));
     const turn = core.processTurn({ messages: msgs, state: session.state, config, tokenCount, renderTags: "text-only" });
     session.state = turn.state;
@@ -58642,7 +58669,7 @@ function prepareOpenai(parsed, req, opts, core, config, prompts, log2, session,
     const { msgs, systemText } = openaiToCore(parsed);
     openaiSystemText = systemText;
     originalMessages = msgs;
-    const tokenCount = effectiveTokenCount(session, msgs);
+    const tokenCount = session.stats.lastInputTokens;
     const activeBefore = new Set(session.state.blocks.filter((b2) => b2.active).map((b2) => b2.blockId));
     const turn = core.processTurn({ messages: msgs, state: session.state, config, tokenCount, renderTags: "text-only" });
     session.state = turn.state;
@@ -58747,7 +58774,7 @@ function prepareResponses(parsed, req, opts, core, config, prompts, log2, sessio
     if (process.env.ACP_DEBUG) {
       log2("info", `[${sessionId}] input items: ${Array.isArray(parsed.input) ? parsed.input.map((i) => i.type).join(",") : "(string)"}`);
     }
-    const tokenCount = effectiveTokenCount(session, msgs);
+    const tokenCount = session.stats.lastInputTokens;
     const turn = core.processTurn({ messages: msgs, state: session.state, config, tokenCount, renderTags });
     session.state = turn.state;
     session.stats.compressCreditTokens = 0;
@@ -59045,15 +59072,54 @@ function buildForwardTarget(req, opts, route, affinity, hopMarker) {
 function isPreflightFailFast(outcome) {
   return "failFast" in outcome;
 }
-async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, core, config, model, route, affinity, anonymous, log2, instanceId) {
+var PREFLIGHT_HOLD_GRACE_DEFAULT_MS = 3e4;
+var PREFLIGHT_KEEPALIVE_MS = 15e3;
+function preflightHoldGraceMs() {
+  const raw = process.env.BILI_PREFLIGHT_HOLD_MS;
+  if (!raw) return PREFLIGHT_HOLD_GRACE_DEFAULT_MS;
+  const v2 = Number(raw);
+  return Number.isFinite(v2) && v2 >= 0 ? Math.floor(v2) : PREFLIGHT_HOLD_GRACE_DEFAULT_MS;
+}
+function beginPreflightHold(res, prepared, log2) {
+  if (res.headersSent || res.destroyed || res.writableEnded) return void 0;
+  const sid = prepared.session.id;
+  const keepAlive = prepared.stream ? ": bili-preflight\n\n" : " ";
+  try {
+    if (prepared.stream) {
+      res.writeHead(200, {
+        "content-type": "text/event-stream",
+        "cache-control": "no-cache",
+        "x-accel-buffering": "no",
+        "x-bili-preflight": "compressing"
+      });
+    } else {
+      res.writeHead(200, { "content-type": "application/json", "x-bili-preflight": "compressing" });
+    }
+  } catch {
+    return void 0;
+  }
+  log2("info", `[${sid}] preflight still running after ${preflightHoldGraceMs()}ms grace \u2014 committed early ${prepared.stream ? "SSE" : "JSON"} headers + keep-alive to hold the client (#568)`);
+  try {
+    res.write(keepAlive);
+  } catch {
+  }
+  const iv = setInterval(() => {
+    try {
+      res.write(keepAlive);
+    } catch {
+      clearInterval(iv);
+    }
+  }, PREFLIGHT_KEEPALIVE_MS);
+  return () => clearInterval(iv);
+}
+async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, core, config, model, route, affinity, log2, instanceId) {
   const session = prepared.session;
   const limit = config.modelContextLimit;
   const imageTokens = imageTokensInRawBody(prepared.protocol, prepared.body);
   const textEstimate = estimateCoreMessages(prepared.processedMessages);
   const overheadEstimate = estimateWireOverhead(prepared.protocol, prepared.body);
   const payloadEstimate = textEstimate + overheadEstimate + imageTokens;
-  const unknownBaseline = anonymous && session.stats.lastInputTokens <= 0;
-  const tokenCount = unknownBaseline ? estimateCoreMessagesUpper(prepared.processedMessages) + overheadEstimate + imageTokens : Math.max(session.stats.lastInputTokens, payloadEstimate);
+  const tokenCount = Math.max(session.stats.lastInputTokens, payloadEstimate);
   if (limit <= 0 || !model || tokenCount < limit) return prepared;
   const learnedMap = session.metadata.learnedContextLimits;
   const learnedLimit = (model ? learnedMap?.[model] : void 0) ?? session.metadata.learnedContextLimit;
@@ -59068,14 +59134,9 @@ async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, c
     log2("error", `[${session.id}] preflight fail-fast ${status2} (retryable=${retryable}): ${message}`);
     return { failFast: true, status: status2, message, retryable, respond: !res.writableEnded };
   };
-  if ((prepared.nudge?.compressibleRanges ?? []).length === 0) {
-    if (!unknownBaseline && payloadEstimate < limit) {
-      log2("warn", `[${session.id}] preflight trigger fired on a stale baseline (~${tokenCount}) but the payload fits (~${payloadEstimate}/${limit}); forwarding as-is`);
-      return prepared;
-    }
-    if (unknownBaseline) {
-      return failFast(502, "no part of the conversation is compressible (nothing left to fold)", false);
-    }
+  if ((prepared.nudge?.compressibleRanges ?? []).length === 0 && payloadEstimate < limit) {
+    log2("warn", `[${session.id}] preflight trigger fired on a stale baseline (~${tokenCount}) but the payload fits (~${payloadEstimate}/${limit}); forwarding as-is`);
+    return prepared;
   }
   log2("warn", `[${session.id}] context ${tokenCount} tokens exceeds model window ${limit} (model=${model}); preflight compressing before forward`);
   const { upstreamUrl, headers, proxyUrl } = buildForwardTarget(req, opts, route, affinity, instanceId);
@@ -59084,32 +59145,41 @@ async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, c
     if (!res.writableEnded) clientAbort.abort();
   });
   const started = Date.now();
-  const result = await preflightCompress(
-    {
-      core,
-      session,
-      config,
-      prompts: prepared.prompts ?? defaultPrompts,
-      protocol: prepared.protocol,
-      url: upstreamUrl,
-      headers,
-      model,
-      proxyUrl,
-      signal: clientAbort.signal,
-      log: log2,
-      imageFloor: imageTokens,
-      wireOverhead: overheadEstimate,
-      unknownBaseline
-    },
-    prepared.originalMessages
-  );
+  let stopHold;
+  const holdTimer = setTimeout(() => {
+    stopHold = beginPreflightHold(res, prepared, log2);
+  }, preflightHoldGraceMs());
+  holdTimer.unref();
+  let result;
+  try {
+    result = await preflightCompress(
+      {
+        core,
+        session,
+        config,
+        prompts: prepared.prompts ?? defaultPrompts,
+        protocol: prepared.protocol,
+        url: upstreamUrl,
+        headers,
+        model,
+        proxyUrl,
+        signal: clientAbort.signal,
+        log: log2,
+        imageFloor: imageTokens,
+        wireOverhead: overheadEstimate
+      },
+      prepared.originalMessages
+    );
+  } finally {
+    clearTimeout(holdTimer);
+    stopHold?.();
+  }
   if (result.compressedRanges > 0) {
     log2("info", `[${session.id}] preflight compressed ${result.compressedRanges} range(s), ~${result.savedTokens} tokens saved (${tokenCount} \u2192 ${session.stats.lastInputTokens}) in ${Date.now() - started}ms; rebuilding payload`);
     const rebuilt = runPrepare();
     session.stats.requests -= 1;
-    const fits = unknownBaseline ? result.fitsWindow : estimateCoreMessages(rebuilt.processedMessages) + overheadEstimate + imageTokens < limit;
-    if (fits) return rebuilt;
-  } else if (unknownBaseline ? result.fitsWindow : estimateCoreMessages(prepared.processedMessages) + overheadEstimate + imageTokens < limit) {
+    if (estimateCoreMessages(rebuilt.processedMessages) + overheadEstimate + imageTokens < limit) return rebuilt;
+  } else if (estimateCoreMessages(prepared.processedMessages) + overheadEstimate + imageTokens < limit) {
     log2("warn", `[${session.id}] preflight made no progress but the payload fits; forwarding as-is`);
     return prepared;
   }
@@ -59124,7 +59194,7 @@ async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, c
 async function forward(req, res, opts, body, prepared, core, config, log2, route, instanceId, affinity) {
   if (prepared?.codexForge) {
     log2("info", `[${prepared.session.id}] codex compact served locally (${prepared.codexForge.kind}); upstream not contacted`);
-    res.writeHead(200, { "content-type": prepared.codexForge.contentType });
+    if (!res.headersSent) res.writeHead(200, { "content-type": prepared.codexForge.contentType });
     res.end(prepared.codexForge.body);
     return;
   }
@@ -59377,6 +59447,18 @@ ${hdrText}
     if (bodyText.length > 600) snippet += " \u2026";
     if (!snippet) snippet = "(no body)";
     log("warn", `[${errSid}] \u2190 upstream ${upstream.status}${reqIdText}: ${snippet}`);
+    if (res.headersSent) {
+      if (prepared?.stream) {
+        emitStreamError(res, prepared.protocol, `upstream HTTP ${upstream.status}: ${snippet}`, (m2) => log("info", m2));
+      } else {
+        try {
+          res.end(errBody ?? void 0);
+        } catch {
+        }
+      }
+      clearUpstreamTimer();
+      return;
+    }
     const errHeaders = { ...respHeaders };
     delete errHeaders["content-length"];
     delete errHeaders["transfer-encoding"];
@@ -59385,7 +59467,7 @@ ${hdrText}
     clearUpstreamTimer();
     return;
   }
-  res.writeHead(upstream.status, respHeaders);
+  if (!res.headersSent) res.writeHead(upstream.status, respHeaders);
   if (!upstream.body) {
     res.end();
     clearUpstreamTimer();