billion-context 0.1.87-pr.591.413 → 0.1.88-pr.558.418

package/dist/index.js

@@ -51107,6 +51107,7 @@ var CHUNK_FRACTION = 0.6;
 var MIN_CHUNK_TOKENS = 2e3;
 var MIN_SUMMARY_CHARS = 50;
 var MAX_SUMMARY_OUTPUT_TOKENS = 8192;
+var MAX_SUMMARY_CALLS_PER_PREFLIGHT = 8;
 function refMaps(messages, state) {
   const refToIdx = /* @__PURE__ */ new Map();
   const idxToRef = /* @__PURE__ */ new Map();
@@ -51146,6 +51147,11 @@ function estimateRawBodyTokens(parsed) {
   walk(parsed);
   return tokens;
 }
+function estimateCoreMessagesUpper(messages) {
+  let chars = 0;
+  for (const m2 of messages) chars += (m2.text ?? "").length;
+  return chars;
+}
 function rangeChars2(messages, startIdx, endIdx) {
   let chars = 0;
   for (let i = startIdx; i <= endIdx && i < messages.length; i++) {
@@ -51165,16 +51171,16 @@ ${text}`);
   }
   return parts.join("\n\n");
 }
-function splitChunks(messages, startIdx, endIdx, budgetTokens) {
+function splitChunks(messages, startIdx, endIdx, budget, minUnits, countText = defaultCountTokens) {
   const chunks = [];
   let cur = startIdx;
   while (cur <= endIdx) {
-    let tokens = 0;
+    let total = 0;
     let last = cur;
     for (let i = cur; i <= endIdx; i++) {
-      const t = defaultCountTokens(messages[i].text ?? "");
-      if (tokens + t > budgetTokens && i > cur) break;
-      tokens += t;
+      const t = countText(messages[i].text ?? "");
+      if (total + t > budget && i > cur && (minUnits <= 0 || total >= minUnits)) break;
+      total += t;
       last = i;
     }
     chunks.push([cur, last]);
@@ -51261,16 +51267,23 @@ function noEmergencyTruncate(config) {
 }
 async function preflightCompress(deps, messages) {
   const limit = deps.config.modelContextLimit;
-  const result = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0) };
+  const result = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0), fitsWindow: true };
   if (limit <= 0) return result;
   const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION));
   const minChars = deps.config.compress.minCompressRange;
-  let currentTokens = deps.session.stats.lastInputTokens;
+  const baselineKnown = deps.unknownBaseline !== true;
+  const countText = baselineKnown ? defaultCountTokens : (text) => text.length;
+  let currentTokens = baselineKnown ? deps.session.stats.lastInputTokens : estimateCoreMessagesUpper(messages);
+  let finalUpper = baselineKnown ? 0 : estimateCoreMessagesUpper(messages);
   let startTokens = -1;
   let failure;
   let activeConfig = deps.config;
   let relaxed = false;
   const relaxedExhaustedDetail = `the payload still exceeds the window after folding everything compressible, including the soft-protected recent zone (last ${deps.config.preserveRecentMessages} messages + most recent user message), which was relaxed under overflow; hard protectedTools remain excluded. Raise the model context window or restart the session to recover.`;
+  const skipSet = /* @__PURE__ */ new Set();
+  let summaryCalls = 0;
+  let budgetHit = false;
+  let rangesTried = 0;
   for (let round = 0; round < MAX_PREFLIGHT_ROUNDS; round++) {
     if (deps.signal?.aborted) {
       failure = ABORTED_FAILURE;
@@ -51285,6 +51298,10 @@ async function preflightCompress(deps, messages) {
     });
     deps.session.state = turn.state;
     currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0));
+    if (!baselineKnown) {
+      finalUpper = estimateCoreMessagesUpper(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0);
+      currentTokens = Math.max(currentTokens, finalUpper);
+    }
     result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0) + (deps.wireOverhead ?? 0);
     if (startTokens < 0) startTokens = currentTokens;
     if (currentTokens < limit) break;
@@ -51293,90 +51310,119 @@ async function preflightCompress(deps, messages) {
       if (!relaxed && result.payloadEstimate >= limit) {
         activeConfig = relaxedConfig(deps.config);
         relaxed = true;
+        summaryCalls = 0;
+        budgetHit = false;
         deps.log("warn", "[preflight] no compressible ranges outside the protected recent zone; relaxing soft protection (preserveRecentMessages/Tokens -> 0) and retrying");
         continue;
       }
       failure = { kind: "exhausted", detail: relaxed ? relaxedExhaustedDetail : "no compressible ranges remain in the conversation" };
       break;
     }
-    const range = [...ranges].sort((a, b2) => refNum3(a.startRef) - refNum3(b2.startRef))[0];
-    const { refToIdx } = refMaps(messages, deps.session.state);
-    const startIdx = refToIdx.get(range.startRef);
-    const endIdx = refToIdx.get(range.endRef);
-    if (startIdx === void 0 || endIdx === void 0 || startIdx > endIdx) {
-      failure = { kind: "exhausted", detail: "the compressible range no longer resolves to payload messages" };
-      break;
-    }
+    const ordered = [...ranges].sort((a, b2) => refNum3(a.startRef) - refNum3(b2.startRef));
     let appliedThisRound = 0;
-    for (const [cs2, ce2] of splitChunks(messages, startIdx, endIdx, budget)) {
+    for (const range of ordered) {
       if (currentTokens < limit) break;
       if (deps.signal?.aborted) {
         failure = ABORTED_FAILURE;
         break;
       }
-      const maps = refMaps(messages, deps.session.state);
-      const startRef = maps.idxToRef.get(cs2);
-      const endRef = maps.idxToRef.get(ce2);
-      if (!startRef || !endRef) continue;
-      if (rangeChars2(messages, cs2, ce2) < minChars) continue;
-      const content = renderRange(messages, cs2, ce2);
-      if (content.length === 0) continue;
-      let summary;
-      try {
-        summary = await summarizeRange(deps, content, startRef, endRef);
-      } catch (err2) {
-        if (err2 instanceof UpstreamHttpError) {
-          failure = {
-            kind: "upstream",
-            status: err2.status,
-            detail: err2.status === 429 ? `the summarization call was rate-limited by the upstream (HTTP 429)` : `the summarization call was rejected by the upstream (HTTP ${err2.status})`
-          };
-          deps.log("warn", `[preflight] summarization failed: HTTP ${err2.status} ${err2.body.slice(0, 200)}`);
-        } else if (deps.signal?.aborted) {
+      if (budgetHit) break;
+      const skipKey = `${range.startRef}:${range.endRef}`;
+      if (skipSet.has(skipKey)) continue;
+      const { refToIdx } = refMaps(messages, deps.session.state);
+      const startIdx = refToIdx.get(range.startRef);
+      const endIdx = refToIdx.get(range.endRef);
+      if (startIdx === void 0 || endIdx === void 0 || startIdx > endIdx) {
+        skipSet.add(skipKey);
+        continue;
+      }
+      rangesTried += 1;
+      for (const [cs2, ce2] of splitChunks(messages, startIdx, endIdx, budget, baselineKnown ? 0 : minChars, countText)) {
+        if (currentTokens < limit) break;
+        if (deps.signal?.aborted) {
           failure = ABORTED_FAILURE;
-          deps.log("warn", `[preflight] summarization aborted: client disconnected`);
-        } else {
-          failure = { kind: "upstream", detail: `the summarization call failed: ${String(err2)}` };
-          deps.log("warn", `[preflight] summarization failed: ${String(err2)}`);
+          break;
+        }
+        if (budgetHit) break;
+        const maps = refMaps(messages, deps.session.state);
+        const startRef = maps.idxToRef.get(cs2);
+        const endRef = maps.idxToRef.get(ce2);
+        if (!startRef || !endRef) continue;
+        if (rangeChars2(messages, cs2, ce2) < minChars) continue;
+        const content = renderRange(messages, cs2, ce2);
+        if (content.length === 0) continue;
+        if (summaryCalls >= MAX_SUMMARY_CALLS_PER_PREFLIGHT) {
+          budgetHit = true;
+          break;
+        }
+        summaryCalls += 1;
+        let summary;
+        try {
+          summary = await summarizeRange(deps, content, startRef, endRef);
+        } catch (err2) {
+          if (err2 instanceof UpstreamHttpError) {
+            failure = {
+              kind: "upstream",
+              status: err2.status,
+              detail: err2.status === 429 ? `the summarization call was rate-limited by the upstream (HTTP 429)` : `the summarization call was rejected by the upstream (HTTP ${err2.status})`
+            };
+            deps.log("warn", `[preflight] summarization failed: HTTP ${err2.status} ${err2.body.slice(0, 200)}`);
+          } else if (deps.signal?.aborted) {
+            failure = ABORTED_FAILURE;
+            deps.log("warn", `[preflight] summarization aborted: client disconnected`);
+          } else {
+            failure = { kind: "upstream", detail: `the summarization call failed: ${String(err2)}` };
+            deps.log("warn", `[preflight] summarization failed: ${String(err2)}`);
+          }
+          break;
+        }
+        if (!summary) {
+          deps.log("warn", `[preflight] range ${skipKey} produced no usable summary; skipping it`);
+          skipSet.add(skipKey);
+          break;
+        }
+        const ctx = {
+          core: deps.core,
+          config: activeConfig,
+          messages,
+          session: deps.session,
+          log: (msg) => deps.log("info", msg)
+        };
+        const creditBefore = deps.session.stats.compressCreditTokens;
+        const applied = applyRanges(parseCompressInput({ content: [{ startId: startRef, endId: endRef, summary, topic: "preflight overflow compress" }] }), ctx);
+        if (applied.startsWith("[Compression FAILED")) {
+          deps.log("warn", `[preflight] ${applied}`);
+          skipSet.add(skipKey);
+          break;
         }
+        const compressed = deps.session.stats.compressCreditTokens - creditBefore;
+        const folded = baselineKnown ? compressed : rangeChars2(messages, cs2, ce2);
+        currentTokens = Math.max(0, currentTokens - folded + countText(summary));
+        deps.session.stats.lastInputTokens += defaultCountTokens(summary);
+        appliedThisRound += 1;
+        result.compressedRanges += 1;
         break;
       }
-      if (!summary) continue;
-      const ctx = {
-        core: deps.core,
-        config: activeConfig,
-        messages,
-        session: deps.session,
-        log: (msg) => deps.log("info", msg)
-      };
-      const creditBefore = deps.session.stats.compressCreditTokens;
-      const applied = applyRanges(parseCompressInput({ content: [{ startId: startRef, endId: endRef, summary, topic: "preflight overflow compress" }] }), ctx);
-      if (applied.startsWith("[Compression FAILED")) {
-        deps.log("warn", `[preflight] ${applied}`);
-        continue;
-      }
-      const compressed = deps.session.stats.compressCreditTokens - creditBefore;
-      currentTokens = Math.max(0, currentTokens - compressed + defaultCountTokens(summary));
-      deps.session.stats.lastInputTokens += defaultCountTokens(summary);
-      appliedThisRound += 1;
-      result.compressedRanges += 1;
-    }
-    if (appliedThisRound === 0) {
-      if (!failure) {
-        failure = { kind: "exhausted", detail: "no range could be compressed (chunks below minCompressRange or the summarization responses were unusable)" };
-      }
-      break;
+      if (appliedThisRound > 0) break;
+      if (failure || budgetHit) break;
     }
+    if (appliedThisRound === 0) break;
   }
-  if (!failure && currentTokens >= limit) {
-    failure = {
-      kind: "exhausted",
-      detail: relaxed ? relaxedExhaustedDetail : `the compress budget was exhausted after ${MAX_PREFLIGHT_ROUNDS} rounds`
-    };
+  if (currentTokens >= limit && !failure) {
+    if (budgetHit) {
+      failure = { kind: "exhausted", detail: `the preflight summarization budget (${MAX_SUMMARY_CALLS_PER_PREFLIGHT} calls per protection regime) was exhausted before the payload fit the window` };
+    } else if (relaxed && result.compressedRanges > 0) {
+      failure = { kind: "exhausted", detail: relaxedExhaustedDetail };
+    } else if (result.compressedRanges === 0) {
+      failure = { kind: "exhausted", detail: `no range could be compressed across ${rangesTried} viable range${rangesTried === 1 ? "" : "s"} (each was below minCompressRange, had an unusable summary, or failed to apply)` };
+    } else {
+      failure = { kind: "exhausted", detail: `the compress budget was exhausted after ${MAX_PREFLIGHT_ROUNDS} rounds` };
+    }
   }
   if (result.compressedRanges > 0) deps.session.stats.lastInputTokens = currentTokens;
   result.savedTokens = Math.max(0, startTokens - currentTokens);
   if (currentTokens >= limit) result.failure = failure;
+  result.fitsWindow = baselineKnown ? result.payloadEstimate < limit : finalUpper < limit;
   return result;
 }
 
@@ -58310,6 +58356,7 @@ ${bodyBuffer.toString("utf8")}`);
             parsed.model,
             route,
             affinity,
+            anonAffinity !== null,
             log2,
             instanceId
           );
@@ -58445,6 +58492,11 @@ function armHostUsageCredit(session, originalMessages, processedMessages, log2)
     log2("info", `[${session.id}] host usage backfill armed: +${session.hostCreditTokens} tok (forwarded view is folded); host usage will report the uncompressed baseline`);
   }
 }
+function effectiveTokenCount(session, msgs) {
+  if (session.stats.lastInputTokens > 0) return session.stats.lastInputTokens;
+  if (!session.metadata.anonymousPrefixAffinity) return 0;
+  return estimateCoreMessagesUpper(msgs);
+}
 function prepareAnthropic(parsed, req, opts, core, config, prompts, log2, session, pluginMode) {
   const sessionId = session.id;
   const stream2 = parsed.stream === true;
@@ -58469,7 +58521,7 @@ function prepareAnthropic(parsed, req, opts, core, config, prompts, log2, sessio
     const { msgs, cacheControls } = anthropicToCore(parsed);
     originalMessages = msgs;
     extractSystem(parsed.system);
-    const tokenCount = session.stats.lastInputTokens;
+    const tokenCount = effectiveTokenCount(session, msgs);
     const activeBefore = new Set(session.state.blocks.filter((b2) => b2.active).map((b2) => b2.blockId));
     const turn = core.processTurn({ messages: msgs, state: session.state, config, tokenCount, renderTags: "text-only" });
     session.state = turn.state;
@@ -58590,7 +58642,7 @@ function prepareOpenai(parsed, req, opts, core, config, prompts, log2, session,
     const { msgs, systemText } = openaiToCore(parsed);
     openaiSystemText = systemText;
     originalMessages = msgs;
-    const tokenCount = session.stats.lastInputTokens;
+    const tokenCount = effectiveTokenCount(session, msgs);
     const activeBefore = new Set(session.state.blocks.filter((b2) => b2.active).map((b2) => b2.blockId));
     const turn = core.processTurn({ messages: msgs, state: session.state, config, tokenCount, renderTags: "text-only" });
     session.state = turn.state;
@@ -58695,7 +58747,7 @@ function prepareResponses(parsed, req, opts, core, config, prompts, log2, sessio
     if (process.env.ACP_DEBUG) {
       log2("info", `[${sessionId}] input items: ${Array.isArray(parsed.input) ? parsed.input.map((i) => i.type).join(",") : "(string)"}`);
     }
-    const tokenCount = session.stats.lastInputTokens;
+    const tokenCount = effectiveTokenCount(session, msgs);
     const turn = core.processTurn({ messages: msgs, state: session.state, config, tokenCount, renderTags });
     session.state = turn.state;
     session.stats.compressCreditTokens = 0;
@@ -58993,14 +59045,15 @@ function buildForwardTarget(req, opts, route, affinity, hopMarker) {
 function isPreflightFailFast(outcome) {
   return "failFast" in outcome;
 }
-async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, core, config, model, route, affinity, log2, instanceId) {
+async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, core, config, model, route, affinity, anonymous, log2, instanceId) {
   const session = prepared.session;
   const limit = config.modelContextLimit;
   const imageTokens = imageTokensInRawBody(prepared.protocol, prepared.body);
   const textEstimate = estimateCoreMessages(prepared.processedMessages);
   const overheadEstimate = estimateWireOverhead(prepared.protocol, prepared.body);
   const payloadEstimate = textEstimate + overheadEstimate + imageTokens;
-  const tokenCount = Math.max(session.stats.lastInputTokens, payloadEstimate);
+  const unknownBaseline = anonymous && session.stats.lastInputTokens <= 0;
+  const tokenCount = unknownBaseline ? estimateCoreMessagesUpper(prepared.processedMessages) + overheadEstimate + imageTokens : Math.max(session.stats.lastInputTokens, payloadEstimate);
   if (limit <= 0 || !model || tokenCount < limit) return prepared;
   const learnedMap = session.metadata.learnedContextLimits;
   const learnedLimit = (model ? learnedMap?.[model] : void 0) ?? session.metadata.learnedContextLimit;
@@ -59015,9 +59068,14 @@ async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, c
     log2("error", `[${session.id}] preflight fail-fast ${status2} (retryable=${retryable}): ${message}`);
     return { failFast: true, status: status2, message, retryable, respond: !res.writableEnded };
   };
-  if ((prepared.nudge?.compressibleRanges ?? []).length === 0 && payloadEstimate < limit) {
-    log2("warn", `[${session.id}] preflight trigger fired on a stale baseline (~${tokenCount}) but the payload fits (~${payloadEstimate}/${limit}); forwarding as-is`);
-    return prepared;
+  if ((prepared.nudge?.compressibleRanges ?? []).length === 0) {
+    if (!unknownBaseline && payloadEstimate < limit) {
+      log2("warn", `[${session.id}] preflight trigger fired on a stale baseline (~${tokenCount}) but the payload fits (~${payloadEstimate}/${limit}); forwarding as-is`);
+      return prepared;
+    }
+    if (unknownBaseline) {
+      return failFast(502, "no part of the conversation is compressible (nothing left to fold)", false);
+    }
   }
   log2("warn", `[${session.id}] context ${tokenCount} tokens exceeds model window ${limit} (model=${model}); preflight compressing before forward`);
   const { upstreamUrl, headers, proxyUrl } = buildForwardTarget(req, opts, route, affinity, instanceId);
@@ -59040,7 +59098,8 @@ async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, c
       signal: clientAbort.signal,
       log: log2,
       imageFloor: imageTokens,
-      wireOverhead: overheadEstimate
+      wireOverhead: overheadEstimate,
+      unknownBaseline
     },
     prepared.originalMessages
   );
@@ -59048,8 +59107,9 @@ async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, c
     log2("info", `[${session.id}] preflight compressed ${result.compressedRanges} range(s), ~${result.savedTokens} tokens saved (${tokenCount} \u2192 ${session.stats.lastInputTokens}) in ${Date.now() - started}ms; rebuilding payload`);
     const rebuilt = runPrepare();
     session.stats.requests -= 1;
-    if (estimateCoreMessages(rebuilt.processedMessages) + overheadEstimate + imageTokens < limit) return rebuilt;
-  } else if (estimateCoreMessages(prepared.processedMessages) + overheadEstimate + imageTokens < limit) {
+    const fits = unknownBaseline ? result.fitsWindow : estimateCoreMessages(rebuilt.processedMessages) + overheadEstimate + imageTokens < limit;
+    if (fits) return rebuilt;
+  } else if (unknownBaseline ? result.fitsWindow : estimateCoreMessages(prepared.processedMessages) + overheadEstimate + imageTokens < limit) {
     log2("warn", `[${session.id}] preflight made no progress but the payload fits; forwarding as-is`);
     return prepared;
   }