billion-context 0.1.76 → 0.1.77-pr.493.322

package/dist/index.js

@@ -50745,7 +50745,7 @@ function summaryPayload(protocol, model, system, content) {
   if (protocol === "openai") {
     return { model, max_tokens: MAX_SUMMARY_OUTPUT_TOKENS, messages: [{ role: "system", content: system }, { role: "user", content }], stream: false };
   }
-  return { model, max_output_tokens: MAX_SUMMARY_OUTPUT_TOKENS, instructions: system, input: [{ role: "user", content }], stream: false };
+  return { model, max_output_tokens: MAX_SUMMARY_OUTPUT_TOKENS, instructions: system, input: [{ role: "user", content }], stream: false, store: false };
 }
 function extractSummaryText(protocol, json) {
   if (protocol === "anthropic") {
@@ -50811,7 +50811,7 @@ TASK: The conversation segment below (messages ${startRef}\u2013${endRef}) must
 var ABORTED_FAILURE = { kind: "aborted", detail: "the client disconnected during preflight compression" };
 async function preflightCompress(deps, messages) {
   const limit = deps.config.modelContextLimit;
-  const result = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) };
+  const result = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) };
   if (limit <= 0) return result;
   const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION));
   const minChars = deps.config.compress.minCompressRange;
@@ -50831,8 +50831,8 @@ async function preflightCompress(deps, messages) {
       renderTags: "text-only"
     });
     deps.session.state = turn.state;
-    currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages));
-    result.payloadEstimate = estimateCoreMessages(turn.messages);
+    currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0));
+    result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0);
     if (startTokens < 0) startTokens = currentTokens;
     if (currentTokens < limit) break;
     const ranges = viableRanges(turn.nudge?.compressibleRanges ?? []);
@@ -50918,6 +50918,78 @@ async function preflightCompress(deps, messages) {
   return result;
 }
 
+// src/image-tokens.ts
+var REMOTE_IMAGE_TOKENS = 4096;
+function imageTokenCap() {
+  const v2 = Number(process.env.BILI_IMAGE_TOKEN_CAP ?? "");
+  return Number.isInteger(v2) && v2 > 0 ? v2 : 0;
+}
+function applyCap(cost) {
+  const cap = imageTokenCap();
+  return cap > 0 ? Math.min(cost, cap) : cost;
+}
+function costForUrl(url) {
+  if (url.startsWith("data:")) {
+    const i = url.indexOf("base64,");
+    if (i >= 0) return applyCap(Math.ceil((url.length - i - "base64,".length) / 4));
+  }
+  return applyCap(REMOTE_IMAGE_TOKENS);
+}
+function isObj(v2) {
+  return typeof v2 === "object" && v2 !== null;
+}
+function urlOf(v2) {
+  if (typeof v2 === "string") return v2;
+  if (isObj(v2) && typeof v2.url === "string") return v2.url;
+  return void 0;
+}
+function imageTokensInParsedBody(protocol, body) {
+  if (!isObj(body)) return 0;
+  let total = 0;
+  if (protocol === "responses") {
+    const input = body.input;
+    if (!Array.isArray(input)) return 0;
+    for (const item of input) {
+      if (!isObj(item) || !Array.isArray(item.content)) continue;
+      for (const part of item.content) {
+        if (!isObj(part) || part.type !== "input_image") continue;
+        const url = urlOf(part.image_url);
+        if (url) total += costForUrl(url);
+      }
+    }
+    return total;
+  }
+  const messages = body.messages;
+  if (!Array.isArray(messages)) return 0;
+  for (const m2 of messages) {
+    if (!isObj(m2) || !Array.isArray(m2.content)) continue;
+    for (const part of m2.content) {
+      if (!isObj(part)) continue;
+      if (protocol === "openai") {
+        if (part.type !== "image_url") continue;
+        const url = urlOf(part.image_url);
+        if (url) total += costForUrl(url);
+      } else {
+        if (part.type !== "image") continue;
+        const src = part.source;
+        if (isObj(src) && src.type === "base64" && typeof src.data === "string") total += applyCap(Math.ceil(src.data.length / 4));
+        else if (isObj(src) && src.type === "url" && typeof src.url === "string") total += costForUrl(src.url);
+      }
+    }
+  }
+  return total;
+}
+function imageTokensInRawBody(protocol, raw) {
+  const s3 = typeof raw === "string" ? raw : raw.toString("utf8");
+  const probe = protocol === "responses" ? s3.includes("input_image") : protocol === "openai" ? s3.includes("image_url") : s3.includes('"type":"image"') || s3.includes('"type": "image"');
+  if (!probe) return 0;
+  try {
+    return imageTokensInParsedBody(protocol, JSON.parse(s3));
+  } catch {
+    return 0;
+  }
+}
+
 // src/web/index.ts
 import { readFileSync as readFileSync3 } from "fs";
 import { dirname as dirname4, join as join3 } from "path";
@@ -51945,6 +52017,7 @@ var CONTEXT_OVERFLOW_PATTERNS = [
   /maximum context length/i,
   /max context length/i,
   /maximum context size/i,
+  /longer than the model'?s context length/i,
   /exceeds the context window/i,
   /out of room in the model/i,
   /exceeded model token limit/i,
@@ -51961,7 +52034,7 @@ function toTokenNumber(s3) {
 function parseOverflowWindow(text) {
   let m2 = text.match(/>\s*(\d[\d,]*)\s*maximum/i);
   if (m2) return toTokenNumber(m2[1]);
-  m2 = text.match(/maximum context length is (\d[\d,]*)/i) ?? text.match(/maximum context length of (\d[\d,]*)/i) ?? text.match(/maximum context size (?:is|of) (\d[\d,]*)/i) ?? text.match(/(?:maximum|max)\s+(?:context\s+)?length\s+(?:is\s+)?(\d[\d,]*)/i) ?? text.match(/limit of (\d[\d,]*)\s*token/i) ?? text.match(/(\d[\d,]*)\s*maximum\b/i);
+  m2 = text.match(/maximum context length is (\d[\d,]*)/i) ?? text.match(/maximum context length of (\d[\d,]*)/i) ?? text.match(/maximum context size (?:is|of) (\d[\d,]*)/i) ?? text.match(/(?:maximum|max)\s+(?:context\s+)?length\s+(?:is\s+)?(\d[\d,]*)/i) ?? text.match(/context length\s*\((\d[\d,]*)\s*token/i) ?? text.match(/limit of (\d[\d,]*)\s*token/i) ?? text.match(/(\d[\d,]*)\s*maximum\b/i);
   if (m2) return toTokenNumber(m2[1]);
   return void 0;
 }
@@ -55344,13 +55417,20 @@ function loadClientConfig(env, cwd) {
   config.dsh = readDshConfig(resolveDshHome(env));
   return config;
 }
-function collectModelWindows(config) {
+function collectModelWindows(config, scope) {
   const out = {};
   const add = (wins) => {
     for (const w2 of wins ?? []) {
       if (!out[w2.id] || w2.contextWindow > out[w2.id]) out[w2.id] = w2.contextWindow;
     }
   };
+  if (scope) {
+    if (scope === "codex") add(config.codex?.modelWindows);
+    else if (scope === "pi") for (const p2 of Object.values(config.pi?.providers ?? {})) add(p2.models);
+    else if (scope === "omp") for (const p2 of Object.values(config.omp?.providers ?? {})) add(p2.models);
+    else if (scope === "opencode") for (const p2 of Object.values(config.opencode?.providers ?? {})) add(p2.models);
+    return out;
+  }
   for (const p2 of Object.values(config.pi?.providers ?? {})) add(p2.models);
   for (const p2 of Object.values(config.omp?.providers ?? {})) add(p2.models);
   for (const p2 of Object.values(config.opencode?.providers ?? {})) add(p2.models);
@@ -57290,7 +57370,7 @@ function emergencyNudge(nudge, escalationPct = EMERGENCY_NUDGE_ESCALATION_PCT) {
 function clampOutgoingOutput(rebuilt, field, ctx, sessionId, log2) {
   const raw = rebuilt[field];
   if (typeof raw !== "number") return;
-  const inputEstimate = estimateInputTokens(ctx.processedMessages, ctx.systemText, ctx.tools, ctx.lastInputTokens);
+  const inputEstimate = estimateInputTokens(ctx.processedMessages, ctx.systemText, ctx.tools, ctx.lastInputTokens) + ctx.imageTokens;
   const capped = clampOutputBudget(raw, inputEstimate, ctx.nativeWindow);
   if (capped !== void 0) {
     rebuilt[field] = capped;
@@ -57355,7 +57435,7 @@ function prepareOpenai(parsed, req, opts, core, config, prompts, log2, session,
     processedMessages = [];
   }
   const rebuilt = { ...parsed, messages: rebuiltMessages, tools: toolsOut };
-  clampOutgoingOutput(rebuilt, typeof parsed.max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow }, sessionId, log2);
+  clampOutgoingOutput(rebuilt, typeof parsed.max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow, imageTokens: imageTokensInParsedBody("openai", rebuilt) }, sessionId, log2);
   delete rebuilt.prompt_cache_retention;
   if (stream2 && rebuilt.stream_options === void 0) {
     rebuilt.stream_options = { include_usage: true };
@@ -57473,7 +57553,7 @@ function prepareResponses(parsed, req, opts, core, config, prompts, log2, sessio
   }
   const rebuilt = { ...parsed, input: rebuiltInput, tools: toolsOut };
   if (!isCompactionTrigger) {
-    clampOutgoingOutput(rebuilt, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow }, sessionId, log2);
+    clampOutgoingOutput(rebuilt, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow, imageTokens: imageTokensInParsedBody("responses", rebuilt) }, sessionId, log2);
   }
   const promptCacheKey = resolvePromptCacheKey(
     rebuilt.prompt_cache_key,
@@ -57698,11 +57778,13 @@ function isPreflightFailFast(outcome) {
 async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, core, config, model, route, affinity, log2, instanceId) {
   const session = prepared.session;
   const limit = config.modelContextLimit;
-  const payloadEstimate = estimateCoreMessages(prepared.processedMessages);
+  const imageTokens = imageTokensInRawBody(prepared.protocol, prepared.body);
+  const payloadEstimate = estimateCoreMessages(prepared.processedMessages) + imageTokens;
   const tokenCount = Math.max(session.stats.lastInputTokens, payloadEstimate);
   if (limit <= 0 || !model || tokenCount < limit) return prepared;
   const failFast = (status2, detail, retryable) => {
-    const message = `context ~${tokenCount} tokens exceeds the model window ${limit} (model=${model}) and preflight compression could not bring it under: ${detail}. The over-window payload was NOT forwarded.`;
+    const imageNote = imageTokens >= limit ? ` Images alone account for ~${imageTokens} tokens (\u2265 window ${limit}); compression cannot remove them \u2014 shrink or remove the images, or raise the window.` : "";
+    const message = `context ~${tokenCount} tokens exceeds the model window ${limit} (model=${model}) and preflight compression could not bring it under: ${detail}.` + imageNote + ` The over-window payload was NOT forwarded.`;
     log2("error", `[${session.id}] preflight fail-fast ${status2} (retryable=${retryable}): ${message}`);
     return { failFast: true, status: status2, message, retryable, respond: !res.writableEnded };
   };
@@ -57732,7 +57814,8 @@ async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, c
       model,
       proxyUrl,
       signal: clientAbort.signal,
-      log: log2
+      log: log2,
+      imageFloor: imageTokens
     },
     prepared.originalMessages
   );
@@ -57901,9 +57984,12 @@ ${hdrText}
       const info = inspectContextOverflow(upstream.status, errBody.toString("utf8"));
       if (info.isOverflow) {
         let reqModel;
+        let rejectedImageTokens = 0;
         try {
           const rawBody = typeof prepared.body === "string" ? prepared.body : prepared.body.toString("utf8");
-          reqModel = JSON.parse(rawBody).model;
+          const parsedBody = JSON.parse(rawBody);
+          reqModel = typeof parsedBody.model === "string" ? parsedBody.model : void 0;
+          rejectedImageTokens = imageTokensInParsedBody(prepared.protocol, parsedBody);
         } catch {
           reqModel = void 0;
         }
@@ -57915,7 +58001,7 @@ ${hdrText}
           s3.metadata.learnedContextLimits = learnedMap;
           log2("warn", `[${s3.id}] upstream context overflow \u2014 learned real window ${info.window} for ${reqModel ?? "(unknown model)"} (was ${prev ?? "unset"}); arming emergency shrink`);
         } else {
-          const payloadEstimate = estimateCoreMessages(prepared.processedMessages);
+          const payloadEstimate = estimateCoreMessages(prepared.processedMessages) + rejectedImageTokens;
           const prev = (reqModel ? learnedMap[reqModel] : void 0) ?? s3.metadata.learnedContextLimit;
           if (payloadEstimate >= 1e3 && (prev === void 0 || payloadEstimate < prev)) {
             if (reqModel) learnedMap[reqModel] = payloadEstimate;
@@ -63640,7 +63726,7 @@ async function runLaunch(params, deps = {}) {
   const routes = discoverRoutes(base, config);
   const biliRoutes = loadRoutes(process.env);
   const domains = dedupeInOrder([...routes.httpsDomains, ...params.mitmDomains ?? []]);
-  const handle2 = await ensureProxyRunning({ host, port, passthrough, debug, mitmDomains: domains, modelWindows: collectModelWindows(config) }, deps);
+  const handle2 = await ensureProxyRunning({ host, port, passthrough, debug, mitmDomains: domains, modelWindows: collectModelWindows(config, base) }, deps);
   console.error(
     `bili: started proxy at ${handle2.origin} (MITM domains: ${domains.length ? domains.join(", ") : "defaults"})` + (routes.httpRewrites.length > 0 ? ` (HTTP /bili/ rewrites: ${routes.httpRewrites.length})` : "") + (routes.httpsRewrites.length > 0 ? ` (HTTPS cert rewrites: ${routes.httpsRewrites.length})` : "") + (params.client === "pi-test" ? " (no extensions)" : "")
   );
@@ -63815,7 +63901,7 @@ async function runTestPi(params, deps = {}) {
     ...discoverDomains("pi", config),
     ...params.mitmDomains ?? []
   ]);
-  const handle2 = await ensureProxyRunning({ host, port, passthrough, debug, mitmDomains: domains, modelWindows: collectModelWindows(config) }, deps);
+  const handle2 = await ensureProxyRunning({ host, port, passthrough, debug, mitmDomains: domains, modelWindows: collectModelWindows(config, "pi") }, deps);
   console.error(
     `bili: started proxy at ${handle2.origin} (MITM domains: ${domains.length ? domains.join(", ") : "defaults"})`
   );