billion-context 0.1.76 → 0.1.77-pr.497.318

package/dist/index.js

@@ -50745,7 +50745,7 @@ function summaryPayload(protocol, model, system, content) {
   if (protocol === "openai") {
     return { model, max_tokens: MAX_SUMMARY_OUTPUT_TOKENS, messages: [{ role: "system", content: system }, { role: "user", content }], stream: false };
   }
-  return { model, max_output_tokens: MAX_SUMMARY_OUTPUT_TOKENS, instructions: system, input: [{ role: "user", content }], stream: false };
+  return { model, max_output_tokens: MAX_SUMMARY_OUTPUT_TOKENS, instructions: system, input: [{ role: "user", content }], stream: false, store: false };
 }
 function extractSummaryText(protocol, json) {
   if (protocol === "anthropic") {
@@ -50811,7 +50811,7 @@ TASK: The conversation segment below (messages ${startRef}\u2013${endRef}) must
 var ABORTED_FAILURE = { kind: "aborted", detail: "the client disconnected during preflight compression" };
 async function preflightCompress(deps, messages) {
   const limit = deps.config.modelContextLimit;
-  const result = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) };
+  const result = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) };
   if (limit <= 0) return result;
   const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION));
   const minChars = deps.config.compress.minCompressRange;
@@ -50831,8 +50831,8 @@ async function preflightCompress(deps, messages) {
       renderTags: "text-only"
     });
     deps.session.state = turn.state;
-    currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages));
-    result.payloadEstimate = estimateCoreMessages(turn.messages);
+    currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0));
+    result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0);
     if (startTokens < 0) startTokens = currentTokens;
     if (currentTokens < limit) break;
     const ranges = viableRanges(turn.nudge?.compressibleRanges ?? []);
@@ -50918,6 +50918,78 @@ async function preflightCompress(deps, messages) {
   return result;
 }
 
+// src/image-tokens.ts
+var REMOTE_IMAGE_TOKENS = 4096;
+function imageTokenCap() {
+  const v2 = Number(process.env.BILI_IMAGE_TOKEN_CAP ?? "");
+  return Number.isInteger(v2) && v2 > 0 ? v2 : 0;
+}
+function applyCap(cost) {
+  const cap = imageTokenCap();
+  return cap > 0 ? Math.min(cost, cap) : cost;
+}
+function costForUrl(url) {
+  if (url.startsWith("data:")) {
+    const i = url.indexOf("base64,");
+    if (i >= 0) return applyCap(Math.ceil((url.length - i - "base64,".length) / 4));
+  }
+  return applyCap(REMOTE_IMAGE_TOKENS);
+}
+function isObj(v2) {
+  return typeof v2 === "object" && v2 !== null;
+}
+function urlOf(v2) {
+  if (typeof v2 === "string") return v2;
+  if (isObj(v2) && typeof v2.url === "string") return v2.url;
+  return void 0;
+}
+function imageTokensInParsedBody(protocol, body) {
+  if (!isObj(body)) return 0;
+  let total = 0;
+  if (protocol === "responses") {
+    const input = body.input;
+    if (!Array.isArray(input)) return 0;
+    for (const item of input) {
+      if (!isObj(item) || !Array.isArray(item.content)) continue;
+      for (const part of item.content) {
+        if (!isObj(part) || part.type !== "input_image") continue;
+        const url = urlOf(part.image_url);
+        if (url) total += costForUrl(url);
+      }
+    }
+    return total;
+  }
+  const messages = body.messages;
+  if (!Array.isArray(messages)) return 0;
+  for (const m2 of messages) {
+    if (!isObj(m2) || !Array.isArray(m2.content)) continue;
+    for (const part of m2.content) {
+      if (!isObj(part)) continue;
+      if (protocol === "openai") {
+        if (part.type !== "image_url") continue;
+        const url = urlOf(part.image_url);
+        if (url) total += costForUrl(url);
+      } else {
+        if (part.type !== "image") continue;
+        const src = part.source;
+        if (isObj(src) && src.type === "base64" && typeof src.data === "string") total += applyCap(Math.ceil(src.data.length / 4));
+        else if (isObj(src) && src.type === "url" && typeof src.url === "string") total += costForUrl(src.url);
+      }
+    }
+  }
+  return total;
+}
+function imageTokensInRawBody(protocol, raw) {
+  const s3 = typeof raw === "string" ? raw : raw.toString("utf8");
+  const probe = protocol === "responses" ? s3.includes("input_image") : protocol === "openai" ? s3.includes("image_url") : s3.includes('"type":"image"') || s3.includes('"type": "image"');
+  if (!probe) return 0;
+  try {
+    return imageTokensInParsedBody(protocol, JSON.parse(s3));
+  } catch {
+    return 0;
+  }
+}
+
 // src/web/index.ts
 import { readFileSync as readFileSync3 } from "fs";
 import { dirname as dirname4, join as join3 } from "path";
@@ -57290,7 +57362,7 @@ function emergencyNudge(nudge, escalationPct = EMERGENCY_NUDGE_ESCALATION_PCT) {
 function clampOutgoingOutput(rebuilt, field, ctx, sessionId, log2) {
   const raw = rebuilt[field];
   if (typeof raw !== "number") return;
-  const inputEstimate = estimateInputTokens(ctx.processedMessages, ctx.systemText, ctx.tools, ctx.lastInputTokens);
+  const inputEstimate = estimateInputTokens(ctx.processedMessages, ctx.systemText, ctx.tools, ctx.lastInputTokens) + ctx.imageTokens;
   const capped = clampOutputBudget(raw, inputEstimate, ctx.nativeWindow);
   if (capped !== void 0) {
     rebuilt[field] = capped;
@@ -57355,7 +57427,7 @@ function prepareOpenai(parsed, req, opts, core, config, prompts, log2, session,
     processedMessages = [];
   }
   const rebuilt = { ...parsed, messages: rebuiltMessages, tools: toolsOut };
-  clampOutgoingOutput(rebuilt, typeof parsed.max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow }, sessionId, log2);
+  clampOutgoingOutput(rebuilt, typeof parsed.max_completion_tokens === "number" ? "max_completion_tokens" : "max_tokens", { systemText: openaiSystemText, tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow, imageTokens: imageTokensInParsedBody("openai", rebuilt) }, sessionId, log2);
   delete rebuilt.prompt_cache_retention;
   if (stream2 && rebuilt.stream_options === void 0) {
     rebuilt.stream_options = { include_usage: true };
@@ -57473,7 +57545,7 @@ function prepareResponses(parsed, req, opts, core, config, prompts, log2, sessio
   }
   const rebuilt = { ...parsed, input: rebuiltInput, tools: toolsOut };
   if (!isCompactionTrigger) {
-    clampOutgoingOutput(rebuilt, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow }, sessionId, log2);
+    clampOutgoingOutput(rebuilt, "max_output_tokens", { systemText: (responsesProjection?.systemParts ?? []).join("\n"), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, nativeWindow, imageTokens: imageTokensInParsedBody("responses", rebuilt) }, sessionId, log2);
   }
   const promptCacheKey = resolvePromptCacheKey(
     rebuilt.prompt_cache_key,
@@ -57698,11 +57770,13 @@ function isPreflightFailFast(outcome) {
 async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, core, config, model, route, affinity, log2, instanceId) {
   const session = prepared.session;
   const limit = config.modelContextLimit;
-  const payloadEstimate = estimateCoreMessages(prepared.processedMessages);
+  const imageTokens = imageTokensInRawBody(prepared.protocol, prepared.body);
+  const payloadEstimate = estimateCoreMessages(prepared.processedMessages) + imageTokens;
   const tokenCount = Math.max(session.stats.lastInputTokens, payloadEstimate);
   if (limit <= 0 || !model || tokenCount < limit) return prepared;
   const failFast = (status2, detail, retryable) => {
-    const message = `context ~${tokenCount} tokens exceeds the model window ${limit} (model=${model}) and preflight compression could not bring it under: ${detail}. The over-window payload was NOT forwarded.`;
+    const imageNote = imageTokens >= limit ? ` Images alone account for ~${imageTokens} tokens (\u2265 window ${limit}); compression cannot remove them \u2014 shrink or remove the images, or raise the window.` : "";
+    const message = `context ~${tokenCount} tokens exceeds the model window ${limit} (model=${model}) and preflight compression could not bring it under: ${detail}.` + imageNote + ` The over-window payload was NOT forwarded.`;
     log2("error", `[${session.id}] preflight fail-fast ${status2} (retryable=${retryable}): ${message}`);
     return { failFast: true, status: status2, message, retryable, respond: !res.writableEnded };
   };
@@ -57732,7 +57806,8 @@ async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, c
       model,
       proxyUrl,
       signal: clientAbort.signal,
-      log: log2
+      log: log2,
+      imageFloor: imageTokens
     },
     prepared.originalMessages
   );
@@ -57901,9 +57976,12 @@ ${hdrText}
       const info = inspectContextOverflow(upstream.status, errBody.toString("utf8"));
       if (info.isOverflow) {
         let reqModel;
+        let rejectedImageTokens = 0;
         try {
           const rawBody = typeof prepared.body === "string" ? prepared.body : prepared.body.toString("utf8");
-          reqModel = JSON.parse(rawBody).model;
+          const parsedBody = JSON.parse(rawBody);
+          reqModel = typeof parsedBody.model === "string" ? parsedBody.model : void 0;
+          rejectedImageTokens = imageTokensInParsedBody(prepared.protocol, parsedBody);
         } catch {
           reqModel = void 0;
         }
@@ -57915,7 +57993,7 @@ ${hdrText}
           s3.metadata.learnedContextLimits = learnedMap;
           log2("warn", `[${s3.id}] upstream context overflow \u2014 learned real window ${info.window} for ${reqModel ?? "(unknown model)"} (was ${prev ?? "unset"}); arming emergency shrink`);
         } else {
-          const payloadEstimate = estimateCoreMessages(prepared.processedMessages);
+          const payloadEstimate = estimateCoreMessages(prepared.processedMessages) + rejectedImageTokens;
           const prev = (reqModel ? learnedMap[reqModel] : void 0) ?? s3.metadata.learnedContextLimit;
           if (payloadEstimate >= 1e3 && (prev === void 0 || payloadEstimate < prev)) {
             if (reqModel) learnedMap[reqModel] = payloadEstimate;