billion-context 0.1.75-pr.486.305 → 0.1.75-pr.489.307

package/dist/index.js

@@ -50696,7 +50696,7 @@ function summaryPayload(protocol, model, system, content) {
   if (protocol === "openai") {
     return { model, max_tokens: MAX_SUMMARY_OUTPUT_TOKENS, messages: [{ role: "system", content: system }, { role: "user", content }], stream: false };
   }
-  return { model, max_output_tokens: MAX_SUMMARY_OUTPUT_TOKENS, instructions: system, input: [{ role: "user", content }], stream: false };
+  return { model, max_output_tokens: MAX_SUMMARY_OUTPUT_TOKENS, instructions: system, input: [{ role: "user", content }], stream: false, store: false };
 }
 function extractSummaryText(protocol, json) {
   if (protocol === "anthropic") {
@@ -50762,7 +50762,7 @@ TASK: The conversation segment below (messages ${startRef}\u2013${endRef}) must
 var ABORTED_FAILURE = { kind: "aborted", detail: "the client disconnected during preflight compression" };
 async function preflightCompress(deps, messages) {
   const limit = deps.config.modelContextLimit;
-  const result = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) };
+  const result = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + (deps.imageFloor ?? 0) };
   if (limit <= 0) return result;
   const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION));
   const minChars = deps.config.compress.minCompressRange;
@@ -50782,8 +50782,8 @@ async function preflightCompress(deps, messages) {
       renderTags: "text-only"
     });
     deps.session.state = turn.state;
-    currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages));
-    result.payloadEstimate = estimateCoreMessages(turn.messages);
+    currentTokens = Math.max(deps.session.stats.lastInputTokens, estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0));
+    result.payloadEstimate = estimateCoreMessages(turn.messages) + (deps.imageFloor ?? 0);
     if (startTokens < 0) startTokens = currentTokens;
     if (currentTokens < limit) break;
     const ranges = viableRanges(turn.nudge?.compressibleRanges ?? []);
@@ -50869,6 +50869,78 @@ async function preflightCompress(deps, messages) {
   return result;
 }
 
+// src/image-tokens.ts
+var REMOTE_IMAGE_TOKENS = 4096;
+function imageTokenCap() {
+  const v2 = Number(process.env.BILI_IMAGE_TOKEN_CAP ?? "");
+  return Number.isInteger(v2) && v2 > 0 ? v2 : 0;
+}
+function applyCap(cost) {
+  const cap = imageTokenCap();
+  return cap > 0 ? Math.min(cost, cap) : cost;
+}
+function costForUrl(url) {
+  if (url.startsWith("data:")) {
+    const i = url.indexOf("base64,");
+    if (i >= 0) return applyCap(Math.ceil((url.length - i - "base64,".length) / 4));
+  }
+  return applyCap(REMOTE_IMAGE_TOKENS);
+}
+function isObj(v2) {
+  return typeof v2 === "object" && v2 !== null;
+}
+function urlOf(v2) {
+  if (typeof v2 === "string") return v2;
+  if (isObj(v2) && typeof v2.url === "string") return v2.url;
+  return void 0;
+}
+function imageTokensInParsedBody(protocol, body) {
+  if (!isObj(body)) return 0;
+  let total = 0;
+  if (protocol === "responses") {
+    const input = body.input;
+    if (!Array.isArray(input)) return 0;
+    for (const item of input) {
+      if (!isObj(item) || !Array.isArray(item.content)) continue;
+      for (const part of item.content) {
+        if (!isObj(part) || part.type !== "input_image") continue;
+        const url = urlOf(part.image_url);
+        if (url) total += costForUrl(url);
+      }
+    }
+    return total;
+  }
+  const messages = body.messages;
+  if (!Array.isArray(messages)) return 0;
+  for (const m2 of messages) {
+    if (!isObj(m2) || !Array.isArray(m2.content)) continue;
+    for (const part of m2.content) {
+      if (!isObj(part)) continue;
+      if (protocol === "openai") {
+        if (part.type !== "image_url") continue;
+        const url = urlOf(part.image_url);
+        if (url) total += costForUrl(url);
+      } else {
+        if (part.type !== "image") continue;
+        const src = part.source;
+        if (isObj(src) && src.type === "base64" && typeof src.data === "string") total += applyCap(Math.ceil(src.data.length / 4));
+        else if (isObj(src) && src.type === "url" && typeof src.url === "string") total += costForUrl(src.url);
+      }
+    }
+  }
+  return total;
+}
+function imageTokensInRawBody(protocol, raw) {
+  const s3 = typeof raw === "string" ? raw : raw.toString("utf8");
+  const probe = protocol === "responses" ? s3.includes("input_image") : protocol === "openai" ? s3.includes("image_url") : s3.includes('"type":"image"') || s3.includes('"type": "image"');
+  if (!probe) return 0;
+  try {
+    return imageTokensInParsedBody(protocol, JSON.parse(s3));
+  } catch {
+    return 0;
+  }
+}
+
 // src/web/index.ts
 import { readFileSync as readFileSync3 } from "fs";
 import { dirname as dirname4, join as join3 } from "path";
@@ -57630,11 +57702,13 @@ function isPreflightFailFast(outcome) {
 async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, core, config, model, route, affinity, log2, instanceId) {
   const session = prepared.session;
   const limit = config.modelContextLimit;
-  const payloadEstimate = estimateCoreMessages(prepared.processedMessages);
+  const imageTokens = imageTokensInRawBody(prepared.protocol, prepared.body);
+  const payloadEstimate = estimateCoreMessages(prepared.processedMessages) + imageTokens;
   const tokenCount = Math.max(session.stats.lastInputTokens, payloadEstimate);
   if (limit <= 0 || !model || tokenCount < limit) return prepared;
   const failFast = (status2, detail, retryable) => {
-    const message = `context ~${tokenCount} tokens exceeds the model window ${limit} (model=${model}) and preflight compression could not bring it under: ${detail}. The over-window payload was NOT forwarded.`;
+    const imageNote = imageTokens >= limit ? ` Images alone account for ~${imageTokens} tokens (\u2265 window ${limit}); compression cannot remove them \u2014 shrink or remove the images, or raise the window.` : "";
+    const message = `context ~${tokenCount} tokens exceeds the model window ${limit} (model=${model}) and preflight compression could not bring it under: ${detail}.` + imageNote + ` The over-window payload was NOT forwarded.`;
     log2("error", `[${session.id}] preflight fail-fast ${status2} (retryable=${retryable}): ${message}`);
     return { failFast: true, status: status2, message, retryable, respond: !res.writableEnded };
   };
@@ -57664,7 +57738,8 @@ async function preflightCompressIfNeeded(prepared, runPrepare, req, res, opts, c
       model,
       proxyUrl,
       signal: clientAbort.signal,
-      log: log2
+      log: log2,
+      imageFloor: imageTokens
     },
     prepared.originalMessages
   );
@@ -57833,9 +57908,12 @@ ${hdrText}
       const info = inspectContextOverflow(upstream.status, errBody.toString("utf8"));
       if (info.isOverflow) {
         let reqModel;
+        let rejectedImageTokens = 0;
         try {
           const rawBody = typeof prepared.body === "string" ? prepared.body : prepared.body.toString("utf8");
-          reqModel = JSON.parse(rawBody).model;
+          const parsedBody = JSON.parse(rawBody);
+          reqModel = typeof parsedBody.model === "string" ? parsedBody.model : void 0;
+          rejectedImageTokens = imageTokensInParsedBody(prepared.protocol, parsedBody);
         } catch {
           reqModel = void 0;
         }
@@ -57847,7 +57925,7 @@ ${hdrText}
           s3.metadata.learnedContextLimits = learnedMap;
           log2("warn", `[${s3.id}] upstream context overflow \u2014 learned real window ${info.window} for ${reqModel ?? "(unknown model)"} (was ${prev ?? "unset"}); arming emergency shrink`);
         } else {
-          const payloadEstimate = estimateCoreMessages(prepared.processedMessages);
+          const payloadEstimate = estimateCoreMessages(prepared.processedMessages) + rejectedImageTokens;
           const prev = (reqModel ? learnedMap[reqModel] : void 0) ?? s3.metadata.learnedContextLimit;
           if (payloadEstimate >= 1e3 && (prev === void 0 || payloadEstimate < prev)) {
             if (reqModel) learnedMap[reqModel] = payloadEstimate;