billion-context 0.1.70 → 0.1.71-pr.378.283

package/dist/index.js

@@ -50369,6 +50369,50 @@ function stripItemContent(it2) {
   }
   return it2;
 }
+function stripIfString(v2) {
+  return typeof v2 === "string" ? stripAcpTags(v2) : v2;
+}
+function stripOpenaiChatText(obj) {
+  if (!obj || typeof obj !== "object") return obj;
+  const o = obj;
+  if (!Array.isArray(o["choices"])) return obj;
+  o["choices"] = o["choices"].map((c) => {
+    if (!c || typeof c !== "object") return c;
+    const ch = c;
+    for (const holder of ["delta", "message"]) {
+      const h = ch[holder];
+      if (h && typeof h === "object") {
+        const hh = { ...h };
+        hh["content"] = stripIfString(hh["content"]);
+        hh["reasoning_content"] = stripIfString(hh["reasoning_content"]);
+        hh["reasoning"] = stripIfString(hh["reasoning"]);
+        ch[holder] = hh;
+      }
+    }
+    return ch;
+  });
+  return obj;
+}
+function stripAnthropicText(obj) {
+  if (!obj || typeof obj !== "object") return obj;
+  const o = obj;
+  const d = o["delta"];
+  if (d && typeof d === "object") {
+    const dd = { ...d };
+    dd["text"] = stripIfString(dd["text"]);
+    dd["thinking"] = stripIfString(dd["thinking"]);
+    o["delta"] = dd;
+  }
+  if (Array.isArray(o["content"])) {
+    o["content"] = o["content"].map((c) => {
+      if (!c || typeof c !== "object") return c;
+      const cc = c;
+      if (typeof cc["text"] !== "string" && typeof cc["thinking"] !== "string") return c;
+      return { ...cc, text: stripIfString(cc["text"]), thinking: stripIfString(cc["thinking"]) };
+    });
+  }
+  return obj;
+}
 function stripResponsesText(obj) {
   if (!obj || typeof obj !== "object") return obj;
   const o = obj;
@@ -53207,6 +53251,86 @@ async function compressLoopResponsesJson(initialResponse, ctx, requestBody, requ
   return current;
 }
 
+// src/fake-completion.ts
+var FAKE_COMPLETION_HINT = "[billion-context] Your last reply described a tool call as plain text instead of invoking it, so no tool actually ran. To act, invoke the tool through the proper tool-calling mechanism (emit a tool_use / tool_calls / function_call block). Do not write tool-call markup as text in your reply.";
+function maxFakeCompletionRetries() {
+  const n = Number.parseInt(process.env.BILI_FAKE_COMPLETION_RETRIES ?? "0", 10);
+  return Number.isFinite(n) && n >= 0 ? n : 0;
+}
+function fakeBufCap() {
+  const n = Number.parseInt(process.env.BILI_FAKE_BUF_CAP ?? String(16 * 1024 * 1024), 10);
+  return Number.isFinite(n) && n > 0 ? n : 16 * 1024 * 1024;
+}
+var ANTHROPIC_TOOL_BLOCK = /"type"\s*:\s*"tool_use"/;
+var OPENAI_TOOL_BLOCK = /"tool_calls"\s*:\s*\[\s*\{/;
+var RESPONSES_TOOL_BLOCK = /"type"\s*:\s*"function_call"/;
+function hasToolBlock(protocol, rawText) {
+  switch (protocol) {
+    case "anthropic":
+      return ANTHROPIC_TOOL_BLOCK.test(rawText);
+    case "openai":
+      return OPENAI_TOOL_BLOCK.test(rawText);
+    case "responses":
+      return RESPONSES_TOOL_BLOCK.test(rawText);
+  }
+}
+var TOOL_OPEN = /\x3c(?!\s*\/)(?:antml:)?(invoke|tool_calls|tool_call|parameter|parameters)\b|\\u003c(?!\s*\/)(?:antml:)?(invoke|tool_calls|tool_call|parameter|parameters)\b/i;
+var TOOL_CLOSE_GLOBAL = /\x3c\/(?:antml:)?(invoke|tool_calls|tool_call|parameter|parameters)\b|\\u003c\/(?:antml:)?(invoke|tool_calls|tool_call|parameter|parameters)\b/gi;
+function hasToolCallStructure(text) {
+  if (!containsToolCallXmlFragment(text)) return false;
+  if (TOOL_OPEN.test(text)) return true;
+  const closes = /* @__PURE__ */ new Set();
+  for (const m2 of text.matchAll(TOOL_CLOSE_GLOBAL)) {
+    const name = m2[1] ?? m2[2];
+    if (name) closes.add(name.toLowerCase());
+  }
+  return closes.size >= 2;
+}
+function isFakeCompletion(protocol, rawText) {
+  return hasToolCallStructure(rawText) && !hasToolBlock(protocol, rawText);
+}
+function injectFakeCompletionHint(protocol, body) {
+  const raw = typeof body === "string" ? body : body.toString("utf8");
+  let obj;
+  try {
+    obj = JSON.parse(raw);
+  } catch {
+    return null;
+  }
+  if (protocol === "responses") {
+    const input = obj.input;
+    if (!Array.isArray(input)) return null;
+    const arr2 = input;
+    const last2 = arr2[arr2.length - 1];
+    if (last2 && typeof last2 === "object" && last2.role === "user") {
+      const c = last2.content;
+      if (typeof c === "string") last2.content = `${c}
+
+${FAKE_COMPLETION_HINT}`;
+      else if (Array.isArray(c)) last2.content = [...c, { type: "input_text", text: FAKE_COMPLETION_HINT }];
+      else last2.content = FAKE_COMPLETION_HINT;
+    } else {
+      arr2.push({ role: "user", content: [{ type: "input_text", text: FAKE_COMPLETION_HINT }] });
+    }
+    return JSON.stringify(obj);
+  }
+  const messages = obj.messages;
+  if (!Array.isArray(messages)) return null;
+  const arr = messages;
+  const last = arr[arr.length - 1];
+  if (last && typeof last === "object" && last.role === "user") {
+    const c = last.content;
+    if (typeof c === "string") last.content = `${c}
+
+${FAKE_COMPLETION_HINT}`;
+    else if (Array.isArray(c)) last.content = [...c, { type: "text", text: FAKE_COMPLETION_HINT }];
+    else last.content = FAKE_COMPLETION_HINT;
+  } else {
+    arr.push({ role: "user", content: FAKE_COMPLETION_HINT });
+  }
+  return JSON.stringify(obj);
+}
+
 // src/stream-openai.ts
 function rewriteOpenaiJsonResponse(body, ctx) {
   if (!body || typeof body !== "object") return body;
@@ -54174,19 +54298,125 @@ function mergeUsageSample(acc, sample) {
   if (sample.cachedTokens !== void 0) acc.cachedTokens = sample.cachedTokens;
   if (sample.outputTokens !== void 0) acc.outputTokens = sample.outputTokens;
 }
-async function pipeThroughWithUsage(stream2, res, session, protocol) {
+async function pipePluginChatWithStrip(stream2, res, session, protocol, log2) {
   const reader = stream2.getReader();
   const decoder = new TextDecoder("utf-8");
   let buf = "";
   const acc = {};
+  const onDrop = (snippet) => {
+    log("warn", `[tag-echo] stripped model-emitted render tag (plugin passthrough): ${snippet.slice(0, 80).replace(/\n/g, " ")}`);
+    log2?.(`[tag-echo] stripped model-emitted render tag from plugin passthrough text`);
+  };
+  const streams = /* @__PURE__ */ new Map();
+  const filterFor = (field, index) => {
+    const key = `${field}:${index}`;
+    let s3 = streams.get(key);
+    if (!s3) {
+      s3 = { filter: createTagEchoFilter(onDrop), field, index };
+      streams.set(key, s3);
+    }
+    return s3;
+  };
+  const anyPending = () => {
+    for (const s3 of streams.values()) if (s3.filter.pending()) return true;
+    return false;
+  };
+  let lastChunkMeta = {};
+  const syntheticTail = (field, index, tail) => {
+    if (protocol === "anthropic") {
+      const deltaType = field === "text" ? "text_delta" : "thinking_delta";
+      return `event: content_block_delta
+data: ${JSON.stringify({ type: "content_block_delta", index, delta: { type: deltaType, [field]: tail } })}
+
+`;
+    }
+    return `data: ${JSON.stringify({ ...lastChunkMeta, object: "chat.completion.chunk", choices: [{ index, delta: { [field]: tail } }] })}
+
+`;
+  };
+  const flushTails = () => {
+    let out = "";
+    for (const s3 of streams.values()) {
+      const tail = s3.filter.flush();
+      if (tail.length > 0) out += syntheticTail(s3.field, s3.index, tail);
+    }
+    return out;
+  };
+  const write = (s3) => {
+    if (!res.write(Buffer.from(s3, "utf8"))) {
+      return new Promise((r) => res.once("drain", () => r()));
+    }
+  };
+  const pushField = (field, index, text) => {
+    const s3 = filterFor(field, index);
+    const clean = s3.filter.push(text);
+    return [clean, clean !== text];
+  };
+  const processOpenai = (ev, rawEvent) => {
+    if (typeof ev["id"] === "string" || typeof ev["model"] === "string") {
+      lastChunkMeta = { id: ev["id"], created: ev["created"], model: ev["model"] };
+    }
+    const choices = ev["choices"];
+    if (!Array.isArray(choices)) {
+      return anyPending() ? flushTails() + rawEvent + "\n\n" : rawEvent + "\n\n";
+    }
+    let rebuilt = null;
+    let droppedText = false;
+    let hadText = false;
+    for (let ci2 = 0; ci2 < choices.length; ci2++) {
+      const ch = choices[ci2];
+      const d = ch?.["delta"];
+      if (!d || typeof d !== "object") continue;
+      const dd = d;
+      for (const field of ["content", "reasoning_content", "reasoning"]) {
+        const v2 = dd[field];
+        if (typeof v2 !== "string") continue;
+        hadText = true;
+        if (!containsRenderTagText(v2) && !anyPending()) continue;
+        const index = typeof ch?.["index"] === "number" ? ch["index"] : ci2;
+        const [clean, changed] = pushField(field, index, v2);
+        if (clean.length === 0) droppedText = true;
+        if (changed) {
+          if (!rebuilt) {
+            rebuilt = { ...ev, choices: choices.map((c) => ({ ...c, delta: { ...c["delta"] } })) };
+          }
+          rebuilt["choices"][ci2]["delta"] = { ...rebuilt["choices"][ci2]["delta"], [field]: clean };
+        }
+      }
+    }
+    if (rebuilt) {
+      if (droppedText && !hadTextOtherThanTextFields(rebuilt["choices"])) {
+        return "";
+      }
+      return rebuildEvent(rawEvent, rebuilt);
+    }
+    if (!hadText && anyPending()) return flushTails() + rawEvent + "\n\n";
+    return rawEvent + "\n\n";
+  };
+  const processAnthropic = (ev, rawEvent) => {
+    if (ev["type"] !== "content_block_delta") {
+      return anyPending() ? flushTails() + rawEvent + "\n\n" : rawEvent + "\n\n";
+    }
+    const d = ev["delta"];
+    const index = typeof ev["index"] === "number" ? ev["index"] : 0;
+    const field = d?.["type"] === "thinking_delta" ? "thinking" : d?.["type"] === "text_delta" ? "text" : null;
+    if (field === null || typeof d?.[field] !== "string") {
+      return rawEvent + "\n\n";
+    }
+    const raw = d[field];
+    if (!containsRenderTagText(raw) && !anyPending()) {
+      return rawEvent + "\n\n";
+    }
+    const [clean, changed] = pushField(field, index, raw);
+    if (!changed) return rawEvent + "\n\n";
+    if (clean.length === 0 && Object.keys(d ?? {}).length <= 2) return "";
+    return rebuildEvent(rawEvent, { ...ev, delta: { ...d, [field]: clean } });
+  };
   try {
     for (; ; ) {
       const { done, value } = await reader.read();
       if (done) break;
       if (value && value.length > 0) {
-        if (!res.write(Buffer.from(value))) {
-          await new Promise((r) => res.once("drain", () => r()));
-        }
         buf = normalizeSseLineEndings(buf + decoder.decode(value, { stream: true }));
         let idx;
         while ((idx = buf.indexOf("\n\n")) !== -1) {
@@ -54195,17 +54425,28 @@ async function pipeThroughWithUsage(stream2, res, session, protocol) {
           const dataLines = rawEvent.split("\n").filter((l) => l.startsWith("data:"));
           if (dataLines.length === 0) continue;
           const jsonStr = dataLines.map((l) => l.slice(5).replace(/^ /, "")).join("\n").trim();
-          if (!jsonStr || jsonStr === "[DONE]") continue;
+          if (!jsonStr) continue;
+          if (jsonStr === "[DONE]") {
+            await write(flushTails() + rawEvent + "\n\n");
+            continue;
+          }
+          let ev;
           try {
-            const ev = JSON.parse(jsonStr);
-            const sample = usageFromSseEvent(ev);
-            if (sample) mergeUsageSample(acc, sample);
+            ev = JSON.parse(jsonStr);
           } catch {
+            await write(rawEvent + "\n\n");
+            continue;
           }
+          const sample = usageFromSseEvent(ev);
+          if (sample) mergeUsageSample(acc, sample);
+          const out = protocol === "anthropic" ? processAnthropic(ev, rawEvent) : processOpenai(ev, rawEvent);
+          if (out.length > 0) await write(out);
         }
       }
       if (res.destroyed || res.writableEnded) break;
     }
+    const rest = flushTails();
+    if (rest.length > 0 && !res.destroyed && !res.writableEnded) await write(rest);
     if (acc.inputTokens !== void 0 || acc.outputTokens !== void 0 || acc.cachedTokens !== void 0) {
       applyUsageSample(session, acc, protocol);
       markDirty(session);
@@ -54215,6 +54456,20 @@ async function pipeThroughWithUsage(stream2, res, session, protocol) {
     res.end();
   }
 }
+function hadTextOtherThanTextFields(choices) {
+  if (!Array.isArray(choices)) return true;
+  for (const c of choices) {
+    if (!c || typeof c !== "object") continue;
+    const ch = c;
+    if (typeof ch["finish_reason"] === "string") return true;
+    const d = ch["delta"];
+    if (!d) continue;
+    for (const k2 of Object.keys(d)) {
+      if (k2 !== "content" && k2 !== "reasoning_content" && k2 !== "reasoning") return true;
+    }
+  }
+  return false;
+}
 async function pipePluginResponsesWithStrip(stream2, res, session, log2) {
   const reader = stream2.getReader();
   const decoder = new TextDecoder("utf-8");
@@ -54353,13 +54608,12 @@ async function pipePluginJson(stream2, res, session, protocol) {
     }
   } catch {
   }
-  if (protocol === "responses") {
+  if (containsRenderTagText(text)) {
     try {
       const json = JSON.parse(text);
-      if (containsRenderTagText(text)) {
-        res.end(Buffer.from(JSON.stringify(stripResponsesText(json)), "utf8"));
-        return;
-      }
+      const stripped = protocol === "responses" ? stripResponsesText(json) : protocol === "anthropic" ? stripAnthropicText(json) : stripOpenaiChatText(json);
+      res.end(Buffer.from(JSON.stringify(stripped), "utf8"));
+      return;
     } catch {
     }
   }
@@ -57329,7 +57583,7 @@ ${hdrText}
       if (prepared.protocol === "responses") {
         await pipePluginResponsesWithStrip(upstream.body, res, prepared.session, (msg) => log2("info", `[${prepared.session.id}] ${msg}`));
       } else {
-        await pipeThroughWithUsage(upstream.body, res, prepared.session, prepared.protocol);
+        await pipePluginChatWithStrip(upstream.body, res, prepared.session, prepared.protocol, (msg) => log2("info", `[${prepared.session.id}] ${msg}`));
       }
     } else {
       await pipePluginJson(upstream.body, res, prepared.session, prepared.protocol);
@@ -57337,10 +57591,24 @@ ${hdrText}
     clearUpstreamTimer();
     return;
   }
+  let responseBody = upstream.body;
+  if (prepared !== null && prepared.stream && maxFakeCompletionRetries() > 0) {
+    const resolvedBuf = await resolveFakeCompletion(upstream.body, {
+      protocol: prepared.protocol,
+      body,
+      upstreamUrl,
+      reqHeaders: buildForwardHeaders(headers),
+      proxyUrl,
+      signal: clientAbort.signal,
+      session: prepared.session,
+      log: log2
+    });
+    responseBody = bufferToStream(resolvedBuf);
+  }
   const useRewriter = prepared !== null && prepared.compressInjected && prepared.processedMessages.length > 0;
   if (!useRewriter || prepared === null) {
     if (prepared && prepared.resetAfterSuccess) {
-      const [toClient, toObserve] = upstream.body.tee();
+      const [toClient, toObserve] = responseBody.tee();
       const observed = observeResponsesTerminalState(toObserve, prepared.stream);
       await pipeThrough(toClient, res);
       clearUpstreamTimer();
@@ -57352,7 +57620,7 @@ ${hdrText}
         log2("warn", `[${prepared.session.id}] native compact response terminal=${terminal}; rebase NOT scheduled`);
       }
     } else {
-      await pipeThrough(upstream.body, res);
+      await pipeThrough(responseBody, res);
       clearUpstreamTimer();
     }
     return;
@@ -57366,10 +57634,10 @@ ${hdrText}
     debug: opts.debug
   };
   if (prepared.stream) {
-    let streamToRead = upstream.body;
+    let streamToRead = responseBody;
     let dumpRaw;
     if (opts.dumpSse) {
-      const [a, b2] = upstream.body.tee();
+      const [a, b2] = responseBody.tee();
       streamToRead = a;
       dumpRaw = dumpStreamToFile(b2, opts.dumpSse, `${Date.now()}-${safeSessionId(prepared.session.id)}-raw.sse`);
     }
@@ -57495,6 +57763,58 @@ async function readStreamToBuffer(stream2, maxBytes = 1 << 20) {
   }
   return Buffer.concat(chunks);
 }
+function bufferToStream(buf) {
+  return new ReadableStream({
+    start(controller) {
+      controller.enqueue(new Uint8Array(buf));
+      controller.close();
+    }
+  });
+}
+async function resolveFakeCompletion(stream2, opts) {
+  let buffer = await readStreamToBuffer(stream2, fakeBufCap());
+  const max = maxFakeCompletionRetries();
+  const sid = opts.session.id;
+  const priorStreak = opts.session.metadata.fakeCompletionStreak ?? 0;
+  if (max > 0 && priorStreak < max && isFakeCompletion(opts.protocol, buffer.toString("utf8"))) {
+    for (let attempt = 1; attempt <= max && !opts.signal.aborted; attempt++) {
+      const hinted = injectFakeCompletionHint(opts.protocol, opts.body);
+      if (hinted === null) break;
+      opts.log("warn", `[${sid}] fake completion (tool-call XML, no tool block); retry ${attempt}/${max} with corrective hint`);
+      let r;
+      try {
+        r = await fetchWithTimeout(
+          opts.upstreamUrl,
+          {
+            method: "POST",
+            headers: opts.reqHeaders,
+            body: hinted,
+            ...opts.proxyUrl ? { dispatcher: proxyDispatcher(opts.proxyUrl) } : {}
+          },
+          void 0,
+          opts.signal
+        );
+      } catch (e) {
+        opts.log("warn", `[${sid}] fake-completion retry failed: ${String(e)}; presenting original`);
+        break;
+      }
+      try {
+        if (!r.response.ok || !r.response.body) {
+          opts.log("warn", `[${sid}] fake-completion retry rejected (HTTP ${r.response.status}); presenting original`);
+          break;
+        }
+        buffer = await readStreamToBuffer(r.response.body, fakeBufCap());
+      } finally {
+        r.clearTimer();
+      }
+      if (!isFakeCompletion(opts.protocol, buffer.toString("utf8"))) break;
+    }
+  }
+  const stillFake = isFakeCompletion(opts.protocol, buffer.toString("utf8"));
+  opts.session.metadata.fakeCompletionStreak = stillFake ? priorStreak + 1 : 0;
+  markDirty(opts.session);
+  return buffer;
+}
 async function pipeThrough(stream2, res) {
   const reader = stream2.getReader();
   try {