billion-context 0.1.72 → 0.1.73-pr.474.289

package/dist/index.js

@@ -53254,6 +53254,86 @@ async function compressLoopResponsesJson(initialResponse, ctx, requestBody, requ
   return current;
 }
 
+// src/fake-completion.ts
+var FAKE_COMPLETION_HINT = "[billion-context] Your last reply described a tool call as plain text instead of invoking it, so no tool actually ran. To act, invoke the tool through the proper tool-calling mechanism (emit a tool_use / tool_calls / function_call block). Do not write tool-call markup as text in your reply.";
+function maxFakeCompletionRetries() {
+  const n = Number.parseInt(process.env.BILI_FAKE_COMPLETION_RETRIES ?? "0", 10);
+  return Number.isFinite(n) && n >= 0 ? n : 0;
+}
+function fakeBufCap() {
+  const n = Number.parseInt(process.env.BILI_FAKE_BUF_CAP ?? String(16 * 1024 * 1024), 10);
+  return Number.isFinite(n) && n > 0 ? n : 16 * 1024 * 1024;
+}
+var ANTHROPIC_TOOL_BLOCK = /"type"\s*:\s*"tool_use"/;
+var OPENAI_TOOL_BLOCK = /"tool_calls"\s*:\s*\[\s*\{/;
+var RESPONSES_TOOL_BLOCK = /"type"\s*:\s*"function_call"/;
+function hasToolBlock(protocol, rawText) {
+  switch (protocol) {
+    case "anthropic":
+      return ANTHROPIC_TOOL_BLOCK.test(rawText);
+    case "openai":
+      return OPENAI_TOOL_BLOCK.test(rawText);
+    case "responses":
+      return RESPONSES_TOOL_BLOCK.test(rawText);
+  }
+}
+var TOOL_OPEN = /\x3c(?!\s*\/)(?:antml:)?(invoke|tool_calls|tool_call|parameter|parameters)\b|\\u003c(?!\s*\/)(?:antml:)?(invoke|tool_calls|tool_call|parameter|parameters)\b/i;
+var TOOL_CLOSE_GLOBAL = /\x3c\/(?:antml:)?(invoke|tool_calls|tool_call|parameter|parameters)\b|\\u003c\/(?:antml:)?(invoke|tool_calls|tool_call|parameter|parameters)\b/gi;
+function hasToolCallStructure(text) {
+  if (!containsToolCallXmlFragment(text)) return false;
+  if (TOOL_OPEN.test(text)) return true;
+  const closes = /* @__PURE__ */ new Set();
+  for (const m2 of text.matchAll(TOOL_CLOSE_GLOBAL)) {
+    const name = m2[1] ?? m2[2];
+    if (name) closes.add(name.toLowerCase());
+  }
+  return closes.size >= 2;
+}
+function isFakeCompletion(protocol, rawText) {
+  return hasToolCallStructure(rawText) && !hasToolBlock(protocol, rawText);
+}
+function injectFakeCompletionHint(protocol, body) {
+  const raw = typeof body === "string" ? body : body.toString("utf8");
+  let obj;
+  try {
+    obj = JSON.parse(raw);
+  } catch {
+    return null;
+  }
+  if (protocol === "responses") {
+    const input = obj.input;
+    if (!Array.isArray(input)) return null;
+    const arr2 = input;
+    const last2 = arr2[arr2.length - 1];
+    if (last2 && typeof last2 === "object" && last2.role === "user") {
+      const c = last2.content;
+      if (typeof c === "string") last2.content = `${c}
+
+${FAKE_COMPLETION_HINT}`;
+      else if (Array.isArray(c)) last2.content = [...c, { type: "input_text", text: FAKE_COMPLETION_HINT }];
+      else last2.content = FAKE_COMPLETION_HINT;
+    } else {
+      arr2.push({ role: "user", content: [{ type: "input_text", text: FAKE_COMPLETION_HINT }] });
+    }
+    return JSON.stringify(obj);
+  }
+  const messages = obj.messages;
+  if (!Array.isArray(messages)) return null;
+  const arr = messages;
+  const last = arr[arr.length - 1];
+  if (last && typeof last === "object" && last.role === "user") {
+    const c = last.content;
+    if (typeof c === "string") last.content = `${c}
+
+${FAKE_COMPLETION_HINT}`;
+    else if (Array.isArray(c)) last.content = [...c, { type: "text", text: FAKE_COMPLETION_HINT }];
+    else last.content = FAKE_COMPLETION_HINT;
+  } else {
+    arr.push({ role: "user", content: FAKE_COMPLETION_HINT });
+  }
+  return JSON.stringify(obj);
+}
+
 // src/stream-openai.ts
 function rewriteOpenaiJsonResponse(body, ctx) {
   if (!body || typeof body !== "object") return body;
@@ -54221,7 +54301,7 @@ function mergeUsageSample(acc, sample) {
   if (sample.cachedTokens !== void 0) acc.cachedTokens = sample.cachedTokens;
   if (sample.outputTokens !== void 0) acc.outputTokens = sample.outputTokens;
 }
-async function pipePluginChatWithStrip(stream2, res, session, protocol, log2) {
+async function pipePluginChatWithStrip(stream2, res, protocol, session, log2) {
   const reader = stream2.getReader();
   const decoder = new TextDecoder("utf-8");
   let buf = "";
@@ -54375,7 +54455,7 @@ data: ${JSON.stringify({ type: "content_block_delta", index, delta: { type: delt
     }
     const rest = flushTails();
     if (rest.length > 0 && !res.destroyed && !res.writableEnded) await write(rest);
-    if (acc.inputTokens !== void 0 || acc.outputTokens !== void 0 || acc.cachedTokens !== void 0) {
+    if (session && (acc.inputTokens !== void 0 || acc.outputTokens !== void 0 || acc.cachedTokens !== void 0)) {
       applyUsageSample(session, acc, protocol);
       markDirty(session);
     }
@@ -54484,7 +54564,7 @@ async function pipePluginResponsesWithStrip(stream2, res, session, log2) {
       const rest = flushTail("");
       if (rest.length > 0) await write(rest);
     }
-    if (acc.inputTokens !== void 0 || acc.outputTokens !== void 0 || acc.cachedTokens !== void 0) {
+    if (session && (acc.inputTokens !== void 0 || acc.outputTokens !== void 0 || acc.cachedTokens !== void 0)) {
       applyUsageSample(session, acc, "responses");
       markDirty(session);
     }
@@ -57511,7 +57591,7 @@ ${hdrText}
       if (prepared.protocol === "responses") {
         await pipePluginResponsesWithStrip(upstream.body, res, prepared.session, (msg) => log2("info", `[${prepared.session.id}] ${msg}`));
       } else {
-        await pipePluginChatWithStrip(upstream.body, res, prepared.session, prepared.protocol, (msg) => log2("info", `[${prepared.session.id}] ${msg}`));
+        await pipePluginChatWithStrip(upstream.body, res, prepared.protocol, prepared.session, (msg) => log2("info", `[${prepared.session.id}] ${msg}`));
       }
     } else {
       await pipePluginJson(upstream.body, res, prepared.session, prepared.protocol);
@@ -57519,10 +57599,24 @@ ${hdrText}
     clearUpstreamTimer();
     return;
   }
+  let responseBody = upstream.body;
+  if (prepared !== null && prepared.stream && maxFakeCompletionRetries() > 0) {
+    const resolvedBuf = await resolveFakeCompletion(upstream.body, {
+      protocol: prepared.protocol,
+      body,
+      upstreamUrl,
+      reqHeaders: buildForwardHeaders(headers),
+      proxyUrl,
+      signal: clientAbort.signal,
+      session: prepared.session,
+      log: log2
+    });
+    responseBody = bufferToStream(resolvedBuf);
+  }
   const useRewriter = prepared !== null && prepared.compressInjected && prepared.processedMessages.length > 0;
   if (!useRewriter || prepared === null) {
     if (prepared && prepared.resetAfterSuccess) {
-      const [toClient, toObserve] = upstream.body.tee();
+      const [toClient, toObserve] = responseBody.tee();
       const observed = observeResponsesTerminalState(toObserve, prepared.stream);
       await pipeThrough(toClient, res);
       clearUpstreamTimer();
@@ -57533,8 +57627,17 @@ ${hdrText}
       } else {
         log2("warn", `[${prepared.session.id}] native compact response terminal=${terminal}; rebase NOT scheduled`);
       }
+    } else if (prepared && prepared.stream && (upstream.headers.get("content-type") ?? "").includes("text/event-stream")) {
+      const p2 = prepared;
+      const tagLog = (msg) => log2("info", `[${p2.session.id}] ${msg}`);
+      if (p2.protocol === "responses") {
+        await pipePluginResponsesWithStrip(responseBody, res, void 0, tagLog);
+      } else {
+        await pipePluginChatWithStrip(responseBody, res, p2.protocol, void 0, tagLog);
+      }
+      clearUpstreamTimer();
     } else {
-      await pipeThrough(upstream.body, res);
+      await pipeThrough(responseBody, res);
       clearUpstreamTimer();
     }
     return;
@@ -57548,10 +57651,10 @@ ${hdrText}
     debug: opts.debug
   };
   if (prepared.stream) {
-    let streamToRead = upstream.body;
+    let streamToRead = responseBody;
     let dumpRaw;
     if (opts.dumpSse) {
-      const [a, b2] = upstream.body.tee();
+      const [a, b2] = responseBody.tee();
       streamToRead = a;
       dumpRaw = dumpStreamToFile(b2, opts.dumpSse, `${Date.now()}-${safeSessionId(prepared.session.id)}-raw.sse`);
     }
@@ -57677,6 +57780,58 @@ async function readStreamToBuffer(stream2, maxBytes = 1 << 20) {
   }
   return Buffer.concat(chunks);
 }
+function bufferToStream(buf) {
+  return new ReadableStream({
+    start(controller) {
+      controller.enqueue(new Uint8Array(buf));
+      controller.close();
+    }
+  });
+}
+async function resolveFakeCompletion(stream2, opts) {
+  let buffer = await readStreamToBuffer(stream2, fakeBufCap());
+  const max = maxFakeCompletionRetries();
+  const sid = opts.session.id;
+  const priorStreak = opts.session.metadata.fakeCompletionStreak ?? 0;
+  if (max > 0 && priorStreak < max && isFakeCompletion(opts.protocol, buffer.toString("utf8"))) {
+    for (let attempt = 1; attempt <= max && !opts.signal.aborted; attempt++) {
+      const hinted = injectFakeCompletionHint(opts.protocol, opts.body);
+      if (hinted === null) break;
+      opts.log("warn", `[${sid}] fake completion (tool-call XML, no tool block); retry ${attempt}/${max} with corrective hint`);
+      let r;
+      try {
+        r = await fetchWithTimeout(
+          opts.upstreamUrl,
+          {
+            method: "POST",
+            headers: opts.reqHeaders,
+            body: hinted,
+            ...opts.proxyUrl ? { dispatcher: proxyDispatcher(opts.proxyUrl) } : {}
+          },
+          void 0,
+          opts.signal
+        );
+      } catch (e) {
+        opts.log("warn", `[${sid}] fake-completion retry failed: ${String(e)}; presenting original`);
+        break;
+      }
+      try {
+        if (!r.response.ok || !r.response.body) {
+          opts.log("warn", `[${sid}] fake-completion retry rejected (HTTP ${r.response.status}); presenting original`);
+          break;
+        }
+        buffer = await readStreamToBuffer(r.response.body, fakeBufCap());
+      } finally {
+        r.clearTimer();
+      }
+      if (!isFakeCompletion(opts.protocol, buffer.toString("utf8"))) break;
+    }
+  }
+  const stillFake = isFakeCompletion(opts.protocol, buffer.toString("utf8"));
+  opts.session.metadata.fakeCompletionStreak = stillFake ? priorStreak + 1 : 0;
+  markDirty(opts.session);
+  return buffer;
+}
 async function pipeThrough(stream2, res) {
   const reader = stream2.getReader();
   try {