jh-web-gateway 2.1.1 → 2.3.0

package/dist/{chunk-7H2RJZN3.js → chunk-E6JMUHPA.js}

@@ -59,7 +59,7 @@ async function findOrOpenJhPage(browser) {
   const contexts = browser.contexts();
   const context = contexts.length > 0 ? contexts[0] : await browser.newContext();
   const page = await context.newPage();
-  await page.goto(JH_URL);
+  await page.goto(JH_URL, { waitUntil: "domcontentloaded", timeout: 3e4 });
   return page;
 }
 
@@ -638,102 +638,103 @@ async function sendChatRequestInner(page, credentials, request, options, isRetry
     }
   };
   await page.route(streamPattern, routeHandler);
-  const postResult = await page.evaluate(
-    async ({
-      apiBase,
-      bearerToken,
-      endpointPath,
-      requestBody
-    }) => {
-      try {
-        const res = await fetch(`${apiBase}/agents/chat/${endpointPath}`, {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            Accept: "text/event-stream",
-            Authorization: `Bearer ${bearerToken}`
-          },
-          body: JSON.stringify(requestBody)
-        });
-        if (!res.ok) {
-          return {
-            error: true,
-            status: res.status,
-            statusText: res.statusText,
-            body: (await res.text()).slice(0, 2e3),
-            contentType: ""
-          };
-        }
-        const ct = res.headers.get("content-type") ?? "";
-        if (ct.includes("text/event-stream")) {
-          const reader = res.body?.getReader();
-          if (!reader) return { error: true, status: 500, statusText: "No body", body: "No SSE body", contentType: ct };
-          const decoder = new TextDecoder();
-          let text = "";
-          while (true) {
-            const { done, value } = await reader.read();
-            if (done) break;
-            text += decoder.decode(value, { stream: true });
+  let result;
+  try {
+    const postResult = await page.evaluate(
+      async ({
+        apiBase,
+        bearerToken,
+        endpointPath,
+        requestBody
+      }) => {
+        try {
+          const res = await fetch(`${apiBase}/agents/chat/${endpointPath}`, {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+              Accept: "text/event-stream",
+              Authorization: `Bearer ${bearerToken}`
+            },
+            body: JSON.stringify(requestBody)
+          });
+          if (!res.ok) {
+            return {
+              error: true,
+              status: res.status,
+              statusText: res.statusText,
+              body: (await res.text()).slice(0, 2e3),
+              contentType: ""
+            };
           }
-          return { error: false, status: 200, statusText: "OK", body: text, contentType: ct };
+          const ct = res.headers.get("content-type") ?? "";
+          if (ct.includes("text/event-stream")) {
+            const reader = res.body?.getReader();
+            if (!reader) return { error: true, status: 500, statusText: "No body", body: "No SSE body", contentType: ct };
+            const decoder = new TextDecoder();
+            let text = "";
+            while (true) {
+              const { done, value } = await reader.read();
+              if (done) break;
+              text += decoder.decode(value, { stream: true });
+            }
+            return { error: false, status: 200, statusText: "OK", body: text, contentType: ct };
+          }
+          const bodyText = await res.text();
+          console.log(`[gateway] POST response content-type: ${ct}, body: ${bodyText.slice(0, 500)}`);
+          return { error: false, status: 200, statusText: "OK", body: bodyText, contentType: ct };
+        } catch (err) {
+          return { error: true, status: 500, statusText: "fetch error", body: String(err), contentType: "" };
         }
-        const bodyText = await res.text();
-        console.log(`[gateway] POST response content-type: ${ct}, body: ${bodyText.slice(0, 500)}`);
-        return { error: false, status: 200, statusText: "OK", body: bodyText, contentType: ct };
-      } catch (err) {
-        return { error: true, status: 500, statusText: "fetch error", body: String(err), contentType: "" };
+      },
+      {
+        apiBase: JH_API_BASE,
+        bearerToken: credentials.bearerToken,
+        endpointPath: endpoint,
+        requestBody: body
       }
-    },
-    {
-      apiBase: JH_API_BASE,
-      bearerToken: credentials.bearerToken,
-      endpointPath: endpoint,
-      requestBody: body
-    }
-  );
-  let result;
-  if (postResult.error) {
-    result = { error: true, status: postResult.status, statusText: postResult.statusText ?? "", body: postResult.body };
-    await page.unroute(streamPattern, routeHandler);
-  } else if (postResult.contentType.includes("text/event-stream")) {
-    result = { error: false, status: 200, statusText: "OK", body: postResult.body };
-    await page.unroute(streamPattern, routeHandler);
-  } else {
-    let streamId;
-    try {
-      streamId = JSON.parse(postResult.body).streamId;
-    } catch {
-    }
-    if (!streamId) {
+    );
+    if (postResult.error) {
+      result = { error: true, status: postResult.status, statusText: postResult.statusText ?? "", body: postResult.body };
+    } else if (postResult.contentType.includes("text/event-stream")) {
       result = { error: false, status: 200, statusText: "OK", body: postResult.body };
-      await page.unroute(streamPattern, routeHandler);
     } else {
-      const streamUrl = `${JH_API_BASE}/agents/chat/stream/${streamId}`;
-      page.evaluate(
-        async ({ url, token }) => {
-          await new Promise((r) => setTimeout(r, 10));
-          try {
-            await fetch(url, {
-              method: "GET",
-              headers: { Accept: "text/event-stream", Authorization: `Bearer ${token}` }
-            });
-          } catch {
-          }
-        },
-        { url: streamUrl, token: credentials.bearerToken }
-      ).catch(() => {
-      });
-      const timeout = new Promise(
-        (res) => setTimeout(() => {
-          if (!sseResolved) {
-            sseResolved = true;
-            res({ error: true, status: 408, statusText: "timeout", body: "Stream capture timed out after 120s" });
-          }
-        }, 12e4)
-      );
-      result = await Promise.race([ssePromise, timeout]);
-      await page.unroute(streamPattern, routeHandler);
+      let streamId;
+      try {
+        streamId = JSON.parse(postResult.body).streamId;
+      } catch {
+      }
+      if (!streamId) {
+        result = { error: false, status: 200, statusText: "OK", body: postResult.body };
+      } else {
+        const streamUrl = `${JH_API_BASE}/agents/chat/stream/${streamId}`;
+        page.evaluate(
+          async ({ url, token }) => {
+            await new Promise((r) => setTimeout(r, 10));
+            try {
+              await fetch(url, {
+                method: "GET",
+                headers: { Accept: "text/event-stream", Authorization: `Bearer ${token}` }
+              });
+            } catch {
+            }
+          },
+          { url: streamUrl, token: credentials.bearerToken }
+        ).catch(() => {
+        });
+        const timeout = new Promise(
+          (res) => setTimeout(() => {
+            if (!sseResolved) {
+              sseResolved = true;
+              res({ error: true, status: 408, statusText: "timeout", body: "Stream capture timed out after 120s" });
+            }
+          }, 12e4)
+        );
+        result = await Promise.race([ssePromise, timeout]);
+      }
     }
+  } finally {
+    await page.unroute(streamPattern, routeHandler).catch(() => {
+    });
   }
   if (result.error) {
     const status = result.status;
@@ -741,7 +742,7 @@ async function sendChatRequestInner(page, credentials, request, options, isRetry
     if (status === 401 && !isRetry) {
       const cdpUrl = options?.cdpUrl ?? "http://127.0.0.1:9222";
       try {
-        await page.reload({ waitUntil: "networkidle" });
+        await page.reload({ waitUntil: "commit" });
         const fresh = await captureCredentials(cdpUrl, 3e4);
         const newCreds = {
           bearerToken: fresh.bearerToken,
@@ -849,11 +850,12 @@ function modelsRouter(_config) {
 
 // src/routes/health.ts
 import { Hono as Hono2 } from "hono";
-function healthRouter(config, startTime) {
+function healthRouter(config, startTime, deps) {
   const app = new Hono2();
   app.get("/", (c) => {
     const uptime = (Date.now() - startTime) / 1e3;
-    const tokenExpiry = config.credentials?.bearerToken ? getTokenExpiry(config.credentials.bearerToken) || null : null;
+    const liveCreds = deps?.getCredentials?.() ?? config.credentials;
+    const tokenExpiry = liveCreds?.bearerToken ? getTokenExpiry(liveCreds.bearerToken) || null : null;
     const tokenExpired = tokenExpiry !== null && Date.now() / 1e3 > tokenExpiry;
     return c.json({
       status: "ok",
@@ -956,8 +958,11 @@ function escapeXmlAttr(s) {
 import { randomBytes as randomBytes2 } from "crypto";
 
 // src/core/tool-parser.ts
-var TOOL_CALL_RE = /<tool_call\s+id="([^"]*)"\s+name="([^"]*)">([\s\S]*?)<\/tool_call>/g;
+var TOOL_CALL_RE = /<tool_call\s+(?=[^>]*\bid="([^"]*)")(?=[^>]*\bname="([^"]*)")(?:[^>]*)>([\s\S]*?)<\/tool_call>/g;
+var TOOL_RESPONSE_RE = /<tool_response\b[^>]*>[\s\S]*?<\/tool_response>/g;
 var THINK_RE = /<think>([\s\S]*?)<\/think>/g;
+var PARTIAL_TOOL_CALL_RE = /<tool_call\b[^>]*>(?:(?!<\/tool_call>)[\s\S])*$/;
+var PARTIAL_TOOL_RESPONSE_RE = /<tool_response\b[^>]*>(?:(?!<\/tool_response>)[\s\S])*$/;
 function parseToolsAndThinking(text) {
   const toolCalls = [];
   let thinking = null;
@@ -966,6 +971,7 @@ function parseToolsAndThinking(text) {
     thinking = thinkMatches.map((m) => m[1]).join("\n");
   }
   let remaining = text.replace(THINK_RE, "");
+  remaining = remaining.replace(TOOL_RESPONSE_RE, "");
   const toolMatches = [...remaining.matchAll(TOOL_CALL_RE)];
   for (const match of toolMatches) {
     const id = match[1];
@@ -999,6 +1005,107 @@ function toOpenAIToolCalls(calls) {
     }
   }));
 }
+var WATCHED_TAGS = ["<tool_call", "<tool_response"];
+var MAX_TAG_PREFIX_LEN = Math.max(...WATCHED_TAGS.map((t) => t.length));
+function findWatchedTagPrefixAtEnd(text) {
+  const maxLen = Math.min(text.length, MAX_TAG_PREFIX_LEN);
+  for (let len = maxLen; len >= 1; len--) {
+    const tail = text.slice(-len);
+    for (const tag of WATCHED_TAGS) {
+      if (tag.startsWith(tail)) {
+        return text.length - len;
+      }
+    }
+  }
+  return -1;
+}
+function findPartialWatchedTag(text) {
+  if (PARTIAL_TOOL_RESPONSE_RE.test(text)) {
+    const idx = text.search(/<tool_response\b/);
+    if (idx >= 0) return idx;
+  }
+  if (PARTIAL_TOOL_CALL_RE.test(text)) {
+    const idx = text.search(/<tool_call\b/);
+    if (idx >= 0) return idx;
+  }
+  return -1;
+}
+function findUnclosedWatchedTag(text) {
+  for (const tag of WATCHED_TAGS) {
+    const idx = text.lastIndexOf(tag);
+    if (idx >= 0) {
+      const afterTag = text.slice(idx);
+      if (!afterTag.includes(">")) {
+        return idx;
+      }
+    }
+  }
+  return -1;
+}
+var StreamingToolBuffer = class {
+  buffer = "";
+  /**
+   * Push a text chunk. Returns an object with:
+   * - `text`: safe-to-emit text (outside any partial/complete tag)
+   * - `completedCalls`: fully parsed tool calls from this chunk
+   */
+  push(chunk) {
+    this.buffer += chunk;
+    const completedCalls = [];
+    let safeText = "";
+    this.buffer = this.buffer.replace(TOOL_RESPONSE_RE, (match2, offset) => {
+      return "";
+    });
+    let match;
+    const re = new RegExp(TOOL_CALL_RE.source, "g");
+    let lastIndex = 0;
+    while ((match = re.exec(this.buffer)) !== null) {
+      safeText += this.buffer.slice(lastIndex, match.index);
+      const id = match[1];
+      const name = match[2];
+      const rawArgs = match[3].trim();
+      let args;
+      try {
+        JSON.parse(rawArgs);
+        args = rawArgs;
+      } catch {
+        args = JSON.stringify(rawArgs);
+      }
+      completedCalls.push({ id, name, arguments: args });
+      lastIndex = match.index + match[0].length;
+    }
+    const remainder = this.buffer.slice(lastIndex);
+    const partialIdx = findPartialWatchedTag(remainder);
+    if (partialIdx >= 0) {
+      if (partialIdx > 0) {
+        safeText += remainder.slice(0, partialIdx);
+      }
+      this.buffer = remainder.slice(partialIdx);
+    } else {
+      const unclosedIdx = findUnclosedWatchedTag(remainder);
+      if (unclosedIdx >= 0) {
+        safeText += remainder.slice(0, unclosedIdx);
+        this.buffer = remainder.slice(unclosedIdx);
+      } else {
+        const prefixStart = findWatchedTagPrefixAtEnd(remainder);
+        if (prefixStart >= 0) {
+          safeText += remainder.slice(0, prefixStart);
+          this.buffer = remainder.slice(prefixStart);
+        } else {
+          safeText += remainder;
+          this.buffer = "";
+        }
+      }
+    }
+    return { text: safeText, completedCalls };
+  }
+  /** Flush any remaining buffer content as raw text (end of stream). */
+  flush() {
+    const remaining = this.buffer;
+    this.buffer = "";
+    return remaining;
+  }
+};
 
 // src/core/stream-translator.ts
 function parseSseEvents(rawSse) {
@@ -1107,6 +1214,9 @@ function translateToStream(rawSse, model, completionId) {
   const events = parseSseEvents(rawSse);
   const chunks = [];
   let lastMessageText = "";
+  const toolBuf = new StreamingToolBuffer();
+  let toolCallIndex = 0;
+  let hasToolCalls = false;
   chunks.push({
     id,
     object: "chat.completion.chunk",
@@ -1115,6 +1225,37 @@ function translateToStream(rawSse, model, completionId) {
     choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }]
   });
   let gotDeltas = false;
+  function processDelta(rawDelta) {
+    const { text, completedCalls } = toolBuf.push(rawDelta);
+    if (text) {
+      chunks.push({
+        id,
+        object: "chat.completion.chunk",
+        created,
+        model,
+        choices: [{ index: 0, delta: { content: text }, finish_reason: null }]
+      });
+    }
+    for (const call of completedCalls) {
+      hasToolCalls = true;
+      const toolDelta = {
+        index: toolCallIndex++,
+        id: call.id,
+        type: "function",
+        function: {
+          name: call.name,
+          arguments: call.arguments
+        }
+      };
+      chunks.push({
+        id,
+        object: "chat.completion.chunk",
+        created,
+        model,
+        choices: [{ index: 0, delta: { tool_calls: [toolDelta] }, finish_reason: null }]
+      });
+    }
+  }
   for (const ev of events) {
     const { type, parsed } = resolveEventType(ev);
     if (isUserEcho(parsed)) continue;
@@ -1123,13 +1264,7 @@ function translateToStream(rawSse, model, completionId) {
       const delta = extractDeltaText(parsed);
       if (delta !== null) {
         gotDeltas = true;
-        chunks.push({
-          id,
-          object: "chat.completion.chunk",
-          created,
-          model,
-          choices: [{ index: 0, delta: { content: delta }, finish_reason: null }]
-        });
+        processDelta(delta);
       }
     }
     if (type === "message" || !type) {
@@ -1138,23 +1273,27 @@ function translateToStream(rawSse, model, completionId) {
         const delta = msgText.slice(lastMessageText.length);
         lastMessageText = msgText;
         if (!gotDeltas && delta) {
-          chunks.push({
-            id,
-            object: "chat.completion.chunk",
-            created,
-            model,
-            choices: [{ index: 0, delta: { content: delta }, finish_reason: null }]
-          });
+          processDelta(delta);
         }
       }
     }
   }
+  const flushed = toolBuf.flush();
+  if (flushed) {
+    chunks.push({
+      id,
+      object: "chat.completion.chunk",
+      created,
+      model,
+      choices: [{ index: 0, delta: { content: flushed }, finish_reason: null }]
+    });
+  }
   chunks.push({
     id,
     object: "chat.completion.chunk",
     created,
     model,
-    choices: [{ index: 0, delta: {}, finish_reason: "stop" }]
+    choices: [{ index: 0, delta: {}, finish_reason: hasToolCalls ? "tool_calls" : "stop" }]
   });
   return chunks;
 }
@@ -1177,7 +1316,7 @@ function translateToCompletion(rawSse, model, completionId) {
     object: "chat.completion",
     created,
     model,
-    choices: [{ index: 0, message, finish_reason: "stop" }],
+    choices: [{ index: 0, message, finish_reason: parsed.toolCalls.length > 0 ? "tool_calls" : "stop" }],
     usage: {
       prompt_tokens: promptTokens,
       completion_tokens: completionTokens,
@@ -1288,11 +1427,14 @@ function chatCompletionsRouter(_config, deps) {
     const { page, queue, release } = acquired;
     const stats = pool.stats;
     console.log(`[chat] Acquired page (pool: ${stats.busy}/${stats.total} busy)`);
+    const fullPrompt = built.systemPrompt ? `${built.systemPrompt}
+
+${built.prompt}` : built.prompt;
     try {
       const response = await queue.enqueue(
         () => sendChatRequest(page, credentials, {
           model,
-          prompt: built.prompt
+          prompt: fullPrompt
         })
       );
       if (shouldStream) {
@@ -1349,7 +1491,7 @@ function chatCompletionsRouter(_config, deps) {
             const retryResponse = await retry.queue.enqueue(
               () => sendChatRequest(retry.page, freshCreds, {
                 model,
-                prompt: built.prompt
+                prompt: fullPrompt
               })
             );
             if (shouldStream) {
@@ -1523,10 +1665,27 @@ var Logger = class {
 };
 
 // src/server.ts
+function requestTrackerMiddleware(tracker) {
+  return async (c, next) => {
+    const id = crypto.randomUUID();
+    c.set("requestId", id);
+    tracker.start(id, c.req.method, c.req.path);
+    try {
+      await next();
+      tracker.end(id, c.res.status);
+    } catch (err) {
+      tracker.end(id, 500);
+      throw err;
+    }
+  };
+}
 function createServer(config, deps) {
   const app = new Hono4();
   const startTime = Date.now();
   const logger = new Logger();
+  if (deps?.requestTracker) {
+    app.use("*", requestTrackerMiddleware(deps.requestTracker));
+  }
   app.use("/v1/*", authMiddleware(config));
   app.use("*", async (c, next) => {
     const start = Date.now();
@@ -1559,7 +1718,7 @@ function createServer(config, deps) {
     });
   });
   app.route("/v1/models", modelsRouter(config));
-  app.route("/health", healthRouter(config, startTime));
+  app.route("/health", healthRouter(config, startTime, deps ? { getCredentials: deps.getCredentials } : void 0));
   if (deps) {
     app.route(
       "/v1/chat/completions",
@@ -1638,14 +1797,6 @@ async function startServer(config, deps) {
     }
     console.log("Shutdown complete.");
   };
-  process.on("SIGINT", async () => {
-    await shutdown();
-    process.exit(0);
-  });
-  process.on("SIGTERM", async () => {
-    await shutdown();
-    process.exit(0);
-  });
   return { close: shutdown };
 }
 
@@ -1668,7 +1819,7 @@ var RequestQueue = class {
       return await task();
     } finally {
       if (this.queue.length > 0) {
-        await new Promise((r) => setTimeout(r, 300));
+        await new Promise((r) => setTimeout(r, 100));
       }
       this.release();
     }
@@ -1718,6 +1869,7 @@ var PagePool = class {
   initPromise = null;
   pagesCreating = 0;
   warmedUp = false;
+  disconnected = false;
   constructor(options = {}) {
     this.targetUrl = options.targetUrl ?? "https://chat.ai.jh.edu";
     this.maxPages = options.maxPages ?? 1;
@@ -1731,6 +1883,11 @@ var PagePool = class {
   }
   async _doInit(browser, seedPage) {
     this.browser = browser;
+    this.disconnected = false;
+    browser.on("disconnected", () => {
+      console.warn("[PagePool] Browser disconnected \u2014 all pages are now invalid");
+      this.disconnected = true;
+    });
     this.pages.push({
       page: seedPage,
       queue: new RequestQueue(this.maxWaitMs),
@@ -1757,6 +1914,13 @@ var PagePool = class {
    * Call `markWarmedUp()` after the first successful request to enable scaling.
    */
   async acquire() {
+    if (this.disconnected) {
+      throw Object.assign(
+        new Error("Browser has disconnected. Restart the gateway to reconnect."),
+        { statusCode: 503 }
+      );
+    }
+    this.evictDeadPages();
     let pooled = this.pages.find((p2) => !p2.inUse);
     if (!pooled && this.warmedUp && this.pages.length + this.pagesCreating < this.maxPages && this.browser) {
       this.pagesCreating++;
@@ -1767,6 +1931,12 @@ var PagePool = class {
       }
     }
     if (!pooled) {
+      if (this.pages.length === 0) {
+        throw Object.assign(
+          new Error("No healthy browser pages available. Restart the gateway."),
+          { statusCode: 503 }
+        );
+      }
       pooled = this.pages.reduce(
         (a, b) => a.queue.pending <= b.queue.pending ? a : b
       );
@@ -1781,6 +1951,9 @@ var PagePool = class {
         if (!this.warmedUp) {
           this.warmedUp = true;
           console.log(`[PagePool] Warm-up complete \u2014 page scaling enabled (max ${this.maxPages})`);
+          if (this.maxPages > 1 && this.pages.length < this.maxPages && this.browser) {
+            this.preWarmPage();
+          }
         }
       }
     };
@@ -1803,7 +1976,7 @@ var PagePool = class {
     }
     const page = await context.newPage();
     try {
-      await page.goto(this.targetUrl, { waitUntil: "networkidle", timeout: 3e4 });
+      await page.goto(this.targetUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
       const finalUrl = page.url();
       if (!finalUrl.includes("chat.ai.jh.edu")) {
         throw new Error(
@@ -1824,6 +1997,36 @@ var PagePool = class {
       throw err;
     }
   }
+  /** Evict pages that have crashed or navigated away from JH */
+  evictDeadPages() {
+    const before = this.pages.length;
+    this.pages = this.pages.filter((p) => {
+      try {
+        if (p.page.isClosed()) return false;
+        const url = p.page.url();
+        if (!url.includes("chat.ai.jh.edu")) {
+          console.warn(`[PagePool] Evicting page \u2014 navigated away: ${url}`);
+          p.page.close().catch(() => {
+          });
+          return false;
+        }
+        return true;
+      } catch {
+        return false;
+      }
+    });
+    const evicted = before - this.pages.length;
+    if (evicted > 0) {
+      console.warn(`[PagePool] Evicted ${evicted} dead/stale page(s)`);
+    }
+  }
+  /** Pre-warm a new page in the background (fire-and-forget) */
+  preWarmPage() {
+    this.pagesCreating++;
+    this.createPage().then(() => console.log("[PagePool] Pre-warmed a new page")).catch((err) => console.warn(`[PagePool] Pre-warm failed: ${err.message}`)).finally(() => {
+      this.pagesCreating--;
+    });
+  }
   /** Close all pages except the seed page */
   async drain() {
     const toClose = this.pages.slice(1);
@@ -2192,4 +2395,4 @@ export {
   TokenRefresher,
   ChromeManager
 };
-//# sourceMappingURL=chunk-7H2RJZN3.js.map
+//# sourceMappingURL=chunk-E6JMUHPA.js.map