min-agent 0.2.0 → 0.3.0

package/dist/tools/web_fetch.js

@@ -1,8 +1,19 @@
 import { tool, jsonSchema } from "ai";
 import { truncateToolOutput } from "../tool-output.js";
-const FIRECRAWL_BASE = "https://fireclawl.xc.lonae.com";
+import { assertSafeUrl, readBodyLimited } from "../http.js";
+import { loadConfig } from "../config.js";
+import { configHint, describeFetchError, resolveBackendBase } from "./backend.js";
+export const FIRECRAWL_BACKEND = {
+    envName: "MIN_AGENT_FIRECRAWL_URL",
+    configKey: "webFetchURL",
+    fallback: "https://fireclawl.xc.lonae.com",
+};
+const MAX_BODY_BYTES = 2 * 1024 * 1024;
+const DIRECT_TIMEOUT_MS = 8000;
+const SCRAPE_TIMEOUT_MS = 35000;
+const BROWSER_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
 export const webFetchTool = tool({
-    description: "Fetch content from a URL. Use this to access web pages, APIs, or any HTTP resource. Returns the response body as text (or markdown for HTML pages). Supports JavaScript-rendered SPA pages via fallback.",
+    description: "Fetch content from a URL. Use this to access web pages, APIs, or any HTTP resource. Returns the response body as text, or markdown for HTML pages (JavaScript-rendered pages go through a scrape backend).",
     inputSchema: jsonSchema({
         type: "object",
         properties: {
@@ -12,72 +23,165 @@ export const webFetchTool = tool({
         required: ["url"],
     }),
     execute: async ({ url, method }) => {
-        // First try: direct fetch
-        const directResult = await directFetch(url, method);
-        // If we got meaningful content, return it
-        if (directResult && hasContent(directResult)) {
-            return directResult;
+        const target = (url ?? "").trim();
+        if (!target)
+            return 'Error: "url" must be a non-empty string.';
+        const blocked = await assertSafeUrl(target);
+        if (blocked)
+            return `Error: ${blocked}`;
+        const verb = (method ?? "GET").trim().toUpperCase() || "GET";
+        const direct = await directFetch(target, verb);
+        // Plain text / JSON responses are what the caller asked for; return them
+        // without paying for a scrape round trip.
+        if (direct.kind === "ok" && !direct.html && hasContent(direct.text)) {
+            return renderBody(direct.text, direct.bodyTruncated);
         }
-        // Fallback: use Firecrawl for SPA/dynamic pages
-        const firecrawlResult = await firecrawlFetch(url);
-        if (firecrawlResult)
-            return firecrawlResult;
-        // Return whatever we got from direct fetch
-        return directResult || "Failed to fetch content from URL";
+        if (direct.kind === "binary") {
+            return `Error: the response is not text (Content-Type: ${direct.contentType}). Use a different tool to download binary content.`;
+        }
+        // A non-GET request cannot be replayed by the scrape backend (it always
+        // performs a GET), so report the direct outcome as-is.
+        if (verb !== "GET") {
+            if (direct.kind === "ok")
+                return renderBody(direct.text, direct.bodyTruncated);
+            return `Error: ${describeDirect(direct, target)}`;
+        }
+        // HTML (or an empty/JS-rendered shell): the scrape backend turns it into
+        // markdown, which is both cheaper and more readable than raw markup.
+        const scraped = await firecrawlFetch(target);
+        if (scraped.kind === "ok")
+            return renderBody(scraped.markdown, scraped.bodyTruncated);
+        if (direct.kind === "ok" && hasContent(direct.text)) {
+            return [
+                `Note: returning raw HTML — the scrape backend could not convert this page (${scraped.reason}).`,
+                renderBody(direct.text, direct.bodyTruncated),
+            ].join("\n\n");
+        }
+        return [
+            `Failed to fetch content from ${target}.`,
+            `Direct request: ${describeDirect(direct, target)}`,
+            `Scrape backend: ${scraped.reason}`,
+            configHint(FIRECRAWL_BACKEND),
+        ].join("\n");
     },
 });
+function renderBody(text, bodyTruncated) {
+    const { content } = truncateToolOutput(text, { direction: "head" });
+    // readBodyLimited cutting at MAX_BODY_BYTES is a different truncation from
+    // truncateToolOutput's (which persists the full text), so disclose it.
+    if (!bodyTruncated)
+        return content;
+    return `${content}\n\n...(response body was cut off at ${MAX_BODY_BYTES / 1024 / 1024} MB)`;
+}
+function describeDirect(outcome, url) {
+    switch (outcome.kind) {
+        case "ok":
+            return `HTTP 200 but the body had almost no readable content (${url} may require JavaScript)`;
+        case "binary":
+            return `non-text response (${outcome.contentType})`;
+        case "http":
+            return `HTTP ${outcome.status}${outcome.statusText ? ` ${outcome.statusText}` : ""}`;
+        case "network":
+            return outcome.message;
+    }
+}
+function isHtml(contentType) {
+    return /html|xhtml/i.test(contentType);
+}
+/** Content types worth handing to the model as text. */
+function isTextual(contentType) {
+    if (!contentType)
+        return true; // Many APIs omit it; the NUL sniff below catches binaries.
+    const ct = contentType.toLowerCase();
+    if (ct.startsWith("text/"))
+        return true;
+    return /(json|xml|javascript|ecmascript|yaml|csv|graphql|x-www-form-urlencoded|\+json)/.test(ct);
+}
 async function directFetch(url, method) {
+    let response;
     try {
-        const response = await fetch(url, {
-            method: method ?? "GET",
+        response = await fetch(url, {
+            method,
             headers: {
-                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
-                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "User-Agent": BROWSER_UA,
+                Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.9,*/*;q=0.8",
             },
-            signal: AbortSignal.timeout(15000),
+            signal: AbortSignal.timeout(DIRECT_TIMEOUT_MS),
         });
-        if (!response.ok)
-            return `HTTP ${response.status}`;
-        const text = await response.text();
-        return truncateToolOutput(text, { direction: "head" }).content;
     }
     catch (err) {
-        return null;
+        return { kind: "network", message: describeFetchError(err) };
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    if (!response.ok)
+        return { kind: "http", status: response.status, statusText: response.statusText };
+    if (!isTextual(contentType))
+        return { kind: "binary", contentType };
+    try {
+        const { text, truncated } = await readBodyLimited(response, MAX_BODY_BYTES);
+        if (text.includes("\u0000"))
+            return { kind: "binary", contentType: contentType || "unknown (binary)" };
+        return { kind: "ok", text, bodyTruncated: truncated, html: isHtml(contentType) || looksLikeHtml(text) };
+    }
+    catch (err) {
+        return { kind: "network", message: describeFetchError(err) };
     }
 }
 async function firecrawlFetch(url) {
+    const resolved = resolveBackendBase(FIRECRAWL_BACKEND, loadConfig().webFetchURL);
+    if (!resolved.ok)
+        return { kind: "unavailable", reason: resolved.error };
+    let response;
     try {
-        const response = await fetch(`${FIRECRAWL_BASE}/v1/scrape`, {
+        response = await fetch(`${resolved.base}/v1/scrape`, {
             method: "POST",
             headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({
-                url,
-                formats: ["markdown"],
-                onlyMainContent: true,
-                waitFor: 3000,
-                timeout: 30000,
-            }),
-            signal: AbortSignal.timeout(35000),
+            body: JSON.stringify({ url, formats: ["markdown"], onlyMainContent: true, waitFor: 3000, timeout: 30000 }),
+            signal: AbortSignal.timeout(SCRAPE_TIMEOUT_MS),
         });
-        if (!response.ok)
-            return null;
-        const data = await response.json();
-        const markdown = data?.data?.markdown || data?.data?.content;
-        if (!markdown)
-            return null;
-        return truncateToolOutput(markdown, { direction: "head" }).content;
+    }
+    catch (err) {
+        return { kind: "unavailable", reason: `could not reach the scrape backend (${describeFetchError(err)})` };
+    }
+    let text;
+    let truncated;
+    try {
+        const body = await readBodyLimited(response, MAX_BODY_BYTES);
+        text = body.text;
+        truncated = body.truncated;
+    }
+    catch (err) {
+        return { kind: "unavailable", reason: `could not read the scrape response (${describeFetchError(err)})` };
+    }
+    let data;
+    try {
+        data = JSON.parse(text);
     }
     catch {
-        return null;
+        return {
+            kind: "unavailable",
+            reason: response.ok ? "the scrape backend returned a non-JSON response" : `HTTP ${response.status} from the scrape backend`,
+        };
     }
+    const record = typeof data === "object" && data !== null ? data : {};
+    if (!response.ok) {
+        const message = typeof record.error === "string" ? record.error : response.statusText;
+        return { kind: "unavailable", reason: `HTTP ${response.status} from the scrape backend${message ? ` (${message})` : ""}` };
+    }
+    const payload = typeof record.data === "object" && record.data !== null ? record.data : {};
+    const markdown = typeof payload.markdown === "string" ? payload.markdown : typeof payload.content === "string" ? payload.content : "";
+    if (!markdown.trim())
+        return { kind: "unavailable", reason: "the scrape backend returned no content for this URL" };
+    return { kind: "ok", markdown, bodyTruncated: truncated };
+}
+function looksLikeHtml(body) {
+    return /^\s*(<!doctype html|<html\b|<head\b)/i.test(body);
 }
-function hasContent(html) {
-    // Check if the response has meaningful content (not just an empty SPA shell)
-    if (html.length < 200)
+/** Whether a body carries enough readable text to be worth returning as-is. */
+function hasContent(body) {
+    if (body.trim().length < 200)
         return false;
-    // SPA shells typically have very little text content outside of script tags
-    const withoutScripts = html.replace(/<script[\s\S]*?<\/script>/gi, "");
+    const withoutScripts = body.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "");
     const textContent = withoutScripts.replace(/<[^>]*>/g, "").trim();
-    // If after removing scripts and tags there's less than 100 chars, it's likely an empty shell
     return textContent.length > 100;
 }