@code-yeongyu/senpi 2026.6.21 → 2026.6.28-4

package/dist/core/extensions/builtin/webfetch/webfetch/fetcher.js

@@ -1,28 +1,27 @@
+import { request } from "undici";
 import { InvalidWebfetchUrlError, WebfetchAbortError, WebfetchResponseTooLargeError, WebfetchTimeoutError, } from "./errors.js";
 export const MAX_RESPONSE_SIZE_BYTES = 5 * 1024 * 1024;
 export const DEFAULT_TIMEOUT_SECONDS = 30;
 export const MAX_TIMEOUT_SECONDS = 120;
+const MAX_REDIRECTS = 20;
 const BROWSER_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36";
+const CHROME_MAJOR_VERSION = "143";
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 export async function fetchUrl(options) {
     validateUrl(options.url);
     const timeoutSeconds = clampTimeout(options.timeoutSeconds);
+    const timeoutMs = timeoutSeconds * 1000;
     const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(new WebfetchTimeoutError(`Request timed out after ${timeoutSeconds}s`)), timeoutSeconds * 1000);
+    const timeout = setTimeout(() => controller.abort(new WebfetchTimeoutError(`Request timed out after ${timeoutSeconds}s`)), timeoutMs);
     const removeAbortForwarder = forwardAbort(options.signal, controller);
     try {
-        const response = await fetch(options.url, {
-            headers: buildHeaders(options.format, BROWSER_USER_AGENT),
+        const response = await requestUrl({
+            url: options.url,
+            format: options.format,
             signal: controller.signal,
+            timeoutMs,
         });
-        if (response.status === 403 && response.headers.get("cf-mitigated") === "challenge") {
-            await cancelBody(response);
-            const retry = await fetch(options.url, {
-                headers: buildHeaders(options.format, "pi-webfetch"),
-                signal: controller.signal,
-            });
-            return await readFetchResponse(options.url, retry, controller.signal);
-        }
-        return await readFetchResponse(options.url, response, controller.signal);
+        return await readHttpResponse(response, controller.signal);
     }
     finally {
         clearTimeout(timeout);
@@ -57,66 +56,126 @@ export function buildAcceptHeader(format) {
             return "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1";
     }
 }
-function buildHeaders(format, userAgent) {
+function buildHeaders(format) {
     return {
         Accept: buildAcceptHeader(format),
         "Accept-Language": "en-US,en;q=0.9",
-        "User-Agent": userAgent,
+        "Sec-CH-UA": `"Google Chrome";v="${CHROME_MAJOR_VERSION}", "Chromium";v="${CHROME_MAJOR_VERSION}", "Not A(Brand";v="24"`,
+        "Sec-CH-UA-Mobile": "?0",
+        "Sec-CH-UA-Platform": '"Windows"',
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "none",
+        "Sec-Fetch-User": "?1",
+        "Upgrade-Insecure-Requests": "1",
+        "User-Agent": BROWSER_USER_AGENT,
     };
 }
-async function readFetchResponse(url, response, signal) {
+async function requestUrl(options) {
+    const headers = buildHeaders(options.format);
+    let currentUrl = options.url;
+    for (let redirectCount = 0; redirectCount <= MAX_REDIRECTS; redirectCount += 1) {
+        const response = await request(currentUrl, {
+            method: "GET",
+            headers,
+            signal: options.signal,
+            headersTimeout: options.timeoutMs,
+            bodyTimeout: options.timeoutMs,
+        });
+        if (!REDIRECT_STATUSES.has(response.statusCode)) {
+            return {
+                url: currentUrl,
+                status: response.statusCode,
+                statusText: response.statusText,
+                headers: response.headers,
+                body: response.body,
+            };
+        }
+        const location = getHeader(response.headers, "location");
+        if (!location) {
+            return {
+                url: currentUrl,
+                status: response.statusCode,
+                statusText: response.statusText,
+                headers: response.headers,
+                body: response.body,
+            };
+        }
+        if (redirectCount === MAX_REDIRECTS) {
+            return {
+                url: currentUrl,
+                status: response.statusCode,
+                statusText: response.statusText,
+                headers: response.headers,
+                body: response.body,
+            };
+        }
+        await discardBody(response.body);
+        currentUrl = new URL(location, currentUrl).toString();
+    }
+    throw new WebfetchAbortError("Redirect resolution aborted");
+}
+async function readHttpResponse(response, signal) {
     await rejectOversizedContentLength(response);
     const body = await readResponseBody(response, signal);
     return {
-        url: response.url || url,
+        url: response.url,
         status: response.status,
         statusText: response.statusText,
-        contentType: response.headers.get("content-type") ?? "",
+        contentType: getHeader(response.headers, "content-type"),
         bytes: body.length,
         body,
         truncated: body.length === MAX_RESPONSE_SIZE_BYTES,
     };
 }
 async function rejectOversizedContentLength(response) {
-    const contentLength = response.headers.get("content-length");
+    const contentLength = getHeader(response.headers, "content-length");
     if (contentLength && Number.parseInt(contentLength, 10) > MAX_RESPONSE_SIZE_BYTES) {
-        await cancelBody(response);
+        await discardBody(response.body);
         throw new WebfetchResponseTooLargeError("Response too large (exceeds 5MB limit)");
     }
 }
-async function cancelBody(response) {
+function getHeader(headers, name) {
+    const value = headers[name.toLowerCase()];
+    if (Array.isArray(value))
+        return value.join(", ");
+    return value ?? "";
+}
+async function discardBody(body) {
     try {
-        await response.body?.cancel();
+        await body.dump({ limit: 1024 });
     }
-    catch {
-        // Preserve the caller's original failure.
+    catch (error) {
+        if (error instanceof Error) {
+            body.destroy(error);
+            return;
+        }
+        throw error;
     }
 }
 async function readResponseBody(response, signal) {
-    if (!response.body)
-        return new Uint8Array();
-    const reader = response.body.getReader();
     const chunks = [];
     let total = 0;
     try {
-        while (true) {
+        for await (const chunk of response.body) {
             if (signal.aborted) {
-                await cancelReader(reader);
+                response.body.destroy();
                 throw new WebfetchAbortError("Request aborted");
             }
-            const read = await reader.read();
-            if (read.done)
-                break;
-            chunks.push(read.value);
-            total += read.value.length;
+            const bytes = toUint8Array(chunk);
+            chunks.push(bytes);
+            total += bytes.length;
             if (total > MAX_RESPONSE_SIZE_BYTES) {
-                await cancelReader(reader);
+                response.body.destroy();
                 throw new WebfetchResponseTooLargeError("Response too large (exceeds 5MB limit)");
             }
         }
     }
-    finally {
-        reader.releaseLock();
+    catch (error) {
+        if (signal.aborted) {
+            throw new WebfetchAbortError("Request aborted");
+        }
+        throw error;
     }
     const body = new Uint8Array(total);
     let offset = 0;
@@ -126,13 +185,12 @@ async function readResponseBody(response, signal) {
     }
     return body;
 }
-async function cancelReader(reader) {
-    try {
-        await reader.cancel();
-    }
-    catch {
-        // Preserve the caller's original failure.
-    }
+function toUint8Array(chunk) {
+    if (chunk instanceof Uint8Array)
+        return chunk;
+    if (typeof chunk === "string")
+        return new TextEncoder().encode(chunk);
+    throw new Error("Unexpected response body chunk");
 }
 function forwardAbort(signal, controller) {
     if (!signal)

package/dist/core/extensions/builtin/webfetch/webfetch/fetcher.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,uBAAuB,EACvB,kBAAkB,EAClB,6BAA6B,EAC7B,oBAAoB,GACpB,MAAM,aAAa,CAAC;AAErB,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACvD,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAC1C,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEvC,MAAM,kBAAkB,GACvB,iHAAiH,CAAC;AAqBnH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB;IACnD,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,UAAU,CACzB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,oBAAoB,CAAC,2BAA2B,cAAc,GAAG,CAAC,CAAC,EAC9F,cAAc,GAAG,IAAI,CACrB,CAAC;IACF,MAAM,oBAAoB,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAEtE,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE;YACzC,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACzD,MAAM,EAAE,UAAU,CAAC,MAAM;SACzB,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,WAAW,EAAE,CAAC;YACrF,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE;gBACtC,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC;gBACpD,MAAM,EAAE,UAAU,CAAC,MAAM;aACzB,CAAC,CAAC;YACH,OAAO,MAAM,iBAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QACvE,CAAC;QAED,OAAO,MAAM,iBAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC;YAAS,CAAC;QACV,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,oBAAoB,EAAE,CAAC;IACxB,CAAC;AACF,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACtC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/D,MAAM,IAAI,uBAAuB,CAAC,yCAAyC,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,CAAC;QACJ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,uBAAuB,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;IAC1D,CAAC;AACF,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,cAAkC;IAC9D,IAAI,cAAc,KAAK,SAAS;QAAE,OAAO,uBAAuB,CAAC;IACjE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,cAAc,IAAI,CAAC;QAAE,OAAO,uBAAuB,CAAC;IAC5F,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,mBAAmB,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAsB;IACvD,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,UAAU;YACd,OAAO,0FAA0F,CAAC;QACnG,KAAK,MAAM;YACV,OAAO,mEAAmE,CAAC;QAC5E,KAAK,MAAM;YACV,OAAO,gGAAgG,CAAC;IAC1G,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,MAAsB,EAAE,SAAiB;IAC9D,OAAO;QACN,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC;QACjC,iBAAiB,EAAE,gBAAgB;QACnC,YAAY,EAAE,SAAS;KACvB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAW,EAAE,QAAkB,EAAE,MAAmB;IACpF,MAAM,4BAA4B,CAAC,QAAQ,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtD,OAAO;QACN,GAAG,EAAE,QAAQ,CAAC,GAAG,IAAI,GAAG;QACxB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE;QACvD,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM,KAAK,uBAAuB;KAClD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,4BAA4B,CAAC,QAAkB;IAC7D,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC7D,IAAI,aAAa,IAAI,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,uBAAuB,EAAE,CAAC;QACnF,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3B,MAAM,IAAI,6BAA6B,CAAC,wCAAwC,CAAC,CAAC;IACnF,CAAC;AACF,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,QAAkB;IAC3C,IAAI,CAAC;QACJ,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACR,0CAA0C;IAC3C,CAAC;AACF,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,QAAkB,EAAE,MAAmB;IACtE,IAAI,CAAC,QAAQ,CAAC,IAAI;QAAE,OAAO,IAAI,UAAU,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACzC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,IAAI,CAAC;QACJ,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC3B,MAAM,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;YACjD,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,IAAI;gBAAE,MAAM;YACrB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxB,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YAC3B,IAAI,KAAK,GAAG,uBAAuB,EAAE,CAAC;gBACrC,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC3B,MAAM,IAAI,6BAA6B,CAAC,wCAAwC,CAAC,CAAC;YACnF,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,MAAM,CAAC,WAAW,EAAE,CAAC;IACtB,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;IACxB,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAA+C;IAC1E,IAAI,CAAC;QACJ,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,0CAA0C;IAC3C,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,MAA+B,EAAE,UAA2B;IACjF,IAAI,CAAC,MAAM;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IAC7B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,QAAQ,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC7D,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC5D,CAAC","sourcesContent":["import {\n\tInvalidWebfetchUrlError,\n\tWebfetchAbortError,\n\tWebfetchResponseTooLargeError,\n\tWebfetchTimeoutError,\n} from \"./errors.ts\";\n\nexport const MAX_RESPONSE_SIZE_BYTES = 5 * 1024 * 1024;\nexport const DEFAULT_TIMEOUT_SECONDS = 30;\nexport const MAX_TIMEOUT_SECONDS = 120;\n\nconst BROWSER_USER_AGENT =\n\t\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36\";\n\nexport type WebfetchFormat = \"markdown\" | \"text\" | \"html\";\n\nexport interface FetchOptions {\n\turl: string;\n\tformat: WebfetchFormat;\n\ttimeoutSeconds?: number;\n\tsignal?: AbortSignal;\n}\n\nexport interface FetchResult {\n\turl: string;\n\tstatus: number;\n\tstatusText: string;\n\tcontentType: string;\n\tbytes: number;\n\tbody: Uint8Array;\n\ttruncated: boolean;\n}\n\nexport async function fetchUrl(options: FetchOptions): Promise<FetchResult> {\n\tvalidateUrl(options.url);\n\n\tconst timeoutSeconds = clampTimeout(options.timeoutSeconds);\n\tconst controller = new AbortController();\n\tconst timeout = setTimeout(\n\t\t() => controller.abort(new WebfetchTimeoutError(`Request timed out after ${timeoutSeconds}s`)),\n\t\ttimeoutSeconds * 1000,\n\t);\n\tconst removeAbortForwarder = forwardAbort(options.signal, controller);\n\n\ttry {\n\t\tconst response = await fetch(options.url, {\n\t\t\theaders: buildHeaders(options.format, BROWSER_USER_AGENT),\n\t\t\tsignal: controller.signal,\n\t\t});\n\n\t\tif (response.status === 403 && response.headers.get(\"cf-mitigated\") === \"challenge\") {\n\t\t\tawait cancelBody(response);\n\t\t\tconst retry = await fetch(options.url, {\n\t\t\t\theaders: buildHeaders(options.format, \"pi-webfetch\"),\n\t\t\t\tsignal: controller.signal,\n\t\t\t});\n\t\t\treturn await readFetchResponse(options.url, retry, controller.signal);\n\t\t}\n\n\t\treturn await readFetchResponse(options.url, response, controller.signal);\n\t} finally {\n\t\tclearTimeout(timeout);\n\t\tremoveAbortForwarder();\n\t}\n}\n\nexport function validateUrl(url: string): void {\n\tif (!url.startsWith(\"http://\") && !url.startsWith(\"https://\")) {\n\t\tthrow new InvalidWebfetchUrlError(\"URL must start with http:// or https://\");\n\t}\n\n\ttry {\n\t\tnew URL(url);\n\t} catch {\n\t\tthrow new InvalidWebfetchUrlError(`Invalid URL: ${url}`);\n\t}\n}\n\nexport function clampTimeout(timeoutSeconds: number | undefined): number {\n\tif (timeoutSeconds === undefined) return DEFAULT_TIMEOUT_SECONDS;\n\tif (!Number.isFinite(timeoutSeconds) || timeoutSeconds <= 0) return DEFAULT_TIMEOUT_SECONDS;\n\treturn Math.min(Math.ceil(timeoutSeconds), MAX_TIMEOUT_SECONDS);\n}\n\nexport function buildAcceptHeader(format: WebfetchFormat): string {\n\tswitch (format) {\n\t\tcase \"markdown\":\n\t\t\treturn \"text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1\";\n\t\tcase \"text\":\n\t\t\treturn \"text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1\";\n\t\tcase \"html\":\n\t\t\treturn \"text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1\";\n\t}\n}\n\nfunction buildHeaders(format: WebfetchFormat, userAgent: string): Record<string, string> {\n\treturn {\n\t\tAccept: buildAcceptHeader(format),\n\t\t\"Accept-Language\": \"en-US,en;q=0.9\",\n\t\t\"User-Agent\": userAgent,\n\t};\n}\n\nasync function readFetchResponse(url: string, response: Response, signal: AbortSignal): Promise<FetchResult> {\n\tawait rejectOversizedContentLength(response);\n\tconst body = await readResponseBody(response, signal);\n\treturn {\n\t\turl: response.url || url,\n\t\tstatus: response.status,\n\t\tstatusText: response.statusText,\n\t\tcontentType: response.headers.get(\"content-type\") ?? \"\",\n\t\tbytes: body.length,\n\t\tbody,\n\t\ttruncated: body.length === MAX_RESPONSE_SIZE_BYTES,\n\t};\n}\n\nasync function rejectOversizedContentLength(response: Response): Promise<void> {\n\tconst contentLength = response.headers.get(\"content-length\");\n\tif (contentLength && Number.parseInt(contentLength, 10) > MAX_RESPONSE_SIZE_BYTES) {\n\t\tawait cancelBody(response);\n\t\tthrow new WebfetchResponseTooLargeError(\"Response too large (exceeds 5MB limit)\");\n\t}\n}\n\nasync function cancelBody(response: Response): Promise<void> {\n\ttry {\n\t\tawait response.body?.cancel();\n\t} catch {\n\t\t// Preserve the caller's original failure.\n\t}\n}\n\nasync function readResponseBody(response: Response, signal: AbortSignal): Promise<Uint8Array> {\n\tif (!response.body) return new Uint8Array();\n\n\tconst reader = response.body.getReader();\n\tconst chunks: Uint8Array[] = [];\n\tlet total = 0;\n\n\ttry {\n\t\twhile (true) {\n\t\t\tif (signal.aborted) {\n\t\t\t\tawait cancelReader(reader);\n\t\t\t\tthrow new WebfetchAbortError(\"Request aborted\");\n\t\t\t}\n\t\t\tconst read = await reader.read();\n\t\t\tif (read.done) break;\n\t\t\tchunks.push(read.value);\n\t\t\ttotal += read.value.length;\n\t\t\tif (total > MAX_RESPONSE_SIZE_BYTES) {\n\t\t\t\tawait cancelReader(reader);\n\t\t\t\tthrow new WebfetchResponseTooLargeError(\"Response too large (exceeds 5MB limit)\");\n\t\t\t}\n\t\t}\n\t} finally {\n\t\treader.releaseLock();\n\t}\n\n\tconst body = new Uint8Array(total);\n\tlet offset = 0;\n\tfor (const chunk of chunks) {\n\t\tbody.set(chunk, offset);\n\t\toffset += chunk.length;\n\t}\n\treturn body;\n}\n\nasync function cancelReader(reader: ReadableStreamDefaultReader<Uint8Array>): Promise<void> {\n\ttry {\n\t\tawait reader.cancel();\n\t} catch {\n\t\t// Preserve the caller's original failure.\n\t}\n}\n\nfunction forwardAbort(signal: AbortSignal | undefined, controller: AbortController): () => void {\n\tif (!signal) return () => {};\n\tif (signal.aborted) {\n\t\tcontroller.abort(signal.reason);\n\t\treturn () => {};\n\t}\n\n\tconst listener = (): void => controller.abort(signal.reason);\n\tsignal.addEventListener(\"abort\", listener, { once: true });\n\treturn () => signal.removeEventListener(\"abort\", listener);\n}\n"]}
+{"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/fetcher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAEjC,OAAO,EACN,uBAAuB,EACvB,kBAAkB,EAClB,6BAA6B,EAC7B,oBAAoB,GACpB,MAAM,aAAa,CAAC;AAErB,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACvD,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAC1C,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AACvC,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,kBAAkB,GACvB,iHAAiH,CAAC;AACnH,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAkC7D,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB;IACnD,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,cAAc,GAAG,IAAI,CAAC;IACxC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,UAAU,CACzB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,oBAAoB,CAAC,2BAA2B,cAAc,GAAG,CAAC,CAAC,EAC9F,SAAS,CACT,CAAC;IACF,MAAM,oBAAoB,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAEtE,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC;YACjC,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,SAAS;SACT,CAAC,CAAC;QAEH,OAAO,MAAM,gBAAgB,CAAC,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;YAAS,CAAC;QACV,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,oBAAoB,EAAE,CAAC;IACxB,CAAC;AACF,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACtC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/D,MAAM,IAAI,uBAAuB,CAAC,yCAAyC,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,CAAC;QACJ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,uBAAuB,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;IAC1D,CAAC;AACF,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,cAAkC;IAC9D,IAAI,cAAc,KAAK,SAAS;QAAE,OAAO,uBAAuB,CAAC;IACjE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,cAAc,IAAI,CAAC;QAAE,OAAO,uBAAuB,CAAC;IAC5F,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,mBAAmB,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAsB;IACvD,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,UAAU;YACd,OAAO,0FAA0F,CAAC;QACnG,KAAK,MAAM;YACV,OAAO,mEAAmE,CAAC;QAC5E,KAAK,MAAM;YACV,OAAO,gGAAgG,CAAC;IAC1G,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,MAAsB;IAC3C,OAAO;QACN,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC;QACjC,iBAAiB,EAAE,gBAAgB;QACnC,WAAW,EAAE,sBAAsB,oBAAoB,oBAAoB,oBAAoB,yBAAyB;QACxH,kBAAkB,EAAE,IAAI;QACxB,oBAAoB,EAAE,WAAW;QACjC,gBAAgB,EAAE,UAAU;QAC5B,gBAAgB,EAAE,UAAU;QAC5B,gBAAgB,EAAE,MAAM;QACxB,gBAAgB,EAAE,IAAI;QACtB,2BAA2B,EAAE,GAAG;QAChC,YAAY,EAAE,kBAAkB;KAChC,CAAC;AACH,CAAC;AASD,KAAK,UAAU,UAAU,CAAC,OAA0B;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC;IAE7B,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,IAAI,aAAa,EAAE,aAAa,IAAI,CAAC,EAAE,CAAC;QAChF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE;YAC1C,MAAM,EAAE,KAAK;YACb,OAAO;YACP,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,cAAc,EAAE,OAAO,CAAC,SAAS;YACjC,WAAW,EAAE,OAAO,CAAC,SAAS;SAC9B,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YACjD,OAAO;gBACN,GAAG,EAAE,UAAU;gBACf,MAAM,EAAE,QAAQ,CAAC,UAAU;gBAC3B,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,IAAI,EAAE,QAAQ,CAAC,IAAI;aACnB,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,OAAO;gBACN,GAAG,EAAE,UAAU;gBACf,MAAM,EAAE,QAAQ,CAAC,UAAU;gBAC3B,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,IAAI,EAAE,QAAQ,CAAC,IAAI;aACnB,CAAC;QACH,CAAC;QAED,IAAI,aAAa,KAAK,aAAa,EAAE,CAAC;YACrC,OAAO;gBACN,GAAG,EAAE,UAAU;gBACf,MAAM,EAAE,QAAQ,CAAC,UAAU;gBAC3B,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,IAAI,EAAE,QAAQ,CAAC,IAAI;aACnB,CAAC;QACH,CAAC;QAED,MAAM,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAC;IACvD,CAAC;IAED,MAAM,IAAI,kBAAkB,CAAC,6BAA6B,CAAC,CAAC;AAC7D,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,QAAsB,EAAE,MAAmB;IAC1E,MAAM,4BAA4B,CAAC,QAAQ,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtD,OAAO;QACN,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,WAAW,EAAE,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;QACxD,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM,KAAK,uBAAuB;KAClD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,4BAA4B,CAAC,QAAsB;IACjE,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IACpE,IAAI,aAAa,IAAI,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,uBAAuB,EAAE,CAAC;QACnF,MAAM,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,IAAI,6BAA6B,CAAC,wCAAwC,CAAC,CAAC;IACnF,CAAC;AACF,CAAC;AAED,SAAS,SAAS,CAAC,OAA4B,EAAE,IAAY;IAC5D,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,OAAO,KAAK,IAAI,EAAE,CAAC;AACpB,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAwB;IAClD,IAAI,CAAC;QACJ,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO;QACR,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;AACF,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,QAAsB,EAAE,MAAmB;IAC1E,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;YACjD,CAAC;YACD,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACtB,IAAI,KAAK,GAAG,uBAAuB,EAAE,CAAC;gBACrC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,IAAI,6BAA6B,CAAC,wCAAwC,CAAC,CAAC;YACnF,CAAC;QACF,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;QACjD,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;IACxB,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IACnC,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,KAAK,CAAC;IAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,YAAY,CAAC,MAA+B,EAAE,UAA2B;IACjF,IAAI,CAAC,MAAM;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IAC7B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,QAAQ,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC7D,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC5D,CAAC","sourcesContent":["import type { IncomingHttpHeaders } from \"node:http\";\nimport { request } from \"undici\";\n\nimport {\n\tInvalidWebfetchUrlError,\n\tWebfetchAbortError,\n\tWebfetchResponseTooLargeError,\n\tWebfetchTimeoutError,\n} from \"./errors.ts\";\n\nexport const MAX_RESPONSE_SIZE_BYTES = 5 * 1024 * 1024;\nexport const DEFAULT_TIMEOUT_SECONDS = 30;\nexport const MAX_TIMEOUT_SECONDS = 120;\nconst MAX_REDIRECTS = 20;\n\nconst BROWSER_USER_AGENT =\n\t\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36\";\nconst CHROME_MAJOR_VERSION = \"143\";\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);\n\nexport type WebfetchFormat = \"markdown\" | \"text\" | \"html\";\n\nexport interface FetchOptions {\n\turl: string;\n\tformat: WebfetchFormat;\n\ttimeoutSeconds?: number;\n\tsignal?: AbortSignal;\n}\n\nexport interface FetchResult {\n\turl: string;\n\tstatus: number;\n\tstatusText: string;\n\tcontentType: string;\n\tbytes: number;\n\tbody: Uint8Array;\n\ttruncated: boolean;\n}\n\ninterface HttpResponse {\n\treadonly url: string;\n\treadonly status: number;\n\treadonly statusText: string;\n\treadonly headers: IncomingHttpHeaders;\n\treadonly body: ResponseBodyStream;\n}\n\ninterface ResponseBodyStream extends AsyncIterable<unknown> {\n\tdestroy(error?: Error): void;\n\tdump(options?: { limit: number; signal?: AbortSignal }): Promise<void>;\n}\n\nexport async function fetchUrl(options: FetchOptions): Promise<FetchResult> {\n\tvalidateUrl(options.url);\n\n\tconst timeoutSeconds = clampTimeout(options.timeoutSeconds);\n\tconst timeoutMs = timeoutSeconds * 1000;\n\tconst controller = new AbortController();\n\tconst timeout = setTimeout(\n\t\t() => controller.abort(new WebfetchTimeoutError(`Request timed out after ${timeoutSeconds}s`)),\n\t\ttimeoutMs,\n\t);\n\tconst removeAbortForwarder = forwardAbort(options.signal, controller);\n\n\ttry {\n\t\tconst response = await requestUrl({\n\t\t\turl: options.url,\n\t\t\tformat: options.format,\n\t\t\tsignal: controller.signal,\n\t\t\ttimeoutMs,\n\t\t});\n\n\t\treturn await readHttpResponse(response, controller.signal);\n\t} finally {\n\t\tclearTimeout(timeout);\n\t\tremoveAbortForwarder();\n\t}\n}\n\nexport function validateUrl(url: string): void {\n\tif (!url.startsWith(\"http://\") && !url.startsWith(\"https://\")) {\n\t\tthrow new InvalidWebfetchUrlError(\"URL must start with http:// or https://\");\n\t}\n\n\ttry {\n\t\tnew URL(url);\n\t} catch {\n\t\tthrow new InvalidWebfetchUrlError(`Invalid URL: ${url}`);\n\t}\n}\n\nexport function clampTimeout(timeoutSeconds: number | undefined): number {\n\tif (timeoutSeconds === undefined) return DEFAULT_TIMEOUT_SECONDS;\n\tif (!Number.isFinite(timeoutSeconds) || timeoutSeconds <= 0) return DEFAULT_TIMEOUT_SECONDS;\n\treturn Math.min(Math.ceil(timeoutSeconds), MAX_TIMEOUT_SECONDS);\n}\n\nexport function buildAcceptHeader(format: WebfetchFormat): string {\n\tswitch (format) {\n\t\tcase \"markdown\":\n\t\t\treturn \"text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1\";\n\t\tcase \"text\":\n\t\t\treturn \"text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1\";\n\t\tcase \"html\":\n\t\t\treturn \"text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1\";\n\t}\n}\n\nfunction buildHeaders(format: WebfetchFormat): Record<string, string> {\n\treturn {\n\t\tAccept: buildAcceptHeader(format),\n\t\t\"Accept-Language\": \"en-US,en;q=0.9\",\n\t\t\"Sec-CH-UA\": `\"Google Chrome\";v=\"${CHROME_MAJOR_VERSION}\", \"Chromium\";v=\"${CHROME_MAJOR_VERSION}\", \"Not A(Brand\";v=\"24\"`,\n\t\t\"Sec-CH-UA-Mobile\": \"?0\",\n\t\t\"Sec-CH-UA-Platform\": '\"Windows\"',\n\t\t\"Sec-Fetch-Dest\": \"document\",\n\t\t\"Sec-Fetch-Mode\": \"navigate\",\n\t\t\"Sec-Fetch-Site\": \"none\",\n\t\t\"Sec-Fetch-User\": \"?1\",\n\t\t\"Upgrade-Insecure-Requests\": \"1\",\n\t\t\"User-Agent\": BROWSER_USER_AGENT,\n\t};\n}\n\ninterface RequestUrlOptions {\n\treadonly url: string;\n\treadonly format: WebfetchFormat;\n\treadonly signal: AbortSignal;\n\treadonly timeoutMs: number;\n}\n\nasync function requestUrl(options: RequestUrlOptions): Promise<HttpResponse> {\n\tconst headers = buildHeaders(options.format);\n\tlet currentUrl = options.url;\n\n\tfor (let redirectCount = 0; redirectCount <= MAX_REDIRECTS; redirectCount += 1) {\n\t\tconst response = await request(currentUrl, {\n\t\t\tmethod: \"GET\",\n\t\t\theaders,\n\t\t\tsignal: options.signal,\n\t\t\theadersTimeout: options.timeoutMs,\n\t\t\tbodyTimeout: options.timeoutMs,\n\t\t});\n\n\t\tif (!REDIRECT_STATUSES.has(response.statusCode)) {\n\t\t\treturn {\n\t\t\t\turl: currentUrl,\n\t\t\t\tstatus: response.statusCode,\n\t\t\t\tstatusText: response.statusText,\n\t\t\t\theaders: response.headers,\n\t\t\t\tbody: response.body,\n\t\t\t};\n\t\t}\n\n\t\tconst location = getHeader(response.headers, \"location\");\n\t\tif (!location) {\n\t\t\treturn {\n\t\t\t\turl: currentUrl,\n\t\t\t\tstatus: response.statusCode,\n\t\t\t\tstatusText: response.statusText,\n\t\t\t\theaders: response.headers,\n\t\t\t\tbody: response.body,\n\t\t\t};\n\t\t}\n\n\t\tif (redirectCount === MAX_REDIRECTS) {\n\t\t\treturn {\n\t\t\t\turl: currentUrl,\n\t\t\t\tstatus: response.statusCode,\n\t\t\t\tstatusText: response.statusText,\n\t\t\t\theaders: response.headers,\n\t\t\t\tbody: response.body,\n\t\t\t};\n\t\t}\n\n\t\tawait discardBody(response.body);\n\t\tcurrentUrl = new URL(location, currentUrl).toString();\n\t}\n\n\tthrow new WebfetchAbortError(\"Redirect resolution aborted\");\n}\n\nasync function readHttpResponse(response: HttpResponse, signal: AbortSignal): Promise<FetchResult> {\n\tawait rejectOversizedContentLength(response);\n\tconst body = await readResponseBody(response, signal);\n\treturn {\n\t\turl: response.url,\n\t\tstatus: response.status,\n\t\tstatusText: response.statusText,\n\t\tcontentType: getHeader(response.headers, \"content-type\"),\n\t\tbytes: body.length,\n\t\tbody,\n\t\ttruncated: body.length === MAX_RESPONSE_SIZE_BYTES,\n\t};\n}\n\nasync function rejectOversizedContentLength(response: HttpResponse): Promise<void> {\n\tconst contentLength = getHeader(response.headers, \"content-length\");\n\tif (contentLength && Number.parseInt(contentLength, 10) > MAX_RESPONSE_SIZE_BYTES) {\n\t\tawait discardBody(response.body);\n\t\tthrow new WebfetchResponseTooLargeError(\"Response too large (exceeds 5MB limit)\");\n\t}\n}\n\nfunction getHeader(headers: IncomingHttpHeaders, name: string): string {\n\tconst value = headers[name.toLowerCase()];\n\tif (Array.isArray(value)) return value.join(\", \");\n\treturn value ?? \"\";\n}\n\nasync function discardBody(body: ResponseBodyStream): Promise<void> {\n\ttry {\n\t\tawait body.dump({ limit: 1024 });\n\t} catch (error) {\n\t\tif (error instanceof Error) {\n\t\t\tbody.destroy(error);\n\t\t\treturn;\n\t\t}\n\t\tthrow error;\n\t}\n}\n\nasync function readResponseBody(response: HttpResponse, signal: AbortSignal): Promise<Uint8Array> {\n\tconst chunks: Uint8Array[] = [];\n\tlet total = 0;\n\n\ttry {\n\t\tfor await (const chunk of response.body) {\n\t\t\tif (signal.aborted) {\n\t\t\t\tresponse.body.destroy();\n\t\t\t\tthrow new WebfetchAbortError(\"Request aborted\");\n\t\t\t}\n\t\t\tconst bytes = toUint8Array(chunk);\n\t\t\tchunks.push(bytes);\n\t\t\ttotal += bytes.length;\n\t\t\tif (total > MAX_RESPONSE_SIZE_BYTES) {\n\t\t\t\tresponse.body.destroy();\n\t\t\t\tthrow new WebfetchResponseTooLargeError(\"Response too large (exceeds 5MB limit)\");\n\t\t\t}\n\t\t}\n\t} catch (error) {\n\t\tif (signal.aborted) {\n\t\t\tthrow new WebfetchAbortError(\"Request aborted\");\n\t\t}\n\t\tthrow error;\n\t}\n\n\tconst body = new Uint8Array(total);\n\tlet offset = 0;\n\tfor (const chunk of chunks) {\n\t\tbody.set(chunk, offset);\n\t\toffset += chunk.length;\n\t}\n\treturn body;\n}\n\nfunction toUint8Array(chunk: unknown): Uint8Array {\n\tif (chunk instanceof Uint8Array) return chunk;\n\tif (typeof chunk === \"string\") return new TextEncoder().encode(chunk);\n\tthrow new Error(\"Unexpected response body chunk\");\n}\n\nfunction forwardAbort(signal: AbortSignal | undefined, controller: AbortController): () => void {\n\tif (!signal) return () => {};\n\tif (signal.aborted) {\n\t\tcontroller.abort(signal.reason);\n\t\treturn () => {};\n\t}\n\n\tconst listener = (): void => controller.abort(signal.reason);\n\tsignal.addEventListener(\"abort\", listener, { once: true });\n\treturn () => signal.removeEventListener(\"abort\", listener);\n}\n"]}

package/dist/core/extensions/builtin/webfetch/webfetch/tool.js

@@ -42,11 +42,11 @@ export const webfetch = defineTool({
         let text = raw;
         let converted = false;
         if (isHtml && format === "markdown") {
-            text = htmlToMarkdown(raw);
+            text = htmlToMarkdown(raw, fetched.url);
             converted = true;
         }
         else if (isHtml && format === "text") {
-            text = htmlToText(raw);
+            text = htmlToText(raw, fetched.url);
             converted = true;
         }
         const details = {

package/dist/core/extensions/builtin/webfetch/webfetch/tool.js.map

@@ -1 +1 @@
-{"version":3,"file":"tool.js","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAuB,MAAM,cAAc,CAAC;AAE3E,MAAM,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAU,CAAC;AAE/D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC1B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,+BAA+B,EAAE,CAAC;IAClE,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,UAAU,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAU,EAAE;QACjD,WAAW,EAAE,4DAA4D;KACzE,CAAC,CACF;IACD,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,2CAA2C,EAAE,CAAC,CAAC;CACjG,CAAC,CAAC;AAwBH,MAAM,CAAC,MAAM,QAAQ,GAAG,UAAU,CAAuC;IACxE,IAAI,EAAE,UAAU;IAChB,KAAK,EAAE,WAAW;IAClB,WAAW,EACV,8EAA8E;QAC9E,6DAA6D;IAC9D,aAAa,EAAE,2DAA2D;IAC1E,gBAAgB,EAAE;QACjB,qDAAqD;QACrD,4EAA4E;QAC5E,kDAAkD;KAClD;IACD,UAAU,EAAE,MAAM;IAClB,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI;QACxD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACpD,QAAQ,EAAE,CAAC;YACV,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,MAAM,CAAC,GAAG,OAAO,MAAM,aAAa,cAAc,IAAI,EAAE,CAAC;YACrG,OAAO,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,cAAc,EAAE;SACvE,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC;YAC9B,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,MAAM;YACN,cAAc;YACd,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;SAC3C,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;QAClG,IAAI,IAAI,GAAG,GAAG,CAAC;QACf,IAAI,SAAS,GAAG,KAAK,CAAC;QAEtB,IAAI,MAAM,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;YACrC,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;YAC3B,SAAS,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,IAAI,MAAM,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACxC,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YACvB,SAAS,GAAG,IAAI,CAAC;QAClB,CAAC;QAED,MAAM,OAAO,GAAoB;YAChC,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,QAAQ,EAAE,OAAO,CAAC,GAAG;YACrB,MAAM;YACN,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,cAAc;YACd,SAAS;YACT,SAAS,EAAE,OAAO,CAAC,SAAS;SAC5B,CAAC;QAEF,OAAO;YACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACjC,OAAO;SACP,CAAC;IACH,CAAC;CACD,CAAC,CAAC;AAEH,MAAM,UAAU,mBAAmB,CAAC,KAAc;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,UAAU,CAAC;IAC3C,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACvC,IAAI,KAAK,KAAK,MAAM;YAAE,OAAO,MAAM,CAAC;IACrC,CAAC;IACD,OAAO,UAAU,CAAC;AACnB,CAAC","sourcesContent":["import { StringEnum } from \"@earendil-works/pi-ai\";\nimport { Type } from \"typebox\";\nimport { defineTool } from \"../../../types.ts\";\n\nimport { htmlToMarkdown, htmlToText } from \"./content.ts\";\nimport { clampTimeout, fetchUrl, type WebfetchFormat } from \"./fetcher.ts\";\n\nconst WEBFETCH_FORMATS = [\"markdown\", \"text\", \"html\"] as const;\n\nconst Params = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch content from\" }),\n\tformat: Type.Optional(\n\t\tStringEnum([\"markdown\", \"text\", \"html\"] as const, {\n\t\t\tdescription: \"The format to return the content in. Defaults to markdown.\",\n\t\t}),\n\t),\n\ttimeout: Type.Optional(Type.Number({ description: \"Optional timeout in seconds. Maximum 120.\" })),\n});\n\nexport interface WebfetchDetails {\n\turl: string;\n\tfinalUrl: string;\n\tformat: WebfetchFormat;\n\tstatus: number;\n\tstatusText: string;\n\tcontentType: string;\n\tbytes: number;\n\ttimeoutSeconds: number;\n\tconverted: boolean;\n\ttruncated: boolean;\n}\n\nexport interface WebfetchProgressDetails {\n\tphase: \"fetching\";\n\turl: string;\n\tformat: WebfetchFormat;\n\ttimeoutSeconds: number;\n}\n\nexport type WebfetchRenderDetails = WebfetchDetails | WebfetchProgressDetails;\n\nexport const webfetch = defineTool<typeof Params, WebfetchRenderDetails>({\n\tname: \"webfetch\",\n\tlabel: \"Web Fetch\",\n\tdescription:\n\t\t\"Fetches content from a URL and returns it as markdown, plain text, or HTML. \" +\n\t\t\"Network use is bounded by timeout and response size limits.\",\n\tpromptSnippet: \"webfetch: retrieve URL content as markdown, text, or html\",\n\tpromptGuidelines: [\n\t\t\"Use webfetch when a specific URL must be retrieved.\",\n\t\t\"Prefer markdown format unless raw HTML or plain text is explicitly needed.\",\n\t\t\"The tool is read-only and does not modify files.\",\n\t],\n\tparameters: Params,\n\tasync execute(_toolCallId, params, signal, onUpdate, _ctx) {\n\t\tconst format = parseWebfetchFormat(params.format);\n\t\tconst timeoutSeconds = clampTimeout(params.timeout);\n\t\tonUpdate?.({\n\t\t\tcontent: [{ type: \"text\", text: `Fetching ${params.url} as ${format} (timeout ${timeoutSeconds}s)` }],\n\t\t\tdetails: { phase: \"fetching\", url: params.url, format, timeoutSeconds },\n\t\t});\n\n\t\tconst fetched = await fetchUrl({\n\t\t\turl: params.url,\n\t\t\tformat,\n\t\t\ttimeoutSeconds,\n\t\t\t...(signal === undefined ? {} : { signal }),\n\t\t});\n\t\tconst raw = new TextDecoder().decode(fetched.body);\n\t\tconst contentType = fetched.contentType.toLowerCase();\n\t\tconst isHtml = contentType.includes(\"text/html\") || contentType.includes(\"application/xhtml+xml\");\n\t\tlet text = raw;\n\t\tlet converted = false;\n\n\t\tif (isHtml && format === \"markdown\") {\n\t\t\ttext = htmlToMarkdown(raw);\n\t\t\tconverted = true;\n\t\t} else if (isHtml && format === \"text\") {\n\t\t\ttext = htmlToText(raw);\n\t\t\tconverted = true;\n\t\t}\n\n\t\tconst details: WebfetchDetails = {\n\t\t\turl: params.url,\n\t\t\tfinalUrl: fetched.url,\n\t\t\tformat,\n\t\t\tstatus: fetched.status,\n\t\t\tstatusText: fetched.statusText,\n\t\t\tcontentType: fetched.contentType,\n\t\t\tbytes: fetched.bytes,\n\t\t\ttimeoutSeconds,\n\t\t\tconverted,\n\t\t\ttruncated: fetched.truncated,\n\t\t};\n\n\t\treturn {\n\t\t\tcontent: [{ type: \"text\", text }],\n\t\t\tdetails,\n\t\t};\n\t},\n});\n\nexport function parseWebfetchFormat(value: unknown): WebfetchFormat {\n\tif (value === undefined) return \"markdown\";\n\tfor (const format of WEBFETCH_FORMATS) {\n\t\tif (value === format) return format;\n\t}\n\treturn \"markdown\";\n}\n"]}
+{"version":3,"file":"tool.js","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAuB,MAAM,cAAc,CAAC;AAE3E,MAAM,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAU,CAAC;AAE/D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC1B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,+BAA+B,EAAE,CAAC;IAClE,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,UAAU,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAU,EAAE;QACjD,WAAW,EAAE,4DAA4D;KACzE,CAAC,CACF;IACD,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,2CAA2C,EAAE,CAAC,CAAC;CACjG,CAAC,CAAC;AAwBH,MAAM,CAAC,MAAM,QAAQ,GAAG,UAAU,CAAuC;IACxE,IAAI,EAAE,UAAU;IAChB,KAAK,EAAE,WAAW;IAClB,WAAW,EACV,8EAA8E;QAC9E,6DAA6D;IAC9D,aAAa,EAAE,2DAA2D;IAC1E,gBAAgB,EAAE;QACjB,qDAAqD;QACrD,4EAA4E;QAC5E,kDAAkD;KAClD;IACD,UAAU,EAAE,MAAM;IAClB,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI;QACxD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACpD,QAAQ,EAAE,CAAC;YACV,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,MAAM,CAAC,GAAG,OAAO,MAAM,aAAa,cAAc,IAAI,EAAE,CAAC;YACrG,OAAO,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,cAAc,EAAE;SACvE,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC;YAC9B,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,MAAM;YACN,cAAc;YACd,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;SAC3C,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;QAClG,IAAI,IAAI,GAAG,GAAG,CAAC;QACf,IAAI,SAAS,GAAG,KAAK,CAAC;QAEtB,IAAI,MAAM,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;YACrC,IAAI,GAAG,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YACxC,SAAS,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,IAAI,MAAM,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACxC,IAAI,GAAG,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YACpC,SAAS,GAAG,IAAI,CAAC;QAClB,CAAC;QAED,MAAM,OAAO,GAAoB;YAChC,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,QAAQ,EAAE,OAAO,CAAC,GAAG;YACrB,MAAM;YACN,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,cAAc;YACd,SAAS;YACT,SAAS,EAAE,OAAO,CAAC,SAAS;SAC5B,CAAC;QAEF,OAAO;YACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACjC,OAAO;SACP,CAAC;IACH,CAAC;CACD,CAAC,CAAC;AAEH,MAAM,UAAU,mBAAmB,CAAC,KAAc;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,UAAU,CAAC;IAC3C,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACvC,IAAI,KAAK,KAAK,MAAM;YAAE,OAAO,MAAM,CAAC;IACrC,CAAC;IACD,OAAO,UAAU,CAAC;AACnB,CAAC","sourcesContent":["import { StringEnum } from \"@earendil-works/pi-ai\";\nimport { Type } from \"typebox\";\nimport { defineTool } from \"../../../types.ts\";\n\nimport { htmlToMarkdown, htmlToText } from \"./content.ts\";\nimport { clampTimeout, fetchUrl, type WebfetchFormat } from \"./fetcher.ts\";\n\nconst WEBFETCH_FORMATS = [\"markdown\", \"text\", \"html\"] as const;\n\nconst Params = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch content from\" }),\n\tformat: Type.Optional(\n\t\tStringEnum([\"markdown\", \"text\", \"html\"] as const, {\n\t\t\tdescription: \"The format to return the content in. Defaults to markdown.\",\n\t\t}),\n\t),\n\ttimeout: Type.Optional(Type.Number({ description: \"Optional timeout in seconds. Maximum 120.\" })),\n});\n\nexport interface WebfetchDetails {\n\turl: string;\n\tfinalUrl: string;\n\tformat: WebfetchFormat;\n\tstatus: number;\n\tstatusText: string;\n\tcontentType: string;\n\tbytes: number;\n\ttimeoutSeconds: number;\n\tconverted: boolean;\n\ttruncated: boolean;\n}\n\nexport interface WebfetchProgressDetails {\n\tphase: \"fetching\";\n\turl: string;\n\tformat: WebfetchFormat;\n\ttimeoutSeconds: number;\n}\n\nexport type WebfetchRenderDetails = WebfetchDetails | WebfetchProgressDetails;\n\nexport const webfetch = defineTool<typeof Params, WebfetchRenderDetails>({\n\tname: \"webfetch\",\n\tlabel: \"Web Fetch\",\n\tdescription:\n\t\t\"Fetches content from a URL and returns it as markdown, plain text, or HTML. \" +\n\t\t\"Network use is bounded by timeout and response size limits.\",\n\tpromptSnippet: \"webfetch: retrieve URL content as markdown, text, or html\",\n\tpromptGuidelines: [\n\t\t\"Use webfetch when a specific URL must be retrieved.\",\n\t\t\"Prefer markdown format unless raw HTML or plain text is explicitly needed.\",\n\t\t\"The tool is read-only and does not modify files.\",\n\t],\n\tparameters: Params,\n\tasync execute(_toolCallId, params, signal, onUpdate, _ctx) {\n\t\tconst format = parseWebfetchFormat(params.format);\n\t\tconst timeoutSeconds = clampTimeout(params.timeout);\n\t\tonUpdate?.({\n\t\t\tcontent: [{ type: \"text\", text: `Fetching ${params.url} as ${format} (timeout ${timeoutSeconds}s)` }],\n\t\t\tdetails: { phase: \"fetching\", url: params.url, format, timeoutSeconds },\n\t\t});\n\n\t\tconst fetched = await fetchUrl({\n\t\t\turl: params.url,\n\t\t\tformat,\n\t\t\ttimeoutSeconds,\n\t\t\t...(signal === undefined ? {} : { signal }),\n\t\t});\n\t\tconst raw = new TextDecoder().decode(fetched.body);\n\t\tconst contentType = fetched.contentType.toLowerCase();\n\t\tconst isHtml = contentType.includes(\"text/html\") || contentType.includes(\"application/xhtml+xml\");\n\t\tlet text = raw;\n\t\tlet converted = false;\n\n\t\tif (isHtml && format === \"markdown\") {\n\t\t\ttext = htmlToMarkdown(raw, fetched.url);\n\t\t\tconverted = true;\n\t\t} else if (isHtml && format === \"text\") {\n\t\t\ttext = htmlToText(raw, fetched.url);\n\t\t\tconverted = true;\n\t\t}\n\n\t\tconst details: WebfetchDetails = {\n\t\t\turl: params.url,\n\t\t\tfinalUrl: fetched.url,\n\t\t\tformat,\n\t\t\tstatus: fetched.status,\n\t\t\tstatusText: fetched.statusText,\n\t\t\tcontentType: fetched.contentType,\n\t\t\tbytes: fetched.bytes,\n\t\t\ttimeoutSeconds,\n\t\t\tconverted,\n\t\t\ttruncated: fetched.truncated,\n\t\t};\n\n\t\treturn {\n\t\t\tcontent: [{ type: \"text\", text }],\n\t\t\tdetails,\n\t\t};\n\t},\n});\n\nexport function parseWebfetchFormat(value: unknown): WebfetchFormat {\n\tif (value === undefined) return \"markdown\";\n\tfor (const format of WEBFETCH_FORMATS) {\n\t\tif (value === format) return format;\n\t}\n\treturn \"markdown\";\n}\n"]}

package/dist/core/extensions/loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../src/core/extensions/loader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuBH,OAAO,EAAkB,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAIhE,OAAO,KAAK,EACX,SAAS,EAET,gBAAgB,EAChB,gBAAgB,EAChB,oBAAoB,EAKpB,MAAM,YAAY,CAAC;AA0HpB,MAAM,MAAM,wBAAwB,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,gBAAgB,GAAG,SAAS,CAAC;AAWrH,wBAAgB,mBAAmB,IAAI,IAAI,CAI1C;AAWD;;;GAGG;AACH,wBAAgB,sBAAsB,IAAI,gBAAgB,CA8CzD;AA+PD;;GAEG;AACH,wBAAsB,wBAAwB,CAC7C,OAAO,EAAE,gBAAgB,EACzB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,gBAAgB,EACzB,aAAa,SAAa,GACxB,OAAO,CAAC,SAAS,CAAC,CAMpB;AAqDD,wBAAsB,cAAc,CACnC,KAAK,EAAE,MAAM,EAAE,EACf,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,wBAAwB,CAAA;CAAE,GACtD,OAAO,CAAC,oBAAoB,CAAC,CAE/B;AAED,wBAAsB,oBAAoB,CACzC,KAAK,EAAE,MAAM,EAAE,EACf,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,oBAAoB,CAAC,CAE/B;AA+GD;;GAEG;AACH,wBAAsB,yBAAyB,CAC9C,eAAe,EAAE,MAAM,EAAE,EACzB,GAAG,EAAE,MAAM,EACX,QAAQ,GAAE,MAAsB,EAChC,QAAQ,CAAC,EAAE,QAAQ,GACjB,OAAO,CAAC,oBAAoB,CAAC,CA2C/B"}
+{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../src/core/extensions/loader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuBH,OAAO,EAAkB,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAKhE,OAAO,KAAK,EACX,SAAS,EAET,gBAAgB,EAChB,gBAAgB,EAChB,oBAAoB,EAKpB,MAAM,YAAY,CAAC;AAiIpB,MAAM,MAAM,wBAAwB,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,gBAAgB,GAAG,SAAS,CAAC;AAWrH,wBAAgB,mBAAmB,IAAI,IAAI,CAI1C;AAWD;;;GAGG;AACH,wBAAgB,sBAAsB,IAAI,gBAAgB,CA8CzD;AAiQD;;GAEG;AACH,wBAAsB,wBAAwB,CAC7C,OAAO,EAAE,gBAAgB,EACzB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,gBAAgB,EACzB,aAAa,SAAa,GACxB,OAAO,CAAC,SAAS,CAAC,CAOpB;AAqDD,wBAAsB,cAAc,CACnC,KAAK,EAAE,MAAM,EAAE,EACf,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,wBAAwB,CAAA;CAAE,GACtD,OAAO,CAAC,oBAAoB,CAAC,CAE/B;AAED,wBAAsB,oBAAoB,CACzC,KAAK,EAAE,MAAM,EAAE,EACf,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,oBAAoB,CAAC,CAE/B;AA+GD;;GAEG;AACH,wBAAsB,yBAAyB,CAC9C,eAAe,EAAE,MAAM,EAAE,EACzB,GAAG,EAAE,MAAM,EACX,QAAQ,GAAE,MAAsB,EAChC,QAAQ,CAAC,EAAE,QAAQ,GACjB,OAAO,CAAC,oBAAoB,CAAC,CA2C/B"}

package/dist/core/extensions/loader.js

@@ -7,7 +7,7 @@ import { createRequire } from "node:module";
 import * as path from "node:path";
 import { fileURLToPath } from "node:url";
 import * as _bundledPiAgentCore from "@earendil-works/pi-agent-core";
-import * as _bundledPiAi from "@earendil-works/pi-ai";
+import * as _bundledPiAiCompat from "@earendil-works/pi-ai/compat";
 import * as _bundledPiAiOauth from "@earendil-works/pi-ai/oauth";
 import * as _bundledPiTui from "@earendil-works/pi-tui";
 import { createJiti } from "jiti/static";
@@ -25,6 +25,7 @@ import { resolvePath } from "../../utils/paths.js";
 import { createEventBus } from "../event-bus.js";
 import { execCommand } from "../exec.js";
 import { createSyntheticSourceInfo } from "../source-info.js";
+import { time } from "../timings.js";
 /** Modules available to extensions via virtualModules (for compiled Bun binary) */
 const VIRTUAL_MODULES = {
     typebox: _bundledTypebox,
@@ -33,16 +34,21 @@ const VIRTUAL_MODULES = {
     "@sinclair/typebox": _bundledTypebox,
     "@sinclair/typebox/compile": _bundledTypeboxCompile,
     "@sinclair/typebox/value": _bundledTypeboxValue,
-    "@mariozechner/pi-agent-core": _bundledPiAgentCore,
-    "@mariozechner/pi-tui": _bundledPiTui,
-    "@mariozechner/pi-ai": _bundledPiAi,
-    "@mariozechner/pi-ai/oauth": _bundledPiAiOauth,
-    "@mariozechner/pi-coding-agent": _bundledPiCodingAgent,
     "@earendil-works/pi-agent-core": _bundledPiAgentCore,
     "@earendil-works/pi-tui": _bundledPiTui,
-    "@earendil-works/pi-ai": _bundledPiAi,
+    // Extensions resolve the pi-ai root to the compat entrypoint (a strict
+    // superset of the core entrypoint): existing extensions using the old
+    // global API keep working at runtime until compat is removed.
+    "@earendil-works/pi-ai": _bundledPiAiCompat,
+    "@earendil-works/pi-ai/compat": _bundledPiAiCompat,
     "@earendil-works/pi-ai/oauth": _bundledPiAiOauth,
     "@earendil-works/pi-coding-agent": _bundledPiCodingAgent,
+    "@mariozechner/pi-agent-core": _bundledPiAgentCore,
+    "@mariozechner/pi-tui": _bundledPiTui,
+    "@mariozechner/pi-ai": _bundledPiAiCompat,
+    "@mariozechner/pi-ai/compat": _bundledPiAiCompat,
+    "@mariozechner/pi-ai/oauth": _bundledPiAiOauth,
+    "@mariozechner/pi-coding-agent": _bundledPiCodingAgent,
     "@code-yeongyu/senpi": _bundledPiCodingAgent,
 };
 const require = createRequire(import.meta.url);
@@ -82,20 +88,22 @@ function getAliases() {
     const piCodingAgentEntry = fs.existsSync(piCodingAgentDistEntry) ? piCodingAgentDistEntry : piCodingAgentSourceEntry;
     const piAgentCoreEntry = resolveWorkspaceOrBundled("@earendil-works/pi-agent-core", "dist/index.js", "agent/dist/index.js", "agent/src/index.ts", "@earendil-works/pi-agent-core");
     const piTuiEntry = resolveWorkspaceOrBundled("@earendil-works/pi-tui", "dist/index.js", "tui/dist/index.js", "tui/src/index.ts", "@earendil-works/pi-tui");
-    const piAiEntry = resolveWorkspaceOrBundled("@earendil-works/pi-ai", "dist/index.js", "ai/dist/index.js", "ai/src/index.ts", "@earendil-works/pi-ai");
+    const piAiCompatEntry = resolveWorkspaceOrBundled("@earendil-works/pi-ai", "dist/compat.js", "ai/dist/compat.js", "ai/src/compat.ts", "@earendil-works/pi-ai/compat");
     const piAiOauthEntry = resolveWorkspaceOrBundled("@earendil-works/pi-ai", "dist/oauth.js", "ai/dist/oauth.js", "ai/src/oauth.ts", "@earendil-works/pi-ai/oauth");
     _aliases = {
+        "@earendil-works/pi-coding-agent": piCodingAgentEntry,
+        "@earendil-works/pi-agent-core": piAgentCoreEntry,
+        "@earendil-works/pi-tui": piTuiEntry,
+        "@earendil-works/pi-ai": piAiCompatEntry,
+        "@earendil-works/pi-ai/compat": piAiCompatEntry,
+        "@earendil-works/pi-ai/oauth": piAiOauthEntry,
         "@code-yeongyu/senpi": piCodingAgentEntry,
         "@mariozechner/pi-coding-agent": piCodingAgentEntry,
         "@mariozechner/pi-agent-core": piAgentCoreEntry,
         "@mariozechner/pi-tui": piTuiEntry,
-        "@mariozechner/pi-ai": piAiEntry,
+        "@mariozechner/pi-ai": piAiCompatEntry,
+        "@mariozechner/pi-ai/compat": piAiCompatEntry,
         "@mariozechner/pi-ai/oauth": piAiOauthEntry,
-        "@earendil-works/pi-agent-core": piAgentCoreEntry,
-        "@earendil-works/pi-tui": piTuiEntry,
-        "@earendil-works/pi-ai": piAiEntry,
-        "@earendil-works/pi-ai/oauth": piAiOauthEntry,
-        "@earendil-works/pi-coding-agent": piCodingAgentEntry,
         typebox: typeboxEntry,
         "typebox/compile": typeboxCompileEntry,
         "typebox/value": typeboxValueEntry,
@@ -347,12 +355,14 @@ async function loadExtension(extensionPath, cwd, eventBus, runtime, getImporter,
     try {
         const factory = factoryResolver?.(extensionPath, resolvedPath) ??
             (await loadExtensionModule(resolvedPath, getImporter(), cacheToken));
+        time(`${extensionPath} module import`, "extensions");
         if (!factory) {
             return { extension: null, error: `Extension does not export a valid factory function: ${extensionPath}` };
         }
         const extension = createExtension(extensionPath, resolvedPath);
         const api = createExtensionAPI(extension, runtime, cwd, eventBus);
         await factory(api);
+        time(`${extensionPath} factory`, "extensions");
         return { extension, error: null };
     }
     catch (err) {
@@ -368,6 +378,7 @@ export async function loadExtensionFromFactory(factory, cwd, eventBus, runtime,
     const resolvedCwd = resolvePath(cwd);
     const api = createExtensionAPI(extension, runtime, resolvedCwd, eventBus);
     await factory(api);
+    time(`${extensionPath} factory`, "extensions");
     return extension;
 }
 /**