npm - @code-yeongyu/senpi - Versions diffs - 2026.6.21 → 2026.6.30 - Mend

@code-yeongyu/senpi 2026.6.21 → 2026.6.30

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (1048) hide show

package/dist/core/extensions/builtin/webfetch/webfetch/content.js CHANGED Viewed

@@ -1,10 +1,50 @@
+import { Readability } from "@mozilla/readability";
+import { JSDOM, VirtualConsole } from "jsdom";
 import TurndownService from "turndown";
 const TAGS_TO_REMOVE = /<(script|style|noscript|iframe|object|embed|meta|link)\b[^>]*>[\s\S]*?<\/\1>/gi;
 const VOID_TAGS_TO_REMOVE = /<(script|style|noscript|iframe|object|embed|meta|link)\b[^>]*\/?>/gi;
 const BLOCK_BREAK_TAGS = /<\/?(address|article|aside|blockquote|br|dd|div|dl|dt|figcaption|figure|footer|h[1-6]|header|hr|li|main|nav|ol|p|pre|section|table|tbody|td|tfoot|th|thead|tr|ul)\b[^>]*>/gi;
+const BLOCK_BREAK_SELECTOR = "address, article, aside, blockquote, dd, div, dl, dt, figcaption, figure, footer, h1, h2, h3, h4, h5, h6, header, hr, li, main, nav, ol, p, pre, section, table, tbody, tfoot, thead, tr, ul";
+const CELL_BREAK_SELECTOR = "td, th";
 const TAGS = /<[^>]+>/g;
-const WHITESPACE = /[\t\f\v ]+/g;
+const WHITESPACE = /[\t\f\v \u00a0]+/g;
 const NEWLINE_RUN = /\n{3,}/g;
+const MIN_EXPLICIT_ARTICLE_TEXT_LENGTH = 30;
+const EXPLICIT_ARTICLE_SELECTORS = [
+    ".article_view",
+    ".tt_article_useless_p_margin",
+    ".entry-content",
+    ".contents_style",
+    ".post-content",
+    ".article-content",
+    ".content-article",
+    "#content .contents_style",
+];
+const TITLE_SELECTORS = [".tit_post", ".entry-title", ".post-title", ".article-title", "h1"];
+const ARTICLE_NOISE_SELECTOR = [
+    "script",
+    "style",
+    "noscript",
+    "iframe",
+    "object",
+    "embed",
+    "meta",
+    "link",
+    "nav",
+    "aside",
+    "footer",
+    ".another_category",
+    ".area_related",
+    ".related",
+    ".revenue_unit_wrap",
+    ".adsbygoogle",
+    ".container_postbtn",
+    ".postbtn_like",
+    ".comments",
+    ".comment",
+    ".tagTrail",
+    ".sidebar",
+].join(", ");
 const ENTITIES = {
     amp: "&",
     apos: "'",
@@ -21,20 +61,143 @@ const turndownService = new TurndownService({
     emDelimiter: "*",
 });
 turndownService.remove(["script", "style", "noscript", "iframe", "object", "embed", "meta", "link"]);
-export function htmlToMarkdown(html) {
-    return turndownService.turndown(html).trim();
+export function htmlToMarkdown(html, url) {
+    const article = extractReadableArticle(html, url);
+    if (!article)
+        return normalizeMarkdown(turndownService.turndown(html));
+    const markdown = normalizeMarkdown(turndownService.turndown(article.content));
+    if (!article.title || article.hasHeading || markdown.startsWith(`# ${article.title}`))
+        return markdown;
+    return `# ${article.title}\n\n${markdown}`.trim();
 }
-export function htmlToText(html) {
-    return decodeHtmlEntities(html
+export function htmlToText(html, url) {
+    const article = extractReadableArticle(html, url);
+    if (article) {
+        const body = htmlFragmentToPlainText(article.content);
+        if (!article.title || article.hasHeading)
+            return body;
+        if (body.startsWith(article.title))
+            return body;
+        return `${article.title}\n\n${body}`.trim();
+    }
+    return htmlFragmentToPlainText(html);
+}
+function htmlFragmentToPlainText(html) {
+    try {
+        const dom = new JSDOM(`<body>${html}</body>`, {
+            contentType: "text/html",
+            virtualConsole: new VirtualConsole(),
+        });
+        try {
+            const document = dom.window.document;
+            for (const element of document.querySelectorAll("script, style, noscript, iframe, object, embed, meta, link")) {
+                element.remove();
+            }
+            for (const element of document.querySelectorAll("br")) {
+                element.replaceWith(document.createTextNode("\n"));
+            }
+            for (const element of document.querySelectorAll(CELL_BREAK_SELECTOR)) {
+                element.after(document.createTextNode("\n"));
+            }
+            for (const element of document.querySelectorAll(BLOCK_BREAK_SELECTOR)) {
+                element.before(document.createTextNode("\n"));
+                element.after(document.createTextNode("\n"));
+            }
+            return normalizePlainText(document.body.textContent ?? "");
+        }
+        finally {
+            dom.window.close();
+        }
+    }
+    catch (error) {
+        if (!(error instanceof Error))
+            throw error;
+    }
+    return htmlFragmentToPlainTextFallback(html);
+}
+function htmlFragmentToPlainTextFallback(html) {
+    return decodeHtmlEntities(normalizePlainText(html
         .replace(TAGS_TO_REMOVE, "")
         .replace(VOID_TAGS_TO_REMOVE, "")
         .replace(BLOCK_BREAK_TAGS, "\n")
-        .replace(TAGS, "")
+        .replace(TAGS, "")));
+}
+function extractReadableArticle(html, url) {
+    try {
+        const dom = new JSDOM(html, {
+            url,
+            contentType: "text/html",
+            virtualConsole: new VirtualConsole(),
+        });
+        try {
+            let explicitArticle;
+            for (const selector of EXPLICIT_ARTICLE_SELECTORS) {
+                const candidate = dom.window.document.querySelector(selector);
+                if (!candidate)
+                    continue;
+                const clonedNode = candidate.cloneNode(true);
+                if (!(clonedNode instanceof dom.window.Element))
+                    continue;
+                for (const noisyElement of clonedNode.querySelectorAll(ARTICLE_NOISE_SELECTOR)) {
+                    noisyElement.remove();
+                }
+                const text = normalizePlainText(clonedNode.textContent ?? "");
+                if (text.length < MIN_EXPLICIT_ARTICLE_TEXT_LENGTH)
+                    continue;
+                explicitArticle = {
+                    title: selectPreferredTitle(dom.window.document, dom.window.document.title),
+                    content: clonedNode.innerHTML,
+                    hasHeading: /<h[1-6]\b/i.test(clonedNode.innerHTML),
+                };
+                break;
+            }
+            if (explicitArticle)
+                return explicitArticle;
+            const article = new Readability(dom.window.document, {
+                charThreshold: 80,
+                keepClasses: false,
+            }).parse();
+            if (!article?.content || !article.textContent)
+                return undefined;
+            return {
+                title: selectPreferredTitle(dom.window.document, article.title ?? ""),
+                content: article.content,
+                hasHeading: /<h[1-6]\b/i.test(article.content),
+            };
+        }
+        finally {
+            dom.window.close();
+        }
+    }
+    catch (error) {
+        if (error instanceof Error)
+            return undefined;
+        throw error;
+    }
+}
+function selectPreferredTitle(document, fallback) {
+    for (const selector of TITLE_SELECTORS) {
+        const title = normalizePlainText(document.querySelector(selector)?.textContent ?? "");
+        if (title)
+            return title;
+    }
+    return normalizePlainText(fallback);
+}
+function normalizePlainText(text) {
+    return text
         .replace(WHITESPACE, " ")
         .replace(/[ \t]+\n/g, "\n")
         .replace(/\n[ \t]+/g, "\n")
         .replace(NEWLINE_RUN, "\n\n")
-        .trim());
+        .trim();
+}
+function normalizeMarkdown(markdown) {
+    return markdown
+        .replace(/\r\n?/g, "\n")
+        .replace(/[ \t]+\n/g, "\n")
+        .replace(/\n[ \t]+/g, "\n")
+        .replace(NEWLINE_RUN, "\n\n")
+        .trim();
 }
 export function decodeHtmlEntities(text) {
     return text.replace(/&(#x[\da-f]+|#\d+|[a-z]+);/gi, (_match, entity) => {

package/dist/core/extensions/builtin/webfetch/webfetch/content.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"content.js","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/content.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,MAAM,UAAU,CAAC;AAEvC,MAAM,cAAc,GAAG,gFAAgF,CAAC;AACxG,MAAM,mBAAmB,GAAG,qEAAqE,CAAC;AAClG,MAAM,gBAAgB,GACrB,6KAA6K,CAAC;AAC/K,MAAM,IAAI,GAAG,UAAU,CAAC;AACxB,MAAM,UAAU,GAAG,aAAa,CAAC;AACjC,MAAM,WAAW,GAAG,SAAS,CAAC;AAE9B,MAAM,QAAQ,GAAqC;IAClD,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,GAAG;IACT,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;CACT,CAAC;AAEF,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC;IAC3C,YAAY,EAAE,KAAK;IACnB,EAAE,EAAE,KAAK;IACT,gBAAgB,EAAE,GAAG;IACrB,cAAc,EAAE,QAAQ;IACxB,WAAW,EAAE,GAAG;CAChB,CAAC,CAAC;AACH,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAErG,MAAM,UAAU,cAAc,CAAC,IAAY;IAC1C,OAAO,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACtC,OAAO,kBAAkB,CACxB,IAAI;SACF,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC;SAChC,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;SAC/B,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;SACjB,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC;SAC5B,IAAI,EAAE,CACR,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC9C,OAAO,IAAI,CAAC,OAAO,CAAC,8BAA8B,EAAE,CAAC,MAAM,EAAE,MAAc,EAAE,EAAE;QAC9E,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,OAAO,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,OAAO,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,MAAM,GAAG,CAAC;IACxD,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACrC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,IAAI,CAAC;QACJ,OAAO,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,EAAE,CAAC;IACX,CAAC;AACF,CAAC","sourcesContent":["import TurndownService from \"turndown\";\n\nconst TAGS_TO_REMOVE = /<(script\|style\|noscript\|iframe\|object\|embed\|meta\|link)\\b[^>]>[\\s\\S]?<\\/\\1>/gi;\nconst VOID_TAGS_TO_REMOVE = /<(script\|style\|noscript\|iframe\|object\|embed\|meta\|link)\\b[^>]\\/?>/gi;\nconst BLOCK_BREAK_TAGS =\n\t/<\\/?(address\|article\|aside\|blockquote\|br\|dd\|div\|dl\|dt\|figcaption\|figure\|footer\|h[1-6]\|header\|hr\|li\|main\|nav\|ol\|p\|pre\|section\|table\|tbody\|td\|tfoot\|th\|thead\|tr\|ul)\\b[^>]>/gi;\nconst TAGS = /<[^>]+>/g;\nconst WHITESPACE = /[\\t\\f\\v ]+/g;\nconst NEWLINE_RUN = /\\n{3,}/g;\n\nconst ENTITIES: Readonly<Record<string, string>> = {\n\tamp: \"&\",\n\tapos: \"'\",\n\tgt: \">\",\n\tlt: \"<\",\n\tnbsp: \" \",\n\tquot: '\"',\n};\n\nconst turndownService = new TurndownService({\n\theadingStyle: \"atx\",\n\thr: \"---\",\n\tbulletListMarker: \"-\",\n\tcodeBlockStyle: \"fenced\",\n\temDelimiter: \"*\",\n});\nturndownService.remove([\"script\", \"style\", \"noscript\", \"iframe\", \"object\", \"embed\", \"meta\", \"link\"]);\n\nexport function htmlToMarkdown(html: string): string {\n\treturn turndownService.turndown(html).trim();\n}\n\nexport function htmlToText(html: string): string {\n\treturn decodeHtmlEntities(\n\t\thtml\n\t\t\t.replace(TAGS_TO_REMOVE, \"\")\n\t\t\t.replace(VOID_TAGS_TO_REMOVE, \"\")\n\t\t\t.replace(BLOCK_BREAK_TAGS, \"\\n\")\n\t\t\t.replace(TAGS, \"\")\n\t\t\t.replace(WHITESPACE, \" \")\n\t\t\t.replace(/[ \\t]+\\n/g, \"\\n\")\n\t\t\t.replace(/\\n[ \\t]+/g, \"\\n\")\n\t\t\t.replace(NEWLINE_RUN, \"\\n\\n\")\n\t\t\t.trim(),\n\t);\n}\n\nexport function decodeHtmlEntities(text: string): string {\n\treturn text.replace(/&(#x[\\da-f]+\|#\\d+\|[a-z]+);/gi, (_match, entity: string) => {\n\t\tif (entity.startsWith(\"#x\")) {\n\t\t\treturn decodeCodePoint(Number.parseInt(entity.slice(2), 16));\n\t\t}\n\t\tif (entity.startsWith(\"#\")) {\n\t\t\treturn decodeCodePoint(Number.parseInt(entity.slice(1), 10));\n\t\t}\n\t\treturn ENTITIES[entity.toLowerCase()] ?? `&${entity};`;\n\t});\n}\n\nfunction decodeCodePoint(value: number): string {\n\tif (!Number.isFinite(value)) return \"\";\n\ttry {\n\t\treturn String.fromCodePoint(value);\n\t} catch {\n\t\treturn \"\";\n\t}\n}\n"]}
1	+ {"version":3,"file":"content.js","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,OAAO,CAAC;AAC9C,OAAO,eAAe,MAAM,UAAU,CAAC;AAQvC,MAAM,cAAc,GAAG,gFAAgF,CAAC;AACxG,MAAM,mBAAmB,GAAG,qEAAqE,CAAC;AAClG,MAAM,gBAAgB,GACrB,6KAA6K,CAAC;AAC/K,MAAM,oBAAoB,GACzB,8LAA8L,CAAC;AAChM,MAAM,mBAAmB,GAAG,QAAQ,CAAC;AACrC,MAAM,IAAI,GAAG,UAAU,CAAC;AACxB,MAAM,UAAU,GAAG,mBAAmB,CAAC;AACvC,MAAM,WAAW,GAAG,SAAS,CAAC;AAC9B,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAC5C,MAAM,0BAA0B,GAAG;IAClC,eAAe;IACf,8BAA8B;IAC9B,gBAAgB;IAChB,iBAAiB;IACjB,eAAe;IACf,kBAAkB;IAClB,kBAAkB;IAClB,0BAA0B;CAC1B,CAAC;AACF,MAAM,eAAe,GAAG,CAAC,WAAW,EAAE,cAAc,EAAE,aAAa,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;AAC7F,MAAM,sBAAsB,GAAG;IAC9B,QAAQ;IACR,OAAO;IACP,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,MAAM;IACN,MAAM;IACN,KAAK;IACL,OAAO;IACP,QAAQ;IACR,mBAAmB;IACnB,eAAe;IACf,UAAU;IACV,oBAAoB;IACpB,cAAc;IACd,oBAAoB;IACpB,eAAe;IACf,WAAW;IACX,UAAU;IACV,WAAW;IACX,UAAU;CACV,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACb,MAAM,QAAQ,GAAqC;IAClD,GAAG,EAAE,GAAG;IACR,IAAI,EAAE,GAAG;IACT,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;CACT,CAAC;AAEF,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC;IAC3C,YAAY,EAAE,KAAK;IACnB,EAAE,EAAE,KAAK;IACT,gBAAgB,EAAE,GAAG;IACrB,cAAc,EAAE,QAAQ;IACxB,WAAW,EAAE,GAAG;CAChB,CAAC,CAAC;AACH,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAErG,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,GAAW;IACvD,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAClD,IAAI,CAAC,OAAO;QAAE,OAAO,iBAAiB,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAEvE,MAAM,QAAQ,GAAG,iBAAiB,CAAC,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9E,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC;QAAE,OAAO,QAAQ,CAAC;IACvG,OAAO,KAAK,OAAO,CAAC,KAAK,OAAO,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY,EAAE,GAAW;IACnD,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAClD,IAAI,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,uBAAuB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtD,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC;QACtD,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAChD,OAAO,GAAG,OAAO,CAAC,KAAK,OAAO,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IAC7C,CAAC;IAED,OAAO,uBAAuB,CAAC,IAAI,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,uBAAuB,CAAC,IAAY;IAC5C,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,SAAS,IAAI,SAAS,EAAE;YAC7C,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,IAAI,cAAc,EAAE;SACpC,CAAC,CAAC;QACH,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;YACrC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,gBAAgB,CAC9C,4DAA4D,CAC5D,EAAE,CAAC;gBACH,OAAO,CAAC,MAAM,EAAE,CAAC;YAClB,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvD,OAAO,CAAC,WAAW,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;YACpD,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,EAAE,CAAC;gBACtE,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;YAC9C,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,EAAE,CAAC;gBACvE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC9C,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC;YAC9C,CAAC;YACD,OAAO,kBAAkB,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAC5D,CAAC;gBAAS,CAAC;YACV,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,CAAC;YAAE,MAAM,KAAK,CAAC;IAC5C,CAAC;IAED,OAAO,+BAA+B,CAAC,IAAI,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,+BAA+B,CAAC,IAAY;IACpD,OAAO,kBAAkB,CACxB,kBAAkB,CACjB,IAAI;SACF,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC;SAChC,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;SAC/B,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CACnB,CACD,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAY,EAAE,GAAW;IACxD,IAAI,CAAC;QACJ,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE;YAC3B,GAAG;YACH,WAAW,EAAE,WAAW;YACxB,cAAc,EAAE,IAAI,cAAc,EAAE;SACpC,CAAC,CAAC;QACH,IAAI,CAAC;YACJ,IAAI,eAA4C,CAAC;YACjD,KAAK,MAAM,QAAQ,IAAI,0BAA0B,EAAE,CAAC;gBACnD,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;gBAC9D,IAAI,CAAC,SAAS;oBAAE,SAAS;gBACzB,MAAM,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAC7C,IAAI,CAAC,CAAC,UAAU,YAAY,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC;oBAAE,SAAS;gBAC1D,KAAK,MAAM,YAAY,IAAI,UAAU,CAAC,gBAAgB,CAAC,sBAAsB,CAAC,EAAE,CAAC;oBAChF,YAAY,CAAC,MAAM,EAAE,CAAC;gBACvB,CAAC;gBACD,MAAM,IAAI,GAAG,kBAAkB,CAAC,UAAU,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;gBAC9D,IAAI,IAAI,CAAC,MAAM,GAAG,gCAAgC;oBAAE,SAAS;gBAC7D,eAAe,GAAG;oBACjB,KAAK,EAAE,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;oBAC3E,OAAO,EAAE,UAAU,CAAC,SAAS;oBAC7B,UAAU,EAAE,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;iBACnD,CAAC;gBACF,MAAM;YACP,CAAC;YACD,IAAI,eAAe;gBAAE,OAAO,eAAe,CAAC;YAE5C,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE;gBACpD,aAAa,EAAE,EAAE;gBACjB,WAAW,EAAE,KAAK;aAClB,CAAC,CAAC,KAAK,EAAE,CAAC;YACX,IAAI,CAAC,OAAO,EAAE,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW;gBAAE,OAAO,SAAS,CAAC;YAChE,OAAO;gBACN,KAAK,EAAE,oBAAoB,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;gBACrE,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,UAAU,EAAE,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;aAC9C,CAAC;QACH,CAAC;gBAAS,CAAC;YACV,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK;YAAE,OAAO,SAAS,CAAC;QAC7C,MAAM,KAAK,CAAC;IACb,CAAC;AACF,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAkB,EAAE,QAAgB;IACjE,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,kBAAkB,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;QACtF,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;IACzB,CAAC;IACD,OAAO,kBAAkB,CAAC,QAAQ,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACvC,OAAO,IAAI;SACT,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC;SAC5B,IAAI,EAAE,CAAC;AACV,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAgB;IAC1C,OAAO,QAAQ;SACb,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC;SAC5B,IAAI,EAAE,CAAC;AACV,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC9C,OAAO,IAAI,CAAC,OAAO,CAAC,8BAA8B,EAAE,CAAC,MAAM,EAAE,MAAc,EAAE,EAAE;QAC9E,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7B,OAAO,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,OAAO,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,OAAO,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,MAAM,GAAG,CAAC;IACxD,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACrC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,IAAI,CAAC;QACJ,OAAO,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,EAAE,CAAC;IACX,CAAC;AACF,CAAC","sourcesContent":["import { Readability } from \"@mozilla/readability\";\nimport { JSDOM, VirtualConsole } from \"jsdom\";\nimport TurndownService from \"turndown\";\n\ninterface ReadableArticle {\n\treadonly title: string;\n\treadonly content: string;\n\treadonly hasHeading: boolean;\n}\n\nconst TAGS_TO_REMOVE = /<(script\|style\|noscript\|iframe\|object\|embed\|meta\|link)\\b[^>]>[\\s\\S]?<\\/\\1>/gi;\nconst VOID_TAGS_TO_REMOVE = /<(script\|style\|noscript\|iframe\|object\|embed\|meta\|link)\\b[^>]\\/?>/gi;\nconst BLOCK_BREAK_TAGS =\n\t/<\\/?(address\|article\|aside\|blockquote\|br\|dd\|div\|dl\|dt\|figcaption\|figure\|footer\|h[1-6]\|header\|hr\|li\|main\|nav\|ol\|p\|pre\|section\|table\|tbody\|td\|tfoot\|th\|thead\|tr\|ul)\\b[^>]>/gi;\nconst BLOCK_BREAK_SELECTOR =\n\t\"address, article, aside, blockquote, dd, div, dl, dt, figcaption, figure, footer, h1, h2, h3, h4, h5, h6, header, hr, li, main, nav, ol, p, pre, section, table, tbody, tfoot, thead, tr, ul\";\nconst CELL_BREAK_SELECTOR = \"td, th\";\nconst TAGS = /<[^>]+>/g;\nconst WHITESPACE = /[\\t\\f\\v \\u00a0]+/g;\nconst NEWLINE_RUN = /\\n{3,}/g;\nconst MIN_EXPLICIT_ARTICLE_TEXT_LENGTH = 30;\nconst EXPLICIT_ARTICLE_SELECTORS = [\n\t\".article_view\",\n\t\".tt_article_useless_p_margin\",\n\t\".entry-content\",\n\t\".contents_style\",\n\t\".post-content\",\n\t\".article-content\",\n\t\".content-article\",\n\t\"#content .contents_style\",\n];\nconst TITLE_SELECTORS = [\".tit_post\", \".entry-title\", \".post-title\", \".article-title\", \"h1\"];\nconst ARTICLE_NOISE_SELECTOR = [\n\t\"script\",\n\t\"style\",\n\t\"noscript\",\n\t\"iframe\",\n\t\"object\",\n\t\"embed\",\n\t\"meta\",\n\t\"link\",\n\t\"nav\",\n\t\"aside\",\n\t\"footer\",\n\t\".another_category\",\n\t\".area_related\",\n\t\".related\",\n\t\".revenue_unit_wrap\",\n\t\".adsbygoogle\",\n\t\".container_postbtn\",\n\t\".postbtn_like\",\n\t\".comments\",\n\t\".comment\",\n\t\".tagTrail\",\n\t\".sidebar\",\n].join(\", \");\nconst ENTITIES: Readonly<Record<string, string>> = {\n\tamp: \"&\",\n\tapos: \"'\",\n\tgt: \">\",\n\tlt: \"<\",\n\tnbsp: \" \",\n\tquot: '\"',\n};\n\nconst turndownService = new TurndownService({\n\theadingStyle: \"atx\",\n\thr: \"---\",\n\tbulletListMarker: \"-\",\n\tcodeBlockStyle: \"fenced\",\n\temDelimiter: \"*\",\n});\nturndownService.remove([\"script\", \"style\", \"noscript\", \"iframe\", \"object\", \"embed\", \"meta\", \"link\"]);\n\nexport function htmlToMarkdown(html: string, url: string): string {\n\tconst article = extractReadableArticle(html, url);\n\tif (!article) return normalizeMarkdown(turndownService.turndown(html));\n\n\tconst markdown = normalizeMarkdown(turndownService.turndown(article.content));\n\tif (!article.title \|\| article.hasHeading \|\| markdown.startsWith(`# ${article.title}`)) return markdown;\n\treturn `# ${article.title}\\n\\n${markdown}`.trim();\n}\n\nexport function htmlToText(html: string, url: string): string {\n\tconst article = extractReadableArticle(html, url);\n\tif (article) {\n\t\tconst body = htmlFragmentToPlainText(article.content);\n\t\tif (!article.title \|\| article.hasHeading) return body;\n\t\tif (body.startsWith(article.title)) return body;\n\t\treturn `${article.title}\\n\\n${body}`.trim();\n\t}\n\n\treturn htmlFragmentToPlainText(html);\n}\n\nfunction htmlFragmentToPlainText(html: string): string {\n\ttry {\n\t\tconst dom = new JSDOM(`<body>${html}</body>`, {\n\t\t\tcontentType: \"text/html\",\n\t\t\tvirtualConsole: new VirtualConsole(),\n\t\t});\n\t\ttry {\n\t\t\tconst document = dom.window.document;\n\t\t\tfor (const element of document.querySelectorAll(\n\t\t\t\t\"script, style, noscript, iframe, object, embed, meta, link\",\n\t\t\t)) {\n\t\t\t\telement.remove();\n\t\t\t}\n\t\t\tfor (const element of document.querySelectorAll(\"br\")) {\n\t\t\t\telement.replaceWith(document.createTextNode(\"\\n\"));\n\t\t\t}\n\t\t\tfor (const element of document.querySelectorAll(CELL_BREAK_SELECTOR)) {\n\t\t\t\telement.after(document.createTextNode(\"\\n\"));\n\t\t\t}\n\t\t\tfor (const element of document.querySelectorAll(BLOCK_BREAK_SELECTOR)) {\n\t\t\t\telement.before(document.createTextNode(\"\\n\"));\n\t\t\t\telement.after(document.createTextNode(\"\\n\"));\n\t\t\t}\n\t\t\treturn normalizePlainText(document.body.textContent ?? \"\");\n\t\t} finally {\n\t\t\tdom.window.close();\n\t\t}\n\t} catch (error) {\n\t\tif (!(error instanceof Error)) throw error;\n\t}\n\n\treturn htmlFragmentToPlainTextFallback(html);\n}\n\nfunction htmlFragmentToPlainTextFallback(html: string): string {\n\treturn decodeHtmlEntities(\n\t\tnormalizePlainText(\n\t\t\thtml\n\t\t\t\t.replace(TAGS_TO_REMOVE, \"\")\n\t\t\t\t.replace(VOID_TAGS_TO_REMOVE, \"\")\n\t\t\t\t.replace(BLOCK_BREAK_TAGS, \"\\n\")\n\t\t\t\t.replace(TAGS, \"\"),\n\t\t),\n\t);\n}\n\nfunction extractReadableArticle(html: string, url: string): ReadableArticle \| undefined {\n\ttry {\n\t\tconst dom = new JSDOM(html, {\n\t\t\turl,\n\t\t\tcontentType: \"text/html\",\n\t\t\tvirtualConsole: new VirtualConsole(),\n\t\t});\n\t\ttry {\n\t\t\tlet explicitArticle: ReadableArticle \| undefined;\n\t\t\tfor (const selector of EXPLICIT_ARTICLE_SELECTORS) {\n\t\t\t\tconst candidate = dom.window.document.querySelector(selector);\n\t\t\t\tif (!candidate) continue;\n\t\t\t\tconst clonedNode = candidate.cloneNode(true);\n\t\t\t\tif (!(clonedNode instanceof dom.window.Element)) continue;\n\t\t\t\tfor (const noisyElement of clonedNode.querySelectorAll(ARTICLE_NOISE_SELECTOR)) {\n\t\t\t\t\tnoisyElement.remove();\n\t\t\t\t}\n\t\t\t\tconst text = normalizePlainText(clonedNode.textContent ?? \"\");\n\t\t\t\tif (text.length < MIN_EXPLICIT_ARTICLE_TEXT_LENGTH) continue;\n\t\t\t\texplicitArticle = {\n\t\t\t\t\ttitle: selectPreferredTitle(dom.window.document, dom.window.document.title),\n\t\t\t\t\tcontent: clonedNode.innerHTML,\n\t\t\t\t\thasHeading: /<h[1-6]\\b/i.test(clonedNode.innerHTML),\n\t\t\t\t};\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\tif (explicitArticle) return explicitArticle;\n\n\t\t\tconst article = new Readability(dom.window.document, {\n\t\t\t\tcharThreshold: 80,\n\t\t\t\tkeepClasses: false,\n\t\t\t}).parse();\n\t\t\tif (!article?.content \|\| !article.textContent) return undefined;\n\t\t\treturn {\n\t\t\t\ttitle: selectPreferredTitle(dom.window.document, article.title ?? \"\"),\n\t\t\t\tcontent: article.content,\n\t\t\t\thasHeading: /<h[1-6]\\b/i.test(article.content),\n\t\t\t};\n\t\t} finally {\n\t\t\tdom.window.close();\n\t\t}\n\t} catch (error) {\n\t\tif (error instanceof Error) return undefined;\n\t\tthrow error;\n\t}\n}\n\nfunction selectPreferredTitle(document: Document, fallback: string): string {\n\tfor (const selector of TITLE_SELECTORS) {\n\t\tconst title = normalizePlainText(document.querySelector(selector)?.textContent ?? \"\");\n\t\tif (title) return title;\n\t}\n\treturn normalizePlainText(fallback);\n}\n\nfunction normalizePlainText(text: string): string {\n\treturn text\n\t\t.replace(WHITESPACE, \" \")\n\t\t.replace(/[ \\t]+\\n/g, \"\\n\")\n\t\t.replace(/\\n[ \\t]+/g, \"\\n\")\n\t\t.replace(NEWLINE_RUN, \"\\n\\n\")\n\t\t.trim();\n}\n\nfunction normalizeMarkdown(markdown: string): string {\n\treturn markdown\n\t\t.replace(/\\r\\n?/g, \"\\n\")\n\t\t.replace(/[ \\t]+\\n/g, \"\\n\")\n\t\t.replace(/\\n[ \\t]+/g, \"\\n\")\n\t\t.replace(NEWLINE_RUN, \"\\n\\n\")\n\t\t.trim();\n}\n\nexport function decodeHtmlEntities(text: string): string {\n\treturn text.replace(/&(#x[\\da-f]+\|#\\d+\|[a-z]+);/gi, (_match, entity: string) => {\n\t\tif (entity.startsWith(\"#x\")) {\n\t\t\treturn decodeCodePoint(Number.parseInt(entity.slice(2), 16));\n\t\t}\n\t\tif (entity.startsWith(\"#\")) {\n\t\t\treturn decodeCodePoint(Number.parseInt(entity.slice(1), 10));\n\t\t}\n\t\treturn ENTITIES[entity.toLowerCase()] ?? `&${entity};`;\n\t});\n}\n\nfunction decodeCodePoint(value: number): string {\n\tif (!Number.isFinite(value)) return \"\";\n\ttry {\n\t\treturn String.fromCodePoint(value);\n\t} catch {\n\t\treturn \"\";\n\t}\n}\n"]}

package/dist/core/extensions/builtin/webfetch/webfetch/fetcher.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/fetcher.ts"],"names":[],"mappings":"~~AAOA~~,eAAO,MAAM,uBAAuB,QAAkB,CAAC;AACvD,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAC1C,eAAO,MAAM,mBAAmB,MAAM,CAAC;~~AAKvC~~,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;AAE1D,MAAM,WAAW,YAAY;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,cAAc,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;CACnB;~~AAED~~,wBAAsB,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,~~CA+B1E~~;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAU7C;AAED,wBAAgB,YAAY,CAAC,cAAc,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAIvE;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAShE"}
1	+ {"version":3,"file":"fetcher.d.ts","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/fetcher.ts"],"names":[],"mappings":"AAUA,eAAO,MAAM,uBAAuB,QAAkB,CAAC;AACvD,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAC1C,eAAO,MAAM,mBAAmB,MAAM,CAAC;AAQvC,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;AAE1D,MAAM,WAAW,YAAY;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,cAAc,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,WAAW,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;CACnB;AAeD,wBAAsB,QAAQ,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAyB1E;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAU7C;AAED,wBAAgB,YAAY,CAAC,cAAc,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAIvE;AAED,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAShE"}

package/dist/core/extensions/builtin/webfetch/webfetch/fetcher.js CHANGED Viewed

@@ -1,28 +1,27 @@
+import { request } from "undici";
 import { InvalidWebfetchUrlError, WebfetchAbortError, WebfetchResponseTooLargeError, WebfetchTimeoutError, } from "./errors.js";
 export const MAX_RESPONSE_SIZE_BYTES = 5 * 1024 * 1024;
 export const DEFAULT_TIMEOUT_SECONDS = 30;
 export const MAX_TIMEOUT_SECONDS = 120;
+const MAX_REDIRECTS = 20;
 const BROWSER_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36";
+const CHROME_MAJOR_VERSION = "143";
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
 export async function fetchUrl(options) {
     validateUrl(options.url);
     const timeoutSeconds = clampTimeout(options.timeoutSeconds);
+    const timeoutMs = timeoutSeconds * 1000;
     const controller = new AbortController();
-    const timeout = setTimeout(() => controller.abort(new WebfetchTimeoutError(`Request timed out after ${timeoutSeconds}s`)), timeoutSeconds * 1000);
+    const timeout = setTimeout(() => controller.abort(new WebfetchTimeoutError(`Request timed out after ${timeoutSeconds}s`)), timeoutMs);
     const removeAbortForwarder = forwardAbort(options.signal, controller);
     try {
-        const response = await fetch(options.url, {
-            headers: buildHeaders(options.format, BROWSER_USER_AGENT),
+        const response = await requestUrl({
+            url: options.url,
+            format: options.format,
             signal: controller.signal,
+            timeoutMs,
         });
-        if (response.status === 403 && response.headers.get("cf-mitigated") === "challenge") {
-            await cancelBody(response);
-            const retry = await fetch(options.url, {
-                headers: buildHeaders(options.format, "pi-webfetch"),
-                signal: controller.signal,
-            });
-            return await readFetchResponse(options.url, retry, controller.signal);
-        }
-        return await readFetchResponse(options.url, response, controller.signal);
+        return await readHttpResponse(response, controller.signal);
     }
     finally {
         clearTimeout(timeout);
@@ -57,66 +56,126 @@ export function buildAcceptHeader(format) {
             return "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1";
     }
 }
-function buildHeaders(format, userAgent) {
+function buildHeaders(format) {
     return {
         Accept: buildAcceptHeader(format),
         "Accept-Language": "en-US,en;q=0.9",
-        "User-Agent": userAgent,
+        "Sec-CH-UA": `"Google Chrome";v="${CHROME_MAJOR_VERSION}", "Chromium";v="${CHROME_MAJOR_VERSION}", "Not A(Brand";v="24"`,
+        "Sec-CH-UA-Mobile": "?0",
+        "Sec-CH-UA-Platform": '"Windows"',
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "none",
+        "Sec-Fetch-User": "?1",
+        "Upgrade-Insecure-Requests": "1",
+        "User-Agent": BROWSER_USER_AGENT,
     };
 }
-async function readFetchResponse(url, response, signal) {
+async function requestUrl(options) {
+    const headers = buildHeaders(options.format);
+    let currentUrl = options.url;
+    for (let redirectCount = 0; redirectCount <= MAX_REDIRECTS; redirectCount += 1) {
+        const response = await request(currentUrl, {
+            method: "GET",
+            headers,
+            signal: options.signal,
+            headersTimeout: options.timeoutMs,
+            bodyTimeout: options.timeoutMs,
+        });
+        if (!REDIRECT_STATUSES.has(response.statusCode)) {
+            return {
+                url: currentUrl,
+                status: response.statusCode,
+                statusText: response.statusText,
+                headers: response.headers,
+                body: response.body,
+            };
+        }
+        const location = getHeader(response.headers, "location");
+        if (!location) {
+            return {
+                url: currentUrl,
+                status: response.statusCode,
+                statusText: response.statusText,
+                headers: response.headers,
+                body: response.body,
+            };
+        }
+        if (redirectCount === MAX_REDIRECTS) {
+            return {
+                url: currentUrl,
+                status: response.statusCode,
+                statusText: response.statusText,
+                headers: response.headers,
+                body: response.body,
+            };
+        }
+        await discardBody(response.body);
+        currentUrl = new URL(location, currentUrl).toString();
+    }
+    throw new WebfetchAbortError("Redirect resolution aborted");
+}
+async function readHttpResponse(response, signal) {
     await rejectOversizedContentLength(response);
     const body = await readResponseBody(response, signal);
     return {
-        url: response.url || url,
+        url: response.url,
         status: response.status,
         statusText: response.statusText,
-        contentType: response.headers.get("content-type") ?? "",
+        contentType: getHeader(response.headers, "content-type"),
         bytes: body.length,
         body,
         truncated: body.length === MAX_RESPONSE_SIZE_BYTES,
     };
 }
 async function rejectOversizedContentLength(response) {
-    const contentLength = response.headers.get("content-length");
+    const contentLength = getHeader(response.headers, "content-length");
     if (contentLength && Number.parseInt(contentLength, 10) > MAX_RESPONSE_SIZE_BYTES) {
-        await cancelBody(response);
+        await discardBody(response.body);
         throw new WebfetchResponseTooLargeError("Response too large (exceeds 5MB limit)");
     }
 }
-async function cancelBody(response) {
+function getHeader(headers, name) {
+    const value = headers[name.toLowerCase()];
+    if (Array.isArray(value))
+        return value.join(", ");
+    return value ?? "";
+}
+async function discardBody(body) {
     try {
-        await response.body?.cancel();
+        await body.dump({ limit: 1024 });
     }
-    catch {
-        // Preserve the caller's original failure.
+    catch (error) {
+        if (error instanceof Error) {
+            body.destroy(error);
+            return;
+        }
+        throw error;
     }
 }
 async function readResponseBody(response, signal) {
-    if (!response.body)
-        return new Uint8Array();
-    const reader = response.body.getReader();
     const chunks = [];
     let total = 0;
     try {
-        while (true) {
+        for await (const chunk of response.body) {
             if (signal.aborted) {
-                await cancelReader(reader);
+                response.body.destroy();
                 throw new WebfetchAbortError("Request aborted");
             }
-            const read = await reader.read();
-            if (read.done)
-                break;
-            chunks.push(read.value);
-            total += read.value.length;
+            const bytes = toUint8Array(chunk);
+            chunks.push(bytes);
+            total += bytes.length;
             if (total > MAX_RESPONSE_SIZE_BYTES) {
-                await cancelReader(reader);
+                response.body.destroy();
                 throw new WebfetchResponseTooLargeError("Response too large (exceeds 5MB limit)");
             }
         }
     }
-    finally {
-        reader.releaseLock();
+    catch (error) {
+        if (signal.aborted) {
+            throw new WebfetchAbortError("Request aborted");
+        }
+        throw error;
     }
     const body = new Uint8Array(total);
     let offset = 0;
@@ -126,13 +185,12 @@ async function readResponseBody(response, signal) {
     }
     return body;
 }
-async function cancelReader(reader) {
-    try {
-        await reader.cancel();
-    }
-    catch {
-        // Preserve the caller's original failure.
-    }
+function toUint8Array(chunk) {
+    if (chunk instanceof Uint8Array)
+        return chunk;
+    if (typeof chunk === "string")
+        return new TextEncoder().encode(chunk);
+    throw new Error("Unexpected response body chunk");
 }
 function forwardAbort(signal, controller) {
     if (!signal)

package/dist/core/extensions/builtin/webfetch/webfetch/fetcher.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/fetcher.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,uBAAuB,EACvB,kBAAkB,EAClB,6BAA6B,EAC7B,oBAAoB,GACpB,MAAM,aAAa,CAAC;AAErB,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACvD,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAC1C,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEvC,MAAM,kBAAkB,GACvB,iHAAiH,CAAC;AAqBnH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB;IACnD,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,UAAU,CACzB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,oBAAoB,CAAC,2BAA2B,cAAc,GAAG,CAAC,CAAC,EAC9F,cAAc,GAAG,IAAI,CACrB,CAAC;IACF,MAAM,oBAAoB,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAEtE,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE;YACzC,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACzD,MAAM,EAAE,UAAU,CAAC,MAAM;SACzB,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,WAAW,EAAE,CAAC;YACrF,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE;gBACtC,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC;gBACpD,MAAM,EAAE,UAAU,CAAC,MAAM;aACzB,CAAC,CAAC;YACH,OAAO,MAAM,iBAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QACvE,CAAC;QAED,OAAO,MAAM,iBAAiB,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC;YAAS,CAAC;QACV,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,oBAAoB,EAAE,CAAC;IACxB,CAAC;AACF,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACtC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/D,MAAM,IAAI,uBAAuB,CAAC,yCAAyC,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,CAAC;QACJ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,uBAAuB,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;IAC1D,CAAC;AACF,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,cAAkC;IAC9D,IAAI,cAAc,KAAK,SAAS;QAAE,OAAO,uBAAuB,CAAC;IACjE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,cAAc,IAAI,CAAC;QAAE,OAAO,uBAAuB,CAAC;IAC5F,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,mBAAmB,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAsB;IACvD,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,UAAU;YACd,OAAO,0FAA0F,CAAC;QACnG,KAAK,MAAM;YACV,OAAO,mEAAmE,CAAC;QAC5E,KAAK,MAAM;YACV,OAAO,gGAAgG,CAAC;IAC1G,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,MAAsB,EAAE,SAAiB;IAC9D,OAAO;QACN,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC;QACjC,iBAAiB,EAAE,gBAAgB;QACnC,YAAY,EAAE,SAAS;KACvB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,GAAW,EAAE,QAAkB,EAAE,MAAmB;IACpF,MAAM,4BAA4B,CAAC,QAAQ,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtD,OAAO;QACN,GAAG,EAAE,QAAQ,CAAC,GAAG,IAAI,GAAG;QACxB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE;QACvD,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM,KAAK,uBAAuB;KAClD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,4BAA4B,CAAC,QAAkB;IAC7D,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC7D,IAAI,aAAa,IAAI,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,uBAAuB,EAAE,CAAC;QACnF,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3B,MAAM,IAAI,6BAA6B,CAAC,wCAAwC,CAAC,CAAC;IACnF,CAAC;AACF,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,QAAkB;IAC3C,IAAI,CAAC;QACJ,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACR,0CAA0C;IAC3C,CAAC;AACF,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,QAAkB,EAAE,MAAmB;IACtE,IAAI,CAAC,QAAQ,CAAC,IAAI;QAAE,OAAO,IAAI,UAAU,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACzC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,IAAI,CAAC;QACJ,OAAO,IAAI,EAAE,CAAC;YACb,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC3B,MAAM,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;YACjD,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,IAAI,CAAC,IAAI;gBAAE,MAAM;YACrB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxB,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YAC3B,IAAI,KAAK,GAAG,uBAAuB,EAAE,CAAC;gBACrC,MAAM,YAAY,CAAC,MAAM,CAAC,CAAC;gBAC3B,MAAM,IAAI,6BAA6B,CAAC,wCAAwC,CAAC,CAAC;YACnF,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,MAAM,CAAC,WAAW,EAAE,CAAC;IACtB,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;IACxB,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAA+C;IAC1E,IAAI,CAAC;QACJ,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACR,0CAA0C;IAC3C,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,MAA+B,EAAE,UAA2B;IACjF,IAAI,CAAC,MAAM;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IAC7B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,QAAQ,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC7D,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC5D,CAAC","sourcesContent":["import {\n\tInvalidWebfetchUrlError,\n\tWebfetchAbortError,\n\tWebfetchResponseTooLargeError,\n\tWebfetchTimeoutError,\n} from \"./errors.ts\";\n\nexport const MAX_RESPONSE_SIZE_BYTES = 5 * 1024 * 1024;\nexport const DEFAULT_TIMEOUT_SECONDS = 30;\nexport const MAX_TIMEOUT_SECONDS = 120;\n\nconst BROWSER_USER_AGENT =\n\t\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36\";\n\nexport type WebfetchFormat = \"markdown\" \| \"text\" \| \"html\";\n\nexport interface FetchOptions {\n\turl: string;\n\tformat: WebfetchFormat;\n\ttimeoutSeconds?: number;\n\tsignal?: AbortSignal;\n}\n\nexport interface FetchResult {\n\turl: string;\n\tstatus: number;\n\tstatusText: string;\n\tcontentType: string;\n\tbytes: number;\n\tbody: Uint8Array;\n\ttruncated: boolean;\n}\n\nexport async function fetchUrl(options: FetchOptions): Promise<FetchResult> {\n\tvalidateUrl(options.url);\n\n\tconst timeoutSeconds = clampTimeout(options.timeoutSeconds);\n\tconst controller = new AbortController();\n\tconst timeout = setTimeout(\n\t\t() => controller.abort(new WebfetchTimeoutError(`Request timed out after ${timeoutSeconds}s`)),\n\t\ttimeoutSeconds * 1000,\n\t);\n\tconst removeAbortForwarder = forwardAbort(options.signal, controller);\n\n\ttry {\n\t\tconst response = await fetch(options.url, {\n\t\t\theaders: buildHeaders(options.format, BROWSER_USER_AGENT),\n\t\t\tsignal: controller.signal,\n\t\t});\n\n\t\tif (response.status === 403 && response.headers.get(\"cf-mitigated\") === \"challenge\") {\n\t\t\tawait cancelBody(response);\n\t\t\tconst retry = await fetch(options.url, {\n\t\t\t\theaders: buildHeaders(options.format, \"pi-webfetch\"),\n\t\t\t\tsignal: controller.signal,\n\t\t\t});\n\t\t\treturn await readFetchResponse(options.url, retry, controller.signal);\n\t\t}\n\n\t\treturn await readFetchResponse(options.url, response, controller.signal);\n\t} finally {\n\t\tclearTimeout(timeout);\n\t\tremoveAbortForwarder();\n\t}\n}\n\nexport function validateUrl(url: string): void {\n\tif (!url.startsWith(\"http://\") && !url.startsWith(\"https://\")) {\n\t\tthrow new InvalidWebfetchUrlError(\"URL must start with http:// or https://\");\n\t}\n\n\ttry {\n\t\tnew URL(url);\n\t} catch {\n\t\tthrow new InvalidWebfetchUrlError(`Invalid URL: ${url}`);\n\t}\n}\n\nexport function clampTimeout(timeoutSeconds: number \| undefined): number {\n\tif (timeoutSeconds === undefined) return DEFAULT_TIMEOUT_SECONDS;\n\tif (!Number.isFinite(timeoutSeconds) \|\| timeoutSeconds <= 0) return DEFAULT_TIMEOUT_SECONDS;\n\treturn Math.min(Math.ceil(timeoutSeconds), MAX_TIMEOUT_SECONDS);\n}\n\nexport function buildAcceptHeader(format: WebfetchFormat): string {\n\tswitch (format) {\n\t\tcase \"markdown\":\n\t\t\treturn \"text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, /;q=0.1\";\n\t\tcase \"text\":\n\t\t\treturn \"text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, /;q=0.1\";\n\t\tcase \"html\":\n\t\t\treturn \"text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, /;q=0.1\";\n\t}\n}\n\nfunction buildHeaders(format: WebfetchFormat, userAgent: string): Record<string, string> {\n\treturn {\n\t\tAccept: buildAcceptHeader(format),\n\t\t\"Accept-Language\": \"en-US,en;q=0.9\",\n\t\t\"User-Agent\": userAgent,\n\t};\n}\n\nasync function readFetchResponse(url: string, response: Response, signal: AbortSignal): Promise<FetchResult> {\n\tawait rejectOversizedContentLength(response);\n\tconst body = await readResponseBody(response, signal);\n\treturn {\n\t\turl: response.url \|\| url,\n\t\tstatus: response.status,\n\t\tstatusText: response.statusText,\n\t\tcontentType: response.headers.get(\"content-type\") ?? \"\",\n\t\tbytes: body.length,\n\t\tbody,\n\t\ttruncated: body.length === MAX_RESPONSE_SIZE_BYTES,\n\t};\n}\n\nasync function rejectOversizedContentLength(response: Response): Promise<void> {\n\tconst contentLength = response.headers.get(\"content-length\");\n\tif (contentLength && Number.parseInt(contentLength, 10) > MAX_RESPONSE_SIZE_BYTES) {\n\t\tawait cancelBody(response);\n\t\tthrow new WebfetchResponseTooLargeError(\"Response too large (exceeds 5MB limit)\");\n\t}\n}\n\nasync function cancelBody(response: Response): Promise<void> {\n\ttry {\n\t\tawait response.body?.cancel();\n\t} catch {\n\t\t// Preserve the caller's original failure.\n\t}\n}\n\nasync function readResponseBody(response: Response, signal: AbortSignal): Promise<Uint8Array> {\n\tif (!response.body) return new Uint8Array();\n\n\tconst reader = response.body.getReader();\n\tconst chunks: Uint8Array[] = [];\n\tlet total = 0;\n\n\ttry {\n\t\twhile (true) {\n\t\t\tif (signal.aborted) {\n\t\t\t\tawait cancelReader(reader);\n\t\t\t\tthrow new WebfetchAbortError(\"Request aborted\");\n\t\t\t}\n\t\t\tconst read = await reader.read();\n\t\t\tif (read.done) break;\n\t\t\tchunks.push(read.value);\n\t\t\ttotal += read.value.length;\n\t\t\tif (total > MAX_RESPONSE_SIZE_BYTES) {\n\t\t\t\tawait cancelReader(reader);\n\t\t\t\tthrow new WebfetchResponseTooLargeError(\"Response too large (exceeds 5MB limit)\");\n\t\t\t}\n\t\t}\n\t} finally {\n\t\treader.releaseLock();\n\t}\n\n\tconst body = new Uint8Array(total);\n\tlet offset = 0;\n\tfor (const chunk of chunks) {\n\t\tbody.set(chunk, offset);\n\t\toffset += chunk.length;\n\t}\n\treturn body;\n}\n\nasync function cancelReader(reader: ReadableStreamDefaultReader<Uint8Array>): Promise<void> {\n\ttry {\n\t\tawait reader.cancel();\n\t} catch {\n\t\t// Preserve the caller's original failure.\n\t}\n}\n\nfunction forwardAbort(signal: AbortSignal \| undefined, controller: AbortController): () => void {\n\tif (!signal) return () => {};\n\tif (signal.aborted) {\n\t\tcontroller.abort(signal.reason);\n\t\treturn () => {};\n\t}\n\n\tconst listener = (): void => controller.abort(signal.reason);\n\tsignal.addEventListener(\"abort\", listener, { once: true });\n\treturn () => signal.removeEventListener(\"abort\", listener);\n}\n"]}
1	+ {"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/fetcher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAEjC,OAAO,EACN,uBAAuB,EACvB,kBAAkB,EAClB,6BAA6B,EAC7B,oBAAoB,GACpB,MAAM,aAAa,CAAC;AAErB,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACvD,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAC1C,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AACvC,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,kBAAkB,GACvB,iHAAiH,CAAC;AACnH,MAAM,oBAAoB,GAAG,KAAK,CAAC;AACnC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAkC7D,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAqB;IACnD,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAEzB,MAAM,cAAc,GAAG,YAAY,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,cAAc,GAAG,IAAI,CAAC;IACxC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,UAAU,CACzB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,oBAAoB,CAAC,2BAA2B,cAAc,GAAG,CAAC,CAAC,EAC9F,SAAS,CACT,CAAC;IACF,MAAM,oBAAoB,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAEtE,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC;YACjC,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,SAAS;SACT,CAAC,CAAC;QAEH,OAAO,MAAM,gBAAgB,CAAC,QAAQ,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;YAAS,CAAC;QACV,YAAY,CAAC,OAAO,CAAC,CAAC;QACtB,oBAAoB,EAAE,CAAC;IACxB,CAAC;AACF,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAW;IACtC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/D,MAAM,IAAI,uBAAuB,CAAC,yCAAyC,CAAC,CAAC;IAC9E,CAAC;IAED,IAAI,CAAC;QACJ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,IAAI,uBAAuB,CAAC,gBAAgB,GAAG,EAAE,CAAC,CAAC;IAC1D,CAAC;AACF,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,cAAkC;IAC9D,IAAI,cAAc,KAAK,SAAS;QAAE,OAAO,uBAAuB,CAAC;IACjE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,cAAc,IAAI,CAAC;QAAE,OAAO,uBAAuB,CAAC;IAC5F,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,mBAAmB,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAsB;IACvD,QAAQ,MAAM,EAAE,CAAC;QAChB,KAAK,UAAU;YACd,OAAO,0FAA0F,CAAC;QACnG,KAAK,MAAM;YACV,OAAO,mEAAmE,CAAC;QAC5E,KAAK,MAAM;YACV,OAAO,gGAAgG,CAAC;IAC1G,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,MAAsB;IAC3C,OAAO;QACN,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC;QACjC,iBAAiB,EAAE,gBAAgB;QACnC,WAAW,EAAE,sBAAsB,oBAAoB,oBAAoB,oBAAoB,yBAAyB;QACxH,kBAAkB,EAAE,IAAI;QACxB,oBAAoB,EAAE,WAAW;QACjC,gBAAgB,EAAE,UAAU;QAC5B,gBAAgB,EAAE,UAAU;QAC5B,gBAAgB,EAAE,MAAM;QACxB,gBAAgB,EAAE,IAAI;QACtB,2BAA2B,EAAE,GAAG;QAChC,YAAY,EAAE,kBAAkB;KAChC,CAAC;AACH,CAAC;AASD,KAAK,UAAU,UAAU,CAAC,OAA0B;IACnD,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC7C,IAAI,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC;IAE7B,KAAK,IAAI,aAAa,GAAG,CAAC,EAAE,aAAa,IAAI,aAAa,EAAE,aAAa,IAAI,CAAC,EAAE,CAAC;QAChF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE;YAC1C,MAAM,EAAE,KAAK;YACb,OAAO;YACP,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,cAAc,EAAE,OAAO,CAAC,SAAS;YACjC,WAAW,EAAE,OAAO,CAAC,SAAS;SAC9B,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YACjD,OAAO;gBACN,GAAG,EAAE,UAAU;gBACf,MAAM,EAAE,QAAQ,CAAC,UAAU;gBAC3B,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,IAAI,EAAE,QAAQ,CAAC,IAAI;aACnB,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,OAAO;gBACN,GAAG,EAAE,UAAU;gBACf,MAAM,EAAE,QAAQ,CAAC,UAAU;gBAC3B,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,IAAI,EAAE,QAAQ,CAAC,IAAI;aACnB,CAAC;QACH,CAAC;QAED,IAAI,aAAa,KAAK,aAAa,EAAE,CAAC;YACrC,OAAO;gBACN,GAAG,EAAE,UAAU;gBACf,MAAM,EAAE,QAAQ,CAAC,UAAU;gBAC3B,UAAU,EAAE,QAAQ,CAAC,UAAU;gBAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,IAAI,EAAE,QAAQ,CAAC,IAAI;aACnB,CAAC;QACH,CAAC;QAED,MAAM,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAC;IACvD,CAAC;IAED,MAAM,IAAI,kBAAkB,CAAC,6BAA6B,CAAC,CAAC;AAC7D,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,QAAsB,EAAE,MAAmB;IAC1E,MAAM,4BAA4B,CAAC,QAAQ,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtD,OAAO;QACN,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,WAAW,EAAE,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;QACxD,KAAK,EAAE,IAAI,CAAC,MAAM;QAClB,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,MAAM,KAAK,uBAAuB;KAClD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,4BAA4B,CAAC,QAAsB;IACjE,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IACpE,IAAI,aAAa,IAAI,MAAM,CAAC,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,uBAAuB,EAAE,CAAC;QACnF,MAAM,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,IAAI,6BAA6B,CAAC,wCAAwC,CAAC,CAAC;IACnF,CAAC;AACF,CAAC;AAED,SAAS,SAAS,CAAC,OAA4B,EAAE,IAAY;IAC5D,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;IAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,OAAO,KAAK,IAAI,EAAE,CAAC;AACpB,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAwB;IAClD,IAAI,CAAC;QACJ,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAClC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO;QACR,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;AACF,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,QAAsB,EAAE,MAAmB;IAC1E,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;YACjD,CAAC;YACD,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACtB,IAAI,KAAK,GAAG,uBAAuB,EAAE,CAAC;gBACrC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,IAAI,6BAA6B,CAAC,wCAAwC,CAAC,CAAC;YACnF,CAAC;QACF,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;QACjD,CAAC;QACD,MAAM,KAAK,CAAC;IACb,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACnC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;IACxB,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IACnC,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,KAAK,CAAC;IAC9C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,YAAY,CAAC,MAA+B,EAAE,UAA2B;IACjF,IAAI,CAAC,MAAM;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IAC7B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,QAAQ,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC7D,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3D,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAC5D,CAAC","sourcesContent":["import type { IncomingHttpHeaders } from \"node:http\";\nimport { request } from \"undici\";\n\nimport {\n\tInvalidWebfetchUrlError,\n\tWebfetchAbortError,\n\tWebfetchResponseTooLargeError,\n\tWebfetchTimeoutError,\n} from \"./errors.ts\";\n\nexport const MAX_RESPONSE_SIZE_BYTES = 5 * 1024 * 1024;\nexport const DEFAULT_TIMEOUT_SECONDS = 30;\nexport const MAX_TIMEOUT_SECONDS = 120;\nconst MAX_REDIRECTS = 20;\n\nconst BROWSER_USER_AGENT =\n\t\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36\";\nconst CHROME_MAJOR_VERSION = \"143\";\nconst REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);\n\nexport type WebfetchFormat = \"markdown\" \| \"text\" \| \"html\";\n\nexport interface FetchOptions {\n\turl: string;\n\tformat: WebfetchFormat;\n\ttimeoutSeconds?: number;\n\tsignal?: AbortSignal;\n}\n\nexport interface FetchResult {\n\turl: string;\n\tstatus: number;\n\tstatusText: string;\n\tcontentType: string;\n\tbytes: number;\n\tbody: Uint8Array;\n\ttruncated: boolean;\n}\n\ninterface HttpResponse {\n\treadonly url: string;\n\treadonly status: number;\n\treadonly statusText: string;\n\treadonly headers: IncomingHttpHeaders;\n\treadonly body: ResponseBodyStream;\n}\n\ninterface ResponseBodyStream extends AsyncIterable<unknown> {\n\tdestroy(error?: Error): void;\n\tdump(options?: { limit: number; signal?: AbortSignal }): Promise<void>;\n}\n\nexport async function fetchUrl(options: FetchOptions): Promise<FetchResult> {\n\tvalidateUrl(options.url);\n\n\tconst timeoutSeconds = clampTimeout(options.timeoutSeconds);\n\tconst timeoutMs = timeoutSeconds * 1000;\n\tconst controller = new AbortController();\n\tconst timeout = setTimeout(\n\t\t() => controller.abort(new WebfetchTimeoutError(`Request timed out after ${timeoutSeconds}s`)),\n\t\ttimeoutMs,\n\t);\n\tconst removeAbortForwarder = forwardAbort(options.signal, controller);\n\n\ttry {\n\t\tconst response = await requestUrl({\n\t\t\turl: options.url,\n\t\t\tformat: options.format,\n\t\t\tsignal: controller.signal,\n\t\t\ttimeoutMs,\n\t\t});\n\n\t\treturn await readHttpResponse(response, controller.signal);\n\t} finally {\n\t\tclearTimeout(timeout);\n\t\tremoveAbortForwarder();\n\t}\n}\n\nexport function validateUrl(url: string): void {\n\tif (!url.startsWith(\"http://\") && !url.startsWith(\"https://\")) {\n\t\tthrow new InvalidWebfetchUrlError(\"URL must start with http:// or https://\");\n\t}\n\n\ttry {\n\t\tnew URL(url);\n\t} catch {\n\t\tthrow new InvalidWebfetchUrlError(`Invalid URL: ${url}`);\n\t}\n}\n\nexport function clampTimeout(timeoutSeconds: number \| undefined): number {\n\tif (timeoutSeconds === undefined) return DEFAULT_TIMEOUT_SECONDS;\n\tif (!Number.isFinite(timeoutSeconds) \|\| timeoutSeconds <= 0) return DEFAULT_TIMEOUT_SECONDS;\n\treturn Math.min(Math.ceil(timeoutSeconds), MAX_TIMEOUT_SECONDS);\n}\n\nexport function buildAcceptHeader(format: WebfetchFormat): string {\n\tswitch (format) {\n\t\tcase \"markdown\":\n\t\t\treturn \"text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, /;q=0.1\";\n\t\tcase \"text\":\n\t\t\treturn \"text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, /;q=0.1\";\n\t\tcase \"html\":\n\t\t\treturn \"text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, /;q=0.1\";\n\t}\n}\n\nfunction buildHeaders(format: WebfetchFormat): Record<string, string> {\n\treturn {\n\t\tAccept: buildAcceptHeader(format),\n\t\t\"Accept-Language\": \"en-US,en;q=0.9\",\n\t\t\"Sec-CH-UA\": `\"Google Chrome\";v=\"${CHROME_MAJOR_VERSION}\", \"Chromium\";v=\"${CHROME_MAJOR_VERSION}\", \"Not A(Brand\";v=\"24\"`,\n\t\t\"Sec-CH-UA-Mobile\": \"?0\",\n\t\t\"Sec-CH-UA-Platform\": '\"Windows\"',\n\t\t\"Sec-Fetch-Dest\": \"document\",\n\t\t\"Sec-Fetch-Mode\": \"navigate\",\n\t\t\"Sec-Fetch-Site\": \"none\",\n\t\t\"Sec-Fetch-User\": \"?1\",\n\t\t\"Upgrade-Insecure-Requests\": \"1\",\n\t\t\"User-Agent\": BROWSER_USER_AGENT,\n\t};\n}\n\ninterface RequestUrlOptions {\n\treadonly url: string;\n\treadonly format: WebfetchFormat;\n\treadonly signal: AbortSignal;\n\treadonly timeoutMs: number;\n}\n\nasync function requestUrl(options: RequestUrlOptions): Promise<HttpResponse> {\n\tconst headers = buildHeaders(options.format);\n\tlet currentUrl = options.url;\n\n\tfor (let redirectCount = 0; redirectCount <= MAX_REDIRECTS; redirectCount += 1) {\n\t\tconst response = await request(currentUrl, {\n\t\t\tmethod: \"GET\",\n\t\t\theaders,\n\t\t\tsignal: options.signal,\n\t\t\theadersTimeout: options.timeoutMs,\n\t\t\tbodyTimeout: options.timeoutMs,\n\t\t});\n\n\t\tif (!REDIRECT_STATUSES.has(response.statusCode)) {\n\t\t\treturn {\n\t\t\t\turl: currentUrl,\n\t\t\t\tstatus: response.statusCode,\n\t\t\t\tstatusText: response.statusText,\n\t\t\t\theaders: response.headers,\n\t\t\t\tbody: response.body,\n\t\t\t};\n\t\t}\n\n\t\tconst location = getHeader(response.headers, \"location\");\n\t\tif (!location) {\n\t\t\treturn {\n\t\t\t\turl: currentUrl,\n\t\t\t\tstatus: response.statusCode,\n\t\t\t\tstatusText: response.statusText,\n\t\t\t\theaders: response.headers,\n\t\t\t\tbody: response.body,\n\t\t\t};\n\t\t}\n\n\t\tif (redirectCount === MAX_REDIRECTS) {\n\t\t\treturn {\n\t\t\t\turl: currentUrl,\n\t\t\t\tstatus: response.statusCode,\n\t\t\t\tstatusText: response.statusText,\n\t\t\t\theaders: response.headers,\n\t\t\t\tbody: response.body,\n\t\t\t};\n\t\t}\n\n\t\tawait discardBody(response.body);\n\t\tcurrentUrl = new URL(location, currentUrl).toString();\n\t}\n\n\tthrow new WebfetchAbortError(\"Redirect resolution aborted\");\n}\n\nasync function readHttpResponse(response: HttpResponse, signal: AbortSignal): Promise<FetchResult> {\n\tawait rejectOversizedContentLength(response);\n\tconst body = await readResponseBody(response, signal);\n\treturn {\n\t\turl: response.url,\n\t\tstatus: response.status,\n\t\tstatusText: response.statusText,\n\t\tcontentType: getHeader(response.headers, \"content-type\"),\n\t\tbytes: body.length,\n\t\tbody,\n\t\ttruncated: body.length === MAX_RESPONSE_SIZE_BYTES,\n\t};\n}\n\nasync function rejectOversizedContentLength(response: HttpResponse): Promise<void> {\n\tconst contentLength = getHeader(response.headers, \"content-length\");\n\tif (contentLength && Number.parseInt(contentLength, 10) > MAX_RESPONSE_SIZE_BYTES) {\n\t\tawait discardBody(response.body);\n\t\tthrow new WebfetchResponseTooLargeError(\"Response too large (exceeds 5MB limit)\");\n\t}\n}\n\nfunction getHeader(headers: IncomingHttpHeaders, name: string): string {\n\tconst value = headers[name.toLowerCase()];\n\tif (Array.isArray(value)) return value.join(\", \");\n\treturn value ?? \"\";\n}\n\nasync function discardBody(body: ResponseBodyStream): Promise<void> {\n\ttry {\n\t\tawait body.dump({ limit: 1024 });\n\t} catch (error) {\n\t\tif (error instanceof Error) {\n\t\t\tbody.destroy(error);\n\t\t\treturn;\n\t\t}\n\t\tthrow error;\n\t}\n}\n\nasync function readResponseBody(response: HttpResponse, signal: AbortSignal): Promise<Uint8Array> {\n\tconst chunks: Uint8Array[] = [];\n\tlet total = 0;\n\n\ttry {\n\t\tfor await (const chunk of response.body) {\n\t\t\tif (signal.aborted) {\n\t\t\t\tresponse.body.destroy();\n\t\t\t\tthrow new WebfetchAbortError(\"Request aborted\");\n\t\t\t}\n\t\t\tconst bytes = toUint8Array(chunk);\n\t\t\tchunks.push(bytes);\n\t\t\ttotal += bytes.length;\n\t\t\tif (total > MAX_RESPONSE_SIZE_BYTES) {\n\t\t\t\tresponse.body.destroy();\n\t\t\t\tthrow new WebfetchResponseTooLargeError(\"Response too large (exceeds 5MB limit)\");\n\t\t\t}\n\t\t}\n\t} catch (error) {\n\t\tif (signal.aborted) {\n\t\t\tthrow new WebfetchAbortError(\"Request aborted\");\n\t\t}\n\t\tthrow error;\n\t}\n\n\tconst body = new Uint8Array(total);\n\tlet offset = 0;\n\tfor (const chunk of chunks) {\n\t\tbody.set(chunk, offset);\n\t\toffset += chunk.length;\n\t}\n\treturn body;\n}\n\nfunction toUint8Array(chunk: unknown): Uint8Array {\n\tif (chunk instanceof Uint8Array) return chunk;\n\tif (typeof chunk === \"string\") return new TextEncoder().encode(chunk);\n\tthrow new Error(\"Unexpected response body chunk\");\n}\n\nfunction forwardAbort(signal: AbortSignal \| undefined, controller: AbortController): () => void {\n\tif (!signal) return () => {};\n\tif (signal.aborted) {\n\t\tcontroller.abort(signal.reason);\n\t\treturn () => {};\n\t}\n\n\tconst listener = (): void => controller.abort(signal.reason);\n\tsignal.addEventListener(\"abort\", listener, { once: true });\n\treturn () => signal.removeEventListener(\"abort\", listener);\n}\n"]}

package/dist/core/extensions/builtin/webfetch/webfetch/tool.js CHANGED Viewed

@@ -42,11 +42,11 @@ export const webfetch = defineTool({
         let text = raw;
         let converted = false;
         if (isHtml && format === "markdown") {
-            text = htmlToMarkdown(raw);
+            text = htmlToMarkdown(raw, fetched.url);
             converted = true;
         }
         else if (isHtml && format === "text") {
-            text = htmlToText(raw);
+            text = htmlToText(raw, fetched.url);
             converted = true;
         }
         const details = {

package/dist/core/extensions/builtin/webfetch/webfetch/tool.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"tool.js","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAuB,MAAM,cAAc,CAAC;AAE3E,MAAM,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAU,CAAC;AAE/D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC1B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,+BAA+B,EAAE,CAAC;IAClE,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,UAAU,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAU,EAAE;QACjD,WAAW,EAAE,4DAA4D;KACzE,CAAC,CACF;IACD,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,2CAA2C,EAAE,CAAC,CAAC;CACjG,CAAC,CAAC;AAwBH,MAAM,CAAC,MAAM,QAAQ,GAAG,UAAU,CAAuC;IACxE,IAAI,EAAE,UAAU;IAChB,KAAK,EAAE,WAAW;IAClB,WAAW,EACV,8EAA8E;QAC9E,6DAA6D;IAC9D,aAAa,EAAE,2DAA2D;IAC1E,gBAAgB,EAAE;QACjB,qDAAqD;QACrD,4EAA4E;QAC5E,kDAAkD;KAClD;IACD,UAAU,EAAE,MAAM;IAClB,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI;QACxD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACpD,QAAQ,EAAE,CAAC;YACV,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,MAAM,CAAC,GAAG,OAAO,MAAM,aAAa,cAAc,IAAI,EAAE,CAAC;YACrG,OAAO,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,cAAc,EAAE;SACvE,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC;YAC9B,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,MAAM;YACN,cAAc;YACd,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;SAC3C,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;QAClG,IAAI,IAAI,GAAG,GAAG,CAAC;QACf,IAAI,SAAS,GAAG,KAAK,CAAC;QAEtB,IAAI,MAAM,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;YACrC,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;~~YAC3B~~,SAAS,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,IAAI,MAAM,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACxC,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;~~YACvB~~,SAAS,GAAG,IAAI,CAAC;QAClB,CAAC;QAED,MAAM,OAAO,GAAoB;YAChC,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,QAAQ,EAAE,OAAO,CAAC,GAAG;YACrB,MAAM;YACN,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,cAAc;YACd,SAAS;YACT,SAAS,EAAE,OAAO,CAAC,SAAS;SAC5B,CAAC;QAEF,OAAO;YACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACjC,OAAO;SACP,CAAC;IACH,CAAC;CACD,CAAC,CAAC;AAEH,MAAM,UAAU,mBAAmB,CAAC,KAAc;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,UAAU,CAAC;IAC3C,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACvC,IAAI,KAAK,KAAK,MAAM;YAAE,OAAO,MAAM,CAAC;IACrC,CAAC;IACD,OAAO,UAAU,CAAC;AACnB,CAAC","sourcesContent":["import { StringEnum } from \"@earendil-works/pi-ai\";\nimport { Type } from \"typebox\";\nimport { defineTool } from \"../../../types.ts\";\n\nimport { htmlToMarkdown, htmlToText } from \"./content.ts\";\nimport { clampTimeout, fetchUrl, type WebfetchFormat } from \"./fetcher.ts\";\n\nconst WEBFETCH_FORMATS = [\"markdown\", \"text\", \"html\"] as const;\n\nconst Params = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch content from\" }),\n\tformat: Type.Optional(\n\t\tStringEnum([\"markdown\", \"text\", \"html\"] as const, {\n\t\t\tdescription: \"The format to return the content in. Defaults to markdown.\",\n\t\t}),\n\t),\n\ttimeout: Type.Optional(Type.Number({ description: \"Optional timeout in seconds. Maximum 120.\" })),\n});\n\nexport interface WebfetchDetails {\n\turl: string;\n\tfinalUrl: string;\n\tformat: WebfetchFormat;\n\tstatus: number;\n\tstatusText: string;\n\tcontentType: string;\n\tbytes: number;\n\ttimeoutSeconds: number;\n\tconverted: boolean;\n\ttruncated: boolean;\n}\n\nexport interface WebfetchProgressDetails {\n\tphase: \"fetching\";\n\turl: string;\n\tformat: WebfetchFormat;\n\ttimeoutSeconds: number;\n}\n\nexport type WebfetchRenderDetails = WebfetchDetails \| WebfetchProgressDetails;\n\nexport const webfetch = defineTool<typeof Params, WebfetchRenderDetails>({\n\tname: \"webfetch\",\n\tlabel: \"Web Fetch\",\n\tdescription:\n\t\t\"Fetches content from a URL and returns it as markdown, plain text, or HTML. \" +\n\t\t\"Network use is bounded by timeout and response size limits.\",\n\tpromptSnippet: \"webfetch: retrieve URL content as markdown, text, or html\",\n\tpromptGuidelines: [\n\t\t\"Use webfetch when a specific URL must be retrieved.\",\n\t\t\"Prefer markdown format unless raw HTML or plain text is explicitly needed.\",\n\t\t\"The tool is read-only and does not modify files.\",\n\t],\n\tparameters: Params,\n\tasync execute(_toolCallId, params, signal, onUpdate, _ctx) {\n\t\tconst format = parseWebfetchFormat(params.format);\n\t\tconst timeoutSeconds = clampTimeout(params.timeout);\n\t\tonUpdate?.({\n\t\t\tcontent: [{ type: \"text\", text: `Fetching ${params.url} as ${format} (timeout ${timeoutSeconds}s)` }],\n\t\t\tdetails: { phase: \"fetching\", url: params.url, format, timeoutSeconds },\n\t\t});\n\n\t\tconst fetched = await fetchUrl({\n\t\t\turl: params.url,\n\t\t\tformat,\n\t\t\ttimeoutSeconds,\n\t\t\t...(signal === undefined ? {} : { signal }),\n\t\t});\n\t\tconst raw = new TextDecoder().decode(fetched.body);\n\t\tconst contentType = fetched.contentType.toLowerCase();\n\t\tconst isHtml = contentType.includes(\"text/html\") \|\| contentType.includes(\"application/xhtml+xml\");\n\t\tlet text = raw;\n\t\tlet converted = false;\n\n\t\tif (isHtml && format === \"markdown\") {\n\t\t\ttext = htmlToMarkdown(raw);\n\t\t\tconverted = true;\n\t\t} else if (isHtml && format === \"text\") {\n\t\t\ttext = htmlToText(raw);\n\t\t\tconverted = true;\n\t\t}\n\n\t\tconst details: WebfetchDetails = {\n\t\t\turl: params.url,\n\t\t\tfinalUrl: fetched.url,\n\t\t\tformat,\n\t\t\tstatus: fetched.status,\n\t\t\tstatusText: fetched.statusText,\n\t\t\tcontentType: fetched.contentType,\n\t\t\tbytes: fetched.bytes,\n\t\t\ttimeoutSeconds,\n\t\t\tconverted,\n\t\t\ttruncated: fetched.truncated,\n\t\t};\n\n\t\treturn {\n\t\t\tcontent: [{ type: \"text\", text }],\n\t\t\tdetails,\n\t\t};\n\t},\n});\n\nexport function parseWebfetchFormat(value: unknown): WebfetchFormat {\n\tif (value === undefined) return \"markdown\";\n\tfor (const format of WEBFETCH_FORMATS) {\n\t\tif (value === format) return format;\n\t}\n\treturn \"markdown\";\n}\n"]}
1	+ {"version":3,"file":"tool.js","sourceRoot":"","sources":["../../../../../../src/core/extensions/builtin/webfetch/webfetch/tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AAC/B,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAuB,MAAM,cAAc,CAAC;AAE3E,MAAM,gBAAgB,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAU,CAAC;AAE/D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC1B,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,+BAA+B,EAAE,CAAC;IAClE,MAAM,EAAE,IAAI,CAAC,QAAQ,CACpB,UAAU,CAAC,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAU,EAAE;QACjD,WAAW,EAAE,4DAA4D;KACzE,CAAC,CACF;IACD,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,2CAA2C,EAAE,CAAC,CAAC;CACjG,CAAC,CAAC;AAwBH,MAAM,CAAC,MAAM,QAAQ,GAAG,UAAU,CAAuC;IACxE,IAAI,EAAE,UAAU;IAChB,KAAK,EAAE,WAAW;IAClB,WAAW,EACV,8EAA8E;QAC9E,6DAA6D;IAC9D,aAAa,EAAE,2DAA2D;IAC1E,gBAAgB,EAAE;QACjB,qDAAqD;QACrD,4EAA4E;QAC5E,kDAAkD;KAClD;IACD,UAAU,EAAE,MAAM;IAClB,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI;QACxD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAClD,MAAM,cAAc,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACpD,QAAQ,EAAE,CAAC;YACV,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,MAAM,CAAC,GAAG,OAAO,MAAM,aAAa,cAAc,IAAI,EAAE,CAAC;YACrG,OAAO,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,cAAc,EAAE;SACvE,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC;YAC9B,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,MAAM;YACN,cAAc;YACd,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;SAC3C,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;QAClG,IAAI,IAAI,GAAG,GAAG,CAAC;QACf,IAAI,SAAS,GAAG,KAAK,CAAC;QAEtB,IAAI,MAAM,IAAI,MAAM,KAAK,UAAU,EAAE,CAAC;YACrC,IAAI,GAAG,cAAc,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YACxC,SAAS,GAAG,IAAI,CAAC;QAClB,CAAC;aAAM,IAAI,MAAM,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;YACxC,IAAI,GAAG,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YACpC,SAAS,GAAG,IAAI,CAAC;QAClB,CAAC;QAED,MAAM,OAAO,GAAoB;YAChC,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,QAAQ,EAAE,OAAO,CAAC,GAAG;YACrB,MAAM;YACN,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,cAAc;YACd,SAAS;YACT,SAAS,EAAE,OAAO,CAAC,SAAS;SAC5B,CAAC;QAEF,OAAO;YACN,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACjC,OAAO;SACP,CAAC;IACH,CAAC;CACD,CAAC,CAAC;AAEH,MAAM,UAAU,mBAAmB,CAAC,KAAc;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,UAAU,CAAC;IAC3C,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACvC,IAAI,KAAK,KAAK,MAAM;YAAE,OAAO,MAAM,CAAC;IACrC,CAAC;IACD,OAAO,UAAU,CAAC;AACnB,CAAC","sourcesContent":["import { StringEnum } from \"@earendil-works/pi-ai\";\nimport { Type } from \"typebox\";\nimport { defineTool } from \"../../../types.ts\";\n\nimport { htmlToMarkdown, htmlToText } from \"./content.ts\";\nimport { clampTimeout, fetchUrl, type WebfetchFormat } from \"./fetcher.ts\";\n\nconst WEBFETCH_FORMATS = [\"markdown\", \"text\", \"html\"] as const;\n\nconst Params = Type.Object({\n\turl: Type.String({ description: \"The URL to fetch content from\" }),\n\tformat: Type.Optional(\n\t\tStringEnum([\"markdown\", \"text\", \"html\"] as const, {\n\t\t\tdescription: \"The format to return the content in. Defaults to markdown.\",\n\t\t}),\n\t),\n\ttimeout: Type.Optional(Type.Number({ description: \"Optional timeout in seconds. Maximum 120.\" })),\n});\n\nexport interface WebfetchDetails {\n\turl: string;\n\tfinalUrl: string;\n\tformat: WebfetchFormat;\n\tstatus: number;\n\tstatusText: string;\n\tcontentType: string;\n\tbytes: number;\n\ttimeoutSeconds: number;\n\tconverted: boolean;\n\ttruncated: boolean;\n}\n\nexport interface WebfetchProgressDetails {\n\tphase: \"fetching\";\n\turl: string;\n\tformat: WebfetchFormat;\n\ttimeoutSeconds: number;\n}\n\nexport type WebfetchRenderDetails = WebfetchDetails \| WebfetchProgressDetails;\n\nexport const webfetch = defineTool<typeof Params, WebfetchRenderDetails>({\n\tname: \"webfetch\",\n\tlabel: \"Web Fetch\",\n\tdescription:\n\t\t\"Fetches content from a URL and returns it as markdown, plain text, or HTML. \" +\n\t\t\"Network use is bounded by timeout and response size limits.\",\n\tpromptSnippet: \"webfetch: retrieve URL content as markdown, text, or html\",\n\tpromptGuidelines: [\n\t\t\"Use webfetch when a specific URL must be retrieved.\",\n\t\t\"Prefer markdown format unless raw HTML or plain text is explicitly needed.\",\n\t\t\"The tool is read-only and does not modify files.\",\n\t],\n\tparameters: Params,\n\tasync execute(_toolCallId, params, signal, onUpdate, _ctx) {\n\t\tconst format = parseWebfetchFormat(params.format);\n\t\tconst timeoutSeconds = clampTimeout(params.timeout);\n\t\tonUpdate?.({\n\t\t\tcontent: [{ type: \"text\", text: `Fetching ${params.url} as ${format} (timeout ${timeoutSeconds}s)` }],\n\t\t\tdetails: { phase: \"fetching\", url: params.url, format, timeoutSeconds },\n\t\t});\n\n\t\tconst fetched = await fetchUrl({\n\t\t\turl: params.url,\n\t\t\tformat,\n\t\t\ttimeoutSeconds,\n\t\t\t...(signal === undefined ? {} : { signal }),\n\t\t});\n\t\tconst raw = new TextDecoder().decode(fetched.body);\n\t\tconst contentType = fetched.contentType.toLowerCase();\n\t\tconst isHtml = contentType.includes(\"text/html\") \|\| contentType.includes(\"application/xhtml+xml\");\n\t\tlet text = raw;\n\t\tlet converted = false;\n\n\t\tif (isHtml && format === \"markdown\") {\n\t\t\ttext = htmlToMarkdown(raw, fetched.url);\n\t\t\tconverted = true;\n\t\t} else if (isHtml && format === \"text\") {\n\t\t\ttext = htmlToText(raw, fetched.url);\n\t\t\tconverted = true;\n\t\t}\n\n\t\tconst details: WebfetchDetails = {\n\t\t\turl: params.url,\n\t\t\tfinalUrl: fetched.url,\n\t\t\tformat,\n\t\t\tstatus: fetched.status,\n\t\t\tstatusText: fetched.statusText,\n\t\t\tcontentType: fetched.contentType,\n\t\t\tbytes: fetched.bytes,\n\t\t\ttimeoutSeconds,\n\t\t\tconverted,\n\t\t\ttruncated: fetched.truncated,\n\t\t};\n\n\t\treturn {\n\t\t\tcontent: [{ type: \"text\", text }],\n\t\t\tdetails,\n\t\t};\n\t},\n});\n\nexport function parseWebfetchFormat(value: unknown): WebfetchFormat {\n\tif (value === undefined) return \"markdown\";\n\tfor (const format of WEBFETCH_FORMATS) {\n\t\tif (value === format) return format;\n\t}\n\treturn \"markdown\";\n}\n"]}

package/dist/core/extensions/loader.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../src/core/extensions/loader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuBH,OAAO,EAAkB,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;~~AAIhE~~,OAAO,KAAK,EACX,SAAS,EAET,gBAAgB,EAChB,gBAAgB,EAChB,oBAAoB,EAKpB,MAAM,YAAY,CAAC;~~AA0HpB~~,MAAM,MAAM,wBAAwB,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,gBAAgB,GAAG,SAAS,CAAC;AAWrH,wBAAgB,mBAAmB,IAAI,IAAI,CAI1C;AAWD;;;GAGG;AACH,wBAAgB,sBAAsB,IAAI,gBAAgB,CA8CzD;~~AA+PD~~;;GAEG;AACH,wBAAsB,wBAAwB,CAC7C,OAAO,EAAE,gBAAgB,EACzB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,gBAAgB,EACzB,aAAa,SAAa,GACxB,OAAO,CAAC,SAAS,CAAC,~~CAMpB~~;AAqDD,wBAAsB,cAAc,CACnC,KAAK,EAAE,MAAM,EAAE,EACf,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,wBAAwB,CAAA;CAAE,GACtD,OAAO,CAAC,oBAAoB,CAAC,CAE/B;AAED,wBAAsB,oBAAoB,CACzC,KAAK,EAAE,MAAM,EAAE,EACf,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,oBAAoB,CAAC,CAE/B;AA+GD;;GAEG;AACH,wBAAsB,yBAAyB,CAC9C,eAAe,EAAE,MAAM,EAAE,EACzB,GAAG,EAAE,MAAM,EACX,QAAQ,GAAE,MAAsB,EAChC,QAAQ,CAAC,EAAE,QAAQ,GACjB,OAAO,CAAC,oBAAoB,CAAC,CA2C/B"}
1	+ {"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../../src/core/extensions/loader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAuBH,OAAO,EAAkB,KAAK,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAKhE,OAAO,KAAK,EACX,SAAS,EAET,gBAAgB,EAChB,gBAAgB,EAChB,oBAAoB,EAKpB,MAAM,YAAY,CAAC;AAiIpB,MAAM,MAAM,wBAAwB,GAAG,CAAC,aAAa,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,KAAK,gBAAgB,GAAG,SAAS,CAAC;AAWrH,wBAAgB,mBAAmB,IAAI,IAAI,CAI1C;AAWD;;;GAGG;AACH,wBAAgB,sBAAsB,IAAI,gBAAgB,CA8CzD;AAiQD;;GAEG;AACH,wBAAsB,wBAAwB,CAC7C,OAAO,EAAE,gBAAgB,EACzB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,QAAQ,EAClB,OAAO,EAAE,gBAAgB,EACzB,aAAa,SAAa,GACxB,OAAO,CAAC,SAAS,CAAC,CAOpB;AAqDD,wBAAsB,cAAc,CACnC,KAAK,EAAE,MAAM,EAAE,EACf,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,gBAAgB,EAC1B,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,wBAAwB,CAAA;CAAE,GACtD,OAAO,CAAC,oBAAoB,CAAC,CAE/B;AAED,wBAAsB,oBAAoB,CACzC,KAAK,EAAE,MAAM,EAAE,EACf,GAAG,EAAE,MAAM,EACX,QAAQ,CAAC,EAAE,QAAQ,EACnB,OAAO,CAAC,EAAE,gBAAgB,GACxB,OAAO,CAAC,oBAAoB,CAAC,CAE/B;AA+GD;;GAEG;AACH,wBAAsB,yBAAyB,CAC9C,eAAe,EAAE,MAAM,EAAE,EACzB,GAAG,EAAE,MAAM,EACX,QAAQ,GAAE,MAAsB,EAChC,QAAQ,CAAC,EAAE,QAAQ,GACjB,OAAO,CAAC,oBAAoB,CAAC,CA2C/B"}