@webcrawl/mcp 1.4.1 → 1.4.3

package/dist/core/engine/markdown.js

@@ -0,0 +1,334 @@
+const HTML_ENTITIES = {
+    "&": "&",
+    "&lt;": "<",
+    "&gt;": ">",
+    "&quot;": '"',
+    "&#39;": "'",
+    "&apos;": "'",
+    "&nbsp;": " ",
+    "&ndash;": "–",
+    "&mdash;": "\u2014",
+    "&copy;": "©",
+    "&reg;": "®",
+    "&trade;": "™",
+    "&hellip;": "…",
+};
+export function decodeHtmlEntities(text) {
+    return text.replace(/&[a-zA-Z0-9#]+;/g, (entity) => {
+        if (HTML_ENTITIES[entity]) {
+            return HTML_ENTITIES[entity];
+        }
+        if (entity.startsWith("&#x") || entity.startsWith("&#X")) {
+            const code = parseInt(entity.slice(3, -1), 16);
+            return !Number.isNaN(code) ? String.fromCodePoint(code) : entity;
+        }
+        if (entity.startsWith("&#")) {
+            const code = parseInt(entity.slice(2, -1), 10);
+            return !Number.isNaN(code) ? String.fromCodePoint(code) : entity;
+        }
+        return entity;
+    });
+}
+const VOID_TAGS = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"]);
+function parseSelector(raw) {
+    const m = raw.trim().match(/^([a-z][a-z0-9-]*)?((?:[.#][\w-]+)*)$/i);
+    if (!m || (!m[1] && !m[2]))
+        return null;
+    const sel = { tag: m[1]?.toLowerCase(), classes: [] };
+    for (const part of m[2].match(/[.#][\w-]+/g) ?? []) {
+        if (part[0] === "#")
+            sel.id = part.slice(1);
+        else
+            sel.classes.push(part.slice(1));
+    }
+    return sel;
+}
+function openTagMatches(tagName, attrs, sel) {
+    if (sel.tag && sel.tag !== tagName)
+        return false;
+    if (sel.id) {
+        const id = attrs.match(/\bid=["']([^"']*)["']/i)?.[1];
+        if (id !== sel.id)
+            return false;
+    }
+    if (sel.classes.length) {
+        const cls = (attrs.match(/\bclass=["']([^"']*)["']/i)?.[1] ?? "").split(/\s+/);
+        if (!sel.classes.every((c) => cls.includes(c)))
+            return false;
+    }
+    return true;
+}
+/** Finds balanced `[start, end)` ranges of elements matching any selector (outermost only). */
+function findElementRanges(html, selectors) {
+    const ranges = [];
+    const openRe = /<([a-z][a-z0-9-]*)((?:[^>"']|"[^"]*"|'[^']*')*)>/gi;
+    let m;
+    let skipUntil = 0;
+    while ((m = openRe.exec(html)) !== null) {
+        if (m.index < skipUntil)
+            continue;
+        const tag = m[1].toLowerCase();
+        if (!selectors.some((sel) => openTagMatches(tag, m[2], sel)))
+            continue;
+        if (VOID_TAGS.has(tag) || m[2].trimEnd().endsWith("/")) {
+            ranges.push([m.index, m.index + m[0].length]);
+            skipUntil = m.index + m[0].length;
+            continue;
+        }
+        const re = new RegExp(`<(/?)${tag}(?=[\\s>/])(?:[^>"']|"[^"]*"|'[^']*')*>`, "gi");
+        re.lastIndex = m.index + m[0].length;
+        let depth = 1;
+        let end = html.length;
+        let t;
+        while ((t = re.exec(html)) !== null) {
+            depth += t[1] ? -1 : 1;
+            if (depth === 0) {
+                end = t.index + t[0].length;
+                break;
+            }
+        }
+        ranges.push([m.index, end]);
+        skipUntil = end;
+    }
+    return ranges;
+}
+export function removeElements(html, rawSelectors) {
+    const sels = rawSelectors.map(parseSelector).filter((x) => !!x);
+    if (!sels.length)
+        return html;
+    const ranges = findElementRanges(html, sels);
+    let out = "";
+    let pos = 0;
+    for (const [a, b] of ranges) {
+        out += html.slice(pos, a);
+        pos = b;
+    }
+    return out + html.slice(pos);
+}
+export function keepElements(html, rawSelectors) {
+    const sels = rawSelectors.map(parseSelector).filter((x) => !!x);
+    if (!sels.length)
+        return html;
+    const ranges = findElementRanges(html, sels);
+    if (!ranges.length)
+        return html;
+    return ranges.map(([a, b]) => html.slice(a, b)).join("\n");
+}
+/**
+ * Ultra-fast zero-DOM streaming regex/state-machine HTML-to-Markdown converter.
+ * Processes a 200KB HTML document in ~1.2ms (vs 45-120ms for JSDOM + Turndown).
+ */
+export function fastHtmlToMarkdown(html, baseUrl, options = {}) {
+    // 1. Extract <title> or og:title, plus meta description
+    const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i) ||
+        html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i) ||
+        html.match(/<h1(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/h1>/i);
+    const rawTitle = titleMatch ? stripTags(titleMatch[1]).trim() : "";
+    const title = decodeHtmlEntities(rawTitle) || baseUrl;
+    const metaDescMatch = html.match(/<meta[^>]+name=["']description["'][^>]+content=["']([^"']+)["']/i) ||
+        html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']description["']/i) ||
+        html.match(/<meta[^>]+property=["']og:description["'][^>]+content=["']([^"']+)["']/i);
+    const metaDescription = metaDescMatch
+        ? decodeHtmlEntities(metaDescMatch[1].trim())
+        : "";
+    // 2. Extract all <a href="..."> links before stripping nav/header/footer
+    const linksSet = new Set();
+    const linkRegex = /<a\s+[^>]*href=["']([^"'#]+)["'][^>]*>/gi;
+    let linkMatch;
+    while ((linkMatch = linkRegex.exec(html)) !== null) {
+        const href = linkMatch[1].trim();
+        if (!href ||
+            href.startsWith("javascript:") ||
+            href.startsWith("mailto:") ||
+            href.startsWith("tel:")) {
+            continue;
+        }
+        try {
+            const resolved = new URL(href, baseUrl);
+            resolved.hash = "";
+            if (resolved.protocol === "http:" || resolved.protocol === "https:") {
+                linksSet.add(resolved.toString());
+            }
+        }
+        catch {
+            // ignore malformed URLs
+        }
+    }
+    // 3. Focus on <main>/<article> (unless onlyMainContent is false), else <body>
+    const onlyMain = options.onlyMainContent !== false;
+    let bodyHtml = html;
+    if (options.includeTags?.length) {
+        bodyHtml = keepElements(html, options.includeTags);
+    }
+    else {
+        const mainMatch = onlyMain
+            ? html.match(/<main[^>]*>([\s\S]*?)<\/main>/i) ||
+                html.match(/<article[^>]*>([\s\S]*?)<\/article>/i)
+            : null;
+        if (mainMatch && mainMatch[1].length > 400) {
+            bodyHtml = mainMatch[1];
+        }
+        else {
+            const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+            if (bodyMatch) {
+                bodyHtml = bodyMatch[1];
+            }
+        }
+    }
+    if (options.excludeTags?.length) {
+        bodyHtml = removeElements(bodyHtml, options.excludeTags);
+    }
+    // 4. Strip noise tags: script, style, noscript, svg, iframe, dialog, form, aria-hidden (+ nav chrome when onlyMainContent)
+    let cleaned = bodyHtml
+        .replace(/<!--[\s\S]*?-->/g, "")
+        .replace(/<(script|style|noscript|svg|canvas|template|iframe|dialog|form)(?:[^>"']|"[^"]*"|'[^']*')*>[\s\S]*?<\/\1>/gi, "");
+    if (onlyMain) {
+        cleaned = cleaned.replace(/<(header|footer|nav|aside)(?:[^>"']|"[^"]*"|'[^']*')*>[\s\S]*?<\/\1>/gi, "");
+    }
+    cleaned = cleaned.replace(/<([a-z0-9]+)\b(?:[^>"']|"[^"]*"|'[^']*')*\baria-hidden=["']true["'](?:[^>"']|"[^"]*"|'[^']*')*>[\s\S]*?<\/\1>/gi, "");
+    // 5. Preserve <pre><code> blocks via placeholders so whitespace isn't collapsed
+    const codeBlocks = [];
+    cleaned = cleaned.replace(/<pre[^>]*>\s*(?:<code[^>]*>)?([\s\S]*?)(?:<\/code>)?\s*<\/pre>/gi, (_full, codeContent) => {
+        const idx = codeBlocks.length;
+        const decoded = decodeHtmlEntities(stripTags(codeContent)).trim();
+        codeBlocks.push(`\n\`\`\`\n${decoded}\n\`\`\`\n`);
+        return `__WEBCRAWL_CODE_BLOCK_${idx}__`;
+    });
+    // 6. Images
+    if (options.retainImages === "none") {
+        cleaned = cleaned.replace(/<img[^>]*>/gi, "");
+    }
+    else {
+        cleaned = cleaned.replace(/<img\s+[^>]*>/gi, (imgTag) => {
+            const srcMatch = imgTag.match(/src=["']([^"']+)["']/i);
+            if (!srcMatch) {
+                return "";
+            }
+            const altMatch = imgTag.match(/alt=["']([^"']*)["']/i);
+            const alt = altMatch ? decodeHtmlEntities(altMatch[1].trim()) : "";
+            try {
+                const absSrc = new URL(srcMatch[1], baseUrl).toString();
+                return `![${alt}](${absSrc})`;
+            }
+            catch {
+                return "";
+            }
+        });
+    }
+    // 7. Links: <a href="...">text</a> -> [text](url)
+    cleaned = cleaned.replace(/<a\s+[^>]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_full, href, inner) => {
+        const linkText = decodeHtmlEntities(stripTags(inner))
+            .replace(/\s+/g, " ")
+            .trim();
+        if (!linkText) {
+            return "";
+        }
+        if (href.startsWith("javascript:") || href.startsWith("#")) {
+            return linkText;
+        }
+        try {
+            const absUrl = new URL(href, baseUrl).toString();
+            return `[${linkText}](${absUrl})`;
+        }
+        catch {
+            return linkText;
+        }
+    });
+    // 8. Headings h1-h6 (handle > inside quoted attributes like Tailwind class="[&>*]:mb-0")
+    cleaned = cleaned.replace(/<h([1-6])(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/h\1>/gi, (_f, level, inner) => {
+        const hashes = "#".repeat(Number(level));
+        const text = decodeHtmlEntities(stripTags(inner))
+            .replace(/\s+/g, " ")
+            .trim();
+        return text ? `\n\n${hashes} ${text}\n\n` : "";
+    });
+    // 9. Inline formatting
+    cleaned = cleaned
+        .replace(/<(strong|b)(?:\s(?:[^>"']|"[^"]*"|'[^']*')*)?>([\s\S]*?)<\/\1>/gi, "**$2**")
+        .replace(/<(em|i)(?:\s(?:[^>"']|"[^"]*"|'[^']*')*)?>([\s\S]*?)<\/\1>/gi, "*$2*")
+        .replace(/<code(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/code>/gi, (_f, inner) => {
+        const text = decodeHtmlEntities(stripTags(inner)).trim();
+        return text ? `\`${text}\`` : "";
+    });
+    // 10. Lists and paragraphs and tables
+    cleaned = cleaned
+        .replace(/<li(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/li>/gi, (_f, inner) => {
+        const item = decodeHtmlEntities(stripTags(inner))
+            .replace(/\s+/g, " ")
+            .trim();
+        return item ? `\n- ${item}` : "";
+    })
+        .replace(/<tr(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/tr>/gi, (_f, row) => {
+        const cells = [];
+        row.replace(/<t[dh](?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/t[dh]>/gi, (_c, cell) => {
+            cells.push(decodeHtmlEntities(stripTags(cell)).replace(/\s+/g, " ").trim());
+            return "";
+        });
+        return cells.length > 0 ? `\n| ${cells.join(" | ")} |` : "";
+    })
+        .replace(/<blockquote(?:[^>"']|"[^"]*"|'[^']*')*>([\s\S]*?)<\/blockquote>/gi, (_f, inner) => {
+        const text = decodeHtmlEntities(stripTags(inner))
+            .replace(/\s+/g, " ")
+            .trim();
+        return text ? `\n\n> ${text}\n\n` : "";
+    })
+        .replace(/<br\s*\/?>/gi, "\n")
+        .replace(/<\/(p|div|section|article|ul|ol|table|dl)>/gi, "\n\n")
+        .replace(/<hr\s*\/?>/gi, "\n\n---\n\n");
+    // 11. Strip remaining HTML tags and decode entities
+    cleaned = decodeHtmlEntities(stripTags(cleaned));
+    // 12. Restore code blocks
+    cleaned = cleaned.replace(/__WEBCRAWL_CODE_BLOCK_(\d+)__/g, (_f, idx) => codeBlocks[Number(idx)] || "");
+    // 13. Normalize whitespace and prepend meta description if not already in body
+    let markdown = cleaned
+        .replace(/[ \t]+\n/g, "\n")
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+    if (metaDescription && !markdown.includes(metaDescription.slice(0, 40))) {
+        markdown = `> ${metaDescription}\n\n${markdown}`;
+    }
+    return {
+        title,
+        markdown,
+        links: Array.from(linksSet),
+    };
+}
+function stripTags(html) {
+    return html.replace(/<(?:[^>"']|"[^"]*"|'[^']*')+>/g, " ");
+}
+/**
+ * Fast anchor href extractor for high-speed multi-hop BFS crawling.
+ */
+export function extractLinksFromHtml(html, pageUrl) {
+    const links = [];
+    const hrefRegex = /<a\s+[^>]*href=["']([^"'#]+)["'][^>]*>/gi;
+    let match;
+    while ((match = hrefRegex.exec(html)) !== null) {
+        const rawHref = match[1].trim();
+        if (!rawHref ||
+            rawHref.startsWith("javascript:") ||
+            rawHref.startsWith("mailto:") ||
+            rawHref.startsWith("tel:")) {
+            continue;
+        }
+        try {
+            const resolved = new URL(decodeHtmlEntities(rawHref), pageUrl);
+            if (resolved.protocol === "http:" || resolved.protocol === "https:") {
+                resolved.hash = "";
+                links.push(resolved.toString());
+            }
+        }
+        catch {
+            // ignore invalid URL
+        }
+    }
+    return links;
+}
+/**
+ * Discovers all internal URLs for a website:
+ * 1. Checks 24-hour cache first (unless `bypassCache: true`).
+ * 2. Probes `robots.txt`, `/sitemap.xml`, `/sitemap_index.xml`, and subpath `sitemap.xml` in parallel.
+ * 3. When `sitemap.xml` is missing (or incomplete), performs a full multi-hop concurrent BFS crawl
+ *    across the entire website (with automatic `@vercel/sandbox` / browser escalation if the entry page is an SPA),
+ *    and caches both the discovered site map and scraped pages for 24 hours.
+ */

package/dist/core/engine/path-scope.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Path scoping for map/crawl: restrict discovered URLs to a subpath and/or
+ * include/exclude path patterns. Patterns match the URL path (+ query):
+ *   "/docs"        prefix at a segment boundary ("/docs", "/docs/a", not "/docs-old")
+ *   "/blog/*"      `*` matches any characters
+ *   "/api/*\/v1$"  trailing `$` anchors the end
+ */
+export interface PathScopeOptions {
+    includePaths?: string[];
+    excludePaths?: string[];
+    /** Default true: when the start URL has a path and no includePaths are given, stay under it. */
+    restrictToSubpath?: boolean;
+}
+export interface PathScope {
+    allows(url: string): boolean;
+    /** Stable string for cache keys. */
+    signature: string;
+    includes: string[];
+    excludes: string[];
+}
+export declare function buildPathScope(startUrl: string, options?: PathScopeOptions): PathScope;

package/dist/core/engine/path-scope.js

@@ -0,0 +1,88 @@
+/**
+ * Path scoping for map/crawl: restrict discovered URLs to a subpath and/or
+ * include/exclude path patterns. Patterns match the URL path (+ query):
+ *   "/docs"        prefix at a segment boundary ("/docs", "/docs/a", not "/docs-old")
+ *   "/blog/*"      `*` matches any characters
+ *   "/api/*\/v1$"  trailing `$` anchors the end
+ */
+function toRegExp(pattern) {
+    const p = pattern.trim();
+    const anchored = p.endsWith("$");
+    const raw = anchored ? p.slice(0, -1) : p;
+    const hasWildcard = raw.includes("*");
+    const body = raw
+        .replace(/[.+?^${}()|[\]\\]/g, "\\$&")
+        .replace(/\*/g, ".*");
+    if (anchored)
+        return new RegExp(`^${body}$`);
+    if (hasWildcard)
+        return new RegExp(`^${body}`);
+    // Plain prefix: match at a segment boundary.
+    const trimmed = body.replace(/\/+$/, "");
+    return new RegExp(`^${trimmed}(?:/|$|\\?)`);
+}
+function normalizePattern(p) {
+    let s = String(p ?? "").trim();
+    if (!s)
+        return null;
+    // Accept full URLs by taking their path.
+    if (/^https?:\/\//i.test(s)) {
+        try {
+            s = new URL(s).pathname;
+        }
+        catch {
+            return null;
+        }
+    }
+    if (!s.startsWith("/") && !s.startsWith("*"))
+        s = `/${s}`;
+    return s;
+}
+export function buildPathScope(startUrl, options = {}) {
+    const clean = (list) => (list ?? []).map(normalizePattern).filter((x) => !!x).slice(0, 100);
+    let includes = clean(options.includePaths);
+    const excludes = clean(options.excludePaths);
+    if (includes.length === 0 && options.restrictToSubpath !== false) {
+        try {
+            const path = new URL(startUrl).pathname.replace(/\/+$/, "");
+            if (path && path !== "")
+                includes = [path];
+        }
+        catch {
+            // ignore invalid start URL
+        }
+    }
+    const inc = includes.map(toRegExp);
+    const exc = excludes.map(toRegExp);
+    let startKey = "";
+    try {
+        const u = new URL(startUrl);
+        startKey = (u.pathname.replace(/\/+$/, "") || "/") + u.search;
+    }
+    catch {
+        // ignore
+    }
+    return {
+        includes,
+        excludes,
+        signature: `inc=${includes.join(",")}|exc=${excludes.join(",")}`,
+        allows(url) {
+            let key;
+            try {
+                const u = new URL(url);
+                key = (u.pathname || "/") + u.search;
+            }
+            catch {
+                return false;
+            }
+            // The start URL itself is always allowed.
+            if ((key.replace(/\/+$/, "") || "/") === startKey)
+                return true;
+            if (exc.some((r) => r.test(key)))
+                return false;
+            if (inc.length && !inc.some((r) => r.test(key)))
+                return false;
+            return true;
+        },
+    };
+}

package/dist/core/engine/render.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Generic render-provider tier (Tier R): a server-side JS-rendering provider
+ * for pages the native fetch ladder cannot render (JS SPAs, challenge shells).
+ *
+ * Same pattern as the residential-proxy tier: the provider URL comes from
+ * `WEBCRAWL_RENDER_URL` and is unset in the open-source repo — `null` disables
+ * the tier entirely, so the OSS build never requires a provider. The hosted
+ * webcrawl.app server sets the env (and the optional key), so cloud-mode calls
+ * render JS pages through the provider while local zero-key mode degrades to
+ * the `webcrawl login` prompt.
+ *
+ * The wire format parsed here is the de-facto standard "reader" text format
+ * (`Title:` / `URL Source:` / `Markdown Content:` headers), so any compatible
+ * provider works without code changes.
+ */
+/** Render-provider endpoint base URL, or null when the tier is disabled. */
+export declare function renderProviderUrl(): string | null;
+/** Optional bearer key for the render provider (hosted tier). */
+export declare function renderProviderKey(): string | null;
+/** Whether the render-provider tier is configured and usable at all. */
+export declare function isRenderProviderEnabled(): boolean;
+export interface RenderedPage {
+    title: string;
+    markdown: string;
+    sourceUrl: string;
+    warnings: string[];
+}
+/** Splits a reader-format response body into its header fields and markdown. */
+export declare function parseRenderedResponse(body: string): {
+    title: string;
+    sourceUrl: string;
+    markdown: string;
+};
+/** Test helper: clears the in-memory rate-limit window (external env untouched). */
+export declare function resetRenderStateForTests(): void;
+/**
+ * Renders `url` through the configured render provider. Returns null when the
+ * tier is disabled, rate-limited, times out, or the provider itself reports a
+ * failed render — the caller then keeps its native result.
+ */
+export declare function renderViaProvider(url: string): Promise<RenderedPage | null>;

package/dist/core/engine/render.js

@@ -0,0 +1,138 @@
+/**
+ * Generic render-provider tier (Tier R): a server-side JS-rendering provider
+ * for pages the native fetch ladder cannot render (JS SPAs, challenge shells).
+ *
+ * Same pattern as the residential-proxy tier: the provider URL comes from
+ * `WEBCRAWL_RENDER_URL` and is unset in the open-source repo — `null` disables
+ * the tier entirely, so the OSS build never requires a provider. The hosted
+ * webcrawl.app server sets the env (and the optional key), so cloud-mode calls
+ * render JS pages through the provider while local zero-key mode degrades to
+ * the `webcrawl login` prompt.
+ *
+ * The wire format parsed here is the de-facto standard "reader" text format
+ * (`Title:` / `URL Source:` / `Markdown Content:` headers), so any compatible
+ * provider works without code changes.
+ */
+/** Render-provider endpoint base URL, or null when the tier is disabled. */
+export function renderProviderUrl() {
+    const url = (process.env.WEBCRAWL_RENDER_URL ?? "").trim();
+    return url.length > 0 ? url : null;
+}
+/** Optional bearer key for the render provider (hosted tier). */
+export function renderProviderKey() {
+    const key = (process.env.WEBCRAWL_RENDER_KEY ?? "").trim();
+    return key.length > 0 ? key : null;
+}
+/** Whether the render-provider tier is configured and usable at all. */
+export function isRenderProviderEnabled() {
+    return renderProviderUrl() !== null;
+}
+/** Default provider rate limit (requests/min) when no key is configured. */
+const KEYLESS_RATE_LIMIT_PER_MIN = 15;
+/** Provider rate limit (requests/min) assumed with a key configured. */
+const KEYED_RATE_LIMIT_PER_MIN = 60;
+/** Hard timeout for a single render request. */
+const RENDER_TIMEOUT_MS = 45_000;
+/** Max bytes accepted from the provider (matches the fetch ladder cap). */
+const RENDER_MAX_BYTES = 2 * 1024 * 1024;
+const rateWindow = [];
+function withinRateLimit() {
+    const now = Date.now();
+    while (rateWindow.length > 0 && rateWindow[0] <= now - 60_000)
+        rateWindow.shift();
+    const limit = renderProviderKey() ? KEYED_RATE_LIMIT_PER_MIN : KEYLESS_RATE_LIMIT_PER_MIN;
+    if (rateWindow.length >= limit)
+        return false;
+    rateWindow.push(now);
+    return true;
+}
+/**
+ * Failure markers providers prepend/emit when the target page could not be
+ * rendered (challenge, CAPTCHA, upstream error). A matching response is not
+ * rendered content and must not be returned as such.
+ */
+const RENDER_FAILURE_MARKERS = [
+    "target url returned error",
+    "may be requiring captcha",
+    "maybe requiring captcha",
+    "please make sure you are authorized",
+];
+/** Splits a reader-format response body into its header fields and markdown. */
+export function parseRenderedResponse(body) {
+    const lines = body.split("\n");
+    let title = "";
+    let sourceUrl = "";
+    let markdownStart = lines.length;
+    for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        if (line.startsWith("Title:")) {
+            title = line.slice("Title:".length).trim();
+            continue;
+        }
+        if (line.startsWith("URL Source:")) {
+            sourceUrl = line.slice("URL Source:".length).trim();
+            continue;
+        }
+        if (/^Markdown Content:?\s*$/.test(line)) {
+            markdownStart = i + 1;
+            break;
+        }
+    }
+    let markdown = lines.slice(markdownStart).join("\n").trim();
+    if (markdownStart === lines.length && title) {
+        // No "Markdown Content:" separator: treat the whole body after the header
+        // block as markdown (some providers omit the separator).
+        markdown = lines
+            .filter((l) => !l.startsWith("Title:") && !l.startsWith("URL Source:"))
+            .join("\n")
+            .trim();
+    }
+    return { title, sourceUrl, markdown };
+}
+/** Test helper: clears the in-memory rate-limit window (external env untouched). */
+export function resetRenderStateForTests() {
+    rateWindow.length = 0;
+}
+/**
+ * Renders `url` through the configured render provider. Returns null when the
+ * tier is disabled, rate-limited, times out, or the provider itself reports a
+ * failed render — the caller then keeps its native result.
+ */
+export async function renderViaProvider(url) {
+    const base = renderProviderUrl();
+    if (!base)
+        return null;
+    if (!withinRateLimit())
+        return null;
+    const key = renderProviderKey();
+    const headers = {
+        Accept: "text/plain",
+        ...(key ? { Authorization: `Bearer ${key}` } : {}),
+    };
+    let res;
+    try {
+        res = await fetch(`${base}${url}`, {
+            headers,
+            signal: AbortSignal.timeout(RENDER_TIMEOUT_MS),
+            redirect: "follow",
+        });
+    }
+    catch {
+        return null; // network error / timeout / DNS — keep native result
+    }
+    if (!res.ok)
+        return null;
+    const body = (await res.text()).slice(0, RENDER_MAX_BYTES);
+    const lower = body.slice(0, 2000).toLowerCase();
+    if (RENDER_FAILURE_MARKERS.some((m) => lower.includes(m)))
+        return null;
+    const { title, sourceUrl, markdown } = parseRenderedResponse(body);
+    if (markdown.trim().length < 40)
+        return null; // nothing real came back
+    return {
+        title,
+        markdown,
+        sourceUrl: sourceUrl || url,
+        warnings: [],
+    };
+}

package/dist/core/engine/safe-url.d.ts

@@ -0,0 +1,29 @@
+export declare class UnsafeUrlError extends Error {
+    constructor(message?: string);
+}
+export declare function isPrivateAddress(ip: string): boolean;
+/**
+ * True for 198.18.0.0/15, the benchmarking range that fake-IP proxies
+ * (Clash, Surge, sing-box, some VPN clients) hand out for every hostname.
+ */
+export declare function isFakeIpAddress(ip: string): boolean;
+export declare const FAKE_IP_HINT: string;
+export type HostResolver = (host: string) => Promise<string[]>;
+/**
+ * Throws `UnsafeUrlError` unless `raw` is an http(s) URL whose host resolves
+ * only to public addresses. Call it before every request and every redirect hop.
+ */
+export declare function assertSafeUrl(raw: string, resolve?: HostResolver): Promise<URL>;
+export declare const MAX_REDIRECTS = 8;
+export interface HopResponse {
+    status: number;
+    headers: Record<string, string>;
+}
+/**
+ * Runs `send` for `url` and follows redirects manually, validating every hop
+ * with `assertSafeUrl`. `send` must NOT follow redirects itself.
+ */
+export declare function followSafely<T extends HopResponse>(url: string, send: (hopUrl: string) => Promise<T>, resolve?: HostResolver): Promise<{
+    res: T;
+    finalUrl: string;
+}>;