@rivi-software/agentic-seo-check 0.0.0-stage → 0.1.1

package/dist/core/crawl.js

@@ -0,0 +1,324 @@
+import { mapConcurrent, sleep } from "./concurrency.js";
+import { DEFAULT_USER_AGENT, fetchPage, isHtmlResponse, probeUrl, toResponseInfo, } from "./fetch.js";
+import { extractPage } from "./html/extract.js";
+import { loadRobotsTxt } from "./robots.js";
+import { isLocalHost } from "./rules/canonical.js";
+import { loadSitemaps } from "./sitemap.js";
+import { hostOf, isSameSite, looksLikeHtml, normalizeUrl, stripWww, } from "./url.js";
+/**
+ * Works through a queue that grows while it runs: each crawled page can add the links it found.
+ * At most `concurrency` items run at once, and with `delayMs` each slot waits that long after its
+ * item before it takes the next one. An item that throws is left out; the rest go on.
+ */
+const runQueue = async (queue, concurrency, delayMs, work) => {
+    const limit = Math.max(1, concurrency);
+    const running = new Set();
+    const launch = (item) => {
+        const task = (async () => {
+            try {
+                await work(item);
+            }
+            catch {
+                // The page is left out of the crawl; the rest of the queue goes on.
+            }
+            if (delayMs > 0) {
+                await sleep(delayMs);
+            }
+        })().finally(() => running.delete(task));
+        running.add(task);
+    };
+    for (;;) {
+        while (running.size < limit) {
+            const item = queue.shift();
+            if (!item) {
+                break;
+            }
+            launch(item);
+        }
+        if (running.size === 0) {
+            return;
+        }
+        // oxlint-disable-next-line eslint/no-await-in-loop -- the queue grows as pages finish, so what to start next is only known once a slot frees
+        await Promise.race(running);
+    }
+};
+/** Adds `source` to the set of pages that link to `target`. */
+const addSource = (sources, target, source) => {
+    let set = sources.get(target);
+    if (!set) {
+        set = new Set();
+        sources.set(target, set);
+    }
+    set.add(source);
+};
+const runProbes = async (startUrl, fetchOpts) => {
+    const probes = {};
+    const url = new URL(startUrl);
+    const soft404Url = new URL(`/agentic-seo-probe-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`, url.origin).href;
+    const soft = await probeUrl(soft404Url, fetchOpts);
+    probes.soft404 = soft.error ? null : { url: soft404Url, status: soft.status };
+    if (isLocalHost(startUrl)) {
+        return probes;
+    }
+    if (url.protocol === "https:") {
+        const httpUrl = `http://${url.host}/`;
+        const res = await probeUrl(httpUrl, { ...fetchOpts, timeoutMs: 10_000 });
+        probes.httpRedirect = res.error
+            ? null
+            : {
+                url: httpUrl,
+                status: res.status,
+                finalUrl: res.finalUrl,
+                redirectsToHttps: res.finalUrl.startsWith("https://"),
+            };
+    }
+    const host = url.hostname;
+    const variantHost = host.startsWith("www.") ? stripWww(host) : `www.${host}`;
+    const variantUrl = `${url.protocol}//${variantHost}/`;
+    const res = await probeUrl(variantUrl, { ...fetchOpts, timeoutMs: 10_000 });
+    probes.hostVariant = res.error
+        ? null
+        : {
+            url: variantUrl,
+            status: res.status,
+            finalUrl: res.finalUrl,
+            redirectsToMain: (hostOf(res.finalUrl) ?? "") === host,
+        };
+    return probes;
+};
+/** The progress callback of a caller that does not listen for progress. */
+const ignoreProgress = () => {
+    // Nobody to tell.
+};
+export const crawlSite = async (startInput, options) => {
+    const started = performance.now();
+    const progress = options.onProgress ?? ignoreProgress;
+    const startUrl = normalizeUrl(startInput) ?? startInput;
+    const { origin } = new URL(startUrl);
+    const startHost = hostOf(startUrl) ?? "";
+    const userAgent = options.fetch.userAgent ?? DEFAULT_USER_AGENT;
+    progress({ phase: "robots", message: `Fetching ${origin}/robots.txt` });
+    const robots = await loadRobotsTxt(origin, userAgent, options.fetch);
+    const allows = options.respectRobots ? robots.allows : () => true;
+    progress({ phase: "sitemap", message: "Fetching sitemaps" });
+    const sitemapCandidates = [
+        ...new Set([...robots.info.sitemaps, new URL("/sitemap.xml", origin).href]),
+    ];
+    const sitemaps = await loadSitemaps(sitemapCandidates, options.fetch);
+    const visited = new Set([startUrl]);
+    const discovered = new Set([startUrl]);
+    const linkChecks = new Map();
+    const linkSources = new Map();
+    const externalSources = new Map();
+    const records = [];
+    const blocked = [];
+    let limitSkips = 0;
+    const accepts = (url) => {
+        if (options.includePatterns.length &&
+            !options.includePatterns.some((re) => re.test(url))) {
+            return false;
+        }
+        if (options.excludePatterns.some((re) => re.test(url))) {
+            return false;
+        }
+        return true;
+    };
+    const queue = [{ url: startUrl, depth: 0 }];
+    /** Queues the internal links a crawled page has, and remembers every link's sources. */
+    const followLinks = (links, from, depth) => {
+        for (const link of links) {
+            const target = link.url ? normalizeUrl(link.url) : null;
+            if (!target) {
+                continue;
+            }
+            if (!link.internal) {
+                addSource(externalSources, target, from);
+                continue;
+            }
+            addSource(linkSources, target, from);
+            if (!looksLikeHtml(target) ||
+                !accepts(target) ||
+                discovered.has(target)) {
+                continue;
+            }
+            discovered.add(target);
+            if (depth + 1 > options.maxDepth) {
+                continue;
+            }
+            if (visited.size >= options.maxPages) {
+                limitSkips += 1;
+                continue;
+            }
+            visited.add(target);
+            queue.push({ url: target, depth: depth + 1 });
+        }
+    };
+    /**
+     * Records where a redirected request ended. False when there is nothing more to crawl there:
+     * the redirect led to a page crawled already, or off the site.
+     */
+    const keepsRedirectTarget = (res, finalKey) => {
+        if (!linkChecks.has(finalKey)) {
+            linkChecks.set(finalKey, {
+                url: res.finalUrl,
+                status: res.status,
+                finalUrl: res.finalUrl,
+                redirectChain: [],
+                error: res.error,
+            });
+        }
+        if (visited.has(finalKey) && records.some((r) => r.url === finalKey)) {
+            return false;
+        }
+        visited.add(finalKey);
+        return isSameSite(finalKey, startHost, options.includeSubdomains);
+    };
+    /** A response that is no HTML page: an asset is skipped, an error page may still be recorded. */
+    const recordUnreadable = (item, res, finalKey) => {
+        const isAsset = res.status !== null &&
+            res.status < 400 &&
+            !res.error &&
+            res.body !== null &&
+            !isHtmlResponse(res);
+        // Record error pages so page-level findings (response.error-status) appear for the start URL.
+        if (isAsset || (item.depth !== 0 && res.status !== null)) {
+            return;
+        }
+        const page = extractPage(res.body ?? "", {
+            url: res.finalUrl,
+            siteHost: startHost,
+            includeSubdomains: options.includeSubdomains,
+        });
+        records.push({
+            url: finalKey,
+            requestedUrl: item.url,
+            page,
+            response: toResponseInfo(res),
+            depth: item.depth,
+        });
+    };
+    const processItem = async (item) => {
+        if (!allows(item.url)) {
+            blocked.push(item.url);
+            return;
+        }
+        const res = await fetchPage(item.url, options.fetch);
+        const requestedKey = item.url;
+        const finalKey = normalizeUrl(res.finalUrl) ?? res.finalUrl;
+        linkChecks.set(requestedKey, {
+            url: item.url,
+            status: res.status,
+            finalUrl: res.finalUrl,
+            redirectChain: res.redirectChain,
+            error: res.error,
+        });
+        if (finalKey !== requestedKey && !keepsRedirectTarget(res, finalKey)) {
+            return;
+        }
+        progress({
+            phase: "crawl",
+            message: `${res.status ?? "ERR"} ${res.finalUrl}`,
+            crawled: records.length + 1,
+            discovered: discovered.size,
+            url: res.finalUrl,
+        });
+        if (res.error || res.body === null || !isHtmlResponse(res)) {
+            recordUnreadable(item, res, finalKey);
+            return;
+        }
+        const page = extractPage(res.body, {
+            url: res.finalUrl,
+            siteHost: startHost,
+            includeSubdomains: options.includeSubdomains,
+        });
+        records.push({
+            url: finalKey,
+            requestedUrl: item.url,
+            page,
+            response: toResponseInfo(res),
+            depth: item.depth,
+        });
+        followLinks(page.links, finalKey, item.depth);
+    };
+    await runQueue(queue, options.concurrency, options.delayMs, processItem);
+    /* Check internal link targets that were discovered but not crawled (limits, depth, non-HTML). */
+    progress({ phase: "links", message: "Checking internal link targets" });
+    const toCheck = [...linkSources.keys()]
+        .filter((u) => !linkChecks.has(u) && allows(u))
+        .slice(0, options.maxLinkChecks);
+    const sitemapExtra = sitemaps.urls
+        .map((u) => normalizeUrl(u) ?? u)
+        .filter((u) => isSameSite(u, startHost, options.includeSubdomains) &&
+        !linkChecks.has(u) &&
+        !toCheck.includes(u) &&
+        allows(u))
+        .slice(0, Math.max(0, options.maxLinkChecks - toCheck.length));
+    for (const target of [...linkSources.keys()].filter((u) => !allows(u))) {
+        if (!blocked.includes(target)) {
+            blocked.push(target);
+        }
+    }
+    await mapConcurrent([...toCheck, ...sitemapExtra], options.concurrency, async (u) => {
+        const res = await probeUrl(u, options.fetch);
+        linkChecks.set(u, {
+            url: u,
+            status: res.status,
+            finalUrl: res.finalUrl,
+            redirectChain: res.redirectChain,
+            error: res.error,
+        });
+        if (options.delayMs) {
+            await sleep(options.delayMs);
+        }
+    });
+    let externalChecks;
+    if (options.checkExternalLinks) {
+        progress({
+            phase: "links",
+            message: `Checking ${Math.min(externalSources.size, options.maxLinkChecks)} external links`,
+        });
+        const checks = new Map();
+        externalChecks = checks;
+        const targets = [...externalSources.keys()].slice(0, options.maxLinkChecks);
+        await mapConcurrent(targets, Math.min(options.concurrency, 4), async (u) => {
+            const res = await probeUrl(u, {
+                ...options.fetch,
+                timeoutMs: Math.min(options.fetch.timeoutMs ?? 15_000, 10_000),
+            });
+            checks.set(u, {
+                url: u,
+                status: res.status,
+                finalUrl: res.finalUrl,
+                redirectChain: res.redirectChain,
+                error: res.error,
+            });
+        });
+    }
+    progress({ phase: "probes", message: "Probing soft-404 and host redirects" });
+    const probes = await runProbes(startUrl, options.fetch);
+    const stats = {
+        pagesCrawled: records.length,
+        pagesDiscovered: discovered.size,
+        linksChecked: linkChecks.size + (externalChecks?.size ?? 0),
+        maxPages: options.maxPages,
+        maxDepth: options.maxDepth,
+        hitPageLimit: limitSkips > 0,
+        durationMs: Math.round(performance.now() - started),
+        respectedRobots: options.respectRobots,
+    };
+    return {
+        startUrl,
+        records,
+        linkChecks,
+        linkSources,
+        externalChecks,
+        externalSources: options.checkExternalLinks ? externalSources : undefined,
+        robots: robots.info,
+        robotsAllows: robots.info.found ? robots.allows : null,
+        blockedByRobots: blocked,
+        sitemaps,
+        probes,
+        stats,
+    };
+};
+//# sourceMappingURL=crawl.js.map

package/dist/core/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/core/crawl.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EACL,kBAAkB,EAClB,SAAS,EACT,cAAc,EACd,QAAQ,EACR,cAAc,GACf,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEnD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAO5C,OAAO,EACL,MAAM,EACN,UAAU,EACV,aAAa,EACb,YAAY,EACZ,QAAQ,GACT,MAAM,UAAU,CAAC;AAsDlB;;;;GAIG;AACH,MAAM,QAAQ,GAAG,KAAK,EACpB,KAAkB,EAClB,WAAmB,EACnB,OAAe,EACf,IAAwC,EACzB,EAAE;IACjB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiB,CAAC;IAEzC,MAAM,MAAM,GAAG,CAAC,IAAe,EAAQ,EAAE;QACvC,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE;YACvB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;YAAC,MAAM,CAAC;gBACP,oEAAoE;YACtE,CAAC;YAED,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QAEzC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC;IAEF,SAAS,CAAC;QACR,OAAO,OAAO,CAAC,IAAI,GAAG,KAAK,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAE3B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM;YACR,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QAED,6IAA6I;QAC7I,MAAM,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;AACH,CAAC,CAAC;AAEF,+DAA+D;AAC/D,MAAM,SAAS,GAAG,CAChB,OAAiC,EACjC,MAAc,EACd,MAAc,EACR,EAAE;IACR,IAAI,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,KAAK,EACrB,QAAgB,EAChB,SAAuB,EACA,EAAE;IACzB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IAE9B,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,sBAAsB,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EACzF,GAAG,CAAC,MAAM,CACX,CAAC,IAAI,CAAC;IAEP,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IAE9E,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK;YAC7B,CAAC,CAAC,IAAI;YACN,CAAC,CAAC;gBACE,GAAG,EAAE,OAAO;gBACZ,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,gBAAgB,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;aACtD,CAAC;IACR,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,WAAW,GAAG,CAAC;IACtD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5E,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC,KAAK;QAC5B,CAAC,CAAC,IAAI;QACN,CAAC,CAAC;YACE,GAAG,EAAE,UAAU;YACf,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,eAAe,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI;SACvD,CAAC;IAEN,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,cAAc,GAAG,GAAS,EAAE;IAChC,kBAAkB;AACpB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,UAAkB,EAClB,OAAqB,EACC,EAAE;IACxB,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,cAAc,CAAC;IACtD,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;IACxD,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAEhE,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,MAAM,aAAa,EAAE,CAAC,CAAC;IACxE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;IAElE,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC,CAAC;IAE7D,MAAM,iBAAiB,GAAG;QACxB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC;KAC5E,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAEtE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAuB,CAAC;IACvD,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,OAAO,GAAG,CAAC,GAAW,EAAW,EAAE;QACvC,IACE,OAAO,CAAC,eAAe,CAAC,MAAM;YAC9B,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACnD,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,OAAO,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,MAAM,KAAK,GAAgB,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAEzD,wFAAwF;IACxF,MAAM,WAAW,GAAG,CAClB,KAAiB,EACjB,IAAY,EACZ,KAAa,EACP,EAAE;QACR,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAExD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,SAAS;YACX,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnB,SAAS,CAAC,eAAe,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,SAAS,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;YAErC,IACE,CAAC,aAAa,CAAC,MAAM,CAAC;gBACtB,CAAC,OAAO,CAAC,MAAM,CAAC;gBAChB,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EACtB,CAAC;gBACD,SAAS;YACX,CAAC;YAED,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAEvB,IAAI,KAAK,GAAG,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACjC,SAAS;YACX,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACrC,UAAU,IAAI,CAAC,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACpB,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;QAChD,CAAC;IACH,CAAC,CAAC;IAEF;;;OAGG;IACH,MAAM,mBAAmB,GAAG,CAAC,GAAgB,EAAE,QAAgB,EAAW,EAAE;QAC1E,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACvB,GAAG,EAAE,GAAG,CAAC,QAAQ;gBACjB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,EAAE;gBACjB,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAEtB,OAAO,UAAU,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpE,CAAC,CAAC;IAEF,iGAAiG;IACjG,MAAM,gBAAgB,GAAG,CACvB,IAAe,EACf,GAAgB,EAChB,QAAgB,EACV,EAAE;QACR,MAAM,OAAO,GACX,GAAG,CAAC,MAAM,KAAK,IAAI;YACnB,GAAG,CAAC,MAAM,GAAG,GAAG;YAChB,CAAC,GAAG,CAAC,KAAK;YACV,GAAG,CAAC,IAAI,KAAK,IAAI;YACjB,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAEvB,8FAA8F;QAC9F,IAAI,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC;YACzD,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE;YACvC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,KAAK,EAAE,IAAe,EAAiB,EAAE;QAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEvB,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC;QAC9B,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC;QAE5D,UAAU,CAAC,GAAG,CAAC,YAAY,EAAE;YAC3B,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,QAAQ,KAAK,YAAY,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;YACrE,OAAO;QACT,CAAC;QAED,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,GAAG,GAAG,CAAC,MAAM,IAAI,KAAK,IAAI,GAAG,CAAC,QAAQ,EAAE;YACjD,OAAO,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC;YAC3B,UAAU,EAAE,UAAU,CAAC,IAAI;YAC3B,GAAG,EAAE,GAAG,CAAC,QAAQ;SAClB,CAAC,CAAC;QAEH,IAAI,GAAG,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3D,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;YAEtC,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE;YACjC,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,QAAQ,EAAE,SAAS;YACnB,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;SAC7C,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,QAAQ;YACb,YAAY,EAAE,IAAI,CAAC,GAAG;YACtB,IAAI;YACJ,QAAQ,EAAE,cAAc,CAAC,GAAG,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;QAEH,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC;IAEF,MAAM,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAEzE,iGAAiG;IACjG,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,CAAC;IAExE,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;SACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;SAC9C,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAEnC,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAChC,MAAM,CACL,CAAC,CAAC,EAAE,EAAE,CACJ,UAAU,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC;QACnD,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QAClB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;QACpB,MAAM,CAAC,CAAC,CAAC,CACZ;SACA,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAEjE,KAAK,MAAM,MAAM,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,aAAa,CACjB,CAAC,GAAG,OAAO,EAAE,GAAG,YAAY,CAAC,EAC7B,OAAO,CAAC,WAAW,EACnB,KAAK,EAAE,CAAC,EAAE,EAAE;QACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE;YAChB,GAAG,EAAE,CAAC;YACN,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,aAAa,EAAE,GAAG,CAAC,aAAa;YAChC,KAAK,EAAE,GAAG,CAAC,KAAK;SACjB,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC,CACF,CAAC;IAEF,IAAI,cAAkD,CAAC;IAEvD,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,QAAQ,CAAC;YACP,KAAK,EAAE,OAAO;YACd,OAAO,EAAE,YAAY,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,iBAAiB;SAC5F,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC5C,cAAc,GAAG,MAAM,CAAC;QACxB,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;QAC5E,MAAM,aAAa,CACjB,OAAO,EACP,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAChC,KAAK,EAAE,CAAC,EAAE,EAAE;YACV,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE;gBAC5B,GAAG,OAAO,CAAC,KAAK;gBAChB,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC;aAC/D,CAAC,CAAC;YAEH,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE;gBACZ,GAAG,EAAE,CAAC;gBACN,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,aAAa,EAAE,GAAG,CAAC,aAAa;gBAChC,KAAK,EAAE,GAAG,CAAC,KAAK;aACjB,CAAC,CAAC;QACL,CAAC,CACF,CAAC;IACJ,CAAC;IAED,QAAQ,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,qCAAqC,EAAE,CAAC,CAAC;IAC9E,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAExD,MAAM,KAAK,GAAe;QACxB,YAAY,EAAE,OAAO,CAAC,MAAM;QAC5B,eAAe,EAAE,UAAU,CAAC,IAAI;QAChC,YAAY,EAAE,UAAU,CAAC,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,CAAC;QAC3D,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,YAAY,EAAE,UAAU,GAAG,CAAC;QAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACnD,eAAe,EAAE,OAAO,CAAC,aAAa;KACvC,CAAC;IAEF,OAAO;QACL,QAAQ;QACR,OAAO;QACP,UAAU;QACV,WAAW;QACX,cAAc;QACd,eAAe,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,SAAS;QACzE,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACtD,eAAe,EAAE,OAAO;QACxB,QAAQ;QACR,MAAM;QACN,KAAK;KACN,CAAC;AACJ,CAAC,CAAC"}

package/dist/core/fetch.d.ts

@@ -0,0 +1,27 @@
+import type { RedirectHop, ResponseInfo } from "./types.js";
+export declare const DEFAULT_USER_AGENT: string;
+export interface FetchOptions {
+    timeoutMs?: number;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    maxRedirects?: number;
+    /** Stop reading the body after this many bytes. */
+    maxBodyBytes?: number;
+}
+export interface FetchResult {
+    requestedUrl: string;
+    finalUrl: string;
+    status: number | null;
+    headers: Record<string, string>;
+    body: string | null;
+    redirectChain: RedirectHop[];
+    timeMs: number;
+    contentType: string | null;
+    error?: string;
+}
+/** GET a URL, following redirects manually so the chain is recorded. */
+export declare const fetchPage: (url: string, options?: FetchOptions, readBody?: boolean) => Promise<FetchResult>;
+/** Status/redirect check without downloading the body. */
+export declare const probeUrl: (url: string, options?: FetchOptions) => Promise<FetchResult>;
+export declare const toResponseInfo: (r: FetchResult) => ResponseInfo;
+export declare const isHtmlResponse: (r: FetchResult) => boolean;

package/dist/core/fetch.js

@@ -0,0 +1,186 @@
+import { VERSION } from "../version.js";
+export const DEFAULT_USER_AGENT = `Mozilla/5.0 (compatible; agentic-seo-check/${VERSION}; +https://github.com/rivi-software/agentic-seo-check)`;
+const KEPT_HEADERS = [
+    "content-type",
+    "x-robots-tag",
+    "link",
+    "content-language",
+    "cache-control",
+    "last-modified",
+    "location",
+    "content-length",
+    "vary",
+];
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const pickHeaders = (headers) => {
+    const out = {};
+    for (const name of KEPT_HEADERS) {
+        const value = headers.get(name);
+        if (value !== null) {
+            out[name] = value;
+        }
+    }
+    return out;
+};
+const result = (requestedUrl, finalUrl, status, headers, body, chain, started, error) => {
+    const r = {
+        requestedUrl,
+        finalUrl,
+        status,
+        headers,
+        body,
+        redirectChain: chain,
+        timeMs: Math.round(performance.now() - started),
+        contentType: headers["content-type"] ?? null,
+    };
+    if (error) {
+        r.error = error;
+    }
+    return r;
+};
+const readLimited = async (res, maxBytes) => {
+    if (!res.body) {
+        return "";
+    }
+    const reader = res.body.getReader();
+    const chunks = [];
+    let total = 0;
+    let truncated = false;
+    for (;;) {
+        // oxlint-disable-next-line eslint/no-await-in-loop -- a stream hands over its chunks one after another
+        const { done, value } = await reader.read();
+        if (done) {
+            break;
+        }
+        if (value) {
+            chunks.push(value);
+            total += value.byteLength;
+            if (total >= maxBytes) {
+                truncated = true;
+                break;
+            }
+        }
+    }
+    if (truncated) {
+        await reader.cancel().catch(() => null);
+    }
+    const charset = /charset=(?<charset>[\w-]+)/iu.exec(res.headers.get("content-type") ?? "")?.groups?.charset;
+    try {
+        return new TextDecoder(charset ?? "utf-8").decode(Buffer.concat(chunks));
+    }
+    catch {
+        return new TextDecoder("utf-8").decode(Buffer.concat(chunks));
+    }
+};
+const fetchHop = async (walk, current) => {
+    const { url, options, readBody, timeoutMs, started, chain } = walk;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+        const res = await fetch(current, {
+            method: "GET",
+            redirect: "manual",
+            signal: controller.signal,
+            headers: {
+                "user-agent": options.userAgent ?? DEFAULT_USER_AGENT,
+                accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                "accept-language": "en,*;q=0.5",
+                ...options.headers,
+            },
+        });
+        const headers = pickHeaders(res.headers);
+        if (REDIRECT_STATUSES.has(res.status) && headers["location"]) {
+            chain.push({ url: current, status: res.status });
+            let nextUrl;
+            try {
+                nextUrl = new URL(headers["location"], current).href;
+            }
+            catch {
+                return {
+                    done: result(url, current, res.status, headers, null, chain, started, `Invalid Location header: ${headers["location"]}`),
+                };
+            }
+            await res.body?.cancel().catch(() => null);
+            if (chain.some((h) => h.url === nextUrl) || nextUrl === current) {
+                return {
+                    done: result(url, nextUrl, res.status, headers, null, chain, started, "Redirect loop"),
+                };
+            }
+            return { next: nextUrl };
+        }
+        let body = null;
+        if (readBody) {
+            body = await readLimited(res, options.maxBodyBytes ?? 5_000_000);
+        }
+        else {
+            await res.body?.cancel().catch(() => null);
+        }
+        return {
+            done: result(url, current, res.status, headers, body, chain, started),
+        };
+    }
+    catch (error) {
+        let message = String(error);
+        if (error instanceof Error) {
+            message =
+                error.name === "AbortError"
+                    ? `Timed out after ${timeoutMs} ms`
+                    : error.message;
+        }
+        return {
+            done: result(url, current, null, {}, null, chain, started, message),
+        };
+    }
+    finally {
+        clearTimeout(timer);
+    }
+};
+/** GET a URL, following redirects manually so the chain is recorded. */
+export const fetchPage = async (url, options = {}, readBody = true) => {
+    const maxRedirects = options.maxRedirects ?? 10;
+    const walk = {
+        url,
+        options,
+        readBody,
+        timeoutMs: options.timeoutMs ?? 15_000,
+        started: performance.now(),
+        chain: [],
+    };
+    let current = url;
+    for (let hop = 0; hop <= maxRedirects; hop += 1) {
+        // oxlint-disable-next-line eslint/no-await-in-loop -- each request goes where the one before it redirected
+        const outcome = await fetchHop(walk, current);
+        if ("done" in outcome) {
+            return outcome.done;
+        }
+        current = outcome.next;
+    }
+    return result(url, current, null, {}, null, walk.chain, walk.started, `More than ${maxRedirects} redirects`);
+};
+/** Status/redirect check without downloading the body. */
+export const probeUrl = (url, options = {}) => fetchPage(url, options, false);
+export const toResponseInfo = (r) => {
+    const info = {
+        status: r.status,
+        finalUrl: r.finalUrl,
+        redirectChain: r.redirectChain,
+        timeMs: r.timeMs,
+        contentType: r.contentType,
+        headers: r.headers,
+    };
+    if (r.error) {
+        info.error = r.error;
+    }
+    return info;
+};
+export const isHtmlResponse = (r) => {
+    const ct = r.contentType?.toLowerCase() ?? "";
+    if (ct) {
+        return ct.includes("text/html") || ct.includes("application/xhtml");
+    }
+    if (!r.body) {
+        return false;
+    }
+    return /<html|<!doctype/iu.test(r.body.slice(0, 2000));
+};
+//# sourceMappingURL=fetch.js.map

package/dist/core/fetch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/core/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,CAAC,MAAM,kBAAkB,GAAG,8CAA8C,OAAO,wDAAwD,CAAC;AAEhJ,MAAM,YAAY,GAAG;IACnB,cAAc;IACd,cAAc;IACd,MAAM;IACN,kBAAkB;IAClB,eAAe;IACf,eAAe;IACf,UAAU;IACV,gBAAgB;IAChB,MAAM;CACP,CAAC;AAuBF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,MAAM,WAAW,GAAG,CAAC,OAAgB,EAAE,EAAE;IACvC,MAAM,GAAG,GAA2B,EAAE,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEhC,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QACpB,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CACb,YAAoB,EACpB,QAAgB,EAChB,MAAqB,EACrB,OAA+B,EAC/B,IAAmB,EACnB,KAAoB,EACpB,OAAe,EACf,KAAc,EACD,EAAE;IACf,MAAM,CAAC,GAAgB;QACrB,YAAY;QACZ,QAAQ;QACR,MAAM;QACN,OAAO;QACP,IAAI;QACJ,aAAa,EAAE,KAAK;QACpB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC/C,WAAW,EAAE,OAAO,CAAC,cAAc,CAAC,IAAI,IAAI;KAC7C,CAAC;IAEF,IAAI,KAAK,EAAE,CAAC;QACV,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,KAAK,EACvB,GAAa,EACb,QAAgB,EACC,EAAE;IACnB,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QACd,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACpC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,SAAS,CAAC;QACR,uGAAuG;QACvG,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAE5C,IAAI,IAAI,EAAE,CAAC;YACT,MAAM;QACR,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;YAE1B,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,OAAO,GAAG,8BAA8B,CAAC,IAAI,CACjD,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CACtC,EAAE,MAAM,EAAE,OAAO,CAAC;IAEnB,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;AACH,CAAC,CAAC;AAeF,MAAM,QAAQ,GAAG,KAAK,EACpB,IAAkB,EAClB,OAAe,EACM,EAAE;IACvB,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IACnE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;IAE9D,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;YAC/B,MAAM,EAAE,KAAK;YACb,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,OAAO,EAAE;gBACP,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;gBACrD,MAAM,EACJ,iEAAiE;gBACnE,iBAAiB,EAAE,YAAY;gBAC/B,GAAG,OAAO,CAAC,OAAO;aACnB;SACF,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAEzC,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YACjD,IAAI,OAAe,CAAC;YAEpB,IAAI,CAAC;gBACH,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,4BAA4B,OAAO,CAAC,UAAU,CAAC,EAAE,CAClD;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAE3C,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;gBAChE,OAAO;oBACL,IAAI,EAAE,MAAM,CACV,GAAG,EACH,OAAO,EACP,GAAG,CAAC,MAAM,EACV,OAAO,EACP,IAAI,EACJ,KAAK,EACL,OAAO,EACP,eAAe,CAChB;iBACF,CAAC;YACJ,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC3B,CAAC;QAED,IAAI,IAAI,GAAkB,IAAI,CAAC;QAE/B,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,IAAI,SAAS,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC;SACtE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAE5B,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO;gBACL,KAAK,CAAC,IAAI,KAAK,YAAY;oBACzB,CAAC,CAAC,mBAAmB,SAAS,KAAK;oBACnC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;QACtB,CAAC;QAED,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC;SACpE,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC,CAAC;AAEF,wEAAwE;AACxE,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAC5B,GAAW,EACX,OAAO,GAAiB,EAAE,EAC1B,QAAQ,GAAG,IAAI,EACO,EAAE;IACxB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC;IAEhD,MAAM,IAAI,GAAiB;QACzB,GAAG;QACH,OAAO;QACP,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,MAAM;QACtC,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE;QAC1B,KAAK,EAAE,EAAE;KACV,CAAC;IAEF,IAAI,OAAO,GAAG,GAAG,CAAC;IAElB,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,YAAY,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAChD,2GAA2G;QAC3G,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;YACtB,OAAO,OAAO,CAAC,IAAI,CAAC;QACtB,CAAC;QAED,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,OAAO,MAAM,CACX,GAAG,EACH,OAAO,EACP,IAAI,EACJ,EAAE,EACF,IAAI,EACJ,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,OAAO,EACZ,aAAa,YAAY,YAAY,CACtC,CAAC;AACJ,CAAC,CAAC;AAEF,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CACtB,GAAW,EACX,OAAO,GAAiB,EAAE,EACJ,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AAE1D,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAgB,EAAE;IAC7D,MAAM,IAAI,GAAiB;QACzB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,OAAO,EAAE,CAAC,CAAC,OAAO;KACnB,CAAC;IAEF,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACZ,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAc,EAAW,EAAE;IACxD,MAAM,EAAE,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAE9C,IAAI,EAAE,EAAE,CAAC;QACP,OAAO,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AACzD,CAAC,CAAC"}

package/dist/core/html/dom.d.ts

@@ -0,0 +1,13 @@
+import type { AnyNode, Document, Element } from "domhandler";
+export type { AnyNode, Document, Element } from "domhandler";
+/** Parse a full HTML document with the WHATWG algorithm (parse5), scripting enabled like a browser. */
+export declare const parseDocument: (html: string) => Document;
+/** Descendants of `root` matching a CSS selector, in document order. */
+export declare const queryAll: (root: AnyNode, selector: string) => Element[];
+/** First descendant of `root` matching a CSS selector. */
+export declare const queryOne: (root: AnyNode, selector: string) => Element | null;
+export declare const attr: (el: Element, name: string) => string | undefined;
+export declare const hasAttr: (el: Element, name: string) => boolean;
+/** DOM `textContent`: concatenated text of all descendants, comments excluded. */
+export declare const text: (node: AnyNode | AnyNode[]) => string;
+export declare const childElements: (el: Element) => Element[];

package/dist/core/html/dom.js

@@ -0,0 +1,17 @@
+import { selectAll, selectOne } from "css-select";
+import { isTag } from "domhandler";
+import { textContent } from "domutils";
+import { parse } from "parse5";
+import { adapter } from "parse5-htmlparser2-tree-adapter";
+/** Parse a full HTML document with the WHATWG algorithm (parse5), scripting enabled like a browser. */
+export const parseDocument = (html) => parse(html, { scriptingEnabled: true, treeAdapter: adapter });
+/** Descendants of `root` matching a CSS selector, in document order. */
+export const queryAll = (root, selector) => selectAll(selector, root);
+/** First descendant of `root` matching a CSS selector. */
+export const queryOne = (root, selector) => selectOne(selector, root);
+export const attr = (el, name) => Object.hasOwn(el.attribs, name) ? el.attribs[name] : undefined;
+export const hasAttr = (el, name) => Object.hasOwn(el.attribs, name);
+/** DOM `textContent`: concatenated text of all descendants, comments excluded. */
+export const text = (node) => textContent(node);
+export const childElements = (el) => el.children.filter(isTag);
+//# sourceMappingURL=dom.js.map

package/dist/core/html/dom.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dom.js","sourceRoot":"","sources":["../../../src/core/html/dom.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAC/B,OAAO,EAAE,OAAO,EAAE,MAAM,iCAAiC,CAAC;AAI1D,uGAAuG;AACvG,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAY,EAAY,EAAE,CACtD,KAAK,CAAC,IAAI,EAAE,EAAE,gBAAgB,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;AAEhE,wEAAwE;AACxE,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAa,EAAE,CACrE,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,0DAA0D;AAC1D,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAa,EAAE,QAAgB,EAAkB,EAAE,CAC1E,SAAS,CAAmB,QAAQ,EAAE,IAAI,CAAC,CAAC;AAE9C,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,EAAW,EAAE,IAAY,EAAsB,EAAE,CACpE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAEjE,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAW,EAAE,IAAY,EAAW,EAAE,CAC5D,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AAElC,kFAAkF;AAClF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAyB,EAAU,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAW,EAAa,EAAE,CACtD,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC"}

package/dist/core/html/extract.d.ts

@@ -0,0 +1,10 @@
+import type { ExtractedPage } from "../types.js";
+export interface ExtractOptions {
+    /** Absolute URL of the document; relative links are resolved against it (or against <base href>). */
+    url: string;
+    /** Treat subdomains of the site host as internal. */
+    includeSubdomains?: boolean;
+    /** Host used to classify links as internal. Defaults to the host of `url`. */
+    siteHost?: string;
+}
+export declare const extractPage: (html: string, options: ExtractOptions) => ExtractedPage;