@agentsean/crawler 1.0.0

package/dist/extract.js

@@ -0,0 +1,268 @@
+import { load } from "cheerio";
+import { parseDocument } from "htmlparser2";
+import { extractMainContent, loremIpsumIn, words } from "./readability.js";
+import { absolutize, originOf, sameOrigin } from "./url.js";
+const BODY_FORCING = /<(?:img|div|iframe|p|table|h[1-6]|section|article|header|footer)\b/i;
+export function extractPage(html, pageUrl, headerCanonical = [], xRobots = null) {
+    const dom = parseDocument(html);
+    const $ = load(dom);
+    const origin = originOf(pageUrl);
+    const titles = $("title")
+        .toArray()
+        .map((el) => $(el).text().trim())
+        .filter(Boolean);
+    const titleInHead = $("head title").length > 0;
+    const metaDescs = $('meta[name="description" i]')
+        .toArray()
+        .map((el) => $(el).attr("content")?.trim() ?? "")
+        .filter(Boolean);
+    const metaDescInHead = $('head meta[name="description" i]').length > 0;
+    const headings = $("h1,h2,h3,h4,h5,h6")
+        .toArray()
+        .map((el) => ({
+        level: Number(el.tagName.slice(1)),
+        text: $(el).text().replace(/\s+/g, " ").trim(),
+    }));
+    const canonicalHtml = $('link[rel="canonical" i]')
+        .toArray()
+        .map((el) => $(el).attr("href")?.trim() ?? "")
+        .filter(Boolean);
+    const canonicalInHead = $('head link[rel="canonical" i]').length > 0;
+    const canonicalOutsideHead = $('link[rel="canonical" i]').length > 0 && !canonicalInHead;
+    const robotsMeta = $('meta[name="robots" i], meta[name="googlebot" i]')
+        .toArray()
+        .map((el) => $(el).attr("content")?.trim() ?? "")
+        .filter(Boolean);
+    const robotsMetaInHead = $('head meta[name="robots" i], head meta[name="googlebot" i]').length > 0;
+    const hreflang = $('link[rel="alternate" i][hreflang]')
+        .toArray()
+        .map((el) => {
+        const href = $(el).attr("href")?.trim() ?? "";
+        const lang = $(el).attr("hreflang")?.trim() ?? "";
+        const inHead = $(el).parents("head").length > 0;
+        return {
+            lang,
+            href,
+            absUrl: absolutize(href, pageUrl) ?? href,
+            inHead,
+            source: inHead ? "head" : "body",
+        };
+    });
+    const jsonLd = $('script[type="application/ld+json"]')
+        .toArray()
+        .map((el) => {
+        const raw = $(el).text();
+        const inHead = $(el).parents("head").length > 0;
+        try {
+            return { raw, parsed: JSON.parse(raw), error: null, inHead };
+        }
+        catch (e) {
+            return {
+                raw,
+                parsed: null,
+                error: e instanceof Error ? e.message : "invalid json",
+                inHead,
+            };
+        }
+    });
+    const openGraph = {};
+    $('meta[property^="og:"]').each((_, el) => {
+        const prop = $(el).attr("property");
+        const content = $(el).attr("content");
+        if (prop && content)
+            openGraph[prop] = content;
+    });
+    const twitterCard = {};
+    $('meta[name^="twitter:"]').each((_, el) => {
+        const name = $(el).attr("name");
+        const content = $(el).attr("content");
+        if (name && content)
+            twitterCard[name] = content;
+    });
+    const links = extractLinks($, pageUrl, origin);
+    const images = extractImages($, pageUrl);
+    const scripts = $("script")
+        .toArray()
+        .map((el) => {
+        const src = $(el).attr("src") ?? null;
+        return {
+            src,
+            absUrl: src ? absolutize(src, pageUrl) : null,
+            async: $(el).attr("async") !== undefined,
+            defer: $(el).attr("defer") !== undefined,
+        };
+    });
+    const stylesheets = $('link[rel="stylesheet" i]')
+        .toArray()
+        .map((el) => {
+        const href = $(el).attr("href") ?? null;
+        return { href, absUrl: href ? absolutize(href, pageUrl) : null };
+    });
+    const allText = $("body").text().replace(/\s+/g, " ").trim();
+    const main = extractMainContent(html, titles[0] ?? "");
+    const boilerplateRatio = allText.length === 0 ? 1 : 1 - Math.min(1, main.text.length / allText.length);
+    const forms = $("form")
+        .toArray()
+        .map((el) => {
+        const action = $(el).attr("action") ?? "";
+        return { action, absAction: absolutize(action || pageUrl, pageUrl) ?? action };
+    });
+    const mixedActive = [];
+    const mixedPassive = [];
+    if (pageUrl.startsWith("https:")) {
+        $("script[src], link[rel='stylesheet'][href], iframe[src]").each((_, el) => {
+            const v = $(el).attr("src") ?? $(el).attr("href") ?? "";
+            if (v.startsWith("http://"))
+                mixedActive.push(v);
+        });
+        $("img[src], video[src], audio[src]").each((_, el) => {
+            const v = $(el).attr("src") ?? "";
+            if (v.startsWith("http://"))
+                mixedPassive.push(v);
+        });
+    }
+    let unsafeTargetBlanks = 0;
+    $("a[target='_blank']").each((_, el) => {
+        const rel = ($(el).attr("rel") ?? "").toLowerCase();
+        if (!rel.includes("noopener") && !rel.includes("noreferrer"))
+            unsafeTargetBlanks++;
+    });
+    const next = $('link[rel="next" i]').attr("href") ?? null;
+    const prev = $('link[rel="prev" i]').attr("href") ?? null;
+    const charset = $("meta[charset]").attr("charset") ??
+        $('meta[http-equiv="content-type" i]').attr("content") ??
+        null;
+    const first1024 = html.slice(0, 1024);
+    const charsetInFirst1024 = /<meta[^>]+charset/i.test(first1024);
+    const spaRootEmpty = /<(?:div|app-root)[^>]+\b(?:id=["'](?:root|app|__next)["']|data-reactroot|ng-version)/i.test(html) && main.wordCount < 30;
+    return {
+        title: titles[0] ?? null,
+        titles,
+        titleInHead,
+        metaDescription: metaDescs[0] ?? null,
+        metaDescriptions: metaDescs,
+        metaDescInHead,
+        h1: headings.filter((h) => h.level === 1).map((h) => h.text),
+        h2: headings.filter((h) => h.level === 2).map((h) => h.text),
+        headings,
+        canonicalHtml,
+        canonicalHeader: headerCanonical,
+        canonicalInHead,
+        canonicalOutsideHead,
+        headBrokenEarly: detectHeadBroken(html),
+        robotsMeta,
+        robotsMetaInHead,
+        xRobotsTag: xRobots,
+        viewport: $('meta[name="viewport" i]').attr("content") ?? null,
+        lang: $("html").attr("lang") ?? null,
+        charset,
+        charsetInFirst1024,
+        hreflang,
+        jsonLd,
+        openGraph,
+        twitterCard,
+        links,
+        images,
+        scripts,
+        stylesheets,
+        wordCount: words(allText),
+        mainWordCount: main.wordCount,
+        mainText: main.text,
+        allText,
+        boilerplateRatio,
+        metaRefresh: $('meta[http-equiv="refresh" i]').attr("content") ?? null,
+        hasNoscriptJsWarning: /please enable javascript|enable js/i.test($("noscript").text()),
+        spaRootEmpty,
+        hashFragmentMeta: $('meta[name="fragment"]').attr("content") === "!",
+        forms,
+        mixedActive,
+        mixedPassive,
+        unsafeTargetBlanks,
+        plugins: $("object, embed").length > 0,
+        loremIpsum: loremIpsumIn(allText) || loremIpsumIn(main.text),
+        paginationRel: {
+            next: next ? (absolutize(next, pageUrl) ?? next) : null,
+            prev: prev ? (absolutize(prev, pageUrl) ?? prev) : null,
+        },
+        htmlLang: $("html").attr("lang") ?? null,
+    };
+}
+function detectHeadBroken(html) {
+    const m = html.match(/<head\b[^>]*>([\s\S]*?)<\/head>/i);
+    const head = m?.[1];
+    if (!head)
+        return false;
+    const canon = head.search(/<link\b[^>]*rel\s*=\s*["']?canonical/i);
+    const slice = canon >= 0 ? head.slice(0, canon) : head;
+    return BODY_FORCING.test(slice);
+}
+function extractLinks($, pageUrl, origin) {
+    const out = [];
+    $("a[href]").each((_, el) => {
+        const href = $(el).attr("href")?.trim() ?? "";
+        if (!href || href.startsWith("javascript:") || href.startsWith("mailto:"))
+            return;
+        const abs = absolutize(href, pageUrl);
+        if (!abs)
+            return;
+        const rel = ($(el).attr("rel") ?? "").toLowerCase().split(/\s+/).filter(Boolean);
+        const position = classifyPosition($(el));
+        out.push({
+            href,
+            absUrl: abs,
+            anchorText: $(el).text().replace(/\s+/g, " ").trim(),
+            rel,
+            target: $(el).attr("target") ?? null,
+            isNofollow: rel.includes("nofollow"),
+            isUgc: rel.includes("ugc"),
+            isSponsored: rel.includes("sponsored"),
+            isInternal: sameOrigin(abs, origin),
+            position,
+        });
+    });
+    return out;
+}
+function extractImages($, pageUrl) {
+    return $("img")
+        .toArray()
+        .map((el) => {
+        const src = $(el).attr("src") ?? "";
+        const attribs = "attribs" in el ? el.attribs : {};
+        return {
+            src,
+            absUrl: absolutize(src, pageUrl) ?? src,
+            alt: $(el).attr("alt") ?? null,
+            hasAltAttr: Object.prototype.hasOwnProperty.call(attribs, "alt"),
+            width: $(el).attr("width") ?? null,
+            height: $(el).attr("height") ?? null,
+            loading: $(el).attr("loading") ?? null,
+            srcset: $(el).attr("srcset") ?? null,
+            decoding: $(el).attr("decoding") ?? null,
+        };
+    });
+}
+function classifyPosition(node) {
+    if (node.closest("nav, [role='navigation']").length)
+        return "nav";
+    if (node.closest("header").length)
+        return "header";
+    if (node.closest("footer").length)
+        return "footer";
+    if (node.closest("aside").length)
+        return "aside";
+    if (node.closest("main, article, [role='main']").length)
+        return "main";
+    return "unknown";
+}
+export function parseLinkHeaderCanonicals(header) {
+    if (!header)
+        return [];
+    const out = [];
+    for (const part of header.split(",")) {
+        const m = part.match(/<([^>]+)>\s*;\s*rel\s*=\s*"?canonical"?/i);
+        if (m?.[1])
+            out.push(m[1]);
+    }
+    return out;
+}
+//# sourceMappingURL=extract.js.map

package/dist/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../src/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAmB,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAS5D,MAAM,YAAY,GAChB,qEAAqE,CAAC;AAExE,MAAM,UAAU,WAAW,CACzB,IAAY,EACZ,OAAe,EACf,kBAA4B,EAAE,EAC9B,UAAyB,IAAI;IAE7B,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IAEjC,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC;SACtB,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;SAChC,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,MAAM,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAE/C,MAAM,SAAS,GAAG,CAAC,CAAC,4BAA4B,CAAC;SAC9C,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,MAAM,cAAc,GAAG,CAAC,CAAC,iCAAiC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAEvE,MAAM,QAAQ,GAAG,CAAC,CAAC,mBAAmB,CAAC;SACpC,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAClC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE;KAC/C,CAAC,CAAC,CAAC;IAEN,MAAM,aAAa,GAAG,CAAC,CAAC,yBAAyB,CAAC;SAC/C,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SAC7C,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,MAAM,eAAe,GAAG,CAAC,CAAC,8BAA8B,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACrE,MAAM,oBAAoB,GACxB,CAAC,CAAC,yBAAyB,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC;IAE9D,MAAM,UAAU,GAAG,CAAC,CAAC,iDAAiD,CAAC;SACpE,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,MAAM,gBAAgB,GACpB,CAAC,CAAC,2DAA2D,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAE5E,MAAM,QAAQ,GAAkB,CAAC,CAAC,mCAAmC,CAAC;SACnE,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAChD,OAAO;YACL,IAAI;YACJ,IAAI;YACJ,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI;YACzC,MAAM;YACN,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;SACjC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,MAAM,MAAM,GAAkB,CAAC,CAAC,oCAAoC,CAAC;SAClE,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAChD,IAAI,CAAC;YACH,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAY,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;QAC1E,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO;gBACL,GAAG;gBACH,MAAM,EAAE,IAAI;gBACZ,KAAK,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc;gBACtD,MAAM;aACP,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEL,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,CAAC,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACxC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpC,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,IAAI,IAAI,OAAO;YAAE,SAAS,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;IACjD,CAAC,CAAC,CAAC;IACH,MAAM,WAAW,GAA2B,EAAE,CAAC;IAC/C,CAAC,CAAC,wBAAwB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACzC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChC,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,IAAI,IAAI,OAAO;YAAE,WAAW,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAEzC,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC;SACxB,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC;QACtC,OAAO;YACL,GAAG;YACH,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI;YAC7C,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,SAAS;YACxC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,SAAS;SACzC,CAAC;IACJ,CAAC,CAAC,CAAC;IACL,MAAM,WAAW,GAAG,CAAC,CAAC,0BAA0B,CAAC;SAC9C,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;QACxC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACnE,CAAC,CAAC,CAAC;IAEL,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACvD,MAAM,gBAAgB,GACpB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAEhF,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC;SACpB,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC1C,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,IAAI,MAAM,EAAE,CAAC;IACjF,CAAC,CAAC,CAAC;IAEL,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjC,CAAC,CAAC,wDAAwD,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACzE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;gBAAE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,kCAAkC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACnD,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC;gBAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,CAAC,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACpD,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC;YAAE,kBAAkB,EAAE,CAAC;IACrF,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,CAAC,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;IAC1D,MAAM,IAAI,GAAG,CAAC,CAAC,oBAAoB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;IAE1D,MAAM,OAAO,GACX,CAAC,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QAClC,CAAC,CAAC,mCAAmC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QACtD,IAAI,CAAC;IACP,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IACtC,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEhE,MAAM,YAAY,GAChB,uFAAuF,CAAC,IAAI,CAC1F,IAAI,CACL,IAAI,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;IAE3B,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI;QACxB,MAAM;QACN,WAAW;QACX,eAAe,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI;QACrC,gBAAgB,EAAE,SAAS;QAC3B,cAAc;QACd,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5D,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5D,QAAQ;QACR,aAAa;QACb,eAAe,EAAE,eAAe;QAChC,eAAe;QACf,oBAAoB;QACpB,eAAe,EAAE,gBAAgB,CAAC,IAAI,CAAC;QACvC,UAAU;QACV,gBAAgB;QAChB,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE,CAAC,CAAC,yBAAyB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI;QAC9D,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI;QACpC,OAAO;QACP,kBAAkB;QAClB,QAAQ;QACR,MAAM;QACN,SAAS;QACT,WAAW;QACX,KAAK;QACL,MAAM;QACN,OAAO;QACP,WAAW;QACX,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC;QACzB,aAAa,EAAE,IAAI,CAAC,SAAS;QAC7B,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,OAAO;QACP,gBAAgB;QAChB,WAAW,EAAE,CAAC,CAAC,8BAA8B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI;QACtE,oBAAoB,EAAE,qCAAqC,CAAC,IAAI,CAC9D,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CACrB;QACD,YAAY;QACZ,gBAAgB,EAAE,CAAC,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG;QACpE,KAAK;QACL,WAAW;QACX,YAAY;QACZ,kBAAkB;QAClB,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,GAAG,CAAC;QACtC,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;QAC5D,aAAa,EAAE;YACb,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;YACvD,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;SACxD;QACD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI;KACzC,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACpB,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,uCAAuC,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,OAAO,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,YAAY,CAAC,CAAa,EAAE,OAAe,EAAE,MAAc;IAClE,MAAM,GAAG,GAAoB,EAAE,CAAC;IAChC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC9C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO;QAClF,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACjF,MAAM,QAAQ,GAAG,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzC,GAAG,CAAC,IAAI,CAAC;YACP,IAAI;YACJ,MAAM,EAAE,GAAG;YACX,UAAU,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE;YACpD,GAAG;YACH,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI;YACpC,UAAU,EAAE,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC;YACpC,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC1B,WAAW,EAAE,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC;YACtC,UAAU,EAAE,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC;YACnC,QAAQ;SACT,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,aAAa,CAAC,CAAa,EAAE,OAAe;IACnD,OAAO,CAAC,CAAC,KAAK,CAAC;SACZ,OAAO,EAAE;SACT,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACV,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,SAAS,IAAI,EAAE,CAAC,CAAC,CAAE,EAAE,CAAC,OAAkC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9E,OAAO;YACL,GAAG;YACH,MAAM,EAAE,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,GAAG;YACvC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI;YAC9B,UAAU,EAAE,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC;YAChE,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI;YAClC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI;YACpC,OAAO,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI;YACtC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI;YACpC,QAAQ,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI;SACzC,CAAC;IACJ,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,gBAAgB,CAAC,IAA4B;IACpD,IAAI,IAAI,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAClE,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM;QAAE,OAAO,QAAQ,CAAC;IACnD,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM;QAAE,OAAO,QAAQ,CAAC;IACnD,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM;QAAE,OAAO,OAAO,CAAC;IACjD,IAAI,IAAI,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC,MAAM;QAAE,OAAO,MAAM,CAAC;IACvE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,MAA0B;IAClE,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IACvB,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;QACjE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/hash.d.ts

@@ -0,0 +1,4 @@
+export declare function sha256Hex(input: string | Uint8Array): string;
+export declare function contentHash(body: string | Uint8Array): string;
+export declare function urlHash(url: string): string;
+//# sourceMappingURL=hash.d.ts.map

package/dist/hash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../src/hash.ts"],"names":[],"mappings":"AAEA,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAE5D;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAE7D;AAED,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAE3C"}

package/dist/hash.js

@@ -0,0 +1,11 @@
+import { createHash } from "node:crypto";
+export function sha256Hex(input) {
+    return createHash("sha256").update(input).digest("hex");
+}
+export function contentHash(body) {
+    return sha256Hex(body);
+}
+export function urlHash(url) {
+    return sha256Hex(url);
+}
+//# sourceMappingURL=hash.js.map

package/dist/hash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash.js","sourceRoot":"","sources":["../src/hash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,UAAU,SAAS,CAAC,KAA0B;IAClD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAyB;IACnD,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,GAAW;IACjC,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC;AACxB,CAAC"}

package/dist/http.d.ts

@@ -0,0 +1,38 @@
+import { Agent } from "undici";
+import type { RedirectHop } from "./types.js";
+export declare const MAX_BODY: number;
+export declare const HTML_GOOGLE_CAP: number;
+export declare const RESOURCE_GOOGLE_CAP: number;
+export type FetchResult = {
+    url: string;
+    finalUrl: string;
+    statusCode: number | null;
+    error: string | null;
+    headers: Record<string, string>;
+    body: Buffer;
+    decoded: Buffer;
+    contentType: string;
+    ttfbMs: number;
+    totalMs: number;
+    wireBytes: number;
+    decodedBytes: number;
+    redirectChain: RedirectHop[];
+    redirectLoop: boolean;
+    exceedsGoogleRedirectLimit: boolean;
+    etag: string | null;
+    lastModified: string | null;
+    notModified: boolean;
+};
+export declare function createCrawlerAgent(connectionsPerOrigin?: number): Agent;
+export declare function fetchUrl(url: string, opts: {
+    agent: Agent;
+    userAgent?: string | undefined;
+    timeoutMs?: number | undefined;
+    maxRedirects?: number | undefined;
+    method?: "GET" | "HEAD" | undefined;
+    etag?: string | undefined;
+    lastModified?: string | undefined;
+    accept?: string | undefined;
+}): Promise<FetchResult>;
+export declare const ROBOTS_MAX_REDIRECTS = 5;
+//# sourceMappingURL=http.d.ts.map

package/dist/http.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAW,MAAM,QAAQ,CAAC;AAGxC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,eAAO,MAAM,QAAQ,QAAmB,CAAC;AACzC,eAAO,MAAM,eAAe,QAAkB,CAAC;AAC/C,eAAO,MAAM,mBAAmB,QAAmB,CAAC;AAKpD,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,WAAW,EAAE,CAAC;IAC7B,YAAY,EAAE,OAAO,CAAC;IACtB,0BAA0B,EAAE,OAAO,CAAC;IACpC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF,wBAAgB,kBAAkB,CAAC,oBAAoB,SAAI,GAAG,KAAK,CAalE;AAID,wBAAsB,QAAQ,CAC5B,GAAG,EAAE,MAAM,EACX,IAAI,EAAE;IACJ,KAAK,EAAE,KAAK,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,GAAG,SAAS,CAAC;IACpC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B,GACA,OAAO,CAAC,WAAW,CAAC,CAqGtB;AAwGD,eAAO,MAAM,oBAAoB,IAAuB,CAAC"}

package/dist/http.js

@@ -0,0 +1,207 @@
+import { Agent, request } from "undici";
+import { acceptEncodingHeader, decodeBody } from "./decode.js";
+import { SEAN_UA } from "./ua.js";
+export const MAX_BODY = 10 * 1024 * 1024;
+export const HTML_GOOGLE_CAP = 2 * 1024 * 1024;
+export const RESOURCE_GOOGLE_CAP = 15 * 1024 * 1024;
+const PAGE_REDIRECT_HOPS = 5;
+const GOOGLE_REDIRECT_HOPS = 10;
+const ROBOTS_REDIRECT_HOPS = 5;
+export function createCrawlerAgent(connectionsPerOrigin = 2) {
+    return new Agent({
+        connections: connectionsPerOrigin,
+        pipelining: 1,
+        keepAliveTimeout: 10_000,
+        keepAliveMaxTimeout: 600_000,
+        headersTimeout: 15_000,
+        bodyTimeout: 30_000,
+        allowH2: true,
+        maxResponseSize: MAX_BODY,
+        clientTtl: 60_000,
+        connect: { rejectUnauthorized: true },
+    });
+}
+const RETRYABLE_STATUS = new Set([408, 425, 429, 500, 502, 503, 504, 522, 524]);
+export async function fetchUrl(url, opts) {
+    const userAgent = opts.userAgent ?? SEAN_UA;
+    const timeoutMs = opts.timeoutMs ?? 20_000;
+    const maxRedirects = opts.maxRedirects ?? PAGE_REDIRECT_HOPS;
+    const chain = [];
+    const seen = new Set();
+    let current = url;
+    let loop = false;
+    let exceeds = false;
+    const t0 = performance.now();
+    let ttfbMs = 0;
+    for (let hop = 0; hop <= GOOGLE_REDIRECT_HOPS; hop++) {
+        if (seen.has(current)) {
+            loop = true;
+            break;
+        }
+        seen.add(current);
+        if (hop > maxRedirects) {
+            exceeds = hop > GOOGLE_REDIRECT_HOPS;
+            break;
+        }
+        const headers = {
+            "user-agent": userAgent,
+            accept: opts.accept ??
+                "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "accept-encoding": acceptEncodingHeader(),
+        };
+        if (opts.etag)
+            headers["if-none-match"] = opts.etag;
+        if (opts.lastModified)
+            headers["if-modified-since"] = opts.lastModified;
+        const result = await attempt(current, {
+            agent: opts.agent,
+            method: opts.method ?? "GET",
+            headers,
+            timeoutMs,
+        });
+        if (ttfbMs === 0)
+            ttfbMs = result.ttfbMs;
+        const location = result.headers.location ?? null;
+        chain.push({ url: current, status: result.statusCode ?? 0, location });
+        if (result.statusCode !== null &&
+            result.statusCode >= 300 &&
+            result.statusCode < 400 &&
+            location) {
+            current = new URL(location, current).href;
+            if (hop + 1 > GOOGLE_REDIRECT_HOPS) {
+                exceeds = true;
+                break;
+            }
+            continue;
+        }
+        const totalMs = performance.now() - t0;
+        return {
+            url,
+            finalUrl: current,
+            statusCode: result.statusCode,
+            error: result.error,
+            headers: result.headers,
+            body: result.wire,
+            decoded: result.decoded,
+            contentType: result.headers["content-type"] ?? "",
+            ttfbMs,
+            totalMs,
+            wireBytes: result.wire.length,
+            decodedBytes: result.decoded.length,
+            redirectChain: chain,
+            redirectLoop: loop,
+            exceedsGoogleRedirectLimit: exceeds,
+            etag: result.headers.etag ?? null,
+            lastModified: result.headers["last-modified"] ?? null,
+            notModified: result.statusCode === 304,
+        };
+    }
+    return {
+        url,
+        finalUrl: current,
+        statusCode: chain.at(-1)?.status ?? null,
+        error: loop ? "redirect_loop" : exceeds ? "redirect_limit" : "redirect",
+        headers: {},
+        body: Buffer.alloc(0),
+        decoded: Buffer.alloc(0),
+        contentType: "",
+        ttfbMs,
+        totalMs: performance.now() - t0,
+        wireBytes: 0,
+        decodedBytes: 0,
+        redirectChain: chain,
+        redirectLoop: loop,
+        exceedsGoogleRedirectLimit: exceeds,
+        etag: null,
+        lastModified: null,
+        notModified: false,
+    };
+}
+async function attempt(url, opts) {
+    let lastError = null;
+    for (let attemptNo = 1; attemptNo <= 3; attemptNo++) {
+        const t0 = performance.now();
+        try {
+            const res = await request(url, {
+                method: opts.method,
+                dispatcher: opts.agent,
+                headers: opts.headers,
+                headersTimeout: opts.timeoutMs,
+                bodyTimeout: opts.timeoutMs,
+            });
+            const ttfbMs = performance.now() - t0;
+            const headers = {};
+            for (const [k, v] of Object.entries(res.headers)) {
+                if (typeof v === "string")
+                    headers[k.toLowerCase()] = v;
+                else if (Array.isArray(v))
+                    headers[k.toLowerCase()] = v.join(", ");
+            }
+            const wire = Buffer.from(new Uint8Array(await res.body.arrayBuffer()));
+            let decoded = wire;
+            try {
+                decoded = await decodeBody(wire, headers["content-encoding"]);
+            }
+            catch {
+                decoded = wire;
+            }
+            if (RETRYABLE_STATUS.has(res.statusCode) && attemptNo < 3) {
+                lastError = `http_${res.statusCode}`;
+                await backoff(attemptNo, headers["retry-after"]);
+                continue;
+            }
+            return {
+                statusCode: res.statusCode,
+                headers,
+                wire,
+                decoded,
+                error: null,
+                ttfbMs,
+            };
+        }
+        catch (e) {
+            lastError = e instanceof Error ? e.message : String(e);
+            if (attemptNo < 3) {
+                await backoff(attemptNo, undefined);
+                continue;
+            }
+            return {
+                statusCode: null,
+                headers: {},
+                wire: Buffer.alloc(0),
+                decoded: Buffer.alloc(0),
+                error: lastError,
+                ttfbMs: performance.now() - t0,
+            };
+        }
+    }
+    return {
+        statusCode: null,
+        headers: {},
+        wire: Buffer.alloc(0),
+        decoded: Buffer.alloc(0),
+        error: lastError,
+        ttfbMs: 0,
+    };
+}
+async function backoff(n, retryAfter) {
+    if (retryAfter) {
+        const seconds = Number(retryAfter);
+        if (Number.isFinite(seconds)) {
+            await sleep(Math.min(seconds * 1000, 300_000));
+            return;
+        }
+        const when = Date.parse(retryAfter);
+        if (!Number.isNaN(when)) {
+            await sleep(Math.min(Math.max(0, when - Date.now()), 300_000));
+            return;
+        }
+    }
+    const cap = Math.min(1000 * 2 ** (n - 1), 60_000);
+    await sleep(Math.random() * cap);
+}
+function sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+}
+export const ROBOTS_MAX_REDIRECTS = ROBOTS_REDIRECT_HOPS;
+//# sourceMappingURL=http.js.map

package/dist/http.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AACxC,OAAO,EAAE,oBAAoB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAGlC,MAAM,CAAC,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACzC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAC/C,MAAM,CAAC,MAAM,mBAAmB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACpD,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAChC,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAuB/B,MAAM,UAAU,kBAAkB,CAAC,oBAAoB,GAAG,CAAC;IACzD,OAAO,IAAI,KAAK,CAAC;QACf,WAAW,EAAE,oBAAoB;QACjC,UAAU,EAAE,CAAC;QACb,gBAAgB,EAAE,MAAM;QACxB,mBAAmB,EAAE,OAAO;QAC5B,cAAc,EAAE,MAAM;QACtB,WAAW,EAAE,MAAM;QACnB,OAAO,EAAE,IAAI;QACb,eAAe,EAAE,QAAQ;QACzB,SAAS,EAAE,MAAM;QACjB,OAAO,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE;KACtC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,GAAW,EACX,IASC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC;IAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,kBAAkB,CAAC;IAC7D,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,oBAAoB,EAAE,GAAG,EAAE,EAAE,CAAC;QACrD,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YACtB,IAAI,GAAG,IAAI,CAAC;YACZ,MAAM;QACR,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClB,IAAI,GAAG,GAAG,YAAY,EAAE,CAAC;YACvB,OAAO,GAAG,GAAG,GAAG,oBAAoB,CAAC;YACrC,MAAM;QACR,CAAC;QAED,MAAM,OAAO,GAA2B;YACtC,YAAY,EAAE,SAAS;YACvB,MAAM,EACJ,IAAI,CAAC,MAAM;gBACX,iEAAiE;YACnE,iBAAiB,EAAE,oBAAoB,EAAE;SAC1C,CAAC;QACF,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,eAAe,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;QACpD,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC;QAExE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE;YACpC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,KAAK;YAC5B,OAAO;YACP,SAAS;SACV,CAAC,CAAC;QACH,IAAI,MAAM,KAAK,CAAC;YAAE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAEzC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;QAEvE,IACE,MAAM,CAAC,UAAU,KAAK,IAAI;YAC1B,MAAM,CAAC,UAAU,IAAI,GAAG;YACxB,MAAM,CAAC,UAAU,GAAG,GAAG;YACvB,QAAQ,EACR,CAAC;YACD,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;YAC1C,IAAI,GAAG,GAAG,CAAC,GAAG,oBAAoB,EAAE,CAAC;gBACnC,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM;YACR,CAAC;YACD,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;QACvC,OAAO;YACL,GAAG;YACH,QAAQ,EAAE,OAAO;YACjB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE;YACjD,MAAM;YACN,OAAO;YACP,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;YAC7B,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;YACnC,aAAa,EAAE,KAAK;YACpB,YAAY,EAAE,IAAI;YAClB,0BAA0B,EAAE,OAAO;YACnC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,IAAI;YACjC,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,IAAI;YACrD,WAAW,EAAE,MAAM,CAAC,UAAU,KAAK,GAAG;SACvC,CAAC;IACJ,CAAC;IAED,OAAO;QACL,GAAG;QACH,QAAQ,EAAE,OAAO;QACjB,UAAU,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,IAAI;QACxC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,UAAU;QACvE,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxB,WAAW,EAAE,EAAE;QACf,MAAM;QACN,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;QAC/B,SAAS,EAAE,CAAC;QACZ,YAAY,EAAE,CAAC;QACf,aAAa,EAAE,KAAK;QACpB,YAAY,EAAE,IAAI;QAClB,0BAA0B,EAAE,OAAO;QACnC,IAAI,EAAE,IAAI;QACV,YAAY,EAAE,IAAI;QAClB,WAAW,EAAE,KAAK;KACnB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,OAAO,CACpB,GAAW,EACX,IAKC;IASD,IAAI,SAAS,GAAkB,IAAI,CAAC;IACpC,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC;QACpD,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE;gBAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,IAAI,CAAC,KAAK;gBACtB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,cAAc,EAAE,IAAI,CAAC,SAAS;gBAC9B,WAAW,EAAE,IAAI,CAAC,SAAS;aAC5B,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtC,MAAM,OAAO,GAA2B,EAAE,CAAC;YAC3C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBACjD,IAAI,OAAO,CAAC,KAAK,QAAQ;oBAAE,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAC;qBACnD,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;oBAAE,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,IAAI,GAAW,MAAM,CAAC,IAAI,CAC9B,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CACnC,CAAC;YACZ,IAAI,OAAO,GAAG,IAAI,CAAC;YACnB,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC;YAChE,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;YACD,IAAI,gBAAgB,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAC1D,SAAS,GAAG,QAAQ,GAAG,CAAC,UAAU,EAAE,CAAC;gBACrC,MAAM,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;gBACjD,SAAS;YACX,CAAC;YACD,OAAO;gBACL,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,OAAO;gBACP,IAAI;gBACJ,OAAO;gBACP,KAAK,EAAE,IAAI;gBACX,MAAM;aACP,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,SAAS,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACvD,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,OAAO,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;gBACpC,SAAS;YACX,CAAC;YACD,OAAO;gBACL,UAAU,EAAE,IAAI;gBAChB,OAAO,EAAE,EAAE;gBACX,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACrB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACxB,KAAK,EAAE,SAAS;gBAChB,MAAM,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;aAC/B,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO;QACL,UAAU,EAAE,IAAI;QAChB,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxB,KAAK,EAAE,SAAS;QAChB,MAAM,EAAE,CAAC;KACV,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,CAAS,EAAE,UAA8B;IAC9D,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QACnC,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;YAC/C,OAAO;QACT,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YAC/D,OAAO;QACT,CAAC;IACH,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,oBAAoB,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+export { crawlSite, ramConcurrency } from "./crawl.js";
+export { persistCrawl, persistFindings } from "./persist.js";
+export { fetchUrl, createCrawlerAgent } from "./http.js";
+export { decodeBody } from "./decode.js";
+export { parseRobotsTxt, isAllowedByRobots, robotsFromFetch, matchRule, } from "./robots.js";
+export { parseSitemapXml } from "./sitemap.js";
+export { extractPage } from "./extract.js";
+export { extractMainContent } from "./readability.js";
+export { simhash64, simhashHex, hamming64, nearDuplicate } from "./simhash.js";
+export { contentHash, sha256Hex, urlHash } from "./hash.js";
+export { SEAN_UA } from "./ua.js";
+export { decideRenderPolicy } from "./adaptive.js";
+export type { CrawledPage, CrawlCheckpoint, CrawlOptions, CrawlResult, PageExtract, RobotsOutcome, ParsedSitemap, OriginProbe, ExtractedLink, ExtractedImage, JsonLdBlock, HreflangAlt, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,SAAS,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC/E,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,YAAY,EACV,WAAW,EACX,eAAe,EACf,YAAY,EACZ,WAAW,EACX,WAAW,EACX,aAAa,EACb,aAAa,EACb,WAAW,EACX,aAAa,EACb,cAAc,EACd,WAAW,EACX,WAAW,GACZ,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,13 @@
+export { crawlSite, ramConcurrency } from "./crawl.js";
+export { persistCrawl, persistFindings } from "./persist.js";
+export { fetchUrl, createCrawlerAgent } from "./http.js";
+export { decodeBody } from "./decode.js";
+export { parseRobotsTxt, isAllowedByRobots, robotsFromFetch, matchRule, } from "./robots.js";
+export { parseSitemapXml } from "./sitemap.js";
+export { extractPage } from "./extract.js";
+export { extractMainContent } from "./readability.js";
+export { simhash64, simhashHex, hamming64, nearDuplicate } from "./simhash.js";
+export { contentHash, sha256Hex, urlHash } from "./hash.js";
+export { SEAN_UA } from "./ua.js";
+export { decideRenderPolicy } from "./adaptive.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,cAAc,EACd,iBAAiB,EACjB,eAAe,EACf,SAAS,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC/E,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC5D,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC"}

package/dist/persist.d.ts

@@ -0,0 +1,20 @@
+import type { SqliteDatabase } from "@agentsean/db";
+import type { CrawlResult } from "./types.js";
+export declare function persistCrawl(db: SqliteDatabase, result: CrawlResult, siteName?: string, opts?: {
+    crawlId?: string | undefined;
+    status?: string | undefined;
+}): Promise<{
+    siteId: string;
+    crawlId: string;
+}>;
+export declare function persistFindings(db: SqliteDatabase, siteId: string, rows: {
+    pageUrl: string | null;
+    ruleId: string;
+    severity: string;
+    autonomyTier: string;
+    title: string;
+    explanation: string | null;
+    evidence: unknown;
+    fingerprint: string;
+}[]): void;
+//# sourceMappingURL=persist.d.ts.map

package/dist/persist.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"persist.d.ts","sourceRoot":"","sources":["../src/persist.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAWpD,OAAO,KAAK,EAAe,WAAW,EAAE,MAAM,YAAY,CAAC;AAE3D,wBAAsB,YAAY,CAChC,EAAE,EAAE,cAAc,EAClB,MAAM,EAAE,WAAW,EACnB,QAAQ,CAAC,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,GACnE,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CA8D9C;AA+FD,wBAAgB,eAAe,CAC7B,EAAE,EAAE,cAAc,EAClB,MAAM,EAAE,MAAM,EACd,IAAI,EAAE;IACJ,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB,EAAE,GACF,IAAI,CA0CN"}